fluent-plugin-kafka 0.12.4 → 0.14.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8b9d681257730f610d10ef3f3ab0f219d3167df23482370bcc3c89b01cf12098
-  data.tar.gz: f628e31d41fdc36f51d93a4fb4f7e68321ad0857765a6088a9b46ce593c435c8
+  metadata.gz: 14cde202b38079778e0d694692f05c19c9576be8622f35a8896df35f33ea9733
+  data.tar.gz: 80cbc1050e85239dabfe78dbffd97a100c939ec4e9227bd68956a7b3d15aa75e
 SHA512:
-  metadata.gz: 57b996881b2981fb4366c31f25f27b4ca19b4d8ca2c4e27227ac4b5d603ae7a6b2719e7a049d8b668ace11620786832b7e970c90c3536de1ed6ba3da1723dd09
-  data.tar.gz: 21ddc3ab9a074b3910719ae59cdf826423f3cd52c1b9179ab86ccdf6db60166776cd3b7218f6e04da1ac101dd130c87486e30658736e7f6347b161776c7bc7ee
+  metadata.gz: 5ec7ed5f16a7d78a0dcd5f6eb15ecd94dd47adc7ae77208896a0ede1d341ede1f5a668141e7ac4e5413734105e276bd483f3f2b8c2041d329a4d619a9c469a76
+  data.tar.gz: 11877dd67f3b0f714b38153368611c8f234b6a96976a00e21117e9523fa5bcde210fd13fa5e22086cf604ee61fc94a3472bc4946c69a6f5623eeace486e7eb75

data/ChangeLog

@@ -1,3 +1,26 @@
+Release 0.14.2 - 2020/08/26
+
+	* in_kafka_group: Add `add_headers` parameter
+	* out_kafka2/out_rdkafka2: Support `discard_kafka_delivery_failed` parameter
+
+Release 0.14.1 - 2020/08/11
+
+	* kafka_producer_ext: Fix regression by v0.14.0 changes
+
+Release 0.14.0 - 2020/08/07
+
+	* Update ruby-kafka dependency to v1.2.0 or later. Check https://github.com/zendesk/ruby-kafka#compatibility
+	* kafka_producer_ext: Follow Paritioner API change
+
+Release 0.13.1 - 2020/07/17
+
+	* in_kafka_group: Support ssl_verify_hostname parameter
+	* out_kafka2/out_rdkafka2: Support topic parameter with placeholders
+
+Release 0.13.0 - 2020/03/09
+
+	* Accept ruby-kafka v1 or later
+
 Release 0.12.4 - 2020/03/03
 
 	* output: Follow rdkafka log level

data/README.md

@@ -118,10 +118,13 @@ Consume events by kafka consumer group features..
       topics <listening topics(separate with comma',')>
       format <input text type (text|json|ltsv|msgpack)> :default => json
       message_key <key (Optional, for text format only, default is message)>
+      kafka_mesasge_key <key (Optional, If specified, set kafka's message key to this key)>
+      add_headers <If true, add kafka's message headers to record>
       add_prefix <tag prefix (Optional)>
       add_suffix <tag suffix (Optional)>
       retry_emit_limit <Wait retry_emit_limit x 1s when BuffereQueueLimitError happens. The default is nil and it means waiting until BufferQueueLimitError is resolved>
-      use_record_time <If true, replace event time with contents of 'time' field of fetched record>
+      use_record_time (Deprecated. Use 'time_source record' instead.) <If true, replace event time with contents of 'time' field of fetched record>
+      time_source <source for message timestamp (now|kafka|record)> :default => now
       time_format <string (Optional when use_record_time is used)>
 
       # ruby-kafka consumer options
@@ -140,7 +143,8 @@ Consuming topic name is used for event tag. So when the target topic name is `ap
 
 ### Output plugin
 
-This `kafka2` plugin is for fluentd v1.0 or later. This will be `out_kafka` plugin in the future.
+This `kafka2` plugin is for fluentd v1 or later. This plugin uses `ruby-kafka` producer for writing data.
+If `ruby-kafka` doesn't fit your kafka environment, check `rdkafka2` plugin instead. This will be `out_kafka` plugin in the future.
 
     <match app.**>
       @type kafka2
@@ -161,6 +165,7 @@ This `kafka2` plugin is for fluentd v1.0 or later. This will be `out_kafka` plug
       headers               (hash)   :default => {}
       headers_from_record   (hash)   :default => {}
       use_default_for_unknown_topic (bool) :default => false
+      discard_kafka_delivery_failed (bool) :default => false (No discard)
 
       <format>
         @type (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
@@ -384,6 +389,7 @@ You need to install rdkafka gem.
       default_message_key   (string) :default => nil
       exclude_topic_key     (bool) :default => false
       exclude_partition_key (bool) :default => false
+      discard_kafka_delivery_failed (bool) :default => false (No discard)
 
       # same with kafka2
       headers               (hash) :default => {}
@@ -443,7 +449,7 @@ See ruby-kafka README for more details: https://github.com/zendesk/ruby-kafka#co
 
 To avoid the problem, there are 2 approaches:
 
-- Upgrade your kafka cluster to latest version. This is better becase recent version is faster and robust.
+- Upgrade your kafka cluster to latest version. This is better because recent version is faster and robust.
 - Downgrade ruby-kafka/fluent-plugin-kafka to work with your older kafka.
 
 ## Contributing

data/fluent-plugin-kafka.gemspec

@@ -13,12 +13,12 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.12.4'
+  gem.version       = '0.14.2'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
   gem.add_dependency 'ltsv'
-  gem.add_dependency 'ruby-kafka', '>= 0.7.8', '< 0.8.0'
+  gem.add_dependency 'ruby-kafka', '>= 1.2.0', '< 2'
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "test-unit", ">= 3.0.8"
 end

data/lib/fluent/plugin/in_kafka.rb

@@ -39,6 +39,8 @@ class Fluent::KafkaInput < Fluent::Input
                :deprecated => "Use 'time_source record' instead."
   config_param :time_source, :enum, :list => [:now, :kafka, :record], :default => :now,
                :desc => "Source for message timestamp."
+  config_param :record_time_key, :string, :default => 'time',
+               :desc => "Time field when time_source is 'record'"
   config_param :get_kafka_client_log, :bool, :default => false
   config_param :time_format, :string, :default => nil,
                :desc => "Time format to be used to parse 'time' field."
@@ -292,9 +294,9 @@ class Fluent::KafkaInput < Fluent::Input
             record_time = Fluent::Engine.now
           when :record
             if @time_format
-              record_time = @time_parser.parse(record['time'])
+              record_time = @time_parser.parse(record[@record_time_key])
             else
-              record_time = record['time']
+              record_time = record[@record_time_key]
             end
           else
             $log.fatal "BUG: invalid time_source: #{@time_source}"

data/lib/fluent/plugin/in_kafka_group.rb

@@ -18,6 +18,8 @@ class Fluent::KafkaGroupInput < Fluent::Input
                :desc => "Supported format: (json|text|ltsv|msgpack)"
   config_param :message_key, :string, :default => 'message',
                :desc => "For 'text' format only."
+  config_param :add_headers, :bool, :default => false,
+               :desc => "Add kafka's message headers to event record"
   config_param :add_prefix, :string, :default => nil,
                :desc => "Tag prefix (Optional)"
   config_param :add_suffix, :string, :default => nil,
@@ -29,6 +31,8 @@ class Fluent::KafkaGroupInput < Fluent::Input
                :deprecated => "Use 'time_source record' instead."
   config_param :time_source, :enum, :list => [:now, :kafka, :record], :default => :now,
                :desc => "Source for message timestamp."
+  config_param :record_time_key, :string, :default => 'time',
+               :desc => "Time field when time_source is 'record'"
   config_param :get_kafka_client_log, :bool, :default => false
   config_param :time_format, :string, :default => nil,
                :desc => "Time format to be used to parse 'time' field."
@@ -166,16 +170,17 @@ class Fluent::KafkaGroupInput < Fluent::Input
       @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_scram_username: @username, sasl_scram_password: @password,
-                         sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl)
+                         sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
     elsif @username != nil && @password != nil
       @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_plain_username: @username, sasl_plain_password: @password,
-                         sasl_over_ssl: @sasl_over_ssl)
+                         sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
     else
       @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
-                         ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab)
+                         ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab,
+                         ssl_verify_hostname: @ssl_verify_hostname)
     end
 
     @consumer = setup_consumer
@@ -198,7 +203,14 @@ class Fluent::KafkaGroupInput < Fluent::Input
   def setup_consumer
     consumer = @kafka.consumer(@consumer_opts)
     @topics.each { |topic|
-      consumer.subscribe(topic, start_from_beginning: @start_from_beginning, max_bytes_per_partition: @max_bytes)
+      if m = /^\/(.+)\/$/.match(topic)
+        topic_or_regex = Regexp.new(m[1])
+        $log.info "Subscribe to topics matching the regex #{topic}"
+      else
+        topic_or_regex = topic
+        $log.info "Subscribe to topic #{topic}"
+      end
+      consumer.subscribe(topic_or_regex, start_from_beginning: @start_from_beginning, max_bytes_per_partition: @max_bytes)
     }
     consumer
   end
@@ -243,9 +255,9 @@ class Fluent::KafkaGroupInput < Fluent::Input
                 record_time = Fluent::Engine.now
               when :record
                 if @time_format
-                  record_time = @time_parser.parse(record['time'].to_s)
+                  record_time = @time_parser.parse(record[@record_time_key].to_s)
                 else
-                  record_time = record['time']
+                  record_time = record[@record_time_key]
                 end
               else
                 log.fatal "BUG: invalid time_source: #{@time_source}"
@@ -253,6 +265,11 @@ class Fluent::KafkaGroupInput < Fluent::Input
               if @kafka_message_key
                 record[@kafka_message_key] = msg.key
               end
+              if @add_headers
+                msg.headers.each_pair { |k, v|
+                  record[k] = v
+                }
+              end
               es.add(record_time, record)
             rescue => e
               log.warn "parser error in #{batch.topic}/#{batch.partition}", :error => e.to_s, :value => msg.value, :offset => msg.offset

data/lib/fluent/plugin/kafka_producer_ext.rb

@@ -69,12 +69,13 @@ module Kafka
         retry_backoff: retry_backoff,
         max_buffer_size: max_buffer_size,
         max_buffer_bytesize: max_buffer_bytesize,
+        partitioner: @partitioner,
       )
     end
   end
 
   class TopicProducer
-    def initialize(topic, cluster:, transaction_manager:, logger:, instrumenter:, compressor:, ack_timeout:, required_acks:, max_retries:, retry_backoff:, max_buffer_size:, max_buffer_bytesize:)
+    def initialize(topic, cluster:, transaction_manager:, logger:, instrumenter:, compressor:, ack_timeout:, required_acks:, max_retries:, retry_backoff:, max_buffer_size:, max_buffer_bytesize:, partitioner:)
       @cluster = cluster
       @transaction_manager = transaction_manager
       @logger = logger
@@ -86,6 +87,7 @@ module Kafka
       @max_buffer_size = max_buffer_size
       @max_buffer_bytesize = max_buffer_bytesize
       @compressor = compressor
+      @partitioner = partitioner
 
       @topic = topic
       @cluster.add_target_topics(Set.new([topic]))
@@ -250,7 +252,7 @@ module Kafka
 
         begin
           if partition.nil?
-            partition = Partitioner.partition_for_key(partition_count, message)
+            partition = @partitioner.call(partition_count, message)
           end
 
           @buffer.write(

data/lib/fluent/plugin/out_kafka2.rb

@@ -15,6 +15,7 @@ module Fluent::Plugin
 Set brokers directly:
 <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,..
 DESC
+    config_param :topic, :string, :default => nil, :desc => "kafka topic. Placeholders are supported"
     config_param :topic_key, :string, :default => 'topic', :desc => "Field for kafka topic"
     config_param :default_topic, :string, :default => nil,
                  :desc => "Default output topic when record doesn't have topic field"
@@ -68,6 +69,7 @@ The codec the producer uses to compress messages.
 Supported codecs depends on ruby-kafka: https://github.com/zendesk/ruby-kafka#compression
 DESC
     config_param :max_send_limit_bytes, :size, :default => nil
+    config_param :discard_kafka_delivery_failed, :bool, :default => false
     config_param :active_support_notification_regex, :string, :default => nil,
                  :desc => <<-DESC
 Add a regular expression to capture ActiveSupport notifications from the Kafka client
@@ -215,7 +217,11 @@ DESC
     # TODO: optimize write performance
     def write(chunk)
       tag = chunk.metadata.tag
-      topic =  (chunk.metadata.variables && chunk.metadata.variables[@topic_key_sym]) || @default_topic || tag
+      topic = if @topic
+                extract_placeholders(@topic, chunk)
+              else
+                (chunk.metadata.variables && chunk.metadata.variables[@topic_key_sym]) || @default_topic || tag
+              end
 
       messages = 0
       record_buf = nil
@@ -262,7 +268,16 @@ DESC
 
         if messages > 0
           log.debug { "#{messages} messages send." }
-          producer.deliver_messages
+          if @discard_kafka_delivery_failed
+            begin
+              producer.deliver_messages
+            rescue Kafka::DeliveryFailed => e
+              log.warn "DeliveryFailed occurred. Discard broken event:", :error => e.to_s, :error_class => e.class.to_s, :tag => tag
+              producer.clear_buffer
+            end
+          else
+            producer.deliver_messages
+          end
         end
       rescue Kafka::UnknownTopicOrPartition
         if @use_default_for_unknown_topic && topic != @default_topic

data/lib/fluent/plugin/out_rdkafka2.rb

@@ -33,6 +33,7 @@ Set brokers directly:
 <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,..
 Brokers: you can choose to use either brokers or zookeeper.
 DESC
+    config_param :topic, :string, :default => nil, :desc => "kafka topic. Placeholders are supported"
     config_param :topic_key, :string, :default => 'topic', :desc => "Field for kafka topic"
     config_param :default_topic, :string, :default => nil,
                  :desc => "Default output topic when record doesn't have topic field"
@@ -72,6 +73,7 @@ The codec the producer uses to compress messages. Used for compression.codec
 Supported codecs: (gzip|snappy)
 DESC
     config_param :max_send_limit_bytes, :size, :default => nil
+    config_param :discard_kafka_delivery_failed, :bool, :default => false
     config_param :rdkafka_buffering_max_ms, :integer, :default => nil, :desc => 'Used for queue.buffering.max.ms'
     config_param :rdkafka_buffering_max_messages, :integer, :default => nil, :desc => 'Used for queue.buffering.max.messages'
     config_param :rdkafka_message_max_bytes, :integer, :default => nil, :desc => 'Used for message.max.bytes'
@@ -278,7 +280,11 @@ DESC
 
     def write(chunk)
       tag = chunk.metadata.tag
-      topic = (chunk.metadata.variables && chunk.metadata.variables[@topic_key_sym]) || @default_topic || tag
+      topic = if @topic
+                extract_placeholders(@topic, chunk)
+              else
+                (chunk.metadata.variables && chunk.metadata.variables[@topic_key_sym]) || @default_topic || tag
+              end
 
       handlers = []
       record_buf = nil
@@ -320,9 +326,13 @@ DESC
         }
       end
     rescue Exception => e
-      log.warn "Send exception occurred: #{e} at #{e.backtrace.first}"
-      # Raise exception to retry sendind messages
-      raise e
+      if @discard_kafka_delivery_failed
+        log.warn "Delivery failed. Discard events:", :error => e.to_s, :error_class => e.class.to_s, :tag => tag
+      else
+        log.warn "Send exception occurred: #{e} at #{e.backtrace.first}"
+        # Raise exception to retry sendind messages
+        raise e
+      end
     end
 
     def enqueue_with_retry(producer, topic, record_buf, message_key, partition, headers)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.12.4
+  version: 0.14.2
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-03-03 00:00:00.000000000 Z
+date: 2020-08-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -51,20 +51,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.7.8
+        version: 1.2.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 0.8.0
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.7.8
+        version: 1.2.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 0.8.0
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement