fluent-plugin-kafka 0.10.0 → 0.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e2c8a3efaae4c3daecbc360c8092aa17a46a0bbcd2e533243f6f7ed375eb3aa4
-  data.tar.gz: e2ae435c6923d71ea46e4fe3a5bfdd599cfb08aeea309da461ab15196cebdc07
+  metadata.gz: 51192d8576ab4ca9653af26130f08f51b61b2004ebeb64759139f08cd90844c9
+  data.tar.gz: 731fea3399de38b4f0b6133704db5abbfe7d904a22ac1d71bb03875770a02e7d
 SHA512:
-  metadata.gz: 233159921bb7c2cd699b2e10939d26bd90e3280bfae56b9e5146dc7ac75619e3d91256e1f404c5ab412726bcf74e980f61b0e52600290b0b927579969aa21937
-  data.tar.gz: f20ce22e379eb1785667d19ff95089b70c9cd33bbe0517f52e4c67b733cebc8128e6096b7376468fe15c1aea4aea3c9347f3413ad1c8535be58579f16f409583
+  metadata.gz: b00872ab8b675af0d00df8d11e14709036d1694ad98613fdbe5ff6ce9de1dd69c47cfe00af3ebc66fcf9a9093566c492145e100436fa032c60dea14192b952c6
+  data.tar.gz: 41a4cda769893e18dcfaa213d96f0c270bd7e059a1a003dda7fb9a5b2ac2cdbc61c49698ee47ad6003d59c652694fb4a4f0a13451e6354b1cb024a695e7f8828

data/ChangeLog

@@ -1,3 +1,10 @@
+Release 0.11.0 - 2019/07/17
+
+	* out_kafka2: Add connect_timeout / socket_timeout parameters
+	* out_kafka2/out_rdkafka2: Add headers / headers_from_record parameters
+	* output: Add use_event_time parameter to use fluentd event time for CreateTime
+	* input: Add get_kafka_client_log parameter. Default is `false`
+
 Release 0.10.0 - 2019/07/03
 
 	* output: Fixed max_send_limit_bytes parameter to be in all kafka outputs

data/README.md

@@ -158,6 +158,8 @@ This plugin is for fluentd v1.0 or later. This will be `out_kafka` plugin in the
       exclude_topic_key     (bool)   :default => false
       exclude_partition_key (bool)   :default => false
       get_kafka_client_log  (bool)   :default => false
+      headers               (hash)   :default => {}
+      headers_from_record   (hash)   :default => {}
       use_default_for_unknown_topic (bool) :default => false
 
       <format>
@@ -245,6 +247,33 @@ If key name `partition_key_key` exists in a message, this plugin set the value o
 
 If key name `message_key_key` exists in a message, this plugin publishes the value of message_key_key to kafka and can be read by consumers. Same message key will be assigned to all messages by setting `default_message_key` in config file. If message_key_key exists and if partition_key_key is not set explicitly, messsage_key_key will be used for partitioning.
 
+#### Headers
+It is possible to set headers on Kafka messages. This only works for kafka2 and rdkafka2 output plugin.
+
+The format is like key1:value1,key2:value2. For example:
+
+    <match app.**>
+      @type kafka2
+      [...]
+      headers some_header_name:some_header_value
+    <match>
+
+You may set header values based on a value of a fluentd record field. For example, imagine a fluentd record like:
+
+    {"source": { "ip": "127.0.0.1" }, "payload": "hello world" }
+
+And the following fluentd config:
+
+    <match app.**>
+      @type kafka2
+      [...]
+      headers_from_record source_ip:$.source.ip
+    <match>
+
+The Kafka message will have a header of source_ip=12.7.0.0.1.
+
+The configuration format is jsonpath. It is descibed in https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-record_accessor
+
 ### Buffered output plugin
 
 This plugin uses ruby-kafka producer for writing data. This plugin works with recent kafka versions. This plugin is for v0.12. If you use v1, see `kafka2`.
@@ -344,6 +373,8 @@ You need to install rdkafka gem.
       default_message_key   (string) :default => nil
       exclude_topic_key     (bool) :default => false
       exclude_partition_key (bool) :default => false
+      headers               (hash)   :default => {}
+      headers_from_record   (hash)   :default => {}
 
       <format>
         @type (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json

data/fluent-plugin-kafka.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.10.0'
+  gem.version       = '0.11.0'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/in_kafka.rb

@@ -36,6 +36,7 @@ class Fluent::KafkaInput < Fluent::Input
   config_param :offset_zk_root_node, :string, :default => '/fluent-plugin-kafka'
   config_param :use_record_time, :bool, :default => false,
                :desc => "Replace message timestamp with contents of 'time' field."
+  config_param :get_kafka_client_log, :bool, :default => false
   config_param :time_format, :string, :default => nil,
                :desc => "Time format to be used to parse 'time' filed."
   config_param :kafka_message_key, :string, :default => nil,
@@ -175,17 +176,18 @@ class Fluent::KafkaInput < Fluent::Input
     opt[:max_wait_time] = @max_wait_time if @max_wait_time
     opt[:min_bytes] = @min_bytes if @min_bytes
 
+    logger = @get_kafka_client_log ? log : nil
     if @scram_mechanism != nil && @username != nil && @password != nil
-      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: log, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_scram_username: @username, sasl_scram_password: @password,
                          sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl)
     elsif @username != nil && @password != nil
-      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: log, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system,sasl_plain_username: @username, sasl_plain_password: @password)
     else
-      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: log, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab)
     end

data/lib/fluent/plugin/in_kafka_group.rb

@@ -26,6 +26,7 @@ class Fluent::KafkaGroupInput < Fluent::Input
                :desc => "How long to stop event consuming when BufferQueueLimitError happens. Wait retry_emit_limit x 1s. The default is waiting until BufferQueueLimitError is resolved"
   config_param :use_record_time, :bool, :default => false,
                :desc => "Replace message timestamp with contents of 'time' field."
+  config_param :get_kafka_client_log, :bool, :default => false
   config_param :time_format, :string, :default => nil,
                :desc => "Time format to be used to parse 'time' filed."
   config_param :kafka_message_key, :string, :default => nil,
@@ -155,17 +156,18 @@ class Fluent::KafkaGroupInput < Fluent::Input
   def start
     super
 
+    logger = @get_kafka_client_log ? log : nil
     if @scram_mechanism != nil && @username != nil && @password != nil
-      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: log, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_scram_username: @username, sasl_scram_password: @password,
                          sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl)
     elsif @username != nil && @password != nil
-      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: log, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_plain_username: @username, sasl_plain_password: @password)
     else
-      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: log, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab)
     end

data/lib/fluent/plugin/kafka_producer_ext.rb

@@ -12,9 +12,7 @@ module Kafka
   EMPTY_HEADER = {}
 
   class Producer
-    def produce_for_buffered(value, key: nil, topic:, partition: nil, partition_key: nil)
-      create_time = Time.now
-
+    def produce_for_buffered(value, key: nil, topic:, partition: nil, partition_key: nil, create_time: Time.now)
       message = PendingMessage.new(
         value: value,
         key: key,
@@ -99,13 +97,11 @@ module Kafka
       @pending_message_queue = PendingMessageQueue.new
     end
 
-    def produce(value, key: nil, partition: nil, partition_key: nil)
-      create_time = Time.now
-
+    def produce(value, key: nil, partition: nil, partition_key: nil, headers: EMPTY_HEADER, create_time: Time.now)
       message = PendingMessage.new(
         value: value,
         key: key,
-        headers: EMPTY_HEADER,
+        headers: headers,
         topic: @topic,
         partition: partition,
         partition_key: partition_key,

data/lib/fluent/plugin/out_kafka2.rb

@@ -8,7 +8,7 @@ module Fluent::Plugin
   class Fluent::Kafka2Output < Output
     Fluent::Plugin.register_output('kafka2', self)
 
-    helpers :inject, :formatter, :event_emitter
+    helpers :inject, :formatter, :event_emitter, :record_accessor
 
     config_param :brokers, :array, :value_type => :string, :default => ['localhost:9092'],
                  :desc => <<-DESC
@@ -39,12 +39,22 @@ DESC
                  :desc => 'Set true to remove partition key from data'
     config_param :exclude_topic_key, :bool, :default => false,
                  :desc => 'Set true to remove topic name key from data'
+    config_param :use_event_time, :bool, :default => false, :desc => 'Use fluentd event time for kafka create_time'
+    config_param :headers, :hash, default: {}, symbolize_keys: true, value_type: :string,
+                 :desc => 'Kafka message headers'
+    config_param :headers_from_record, :hash, default: {}, symbolize_keys: true, value_type: :string,
+                 :desc => 'Kafka message headers where the header value is a jsonpath to a record value'
 
     config_param :get_kafka_client_log, :bool, :default => false
 
     config_param :ignore_exceptions, :array, :default => [], value_type: :string, :desc => "Ignorable exception list"
     config_param :exception_backup, :bool, :default => true, :desc => "Chunk backup flag when ignore exception occured"
 
+    config_param :connect_timeout, :integer, :default => nil,
+                 :desc => "The timeout setting for connecting to brokers"
+    config_param :socket_timeout, :integer, :default => nil,
+                 :desc => "The timeout setting for socket connection"
+
     # ruby-kafka producer options
     config_param :max_send_retries, :integer, :default => 2,
                  :desc => "Number of times to retry sending of messages to a leader."
@@ -84,17 +94,17 @@ DESC
       begin
         logger = @get_kafka_client_log ? log : nil
         if @scram_mechanism != nil && @username != nil && @password != nil
-          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_client_cert_chain: read_ssl_file(@ssl_client_cert_chain),
                              ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_scram_username: @username, sasl_scram_password: @password,
                              sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
         elsif @username != nil && @password != nil
-          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_client_cert_chain: read_ssl_file(@ssl_client_cert_chain),
                              ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_plain_username: @username, sasl_plain_password: @password, sasl_over_ssl: @sasl_over_ssl,
                              ssl_verify_hostname: @ssl_verify_hostname)
         else
-          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_client_cert_chain: read_ssl_file(@ssl_client_cert_chain),
                              ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab, sasl_over_ssl: @sasl_over_ssl,
                              ssl_verify_hostname: @ssl_verify_hostname)
@@ -155,6 +165,11 @@ DESC
       end
 
       @topic_key_sym = @topic_key.to_sym
+
+      @headers_from_record_accessors = {}
+      @headers_from_record.each do |key, value|
+        @headers_from_record_accessors[key] = record_accessor_create(value)
+      end
     end
 
     def multi_workers_ready?
@@ -205,6 +220,8 @@ DESC
       messages = 0
       record_buf = nil
 
+      headers = @headers.clone
+
       begin
         producer = @kafka.topic_producer(topic, @producer_opts)
         chunk.msgpack_each { |time, record|
@@ -215,6 +232,10 @@ DESC
             partition = (@exclude_partition ? record.delete(@partition_key) : record[@partition_key]) || @default_partition
             message_key = (@exclude_message_key ? record.delete(@message_key_key) : record[@message_key_key]) || @default_message_key
 
+            @headers_from_record_accessors.each do |key, header_accessor|
+              headers[key] = header_accessor.call(record)
+            end
+
             record_buf = @formatter_proc.call(tag, time, record)
             record_buf_bytes = record_buf.bytesize
             if @max_send_limit_bytes && record_buf_bytes > @max_send_limit_bytes
@@ -229,7 +250,8 @@ DESC
           log.trace { "message will send to #{topic} with partition_key: #{partition_key}, partition: #{partition}, message_key: #{message_key} and value: #{record_buf}." }
           messages += 1
 
-          producer.produce(record_buf, key: message_key, partition_key: partition_key, partition: partition)
+          producer.produce(record_buf, key: message_key, partition_key: partition_key, partition: partition, headers: headers,
+                           create_time: @use_event_time ? Time.at(time) : Time.now)
         }
 
         if messages > 0

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -46,14 +46,15 @@ DESC
                :desc => <<-DESC
 Set true to remove partition from data
 DESC
-   config_param :exclude_message_key, :bool, :default => false,
+  config_param :exclude_message_key, :bool, :default => false,
                :desc => <<-DESC
 Set true to remove message key from data
 DESC
-   config_param :exclude_topic_key, :bool, :default => false,
+  config_param :exclude_topic_key, :bool, :default => false,
                 :desc => <<-DESC
 Set true to remove topic name key from data
 DESC
+  config_param :use_event_time, :bool, :default => false, :desc => 'Use fluentd event time for kafka create_time'
 
   config_param :kafka_agg_max_bytes, :size, :default => 4*1024  #4k
   config_param :kafka_agg_max_messages, :integer, :default => nil
@@ -339,7 +340,8 @@ DESC
         end
         log.trace { "message will send to #{topic} with partition_key: #{partition_key}, partition: #{partition}, message_key: #{message_key} and value: #{record_buf}." }
         messages += 1
-        producer.produce_for_buffered(record_buf, topic: topic, key: message_key, partition_key: partition_key, partition: partition)
+        producer.produce_for_buffered(record_buf, topic: topic, key: message_key, partition_key: partition_key, partition: partition,
+                                      create_time: @use_event_time ? Time.at(time) : Time.now)
         messages_bytes += record_buf_bytes
 
         records_by_topic[topic] += 1

data/lib/fluent/plugin/out_rdkafka2.rb

@@ -24,7 +24,7 @@ module Fluent::Plugin
   class Fluent::Rdkafka2Output < Output
     Fluent::Plugin.register_output('rdkafka2', self)
 
-    helpers :inject, :formatter
+    helpers :inject, :formatter, :record_accessor
 
     config_param :brokers, :string, :default => 'localhost:9092',
                  :desc => <<-DESC
@@ -54,6 +54,11 @@ DESC
                  :desc => <<-DESC
 Set true to remove topic key from data
 DESC
+    config_param :headers, :hash, default: {}, symbolize_keys: true, value_type: :string,
+                 :desc => 'Kafka message headers'
+    config_param :headers_from_record, :hash, default: {}, symbolize_keys: true, value_type: :string,
+                 :desc => 'Kafka message headers where the header value is a jsonpath to a record value'
+
     config_param :max_send_retries, :integer, :default => 2,
                  :desc => "Number of times to retry sending of messages to a leader. Used for message.send.max.retries"
     config_param :required_acks, :integer, :default => -1,
@@ -128,6 +133,11 @@ DESC
       end
       @formatter_proc = setup_formatter(formatter_conf)
       @topic_key_sym = @topic_key.to_sym
+
+      @headers_from_record_accessors = {}
+      @headers_from_record.each do |key, value|
+        @headers_from_record_accessors[key] = record_accessor_create(value)
+      end
     end
 
     def build_config
@@ -236,6 +246,8 @@ DESC
       record_buf = nil
       record_buf_bytes = nil
 
+      headers = @headers.clone
+
       begin
         producer = get_producer
         chunk.msgpack_each { |time, record|
@@ -245,6 +257,10 @@ DESC
             partition = (@exclude_partition ? record.delete(@partition_key) : record[@partition_key]) || @default_partition
             message_key = (@exclude_message_key ? record.delete(@message_key_key) : record[@message_key_key]) || @default_message_key
 
+            @headers_from_record_accessors.each do |key, header_accessor|
+              headers[key] = header_accessor.call(record)
+            end
+
             record_buf = @formatter_proc.call(tag, time, record)
             record_buf_bytes = record_buf.bytesize
             if @max_send_limit_bytes && record_buf_bytes > @max_send_limit_bytes
@@ -256,7 +272,7 @@ DESC
             next
           end
 
-          handlers << enqueue_with_retry(producer, topic, record_buf, message_key, partition)
+          handlers << enqueue_with_retry(producer, topic, record_buf, message_key, partition, headers)
         }
         handlers.each { |handler|
           handler.wait(@rdkafka_delivery_handle_poll_timeout) if @rdkafka_delivery_handle_poll_timeout != 0
@@ -268,11 +284,11 @@ DESC
       raise e
     end
 
-    def enqueue_with_retry(producer, topic, record_buf, message_key, partition)
+    def enqueue_with_retry(producer, topic, record_buf, message_key, partition, headers)
       attempt = 0
       loop do
         begin
-          return producer.produce(topic: topic, payload: record_buf, key: message_key, partition: partition)
+          return producer.produce(topic: topic, payload: record_buf, key: message_key, partition: partition, headers: headers)
         rescue Exception => e
           if e.respond_to?(:code) && e.code == :queue_full
             if attempt <= @max_enqueue_retries

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.10.0
+  version: 0.11.0
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-07-03 00:00:00.000000000 Z
+date: 2019-07-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd