fluent-plugin-kafka 0.17.3 → 0.18.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 380a588c3128af581568815d5fdec9edea856c702832de8be87a9e83a012a8b7
-  data.tar.gz: afd5d2e7ba2b89a6948637cbb325b3b0683e0c23ac37a591f717902a78a5ba1b
+  metadata.gz: 0a8590f6d34bbdb2faa991bba6e32a1424623c7ddd9609dc1d2ffcbeabaa20e1
+  data.tar.gz: a653d9dba00fa82f18071304be9093cd6d620bf4bda29d5e00a75113d05e8e3e
 SHA512:
-  metadata.gz: fe4f49213a66d2e58bb4177f888ab3c0f3cf0fefe3ca2fa5972bc6394ee46c34c7f054088efd035c10e3beb4d277a5368f0f17ff9aa81954d0340e0962e31902
-  data.tar.gz: 1c59d58e36828d61eb8d42b9c60578850dd8a439b21faadc0261d7f6b873a7d3874797923b341b9fc6fc6ffce59e1bff4ac90433e8b329580df2031b41d4005a
+  metadata.gz: 69a784e7d69e2229a036110997b2691143008d4ca786e18415e9a806a6d577b38465a4a3931166418a5b7e022dddbc62d014c0c784277ccaaecc9b6570822564
+  data.tar.gz: 9f14d8d10a45bf390c801c93d79b893f4248ded7e37dc87f8c4453b04158754e1006aa7603874986e8c93812fe899197219abccb3bdb0ae4daacef6d516a0af3

data/.github/workflows/linux.yml

@@ -1,7 +1,9 @@
 name: linux
 on:
-  - push
-  - pull_request
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
 jobs:
   build:
     runs-on: ${{ matrix.os }}
@@ -10,7 +12,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.4', '2.5', '2.6', '2.7', '3.0' ]
+        ruby: [ '3.1', '3.0', '2.7', '2.6' ]
         os:
           - ubuntu-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/ChangeLog

@@ -1,3 +1,14 @@
+Release 0.18.0 - 2022/07/21
+	* out_kafka2: Keep alive Kafka connections between flushes
+	* out_rdkafka2: Enable to set SASL credentials via `username` and `password` parameters
+	* out_kafka2/out_rdkafka2: Add `record_key` parameter
+
+Release 0.17.5 - 2022/03/18
+	* out_kafka2: Add `resolve_seed_brokers` parameter
+
+Release 0.17.4 - 2022/01/25
+	* in_kafka_group: Add `refresh_topic_interval` parameter
+
 Release 0.17.3 - 2021/11/26
 	* output: Suppress large warning logs for events skipped by `max_send_limit_bytes`
 

data/README.md

@@ -135,6 +135,7 @@ Consume events by kafka consumer group features..
       offset_commit_interval  (integer) :default => nil (Use default of ruby-kafka)
       offset_commit_threshold (integer) :default => nil (Use default of ruby-kafka)
       fetcher_max_queue_size  (integer) :default => nil (Use default of ruby-kafka)
+      refresh_topic_interval  (integer) :default => nil (Use default of ruby-kafka)
       start_from_beginning    (bool)    :default => true
     </source>
 
@@ -192,6 +193,7 @@ If `ruby-kafka` doesn't fit your kafka environment, check `rdkafka2` plugin inst
       message_key_key       (string) :default => 'message_key'
       default_topic         (string) :default => nil
       default_partition_key (string) :default => nil
+      record_key            (string) :default => nil
       default_message_key   (string) :default => nil
       exclude_topic_key     (bool)   :default => false
       exclude_partition_key (bool)   :default => false
@@ -204,6 +206,7 @@ If `ruby-kafka` doesn't fit your kafka environment, check `rdkafka2` plugin inst
       use_default_for_unknown_topic (bool) :default => false
       discard_kafka_delivery_failed (bool) :default => false (No discard)
       partitioner_hash_function (enum) (crc32|murmur2) :default => 'crc32'
+      share_producer        (bool)   :default => false
 
       <format>
         @type (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
@@ -334,6 +337,40 @@ For example, `$.source.ip` can be extracted with config `headers_from_record` an
 
 > Using this config to remove unused fields is discouraged. A [filter plugin](https://docs.fluentd.org/v/0.12/filter) can be used for this purpose.
 
+#### Send only a sub field as a message payload
+
+If `record_key` is provided, the plugin sends only a sub field given by that key.
+The configuration format is jsonpath.
+
+e.g. When the following configuration and the incoming record are given:
+
+configuration:
+
+    <match **>
+      @type kafka2
+      [...]
+      record_key '$.data'
+    </match>
+
+record:
+
+    {
+        "specversion" : "1.0",
+        "type" : "com.example.someevent",
+        "id" : "C234-1234-1234",
+        "time" : "2018-04-05T17:31:00Z",
+        "datacontenttype" : "application/json",
+        "data" : {
+            "appinfoA" : "abc",
+            "appinfoB" : 123,
+            "appinfoC" : true
+        },
+        ...
+    }
+
+only the `data` field will be serialized by the formatter and sent to Kafka.
+The toplevel `data` key will be removed.
+
 ### Buffered output plugin
 
 This plugin uses ruby-kafka producer for writing data. This plugin is for v0.12. If you use v1, see `kafka2`.
@@ -459,6 +496,7 @@ You need to install rdkafka gem.
       # same with kafka2
       headers               (hash) :default => {}
       headers_from_record   (hash) :default => {}
+      record_key            (string) :default => nil
 
       <format>
         @type (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json

data/fluent-plugin-kafka.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.17.3'
+  gem.version       = '0.18.0'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/in_kafka_group.rb

@@ -67,6 +67,8 @@ class Fluent::KafkaGroupInput < Fluent::Input
                :desc => "The number of messages that can be processed before their offsets are committed"
   config_param :fetcher_max_queue_size, :integer, :default => nil,
                :desc => "The number of fetched messages per partition that are queued in fetcher queue"
+  config_param :refresh_topic_interval, :integer, :default => nil,
+               :desc => "The interval of refreshing the topic list in seconds. Zero or unset disables this"
   config_param :start_from_beginning, :bool, :default => true,
                :desc => "Whether to start from the beginning of the topic or just subscribe to new messages being produced"
 
@@ -128,6 +130,7 @@ class Fluent::KafkaGroupInput < Fluent::Input
     @consumer_opts[:offset_commit_interval] = @offset_commit_interval if @offset_commit_interval
     @consumer_opts[:offset_commit_threshold] = @offset_commit_threshold if @offset_commit_threshold
     @consumer_opts[:fetcher_max_queue_size] = @fetcher_max_queue_size if @fetcher_max_queue_size
+    @consumer_opts[:refresh_topic_interval] = @refresh_topic_interval if @refresh_topic_interval
 
     @fetch_opts = {}
     @fetch_opts[:max_wait_time] = @max_wait_time if @max_wait_time

data/lib/fluent/plugin/out_kafka2.rb

@@ -27,6 +27,11 @@ DESC
     config_param :partitioner_hash_function, :enum, list: [:crc32, :murmur2], :default => :crc32,
                  :desc => "Specify kafka patrtitioner hash algorithm"
     config_param :default_partition, :integer, :default => nil
+    config_param :record_key, :string, :default => nil,
+                 :desc => <<-DESC
+A jsonpath to a record value pointing to the field which will be passed to the formatter and sent as the Kafka message payload.
+If defined, only this field in the record will be sent to Kafka as the message payload.
+DESC
     config_param :use_default_for_unknown_topic, :bool, :default => false, :desc => "If true, default_topic is used when topic not found"
     config_param :client_id, :string, :default => 'fluentd'
     config_param :idempotent, :bool, :default => false, :desc => 'Enable idempotent producer'
@@ -49,6 +54,8 @@ DESC
                  :desc => 'Kafka message headers'
     config_param :headers_from_record, :hash, default: {}, symbolize_keys: true, value_type: :string,
                  :desc => 'Kafka message headers where the header value is a jsonpath to a record value'
+    config_param :resolve_seed_brokers, :bool, :default => false,
+                 :desc => "support brokers' hostname with multiple addresses"
 
     config_param :get_kafka_client_log, :bool, :default => false
 
@@ -79,6 +86,7 @@ DESC
 Add a regular expression to capture ActiveSupport notifications from the Kafka client
 requires activesupport gem - records will be generated under fluent_kafka_stats.**
 DESC
+    config_param :share_producer, :bool, :default => false, :desc => 'share kafka producer between flush threads'
 
     config_section :buffer do
       config_set_default :chunk_keys, ["topic"]
@@ -94,6 +102,12 @@ DESC
       super
 
       @kafka = nil
+      @producers = nil
+      @producers_mutex = nil
+      @shared_producer = nil
+
+      @writing_threads_mutex = Mutex.new
+      @writing_threads = Set.new
     end
 
     def refresh_client(raise_error = true)
@@ -103,19 +117,19 @@ DESC
           @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert_file_path: @ssl_ca_cert,
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_client_cert_chain: read_ssl_file(@ssl_client_cert_chain),
                              ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_scram_username: @username, sasl_scram_password: @password,
-                             sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname,
+                             sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname, resolve_seed_brokers: @resolve_seed_brokers,
                              partitioner: Kafka::Partitioner.new(hash_function: @partitioner_hash_function))
         elsif @username != nil && @password != nil
           @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert_file_path: @ssl_ca_cert,
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_client_cert_chain: read_ssl_file(@ssl_client_cert_chain),
                              ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_plain_username: @username, sasl_plain_password: @password, sasl_over_ssl: @sasl_over_ssl,
-                             ssl_verify_hostname: @ssl_verify_hostname,
+                             ssl_verify_hostname: @ssl_verify_hostname, resolve_seed_brokers: @resolve_seed_brokers,
                              partitioner: Kafka::Partitioner.new(hash_function: @partitioner_hash_function))
         else
           @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert_file_path: @ssl_ca_cert,
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_client_cert_chain: read_ssl_file(@ssl_client_cert_chain),
                              ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab, sasl_over_ssl: @sasl_over_ssl,
-                             ssl_verify_hostname: @ssl_verify_hostname,
+                             ssl_verify_hostname: @ssl_verify_hostname, resolve_seed_brokers: @resolve_seed_brokers,
                              partitioner: Kafka::Partitioner.new(hash_function: @partitioner_hash_function))
         end
         log.info "initialized kafka producer: #{@client_id}"
@@ -183,15 +197,29 @@ DESC
       @exclude_field_accessors = @exclude_fields.map do |field|
         record_accessor_create(field)
       end
+
+      @record_field_accessor = nil
+      @record_field_accessor = record_accessor_create(@record_key) unless @record_key.nil?
     end
 
     def multi_workers_ready?
       true
     end
 
+    def create_producer
+      @kafka.producer(**@producer_opts)
+    end
+
     def start
       super
       refresh_client
+
+      if @share_producer
+        @shared_producer = create_producer
+      else
+        @producers = {}
+        @producers_mutex = Mutex.new
+      end
     end
 
     def close
@@ -204,6 +232,56 @@ DESC
       @kafka = nil
     end
 
+    def wait_writing_threads
+      done = false
+      until done do
+        @writing_threads_mutex.synchronize do
+          done = true if @writing_threads.empty?
+        end
+        sleep(1) unless done
+      end
+    end
+
+    def shutdown
+      super
+      wait_writing_threads
+      shutdown_producers
+    end
+
+    def shutdown_producers
+      if @share_producer
+        @shared_producer.shutdown
+        @shared_producer = nil
+      else
+        @producers_mutex.synchronize {
+          shutdown_threads = @producers.map { |key, producer|
+            th = Thread.new {
+              producer.shutdown
+            }
+            th.abort_on_exception = true
+            th
+          }
+          shutdown_threads.each { |th| th.join }
+          @producers = {}
+        }
+      end
+    end
+
+    def get_producer
+      if @share_producer
+        @shared_producer
+      else
+        @producers_mutex.synchronize {
+          producer = @producers[Thread.current.object_id]
+          unless producer
+            producer = create_producer
+            @producers[Thread.current.object_id] = producer
+          end
+          producer
+        }
+      end
+    end
+
     def setup_formatter(conf)
       type = conf['@type']
       case type
@@ -227,6 +305,8 @@ DESC
 
     # TODO: optimize write performance
     def write(chunk)
+      @writing_threads_mutex.synchronize { @writing_threads.add(Thread.current) }
+
       tag = chunk.metadata.tag
       topic = if @topic
                 extract_placeholders(@topic, chunk)
@@ -235,13 +315,12 @@ DESC
               end
 
       messages = 0
-      record_buf = nil
 
       base_headers = @headers
       mutate_headers = !@headers_from_record_accessors.empty?
 
       begin
-        producer = @kafka.topic_producer(topic, **@producer_opts)
+        producer = get_producer
         chunk.msgpack_each { |time, record|
           begin
             record = inject_values_to_record(tag, time, record)
@@ -265,6 +344,7 @@ DESC
               end
             end
 
+            record = @record_field_accessor.call(record) unless @record_field_accessor.nil?
             record_buf = @formatter_proc.call(tag, time, record)
             record_buf_bytes = record_buf.bytesize
             if @max_send_limit_bytes && record_buf_bytes > @max_send_limit_bytes
@@ -281,7 +361,7 @@ DESC
           messages += 1
 
           producer.produce(record_buf, key: message_key, partition_key: partition_key, partition: partition, headers: headers,
-                           create_time: @use_event_time ? Time.at(time) : Time.now)
+                           create_time: @use_event_time ? Time.at(time) : Time.now, topic: topic)
         }
 
         if messages > 0
@@ -299,7 +379,6 @@ DESC
         end
       rescue Kafka::UnknownTopicOrPartition
         if @use_default_for_unknown_topic && topic != @default_topic
-          producer.shutdown if producer
           log.warn "'#{topic}' topic not found. Retry with '#{default_topic}' topic"
           topic = @default_topic
           retry
@@ -319,7 +398,7 @@ DESC
       # Raise exception to retry sendind messages
       raise e unless ignore
     ensure
-      producer.shutdown if producer
+      @writing_threads_mutex.synchronize { @writing_threads.delete(Thread.current) }
     end
   end
 end

data/lib/fluent/plugin/out_rdkafka2.rb

@@ -73,6 +73,11 @@ DESC
                  :desc => <<-DESC
 The codec the producer uses to compress messages. Used for compression.codec
 Supported codecs: (gzip|snappy)
+DESC
+    config_param :record_key, :string, :default => nil,
+                 :desc => <<-DESC
+A jsonpath to a record value pointing to the field which will be passed to the formatter and sent as the Kafka message payload.
+If defined, only this field in the record will be sent to Kafka as the message payload.
 DESC
     config_param :use_event_time, :bool, :default => false, :desc => 'Use fluentd event time for rdkafka timestamp'
     config_param :max_send_limit_bytes, :size, :default => nil
@@ -230,6 +235,9 @@ DESC
       end
 
       @enqueue_rate = EnqueueRate.new(@max_enqueue_bytes_per_second) unless @max_enqueue_bytes_per_second.nil?
+
+      @record_field_accessor = nil
+      @record_field_accessor = record_accessor_create(@record_key) unless @record_key.nil?
     end
 
     def build_config
@@ -270,6 +278,8 @@ DESC
       config[:"queue.buffering.max.messages"] = @rdkafka_buffering_max_messages if @rdkafka_buffering_max_messages
       config[:"message.max.bytes"] = @rdkafka_message_max_bytes if @rdkafka_message_max_bytes
       config[:"batch.num.messages"] = @rdkafka_message_max_num if @rdkafka_message_max_num
+      config[:"sasl.username"] = @username if @username
+      config[:"sasl.password"] = @password if @password
 
       @rdkafka_options.each { |k, v|
         config[k.to_sym] = v
@@ -371,8 +381,6 @@ DESC
               end
 
       handlers = []
-      record_buf = nil
-      record_buf_bytes = nil
 
       headers = @headers.clone
 
@@ -395,6 +403,7 @@ DESC
               end
             end
 
+            record = @record_field_accessor.call(record) unless @record_field_accessor.nil?
             record_buf = @formatter_proc.call(tag, time, record)
             record_buf_bytes = record_buf.bytesize
             if @max_send_limit_bytes && record_buf_bytes > @max_send_limit_bytes

data/test/plugin/test_in_kafka_group.rb

@@ -14,6 +14,7 @@ class KafkaGroupInputTest < Test::Unit::TestCase
     brokers localhost:9092
     consumer_group fluentd
     format text
+    refresh_topic_interval 0
     @label @kafka
     topics #{TOPIC_NAME}
   ]
@@ -52,6 +53,7 @@ class KafkaGroupInputTest < Test::Unit::TestCase
         brokers localhost:9092
         format text
         @label @kafka
+        refresh_topic_interval 0
         topics #{TOPIC_NAME}
       ]
       d = create_driver

data/test/plugin/test_out_kafka2.rb

@@ -60,6 +60,13 @@ class Kafka2OutputTest < Test::Unit::TestCase
     assert_equal true, d.instance.multi_workers_ready?
   end
 
+  def test_resolve_seed_brokers
+    d = create_driver(config + config_element('ROOT', '', {"resolve_seed_brokers" => true}))
+    assert_nothing_raised do
+      d.instance.refresh_client
+    end
+  end
+
   class WriteTest < self
     TOPIC_NAME = "kafka-output-#{SecureRandom.uuid}"
 
@@ -98,6 +105,21 @@ class Kafka2OutputTest < Test::Unit::TestCase
       assert_equal([expected_message], actual_messages)
     end
 
+    def test_record_key
+      conf = config(default_topic: TOPIC_NAME) +
+        config_element('ROOT', '', {"record_key" => "$.data"}, [])
+      target_driver = create_target_driver
+      target_driver.run(expect_records: 1, timeout: 5) do
+        sleep 2
+        d = create_driver(conf)
+        d.run do
+          d.feed('test', event_time, {'data' => {'a' => 'b', 'foo' => 'bar', 'message' => 'test'}, 'message_key' => '123456'})
+        end
+      end
+      actual_messages = target_driver.events.collect { |event| event[2] }
+      assert_equal([{'a' => 'b', 'foo' => 'bar', 'message' => 'test'}], actual_messages)
+    end
+
     def test_exclude_fields
       conf = config(default_topic: TOPIC_NAME) +
              config_element('ROOT', '', {"exclude_fields" => "$.foo"}, [])

data/test/plugin/test_out_rdkafka2.rb

@@ -163,5 +163,20 @@ class Rdkafka2OutputTest < Test::Unit::TestCase
       actual_messages = target_driver.events.collect { |event| event[2] }
       assert_equal(expected_messages, actual_messages)
     end
+
+    def test_record_key
+      conf = config(default_topic: TOPIC_NAME) +
+             config_element('ROOT', '', {"record_key" => "$.data"}, [])
+      target_driver = create_target_driver
+      target_driver.run(expect_records: 1, timeout: 5) do
+        sleep 2
+        d = create_driver(conf)
+        d.run do
+          d.feed('test', event_time, {'data' => {'a' => 'b', 'foo' => 'bar', 'message' => 'test'}, 'message_key' => '123456'})
+        end
+      end
+      actual_messages = target_driver.events.collect { |event| event[2] }
+      assert_equal([{'a' => 'b', 'foo' => 'bar', 'message' => 'test'}], actual_messages)
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.17.3
+  version: 0.18.0
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-11-26 00:00:00.000000000 Z
+date: 2022-07-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -193,7 +193,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.5
+rubygems_version: 3.3.5
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin for Apache Kafka > 0.8