fluent-plugin-kafka 0.17.5 → 0.19.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: eb7f693666ff7fd4bdb43b249d3e60e882ebf99e80402de2a5310b61668ef9e7
-  data.tar.gz: d4b141409cf83402ad1e28b9579317c672b20c9760b8f58b8d54800e8c46313b
+  metadata.gz: 342b66e96b3175aed8f7fe9bc0f29bb03f2737924229eaa63d7ffd408f5209cf
+  data.tar.gz: 89a9361a2c599eca1419668136ecb722c6eaa95e9ebc55f2e2a2be3f1346cafc
 SHA512:
-  metadata.gz: 8f86c7c82fbf5db63ef0c92737ba4935a3a60534d434dfb9552c2ad624ec1e28fea9876c689a29ca6b16c16abc3818fdfb24bd7540495f980daa8a5509b29ca0
-  data.tar.gz: 8b1dfba11d40e12f9b0b1fbf4ff947678fb5ef2ab3fbd3969e363935c8e895c29f497273ddb86c80c4e1ebc9d254045275dc7e08b07788d4573799c97ce7a393
+  metadata.gz: '0172977463edc0ff6085588bcded5bf3d33412e088422bf48954672507d7661bda68735d1415fcc9dcaebd132eff355cc714be619c15a107962e12dc69eba4e1'
+  data.tar.gz: 00054cf6be46a6041fd6f8712f3159f832f91deffcf34feef45fff919d2301b2e04b5ec0493a2d9086af87c3ce9ec37ca0f004a9fac006ee4e3c17d20b372984

data/.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -1,5 +1,6 @@
 name: Bug Report
 description: Create a report with a procedure for reproducing the bug
+labels: "waiting-for-triage"
 body:
   - type: markdown
     attributes:

data/.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -1,5 +1,6 @@
 name: Feature request
 description: Suggest an idea for this project
+labels: "waiting-for-triage"
 body:
   - type: markdown
     attributes:

data/.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: 'github-actions'
+    directory: '/'
+    schedule:
+      interval: 'weekly'

data/.github/workflows/linux.yml

@@ -12,12 +12,15 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '3.1', '3.0', '2.7', '2.6' ]
+        ruby: [ '3.2', '3.1', '3.0', '2.7' ]
         os:
           - ubuntu-latest
-    name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
+        rdkafka_versions:
+          - { min: '>= 0.6.0', max: '< 0.12.0' }
+          - { min: '>= 0.12.0', max: '>= 0.12.0' }
+    name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }} with rdkafka gem version (min ${{ matrix.rdkafka_versions.min }} max ${{ matrix.rdkafka_versions.max }})
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}
@@ -33,6 +36,8 @@ jobs:
     - name: unit testing
       env:
         CI: true
+        RDKAFKA_VERSION_MIN_RANGE: ${{ matrix.rdkafka_versions.min }}
+        RDKAFKA_VERSION_MAX_RANGE: ${{ matrix.rdkafka_versions.max }}
       run: |
         sudo ./ci/prepare-kafka-server.sh
         gem install bundler rake

data/.github/workflows/stale-actions.yml

@@ -7,7 +7,7 @@ jobs:
   stale:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/stale@v3
+    - uses: actions/stale@v8
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         days-before-stale: 90
@@ -18,5 +18,7 @@ jobs:
         close-pr-message: "This PR was automatically closed because of stale in 30 days"
         stale-pr-label: "stale"
         stale-issue-label: "stale"
-        exempt-issue-labels: "bug,enhancement,help wanted"
-        exempt-pr-labels: "bug,enhancement,help wanted"
+        exempt-issue-labels: "bug,enhancement,help wanted,waiting-for-triage"
+        exempt-pr-labels: "bug,enhancement,help wanted,waiting-for-triage"
+        exempt-all-assignees: true
+        exempt-all-milestones: true

data/ChangeLog

@@ -1,3 +1,20 @@
+Release 0.19.1 - 2023/09/20
+	* out_rdkafka2: Add `use_default_for_unknown_topic` & `use_default_for_unknown_partition_error`
+
+Release 0.19.0 - 2023/04/26
+	* out_kafka2: Add support for AWS IAM authentication
+	* in_kafka, in_kafka_group, out_kafka2: Add support for ssl client cert key password
+	* out_rdkafka2: Mask `ssl_client_cert_key_password` on dumping it to log
+	* out_rdkafka2: Support rdkafka-ruby 0.12
+
+Release 0.18.1 - 2022/08/17
+	* out_kafka2: Fix a bug that it doesn't respect `chunk_limit_records` and `chunk_limit_size`
+
+Release 0.18.0 - 2022/07/21
+	* out_kafka2: Keep alive Kafka connections between flushes
+	* out_rdkafka2: Enable to set SASL credentials via `username` and `password` parameters
+	* out_kafka2/out_rdkafka2: Add `record_key` parameter
+
 Release 0.17.5 - 2022/03/18
 	* out_kafka2: Add `resolve_seed_brokers` parameter
 

data/Gemfile

@@ -3,4 +3,4 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in fluent-plugin-kafka.gemspec
 gemspec
 
-gem 'rdkafka', '>= 0.6.0' if ENV["USE_RDKAFKA"]
+gem 'rdkafka', ENV['RDKAFKA_VERSION_MIN_RANGE'], ENV['RDKAFKA_VERSION_MAX_RANGE'] if ENV['USE_RDKAFKA']

data/README.md

@@ -36,6 +36,7 @@ If you want to use zookeeper related parameters, you also need to install zookee
 - ssl_ca_cert
 - ssl_client_cert
 - ssl_client_cert_key
+- ssl_client_cert_key_password
 - ssl_ca_certs_from_system
 
 Set path to SSL related files. See [Encryption and Authentication using SSL](https://github.com/zendesk/ruby-kafka#encryption-and-authentication-using-ssl) for more detail.
@@ -187,12 +188,17 @@ If `ruby-kafka` doesn't fit your kafka environment, check `rdkafka2` plugin inst
       @type kafka2
 
       brokers               <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,.. # Set brokers directly
+
+      # Kafka topic, placerholders are supported. Chunk keys are required in the Buffer section inorder for placeholders
+      # to work.
+      topic                 (string) :default => nil
       topic_key             (string) :default => 'topic'
       partition_key         (string) :default => 'partition'
       partition_key_key     (string) :default => 'partition_key'
       message_key_key       (string) :default => 'message_key'
       default_topic         (string) :default => nil
       default_partition_key (string) :default => nil
+      record_key            (string) :default => nil
       default_message_key   (string) :default => nil
       exclude_topic_key     (bool)   :default => false
       exclude_partition_key (bool)   :default => false
@@ -205,6 +211,17 @@ If `ruby-kafka` doesn't fit your kafka environment, check `rdkafka2` plugin inst
       use_default_for_unknown_topic (bool) :default => false
       discard_kafka_delivery_failed (bool) :default => false (No discard)
       partitioner_hash_function (enum) (crc32|murmur2) :default => 'crc32'
+      share_producer        (bool)   :default => false
+
+      # If you intend to rely on AWS IAM auth to MSK with long lived credentials
+      # https://docs.aws.amazon.com/msk/latest/developerguide/iam-access-control.html
+      #
+      # For AWS STS support, see status in
+      # - https://github.com/zendesk/ruby-kafka/issues/944
+      # - https://github.com/zendesk/ruby-kafka/pull/951
+      sasl_aws_msk_iam_access_key_id (string) :default => nil
+      sasl_aws_msk_iam_secret_key_id (string) :default => nil
+      sasl_aws_msk_iam_aws_region    (string) :default => nil
 
       <format>
         @type (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
@@ -241,13 +258,12 @@ ruby-kafka's log is routed to fluentd log so you can see ruby-kafka's log in flu
 
 Supports following ruby-kafka's producer options.
 
-- max_send_retries - default: 1 - Number of times to retry sending of messages to a leader.
+- max_send_retries - default: 2 - Number of times to retry sending of messages to a leader.
 - required_acks - default: -1 - The number of acks required per request. If you need flush performance, set lower value, e.g. 1, 2.
 - ack_timeout - default: nil - How long the producer waits for acks. The unit is seconds.
 - compression_codec - default: nil - The codec the producer uses to compress messages.
 - max_send_limit_bytes - default: nil - Max byte size to send message to avoid MessageSizeTooLarge. For example, if you set 1000000(message.max.bytes in kafka), Message more than 1000000 byes will be dropped.
 - discard_kafka_delivery_failed - default: false - discard the record where [Kafka::DeliveryFailed](http://www.rubydoc.info/gems/ruby-kafka/Kafka/DeliveryFailed) occurred
-- monitoring_list - default: [] - library to be used to monitor. statsd and datadog are supported
 
 If you want to know about detail of monitoring, see also https://github.com/zendesk/ruby-kafka#monitoring
 
@@ -335,6 +351,40 @@ For example, `$.source.ip` can be extracted with config `headers_from_record` an
 
 > Using this config to remove unused fields is discouraged. A [filter plugin](https://docs.fluentd.org/v/0.12/filter) can be used for this purpose.
 
+#### Send only a sub field as a message payload
+
+If `record_key` is provided, the plugin sends only a sub field given by that key.
+The configuration format is jsonpath.
+
+e.g. When the following configuration and the incoming record are given:
+
+configuration:
+
+    <match **>
+      @type kafka2
+      [...]
+      record_key '$.data'
+    </match>
+
+record:
+
+    {
+        "specversion" : "1.0",
+        "type" : "com.example.someevent",
+        "id" : "C234-1234-1234",
+        "time" : "2018-04-05T17:31:00Z",
+        "datacontenttype" : "application/json",
+        "data" : {
+            "appinfoA" : "abc",
+            "appinfoB" : 123,
+            "appinfoC" : true
+        },
+        ...
+    }
+
+only the `data` field will be serialized by the formatter and sent to Kafka.
+The toplevel `data` key will be removed.
+
 ### Buffered output plugin
 
 This plugin uses ruby-kafka producer for writing data. This plugin is for v0.12. If you use v1, see `kafka2`.
@@ -384,6 +434,16 @@ Support of fluentd v0.12 has ended. `kafka_buffered` will be an alias of `kafka2
       monitoring_list              (array)   :default => []
     </match>
 
+`kafka_buffered` supports the following `ruby-kafka` parameters:
+
+- max_send_retries - default: 2 - Number of times to retry sending of messages to a leader.
+- required_acks - default: -1 - The number of acks required per request. If you need flush performance, set lower value, e.g. 1, 2.
+- ack_timeout - default: nil - How long the producer waits for acks. The unit is seconds.
+- compression_codec - default: nil - The codec the producer uses to compress messages.
+- max_send_limit_bytes - default: nil - Max byte size to send message to avoid MessageSizeTooLarge. For example, if you set 1000000(message.max.bytes in kafka), Message more than 1000000 byes will be dropped.
+- discard_kafka_delivery_failed - default: false - discard the record where [Kafka::DeliveryFailed](http://www.rubydoc.info/gems/ruby-kafka/Kafka/DeliveryFailed) occurred
+- monitoring_list - default: [] - library to be used to monitor. statsd and datadog are supported
+
 `kafka_buffered` has two additional parameters:
 
 - kafka_agg_max_bytes - default: 4096 - Maximum value of total message size to be included in one batch transmission.
@@ -450,6 +510,8 @@ You need to install rdkafka gem.
       partition_key_key     (string) :default => 'partition_key'
       message_key_key       (string) :default => 'message_key'
       default_topic         (string) :default => nil
+      use_default_for_unknown_topic           (bool) :default => false
+      use_default_for_unknown_partition_error (bool) :default => false
       default_partition_key (string) :default => nil
       default_message_key   (string) :default => nil
       exclude_topic_key     (bool) :default => false
@@ -460,6 +522,7 @@ You need to install rdkafka gem.
       # same with kafka2
       headers               (hash) :default => {}
       headers_from_record   (hash) :default => {}
+      record_key            (string) :default => nil
 
       <format>
         @type (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json

data/examples/README.md

@@ -0,0 +1,3 @@
+# Examples
+
+This directory contains example Fluentd config for this plugin

data/examples/out_kafka2/dynamic_topic_based_on_tag.conf

@@ -0,0 +1,32 @@
+<source>
+    @type sample
+    sample {"hello": "world"}
+    rate 7000
+    tag sample.hello.world
+</source>
+
+<match sample.**>
+    @type kafka2
+
+    brokers "broker:29092"
+
+    # Writes to topic `events.sample.hello.world`
+    topic "events.${tag}"
+
+    # Writes to topic `hello.world`
+    # topic "${tag[1]}.${tag[2]}"
+
+    <format>
+        @type json
+    </format>
+
+    <buffer tag>
+      flush_at_shutdown true
+      flush_mode interval
+      flush_interval 1s
+      chunk_limit_size 3MB
+      chunk_full_threshold 1
+      total_limit_size 1024MB
+      overflow_action block
+    </buffer>
+</match>

data/examples/out_kafka2/protobuf-formatter.conf

@@ -0,0 +1,23 @@
+<source>
+    @type sample
+    sample {"hello": "world", "some_record":{"event":"message"}}
+    rate 7000
+    tag sample.hello.world
+</source>
+
+<match sample.**>
+    @type kafka2
+
+    brokers "broker:29092"
+
+    record_key "some_record"
+    default_topic "events"
+
+    <format>
+        # requires the fluent-plugin-formatter-protobuf gem
+        # see its docs for full usage
+        @type protobuf
+        class_name SomeRecord
+        include_paths ["/opt/fluent-plugin-formatter-protobuf/some_record_pb.rb"]
+    </format>
+</match>

data/examples/out_kafka2/record_key.conf

@@ -0,0 +1,31 @@
+<source>
+    @type sample
+    sample {"hello": "world", "some_record":{"event":"message"}}
+    rate 7000
+    tag sample.hello.world
+</source>
+
+<match sample.**>
+    @type kafka2
+
+    brokers "broker:29092"
+
+    # {"event": "message"} will be formatted and sent to Kafka
+    record_key "some_record"
+
+    default_topic "events"
+
+    <format>
+        @type json
+    </format>
+
+    <buffer>
+      flush_at_shutdown true
+      flush_mode interval
+      flush_interval 1s
+      chunk_limit_size 3MB
+      chunk_full_threshold 1
+      total_limit_size 1024MB
+      overflow_action block
+    </buffer>
+</match>

data/fluent-plugin-kafka.gemspec

@@ -13,12 +13,12 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.17.5'
+  gem.version       = '0.19.1'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
   gem.add_dependency 'ltsv'
-  gem.add_dependency 'ruby-kafka', '>= 1.4.0', '< 2'
+  gem.add_dependency 'ruby-kafka', '>= 1.5.0', '< 2'
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "test-unit", ">= 3.0.8"
   gem.add_development_dependency "test-unit-rr", "~> 1.0"

data/lib/fluent/plugin/in_kafka.rb

@@ -200,16 +200,19 @@ class Fluent::KafkaInput < Fluent::Input
     if @scram_mechanism != nil && @username != nil && @password != nil
       @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert_file_path: @ssl_ca_cert,
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+                         ssl_client_cert_key_password: @ssl_client_cert_key_password,
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_scram_username: @username, sasl_scram_password: @password,
                          sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
     elsif @username != nil && @password != nil
       @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert_file_path: @ssl_ca_cert,
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+                         ssl_client_cert_key_password: @ssl_client_cert_key_password,
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system,sasl_plain_username: @username, sasl_plain_password: @password,
                          sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
     else
       @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert_file_path: @ssl_ca_cert,
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+                         ssl_client_cert_key_password: @ssl_client_cert_key_password,
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab,
                          ssl_verify_hostname: @ssl_verify_hostname)
     end

data/lib/fluent/plugin/in_kafka_group.rb

@@ -188,16 +188,19 @@ class Fluent::KafkaGroupInput < Fluent::Input
     if @scram_mechanism != nil && @username != nil && @password != nil
       @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert_file_path: @ssl_ca_cert,
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+                         ssl_client_cert_key_password: @ssl_client_cert_key_password,
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_scram_username: @username, sasl_scram_password: @password,
                          sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
     elsif @username != nil && @password != nil
       @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert_file_path: @ssl_ca_cert,
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+                         ssl_client_cert_key_password: @ssl_client_cert_key_password,
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_plain_username: @username, sasl_plain_password: @password,
                          sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
     else
       @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert_file_path: @ssl_ca_cert,
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+                         ssl_client_cert_key_password: @ssl_client_cert_key_password,
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab,
                          ssl_verify_hostname: @ssl_verify_hostname)
     end

data/lib/fluent/plugin/kafka_plugin_util.rb

@@ -1,5 +1,18 @@
 module Fluent
   module KafkaPluginUtil
+    module AwsIamSettings
+      def self.included(klass)
+        klass.instance_eval do
+          config_param :sasl_aws_msk_iam_access_key_id, :string, :default => nil, secret: true,
+                       desc: "AWS access key Id for IAM authentication to MSK."
+          config_param :sasl_aws_msk_iam_secret_key_id, :string, :default => nil, secret: true,
+                       desc: "AWS access key secret for IAM authentication to MSK."
+          config_param :sasl_aws_msk_iam_aws_region, :string, :default => nil,
+                       desc: "AWS region for IAM authentication to MSK."
+        end
+      end
+    end
+
     module SSLSettings
       def self.included(klass)
         klass.instance_eval {
@@ -10,6 +23,8 @@ module Fluent
                        :desc => "a PEM encoded client cert to use with and SSL connection. Must be used in combination with ssl_client_cert_key."
           config_param :ssl_client_cert_key, :string, :default => nil,
                        :desc => "a PEM encoded client cert key to use with and SSL connection. Must be used in combination with ssl_client_cert."
+          config_param :ssl_client_cert_key_password, :string, :default => nil, secret: true,
+                       :desc => "a PEM encoded client cert key password to use with SSL connection."
           config_param :ssl_client_cert_chain, :string, :default => nil,
                        :desc => "an extra PEM encoded cert to use with and SSL connection."
           config_param :ssl_ca_certs_from_system, :bool, :default => false,

data/lib/fluent/plugin/kafka_producer_ext.rb

@@ -38,9 +38,15 @@ module Kafka
 end
 
 # for out_kafka2
+# Majority (if not all) of this code is lifted from https://github.com/zendesk/ruby-kafka/blob/master/lib/kafka/producer.rb
+# with the main difference where we have removed any checks regarding max_buffer_bytesize and max_buffer_size
+# The reason for doing this is to provide a better UX for our users where they only need to set those bounds in
+# the Buffer section using `chunk_limit_size` and `chunk_limit_records`.
+#
+# We should reconsider this in the future in case the `ruby-kafka` library drastically changes its internal.
 module Kafka
   class Client
-    def topic_producer(topic, compression_codec: nil, compression_threshold: 1, ack_timeout: 5, required_acks: :all, max_retries: 2, retry_backoff: 1, max_buffer_size: 1000, max_buffer_bytesize: 10_000_000, idempotent: false, transactional: false, transactional_id: nil, transactional_timeout: 60)
+    def custom_producer(compression_codec: nil, compression_threshold: 1, ack_timeout: 5, required_acks: :all, max_retries: 2, retry_backoff: 1, max_buffer_size: 1000, max_buffer_bytesize: 10_000_000, idempotent: false, transactional: false, transactional_id: nil, transactional_timeout: 60)
       cluster = initialize_cluster
       compressor = Compressor.new(
         codec_name: compression_codec,
@@ -57,8 +63,7 @@ module Kafka
         transactional_timeout: transactional_timeout,
       )
 
-      TopicProducer.new(topic,
-        cluster: cluster,
+      CustomProducer.new(cluster: cluster,
         transaction_manager: transaction_manager,
         logger: @logger,
         instrumenter: @instrumenter,
@@ -74,8 +79,8 @@ module Kafka
     end
   end
 
-  class TopicProducer
-    def initialize(topic, cluster:, transaction_manager:, logger:, instrumenter:, compressor:, ack_timeout:, required_acks:, max_retries:, retry_backoff:, max_buffer_size:, max_buffer_bytesize:, partitioner:)
+  class CustomProducer
+    def initialize(cluster:, transaction_manager:, logger:, instrumenter:, compressor:, ack_timeout:, required_acks:, max_retries:, retry_backoff:, max_buffer_size:, max_buffer_bytesize:, partitioner:)
       @cluster = cluster
       @transaction_manager = transaction_manager
       @logger = logger
@@ -88,10 +93,6 @@ module Kafka
       @max_buffer_bytesize = max_buffer_bytesize
       @compressor = compressor
       @partitioner = partitioner
-
-      @topic = topic
-      @cluster.add_target_topics(Set.new([topic]))
-
       # A buffer organized by topic/partition.
       @buffer = MessageBuffer.new
 
@@ -99,12 +100,12 @@ module Kafka
       @pending_message_queue = PendingMessageQueue.new
     end
 
-    def produce(value, key: nil, partition: nil, partition_key: nil, headers: EMPTY_HEADER, create_time: Time.now)
+    def produce(value, key: nil, partition: nil, partition_key: nil, headers: EMPTY_HEADER, create_time: Time.now, topic: nil)
       message = PendingMessage.new(
         value: value,
         key: key,
         headers: headers,
-        topic: @topic,
+        topic: topic,
         partition: partition,
         partition_key: partition_key,
         create_time: create_time
@@ -245,12 +246,13 @@ module Kafka
 
     def assign_partitions!
       failed_messages = []
-      partition_count = @cluster.partitions_for(@topic).count
 
       @pending_message_queue.each do |message|
         partition = message.partition
 
         begin
+          partition_count = @cluster.partitions_for(message.topic).count
+
           if partition.nil?
             partition = @partitioner.call(partition_count, message)
           end

data/lib/fluent/plugin/out_kafka2.rb

@@ -27,6 +27,11 @@ DESC
     config_param :partitioner_hash_function, :enum, list: [:crc32, :murmur2], :default => :crc32,
                  :desc => "Specify kafka patrtitioner hash algorithm"
     config_param :default_partition, :integer, :default => nil
+    config_param :record_key, :string, :default => nil,
+                 :desc => <<-DESC
+A jsonpath to a record value pointing to the field which will be passed to the formatter and sent as the Kafka message payload.
+If defined, only this field in the record will be sent to Kafka as the message payload.
+DESC
     config_param :use_default_for_unknown_topic, :bool, :default => false, :desc => "If true, default_topic is used when topic not found"
     config_param :client_id, :string, :default => 'fluentd'
     config_param :idempotent, :bool, :default => false, :desc => 'Enable idempotent producer'
@@ -81,6 +86,7 @@ DESC
 Add a regular expression to capture ActiveSupport notifications from the Kafka client
 requires activesupport gem - records will be generated under fluent_kafka_stats.**
 DESC
+    config_param :share_producer, :bool, :default => false, :desc => 'share kafka producer between flush threads'
 
     config_section :buffer do
       config_set_default :chunk_keys, ["topic"]
@@ -89,6 +95,7 @@ DESC
       config_set_default :@type, 'json'
     end
 
+    include Fluent::KafkaPluginUtil::AwsIamSettings
     include Fluent::KafkaPluginUtil::SSLSettings
     include Fluent::KafkaPluginUtil::SaslSettings
 
@@ -96,30 +103,58 @@ DESC
       super
 
       @kafka = nil
+      @producers = nil
+      @producers_mutex = nil
+      @shared_producer = nil
+
+      @writing_threads_mutex = Mutex.new
+      @writing_threads = Set.new
     end
 
     def refresh_client(raise_error = true)
       begin
         logger = @get_kafka_client_log ? log : nil
+        use_long_lived_aws_credentials = @sasl_aws_msk_iam_access_key_id != nil && @sasl_aws_msk_iam_secret_key_id != nil
         if @scram_mechanism != nil && @username != nil && @password != nil
-          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert_file_path: @ssl_ca_cert,
-                             ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_client_cert_chain: read_ssl_file(@ssl_client_cert_chain),
-                             ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_scram_username: @username, sasl_scram_password: @password,
-                             sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname, resolve_seed_brokers: @resolve_seed_brokers,
-                             partitioner: Kafka::Partitioner.new(hash_function: @partitioner_hash_function))
+          sasl_params = {
+            sasl_scram_username: @username,
+            sasl_scram_password: @password,
+            sasl_scram_mechanism: @scram_mechanism,
+          }
         elsif @username != nil && @password != nil
-          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert_file_path: @ssl_ca_cert,
-                             ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_client_cert_chain: read_ssl_file(@ssl_client_cert_chain),
-                             ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_plain_username: @username, sasl_plain_password: @password, sasl_over_ssl: @sasl_over_ssl,
-                             ssl_verify_hostname: @ssl_verify_hostname, resolve_seed_brokers: @resolve_seed_brokers,
-                             partitioner: Kafka::Partitioner.new(hash_function: @partitioner_hash_function))
+          sasl_params = {
+            sasl_plain_username: @username,
+            sasl_plain_password: @password,
+          }
+        elsif use_long_lived_aws_credentials
+          sasl_params = {
+            sasl_aws_msk_iam_access_key_id: @sasl_aws_msk_iam_access_key_id,
+            sasl_aws_msk_iam_secret_key_id: @sasl_aws_msk_iam_secret_key_id,
+            sasl_aws_msk_iam_aws_region: @sasl_aws_msk_iam_aws_region,
+          }
         else
-          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert_file_path: @ssl_ca_cert,
-                             ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_client_cert_chain: read_ssl_file(@ssl_client_cert_chain),
-                             ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab, sasl_over_ssl: @sasl_over_ssl,
-                             ssl_verify_hostname: @ssl_verify_hostname, resolve_seed_brokers: @resolve_seed_brokers,
-                             partitioner: Kafka::Partitioner.new(hash_function: @partitioner_hash_function))
+          sasl_params = {
+            sasl_gssapi_principal: @principal,
+            sasl_gssapi_keytab: @keytab,
+          }
         end
+        @kafka = Kafka.new(
+          seed_brokers: @seed_brokers,
+          client_id: @client_id,
+          logger: logger,
+          connect_timeout: @connect_timeout,
+          socket_timeout: @socket_timeout,
+          ssl_ca_cert_file_path: @ssl_ca_cert,
+          ssl_client_cert: read_ssl_file(@ssl_client_cert),
+          ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
+          ssl_client_cert_key_password: @ssl_client_cert_key_password,
+          ssl_client_cert_chain: read_ssl_file(@ssl_client_cert_chain),
+          ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
+          ssl_verify_hostname: @ssl_verify_hostname,
+          resolve_seed_brokers: @resolve_seed_brokers,
+          partitioner: Kafka::Partitioner.new(hash_function: @partitioner_hash_function),
+          sasl_over_ssl: @sasl_over_ssl,
+          **sasl_params)
         log.info "initialized kafka producer: #{@client_id}"
       rescue Exception => e
         if raise_error # During startup, error should be reported to engine and stop its phase for safety.
@@ -185,15 +220,29 @@ DESC
       @exclude_field_accessors = @exclude_fields.map do |field|
         record_accessor_create(field)
       end
+
+      @record_field_accessor = nil
+      @record_field_accessor = record_accessor_create(@record_key) unless @record_key.nil?
     end
 
     def multi_workers_ready?
       true
     end
 
+    def create_producer
+      @kafka.custom_producer(**@producer_opts)
+    end
+
     def start
       super
       refresh_client
+
+      if @share_producer
+        @shared_producer = create_producer
+      else
+        @producers = {}
+        @producers_mutex = Mutex.new
+      end
     end
 
     def close
@@ -206,6 +255,56 @@ DESC
       @kafka = nil
     end
 
+    def wait_writing_threads
+      done = false
+      until done do
+        @writing_threads_mutex.synchronize do
+          done = true if @writing_threads.empty?
+        end
+        sleep(1) unless done
+      end
+    end
+
+    def shutdown
+      super
+      wait_writing_threads
+      shutdown_producers
+    end
+
+    def shutdown_producers
+      if @share_producer
+        @shared_producer.shutdown
+        @shared_producer = nil
+      else
+        @producers_mutex.synchronize {
+          shutdown_threads = @producers.map { |key, producer|
+            th = Thread.new {
+              producer.shutdown
+            }
+            th.abort_on_exception = true
+            th
+          }
+          shutdown_threads.each { |th| th.join }
+          @producers = {}
+        }
+      end
+    end
+
+    def get_producer
+      if @share_producer
+        @shared_producer
+      else
+        @producers_mutex.synchronize {
+          producer = @producers[Thread.current.object_id]
+          unless producer
+            producer = create_producer
+            @producers[Thread.current.object_id] = producer
+          end
+          producer
+        }
+      end
+    end
+
     def setup_formatter(conf)
       type = conf['@type']
       case type
@@ -229,6 +328,8 @@ DESC
 
     # TODO: optimize write performance
     def write(chunk)
+      @writing_threads_mutex.synchronize { @writing_threads.add(Thread.current) }
+
       tag = chunk.metadata.tag
       topic = if @topic
                 extract_placeholders(@topic, chunk)
@@ -237,13 +338,12 @@ DESC
               end
 
       messages = 0
-      record_buf = nil
 
       base_headers = @headers
       mutate_headers = !@headers_from_record_accessors.empty?
 
       begin
-        producer = @kafka.topic_producer(topic, **@producer_opts)
+        producer = get_producer
         chunk.msgpack_each { |time, record|
           begin
             record = inject_values_to_record(tag, time, record)
@@ -267,6 +367,7 @@ DESC
               end
             end
 
+            record = @record_field_accessor.call(record) unless @record_field_accessor.nil?
             record_buf = @formatter_proc.call(tag, time, record)
             record_buf_bytes = record_buf.bytesize
             if @max_send_limit_bytes && record_buf_bytes > @max_send_limit_bytes
@@ -283,7 +384,7 @@ DESC
           messages += 1
 
           producer.produce(record_buf, key: message_key, partition_key: partition_key, partition: partition, headers: headers,
-                           create_time: @use_event_time ? Time.at(time) : Time.now)
+                           create_time: @use_event_time ? Time.at(time) : Time.now, topic: topic)
         }
 
         if messages > 0
@@ -301,7 +402,6 @@ DESC
         end
       rescue Kafka::UnknownTopicOrPartition
         if @use_default_for_unknown_topic && topic != @default_topic
-          producer.shutdown if producer
           log.warn "'#{topic}' topic not found. Retry with '#{default_topic}' topic"
           topic = @default_topic
           retry
@@ -321,7 +421,7 @@ DESC
       # Raise exception to retry sendind messages
       raise e unless ignore
     ensure
-      producer.shutdown if producer
+      @writing_threads_mutex.synchronize { @writing_threads.delete(Thread.current) }
     end
   end
 end

data/lib/fluent/plugin/out_rdkafka2.rb

@@ -5,9 +5,37 @@ require 'fluent/plugin/kafka_plugin_util'
 
 require 'rdkafka'
 
+# This is required for `rdkafka` version >= 0.12.0
+# Overriding the close method in order to provide a time limit for when it should be forcibly closed
+class Rdkafka::Producer::Client
+  # return false if producer is forcefully closed, otherwise return true
+  def close(timeout=nil)
+    return unless @native
+
+    # Indicate to polling thread that we're closing
+    @polling_thread[:closing] = true
+    # Wait for the polling thread to finish up
+    thread = @polling_thread.join(timeout)
+
+    Rdkafka::Bindings.rd_kafka_destroy(@native)
+
+    @native = nil
+
+    return !thread.nil?
+  end
+end
+
 class Rdkafka::Producer
   # return false if producer is forcefully closed, otherwise return true
   def close(timeout = nil)
+    rdkafka_version = Rdkafka::VERSION || '0.0.0'
+    # Rdkafka version >= 0.12.0 changed its internals
+    if Gem::Version::create(rdkafka_version) >= Gem::Version.create('0.12.0')
+      ObjectSpace.undefine_finalizer(self)
+
+      return @client.close(timeout)
+    end
+
     @closing = true
     # Wait for the polling thread to finish up
     # If the broker isn't alive, the thread doesn't exit
@@ -37,6 +65,8 @@ DESC
     config_param :topic_key, :string, :default => 'topic', :desc => "Field for kafka topic"
     config_param :default_topic, :string, :default => nil,
                  :desc => "Default output topic when record doesn't have topic field"
+    config_param :use_default_for_unknown_topic, :bool, :default => false, :desc => "If true, default_topic is used when topic not found"
+    config_param :use_default_for_unknown_partition_error, :bool, :default => false, :desc => "If true, default_topic is used when received unknown_partition error"
     config_param :message_key_key, :string, :default => 'message_key', :desc => "Field for kafka message key"
     config_param :default_message_key, :string, :default => nil
     config_param :partition_key, :string, :default => 'partition', :desc => "Field for kafka partition"
@@ -73,6 +103,11 @@ DESC
                  :desc => <<-DESC
 The codec the producer uses to compress messages. Used for compression.codec
 Supported codecs: (gzip|snappy)
+DESC
+    config_param :record_key, :string, :default => nil,
+                 :desc => <<-DESC
+A jsonpath to a record value pointing to the field which will be passed to the formatter and sent as the Kafka message payload.
+If defined, only this field in the record will be sent to Kafka as the message payload.
 DESC
     config_param :use_event_time, :bool, :default => false, :desc => 'Use fluentd event time for rdkafka timestamp'
     config_param :max_send_limit_bytes, :size, :default => nil
@@ -90,7 +125,6 @@ DESC
     config_param :max_enqueue_bytes_per_second, :size, :default => nil, :desc => 'The maximum number of enqueueing bytes per second'
 
     config_param :service_name, :string, :default => nil, :desc => 'Used for sasl.kerberos.service.name'
-    config_param :ssl_client_cert_key_password, :string, :default => nil, :desc => 'Used for ssl.key.password'
 
     config_section :buffer do
       config_set_default :chunk_keys, ["topic"]
@@ -201,6 +235,9 @@ DESC
       @rdkafka = Rdkafka::Config.new(config)
 
       if @default_topic.nil?
+        if @use_default_for_unknown_topic || @use_default_for_unknown_partition_error
+          raise Fluent::ConfigError, "default_topic must be set when use_default_for_unknown_topic or use_default_for_unknown_partition_error is true"
+        end
         if @chunk_keys.include?(@topic_key) && !@chunk_key_tag
           log.warn "Use '#{@topic_key}' field of event record for topic but no fallback. Recommend to set default_topic or set 'tag' in buffer chunk keys like <buffer #{@topic_key},tag>"
         end
@@ -230,6 +267,9 @@ DESC
       end
 
       @enqueue_rate = EnqueueRate.new(@max_enqueue_bytes_per_second) unless @max_enqueue_bytes_per_second.nil?
+
+      @record_field_accessor = nil
+      @record_field_accessor = record_accessor_create(@record_key) unless @record_key.nil?
     end
 
     def build_config
@@ -270,6 +310,8 @@ DESC
       config[:"queue.buffering.max.messages"] = @rdkafka_buffering_max_messages if @rdkafka_buffering_max_messages
       config[:"message.max.bytes"] = @rdkafka_message_max_bytes if @rdkafka_message_max_bytes
       config[:"batch.num.messages"] = @rdkafka_message_max_num if @rdkafka_message_max_num
+      config[:"sasl.username"] = @username if @username
+      config[:"sasl.password"] = @password if @password
 
       @rdkafka_options.each { |k, v|
         config[k.to_sym] = v
@@ -371,8 +413,6 @@ DESC
               end
 
       handlers = []
-      record_buf = nil
-      record_buf_bytes = nil
 
       headers = @headers.clone
 
@@ -395,6 +435,7 @@ DESC
               end
             end
 
+            record = @record_field_accessor.call(record) unless @record_field_accessor.nil?
             record_buf = @formatter_proc.call(tag, time, record)
             record_buf_bytes = record_buf.bytesize
             if @max_send_limit_bytes && record_buf_bytes > @max_send_limit_bytes
@@ -430,17 +471,25 @@ DESC
 
     def enqueue_with_retry(producer, topic, record_buf, message_key, partition, headers, time)
       attempt = 0
+      actual_topic = topic
+
       loop do
         begin
           @enqueue_rate.raise_if_limit_exceeded(record_buf.bytesize) if @enqueue_rate
-          return producer.produce(topic: topic, payload: record_buf, key: message_key, partition: partition, headers: headers, timestamp: @use_event_time ? Time.at(time) : nil)
+          return producer.produce(topic: actual_topic, payload: record_buf, key: message_key, partition: partition, headers: headers, timestamp: @use_event_time ? Time.at(time) : nil)
         rescue EnqueueRate::LimitExceeded => e
           @enqueue_rate.revert if @enqueue_rate
           duration = e.next_retry_clock - Fluent::Clock.now
           sleep(duration) if duration > 0.0
         rescue Exception => e
           @enqueue_rate.revert if @enqueue_rate
-          if e.respond_to?(:code) && e.code == :queue_full
+
+          if !e.respond_to?(:code)
+            raise e
+          end
+
+          case e.code
+          when :queue_full
             if attempt <= @max_enqueue_retries
               log.warn "Failed to enqueue message; attempting retry #{attempt} of #{@max_enqueue_retries} after #{@enqueue_retry_backoff}s"
               sleep @enqueue_retry_backoff
@@ -448,6 +497,25 @@ DESC
             else
               raise "Failed to enqueue message although tried retry #{@max_enqueue_retries} times"
             end
+          # https://github.com/confluentinc/librdkafka/blob/c282ba2423b2694052393c8edb0399a5ef471b3f/src/rdkafka.h#LL309C9-L309C41
+          # RD_KAFKA_RESP_ERR__UNKNOWN_TOPIC
+          when :unknown_topic
+            if @use_default_for_unknown_topic && actual_topic != @default_topic
+              log.debug "'#{actual_topic}' topic not found. Retry with '#{@default_topic}' topic"
+              actual_topic = @default_topic
+              retry
+            end
+            raise e
+          # https://github.com/confluentinc/librdkafka/blob/c282ba2423b2694052393c8edb0399a5ef471b3f/src/rdkafka.h#L305
+          # RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION
+          when :unknown_partition
+            if @use_default_for_unknown_partition_error && actual_topic != @default_topic
+              log.debug "failed writing to topic '#{actual_topic}' with error '#{e.to_s}'. Writing message to topic '#{@default_topic}'"
+              actual_topic = @default_topic
+              retry
+            end
+
+            raise e
           else
             raise e
           end

data/test/plugin/test_out_kafka2.rb

@@ -105,6 +105,21 @@ class Kafka2OutputTest < Test::Unit::TestCase
       assert_equal([expected_message], actual_messages)
     end
 
+    def test_record_key
+      conf = config(default_topic: TOPIC_NAME) +
+        config_element('ROOT', '', {"record_key" => "$.data"}, [])
+      target_driver = create_target_driver
+      target_driver.run(expect_records: 1, timeout: 5) do
+        sleep 2
+        d = create_driver(conf)
+        d.run do
+          d.feed('test', event_time, {'data' => {'a' => 'b', 'foo' => 'bar', 'message' => 'test'}, 'message_key' => '123456'})
+        end
+      end
+      actual_messages = target_driver.events.collect { |event| event[2] }
+      assert_equal([{'a' => 'b', 'foo' => 'bar', 'message' => 'test'}], actual_messages)
+    end
+
     def test_exclude_fields
       conf = config(default_topic: TOPIC_NAME) +
              config_element('ROOT', '', {"exclude_fields" => "$.foo"}, [])

data/test/plugin/test_out_rdkafka2.rb

@@ -163,5 +163,20 @@ class Rdkafka2OutputTest < Test::Unit::TestCase
       actual_messages = target_driver.events.collect { |event| event[2] }
       assert_equal(expected_messages, actual_messages)
     end
+
+    def test_record_key
+      conf = config(default_topic: TOPIC_NAME) +
+             config_element('ROOT', '', {"record_key" => "$.data"}, [])
+      target_driver = create_target_driver
+      target_driver.run(expect_records: 1, timeout: 5) do
+        sleep 2
+        d = create_driver(conf)
+        d.run do
+          d.feed('test', event_time, {'data' => {'a' => 'b', 'foo' => 'bar', 'message' => 'test'}, 'message_key' => '123456'})
+        end
+      end
+      actual_messages = target_driver.events.collect { |event| event[2] }
+      assert_equal([{'a' => 'b', 'foo' => 'bar', 'message' => 'test'}], actual_messages)
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.17.5
+  version: 0.19.1
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-03-18 00:00:00.000000000 Z
+date: 2023-09-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -51,7 +51,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.4.0
+        version: 1.5.0
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -61,7 +61,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.4.0
+        version: 1.5.0
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -146,6 +146,7 @@ files:
 - ".github/ISSUE_TEMPLATE/bug_report.yaml"
 - ".github/ISSUE_TEMPLATE/config.yml"
 - ".github/ISSUE_TEMPLATE/feature_request.yaml"
+- ".github/dependabot.yml"
 - ".github/workflows/linux.yml"
 - ".github/workflows/stale-actions.yml"
 - ".gitignore"
@@ -155,6 +156,10 @@ files:
 - README.md
 - Rakefile
 - ci/prepare-kafka-server.sh
+- examples/README.md
+- examples/out_kafka2/dynamic_topic_based_on_tag.conf
+- examples/out_kafka2/protobuf-formatter.conf
+- examples/out_kafka2/record_key.conf
 - fluent-plugin-kafka.gemspec
 - lib/fluent/plugin/in_kafka.rb
 - lib/fluent/plugin/in_kafka_group.rb
@@ -193,7 +198,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.5
+rubygems_version: 3.3.5
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin for Apache Kafka > 0.8