fluent-plugin-kafka 0.19.4 → 0.19.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 242a6673fffdacfbab759884cadaf66e22b33e72cbb60ff49cc9201a28366125
-  data.tar.gz: 0ac9e110d4f43cb6d25e9cf416084ebcb05d3467671db28bae38310d6c8741dc
+  metadata.gz: c01dcccf05f4c04f7dc5eb08c7f2da880ea7f892242931e95a009fe8cb18c270
+  data.tar.gz: d446556bb8e8c65b7a2b16460aac3fab05720f444a5551bb7da486e5941e8c98
 SHA512:
-  metadata.gz: 5ed4bc59126e9dbb6c59c4dc35c7b94d837652ed0f48a4fecdba62071ab4b284389adbe0d2cf493e926757d8aaedfc8283f6192e7e9d71c882176bd3d044d1c2
-  data.tar.gz: e2e1a07d54ac22a342b2f27ddcacb69e5973d6f606000c0e2fcd601f97a6f9c6ed8c488d759a9af3c95e16a70c6f6aca691567734a2fae852b9acc01d34cae46
+  metadata.gz: efe8546b9cdd28016b8872cdcbf91402b3cfd11b50ef1a63e38215b46128399bd05ed0d211cf03e8cbee1122b5316f15f95c3bc906937d2244340e4a9b68cdd8
+  data.tar.gz: 100e538e93d736579a5886db60e3dc769807efc0ac516f352def62b126f7d636e594161ea72c490fffcf3c8a4f0bd22ada8fc59be40873aa407f9aa034e20e75

data/.github/workflows/linux.yml

@@ -19,7 +19,14 @@ jobs:
           - { min: '>= 0.6.0', max: '< 0.12.0' }
           - { min: '>= 0.12.0', max: '< 0.14.0' }
           - { min: '>= 0.14.0', max: '< 0.16.0' }
-          - { min: '>= 0.16.0', max: '>= 0.16.0' }
+          - { min: '>= 0.16.0', max: '< 0.25.0' }
+          - { min: '>= 0.25.0', max: '>= 0.25.0' }
+        # rdkafka 0.25.0 supports Ruby 3.2+, so we exclude it for Ruby 3.0 and 3.1
+        exclude:
+          - ruby: '3.1'
+            rdkafka_versions: { min: '>= 0.25.0', max: '>= 0.25.0' }
+          - ruby: '3.0'
+            rdkafka_versions: { min: '>= 0.25.0', max: '>= 0.25.0' }
         # rdkafka 0.15.2 is the last version which supports Ruby 2.7
         include:
           - ruby: '2.7'
@@ -30,7 +37,7 @@ jobs:
             rdkafka_versions: { min: '>= 0.12.0', max: '= 0.15.2' }
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }} with rdkafka gem version (min ${{ matrix.rdkafka_versions.min }} max ${{ matrix.rdkafka_versions.max }})
     steps:
-    - uses: actions/checkout@v4
+    - uses: actions/checkout@v6
     - uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}

data/.github/workflows/stale-actions.yml

@@ -7,7 +7,7 @@ jobs:
   stale:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/stale@v9
+    - uses: actions/stale@v10
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         days-before-stale: 90

data/ChangeLog

@@ -1,3 +1,13 @@
+Release 0.19.6 - 2026/03/04
+	* out_rdkafka2: Auto-configure SASL PLAIN when username and password are set (#547)
+	* out_rdkafka2: Fix NoMethodError during graceful reload (#546)
+	* out_rdkafka: Fix ArgumentError for ssl_client_cert_key_password (#545)
+	* out_rdkafka, out_rdkafka2: Fix deprecation warning for max_wait_timeout in rdkafka-ruby >= 0.25.0 (#544)
+
+Release 0.19.5 - 2025/07/11
+	* in_kafka_group: Add `sasl_aws_msk_iam_access_key_id`, `sasl_aws_msk_iam_secret_access_key`
+			and `sasl_aws_msk_iam_aws_region` options (#531)
+
 Release 0.19.4 - 2025/03/24
 	* Support Ruby 3.4. (#526)
 

data/fluent-plugin-kafka.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.19.4'
+  gem.version       = '0.19.6'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/in_kafka_group.rb

@@ -74,6 +74,7 @@ class Fluent::KafkaGroupInput < Fluent::Input
 
   include Fluent::KafkaPluginUtil::SSLSettings
   include Fluent::KafkaPluginUtil::SaslSettings
+  include Fluent::KafkaPluginUtil::AwsIamSettings
 
   class ForShutdown < StandardError
   end
@@ -185,6 +186,7 @@ class Fluent::KafkaGroupInput < Fluent::Input
     super
 
     logger = @get_kafka_client_log ? log : nil
+    use_aws_msk_iam = @sasl_aws_msk_iam_access_key_id != nil && @sasl_aws_msk_iam_secret_key_id != nil
     if @scram_mechanism != nil && @username != nil && @password != nil
       @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert_file_path: @ssl_ca_cert,
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
@@ -197,6 +199,10 @@ class Fluent::KafkaGroupInput < Fluent::Input
                          ssl_client_cert_key_password: @ssl_client_cert_key_password,
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_plain_username: @username, sasl_plain_password: @password,
                          sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
+    elsif use_aws_msk_iam
+      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout,
+                         sasl_aws_msk_iam_secret_key_id: @sasl_aws_msk_iam_secret_key_id, sasl_aws_msk_iam_access_key_id: @sasl_aws_msk_iam_access_key_id, sasl_aws_msk_iam_aws_region: @sasl_aws_msk_iam_aws_region,
+                         ssl_ca_certs_from_system: @ssl_ca_certs_from_system)
     else
       @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert_file_path: @ssl_ca_cert,
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),

data/lib/fluent/plugin/out_rdkafka.rb

@@ -78,7 +78,6 @@ DESC
   config_param :enqueue_retry_backoff, :integer, :default => 3
 
   config_param :service_name, :string, :default => nil
-  config_param :ssl_client_cert_key_password, :string, :default => nil
 
   include Fluent::KafkaPluginUtil::SSLSettings
   include Fluent::KafkaPluginUtil::SaslSettings
@@ -87,6 +86,8 @@ DESC
     super
     @producers = {}
     @producers_mutex = Mutex.new
+
+    @use_max_wait_timeout_ms = Gem::Version.new(Rdkafka::VERSION) >= Gem::Version.new('0.25.0')
   end
 
   def configure(conf)
@@ -291,7 +292,13 @@ DESC
         handler = enqueue_with_retry(producer, topic, record_buf, message_key, partition, time)
         handler
       }.each { |handler|
-        handler.wait(max_wait_timeout: @rdkafka_delivery_handle_poll_timeout) if @rdkafka_delivery_handle_poll_timeout != 0
+        if @rdkafka_delivery_handle_poll_timeout != 0
+          if @use_max_wait_timeout_ms
+            handler.wait(max_wait_timeout_ms: @rdkafka_delivery_handle_poll_timeout * 1000)
+          else
+            handler.wait(max_wait_timeout: @rdkafka_delivery_handle_poll_timeout)
+          end
+        end
       }
     end
   rescue Exception => e

data/lib/fluent/plugin/out_rdkafka2.rb

@@ -168,12 +168,14 @@ DESC
     def initialize
       super
 
-      @producers = nil
-      @producers_mutex = nil
+      @producers = {}
+      @producers_mutex = Mutex.new
       @shared_producer = nil
       @enqueue_rate = nil
       @writing_threads_mutex = Mutex.new
       @writing_threads = Set.new
+
+      @use_max_wait_timeout_ms = Gem::Version.new(Rdkafka::VERSION) >= Gem::Version.new('0.25.0')
     end
 
     def configure(conf)
@@ -266,6 +268,11 @@ DESC
         config[:"sasl.kerberos.keytab"] = @keytab if @keytab
       end
 
+      if @username && @password
+        sasl = true
+        config[:"sasl.mechanisms"] = 'PLAIN'
+      end
+
       if ssl && sasl
         security_protocol = "SASL_SSL"
       elsif ssl && !sasl
@@ -301,7 +308,6 @@ DESC
         @shared_producer = @rdkafka.producer
       else
         @producers = {}
-        @producers_mutex = Mutex.new
       end
 
       super
@@ -430,7 +436,11 @@ DESC
           end
         }
         handlers.each { |handler|
-          handler.wait(max_wait_timeout: @rdkafka_delivery_handle_poll_timeout)
+          if @use_max_wait_timeout_ms
+            handler.wait(max_wait_timeout_ms: @rdkafka_delivery_handle_poll_timeout * 1000)
+          else
+            handler.wait(max_wait_timeout: @rdkafka_delivery_handle_poll_timeout)
+          end
         }
       end
     rescue Exception => e

data/test/plugin/test_in_kafka_group.rb

@@ -39,6 +39,7 @@ class KafkaGroupInputTest < Test::Unit::TestCase
   class ConsumeTest < self
     def setup
       @kafka = Kafka.new(["localhost:9092"], client_id: 'kafka')
+      @kafka.create_topic(TOPIC_NAME)
       @producer = @kafka.producer
     end
 
@@ -59,6 +60,7 @@ class KafkaGroupInputTest < Test::Unit::TestCase
       d = create_driver
 
       d.run(expect_records: 1, timeout: 10) do
+        sleep 0.1
         @producer.produce("Hello, fluent-plugin-kafka!", topic: TOPIC_NAME)
         @producer.deliver_messages
       end

data/test/plugin/test_out_rdkafka2.rb

@@ -55,6 +55,18 @@ class Rdkafka2OutputTest < Test::Unit::TestCase
     assert_equal 'localhost:9092', d.instance.brokers
   end
 
+  def test_configure_sasl_plain
+    conf = base_config + config_element('ROOT', '', {"username" => "testuser", "password" => "testpass"}, [])
+    d = create_driver(conf)
+
+    config = d.instance.build_config
+
+    assert_equal 'PLAIN', config[:"sasl.mechanisms"]
+    assert_equal 'SASL_PLAINTEXT', config[:"security.protocol"]
+    assert_equal 'testuser', config[:"sasl.username"]
+    assert_equal 'testpass', config[:"sasl.password"]
+  end
+
   def test_mutli_worker_support
     d = create_driver
     assert_equal true, d.instance.multi_workers_ready?

metadata

@@ -1,15 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.19.4
+  version: 0.19.6
 platform: ruby
 authors:
 - Hidemasa Togashi
 - Masahiro Nakagawa
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-03-25 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -202,7 +201,6 @@ homepage: https://github.com/fluent/fluent-plugin-kafka
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -217,8 +215,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.22
-signing_key:
+rubygems_version: 4.0.6
 specification_version: 4
 summary: Fluentd plugin for Apache Kafka > 0.8
 test_files: