fluent-plugin-kafka 0.19.5 → 0.19.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 17961fc6c8b1aa236a49f6e464ad5cab9e10731ee208923f0a8e5b656d2c4c65
-  data.tar.gz: c36db6868dbb56daef12196359c6e258e03783f831cc00048598d1e3fcb29a63
+  metadata.gz: 2a709cad3615bca0c45f0606644142278acbb75b8e43f766dd4a1821a624aa75
+  data.tar.gz: 269bfdaa2ef8fb7b6cbb254971cc728769aba7215a472a73b34af2c04d735f6f
 SHA512:
-  metadata.gz: cd7e1a087448d229fd1d6647035ecbecb01fd298bf26b86783be88d14dfe681e1dbedca880eddd201bb7c1bbbbe1a9a2c6cdc82d5d2797c68926d5c938f0428c
-  data.tar.gz: e9bdab87031ec6c55f91c15e8e1f45f384bc21b54dffb26046aa6f255f6151ad7433ae887f6ba5c8e7f612162a898909e413eae972e3411368afbf4b87202e58
+  metadata.gz: b15df023bd526fb22752da54b460ed12ce9a2d3c35e3ad9959c86d1ba8ac06fb9915799da95135fec3c6911ae776375ee8bc13f4803cad6663e2b60351b302c1
+  data.tar.gz: f9bfe163282d53f4f583ac905bffd50cab6eeaaaf5da2b9bf7511c7dad064c3aff458252af784ff8a52a3b6b5b63c80d47fccc80677692f58dc8a9fed814e6f4

data/.github/dependabot.yml

@@ -3,4 +3,4 @@ updates:
   - package-ecosystem: 'github-actions'
     directory: '/'
     schedule:
-      interval: 'weekly'
+      interval: 'monthly'

data/.github/workflows/add-to-project.yml

@@ -10,8 +10,15 @@ jobs:
     name: Add issue to project
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/add-to-project@v1.0.2
+      - name: Generate token
+        id: generate_token
+        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
+        with:
+          app-id: ${{ secrets.PROJECT_APP_ID }}
+          private-key: ${{ secrets.PROJECT_APP_PRIVATE_KEY }}
+          owner: fluent
+      - uses: actions/add-to-project@5afcf98fcd03f1c2f92c3c83f58ae24323cc57fd # v2.0.0
         with:
           project-url: https://github.com/orgs/fluent/projects/4
-          github-token: ${{ secrets.ADD_TO_PROJECT_PAT }}
+          github-token: ${{ steps.generate_token.outputs.token }}
           labeled: waiting-for-triage

data/.github/workflows/linux.yml

@@ -4,22 +4,38 @@ on:
     branches: [master]
   pull_request:
     branches: [master]
+  schedule:
+    - cron: '0 0 1 * *'
 jobs:
+  ruby-versions:
+    uses: ruby/actions/.github/workflows/ruby_versions.yml@master
+    with:
+      engine: cruby
+      min_version: 3.0
   build:
+    needs: ruby-versions
     runs-on: ${{ matrix.os }}
     env:
       USE_RDKAFKA: 1
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '3.4', '3.3', '3.2', '3.1', '3.0' ]
+        ruby: ${{ fromJson(needs.ruby-versions.outputs.versions) }}
         os:
           - ubuntu-latest
         rdkafka_versions:
           - { min: '>= 0.6.0', max: '< 0.12.0' }
           - { min: '>= 0.12.0', max: '< 0.14.0' }
           - { min: '>= 0.14.0', max: '< 0.16.0' }
-          - { min: '>= 0.16.0', max: '>= 0.16.0' }
+          - { min: '>= 0.16.0', max: '< 0.25.0' }
+          - { min: '>= 0.25.0', max: '>= 0.25.0' }
+        # rdkafka 0.25.0 supports Ruby 3.2+, so we exclude it for Ruby 3.0 and 3.1
+        exclude:
+          - ruby: head
+          - ruby: '3.1'
+            rdkafka_versions: { min: '>= 0.25.0', max: '>= 0.25.0' }
+          - ruby: '3.0'
+            rdkafka_versions: { min: '>= 0.25.0', max: '>= 0.25.0' }
         # rdkafka 0.15.2 is the last version which supports Ruby 2.7
         include:
           - ruby: '2.7'
@@ -30,17 +46,17 @@ jobs:
             rdkafka_versions: { min: '>= 0.12.0', max: '= 0.15.2' }
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }} with rdkafka gem version (min ${{ matrix.rdkafka_versions.min }} max ${{ matrix.rdkafka_versions.max }})
     steps:
-    - uses: actions/checkout@v4
-    - uses: ruby/setup-ruby@v1
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+    - uses: ruby/setup-ruby@afeafc3d1ab54a631816aba4c914a0081c12ff2f # v1.310.0
       with:
         ruby-version: ${{ matrix.ruby }}
     - name: Install confluent-kafka
       run: |
         sudo apt install -V -y gnupg2 wget
-        wget https://packages.confluent.io/deb/6.0/archive.key
+        wget https://packages.confluent.io/deb/7.9/archive.key
         sudo gpg2 --homedir /tmp --no-default-keyring --keyring gnupg-ring:/usr/share/keyrings/confluent-archive-keyring.gpg --import archive.key
         sudo chmod 644 /usr/share/keyrings/confluent-archive-keyring.gpg
-        sudo sh -c 'echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/confluent-archive-keyring.gpg] https://packages.confluent.io/deb/6.0 stable main" > /etc/apt/sources.list.d/confluent.list'
+        sudo sh -c 'echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/confluent-archive-keyring.gpg] https://packages.confluent.io/deb/7.9 stable main" > /etc/apt/sources.list.d/confluent.list'
         sudo apt update
         sudo apt install -y confluent-community-2.13 openjdk-11-jre netcat-openbsd
     - name: unit testing
@@ -50,6 +66,5 @@ jobs:
         RDKAFKA_VERSION_MAX_RANGE: ${{ matrix.rdkafka_versions.max }}
       run: |
         sudo ./ci/prepare-kafka-server.sh
-        gem install rake
         bundle install --jobs 4 --retry 3
         bundle exec rake test

data/.github/workflows/stale-actions.yml

@@ -6,8 +6,11 @@ on:
 jobs:
   stale:
     runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
     steps:
-    - uses: actions/stale@v9
+    - uses: actions/stale@eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899 # v10.3.0
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         days-before-stale: 90

data/ChangeLog

@@ -1,3 +1,12 @@
+Release 0.19.7 - 2026/06/02
+    * in_rdkafka_group: support regexp pattern in topics (#541)
+
+Release 0.19.6 - 2026/03/04
+	* out_rdkafka2: Auto-configure SASL PLAIN when username and password are set (#547)
+	* out_rdkafka2: Fix NoMethodError during graceful reload (#546)
+	* out_rdkafka: Fix ArgumentError for ssl_client_cert_key_password (#545)
+	* out_rdkafka, out_rdkafka2: Fix deprecation warning for max_wait_timeout in rdkafka-ruby >= 0.25.0 (#544)
+
 Release 0.19.5 - 2025/07/11
 	* in_kafka_group: Add `sasl_aws_msk_iam_access_key_id`, `sasl_aws_msk_iam_secret_access_key`
 			and `sasl_aws_msk_iam_aws_region` options (#531)

data/README.md

@@ -177,6 +177,10 @@ With the introduction of the rdkafka-ruby based input plugin we hope to support
 
 See also [rdkafka-ruby](https://github.com/appsignal/rdkafka-ruby) and [librdkafka](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md) for more detailed documentation about Kafka consumer options.
 
+`topics` supports POSIX Extended Regular Expression pattern (not Ruby regex syntax) since v0.19.6. If you want to use regex pattern, use `/pattern/` like `/foo.*/`.
+
+**Note**: A caret (`^`) is automatically added to the beginning of the pattern.
+
 Consuming topic name is used for event tag. So when the target topic name is `app_event`, the tag is `app_event`. If you want to modify tag, use `add_prefix` or `add_suffix` parameter. With `add_prefix kafka`, the tag is `kafka.app_event`.
 
 ### Output plugin

data/ci/prepare-kafka-server.sh

@@ -1,5 +1,7 @@
 #!/bin/sh
 
+set -x
+
 export KAFKA_OPTS=-Dzookeeper.4lw.commands.whitelist=ruok
 /usr/bin/zookeeper-server-start /etc/kafka/zookeeper.properties  &
 N_POLLING=30
@@ -30,4 +32,14 @@ while true ; do
 	exit 1
     fi
 done
-/usr/bin/kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test
+# kafka-topics --version fails to connect zookeeper here.
+case "$(java -cp "/usr/share/java/kafka/*" kafka.Kafka --version 2>/dev/null)" in
+    7.*)
+        /usr/bin/kafka-topics --create --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1 --topic test
+        ;;
+    *)
+        # Deprecated zookeeper
+        /usr/bin/kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test
+        ;;
+esac
+

data/fluent-plugin-kafka.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.19.5'
+  gem.version       = '0.19.7'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/in_rdkafka_group.rb

@@ -76,13 +76,25 @@ class Fluent::Plugin::RdKafkaGroupInput < Fluent::Plugin::Input
     end
     config_array
   end
+  private :_config_to_array
+
+  def regex_pattern?(topic)
+    topic.start_with?('/') && topic.end_with?('/') ? true : false
+  end
+  private :regex_pattern?
+
+  def _config_regex_pattern(topic)
+    # librdkafka recognizes string as regex pattern if the topic name starts with '^'.
+    # https://github.com/confluentinc/librdkafka/blob/570c785e9e35812db8f50254bd2f7e0cf47def39/src/rdkafka.h#L4148
+    # https://github.com/confluentinc/librdkafka/blob/e1db7eaa517f0a6438bc846a9c49ede73b9ea211/src/rdkafka_topic.c#L2064
+    [topic[1..-2].prepend('^')]
+  end
+  private :_config_regex_pattern
 
   def multi_workers_ready?
     true
   end
 
-  private :_config_to_array
-
   def configure(conf)
     compat_parameters_convert(conf, :parser)
 
@@ -93,7 +105,12 @@ class Fluent::Plugin::RdKafkaGroupInput < Fluent::Plugin::Input
     log.info "Will watch for topics #{@topics} at brokers " \
               "#{@kafka_configs["bootstrap.servers"]} and '#{@kafka_configs["group.id"]}' group"
 
-    @topics = _config_to_array(@topics)
+    topics = @topics.strip
+    if regex_pattern?(topics)
+      @topics = _config_regex_pattern(topics)
+    else
+      @topics = _config_to_array(topics)
+    end
 
     parser_conf = conf.elements('parse').first
     unless parser_conf

data/lib/fluent/plugin/out_rdkafka.rb

@@ -78,7 +78,6 @@ DESC
   config_param :enqueue_retry_backoff, :integer, :default => 3
 
   config_param :service_name, :string, :default => nil
-  config_param :ssl_client_cert_key_password, :string, :default => nil
 
   include Fluent::KafkaPluginUtil::SSLSettings
   include Fluent::KafkaPluginUtil::SaslSettings
@@ -87,6 +86,8 @@ DESC
     super
     @producers = {}
     @producers_mutex = Mutex.new
+
+    @use_max_wait_timeout_ms = Gem::Version.new(Rdkafka::VERSION) >= Gem::Version.new('0.25.0')
   end
 
   def configure(conf)
@@ -291,7 +292,13 @@ DESC
         handler = enqueue_with_retry(producer, topic, record_buf, message_key, partition, time)
         handler
       }.each { |handler|
-        handler.wait(max_wait_timeout: @rdkafka_delivery_handle_poll_timeout) if @rdkafka_delivery_handle_poll_timeout != 0
+        if @rdkafka_delivery_handle_poll_timeout != 0
+          if @use_max_wait_timeout_ms
+            handler.wait(max_wait_timeout_ms: @rdkafka_delivery_handle_poll_timeout * 1000)
+          else
+            handler.wait(max_wait_timeout: @rdkafka_delivery_handle_poll_timeout)
+          end
+        end
       }
     end
   rescue Exception => e

data/lib/fluent/plugin/out_rdkafka2.rb

@@ -168,12 +168,14 @@ DESC
     def initialize
       super
 
-      @producers = nil
-      @producers_mutex = nil
+      @producers = {}
+      @producers_mutex = Mutex.new
       @shared_producer = nil
       @enqueue_rate = nil
       @writing_threads_mutex = Mutex.new
       @writing_threads = Set.new
+
+      @use_max_wait_timeout_ms = Gem::Version.new(Rdkafka::VERSION) >= Gem::Version.new('0.25.0')
     end
 
     def configure(conf)
@@ -266,6 +268,11 @@ DESC
         config[:"sasl.kerberos.keytab"] = @keytab if @keytab
       end
 
+      if @username && @password
+        sasl = true
+        config[:"sasl.mechanisms"] = 'PLAIN'
+      end
+
       if ssl && sasl
         security_protocol = "SASL_SSL"
       elsif ssl && !sasl
@@ -301,7 +308,6 @@ DESC
         @shared_producer = @rdkafka.producer
       else
         @producers = {}
-        @producers_mutex = Mutex.new
       end
 
       super
@@ -430,7 +436,11 @@ DESC
           end
         }
         handlers.each { |handler|
-          handler.wait(max_wait_timeout: @rdkafka_delivery_handle_poll_timeout)
+          if @use_max_wait_timeout_ms
+            handler.wait(max_wait_timeout_ms: @rdkafka_delivery_handle_poll_timeout * 1000)
+          else
+            handler.wait(max_wait_timeout: @rdkafka_delivery_handle_poll_timeout)
+          end
         }
       end
     rescue Exception => e

data/test/plugin/test_in_kafka_group.rb

@@ -39,6 +39,7 @@ class KafkaGroupInputTest < Test::Unit::TestCase
   class ConsumeTest < self
     def setup
       @kafka = Kafka.new(["localhost:9092"], client_id: 'kafka')
+      @kafka.create_topic(TOPIC_NAME)
       @producer = @kafka.producer
     end
 
@@ -59,6 +60,7 @@ class KafkaGroupInputTest < Test::Unit::TestCase
       d = create_driver
 
       d.run(expect_records: 1, timeout: 10) do
+        sleep 0.1
         @producer.produce("Hello, fluent-plugin-kafka!", topic: TOPIC_NAME)
         @producer.deliver_messages
       end

data/test/plugin/test_in_rdkafka_group.rb

@@ -0,0 +1,128 @@
+require 'helper'
+require 'fluent/test/driver/input'
+require 'securerandom'
+
+class RdkafkaGroupInputTest < Test::Unit::TestCase
+
+  def have_rdkafka
+    begin
+      require 'fluent/plugin/in_rdkafka_group'
+      true
+    rescue LoadError
+      false
+    end
+  end
+
+  def setup
+    omit_unless(have_rdkafka, "rdkafka isn't installed")
+    Fluent::Test.setup
+  end
+
+  TOPIC_NAME = "kafka-input-#{SecureRandom.uuid}"
+
+  CONFIG = %[
+    topics #{TOPIC_NAME}
+    kafka_configs {"bootstrap.servers": "localhost:9092", "group.id": "test_group"}
+    <parse>
+      @type none
+    </parse>
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::RdKafkaGroupInput).configure(conf)
+  end
+
+
+  def test_configure
+    d = create_driver
+    assert_equal [TOPIC_NAME], d.instance.topics
+    assert_equal 'localhost:9092', d.instance.kafka_configs['bootstrap.servers']
+  end
+
+  def test_multi_worker_support
+    d = create_driver
+    assert_true d.instance.multi_workers_ready?
+  end
+
+  class ConsumeTest < self
+    TOPIC_NAME = "kafka-input-#{SecureRandom.uuid}"
+
+    def setup
+      @kafka = Kafka.new(["localhost:9092"], client_id: 'kafka')
+      @producer = @kafka.producer
+      @kafka.create_topic(TOPIC_NAME)
+    end
+
+    def teardown
+      @kafka.delete_topic(TOPIC_NAME)
+      @kafka.close
+    end
+
+    def test_consume
+      conf = %[
+        topics #{TOPIC_NAME}
+        kafka_configs {"bootstrap.servers": "localhost:9092", "group.id": "test_group"}
+        <parse>
+          @type none
+        </parse>
+      ]
+
+      d = create_driver(conf)
+
+      d.run(expect_records: 1, timeout: 10) do
+        sleep 0.1
+        @producer.produce("Hello, fluent-plugin-kafka!", topic: TOPIC_NAME)
+        @producer.deliver_messages
+      end
+
+      expected = {'message'  => 'Hello, fluent-plugin-kafka!'}
+      assert_equal expected, d.events[0][2]
+    end
+  end
+
+  class ConsumeTopicWithRegexpTest < self
+    TOPIC_NAME1       = "kafka-input-1-#{SecureRandom.uuid}"
+    TOPIC_NAME2       = "kafka-input-22-#{SecureRandom.uuid}"
+    UNMATCHED_TOPIC   = "kafka-input-333-#{SecureRandom.uuid}"
+
+    TOPIC_NAME_REGEXP = "/kafka-input-[0-9]{1,2}-.*/"
+
+    def setup
+      @kafka = Kafka.new(["localhost:9092"], client_id: 'kafka')
+      @producer = @kafka.producer
+      @kafka.create_topic(TOPIC_NAME1)
+      @kafka.create_topic(TOPIC_NAME2)
+      @kafka.create_topic(UNMATCHED_TOPIC)
+    end
+
+    def teardown
+      @kafka.delete_topic(TOPIC_NAME1)
+      @kafka.delete_topic(TOPIC_NAME2)
+      @kafka.delete_topic(UNMATCHED_TOPIC)
+      @kafka.close
+    end
+
+    def test_consume_with_regexp
+      conf = %[
+        topics #{TOPIC_NAME_REGEXP}
+        kafka_configs {"bootstrap.servers": "localhost:9092", "group.id": "test_group"}
+        <parse>
+          @type none
+        </parse>
+      ]
+      d = create_driver(conf)
+
+      d.run(expect_records: 2, timeout: 10) do
+        sleep 0.1
+        @producer.produce("Hello, fluent-plugin-kafka! in topic 1", topic: TOPIC_NAME1)
+        @producer.produce("Hello, fluent-plugin-kafka! in topic 2", topic: TOPIC_NAME2)
+        @producer.produce("Should be ignored", topic: UNMATCHED_TOPIC)
+        @producer.deliver_messages
+      end
+      expected_message_pattern = /Hello, fluent-plugin-kafka! in topic [12]/
+      assert_equal 2, d.events.size
+      assert_match(expected_message_pattern, d.events[0][2]['message'])
+      assert_match(expected_message_pattern, d.events[1][2]['message'])
+    end
+  end
+end

data/test/plugin/test_out_rdkafka2.rb

@@ -55,6 +55,18 @@ class Rdkafka2OutputTest < Test::Unit::TestCase
     assert_equal 'localhost:9092', d.instance.brokers
   end
 
+  def test_configure_sasl_plain
+    conf = base_config + config_element('ROOT', '', {"username" => "testuser", "password" => "testpass"}, [])
+    d = create_driver(conf)
+
+    config = d.instance.build_config
+
+    assert_equal 'PLAIN', config[:"sasl.mechanisms"]
+    assert_equal 'SASL_PLAINTEXT', config[:"security.protocol"]
+    assert_equal 'testuser', config[:"sasl.username"]
+    assert_equal 'testpass', config[:"sasl.password"]
+  end
+
   def test_mutli_worker_support
     d = create_driver
     assert_equal true, d.instance.multi_workers_ready?

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.19.5
+  version: 0.19.7
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -192,6 +192,7 @@ files:
 - test/helper.rb
 - test/plugin/test_in_kafka.rb
 - test/plugin/test_in_kafka_group.rb
+- test/plugin/test_in_rdkafka_group.rb
 - test/plugin/test_kafka_plugin_util.rb
 - test/plugin/test_out_kafka.rb
 - test/plugin/test_out_kafka2.rb
@@ -215,13 +216,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.7
+rubygems_version: 4.0.10
 specification_version: 4
 summary: Fluentd plugin for Apache Kafka > 0.8
 test_files:
 - test/helper.rb
 - test/plugin/test_in_kafka.rb
 - test/plugin/test_in_kafka_group.rb
+- test/plugin/test_in_rdkafka_group.rb
 - test/plugin/test_kafka_plugin_util.rb
 - test/plugin/test_out_kafka.rb
 - test/plugin/test_out_kafka2.rb