fluent-plugin-kafka 0.19.2 → 0.19.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 62d114017cabc47f58cd39db1ea21dd8cfb6d76126c4804741e25766a5b4576c
-  data.tar.gz: f5e4ea9adb96a8f2f46077c0ac89b51e1c177ea0933242ed6cbb6472e40577a2
+  metadata.gz: 242a6673fffdacfbab759884cadaf66e22b33e72cbb60ff49cc9201a28366125
+  data.tar.gz: 0ac9e110d4f43cb6d25e9cf416084ebcb05d3467671db28bae38310d6c8741dc
 SHA512:
-  metadata.gz: 80f4d2ce4d23d95837d3b344841339fc517581bc3347e17e5352a135b088756317b970b559eb66f05e0216ce8841c9e374c5ed0fd09e48d22346249ec9aec0b9
-  data.tar.gz: 8d89ff55bf6ac92e5d2c7a3942882a6b307e6fe19ede620bf299d54feb6f2f1d70fcdece731008324a993c96c69f44456a71798c9af879666de9d017b38e39c3
+  metadata.gz: 5ed4bc59126e9dbb6c59c4dc35c7b94d837652ed0f48a4fecdba62071ab4b284389adbe0d2cf493e926757d8aaedfc8283f6192e7e9d71c882176bd3d044d1c2
+  data.tar.gz: e2e1a07d54ac22a342b2f27ddcacb69e5973d6f606000c0e2fcd601f97a6f9c6ed8c488d759a9af3c95e16a70c6f6aca691567734a2fae852b9acc01d34cae46

data/.github/workflows/add-to-project.yml

@@ -0,0 +1,17 @@
+name: Add bugs to fluent project
+
+on:
+  issues:
+    types:
+      - opened
+
+jobs:
+  add-to-project:
+    name: Add issue to project
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/add-to-project@v1.0.2
+        with:
+          project-url: https://github.com/orgs/fluent/projects/4
+          github-token: ${{ secrets.ADD_TO_PROJECT_PAT }}
+          labeled: waiting-for-triage

data/.github/workflows/linux.yml

@@ -12,12 +12,22 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '3.2', '3.1', '3.0', '2.7' ]
+        ruby: [ '3.4', '3.3', '3.2', '3.1', '3.0' ]
         os:
           - ubuntu-latest
         rdkafka_versions:
           - { min: '>= 0.6.0', max: '< 0.12.0' }
-          - { min: '>= 0.12.0', max: '>= 0.12.0' }
+          - { min: '>= 0.12.0', max: '< 0.14.0' }
+          - { min: '>= 0.14.0', max: '< 0.16.0' }
+          - { min: '>= 0.16.0', max: '>= 0.16.0' }
+        # rdkafka 0.15.2 is the last version which supports Ruby 2.7
+        include:
+          - ruby: '2.7'
+            os: ubuntu-latest
+            rdkafka_versions: { min: '>= 0.6.0', max: '< 0.12.0' }
+          - ruby: '2.7'
+            os: ubuntu-latest
+            rdkafka_versions: { min: '>= 0.12.0', max: '= 0.15.2' }
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }} with rdkafka gem version (min ${{ matrix.rdkafka_versions.min }} max ${{ matrix.rdkafka_versions.max }})
     steps:
     - uses: actions/checkout@v4
@@ -40,6 +50,6 @@ jobs:
         RDKAFKA_VERSION_MAX_RANGE: ${{ matrix.rdkafka_versions.max }}
       run: |
         sudo ./ci/prepare-kafka-server.sh
-        gem install bundler rake
+        gem install rake
         bundle install --jobs 4 --retry 3
         bundle exec rake test

data/.github/workflows/stale-actions.yml

@@ -7,7 +7,7 @@ jobs:
   stale:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/stale@v8
+    - uses: actions/stale@v9
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         days-before-stale: 90

data/ChangeLog

@@ -1,3 +1,16 @@
+Release 0.19.4 - 2025/03/24
+	* Support Ruby 3.4. (#526)
+
+Release 0.19.3 - 2024/08/02
+	* out_rdkafka2: Add `unrecoverable_error_codes` parameter to handle
+			specific error code as unrecoverable errors. `topic_authorization_failed`
+			and `msg_size_too_large` are treated as such unrecoverable error by default. (#510)
+	* out_rdkafka2: Add missing closing timeout feature to keep compatibility with
+			rdkafka-ruby 0.12.x or later. (#505)
+	* out_rdkafka2: Add `idempotent` parameter to enable idempotence in Kafka producer. (#501)
+	* out_kafka2: Fix errors while sending data to EventHub by adding
+		      broker pool to take care of fetching metadata (#503)
+
 Release 0.19.2 - 2023/10/13
 	* out_rdkafka2: Add `discard_kafka_delivery_failed_regex`
 

data/README.md

@@ -212,6 +212,7 @@ If `ruby-kafka` doesn't fit your kafka environment, check `rdkafka2` plugin inst
       discard_kafka_delivery_failed (bool) :default => false (No discard)
       partitioner_hash_function (enum) (crc32|murmur2) :default => 'crc32'
       share_producer        (bool)   :default => false
+      idempotent            (bool)   :default => false
 
       # If you intend to rely on AWS IAM auth to MSK with long lived credentials
       # https://docs.aws.amazon.com/msk/latest/developerguide/iam-access-control.html
@@ -509,7 +510,6 @@ You need to install rdkafka gem.
       partition_key         (string) :default => 'partition'
       partition_key_key     (string) :default => 'partition_key'
       message_key_key       (string) :default => 'message_key'
-      default_topic         (string) :default => nil
       use_default_for_unknown_topic           (bool) :default => false
       use_default_for_unknown_partition_error (bool) :default => false
       default_partition_key (string) :default => nil
@@ -558,6 +558,8 @@ You need to install rdkafka gem.
       # load of both Fluentd and Kafka when excessive messages are attempted
       # to send. Default is no limit.
       max_enqueue_bytes_per_second (integer) :default => nil
+      unrecoverable_error_codes (array) :default => ["topic_authorization_failed", "msg_size_too_large"]
+
     </match>
 
 `rdkafka2` supports `discard_kafka_delivery_failed_regex` parameter:

data/fluent-plugin-kafka.gemspec

@@ -13,12 +13,16 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.19.2'
+  gem.version       = '0.19.4'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
   gem.add_dependency 'ltsv'
   gem.add_dependency 'ruby-kafka', '>= 1.5.0', '< 2'
+
+  # gems that aren't default gems as of Ruby 3.4
+  gem.add_dependency("bigdecimal", ["~> 3.1"])
+
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "test-unit", ">= 3.0.8"
   gem.add_development_dependency "test-unit-rr", "~> 1.0"

data/lib/fluent/plugin/kafka_producer_ext.rb

@@ -93,6 +93,10 @@ module Kafka
       @max_buffer_bytesize = max_buffer_bytesize
       @compressor = compressor
       @partitioner = partitioner
+
+      # The set of topics that are produced to.
+      @target_topics = Set.new
+
       # A buffer organized by topic/partition.
       @buffer = MessageBuffer.new
 
@@ -116,7 +120,8 @@ module Kafka
       if @transaction_manager.transactional? && !@transaction_manager.in_transaction?
         raise 'You must trigger begin_transaction before producing messages'
       end
-
+      
+      @target_topics.add(topic)
       @pending_message_queue.write(message)
 
       nil
@@ -187,7 +192,7 @@ module Kafka
     def deliver_messages_with_retries
       attempt = 0
 
-      #@cluster.add_target_topics(@target_topics)
+      @cluster.add_target_topics(@target_topics)
 
       operation = ProduceOperation.new(
         cluster: @cluster,

data/lib/fluent/plugin/out_rdkafka2.rb

@@ -5,48 +5,19 @@ require 'fluent/plugin/kafka_plugin_util'
 
 require 'rdkafka'
 
-# This is required for `rdkafka` version >= 0.12.0
-# Overriding the close method in order to provide a time limit for when it should be forcibly closed
-class Rdkafka::Producer::Client
-  # return false if producer is forcefully closed, otherwise return true
-  def close(timeout=nil)
-    return unless @native
-
-    # Indicate to polling thread that we're closing
-    @polling_thread[:closing] = true
-    # Wait for the polling thread to finish up
-    thread = @polling_thread.join(timeout)
-
-    Rdkafka::Bindings.rd_kafka_destroy(@native)
-
-    @native = nil
-
-    return !thread.nil?
-  end
-end
-
-class Rdkafka::Producer
-  # return false if producer is forcefully closed, otherwise return true
-  def close(timeout = nil)
-    rdkafka_version = Rdkafka::VERSION || '0.0.0'
-    # Rdkafka version >= 0.12.0 changed its internals
-    if Gem::Version::create(rdkafka_version) >= Gem::Version.create('0.12.0')
-      ObjectSpace.undefine_finalizer(self)
-
-      return @client.close(timeout)
-    end
-
-    @closing = true
-    # Wait for the polling thread to finish up
-    # If the broker isn't alive, the thread doesn't exit
-    if timeout
-      thr = @polling_thread.join(timeout)
-      return !!thr
-    else
-      @polling_thread.join
-      return true
-    end
+begin
+  rdkafka_version = Gem::Version::create(Rdkafka::VERSION)
+  if rdkafka_version < Gem::Version.create('0.12.0')
+    require_relative 'rdkafka_patch/0_11_0'
+  elsif rdkafka_version == Gem::Version.create('0.12.0')
+    require_relative 'rdkafka_patch/0_12_0'
+  elsif rdkafka_version >= Gem::Version.create('0.14.0')
+    require_relative 'rdkafka_patch/0_14_0'
+  elsif rdkafka_version >= Gem::Version.create('0.16.0')
+    require_relative 'rdkafka_patch/0_16_0'
   end
+rescue LoadError, NameError
+  raise "unable to patch rdkafka."
 end
 
 module Fluent::Plugin
@@ -71,6 +42,7 @@ DESC
     config_param :default_message_key, :string, :default => nil
     config_param :partition_key, :string, :default => 'partition', :desc => "Field for kafka partition"
     config_param :default_partition, :integer, :default => nil
+    config_param :idempotent, :bool, :default => false, :desc => 'Enable idempotent producer'
     config_param :output_data_type, :string, :default => 'json', :obsoleted => "Use <format> section instead"
     config_param :output_include_tag, :bool, :default => false, :obsoleted => "Use <inject> section instead"
     config_param :output_include_time, :bool, :default => false, :obsoleted => "Use <inject> section instead"
@@ -126,6 +98,8 @@ DESC
     config_param :max_enqueue_bytes_per_second, :size, :default => nil, :desc => 'The maximum number of enqueueing bytes per second'
 
     config_param :service_name, :string, :default => nil, :desc => 'Used for sasl.kerberos.service.name'
+    config_param :unrecoverable_error_codes, :array, :default => ["topic_authorization_failed", "msg_size_too_large"],
+                 :desc => 'Handle some of the error codes should be unrecoverable if specified'
 
     config_section :buffer do
       config_set_default :chunk_keys, ["topic"]
@@ -313,6 +287,7 @@ DESC
       config[:"batch.num.messages"] = @rdkafka_message_max_num if @rdkafka_message_max_num
       config[:"sasl.username"] = @username if @username
       config[:"sasl.password"] = @password if @password
+      config[:"enable.idempotence"] = @idempotent if @idempotent
 
       @rdkafka_options.each { |k, v|
         config[k.to_sym] = v
@@ -522,7 +497,12 @@ DESC
 
             raise e
           else
-            raise e
+            if unrecoverable_error_codes.include?(e.code.to_s)
+              # some of the errors should be handled as an unrecoverable error
+              raise Fluent::UnrecoverableError, "Rejected due to #{e}"
+            else
+              raise e
+            end
           end
         end
       end

data/lib/fluent/plugin/rdkafka_patch/0_11_0.rb

@@ -0,0 +1,15 @@
+class Rdkafka::Producer
+  # return false if producer is forcefully closed, otherwise return true
+  def close(timeout = nil)
+    @closing = true
+    # Wait for the polling thread to finish up
+    # If the broker isn't alive, the thread doesn't exit
+    if timeout
+      thr = @polling_thread.join(timeout)
+      return !!thr
+    else
+      @polling_thread.join
+      return true
+    end
+  end
+end

data/lib/fluent/plugin/rdkafka_patch/0_12_0.rb

@@ -0,0 +1,27 @@
+# This is required for `rdkafka` version >= 0.12.0
+# Overriding the close method in order to provide a time limit for when it should be forcibly closed
+class Rdkafka::Producer::Client
+  # return false if producer is forcefully closed, otherwise return true
+  def close(timeout=nil)
+    return unless @native
+
+    # Indicate to polling thread that we're closing
+    @polling_thread[:closing] = true
+    # Wait for the polling thread to finish up
+    thread = @polling_thread.join(timeout)
+
+    Rdkafka::Bindings.rd_kafka_destroy(@native)
+
+    @native = nil
+
+    return !thread.nil?
+  end
+end
+
+class Rdkafka::Producer
+  def close(timeout = nil)
+    ObjectSpace.undefine_finalizer(self)
+
+    return @client.close(timeout)
+  end
+end

data/lib/fluent/plugin/rdkafka_patch/0_14_0.rb

@@ -0,0 +1,44 @@
+class Rdkafka::NativeKafka
+  # return false if producer is forcefully closed, otherwise return true
+  def close(timeout=nil, object_id=nil)
+    return true if closed?
+
+    synchronize do
+      # Indicate to the outside world that we are closing
+      @closing = true
+
+      thread_status = :unknown
+      if @polling_thread
+        # Indicate to polling thread that we're closing
+        @polling_thread[:closing] = true
+
+        # Wait for the polling thread to finish up,
+        # this can be aborted in practice if this
+        # code runs from a finalizer.
+        thread_status = @polling_thread.join(timeout)
+      end
+
+      # Destroy the client after locking both mutexes
+      @poll_mutex.lock
+
+      # This check prevents a race condition, where we would enter the close in two threads
+      # and after unlocking the primary one that hold the lock but finished, ours would be unlocked
+      # and would continue to run, trying to destroy inner twice
+      if @inner
+        Rdkafka::Bindings.rd_kafka_destroy(@inner)
+        @inner = nil
+        @opaque = nil
+      end
+
+      !thread_status.nil?
+    end
+  end
+end
+
+class Rdkafka::Producer
+  def close(timeout = nil)
+    return true if closed?
+    ObjectSpace.undefine_finalizer(self)
+    @native_kafka.close(timeout)
+  end
+end

data/lib/fluent/plugin/rdkafka_patch/0_16_0.rb

@@ -0,0 +1,55 @@
+class Rdkafka::NativeKafka
+  # return false if producer is forcefully closed, otherwise return true
+  def close(timeout=nil, object_id=nil)
+    return true if closed?
+
+    synchronize do
+      # Indicate to the outside world that we are closing
+      @closing = true
+
+      thread_status = :unknown
+      if @polling_thread
+        # Indicate to polling thread that we're closing
+        @polling_thread[:closing] = true
+
+        # Wait for the polling thread to finish up,
+        # this can be aborted in practice if this
+        # code runs from a finalizer.
+        thread_status = @polling_thread.join(timeout)
+      end
+
+      # Destroy the client after locking both mutexes
+      @poll_mutex.lock
+
+      # This check prevents a race condition, where we would enter the close in two threads
+      # and after unlocking the primary one that hold the lock but finished, ours would be unlocked
+      # and would continue to run, trying to destroy inner twice
+      retun unless @inner
+
+      Rdkafka::Bindings.rd_kafka_destroy(@inner)
+      @inner = nil
+      @opaque = nil
+
+      !thread_status.nil?
+    end
+  end
+end
+
+class Rdkafka::Producer
+  def close(timeout = nil)
+    return true if closed?
+    ObjectSpace.undefine_finalizer(self)
+
+    @native_kafka.close(timeout) do
+      # We need to remove the topics references objects before we destroy the producer,
+      # otherwise they would leak out
+      @topics_refs_map.each_value do |refs|
+        refs.each_value do |ref|
+          Rdkafka::Bindings.rd_kafka_topic_destroy(ref)
+        end
+      end
+    end
+
+    @topics_refs_map.clear
+  end
+end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.19.2
+  version: 0.19.4
 platform: ruby
 authors:
 - Hidemasa Togashi
 - Masahiro Nakagawa
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-10-13 00:00:00.000000000 Z
+date: 2025-03-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -65,6 +65,20 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
+- !ruby/object:Gem::Dependency
+  name: bigdecimal
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.1'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -147,6 +161,7 @@ files:
 - ".github/ISSUE_TEMPLATE/config.yml"
 - ".github/ISSUE_TEMPLATE/feature_request.yaml"
 - ".github/dependabot.yml"
+- ".github/workflows/add-to-project.yml"
 - ".github/workflows/linux.yml"
 - ".github/workflows/stale-actions.yml"
 - ".gitignore"
@@ -171,6 +186,10 @@ files:
 - lib/fluent/plugin/out_kafka_buffered.rb
 - lib/fluent/plugin/out_rdkafka.rb
 - lib/fluent/plugin/out_rdkafka2.rb
+- lib/fluent/plugin/rdkafka_patch/0_11_0.rb
+- lib/fluent/plugin/rdkafka_patch/0_12_0.rb
+- lib/fluent/plugin/rdkafka_patch/0_14_0.rb
+- lib/fluent/plugin/rdkafka_patch/0_16_0.rb
 - test/helper.rb
 - test/plugin/test_in_kafka.rb
 - test/plugin/test_in_kafka_group.rb
@@ -183,7 +202,7 @@ homepage: https://github.com/fluent/fluent-plugin-kafka
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -198,8 +217,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.5
-signing_key: 
+rubygems_version: 3.5.22
+signing_key:
 specification_version: 4
 summary: Fluentd plugin for Apache Kafka > 0.8
 test_files: