fluent-plugin-kafka 0.19.1 → 0.19.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 342b66e96b3175aed8f7fe9bc0f29bb03f2737924229eaa63d7ffd408f5209cf
-  data.tar.gz: 89a9361a2c599eca1419668136ecb722c6eaa95e9ebc55f2e2a2be3f1346cafc
+  metadata.gz: 8be5590390d2220ee64b54e442178369ae55be6a111cdb60c9815fe2f96e9337
+  data.tar.gz: 8a76798c2d4c0f8abaaf3b0c209c0a4e6770aeb825756ef6715861e392f6b051
 SHA512:
-  metadata.gz: '0172977463edc0ff6085588bcded5bf3d33412e088422bf48954672507d7661bda68735d1415fcc9dcaebd132eff355cc714be619c15a107962e12dc69eba4e1'
-  data.tar.gz: 00054cf6be46a6041fd6f8712f3159f832f91deffcf34feef45fff919d2301b2e04b5ec0493a2d9086af87c3ce9ec37ca0f004a9fac006ee4e3c17d20b372984
+  metadata.gz: 2d8fc83684627125a52398ff3561ba4e0ac878bac7b6ab7075ab9d07f3a018ceaa132b013652290198f64333a336f614384a0b7c18392d05e6494bb3fe259ab4
+  data.tar.gz: 98257dcd81f581697a702ba85d6e2c01c497c5dc98d9dfbc3f908856e6b1d418380ce5c6fb9418fe69c2fc7ec5889b3c84c1aeb6ba4182a6cf3c08a7bd87fff5

data/.github/workflows/linux.yml

@@ -12,15 +12,29 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '3.2', '3.1', '3.0', '2.7' ]
+        ruby: [ '3.3', '3.2', '3.1', '3.0' ]
         os:
           - ubuntu-latest
         rdkafka_versions:
           - { min: '>= 0.6.0', max: '< 0.12.0' }
-          - { min: '>= 0.12.0', max: '>= 0.12.0' }
+          - { min: '>= 0.12.0', max: '< 0.14.0' }
+          - { min: '>= 0.14.0', max: '< 0.16.0' }
+          - { min: '>= 0.16.0', max: '>= 0.16.0' }
+        bundler_version:
+          - '2.5.16'
+        # rdkafka 0.15.2 is the last version which supports Ruby 2.7
+        include:
+          - ruby: '2.7'
+            os: ubuntu-latest
+            rdkafka_versions: { min: '>= 0.6.0', max: '< 0.12.0' }
+            bundler_version: '2.4.22'
+          - ruby: '2.7'
+            os: ubuntu-latest
+            rdkafka_versions: { min: '>= 0.12.0', max: '= 0.15.2' }
+            bundler_version: '2.4.22'
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }} with rdkafka gem version (min ${{ matrix.rdkafka_versions.min }} max ${{ matrix.rdkafka_versions.max }})
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}
@@ -40,6 +54,7 @@ jobs:
         RDKAFKA_VERSION_MAX_RANGE: ${{ matrix.rdkafka_versions.max }}
       run: |
         sudo ./ci/prepare-kafka-server.sh
-        gem install bundler rake
-        bundle install --jobs 4 --retry 3
-        bundle exec rake test
+        gem install bundler -v ${{ matrix.bundler_version }}
+        gem install rake
+        bundle _${{ matrix.bundler_version }}_ install --jobs 4 --retry 3
+        bundle _${{ matrix.bundler_version }}_ exec rake test

data/.github/workflows/stale-actions.yml

@@ -7,7 +7,7 @@ jobs:
   stale:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/stale@v8
+    - uses: actions/stale@v9
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         days-before-stale: 90

data/ChangeLog

@@ -1,3 +1,16 @@
+Release 0.19.3 - 2024/08/02
+	* out_rdkafka2: Add `unrecoverable_error_codes` parameter to handle
+			specific error code as unrecoverable errors. `topic_authorization_failed`
+			and `msg_size_too_large` are treated as such unrecoverable error by default. (#510)
+	* out_rdkafka2: Add missing closing timeout feature to keep compatibility with
+			rdkafka-ruby 0.12.x or later. (#505)
+	* out_rdkafka2: Add `idempotent` parameter to enable idempotence in Kafka producer. (#501)
+	* out_kafka2: Fix errors while sending data to EventHub by adding
+		      broker pool to take care of fetching metadata (#503)
+
+Release 0.19.2 - 2023/10/13
+	* out_rdkafka2: Add `discard_kafka_delivery_failed_regex`
+
 Release 0.19.1 - 2023/09/20
 	* out_rdkafka2: Add `use_default_for_unknown_topic` & `use_default_for_unknown_partition_error`
 

data/README.md

@@ -212,6 +212,7 @@ If `ruby-kafka` doesn't fit your kafka environment, check `rdkafka2` plugin inst
       discard_kafka_delivery_failed (bool) :default => false (No discard)
       partitioner_hash_function (enum) (crc32|murmur2) :default => 'crc32'
       share_producer        (bool)   :default => false
+      idempotent            (bool)   :default => false
 
       # If you intend to rely on AWS IAM auth to MSK with long lived credentials
       # https://docs.aws.amazon.com/msk/latest/developerguide/iam-access-control.html
@@ -509,7 +510,6 @@ You need to install rdkafka gem.
       partition_key         (string) :default => 'partition'
       partition_key_key     (string) :default => 'partition_key'
       message_key_key       (string) :default => 'message_key'
-      default_topic         (string) :default => nil
       use_default_for_unknown_topic           (bool) :default => false
       use_default_for_unknown_partition_error (bool) :default => false
       default_partition_key (string) :default => nil
@@ -517,6 +517,7 @@ You need to install rdkafka gem.
       exclude_topic_key     (bool) :default => false
       exclude_partition_key (bool) :default => false
       discard_kafka_delivery_failed (bool) :default => false (No discard)
+      discard_kafka_delivery_failed_regex (regexp) :default => nil (No discard)
       use_event_time        (bool) :default => false
 
       # same with kafka2
@@ -557,8 +558,13 @@ You need to install rdkafka gem.
       # load of both Fluentd and Kafka when excessive messages are attempted
       # to send. Default is no limit.
       max_enqueue_bytes_per_second (integer) :default => nil
+      unrecoverable_error_codes (array) :default => ["topic_authorization_failed", "msg_size_too_large"]
+
     </match>
 
+`rdkafka2` supports `discard_kafka_delivery_failed_regex` parameter:
+- `discard_kafka_delivery_failed_regex` - default: nil - discard the record where the Kafka::DeliveryFailed occurred and the emitted message matches the given regex pattern, such as `/unknown_topic/`. 
+
 If you use v0.12, use `rdkafka` instead.
 
     <match kafka.**>

data/fluent-plugin-kafka.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.19.1'
+  gem.version       = '0.19.3'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/kafka_producer_ext.rb

@@ -93,6 +93,10 @@ module Kafka
       @max_buffer_bytesize = max_buffer_bytesize
       @compressor = compressor
       @partitioner = partitioner
+
+      # The set of topics that are produced to.
+      @target_topics = Set.new
+
       # A buffer organized by topic/partition.
       @buffer = MessageBuffer.new
 
@@ -116,7 +120,8 @@ module Kafka
       if @transaction_manager.transactional? && !@transaction_manager.in_transaction?
         raise 'You must trigger begin_transaction before producing messages'
       end
-
+      
+      @target_topics.add(topic)
       @pending_message_queue.write(message)
 
       nil
@@ -187,7 +192,7 @@ module Kafka
     def deliver_messages_with_retries
       attempt = 0
 
-      #@cluster.add_target_topics(@target_topics)
+      @cluster.add_target_topics(@target_topics)
 
       operation = ProduceOperation.new(
         cluster: @cluster,

data/lib/fluent/plugin/out_rdkafka2.rb

@@ -5,48 +5,19 @@ require 'fluent/plugin/kafka_plugin_util'
 
 require 'rdkafka'
 
-# This is required for `rdkafka` version >= 0.12.0
-# Overriding the close method in order to provide a time limit for when it should be forcibly closed
-class Rdkafka::Producer::Client
-  # return false if producer is forcefully closed, otherwise return true
-  def close(timeout=nil)
-    return unless @native
-
-    # Indicate to polling thread that we're closing
-    @polling_thread[:closing] = true
-    # Wait for the polling thread to finish up
-    thread = @polling_thread.join(timeout)
-
-    Rdkafka::Bindings.rd_kafka_destroy(@native)
-
-    @native = nil
-
-    return !thread.nil?
-  end
-end
-
-class Rdkafka::Producer
-  # return false if producer is forcefully closed, otherwise return true
-  def close(timeout = nil)
-    rdkafka_version = Rdkafka::VERSION || '0.0.0'
-    # Rdkafka version >= 0.12.0 changed its internals
-    if Gem::Version::create(rdkafka_version) >= Gem::Version.create('0.12.0')
-      ObjectSpace.undefine_finalizer(self)
-
-      return @client.close(timeout)
-    end
-
-    @closing = true
-    # Wait for the polling thread to finish up
-    # If the broker isn't alive, the thread doesn't exit
-    if timeout
-      thr = @polling_thread.join(timeout)
-      return !!thr
-    else
-      @polling_thread.join
-      return true
-    end
+begin
+  rdkafka_version = Gem::Version::create(Rdkafka::VERSION)
+  if rdkafka_version < Gem::Version.create('0.12.0')
+    require_relative 'rdkafka_patch/0_11_0'
+  elsif rdkafka_version == Gem::Version.create('0.12.0')
+    require_relative 'rdkafka_patch/0_12_0'
+  elsif rdkafka_version >= Gem::Version.create('0.14.0')
+    require_relative 'rdkafka_patch/0_14_0'
+  elsif rdkafka_version >= Gem::Version.create('0.16.0')
+    require_relative 'rdkafka_patch/0_16_0'
   end
+rescue LoadError, NameError
+  raise "unable to patch rdkafka."
 end
 
 module Fluent::Plugin
@@ -71,6 +42,7 @@ DESC
     config_param :default_message_key, :string, :default => nil
     config_param :partition_key, :string, :default => 'partition', :desc => "Field for kafka partition"
     config_param :default_partition, :integer, :default => nil
+    config_param :idempotent, :bool, :default => false, :desc => 'Enable idempotent producer'
     config_param :output_data_type, :string, :default => 'json', :obsoleted => "Use <format> section instead"
     config_param :output_include_tag, :bool, :default => false, :obsoleted => "Use <inject> section instead"
     config_param :output_include_time, :bool, :default => false, :obsoleted => "Use <inject> section instead"
@@ -112,6 +84,7 @@ DESC
     config_param :use_event_time, :bool, :default => false, :desc => 'Use fluentd event time for rdkafka timestamp'
     config_param :max_send_limit_bytes, :size, :default => nil
     config_param :discard_kafka_delivery_failed, :bool, :default => false
+    config_param :discard_kafka_delivery_failed_regex, :regexp, :default => nil
     config_param :rdkafka_buffering_max_ms, :integer, :default => nil, :desc => 'Used for queue.buffering.max.ms'
     config_param :rdkafka_buffering_max_messages, :integer, :default => nil, :desc => 'Used for queue.buffering.max.messages'
     config_param :rdkafka_message_max_bytes, :integer, :default => nil, :desc => 'Used for message.max.bytes'
@@ -125,6 +98,8 @@ DESC
     config_param :max_enqueue_bytes_per_second, :size, :default => nil, :desc => 'The maximum number of enqueueing bytes per second'
 
     config_param :service_name, :string, :default => nil, :desc => 'Used for sasl.kerberos.service.name'
+    config_param :unrecoverable_error_codes, :array, :default => ["topic_authorization_failed", "msg_size_too_large"],
+                 :desc => 'Handle some of the error codes should be unrecoverable if specified'
 
     config_section :buffer do
       config_set_default :chunk_keys, ["topic"]
@@ -312,6 +287,7 @@ DESC
       config[:"batch.num.messages"] = @rdkafka_message_max_num if @rdkafka_message_max_num
       config[:"sasl.username"] = @username if @username
       config[:"sasl.password"] = @password if @password
+      config[:"enable.idempotence"] = @idempotent if @idempotent
 
       @rdkafka_options.each { |k, v|
         config[k.to_sym] = v
@@ -461,9 +437,13 @@ DESC
       if @discard_kafka_delivery_failed
         log.warn "Delivery failed. Discard events:", :error => e.to_s, :error_class => e.class.to_s, :tag => tag
       else
-        log.warn "Send exception occurred: #{e} at #{e.backtrace.first}"
-        # Raise exception to retry sendind messages
-        raise e
+	if @discard_kafka_delivery_failed_regex != nil && @discard_kafka_delivery_failed_regex.match?(e.to_s)
+          log.warn "Delivery failed and matched regexp pattern #{@discard_kafka_delivery_failed_regex}. Discard events:", :error => e.to_s, :error_class => e.class.to_s, :tag => tag
+	else
+	  log.warn "Send exception occurred: #{e} at #{e.backtrace.first}"
+          # Raise exception to retry sendind messages
+          raise e
+	end
       end
     ensure
       @writing_threads_mutex.synchronize { @writing_threads.delete(Thread.current) }
@@ -517,7 +497,12 @@ DESC
 
             raise e
           else
-            raise e
+            if unrecoverable_error_codes.include?(e.code.to_s)
+              # some of the errors should be handled as an unrecoverable error
+              raise Fluent::UnrecoverableError, "Rejected due to #{e}"
+            else
+              raise e
+            end
           end
         end
       end

data/lib/fluent/plugin/rdkafka_patch/0_11_0.rb

@@ -0,0 +1,15 @@
+class Rdkafka::Producer
+  # return false if producer is forcefully closed, otherwise return true
+  def close(timeout = nil)
+    @closing = true
+    # Wait for the polling thread to finish up
+    # If the broker isn't alive, the thread doesn't exit
+    if timeout
+      thr = @polling_thread.join(timeout)
+      return !!thr
+    else
+      @polling_thread.join
+      return true
+    end
+  end
+end

data/lib/fluent/plugin/rdkafka_patch/0_12_0.rb

@@ -0,0 +1,27 @@
+# This is required for `rdkafka` version >= 0.12.0
+# Overriding the close method in order to provide a time limit for when it should be forcibly closed
+class Rdkafka::Producer::Client
+  # return false if producer is forcefully closed, otherwise return true
+  def close(timeout=nil)
+    return unless @native
+
+    # Indicate to polling thread that we're closing
+    @polling_thread[:closing] = true
+    # Wait for the polling thread to finish up
+    thread = @polling_thread.join(timeout)
+
+    Rdkafka::Bindings.rd_kafka_destroy(@native)
+
+    @native = nil
+
+    return !thread.nil?
+  end
+end
+
+class Rdkafka::Producer
+  def close(timeout = nil)
+    ObjectSpace.undefine_finalizer(self)
+
+    return @client.close(timeout)
+  end
+end

data/lib/fluent/plugin/rdkafka_patch/0_14_0.rb

@@ -0,0 +1,44 @@
+class Rdkafka::NativeKafka
+  # return false if producer is forcefully closed, otherwise return true
+  def close(timeout=nil, object_id=nil)
+    return true if closed?
+
+    synchronize do
+      # Indicate to the outside world that we are closing
+      @closing = true
+
+      thread_status = :unknown
+      if @polling_thread
+        # Indicate to polling thread that we're closing
+        @polling_thread[:closing] = true
+
+        # Wait for the polling thread to finish up,
+        # this can be aborted in practice if this
+        # code runs from a finalizer.
+        thread_status = @polling_thread.join(timeout)
+      end
+
+      # Destroy the client after locking both mutexes
+      @poll_mutex.lock
+
+      # This check prevents a race condition, where we would enter the close in two threads
+      # and after unlocking the primary one that hold the lock but finished, ours would be unlocked
+      # and would continue to run, trying to destroy inner twice
+      if @inner
+        Rdkafka::Bindings.rd_kafka_destroy(@inner)
+        @inner = nil
+        @opaque = nil
+      end
+
+      !thread_status.nil?
+    end
+  end
+end
+
+class Rdkafka::Producer
+  def close(timeout = nil)
+    return true if closed?
+    ObjectSpace.undefine_finalizer(self)
+    @native_kafka.close(timeout)
+  end
+end

data/lib/fluent/plugin/rdkafka_patch/0_16_0.rb

@@ -0,0 +1,55 @@
+class Rdkafka::NativeKafka
+  # return false if producer is forcefully closed, otherwise return true
+  def close(timeout=nil, object_id=nil)
+    return true if closed?
+
+    synchronize do
+      # Indicate to the outside world that we are closing
+      @closing = true
+
+      thread_status = :unknown
+      if @polling_thread
+        # Indicate to polling thread that we're closing
+        @polling_thread[:closing] = true
+
+        # Wait for the polling thread to finish up,
+        # this can be aborted in practice if this
+        # code runs from a finalizer.
+        thread_status = @polling_thread.join(timeout)
+      end
+
+      # Destroy the client after locking both mutexes
+      @poll_mutex.lock
+
+      # This check prevents a race condition, where we would enter the close in two threads
+      # and after unlocking the primary one that hold the lock but finished, ours would be unlocked
+      # and would continue to run, trying to destroy inner twice
+      retun unless @inner
+
+      Rdkafka::Bindings.rd_kafka_destroy(@inner)
+      @inner = nil
+      @opaque = nil
+
+      !thread_status.nil?
+    end
+  end
+end
+
+class Rdkafka::Producer
+  def close(timeout = nil)
+    return true if closed?
+    ObjectSpace.undefine_finalizer(self)
+
+    @native_kafka.close(timeout) do
+      # We need to remove the topics references objects before we destroy the producer,
+      # otherwise they would leak out
+      @topics_refs_map.each_value do |refs|
+        refs.each_value do |ref|
+          Rdkafka::Bindings.rd_kafka_topic_destroy(ref)
+        end
+      end
+    end
+
+    @topics_refs_map.clear
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.19.1
+  version: 0.19.3
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-09-20 00:00:00.000000000 Z
+date: 2024-08-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -171,6 +171,10 @@ files:
 - lib/fluent/plugin/out_kafka_buffered.rb
 - lib/fluent/plugin/out_rdkafka.rb
 - lib/fluent/plugin/out_rdkafka2.rb
+- lib/fluent/plugin/rdkafka_patch/0_11_0.rb
+- lib/fluent/plugin/rdkafka_patch/0_12_0.rb
+- lib/fluent/plugin/rdkafka_patch/0_14_0.rb
+- lib/fluent/plugin/rdkafka_patch/0_16_0.rb
 - test/helper.rb
 - test/plugin/test_in_kafka.rb
 - test/plugin/test_in_kafka_group.rb