fluent-plugin-kafka 0.19.8 → 0.19.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f3e5a24c5a5c936b9d502631d6382faa9315a54f0813e5b636cf025ecb64212c
-  data.tar.gz: 43c1b088840c92c330c70c456d92e3849d9de326b6b772c55bae63affea9f3e3
+  metadata.gz: 2eeb3b37a48db8cde6fda0eb9e413f507076c231abd06456a35540a10eed5667
+  data.tar.gz: 90b95493d3445829123ee569af66f581b109dcea7f91cf3d9aee6b2b6b9caeab
 SHA512:
-  metadata.gz: 608800269cb2f306911e4a1a6a6aeea8b7e0e69fde4b34b5a165f67d4a306e82226de890151ffff813a9ba91ef4399d87aeb531243cb63cdf625a023cf92d3e7
-  data.tar.gz: e818390d3f8946d0449a80d8c8ddd612e4a96632b2d6765fdcbf3beeb394c0cdc7c95e0cdce8c0780a05e66e39812328ddb48cd32135476cf5a85dac244485ef
+  metadata.gz: 3abf77244f8a7b239e92f63c798f0dc70b0d078a3b75de8f17ddc9e7cb17a8108088cc4f4bc3e44005496c68c89e960ca727def67cb7418f8e001bb713c938b4
+  data.tar.gz: 892d0c09e810b9fe78f87917a8468ac392b206991e99c4080f755e80953db79d8637e41717ef3ae0acbf13baa26e2fd1fe032bef7650ca70bc62df44b1dead7a

data/.github/workflows/linux.yml

@@ -11,7 +11,7 @@ jobs:
     uses: ruby/actions/.github/workflows/ruby_versions.yml@master
     with:
       engine: cruby
-      min_version: 3.0
+      min_version: 3.2
   build:
     needs: ruby-versions
     runs-on: ${{ matrix.os }}
@@ -29,25 +29,12 @@ jobs:
           - { min: '>= 0.14.0', max: '< 0.16.0' }
           - { min: '>= 0.16.0', max: '< 0.25.0' }
           - { min: '>= 0.25.0', max: '>= 0.25.0' }
-        # rdkafka 0.25.0 supports Ruby 3.2+, so we exclude it for Ruby 3.0 and 3.1
         exclude:
           - ruby: head
-          - ruby: '3.1'
-            rdkafka_versions: { min: '>= 0.25.0', max: '>= 0.25.0' }
-          - ruby: '3.0'
-            rdkafka_versions: { min: '>= 0.25.0', max: '>= 0.25.0' }
-        # rdkafka 0.15.2 is the last version which supports Ruby 2.7
-        include:
-          - ruby: '2.7'
-            os: ubuntu-latest
-            rdkafka_versions: { min: '>= 0.6.0', max: '< 0.12.0' }
-          - ruby: '2.7'
-            os: ubuntu-latest
-            rdkafka_versions: { min: '>= 0.12.0', max: '= 0.15.2' }
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }} with rdkafka gem version (min ${{ matrix.rdkafka_versions.min }} max ${{ matrix.rdkafka_versions.max }})
     steps:
     - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-    - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0
+    - uses: ruby/setup-ruby@14594264cd68ce8a2345dd349bc3d138a4ef85c8 # v1.327.0
       with:
         ruby-version: ${{ matrix.ruby }}
     - name: Install confluent-kafka

data/ChangeLog

@@ -1,3 +1,25 @@
+Release 0.19.9 - 2026/10/30
+	* out_kafka2: Fix duplicated records and failing writes with
+				`share_producer true`. All flush threads used one ruby-kafka producer
+				at the same time, although it is not thread safe. NOTE: only one flush
+				thread at a time now uses the shared producer, so chunks are no longer
+				sent in parallel even if `flush_thread_count` is 2 or more. Keep
+				`share_producer false` to send them in parallel. (#589)
+	* out_kafka2: Fix later writes failing until restart after a delivery
+				error. Since 0.18.0, the messages of a failed delivery stayed in the
+				producer, so later writes could keep failing with
+				`Kafka::MessageSizeTooLarge`. The producer is now discarded after a
+				failure and created again on the next write. (#588, #589)
+	* in_kafka: Fix endless re-fetching of a batch with a record larger than
+				`chunk_limit_size`. The offset did not advance, and the records that
+				did fit were emitted again on every `interval`. NOTE: such a record is
+				now dropped with a warning. Use `<label @ERROR>` to keep it. (#584)
+	* in_kafka_group: Fix endless consumer restarts on a record larger than
+				`chunk_limit_size`. The consumer restarted without committing offsets,
+				so the same batch was fetched forever, and the records that did fit
+				were emitted again each time. NOTE: such a record is now dropped with a
+				warning. Use `<label @ERROR>` to keep it. (#583)
+
 Release 0.19.8 - 2026/09/04
 	* in_kafka: Emit each record with its own tag when `tag_source record` is
 				set. The whole fetch batch used to be emitted with the tag of the

data/README.md

@@ -263,6 +263,8 @@ The `<formatter name>` in `<format>` uses fluentd's formatter plugins. See [form
 
 **Note:** Java based Kafka client uses `murmur2` as partitioner function by default. If you want to use same partitioning behavior with fluent-plugin-kafka, change it to `murmur2` instead of `crc32`. Note that for using `murmur2` hash partitioner function, you must install `digest-murmurhash` gem.
 
+**Note:** `share_producer true` makes all flush threads use one ruby-kafka producer. This reduces the connections to the brokers. However, the ruby-kafka producer is not thread safe, so the flush threads send chunks one at a time. To send chunks in parallel with `flush_thread_count`, keep `share_producer` false. `rdkafka2` can share its producer without this limit, because librdkafka is thread safe.
+
 ruby-kafka sometimes returns `Kafka::DeliveryFailed` error without good information.
 In this case, `get_kafka_client_log` is useful for identifying the error cause.
 ruby-kafka's log is routed to fluentd log so you can see ruby-kafka's log in fluentd logs.
@@ -273,7 +275,7 @@ Supports following ruby-kafka's producer options.
 - required_acks - default: -1 - The number of acks required per request. If you need flush performance, set lower value, e.g. 1, 2.
 - ack_timeout - default: nil - How long the producer waits for acks. The unit is seconds.
 - compression_codec - default: nil - The codec the producer uses to compress messages.
-- max_send_limit_bytes - default: nil - Max byte size to send message to avoid MessageSizeTooLarge. For example, if you set 1000000(message.max.bytes in kafka), Message more than 1000000 byes will be dropped.
+- max_send_limit_bytes - default: nil - Max byte size of one record after formatting. A larger record is dropped with a warning, and the other records in the chunk are sent. For example, if you set 1000000, a record larger than 1000000 bytes is dropped. This does not limit the size of a record batch, so it does not prevent MessageSizeTooLarge by itself. See [FAQ](#why-does-fluentd-fail-with-kafkamessagesizetoolarge-or-msg_size_too_large).
 - discard_kafka_delivery_failed - default: false - discard the record where [Kafka::DeliveryFailed](http://www.rubydoc.info/gems/ruby-kafka/Kafka/DeliveryFailed) occurred
 
 If you want to know about detail of monitoring, see also https://github.com/zendesk/ruby-kafka#monitoring
@@ -451,7 +453,7 @@ Support of fluentd v0.12 has ended. `kafka_buffered` will be an alias of `kafka2
 - required_acks - default: -1 - The number of acks required per request. If you need flush performance, set lower value, e.g. 1, 2.
 - ack_timeout - default: nil - How long the producer waits for acks. The unit is seconds.
 - compression_codec - default: nil - The codec the producer uses to compress messages.
-- max_send_limit_bytes - default: nil - Max byte size to send message to avoid MessageSizeTooLarge. For example, if you set 1000000(message.max.bytes in kafka), Message more than 1000000 byes will be dropped.
+- max_send_limit_bytes - default: nil - Max byte size of one record after formatting. A larger record is dropped with a warning, and the other records in the chunk are sent. For example, if you set 1000000, a record larger than 1000000 bytes is dropped. This does not limit the size of a record batch, so it does not prevent MessageSizeTooLarge by itself. See [FAQ](#why-does-fluentd-fail-with-kafkamessagesizetoolarge-or-msg_size_too_large).
 - discard_kafka_delivery_failed - default: false - discard the record where [Kafka::DeliveryFailed](http://www.rubydoc.info/gems/ruby-kafka/Kafka/DeliveryFailed) occurred
 - monitoring_list - default: [] - library to be used to monitor. statsd and datadog are supported
 
@@ -564,7 +566,8 @@ You need to install rdkafka gem.
       share_producer (bool) :default => false
       # Timeout for polling message wait. If 0, no wait.
       rdkafka_delivery_handle_poll_timeout (integer) :default => 30
-      # If the record size is larger than this value, such records are ignored. Default is no limit
+      # Max byte size of one record after formatting. Larger records are dropped.
+      # This does not limit the size of a record batch. Default is no limit
       max_send_limit_bytes (integer) :default => nil
       # The maximum number of enqueueing bytes per second. It can reduce the
       # load of both Fluentd and Kafka when excessive messages are attempted
@@ -608,6 +611,40 @@ To avoid the problem, there are 2 approaches:
 - Upgrade your kafka cluster to latest version. This is better because recent version is faster and robust.
 - Downgrade ruby-kafka/fluent-plugin-kafka to work with your older kafka.
 
+### Why does Fluentd fail with Kafka::MessageSizeTooLarge (or msg_size_too_large)?
+
+Kafka rejects a record batch that is larger than the size limit of the topic. The limit is `max.message.bytes` of the topic if it is set, and `message.max.bytes` of the broker otherwise. The topic value can be larger or smaller than the broker value. Below, "the topic limit" means this value. If one output writes to several topics, use the smallest one.
+
+`kafka2` sends all records of a buffer chunk at once, and the records for the same partition go into one record batch. In the worst case, the whole chunk becomes one record batch. To make the batches smaller, lower `chunk_limit_size` in `<buffer>`.
+
+`chunk_limit_size` is the size of a chunk in the buffer, where records are kept as msgpack. The records sent to Kafka can be larger, because of `<format>`, `<inject>` and the overhead of each Kafka record. Start with about half of the topic limit, and lower it if the error continues. If `<buffer>` has `compress gzip` or `compress zstd`, `chunk_limit_size` limits the compressed size, and a chunk can hold several times more data. In that case, divide it further by the compression ratio of your data. `compression_codec` works the other way. Kafka checks the size of a record batch after compression, so compressed batches leave more room. How much depends on how well your data compresses.
+
+`max_send_limit_bytes` does not make the batch smaller. It checks only the formatted value of each record, and drops the records larger than this setting. The message key, headers and the overhead of the batch are not counted. To drop a single record that Kafka would reject, set it below the topic limit with some room for them.
+
+The following example assumes a topic limit of 1MB.
+
+    <match app.**>
+      @type kafka2
+      brokers <broker1_host>:<broker1_port>
+      default_topic app
+
+      max_send_limit_bytes 1000000
+
+      <format>
+        @type json
+      </format>
+
+      <buffer topic>
+        chunk_limit_size 512k
+      </buffer>
+    </match>
+
+Fluentd may log `chunk bytes limit exceeds for an emitted event stream`. This warning comes from the Fluentd buffer. It means that one event stream emitted to the output was larger than `chunk_limit_size`, and Fluentd splits it into several chunks. The number in the warning is the size of that event stream in the buffer, not the growth by formatting. If a single record is larger than `chunk_limit_size`, that record cannot be stored, and Fluentd raises `BufferChunkOverflowError`.
+
+`kafka_buffered` splits a chunk into smaller deliveries by `kafka_agg_max_bytes` (default: 4096) and `kafka_agg_max_messages`, so `chunk_limit_size` does not decide its batch size.
+
+`rdkafka2` passes records to librdkafka one by one, and librdkafka builds the record batches within its own `batch.size` and `message.max.bytes`. You can set them in `rdkafka_options`. Keep them below the topic limit. `msg_size_too_large` is in the default of `unrecoverable_error_codes`, so Fluentd does not retry a chunk that hits it.
+
 ## Contributing
 
 1. Fork it

data/fluent-plugin-kafka.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.19.8'
+  gem.version       = '0.19.9'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/in_kafka.rb

@@ -62,6 +62,12 @@ class Fluent::KafkaInput < Fluent::Input
   include Fluent::KafkaPluginUtil::SSLSettings
   include Fluent::KafkaPluginUtil::SaslSettings
 
+  BufferChunkOverflowError = if defined?(Fluent::Plugin::Buffer::BufferChunkOverflowError)
+                               Fluent::Plugin::Buffer::BufferChunkOverflowError
+                             else
+                               Fluent::BufferChunkLimitError
+                             end
+
   unless method_defined?(:router)
     define_method("router") { Fluent::Engine }
   end
@@ -348,7 +354,11 @@ class Fluent::KafkaInput < Fluent::Input
       offset = messages.last.offset + 1
 
       event_streams.each { |tag, es|
-        @router.emit_stream(tag, es)
+        begin
+          @router.emit_stream(tag, es)
+        rescue BufferChunkOverflowError => e
+          $log.warn "Skipped records larger than the buffer chunk limit size in #{@topic_entry.topic}/#{@topic_entry.partition}", :tag => tag, :error => e.to_s
+        end
       }
 
       if @offset_manager

data/lib/fluent/plugin/in_kafka_group.rb

@@ -85,6 +85,12 @@ class Fluent::KafkaGroupInput < Fluent::Input
                   Fluent::BufferQueueLimitError
                 end
 
+  BufferChunkOverflowError = if defined?(Fluent::Plugin::Buffer::BufferChunkOverflowError)
+                               Fluent::Plugin::Buffer::BufferChunkOverflowError
+                             else
+                               Fluent::BufferChunkLimitError
+                             end
+
   unless method_defined?(:router)
     define_method("router") { Fluent::Engine }
   end
@@ -380,6 +386,8 @@ class Fluent::KafkaGroupInput < Fluent::Input
     retries = 0
     begin
       router.emit_stream(tag, es)
+    rescue BufferChunkOverflowError => e
+      log.warn "Skipped records larger than the buffer chunk limit size", :tag => tag, :error => e.to_s
     rescue BufferError
       raise ForShutdown if @consumer.nil?
 

data/lib/fluent/plugin/out_kafka2.rb

@@ -198,6 +198,10 @@ DESC
         end
       end
 
+      if @share_producer && @buffer_config.flush_thread_count > 1
+        log.warn "share_producer sends one chunk at a time even with flush_thread_count #{@buffer_config.flush_thread_count}. Set share_producer false to send chunks in parallel"
+      end
+
       @producer_opts = {max_retries: @max_send_retries, required_acks: @required_acks, idempotent: @idempotent}
       @producer_opts[:ack_timeout] = @ack_timeout if @ack_timeout
       @producer_opts[:compression_codec] = @compression_codec.to_sym if @compression_codec
@@ -240,6 +244,7 @@ DESC
 
       if @share_producer
         @shared_producer = create_producer
+        @shared_producer_mutex = Mutex.new
       else
         @producers = {}
         @producers_mutex = Mutex.new
@@ -274,8 +279,10 @@ DESC
 
     def shutdown_producers
       if @share_producer
-        @shared_producer.shutdown
-        @shared_producer = nil
+        @shared_producer_mutex.synchronize {
+          @shared_producer.shutdown if @shared_producer
+          @shared_producer = nil
+        }
       else
         @producers_mutex.synchronize {
           shutdown_threads = @producers.map { |key, producer|
@@ -293,7 +300,7 @@ DESC
 
     def get_producer
       if @share_producer
-        @shared_producer
+        @shared_producer ||= create_producer
       else
         @producers_mutex.synchronize {
           producer = @producers[Thread.current.object_id]
@@ -306,6 +313,17 @@ DESC
       end
     end
 
+    def discard_producer
+      if @share_producer
+        producer, @shared_producer = @shared_producer, nil
+      else
+        producer = @producers_mutex.synchronize { @producers.delete(Thread.current.object_id) }
+      end
+      producer.shutdown if producer
+    rescue => e
+      log.warn "failed to shut down the discarded producer:", :error => e.to_s, :error_class => e.class.to_s
+    end
+
     def setup_formatter(conf)
       type = conf['@type']
       case type
@@ -327,10 +345,20 @@ DESC
       end
     end
 
-    # TODO: optimize write performance
     def write(chunk)
       @writing_threads_mutex.synchronize { @writing_threads.add(Thread.current) }
 
+      if @share_producer
+        @shared_producer_mutex.synchronize { write_chunk(chunk) }
+      else
+        write_chunk(chunk)
+      end
+    ensure
+      @writing_threads_mutex.synchronize { @writing_threads.delete(Thread.current) }
+    end
+
+    # TODO: optimize write performance
+    def write_chunk(chunk)
       tag = chunk.metadata.tag
       topic = if @topic
                 extract_placeholders(@topic, chunk)
@@ -418,12 +446,11 @@ DESC
       log.warn "Exception ignored in tag : #{tag}" if ignore
       # For safety, refresh client and its producers
       refresh_client(false)
+      discard_producer
       # raise UnrecoverableError for backup ignored exception chunk
       raise Fluent::UnrecoverableError if ignore && exception_backup
       # Raise exception to retry sendind messages
       raise e unless ignore
-    ensure
-      @writing_threads_mutex.synchronize { @writing_threads.delete(Thread.current) }
     end
   end
 end

data/test/plugin/test_in_kafka.rb

@@ -52,6 +52,18 @@ class KafkaInputTest < Test::Unit::TestCase
       end
     end
 
+    class OverflowRouter < FakeRouter
+      def initialize(overflow_tag)
+        super()
+        @overflow_tag = overflow_tag
+      end
+
+      def emit_stream(tag, es)
+        raise Fluent::KafkaInput::BufferChunkOverflowError, "too large" if tag == @overflow_tag
+        super
+      end
+    end
+
     def create_topic_watcher(messages, router, tag_source: :record, add_prefix: nil, add_suffix: nil)
       kafka = Object.new
       kafka.define_singleton_method(:fetch_messages) { |**args| messages }
@@ -159,6 +171,20 @@ class KafkaInputTest < Test::Unit::TestCase
       assert_equal([[TOPIC_NAME, ['record 1', 'record 2']]],
                    router.emitted.map { |tag, records| [tag, records.map { |r| r['message'] }] })
     end
+
+    def test_consume_skips_buffer_chunk_overflow
+      messages = [
+        tagged_message('app.large', 'record 1', 0),
+        tagged_message('app.small', 'record 2', 1),
+      ]
+      router = OverflowRouter.new('app.large')
+      watcher = create_topic_watcher(messages, router)
+      watcher.consume
+
+      assert_equal([['app.small', ['record 2']]],
+                   router.emitted.map { |tag, records| [tag, records.map { |r| r['message'] }] })
+      assert_equal(2, watcher.instance_variable_get(:@next_offset))
+    end
   end
 
   class ConsumeTest < self

data/test/plugin/test_in_kafka_group.rb

@@ -36,6 +36,19 @@ class KafkaGroupInputTest < Test::Unit::TestCase
     assert_true d.instance.multi_workers_ready?
   end
 
+  def test_emit_events_with_buffer_chunk_overflow
+    d = create_driver
+    es = Fluent::MultiEventStream.new
+    router = Object.new
+    mock(router).emit_stream("test", es).once { raise Fluent::Plugin::Buffer::BufferChunkOverflowError, "too large" }
+    stub(d.instance).router { router }
+
+    assert_nothing_raised do
+      d.instance.emit_events("test", es)
+    end
+    assert_true d.logs.any? { |log| log.include?("Skipped records larger than the buffer chunk limit size") && log.include?("too large") }
+  end
+
   class ConsumeTest < self
     def setup
       @kafka = Kafka.new(["localhost:9092"], client_id: 'kafka')

data/test/plugin/test_out_kafka2.rb

@@ -144,6 +144,79 @@ class Kafka2OutputTest < Test::Unit::TestCase
     assert_equal [10], producer.produced.collect { |message| message[:partition] }
   end
 
+  data("per thread producer" => false,
+       "shared producer" => true)
+  def test_write_failure_replaces_producer(share_producer)
+    d = create_driver(base_config + config_element('ROOT', '', {"default_topic" => "kitagawakeiko",
+                                                                "brokers" => "localhost:1",
+                                                                "share_producer" => share_producer}))
+    d.instance_start
+    producer = d.instance.get_producer
+    mock.proxy(producer).shutdown
+
+    assert_raise(Kafka::ConnectionError) {
+      d.instance.write(create_chunk([{"a" => "b"}]))
+    }
+
+    assert_not_same producer, d.instance.get_producer
+  ensure
+    d.instance_shutdown if d
+  end
+
+  data("per thread producer" => false,
+       "shared producer" => true)
+  def test_write_failure_keeps_error_when_producer_shutdown_fails(share_producer)
+    d = create_driver(base_config + config_element('ROOT', '', {"default_topic" => "kitagawakeiko",
+                                                                "brokers" => "localhost:1",
+                                                                "share_producer" => share_producer}))
+    d.instance_start
+    producer = d.instance.get_producer
+    stub(producer).shutdown { raise IOError }
+
+    assert_raise(Kafka::ConnectionError) {
+      d.instance.write(create_chunk([{"a" => "b"}]))
+    }
+  ensure
+    d.instance_shutdown if d
+  end
+
+  class SlowProducer < DummyProducer
+    attr_reader :max_concurrent_deliveries
+
+    def initialize
+      @mutex = Mutex.new
+      @deliveries = 0
+      @max_concurrent_deliveries = 0
+    end
+
+    def deliver_messages
+      @mutex.synchronize {
+        @deliveries += 1
+        @max_concurrent_deliveries = [@max_concurrent_deliveries, @deliveries].max
+      }
+      sleep 0.1
+      @mutex.synchronize { @deliveries -= 1 }
+    end
+
+    def shutdown
+    end
+  end
+
+  def test_shared_producer_delivers_one_chunk_at_a_time
+    d = create_driver(config + config_element('ROOT', '', {"share_producer" => true}))
+    producer = SlowProducer.new
+    stub(d.instance).create_producer { producer }
+    d.instance_start
+
+    2.times.map {
+      Thread.new { d.instance.write(create_chunk([{"a" => "b"}])) }
+    }.each(&:join)
+
+    assert_equal 1, producer.max_concurrent_deliveries
+  ensure
+    d.instance_shutdown if d
+  end
+
   data("crc32" => "crc32",
        "murmur2" => "murmur2")
   def test_partitioner_hash_function(data)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.19.8
+  version: 0.19.9
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -218,7 +218,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.18
+rubygems_version: 4.0.20
 specification_version: 4
 summary: Fluentd plugin for Apache Kafka > 0.8
 test_files: