fluent-plugin-kafka 0.18.0 → 0.18.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0a8590f6d34bbdb2faa991bba6e32a1424623c7ddd9609dc1d2ffcbeabaa20e1
-  data.tar.gz: a653d9dba00fa82f18071304be9093cd6d620bf4bda29d5e00a75113d05e8e3e
+  metadata.gz: 8b47d0dafccd1d84a2adcef4e9a0830b26998fd21c4095a974887ffa201fc64c
+  data.tar.gz: 744716215237149802687884a5c306e4684f1d2e89959d13d04b64f3116a65f2
 SHA512:
-  metadata.gz: 69a784e7d69e2229a036110997b2691143008d4ca786e18415e9a806a6d577b38465a4a3931166418a5b7e022dddbc62d014c0c784277ccaaecc9b6570822564
-  data.tar.gz: 9f14d8d10a45bf390c801c93d79b893f4248ded7e37dc87f8c4453b04158754e1006aa7603874986e8c93812fe899197219abccb3bdb0ae4daacef6d516a0af3
+  metadata.gz: 0215e2229d272a24abef39f8179457a4585d582a388f8b03bc70a1d4a066d27422878c91277d5b7412b3ddd930716b337ff56a75e4fb8de00366063b7c47c52a
+  data.tar.gz: 6f47fa553a9cf2cf2d55b725d5b8f562e6aef1c3ef92aee597bbf9f84aac7e0fb7fac5d67864d935c02b1dc5a890563a9b530b75ae47d3bac1763fbc021f20bc

data/ChangeLog

@@ -1,3 +1,6 @@
+Release 0.18.1 - 2022/08/17
+	* out_kafka2: Fix a bug that it doesn't respect `chunk_limit_records` and `chunk_limit_size`
+
 Release 0.18.0 - 2022/07/21
 	* out_kafka2: Keep alive Kafka connections between flushes
 	* out_rdkafka2: Enable to set SASL credentials via `username` and `password` parameters

data/fluent-plugin-kafka.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.18.0'
+  gem.version       = '0.18.1'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/kafka_producer_ext.rb

@@ -38,9 +38,15 @@ module Kafka
 end
 
 # for out_kafka2
+# Majority (if not all) of this code is lifted from https://github.com/zendesk/ruby-kafka/blob/master/lib/kafka/producer.rb
+# with the main difference where we have removed any checks regarding max_buffer_bytesize and max_buffer_size
+# The reason for doing this is to provide a better UX for our users where they only need to set those bounds in
+# the Buffer section using `chunk_limit_size` and `chunk_limit_records`.
+#
+# We should reconsider this in the future in case the `ruby-kafka` library drastically changes its internal.
 module Kafka
   class Client
-    def topic_producer(topic, compression_codec: nil, compression_threshold: 1, ack_timeout: 5, required_acks: :all, max_retries: 2, retry_backoff: 1, max_buffer_size: 1000, max_buffer_bytesize: 10_000_000, idempotent: false, transactional: false, transactional_id: nil, transactional_timeout: 60)
+    def custom_producer(compression_codec: nil, compression_threshold: 1, ack_timeout: 5, required_acks: :all, max_retries: 2, retry_backoff: 1, max_buffer_size: 1000, max_buffer_bytesize: 10_000_000, idempotent: false, transactional: false, transactional_id: nil, transactional_timeout: 60)
       cluster = initialize_cluster
       compressor = Compressor.new(
         codec_name: compression_codec,
@@ -57,8 +63,7 @@ module Kafka
         transactional_timeout: transactional_timeout,
       )
 
-      TopicProducer.new(topic,
-        cluster: cluster,
+      CustomProducer.new(cluster: cluster,
         transaction_manager: transaction_manager,
         logger: @logger,
         instrumenter: @instrumenter,
@@ -74,8 +79,8 @@ module Kafka
     end
   end
 
-  class TopicProducer
-    def initialize(topic, cluster:, transaction_manager:, logger:, instrumenter:, compressor:, ack_timeout:, required_acks:, max_retries:, retry_backoff:, max_buffer_size:, max_buffer_bytesize:, partitioner:)
+  class CustomProducer
+    def initialize(cluster:, transaction_manager:, logger:, instrumenter:, compressor:, ack_timeout:, required_acks:, max_retries:, retry_backoff:, max_buffer_size:, max_buffer_bytesize:, partitioner:)
       @cluster = cluster
       @transaction_manager = transaction_manager
       @logger = logger
@@ -88,10 +93,6 @@ module Kafka
       @max_buffer_bytesize = max_buffer_bytesize
       @compressor = compressor
       @partitioner = partitioner
-
-      @topic = topic
-      @cluster.add_target_topics(Set.new([topic]))
-
       # A buffer organized by topic/partition.
       @buffer = MessageBuffer.new
 
@@ -99,12 +100,12 @@ module Kafka
       @pending_message_queue = PendingMessageQueue.new
     end
 
-    def produce(value, key: nil, partition: nil, partition_key: nil, headers: EMPTY_HEADER, create_time: Time.now)
+    def produce(value, key: nil, partition: nil, partition_key: nil, headers: EMPTY_HEADER, create_time: Time.now, topic: nil)
       message = PendingMessage.new(
         value: value,
         key: key,
         headers: headers,
-        topic: @topic,
+        topic: topic,
         partition: partition,
         partition_key: partition_key,
         create_time: create_time
@@ -245,12 +246,13 @@ module Kafka
 
     def assign_partitions!
       failed_messages = []
-      partition_count = @cluster.partitions_for(@topic).count
 
       @pending_message_queue.each do |message|
         partition = message.partition
 
         begin
+          partition_count = @cluster.partitions_for(message.topic).count
+
           if partition.nil?
             partition = @partitioner.call(partition_count, message)
           end

data/lib/fluent/plugin/out_kafka2.rb

@@ -207,7 +207,7 @@ DESC
     end
 
     def create_producer
-      @kafka.producer(**@producer_opts)
+      @kafka.custom_producer(**@producer_opts)
     end
 
     def start

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.18.0
+  version: 0.18.1
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-07-21 00:00:00.000000000 Z
+date: 2022-08-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd