fluent-plugin-kafka 0.11.1 → 0.11.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fa8bd187d82171d2d909caee040959b99380852c82c94e7ffb12d9c9b4ab824e
-  data.tar.gz: ef617b72fd2908a3119fd086c330f01da867c16314d85dc78151d7489e557332
+  metadata.gz: a32cfbf1850b74e02b8b698db2b3676f89239e647debf62ae7d71d0b3afc229c
+  data.tar.gz: c076287c2ea443fd645193432e93ab6ed0c1263043dac7c171943ac4a268da73
 SHA512:
-  metadata.gz: fda25bea7239416714baae9ba577aad680617f1482f48aae8243b570d5e3fd090cab86fa6da0efb7ea8b4542745e4ad13cb131bcaacf9a5e0c653df044ced319
-  data.tar.gz: 53a84fed9f9bb11fdf7b4306af0e1900c216afc4546dc012c532db459d69f95047205c1e020884410cd30ebef915ed8e424f0734b97afc59169a2fe3aa459f5a
+  metadata.gz: 5bb0d0d1628ad5c3193c1fd9c47af0531330dd6f6359b2cdd6d6e13821b2c023fee9d82b8758a022e6502bf09dc7c594583da2815d87d11dde1e7c0601517bf9
+  data.tar.gz: 05e1f5c8e66ef96a938ee79eacbdd95ecccea3f7fdce76e4888fc38c9b06c5166b57b9df5e7fdc4aeb09bae4099f1fd7907251a6d99cc3a384cb0568939739f6

data/ChangeLog

@@ -1,3 +1,7 @@
+Release 0.11.2 - 2019/09/27
+
+	* out_rdkafka2: Add share_producer parameter
+
 Release 0.11.1 - 2019/08/22
 
 	* out_kafka2: Fix mutating issue of headers

data/README.md

@@ -276,7 +276,7 @@ The configuration format is jsonpath. It is descibed in https://docs.fluentd.org
 
 ### Buffered output plugin
 
-This plugin uses ruby-kafka producer for writing data. This plugin works with recent kafka versions. This plugin is for v0.12. If you use v1, see `kafka2`.
+This plugin uses ruby-kafka producer for writing data. This plugin is for v0.12. If you use v1, see `kafka2`.
 
     <match app.**>
       @type kafka_buffered
@@ -373,8 +373,10 @@ You need to install rdkafka gem.
       default_message_key   (string) :default => nil
       exclude_topic_key     (bool) :default => false
       exclude_partition_key (bool) :default => false
-      headers               (hash)   :default => {}
-      headers_from_record   (hash)   :default => {}
+
+      # same with kafka2
+      headers               (hash) :default => {}
+      headers_from_record   (hash) :default => {}
 
       <format>
         @type (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
@@ -396,6 +398,15 @@ You need to install rdkafka gem.
       rdkafka_options {
         "log_level" : 7
       }
+
+      # rdkafka2 specific parameters
+
+      # share kafka producer between flush threads. This is mainly for reducing kafka operations like kerberos
+      share_producer (bool) :default => false
+      # Timeout for polling message wait. If 0, no wait.
+      rdkafka_delivery_handle_poll_timeout (integer) :default => 30
+      # If the record size is larger than this value, such records are ignored. Default is no limit
+      max_send_limit_bytes (integer) :default => nil
     </match>
 
 If you use v0.12, use `rdkafka` instead.

data/fluent-plugin-kafka.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.11.1'
+  gem.version       = '0.11.2'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/out_rdkafka2.rb

@@ -77,6 +77,7 @@ DESC
     config_param :rdkafka_message_max_num, :integer, :default => nil, :desc => 'Used for batch.num.messages'
     config_param :rdkafka_delivery_handle_poll_timeout, :integer, :default => 30, :desc => 'Timeout for polling message wait'
     config_param :rdkafka_options, :hash, :default => {}, :desc => 'Set any rdkafka configuration. See https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md'
+    config_param :share_producer, :bool, :default => false, :desc => 'share kafka producer between flush threads'
 
     config_param :max_enqueue_retries, :integer, :default => 3
     config_param :enqueue_retry_backoff, :integer, :default => 3
@@ -97,8 +98,10 @@ DESC
 
     def initialize
       super
-      @producers = {}
-      @producers_mutex = Mutex.new
+
+      @producers = nil
+      @producers_mutex = nil
+      @shared_producer = nil
     end
 
     def configure(conf)
@@ -187,6 +190,13 @@ DESC
     end
 
     def start
+      if @share_producer
+        @shared_producer = @rdkafka.producer
+      else
+        @producers = {}
+        @producers_mutex = Mutex.new
+      end
+
       super
     end
 
@@ -200,30 +210,43 @@ DESC
     end
 
     def shutdown_producers
-      @producers_mutex.synchronize {
-        shutdown_threads = @producers.map { |key, producer|
-          th = Thread.new {
-            unless producer.close(10)
-              log.warn("Queue is forcefully closed after 10 seconds wait")
-            end
+      if @share_producer
+        close_producer(@shared_producer)
+        @shared_producer = nil
+      else
+        @producers_mutex.synchronize {
+          shutdown_threads = @producers.map { |key, producer|
+            th = Thread.new {
+              close_producer(producer)
+            }
+            th.abort_on_exception = true
+            th
           }
-          th.abort_on_exception = true
-          th
+          shutdown_threads.each { |th| th.join }
+          @producers = {}
         }
-        shutdown_threads.each { |th| th.join }
-        @producers = {}
-      }
+      end
+    end
+
+    def close_producer(producer)
+      unless producer.close(10)
+        log.warn("Queue is forcefully closed after 10 seconds wait")
+      end
     end
 
     def get_producer
-      @producers_mutex.synchronize {
-        producer = @producers[Thread.current.object_id]
-        unless producer
-          producer = @rdkafka.producer
-          @producers[Thread.current.object_id] = producer
-        end
-        producer
-      }
+      if @share_producer
+        @shared_producer
+      else
+        @producers_mutex.synchronize {
+          producer = @producers[Thread.current.object_id]
+          unless producer
+            producer = @rdkafka.producer
+            @producers[Thread.current.object_id] = producer
+          end
+          producer
+        }
+      end
     end
 
     def setup_formatter(conf)
@@ -272,10 +295,13 @@ DESC
             next
           end
 
-          handlers << enqueue_with_retry(producer, topic, record_buf, message_key, partition, headers)
+          handler = enqueue_with_retry(producer, topic, record_buf, message_key, partition, headers)
+          if @rdkafka_delivery_handle_poll_timeout != 0
+            handlers << handler
+          end
         }
         handlers.each { |handler|
-          handler.wait(@rdkafka_delivery_handle_poll_timeout) if @rdkafka_delivery_handle_poll_timeout != 0
+          handler.wait(@rdkafka_delivery_handle_poll_timeout)
         }
       end
     rescue Exception => e

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.11.1
+  version: 0.11.2
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-08-22 00:00:00.000000000 Z
+date: 2019-09-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd