fluent-plugin-kafka 0.17.1 → 0.17.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 230266adf4ba3d77b8b2fd743863377c4dd2532297f45e4489df62bfcbee1db5
-  data.tar.gz: 4b82d2f33bbbb3294f547a7af154783c432c6316f1155a0e3a822361232743e4
+  metadata.gz: 16968c0e56c22f64225e8e41e905294defc7240df6054813151d3904a79a4107
+  data.tar.gz: 71b13953b11048f201c8a8a275350e983a51377b45e84440196b701063e317c4
 SHA512:
-  metadata.gz: 9321cd8bd10dcd603b653c101e6695b7071d7b9dc61fb49bc79248b22cbe9f1db46416c40132243c9862e9c4a3888c818f7a8906906ccf0dda545032f2ac53fd
-  data.tar.gz: 832517ac39c4f775d95775454b07bc0f3ecc2f27b1c0bd4c3f8025e795dfbe869d3e349084f868bdb54cfa72188d4cbd96173192f728fae4281f7f84e5d973aa
+  metadata.gz: aedae36f4b7a29408bc96838da3e158a98ddac136987408dcd4bc347068e3d27839e844d1d3d4a239e4f175e023f19d9ecd82e14d4ab089fd43625f60dcca17c
+  data.tar.gz: 4a72ea62a754b689944d3f2de81cc495c9ef92df97c127b80b46ace6824aca00df5b4cc499eb3993d0edbd3f156f140c6c8ba718fb82f65b5328170adf26f622

data/ChangeLog

@@ -1,3 +1,8 @@
+Release 0.17.2 - 2021/10/14
+	* out_rdkafka2: Add `max_enqueue_bytes_per_second` parameter
+	* out_rdkafka2: Support `use_event_time` parameter
+	* out_rdkafka2: Fix a potential bug that the plugin might exit without receiving responses from Kafka.
+
 Release 0.17.1 - 2021/09/24
 	* out_rdkafka/out_rdkafka2: Support rdkafka 0.9.0 or later
 	* out_rdkafka/out_rdkafka2: Add `exclude_fields` parameter

data/README.md

@@ -200,6 +200,7 @@ If `ruby-kafka` doesn't fit your kafka environment, check `rdkafka2` plugin inst
       get_kafka_client_log  (bool)   :default => false
       headers               (hash)   :default => {}
       headers_from_record   (hash)   :default => {}
+      use_event_time        (bool)   :default => false
       use_default_for_unknown_topic (bool) :default => false
       discard_kafka_delivery_failed (bool) :default => false (No discard)
       partitioner_hash_function (enum) (crc32|murmur2) :default => 'crc32'
@@ -363,6 +364,7 @@ Support of fluentd v0.12 has ended. `kafka_buffered` will be an alias of `kafka2
       exclude_topic_key     (bool) :default => false
       exclude_partition_key (bool) :default => false
       get_kafka_client_log  (bool) :default => false
+      use_event_time        (bool) :default => false
       partitioner_hash_function (enum) (crc32|murmur2) :default => 'crc32'
 
       # See fluentd document for buffer related parameters: https://docs.fluentd.org/v/0.12/buffer
@@ -452,6 +454,7 @@ You need to install rdkafka gem.
       exclude_topic_key     (bool) :default => false
       exclude_partition_key (bool) :default => false
       discard_kafka_delivery_failed (bool) :default => false (No discard)
+      use_event_time        (bool) :default => false
 
       # same with kafka2
       headers               (hash) :default => {}
@@ -486,6 +489,10 @@ You need to install rdkafka gem.
       rdkafka_delivery_handle_poll_timeout (integer) :default => 30
       # If the record size is larger than this value, such records are ignored. Default is no limit
       max_send_limit_bytes (integer) :default => nil
+      # The maximum number of enqueueing bytes per second. It can reduce the
+      # load of both Fluentd and Kafka when excessive messages are attempted
+      # to send. Default is no limit.
+      max_enqueue_bytes_per_second (integer) :default => nil
     </match>
 
 If you use v0.12, use `rdkafka` instead.

data/fluent-plugin-kafka.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.17.1'
+  gem.version       = '0.17.2'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/out_rdkafka.rb

@@ -65,6 +65,7 @@ DESC
 The codec the producer uses to compress messages.
 Supported codecs: (gzip|snappy)
 DESC
+  config_param :use_event_time, :bool, :default => false, :desc => 'Use fluentd event time for rdkafka timestamp'
   config_param :max_send_limit_bytes, :size, :default => nil
   config_param :rdkafka_buffering_max_ms, :integer, :default => nil
   config_param :rdkafka_buffering_max_messages, :integer, :default => nil
@@ -286,7 +287,7 @@ DESC
         end
 
         producer = get_producer
-        handler = enqueue_with_retry(producer, topic, record_buf, message_key, partition)
+        handler = enqueue_with_retry(producer, topic, record_buf, message_key, partition, time)
         handler
       }.each { |handler|
         handler.wait(max_wait_timeout: @rdkafka_delivery_handle_poll_timeout) if @rdkafka_delivery_handle_poll_timeout != 0
@@ -298,11 +299,11 @@ DESC
     raise e
   end
 
-  def enqueue_with_retry(producer, topic, record_buf, message_key, partition)
+  def enqueue_with_retry(producer, topic, record_buf, message_key, partition, time)
     attempt = 0
     loop do
       begin
-        handler = producer.produce(topic: topic, payload: record_buf, key: message_key, partition: partition)
+        handler = producer.produce(topic: topic, payload: record_buf, key: message_key, partition: partition, timestamp: @use_event_time ? Time.at(time) : nil)
         return handler
       rescue Exception => e
         if e.respond_to?(:code) && e.code == :queue_full

data/lib/fluent/plugin/out_rdkafka2.rb

@@ -74,6 +74,7 @@ DESC
 The codec the producer uses to compress messages. Used for compression.codec
 Supported codecs: (gzip|snappy)
 DESC
+    config_param :use_event_time, :bool, :default => false, :desc => 'Use fluentd event time for rdkafka timestamp'
     config_param :max_send_limit_bytes, :size, :default => nil
     config_param :discard_kafka_delivery_failed, :bool, :default => false
     config_param :rdkafka_buffering_max_ms, :integer, :default => nil, :desc => 'Used for queue.buffering.max.ms'
@@ -86,6 +87,7 @@ DESC
 
     config_param :max_enqueue_retries, :integer, :default => 3
     config_param :enqueue_retry_backoff, :integer, :default => 3
+    config_param :max_enqueue_bytes_per_second, :size, :default => nil, :desc => 'The maximum number of enqueueing bytes per second'
 
     config_param :service_name, :string, :default => nil, :desc => 'Used for sasl.kerberos.service.name'
     config_param :ssl_client_cert_key_password, :string, :default => nil, :desc => 'Used for ssl.key.password'
@@ -101,12 +103,68 @@ DESC
     include Fluent::KafkaPluginUtil::SSLSettings
     include Fluent::KafkaPluginUtil::SaslSettings
 
+    class EnqueueRate
+      class LimitExceeded < StandardError
+        attr_reader :next_retry_clock
+        def initialize(next_retry_clock)
+          @next_retry_clock = next_retry_clock
+        end
+      end
+
+      def initialize(limit_bytes_per_second)
+        @mutex = Mutex.new
+        @start_clock = Fluent::Clock.now
+        @bytes_per_second = 0
+        @limit_bytes_per_second = limit_bytes_per_second
+        @commits = {}
+      end
+
+      def raise_if_limit_exceeded(bytes_to_enqueue)
+        return if @limit_bytes_per_second.nil?
+
+        @mutex.synchronize do
+          @commits[Thread.current] = {
+            clock: Fluent::Clock.now,
+            bytesize: bytes_to_enqueue,
+          }
+
+          @bytes_per_second += @commits[Thread.current][:bytesize]
+          duration = @commits[Thread.current][:clock] - @start_clock
+
+          if duration < 1.0
+            if @bytes_per_second > @limit_bytes_per_second
+              raise LimitExceeded.new(@start_clock + 1.0)
+            end
+          else
+            @start_clock = @commits[Thread.current][:clock]
+            @bytes_per_second = @commits[Thread.current][:bytesize]
+          end
+        end
+      end
+
+      def revert
+        return if @limit_bytes_per_second.nil?
+
+        @mutex.synchronize do
+          return unless @commits[Thread.current]
+          return unless @commits[Thread.current][:clock]
+          if @commits[Thread.current][:clock] >= @start_clock
+            @bytes_per_second -= @commits[Thread.current][:bytesize]
+          end
+          @commits[Thread.current] = nil
+        end
+      end
+    end
+
     def initialize
       super
 
       @producers = nil
       @producers_mutex = nil
       @shared_producer = nil
+      @enqueue_rate = nil
+      @writing_threads_mutex = Mutex.new
+      @writing_threads = Set.new
     end
 
     def configure(conf)
@@ -170,6 +228,8 @@ DESC
       @exclude_field_accessors = @exclude_fields.map do |field|
         record_accessor_create(field)
       end
+
+      @enqueue_rate = EnqueueRate.new(@max_enqueue_bytes_per_second) unless @max_enqueue_bytes_per_second.nil?
     end
 
     def build_config
@@ -233,8 +293,19 @@ DESC
       true
     end
 
+    def wait_writing_threads
+      done = false
+      until done do
+        @writing_threads_mutex.synchronize do
+          done = true if @writing_threads.empty?
+        end
+        sleep(1) unless done
+      end
+    end
+
     def shutdown
       super
+      wait_writing_threads
       shutdown_producers
     end
 
@@ -291,6 +362,7 @@ DESC
     end
 
     def write(chunk)
+      @writing_threads_mutex.synchronize { @writing_threads.add(Thread.current) }
       tag = chunk.metadata.tag
       topic = if @topic
                 extract_placeholders(@topic, chunk)
@@ -334,7 +406,7 @@ DESC
             next
           end
 
-          handler = enqueue_with_retry(producer, topic, record_buf, message_key, partition, headers)
+          handler = enqueue_with_retry(producer, topic, record_buf, message_key, partition, headers, time)
           if @rdkafka_delivery_handle_poll_timeout != 0
             handlers << handler
           end
@@ -351,14 +423,22 @@ DESC
         # Raise exception to retry sendind messages
         raise e
       end
+    ensure
+      @writing_threads_mutex.synchronize { @writing_threads.delete(Thread.current) }
     end
 
-    def enqueue_with_retry(producer, topic, record_buf, message_key, partition, headers)
+    def enqueue_with_retry(producer, topic, record_buf, message_key, partition, headers, time)
       attempt = 0
       loop do
         begin
-          return producer.produce(topic: topic, payload: record_buf, key: message_key, partition: partition, headers: headers)
+          @enqueue_rate.raise_if_limit_exceeded(record_buf.bytesize) if @enqueue_rate
+          return producer.produce(topic: topic, payload: record_buf, key: message_key, partition: partition, headers: headers, timestamp: @use_event_time ? Time.at(time) : nil)
+        rescue EnqueueRate::LimitExceeded => e
+          @enqueue_rate.revert if @enqueue_rate
+          duration = e.next_retry_clock - Fluent::Clock.now
+          sleep(duration) if duration > 0.0
         rescue Exception => e
+          @enqueue_rate.revert if @enqueue_rate
           if e.respond_to?(:code) && e.code == :queue_full
             if attempt <= @max_enqueue_retries
               log.warn "Failed to enqueue message; attempting retry #{attempt} of #{@max_enqueue_retries} after #{@enqueue_retry_backoff}s"

data/test/plugin/test_out_rdkafka2.rb

@@ -102,6 +102,31 @@ class Rdkafka2OutputTest < Test::Unit::TestCase
       assert_equal([expected_message], actual_messages)
     end
 
+    def test_write_with_use_event_time
+      input_config = %[
+        @type kafka
+        brokers localhost:9092
+        format json
+        @label @kafka
+        topics #{TOPIC_NAME}
+        time_source kafka
+      ]
+      target_driver = create_target_driver(input_config)
+      expected_message = {"a" => 2}
+      now = event_time
+      target_driver.run(expect_records: 1, timeout: 5) do
+        sleep 2
+        d = create_driver(config(default_topic: TOPIC_NAME) + config_element('ROOT', '', {"use_event_time" => true}))
+        d.run do
+          d.feed("test", now, expected_message)
+        end
+      end
+      actual_time = target_driver.events.collect { |event| event[1] }.last
+      assert_in_delta(actual_time, now, 0.001) # expects millseconds precision
+      actual_messages = target_driver.events.collect { |event| event[2] }
+      assert_equal([expected_message], actual_messages)
+    end
+
     def test_exclude_fields
       conf = config(default_topic: TOPIC_NAME) +
              config_element('ROOT', '', {"exclude_fields" => "$.foo"}, [])
@@ -116,5 +141,27 @@ class Rdkafka2OutputTest < Test::Unit::TestCase
       actual_messages = target_driver.events.collect { |event| event[2] }
       assert_equal([{'a' => 'b', 'message' => 'test'}], actual_messages)
     end
+
+    def test_max_enqueue_bytes_per_second
+      conf = config(default_topic: TOPIC_NAME) +
+             config_element('ROOT', '', {"max_enqueue_bytes_per_second" => 32 * 3}, [])
+      target_driver = create_target_driver
+      expected_messages = []
+      target_driver.run(expect_records: 9, timeout: 10) do
+        sleep 2
+        d = create_driver(conf)
+        start_time = Fluent::Clock.now
+        d.run do
+          9.times do |i|
+            message = {"message" => "32bytes message: #{i}"}
+            d.feed("test", event_time, message)
+            expected_messages << message
+          end
+        end
+        assert_in_delta(2.0, Fluent::Clock.now - start_time, 0.5)
+      end
+      actual_messages = target_driver.events.collect { |event| event[2] }
+      assert_equal(expected_messages, actual_messages)
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.17.1
+  version: 0.17.2
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-09-24 00:00:00.000000000 Z
+date: 2021-10-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd