fluent-plugin-kafka 0.17.0 → 0.17.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 15805411e4029813123b9b636b6faadb937cf38c9841adcd9d998a0f54d8b687
-  data.tar.gz: 75fef11595c86beb4a54d2d2ff659f77075e328426463f7ef830982c9724ff16
+  metadata.gz: 5cc122034295e37318cd7510ef3347eeda14cc43b8c0132053cb944d68141feb
+  data.tar.gz: e725b07eaa95f639b2122f1a4c8342101314f2f721e3625c73889dc8caf9aead
 SHA512:
-  metadata.gz: db08ffbde4fe36ce38abe6eebf83e6e0dc157bc5d8dd95141ef911d0c5b59074a208c3a8f836e258ffddc34f73fda303006eaef1cd8fc12ae4dd4c79d101d0c7
-  data.tar.gz: 9b3b098b1bc58654924d50ac8d685af007763942c1cc68187bbd6d39510a27ca31ae795df0de312d97ad67463ae0c428882118640fce052119288e27e3fb0df5
+  metadata.gz: 2e432e7f2670132022b18fa9460b8eda69a18a4dd3a35aa775619c6a45ff8cb6ea5bad869ebc5cefe804b9bb4261ab12150cb77ad10af62dc2e54fd6de435aec
+  data.tar.gz: edbebd57c325292d197d342ff8f5151aa1fcfbd47128fc09d1b71e2bf4d7ccf196d54b48df106f1b9f655fe334ab1f9fe907fce8b4f1b1d20edd9c8254c6c8cd

data/.github/workflows/linux.yml

@@ -5,6 +5,8 @@ on:
 jobs:
   build:
     runs-on: ${{ matrix.os }}
+    env:
+      USE_RDKAFKA: 1
     strategy:
       fail-fast: false
       matrix:

data/ChangeLog

@@ -1,3 +1,19 @@
+Release 0.17.4 - 2022/01/25
+	* in_kafka_group: Add `refresh_topic_interval` parameter
+
+Release 0.17.3 - 2021/11/26
+	* output: Suppress large warning logs for events skipped by `max_send_limit_bytes`
+
+Release 0.17.2 - 2021/10/14
+	* out_rdkafka2: Add `max_enqueue_bytes_per_second` parameter
+	* out_rdkafka2: Support `use_event_time` parameter
+	* out_rdkafka2: Fix a potential bug that the plugin might exit without receiving responses from Kafka.
+
+Release 0.17.1 - 2021/09/24
+	* out_rdkafka/out_rdkafka2: Support rdkafka 0.9.0 or later
+	* out_rdkafka/out_rdkafka2: Add `exclude_fields` parameter
+	* out_kafka2.rb: Fix one more Ruby 3.0 keyword arguments issue
+
 Release 0.17.0 - 2021/08/30
 	* out_kafka/out_kafka_buffered/out_kafka2: Provide murmur2 partitioner hash function choice
 	* in_kafka/in_kafka_group/out_kafka/out_kafka_buffered/out_kafka2: Use Ruby Kafka's ssl_ca_cert_file_path parameter to feed the CA certs

data/Gemfile

@@ -2,3 +2,5 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kafka.gemspec
 gemspec
+
+gem 'rdkafka', '>= 0.6.0' if ENV["USE_RDKAFKA"]

data/README.md

@@ -40,14 +40,14 @@ If you want to use zookeeper related parameters, you also need to install zookee
 
 Set path to SSL related files. See [Encryption and Authentication using SSL](https://github.com/zendesk/ruby-kafka#encryption-and-authentication-using-ssl) for more detail.
 
-#### SASL authentication 
+#### SASL authentication
 
 ##### with GSSAPI
 
 - principal
 - keytab
 
-Set principal and path to keytab for SASL/GSSAPI authentication. 
+Set principal and path to keytab for SASL/GSSAPI authentication.
 See [Authentication using SASL](https://github.com/zendesk/ruby-kafka#authentication-using-sasl) for more details.
 
 ##### with Plain/SCRAM
@@ -57,7 +57,7 @@ See [Authentication using SASL](https://github.com/zendesk/ruby-kafka#authentica
 - scram_mechanism
 - sasl_over_ssl
 
-Set username, password, scram_mechanism and sasl_over_ssl for SASL/Plain or Scram authentication. 
+Set username, password, scram_mechanism and sasl_over_ssl for SASL/Plain or Scram authentication.
 See [Authentication using SASL](https://github.com/zendesk/ruby-kafka#authentication-using-sasl) for more details.
 
 ### Input plugin (@type 'kafka')
@@ -119,7 +119,7 @@ Consume events by kafka consumer group features..
       topics <listening topics(separate with comma',')>
       format <input text type (text|json|ltsv|msgpack)> :default => json
       message_key <key (Optional, for text format only, default is message)>
-      kafka_mesasge_key <key (Optional, If specified, set kafka's message key to this key)>
+      kafka_message_key <key (Optional, If specified, set kafka's message key to this key)>
       add_headers <If true, add kafka's message headers to record>
       add_prefix <tag prefix (Optional)>
       add_suffix <tag suffix (Optional)>
@@ -135,6 +135,7 @@ Consume events by kafka consumer group features..
       offset_commit_interval  (integer) :default => nil (Use default of ruby-kafka)
       offset_commit_threshold (integer) :default => nil (Use default of ruby-kafka)
       fetcher_max_queue_size  (integer) :default => nil (Use default of ruby-kafka)
+      refresh_topic_interval  (integer) :default => nil (Use default of ruby-kafka)
       start_from_beginning    (bool)    :default => true
     </source>
 
@@ -155,7 +156,7 @@ With the introduction of the rdkafka-ruby based input plugin we hope to support
       topics <listening topics(separate with comma',')>
       format <input text type (text|json|ltsv|msgpack)> :default => json
       message_key <key (Optional, for text format only, default is message)>
-      kafka_mesasge_key <key (Optional, If specified, set kafka's message key to this key)>
+      kafka_message_key <key (Optional, If specified, set kafka's message key to this key)>
       add_headers <If true, add kafka's message headers to record>
       add_prefix <tag prefix (Optional)>
       add_suffix <tag suffix (Optional)>
@@ -200,6 +201,7 @@ If `ruby-kafka` doesn't fit your kafka environment, check `rdkafka2` plugin inst
       get_kafka_client_log  (bool)   :default => false
       headers               (hash)   :default => {}
       headers_from_record   (hash)   :default => {}
+      use_event_time        (bool)   :default => false
       use_default_for_unknown_topic (bool) :default => false
       discard_kafka_delivery_failed (bool) :default => false (No discard)
       partitioner_hash_function (enum) (crc32|murmur2) :default => 'crc32'
@@ -316,6 +318,23 @@ The Kafka message will have a header of source_ip=12.7.0.0.1.
 
 The configuration format is jsonpath. It is descibed in https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-record_accessor
 
+#### Excluding fields
+Fields can be excluded from output data. Only works for kafka2 and rdkafka2 output plugin.
+
+Fields must be specified using an array of dot notation `$.`, for example:
+
+    <match app.**>
+      @type kafka2
+      [...]
+      exclude_fields $.source.ip,$.HTTP_FOO
+    <match>
+
+This config can be used to remove fields used on another configs.
+
+For example, `$.source.ip` can be extracted with config `headers_from_record` and excluded from message payload.
+
+> Using this config to remove unused fields is discouraged. A [filter plugin](https://docs.fluentd.org/v/0.12/filter) can be used for this purpose.
+
 ### Buffered output plugin
 
 This plugin uses ruby-kafka producer for writing data. This plugin is for v0.12. If you use v1, see `kafka2`.
@@ -346,6 +365,7 @@ Support of fluentd v0.12 has ended. `kafka_buffered` will be an alias of `kafka2
       exclude_topic_key     (bool) :default => false
       exclude_partition_key (bool) :default => false
       get_kafka_client_log  (bool) :default => false
+      use_event_time        (bool) :default => false
       partitioner_hash_function (enum) (crc32|murmur2) :default => 'crc32'
 
       # See fluentd document for buffer related parameters: https://docs.fluentd.org/v/0.12/buffer
@@ -435,6 +455,7 @@ You need to install rdkafka gem.
       exclude_topic_key     (bool) :default => false
       exclude_partition_key (bool) :default => false
       discard_kafka_delivery_failed (bool) :default => false (No discard)
+      use_event_time        (bool) :default => false
 
       # same with kafka2
       headers               (hash) :default => {}
@@ -469,6 +490,10 @@ You need to install rdkafka gem.
       rdkafka_delivery_handle_poll_timeout (integer) :default => 30
       # If the record size is larger than this value, such records are ignored. Default is no limit
       max_send_limit_bytes (integer) :default => nil
+      # The maximum number of enqueueing bytes per second. It can reduce the
+      # load of both Fluentd and Kafka when excessive messages are attempted
+      # to send. Default is no limit.
+      max_enqueue_bytes_per_second (integer) :default => nil
     </match>
 
 If you use v0.12, use `rdkafka` instead.

data/fluent-plugin-kafka.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.17.0'
+  gem.version       = '0.17.4'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/in_kafka.rb

@@ -71,6 +71,7 @@ class Fluent::KafkaInput < Fluent::Input
     require 'kafka'
 
     @time_parser = nil
+    @zookeeper = nil
   end
 
   def configure(conf)

data/lib/fluent/plugin/in_kafka_group.rb

@@ -67,6 +67,8 @@ class Fluent::KafkaGroupInput < Fluent::Input
                :desc => "The number of messages that can be processed before their offsets are committed"
   config_param :fetcher_max_queue_size, :integer, :default => nil,
                :desc => "The number of fetched messages per partition that are queued in fetcher queue"
+  config_param :refresh_topic_interval, :integer, :default => nil,
+               :desc => "The interval of refreshing the topic list in seconds. Zero or unset disables this"
   config_param :start_from_beginning, :bool, :default => true,
                :desc => "Whether to start from the beginning of the topic or just subscribe to new messages being produced"
 
@@ -128,6 +130,7 @@ class Fluent::KafkaGroupInput < Fluent::Input
     @consumer_opts[:offset_commit_interval] = @offset_commit_interval if @offset_commit_interval
     @consumer_opts[:offset_commit_threshold] = @offset_commit_threshold if @offset_commit_threshold
     @consumer_opts[:fetcher_max_queue_size] = @fetcher_max_queue_size if @fetcher_max_queue_size
+    @consumer_opts[:refresh_topic_interval] = @refresh_topic_interval if @refresh_topic_interval
 
     @fetch_opts = {}
     @fetch_opts[:max_wait_time] = @max_wait_time if @max_wait_time

data/lib/fluent/plugin/out_kafka.rb

@@ -88,6 +88,7 @@ DESC
     require 'kafka'
 
     @kafka = nil
+    @field_separator = nil
   end
 
   def refresh_client
@@ -239,7 +240,8 @@ DESC
         record_buf = @formatter_proc.call(tag, time, record)
         record_buf_bytes = record_buf.bytesize
         if @max_send_limit_bytes && record_buf_bytes > @max_send_limit_bytes
-          log.warn "record size exceeds max_send_limit_bytes. Skip event:", :time => time, :record => record
+          log.warn "record size exceeds max_send_limit_bytes. Skip event:", :time => time, :record_size => record_buf_bytes
+          log.debug "Skipped event:", :record => record
           next
         end
         log.trace { "message will send to #{topic} with partition_key: #{partition_key}, partition: #{partition}, message_key: #{message_key} and value: #{record_buf}." }

data/lib/fluent/plugin/out_kafka2.rb

@@ -42,6 +42,8 @@ DESC
                  :desc => 'Set true to remove message key from data'
     config_param :exclude_topic_key, :bool, :default => false,
                  :desc => 'Set true to remove topic name key from data'
+    config_param :exclude_fields, :array, :default => [], value_type: :string,
+                 :desc => 'Fields to remove from data where the value is a jsonpath to a record value'
     config_param :use_event_time, :bool, :default => false, :desc => 'Use fluentd event time for kafka create_time'
     config_param :headers, :hash, default: {}, symbolize_keys: true, value_type: :string,
                  :desc => 'Kafka message headers'
@@ -177,6 +179,10 @@ DESC
       @headers_from_record.each do |key, value|
         @headers_from_record_accessors[key] = record_accessor_create(value)
       end
+
+      @exclude_field_accessors = @exclude_fields.map do |field|
+        record_accessor_create(field)
+      end
     end
 
     def multi_workers_ready?
@@ -235,7 +241,7 @@ DESC
       mutate_headers = !@headers_from_record_accessors.empty?
 
       begin
-        producer = @kafka.topic_producer(topic, @producer_opts)
+        producer = @kafka.topic_producer(topic, **@producer_opts)
         chunk.msgpack_each { |time, record|
           begin
             record = inject_values_to_record(tag, time, record)
@@ -253,10 +259,17 @@ DESC
               headers = base_headers
             end
 
+            unless @exclude_fields.empty?
+              @exclude_field_accessors.each do |exclude_field_accessor|
+                exclude_field_accessor.delete(record)
+              end
+            end
+
             record_buf = @formatter_proc.call(tag, time, record)
             record_buf_bytes = record_buf.bytesize
             if @max_send_limit_bytes && record_buf_bytes > @max_send_limit_bytes
-              log.warn "record size exceeds max_send_limit_bytes. Skip event:", :time => time, :record => record
+              log.warn "record size exceeds max_send_limit_bytes. Skip event:", :time => time, :record_size => record_buf_bytes
+              log.debug "Skipped event:", :record => record
               next
             end
           rescue StandardError => e

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -107,6 +107,7 @@ DESC
     @kafka = nil
     @producers = {}
     @producers_mutex = Mutex.new
+    @field_separator = nil
   end
 
   def multi_workers_ready?
@@ -331,7 +332,8 @@ DESC
           record_buf = @formatter_proc.call(tag, time, record)
           record_buf_bytes = record_buf.bytesize
           if @max_send_limit_bytes && record_buf_bytes > @max_send_limit_bytes
-            log.warn "record size exceeds max_send_limit_bytes. Skip event:", :time => time, :record => record
+            log.warn "record size exceeds max_send_limit_bytes. Skip event:", :time => time, :record_size => record_buf_bytes
+            log.debug "Skipped event:", :record => record
             next
           end
         rescue StandardError => e

data/lib/fluent/plugin/out_rdkafka.rb

@@ -65,6 +65,7 @@ DESC
 The codec the producer uses to compress messages.
 Supported codecs: (gzip|snappy)
 DESC
+  config_param :use_event_time, :bool, :default => false, :desc => 'Use fluentd event time for rdkafka timestamp'
   config_param :max_send_limit_bytes, :size, :default => nil
   config_param :rdkafka_buffering_max_ms, :integer, :default => nil
   config_param :rdkafka_buffering_max_messages, :integer, :default => nil
@@ -91,23 +92,29 @@ DESC
   def configure(conf)
     super
     log.instance_eval {
-      def add(level, &block)
-        return unless block
+      def add(level, message = nil)
+        if message.nil?
+          if block_given?
+            message = yield
+          else
+            return
+          end
+        end
 
         # Follow rdkakfa's log level. See also rdkafka-ruby's bindings.rb: https://github.com/appsignal/rdkafka-ruby/blob/e5c7261e3f2637554a5c12b924be297d7dca1328/lib/rdkafka/bindings.rb#L117
         case level
         when Logger::FATAL
-          self.fatal(block.call)
+          self.fatal(message)
         when Logger::ERROR
-          self.error(block.call)
+          self.error(message)
         when Logger::WARN
-          self.warn(block.call)
+          self.warn(message)
         when Logger::INFO
-          self.info(block.call)
+          self.info(message)
         when Logger::DEBUG
-          self.debug(block.call)
+          self.debug(message)
         else
-          self.trace(block.call)
+          self.trace(message)
         end
       end
     }
@@ -271,7 +278,8 @@ DESC
           record_buf = @formatter_proc.call(tag, time, record)
           record_buf_bytes = record_buf.bytesize
           if @max_send_limit_bytes && record_buf_bytes > @max_send_limit_bytes
-            log.warn "record size exceeds max_send_limit_bytes. Skip event:", :time => time, :record => record
+            log.warn "record size exceeds max_send_limit_bytes. Skip event:", :time => time, :record_size => record_buf_bytes
+            log.debug "Skipped event:", :record => record
             next
           end
         rescue StandardError => e
@@ -280,7 +288,7 @@ DESC
         end
 
         producer = get_producer
-        handler = enqueue_with_retry(producer, topic, record_buf, message_key, partition)
+        handler = enqueue_with_retry(producer, topic, record_buf, message_key, partition, time)
         handler
       }.each { |handler|
         handler.wait(max_wait_timeout: @rdkafka_delivery_handle_poll_timeout) if @rdkafka_delivery_handle_poll_timeout != 0
@@ -292,11 +300,11 @@ DESC
     raise e
   end
 
-  def enqueue_with_retry(producer, topic, record_buf, message_key, partition)
+  def enqueue_with_retry(producer, topic, record_buf, message_key, partition, time)
     attempt = 0
     loop do
       begin
-        handler = producer.produce(topic: topic, payload: record_buf, key: message_key, partition: partition)
+        handler = producer.produce(topic: topic, payload: record_buf, key: message_key, partition: partition, timestamp: @use_event_time ? Time.at(time) : nil)
         return handler
       rescue Exception => e
         if e.respond_to?(:code) && e.code == :queue_full

data/lib/fluent/plugin/out_rdkafka2.rb

@@ -56,6 +56,8 @@ DESC
                  :desc => <<-DESC
 Set true to remove topic key from data
 DESC
+    config_param :exclude_fields, :array, :default => [], value_type: :string,
+                 :desc => 'Fields to remove from data where the value is a jsonpath to a record value'
     config_param :headers, :hash, default: {}, symbolize_keys: true, value_type: :string,
                  :desc => 'Kafka message headers'
     config_param :headers_from_record, :hash, default: {}, symbolize_keys: true, value_type: :string,
@@ -72,6 +74,7 @@ DESC
 The codec the producer uses to compress messages. Used for compression.codec
 Supported codecs: (gzip|snappy)
 DESC
+    config_param :use_event_time, :bool, :default => false, :desc => 'Use fluentd event time for rdkafka timestamp'
     config_param :max_send_limit_bytes, :size, :default => nil
     config_param :discard_kafka_delivery_failed, :bool, :default => false
     config_param :rdkafka_buffering_max_ms, :integer, :default => nil, :desc => 'Used for queue.buffering.max.ms'
@@ -84,6 +87,7 @@ DESC
 
     config_param :max_enqueue_retries, :integer, :default => 3
     config_param :enqueue_retry_backoff, :integer, :default => 3
+    config_param :max_enqueue_bytes_per_second, :size, :default => nil, :desc => 'The maximum number of enqueueing bytes per second'
 
     config_param :service_name, :string, :default => nil, :desc => 'Used for sasl.kerberos.service.name'
     config_param :ssl_client_cert_key_password, :string, :default => nil, :desc => 'Used for ssl.key.password'
@@ -99,34 +103,96 @@ DESC
     include Fluent::KafkaPluginUtil::SSLSettings
     include Fluent::KafkaPluginUtil::SaslSettings
 
+    class EnqueueRate
+      class LimitExceeded < StandardError
+        attr_reader :next_retry_clock
+        def initialize(next_retry_clock)
+          @next_retry_clock = next_retry_clock
+        end
+      end
+
+      def initialize(limit_bytes_per_second)
+        @mutex = Mutex.new
+        @start_clock = Fluent::Clock.now
+        @bytes_per_second = 0
+        @limit_bytes_per_second = limit_bytes_per_second
+        @commits = {}
+      end
+
+      def raise_if_limit_exceeded(bytes_to_enqueue)
+        return if @limit_bytes_per_second.nil?
+
+        @mutex.synchronize do
+          @commits[Thread.current] = {
+            clock: Fluent::Clock.now,
+            bytesize: bytes_to_enqueue,
+          }
+
+          @bytes_per_second += @commits[Thread.current][:bytesize]
+          duration = @commits[Thread.current][:clock] - @start_clock
+
+          if duration < 1.0
+            if @bytes_per_second > @limit_bytes_per_second
+              raise LimitExceeded.new(@start_clock + 1.0)
+            end
+          else
+            @start_clock = @commits[Thread.current][:clock]
+            @bytes_per_second = @commits[Thread.current][:bytesize]
+          end
+        end
+      end
+
+      def revert
+        return if @limit_bytes_per_second.nil?
+
+        @mutex.synchronize do
+          return unless @commits[Thread.current]
+          return unless @commits[Thread.current][:clock]
+          if @commits[Thread.current][:clock] >= @start_clock
+            @bytes_per_second -= @commits[Thread.current][:bytesize]
+          end
+          @commits[Thread.current] = nil
+        end
+      end
+    end
+
     def initialize
       super
 
       @producers = nil
       @producers_mutex = nil
       @shared_producer = nil
+      @enqueue_rate = nil
+      @writing_threads_mutex = Mutex.new
+      @writing_threads = Set.new
     end
 
     def configure(conf)
       super
       log.instance_eval {
-        def add(level, &block)
-          return unless block
+        def add(level, message = nil)
+          if message.nil?
+            if block_given?
+              message = yield
+            else
+              return
+            end
+          end
 
           # Follow rdkakfa's log level. See also rdkafka-ruby's bindings.rb: https://github.com/appsignal/rdkafka-ruby/blob/e5c7261e3f2637554a5c12b924be297d7dca1328/lib/rdkafka/bindings.rb#L117
           case level
           when Logger::FATAL
-            self.fatal(block.call)
+            self.fatal(message)
           when Logger::ERROR
-            self.error(block.call)
+            self.error(message)
           when Logger::WARN
-            self.warn(block.call)
+            self.warn(message)
           when Logger::INFO
-            self.info(block.call)
+            self.info(message)
           when Logger::DEBUG
-            self.debug(block.call)
+            self.debug(message)
           else
-            self.trace(block.call)
+            self.trace(message)
           end
         end
       }
@@ -158,6 +224,12 @@ DESC
       @headers_from_record.each do |key, value|
         @headers_from_record_accessors[key] = record_accessor_create(value)
       end
+
+      @exclude_field_accessors = @exclude_fields.map do |field|
+        record_accessor_create(field)
+      end
+
+      @enqueue_rate = EnqueueRate.new(@max_enqueue_bytes_per_second) unless @max_enqueue_bytes_per_second.nil?
     end
 
     def build_config
@@ -221,8 +293,19 @@ DESC
       true
     end
 
+    def wait_writing_threads
+      done = false
+      until done do
+        @writing_threads_mutex.synchronize do
+          done = true if @writing_threads.empty?
+        end
+        sleep(1) unless done
+      end
+    end
+
     def shutdown
       super
+      wait_writing_threads
       shutdown_producers
     end
 
@@ -279,6 +362,7 @@ DESC
     end
 
     def write(chunk)
+      @writing_threads_mutex.synchronize { @writing_threads.add(Thread.current) }
       tag = chunk.metadata.tag
       topic = if @topic
                 extract_placeholders(@topic, chunk)
@@ -305,10 +389,17 @@ DESC
               headers[key] = header_accessor.call(record)
             end
 
+            unless @exclude_fields.empty?
+              @exclude_field_accessors.each do |exclude_field_acessor|
+                exclude_field_acessor.delete(record)
+              end
+            end
+
             record_buf = @formatter_proc.call(tag, time, record)
             record_buf_bytes = record_buf.bytesize
             if @max_send_limit_bytes && record_buf_bytes > @max_send_limit_bytes
-              log.warn "record size exceeds max_send_limit_bytes. Skip event:", :time => time, :record => record
+              log.warn "record size exceeds max_send_limit_bytes. Skip event:", :time => time, :record_size => record_buf_bytes
+              log.debug "Skipped event:", :record => record
               next
             end
           rescue StandardError => e
@@ -316,7 +407,7 @@ DESC
             next
           end
 
-          handler = enqueue_with_retry(producer, topic, record_buf, message_key, partition, headers)
+          handler = enqueue_with_retry(producer, topic, record_buf, message_key, partition, headers, time)
           if @rdkafka_delivery_handle_poll_timeout != 0
             handlers << handler
           end
@@ -333,14 +424,22 @@ DESC
         # Raise exception to retry sendind messages
         raise e
       end
+    ensure
+      @writing_threads_mutex.synchronize { @writing_threads.delete(Thread.current) }
     end
 
-    def enqueue_with_retry(producer, topic, record_buf, message_key, partition, headers)
+    def enqueue_with_retry(producer, topic, record_buf, message_key, partition, headers, time)
       attempt = 0
       loop do
         begin
-          return producer.produce(topic: topic, payload: record_buf, key: message_key, partition: partition, headers: headers)
+          @enqueue_rate.raise_if_limit_exceeded(record_buf.bytesize) if @enqueue_rate
+          return producer.produce(topic: topic, payload: record_buf, key: message_key, partition: partition, headers: headers, timestamp: @use_event_time ? Time.at(time) : nil)
+        rescue EnqueueRate::LimitExceeded => e
+          @enqueue_rate.revert if @enqueue_rate
+          duration = e.next_retry_clock - Fluent::Clock.now
+          sleep(duration) if duration > 0.0
         rescue Exception => e
+          @enqueue_rate.revert if @enqueue_rate
           if e.respond_to?(:code) && e.code == :queue_full
             if attempt <= @max_enqueue_retries
               log.warn "Failed to enqueue message; attempting retry #{attempt} of #{@max_enqueue_retries} after #{@enqueue_retry_backoff}s"

data/test/plugin/test_in_kafka_group.rb

@@ -14,6 +14,7 @@ class KafkaGroupInputTest < Test::Unit::TestCase
     brokers localhost:9092
     consumer_group fluentd
     format text
+    refresh_topic_interval 0
     @label @kafka
     topics #{TOPIC_NAME}
   ]
@@ -52,6 +53,7 @@ class KafkaGroupInputTest < Test::Unit::TestCase
         brokers localhost:9092
         format text
         @label @kafka
+        refresh_topic_interval 0
         topics #{TOPIC_NAME}
       ]
       d = create_driver

data/test/plugin/test_out_kafka2.rb

@@ -1,6 +1,8 @@
 require 'helper'
 require 'fluent/test/helpers'
-require 'fluent/output'
+require 'fluent/test/driver/input'
+require 'fluent/test/driver/output'
+require 'securerandom'
 
 class Kafka2OutputTest < Test::Unit::TestCase
   include Fluent::Test::Helpers
@@ -15,8 +17,8 @@ class Kafka2OutputTest < Test::Unit::TestCase
                    ])
   end
 
-  def config
-    base_config + config_element('ROOT', '', {"default_topic" => "kitagawakeiko",
+  def config(default_topic: "kitagawakeiko")
+    base_config + config_element('ROOT', '', {"default_topic" => default_topic,
                                               "brokers" => "localhost:9092"}, [
                                  ])
   end
@@ -57,4 +59,58 @@ class Kafka2OutputTest < Test::Unit::TestCase
     d = create_driver
     assert_equal true, d.instance.multi_workers_ready?
   end
+
+  class WriteTest < self
+    TOPIC_NAME = "kafka-output-#{SecureRandom.uuid}"
+
+    INPUT_CONFIG = %[
+      @type kafka
+      brokers localhost:9092
+      format json
+      @label @kafka
+      topics #{TOPIC_NAME}
+    ]
+
+    def create_target_driver(conf = INPUT_CONFIG)
+      Fluent::Test::Driver::Input.new(Fluent::KafkaInput).configure(conf)
+    end
+
+    def setup
+      @kafka = Kafka.new(["localhost:9092"], client_id: 'kafka')
+    end
+
+    def teardown
+      @kafka.delete_topic(TOPIC_NAME)
+      @kafka.close
+    end
+
+    def test_write
+      target_driver = create_target_driver
+      expected_message = {"a" => 2}
+      target_driver.run(expect_records: 1, timeout: 5) do
+        sleep 2
+        d = create_driver(config(default_topic: TOPIC_NAME))
+        d.run do
+          d.feed("test", event_time, expected_message)
+        end
+      end
+      actual_messages = target_driver.events.collect { |event| event[2] }
+      assert_equal([expected_message], actual_messages)
+    end
+
+    def test_exclude_fields
+      conf = config(default_topic: TOPIC_NAME) +
+             config_element('ROOT', '', {"exclude_fields" => "$.foo"}, [])
+      target_driver = create_target_driver
+      target_driver.run(expect_records: 1, timeout: 5) do
+        sleep 2
+        d = create_driver(conf)
+        d.run do
+          d.feed('test', event_time, {'a' => 'b', 'foo' => 'bar', 'message' => 'test'})
+        end
+      end
+      actual_messages = target_driver.events.collect { |event| event[2] }
+      assert_equal([{'a' => 'b', 'message' => 'test'}], actual_messages)
+    end
+  end
 end

data/test/plugin/test_out_rdkafka2.rb

@@ -0,0 +1,167 @@
+require 'helper'
+require 'fluent/test/helpers'
+require 'fluent/test/driver/input'
+require 'fluent/test/driver/output'
+require 'securerandom'
+
+class Rdkafka2OutputTest < Test::Unit::TestCase
+  include Fluent::Test::Helpers
+
+  def have_rdkafka
+    begin
+      require 'fluent/plugin/out_rdkafka2'
+      true
+    rescue LoadError
+      false
+    end
+  end
+
+  def setup
+    omit_unless(have_rdkafka, "rdkafka isn't installed")
+    Fluent::Test.setup
+  end
+
+  def base_config
+    config_element('ROOT', '', {"@type" => "rdkafka2"}, [
+                     config_element('format', "", {"@type" => "json"})
+                   ])
+  end
+
+  def config(default_topic: "kitagawakeiko")
+    base_config + config_element('ROOT', '', {"default_topic" => default_topic,
+                                              "brokers" => "localhost:9092"}, [
+                                 ])
+  end
+
+  def create_driver(conf = config, tag='test')
+    Fluent::Test::Driver::Output.new(Fluent::Rdkafka2Output).configure(conf)
+  end
+
+  def test_configure
+    assert_nothing_raised(Fluent::ConfigError) {
+      create_driver(base_config)
+    }
+
+    assert_nothing_raised(Fluent::ConfigError) {
+      create_driver(config)
+    }
+
+    assert_nothing_raised(Fluent::ConfigError) {
+      create_driver(config + config_element('buffer', "", {"@type" => "memory"}))
+    }
+
+    d = create_driver
+    assert_equal 'kitagawakeiko', d.instance.default_topic
+    assert_equal 'localhost:9092', d.instance.brokers
+  end
+
+  def test_mutli_worker_support
+    d = create_driver
+    assert_equal true, d.instance.multi_workers_ready?
+  end
+
+  class WriteTest < self
+    TOPIC_NAME = "kafka-output-#{SecureRandom.uuid}"
+
+    INPUT_CONFIG = %[
+      @type kafka
+      brokers localhost:9092
+      format json
+      @label @kafka
+      topics #{TOPIC_NAME}
+    ]
+
+    def create_target_driver(conf = INPUT_CONFIG)
+      Fluent::Test::Driver::Input.new(Fluent::KafkaInput).configure(conf)
+    end
+
+    def setup
+      @kafka = nil
+      omit_unless(have_rdkafka, "rdkafka isn't installed")
+      @kafka = Kafka.new(["localhost:9092"], client_id: 'kafka')
+    end
+
+    def teardown
+      if @kafka
+        @kafka.delete_topic(TOPIC_NAME)
+        @kafka.close
+      end
+    end
+
+    def test_write
+      target_driver = create_target_driver
+      expected_message = {"a" => 2}
+      target_driver.run(expect_records: 1, timeout: 5) do
+        sleep 2
+        d = create_driver(config(default_topic: TOPIC_NAME))
+        d.run do
+          d.feed("test", event_time, expected_message)
+        end
+      end
+      actual_messages = target_driver.events.collect { |event| event[2] }
+      assert_equal([expected_message], actual_messages)
+    end
+
+    def test_write_with_use_event_time
+      input_config = %[
+        @type kafka
+        brokers localhost:9092
+        format json
+        @label @kafka
+        topics #{TOPIC_NAME}
+        time_source kafka
+      ]
+      target_driver = create_target_driver(input_config)
+      expected_message = {"a" => 2}
+      now = event_time
+      target_driver.run(expect_records: 1, timeout: 5) do
+        sleep 2
+        d = create_driver(config(default_topic: TOPIC_NAME) + config_element('ROOT', '', {"use_event_time" => true}))
+        d.run do
+          d.feed("test", now, expected_message)
+        end
+      end
+      actual_time = target_driver.events.collect { |event| event[1] }.last
+      assert_in_delta(actual_time, now, 0.001) # expects millseconds precision
+      actual_messages = target_driver.events.collect { |event| event[2] }
+      assert_equal([expected_message], actual_messages)
+    end
+
+    def test_exclude_fields
+      conf = config(default_topic: TOPIC_NAME) +
+             config_element('ROOT', '', {"exclude_fields" => "$.foo"}, [])
+      target_driver = create_target_driver
+      target_driver.run(expect_records: 1, timeout: 5) do
+        sleep 2
+        d = create_driver(conf)
+        d.run do
+          d.feed('test', event_time, {'a' => 'b', 'foo' => 'bar', 'message' => 'test'})
+        end
+      end
+      actual_messages = target_driver.events.collect { |event| event[2] }
+      assert_equal([{'a' => 'b', 'message' => 'test'}], actual_messages)
+    end
+
+    def test_max_enqueue_bytes_per_second
+      conf = config(default_topic: TOPIC_NAME) +
+             config_element('ROOT', '', {"max_enqueue_bytes_per_second" => 32 * 3}, [])
+      target_driver = create_target_driver
+      expected_messages = []
+      target_driver.run(expect_records: 9, timeout: 10) do
+        sleep 2
+        d = create_driver(conf)
+        start_time = Fluent::Clock.now
+        d.run do
+          9.times do |i|
+            message = {"message" => "32bytes message: #{i}"}
+            d.feed("test", event_time, message)
+            expected_messages << message
+          end
+        end
+        assert_in_delta(2.0, Fluent::Clock.now - start_time, 0.5)
+      end
+      actual_messages = target_driver.events.collect { |event| event[2] }
+      assert_equal(expected_messages, actual_messages)
+    end
+  end
+end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.17.0
+  version: 0.17.4
 platform: ruby
 authors:
 - Hidemasa Togashi
 - Masahiro Nakagawa
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-30 00:00:00.000000000 Z
+date: 2022-01-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -173,11 +173,12 @@ files:
 - test/plugin/test_out_kafka.rb
 - test/plugin/test_out_kafka2.rb
 - test/plugin/test_out_kafka_buffered.rb
+- test/plugin/test_out_rdkafka2.rb
 homepage: https://github.com/fluent/fluent-plugin-kafka
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -192,8 +193,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
-signing_key:
+rubygems_version: 3.2.5
+signing_key: 
 specification_version: 4
 summary: Fluentd plugin for Apache Kafka > 0.8
 test_files:
@@ -204,3 +205,4 @@ test_files:
 - test/plugin/test_out_kafka.rb
 - test/plugin/test_out_kafka2.rb
 - test/plugin/test_out_kafka_buffered.rb
+- test/plugin/test_out_rdkafka2.rb