fluent-plugin-kafka 0.14.1 → 0.15.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 27a59b5835dff5d64dcf78bd5d3bf945341c9b734de476b16f9217afa2839a22
-  data.tar.gz: fdac125fa11e88712059f0f0794ca199ef248d562b1e1aad389e6b0dace9c777
+  metadata.gz: 730f14a0d24b98ac7df9683bc0836f826ff80def0ee61c833bc4c1e61bf435ac
+  data.tar.gz: 0a647554fb96f88ae75c8b872399545bfdb6b26d12247f3715c56dda2e628d57
 SHA512:
-  metadata.gz: faf2abd472b6af6b010409750d6b0e3483ba8748af8930c4dad05d6e6b4d9aca5dc7c55d29b2e4ed3b092b12612f0d19e46b9878463ecbe417e417d7c3ee522b
-  data.tar.gz: f51803596ea03e0f6dfc9f83abaf070179136f497dc89ad48eed0582c8d580ad88e105f5f4e075096421f6dc0acf8bf1e58fd75738e7c49636acca01ec347a46
+  metadata.gz: 802bef06a0cb0703e0e3c1d8c9581a96007b23a6b574d2c64f3bca362f6d4c531cbc4c9c17c78162705d655fc2e864af16182772a1db0ce2150c0e853b4f55ef
+  data.tar.gz: 0d3695bc1289b75cee1071fcf8e697250de9f0177b3facd69dd62e931a7b705fd93b24f886e683586d01f9daf9888d487ff6058da791d3779d09c076af854dd8

data/ChangeLog

@@ -1,3 +1,25 @@
+Release 0.15.3 - 2020/12/08
+
+	* in_kafka: Fix `record_time_key` parameter not working
+
+Release 0.15.2 - 2020/09/30
+
+	* input: Support 3rd party parser
+
+Release 0.15.1 - 2020/09/17
+
+	* out_kafka2: Fix wrong class name for configuration error
+
+Release 0.15.0 - 2020/09/14
+
+	* Add experimental `in_rdkafka_group`
+	* in_kafka: Expose `ssl_verify_hostname` parameter
+
+Release 0.14.2 - 2020/08/26
+
+	* in_kafka_group: Add `add_headers` parameter
+	* out_kafka2/out_rdkafka2: Support `discard_kafka_delivery_failed` parameter
+
 Release 0.14.1 - 2020/08/11
 
 	* kafka_producer_ext: Fix regression by v0.14.0 changes
@@ -10,6 +32,7 @@ Release 0.14.0 - 2020/08/07
 Release 0.13.1 - 2020/07/17
 
 	* in_kafka_group: Support ssl_verify_hostname parameter
+	* in_kafka_group: Support regex based topics
 	* out_kafka2/out_rdkafka2: Support topic parameter with placeholders
 
 Release 0.13.0 - 2020/03/09

data/README.md

@@ -118,6 +118,8 @@ Consume events by kafka consumer group features..
       topics <listening topics(separate with comma',')>
       format <input text type (text|json|ltsv|msgpack)> :default => json
       message_key <key (Optional, for text format only, default is message)>
+      kafka_mesasge_key <key (Optional, If specified, set kafka's message key to this key)>
+      add_headers <If true, add kafka's message headers to record>
       add_prefix <tag prefix (Optional)>
       add_suffix <tag suffix (Optional)>
       retry_emit_limit <Wait retry_emit_limit x 1s when BuffereQueueLimitError happens. The default is nil and it means waiting until BufferQueueLimitError is resolved>
@@ -137,11 +139,47 @@ Consume events by kafka consumer group features..
 
 See also [ruby-kafka README](https://github.com/zendesk/ruby-kafka#consuming-messages-from-kafka) for more detailed documentation about ruby-kafka options.
 
+`topics` supports regex pattern since v0.13.1. If you want to use regex pattern, use `/pattern/` like `/foo.*/`.
+
+Consuming topic name is used for event tag. So when the target topic name is `app_event`, the tag is `app_event`. If you want to modify tag, use `add_prefix` or `add_suffix` parameter. With `add_prefix kafka`, the tag is `kafka.app_event`.
+
+### Input plugin (@type 'rdkafka_group', supports kafka consumer groups, uses rdkafka-ruby)
+
+:warning: **The in_rdkafka_group consumer was not yet tested under heavy production load. Use it at your own risk!**
+
+With the introduction of the rdkafka-ruby based input plugin we hope to support Kafka brokers above version 2.1 where we saw [compatibility issues](https://github.com/fluent/fluent-plugin-kafka/issues/315) when using the ruby-kafka based @kafka_group input type. The rdkafka-ruby lib wraps the highly performant and production ready librdkafka C lib.
+
+    <source>
+      @type rdkafka_group
+      topics <listening topics(separate with comma',')>
+      format <input text type (text|json|ltsv|msgpack)> :default => json
+      message_key <key (Optional, for text format only, default is message)>
+      kafka_mesasge_key <key (Optional, If specified, set kafka's message key to this key)>
+      add_headers <If true, add kafka's message headers to record>
+      add_prefix <tag prefix (Optional)>
+      add_suffix <tag suffix (Optional)>
+      retry_emit_limit <Wait retry_emit_limit x 1s when BuffereQueueLimitError happens. The default is nil and it means waiting until BufferQueueLimitError is resolved>
+      use_record_time (Deprecated. Use 'time_source record' instead.) <If true, replace event time with contents of 'time' field of fetched record>
+      time_source <source for message timestamp (now|kafka|record)> :default => now
+      time_format <string (Optional when use_record_time is used)>
+
+      # kafka consumer options
+      max_wait_time_ms 500
+      max_batch_size 10000
+      kafka_configs {
+        "bootstrap.servers": "brokers <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>",
+        "group.id": "<consumer group name>"
+      }
+    </source>
+
+See also [rdkafka-ruby](https://github.com/appsignal/rdkafka-ruby) and [librdkafka](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md) for more detailed documentation about Kafka consumer options.
+
 Consuming topic name is used for event tag. So when the target topic name is `app_event`, the tag is `app_event`. If you want to modify tag, use `add_prefix` or `add_suffix` parameter. With `add_prefix kafka`, the tag is `kafka.app_event`.
 
 ### Output plugin
 
-This `kafka2` plugin is for fluentd v1.0 or later. This will be `out_kafka` plugin in the future.
+This `kafka2` plugin is for fluentd v1 or later. This plugin uses `ruby-kafka` producer for writing data.
+If `ruby-kafka` doesn't fit your kafka environment, check `rdkafka2` plugin instead. This will be `out_kafka` plugin in the future.
 
     <match app.**>
       @type kafka2
@@ -162,6 +200,7 @@ This `kafka2` plugin is for fluentd v1.0 or later. This will be `out_kafka` plug
       headers               (hash)   :default => {}
       headers_from_record   (hash)   :default => {}
       use_default_for_unknown_topic (bool) :default => false
+      discard_kafka_delivery_failed (bool) :default => false (No discard)
 
       <format>
         @type (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
@@ -385,6 +424,7 @@ You need to install rdkafka gem.
       default_message_key   (string) :default => nil
       exclude_topic_key     (bool) :default => false
       exclude_partition_key (bool) :default => false
+      discard_kafka_delivery_failed (bool) :default => false (No discard)
 
       # same with kafka2
       headers               (hash) :default => {}

data/fluent-plugin-kafka.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.14.1'
+  gem.version       = '0.15.3'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/in_kafka.rb

@@ -113,7 +113,7 @@ class Fluent::KafkaInput < Fluent::Input
 
     require 'zookeeper' if @offset_zookeeper
 
-    @parser_proc = setup_parser
+    @parser_proc = setup_parser(conf)
 
     @time_source = :record if @use_record_time
 
@@ -126,7 +126,7 @@ class Fluent::KafkaInput < Fluent::Input
     end
   end
 
-  def setup_parser
+  def setup_parser(conf)
     case @format
     when 'json'
       begin
@@ -165,6 +165,14 @@ class Fluent::KafkaInput < Fluent::Input
         add_offset_in_hash(r, te, msg.offset) if @add_offset_in_record
         r
       }
+    else
+      @custom_parser = Fluent::Plugin.new_parser(conf['format'])
+      @custom_parser.configure(conf)
+      Proc.new { |msg|
+        @custom_parser.parse(msg.value) {|_time, record|
+          record
+        }
+      }
     end
   end
 
@@ -188,16 +196,17 @@ class Fluent::KafkaInput < Fluent::Input
       @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_scram_username: @username, sasl_scram_password: @password,
-                         sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl)
+                         sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
     elsif @username != nil && @password != nil
       @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system,sasl_plain_username: @username, sasl_plain_password: @password,
-                         sasl_over_ssl: @sasl_over_ssl)
+                         sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
     else
       @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
-                         ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab)
+                         ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab,
+                         ssl_verify_hostname: @ssl_verify_hostname)
     end
 
     @zookeeper = Zookeeper.new(@offset_zookeeper) if @offset_zookeeper
@@ -215,6 +224,7 @@ class Fluent::KafkaInput < Fluent::Input
         router,
         @kafka_message_key,
         @time_source,
+        @record_time_key,
         opt)
     }
     @topic_watchers.each {|tw|
@@ -239,7 +249,7 @@ class Fluent::KafkaInput < Fluent::Input
   end
 
   class TopicWatcher < Coolio::TimerWatcher
-    def initialize(topic_entry, kafka, interval, parser, add_prefix, add_suffix, offset_manager, router, kafka_message_key, time_source, options={})
+    def initialize(topic_entry, kafka, interval, parser, add_prefix, add_suffix, offset_manager, router, kafka_message_key, time_source, record_time_key, options={})
       @topic_entry = topic_entry
       @kafka = kafka
       @callback = method(:consume)
@@ -251,6 +261,7 @@ class Fluent::KafkaInput < Fluent::Input
       @router = router
       @kafka_message_key = kafka_message_key
       @time_source = time_source
+      @record_time_key = record_time_key
 
       @next_offset = @topic_entry.offset
       if @topic_entry.offset == -1 && offset_manager

data/lib/fluent/plugin/in_kafka_group.rb

@@ -18,6 +18,8 @@ class Fluent::KafkaGroupInput < Fluent::Input
                :desc => "Supported format: (json|text|ltsv|msgpack)"
   config_param :message_key, :string, :default => 'message',
                :desc => "For 'text' format only."
+  config_param :add_headers, :bool, :default => false,
+               :desc => "Add kafka's message headers to event record"
   config_param :add_prefix, :string, :default => nil,
                :desc => "Tag prefix (Optional)"
   config_param :add_suffix, :string, :default => nil,
@@ -115,7 +117,7 @@ class Fluent::KafkaGroupInput < Fluent::Input
       @max_wait_time = conf['max_wait_ms'].to_i / 1000
     end
 
-    @parser_proc = setup_parser
+    @parser_proc = setup_parser(conf)
 
     @consumer_opts = {:group_id => @consumer_group}
     @consumer_opts[:session_timeout] = @session_timeout if @session_timeout
@@ -138,7 +140,7 @@ class Fluent::KafkaGroupInput < Fluent::Input
     end
   end
 
-  def setup_parser
+  def setup_parser(conf)
     case @format
     when 'json'
       begin
@@ -157,6 +159,14 @@ class Fluent::KafkaGroupInput < Fluent::Input
       Proc.new { |msg| MessagePack.unpack(msg.value) }
     when 'text'
       Proc.new { |msg| {@message_key => msg.value} }
+    else
+      @custom_parser = Fluent::Plugin.new_parser(conf['format'])
+      @custom_parser.configure(conf)
+      Proc.new { |msg|
+        @custom_parser.parse(msg.value) {|_time, record|
+          record
+        }
+      }
     end
   end
 
@@ -263,6 +273,11 @@ class Fluent::KafkaGroupInput < Fluent::Input
               if @kafka_message_key
                 record[@kafka_message_key] = msg.key
               end
+              if @add_headers
+                msg.headers.each_pair { |k, v|
+                  record[k] = v
+                }
+              end
               es.add(record_time, record)
             rescue => e
               log.warn "parser error in #{batch.topic}/#{batch.partition}", :error => e.to_s, :value => msg.value, :offset => msg.offset

data/lib/fluent/plugin/in_rdkafka_group.rb

@@ -0,0 +1,305 @@
+require 'fluent/plugin/input'
+require 'fluent/time'
+require 'fluent/plugin/kafka_plugin_util'
+
+require 'rdkafka'
+
+class Fluent::Plugin::RdKafkaGroupInput < Fluent::Plugin::Input
+  Fluent::Plugin.register_input('rdkafka_group', self)
+
+  helpers :thread, :parser, :compat_parameters
+
+  config_param :topics, :string,
+               :desc => "Listening topics(separate with comma',')."
+
+  config_param :format, :string, :default => 'json',
+               :desc => "Supported format: (json|text|ltsv|msgpack)"
+  config_param :message_key, :string, :default => 'message',
+               :desc => "For 'text' format only."
+  config_param :add_headers, :bool, :default => false,
+               :desc => "Add kafka's message headers to event record"
+  config_param :add_prefix, :string, :default => nil,
+               :desc => "Tag prefix (Optional)"
+  config_param :add_suffix, :string, :default => nil,
+               :desc => "Tag suffix (Optional)"
+  config_param :use_record_time, :bool, :default => false,
+               :desc => "Replace message timestamp with contents of 'time' field.",
+               :deprecated => "Use 'time_source record' instead."
+  config_param :time_source, :enum, :list => [:now, :kafka, :record], :default => :now,
+               :desc => "Source for message timestamp."
+  config_param :record_time_key, :string, :default => 'time',
+               :desc => "Time field when time_source is 'record'"
+  config_param :time_format, :string, :default => nil,
+               :desc => "Time format to be used to parse 'time' field."
+  config_param :kafka_message_key, :string, :default => nil,
+               :desc => "Set kafka's message key to this field"
+
+  config_param :retry_emit_limit, :integer, :default => nil,
+               :desc => "How long to stop event consuming when BufferQueueLimitError happens. Wait retry_emit_limit x 1s. The default is waiting until BufferQueueLimitError is resolved"
+  config_param :retry_wait_seconds, :integer, :default => 30
+  config_param :disable_retry_limit, :bool, :default => false,
+               :desc => "If set true, it disables retry_limit and make Fluentd retry indefinitely (default: false)"
+  config_param :retry_limit, :integer, :default => 10,
+               :desc => "The maximum number of retries for connecting kafka (default: 10)"
+
+  config_param :max_wait_time_ms, :integer, :default => 250,
+               :desc => "How long to block polls in milliseconds until the server sends us data."
+  config_param :max_batch_size, :integer, :default => 10000,
+               :desc => "Maximum number of log lines emitted in a single batch."
+
+  config_param :kafka_configs, :hash, :default => {},
+               :desc => "Kafka configuration properties as desribed in https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md"
+
+  config_section :parse do
+    config_set_default :@type, 'json'
+  end
+
+  include Fluent::KafkaPluginUtil::SSLSettings
+  include Fluent::KafkaPluginUtil::SaslSettings
+
+  class ForShutdown < StandardError
+  end
+
+  BufferError = Fluent::Plugin::Buffer::BufferOverflowError
+
+  def initialize
+    super
+
+    @time_parser = nil
+    @retry_count = 1
+  end
+
+  def _config_to_array(config)
+    config_array = config.split(',').map {|k| k.strip }
+    if config_array.empty?
+      raise Fluent::ConfigError, "kafka_group: '#{config}' is a required parameter"
+    end
+    config_array
+  end
+
+  def multi_workers_ready?
+    true
+  end
+
+  private :_config_to_array
+
+  def configure(conf)
+    compat_parameters_convert(conf, :parser)
+
+    super
+
+    log.warn "The in_rdkafka_group consumer was not yet tested under heavy production load. Use it at your own risk!"
+
+    log.info "Will watch for topics #{@topics} at brokers " \
+              "#{@kafka_configs["bootstrap.servers"]} and '#{@kafka_configs["group.id"]}' group"
+
+    @topics = _config_to_array(@topics)
+
+    parser_conf = conf.elements('parse').first
+    unless parser_conf
+      raise Fluent::ConfigError, "<parse> section or format parameter is required."
+    end
+    unless parser_conf["@type"]
+      raise Fluent::ConfigError, "parse/@type is required."
+    end
+    @parser_proc = setup_parser(parser_conf)
+
+    @time_source = :record if @use_record_time
+
+    if @time_source == :record and @time_format
+      @time_parser = Fluent::TimeParser.new(@time_format)
+    end
+  end
+
+  def setup_parser(parser_conf)
+    format = parser_conf["@type"]
+    case format
+    when 'json'
+      begin
+        require 'oj'
+        Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
+        Proc.new { |msg| Oj.load(msg.payload) }
+      rescue LoadError
+        require 'yajl'
+        Proc.new { |msg| Yajl::Parser.parse(msg.payload) }
+      end
+    when 'ltsv'
+      require 'ltsv'
+      Proc.new { |msg| LTSV.parse(msg.payload, {:symbolize_keys => false}).first }
+    when 'msgpack'
+      require 'msgpack'
+      Proc.new { |msg| MessagePack.unpack(msg.payload) }
+    when 'text'
+      Proc.new { |msg| {@message_key => msg.payload} }
+    else
+      @custom_parser = parser_create(usage: 'in-rdkafka-plugin', conf: parser_conf)
+      Proc.new { |msg|
+        @custom_parser.parse(msg.payload) {|_time, record|
+          record
+        }
+      }
+    end
+  end
+
+  def start
+    super
+
+    @consumer = setup_consumer
+
+    thread_create(:in_rdkafka_group, &method(:run))
+  end
+
+  def shutdown
+    # This nil assignment should be guarded by mutex in multithread programming manner.
+    # But the situation is very low contention, so we don't use mutex for now.
+    # If the problem happens, we will add a guard for consumer.
+    consumer = @consumer
+    @consumer = nil
+    consumer.close
+
+    super
+  end
+
+  def setup_consumer
+    consumer = Rdkafka::Config.new(@kafka_configs).consumer
+    consumer.subscribe(*@topics)
+    consumer
+  end
+
+  def reconnect_consumer
+    log.warn "Stopping Consumer"
+    consumer = @consumer
+    @consumer = nil
+    if consumer
+      consumer.close
+    end
+    log.warn "Could not connect to broker. retry_time:#{@retry_count}. Next retry will be in #{@retry_wait_seconds} seconds"
+    @retry_count = @retry_count + 1
+    sleep @retry_wait_seconds
+    @consumer = setup_consumer
+    log.warn "Re-starting consumer #{Time.now.to_s}"
+    @retry_count = 0
+  rescue =>e
+    log.error "unexpected error during re-starting consumer object access", :error => e.to_s
+    log.error_backtrace
+    if @retry_count <= @retry_limit or disable_retry_limit
+      reconnect_consumer
+    end
+  end
+
+  class Batch
+    attr_reader :topic
+    attr_reader :messages
+
+    def initialize(topic)
+      @topic = topic
+      @messages = []
+    end
+  end
+
+  # Executes the passed codeblock on a batch of messages.
+  # It is guaranteed that every message in a given batch belongs to the same topic, because the tagging logic in :run expects that property.
+  # The number of maximum messages in a batch is capped by the :max_batch_size configuration value. It ensures that consuming from a single
+  # topic for a long time (e.g. with `auto.offset.reset` set to `earliest`) does not lead to memory exhaustion. Also, calling consumer.poll
+  # advances thes consumer offset, so in case the process crashes we might lose at most :max_batch_size messages.
+  def each_batch(&block)
+    batch = nil
+    message = nil
+    while @consumer
+      message = @consumer.poll(@max_wait_time_ms)
+      if message
+        if not batch
+          batch = Batch.new(message.topic)
+        elsif batch.topic != message.topic || batch.messages.size >= @max_batch_size
+          yield batch
+          batch = Batch.new(message.topic)
+        end
+        batch.messages << message
+      else
+        yield batch if batch
+        batch = nil
+      end
+    end
+    yield batch if batch
+  end
+
+  def run
+    while @consumer
+      begin
+        each_batch { |batch|
+          log.debug "A new batch for topic #{batch.topic} with #{batch.messages.size} messages"
+          es = Fluent::MultiEventStream.new
+          tag = batch.topic
+          tag = @add_prefix + "." + tag if @add_prefix
+          tag = tag + "." + @add_suffix if @add_suffix
+
+          batch.messages.each { |msg|
+            begin
+              record = @parser_proc.call(msg)
+              case @time_source
+              when :kafka
+                record_time = Fluent::EventTime.from_time(msg.timestamp)
+              when :now
+                record_time = Fluent::Engine.now
+              when :record
+                if @time_format
+                  record_time = @time_parser.parse(record[@record_time_key].to_s)
+                else
+                  record_time = record[@record_time_key]
+                end
+              else
+                log.fatal "BUG: invalid time_source: #{@time_source}"
+              end
+              if @kafka_message_key
+                record[@kafka_message_key] = msg.key
+              end
+              if @add_headers
+                msg.headers.each_pair { |k, v|
+                  record[k] = v
+                }
+              end
+              es.add(record_time, record)
+            rescue => e
+              log.warn "parser error in #{msg.topic}/#{msg.partition}", :error => e.to_s, :value => msg.payload, :offset => msg.offset
+              log.debug_backtrace
+            end
+          }
+
+          unless es.empty?
+            emit_events(tag, es)
+          end
+        }
+      rescue ForShutdown
+      rescue => e
+        log.error "unexpected error during consuming events from kafka. Re-fetch events.", :error => e.to_s
+        log.error_backtrace
+        reconnect_consumer
+      end
+    end
+  rescue => e
+    log.error "unexpected error during consumer object access", :error => e.to_s
+    log.error_backtrace
+  end
+
+  def emit_events(tag, es)
+    retries = 0
+    begin
+      router.emit_stream(tag, es)
+    rescue BufferError
+      raise ForShutdown if @consumer.nil?
+
+      if @retry_emit_limit.nil?
+        sleep 1
+        retry
+      end
+
+      if retries < @retry_emit_limit
+        retries += 1
+        sleep 1
+        retry
+      else
+        raise RuntimeError, "Exceeds retry_emit_limit"
+      end
+    end
+  end
+end

data/lib/fluent/plugin/out_kafka2.rb

@@ -69,6 +69,7 @@ The codec the producer uses to compress messages.
 Supported codecs depends on ruby-kafka: https://github.com/zendesk/ruby-kafka#compression
 DESC
     config_param :max_send_limit_bytes, :size, :default => nil
+    config_param :discard_kafka_delivery_failed, :bool, :default => false
     config_param :active_support_notification_regex, :string, :default => nil,
                  :desc => <<-DESC
 Add a regular expression to capture ActiveSupport notifications from the Kafka client
@@ -127,7 +128,7 @@ DESC
         @seed_brokers = @brokers
         log.info "brokers has been set: #{@seed_brokers}"
       else
-        raise Fluent::Config, 'No brokers specified. Need one broker at least.'
+        raise Fluent::ConfigError, 'No brokers specified. Need one broker at least.'
       end
 
       formatter_conf = conf.elements('format').first
@@ -267,7 +268,16 @@ DESC
 
         if messages > 0
           log.debug { "#{messages} messages send." }
-          producer.deliver_messages
+          if @discard_kafka_delivery_failed
+            begin
+              producer.deliver_messages
+            rescue Kafka::DeliveryFailed => e
+              log.warn "DeliveryFailed occurred. Discard broken event:", :error => e.to_s, :error_class => e.class.to_s, :tag => tag
+              producer.clear_buffer
+            end
+          else
+            producer.deliver_messages
+          end
         end
       rescue Kafka::UnknownTopicOrPartition
         if @use_default_for_unknown_topic && topic != @default_topic

data/lib/fluent/plugin/out_rdkafka2.rb

@@ -73,6 +73,7 @@ The codec the producer uses to compress messages. Used for compression.codec
 Supported codecs: (gzip|snappy)
 DESC
     config_param :max_send_limit_bytes, :size, :default => nil
+    config_param :discard_kafka_delivery_failed, :bool, :default => false
     config_param :rdkafka_buffering_max_ms, :integer, :default => nil, :desc => 'Used for queue.buffering.max.ms'
     config_param :rdkafka_buffering_max_messages, :integer, :default => nil, :desc => 'Used for queue.buffering.max.messages'
     config_param :rdkafka_message_max_bytes, :integer, :default => nil, :desc => 'Used for message.max.bytes'
@@ -325,9 +326,13 @@ DESC
         }
       end
     rescue Exception => e
-      log.warn "Send exception occurred: #{e} at #{e.backtrace.first}"
-      # Raise exception to retry sendind messages
-      raise e
+      if @discard_kafka_delivery_failed
+        log.warn "Delivery failed. Discard events:", :error => e.to_s, :error_class => e.class.to_s, :tag => tag
+      else
+        log.warn "Send exception occurred: #{e} at #{e.backtrace.first}"
+        # Raise exception to retry sendind messages
+        raise e
+      end
     end
 
     def enqueue_with_retry(producer, topic, record_buf, message_key, partition, headers)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.14.1
+  version: 0.15.3
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-08-11 00:00:00.000000000 Z
+date: 2020-12-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -111,6 +111,7 @@ files:
 - fluent-plugin-kafka.gemspec
 - lib/fluent/plugin/in_kafka.rb
 - lib/fluent/plugin/in_kafka_group.rb
+- lib/fluent/plugin/in_rdkafka_group.rb
 - lib/fluent/plugin/kafka_plugin_util.rb
 - lib/fluent/plugin/kafka_producer_ext.rb
 - lib/fluent/plugin/out_kafka.rb