fluent-plugin-kafka 0.7.9 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ec635f2018cccc5a37647a4ad478ff1f3cd07754
-  data.tar.gz: 2d798630940b044b2576610e0da5de71b5856b08
+  metadata.gz: faa29645a4c13d7400459f16cbf7720e3bf8a3cc
+  data.tar.gz: 56a4b88bbfc9f42efe9f87bc845552bc17b1f019
 SHA512:
-  metadata.gz: 8fa8b7f16274b51087abddf398ef166c3a3aab1ecc2369c8fed60bf1424fe2947ae5244435d98585516607e06a0c40fac5ef6c1c39c6fff99d76def2ec07b419
-  data.tar.gz: 612620ed1e78bcf4bee2e02d17d4be84914890638da0b1c869c00a5fa5847c5e8630d88e5f4c0ebf2d229c6605b7b4fbb0ac52e1781bde7a21b67584acc14b99
+  metadata.gz: 13253507831d6f1e07e0937f3b4d01e01ffa2ea60605ad3cb1c326f3a4606c3a1173d07d3688250b8866ae3a20a69662c05e688ba746cc677c448a643082e528
+  data.tar.gz: 4fb3eb6ed6ffd61b12efa4e8da930a573b5420a4681885a66b54c9052a5907bc39ad6c17d6d65d7aefaa9f8546fe8453cee199ad52a48a1d4a799636b8f0165f

data/ChangeLog

@@ -1,3 +1,9 @@
+Release 0.8.0 - 2018/10/18
+
+	* output: Support SASL without SSL
+	* output: Add rdkafka based output
+	* Update ruby-kafka dependency to v0.7 or later
+
 Release 0.7.9 - 2018/09/11
 
 	* in_kafka_group: Add fetcher_max_queue_size parameter

data/README.md

@@ -18,7 +18,7 @@ And then execute:
 
 Or install it yourself as:
 
-    $ gem install fluent-plugin-kafka
+    $ gem install fluent-plugin-kafka --no-document
 
 If you want to use zookeeper related parameters, you also need to install zookeeper gem. zookeeper gem includes native extension, so development tools are needed, e.g. gcc, make and etc.
 
@@ -192,7 +192,7 @@ See also [Kafka::Client](http://www.rubydoc.info/gems/ruby-kafka/Kafka/Client) f
 This plugin supports compression codec "snappy" also.
 Install snappy module before you use snappy compression.
 
-    $ gem install snappy
+    $ gem install snappy --no-document
 
 snappy gem uses native extension, so you need to install several packages before.
 On Ubuntu, need development packages and snappy library.
@@ -219,7 +219,7 @@ If key name `message_key` exists in a message, this plugin publishes the value o
 
 ### Output plugin
 
-This plugin is for v1.0. This will be `out_kafka` plugin in the future.
+This plugin is for fluentd v1.0 or later. This will be `out_kafka` plugin in the future.
 
     <match app.**>
       @type kafka2
@@ -290,6 +290,26 @@ This plugin uses ruby-kafka producer for writing data. For performance and relia
 
 This plugin also supports ruby-kafka related parameters. See Buffered output plugin section.
 
+### rdkafka based output plugin
+
+This plugin uses `rdkafka` instead of `ruby-kafka` for ruby client.
+You need to install rdkafka gem.
+
+    # rdkafka is C extension library so need development tools like ruby-devel, gcc and etc
+    $ gem install rdkafka --no-document
+
+    <match kafka.**>
+      @type rdkafka
+
+      default_topic kafka
+      flush_interval 1s
+      output_data_type json
+
+      rdkafka_options {
+        "log_level" : 7
+      }
+    </match>
+
 ## Contributing
 
 1. Fork it

data/fluent-plugin-kafka.gemspec

@@ -13,12 +13,12 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.7.9'
+  gem.version       = '0.8.0'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
   gem.add_dependency 'ltsv'
-  gem.add_dependency 'ruby-kafka', '>= 0.4.1', '< 0.7.0'
+  gem.add_dependency 'ruby-kafka', '>= 0.7.1', '< 0.8.0'
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "test-unit", ">= 3.0.8"
 end

data/lib/fluent/plugin/kafka_producer_ext.rb

@@ -9,19 +9,28 @@ require 'kafka/producer'
 
 # for out_kafka_buffered
 module Kafka
+  EMPTY_HEADER = {}
+
   class Producer
-    def produce2(value, key: nil, topic:, partition: nil, partition_key: nil)
+    def produce_for_buffered(value, key: nil, topic:, partition: nil, partition_key: nil)
       create_time = Time.now
 
       message = PendingMessage.new(
-        value,
-        key,
-        topic,
-        partition,
-        partition_key,
-        create_time
+        value: value,
+        key: key,
+        headers: EMPTY_HEADER,
+        topic: topic,
+        partition: partition,
+        partition_key: partition_key,
+        create_time: create_time
       )
 
+      # If the producer is in transactional mode, all the message production
+      # must be used when the producer is currently in transaction
+      if @transaction_manager.transactional? && !@transaction_manager.in_transaction?
+        raise 'You must trigger begin_transaction before producing messages'
+      end
+
       @target_topics.add(topic)
       @pending_message_queue.write(message)
 
@@ -33,15 +42,26 @@ end
 # for out_kafka2
 module Kafka
   class Client
-    def topic_producer(topic, compression_codec: nil, compression_threshold: 1, ack_timeout: 5, required_acks: :all, max_retries: 2, retry_backoff: 1, max_buffer_size: 1000, max_buffer_bytesize: 10_000_000)
+    def topic_producer(topic, compression_codec: nil, compression_threshold: 1, ack_timeout: 5, required_acks: :all, max_retries: 2, retry_backoff: 1, max_buffer_size: 1000, max_buffer_bytesize: 10_000_000, idempotent: false, transactional: false, transactional_id: nil, transactional_timeout: 60)
+      cluster = initialize_cluster
       compressor = Compressor.new(
         codec_name: compression_codec,
         threshold: compression_threshold,
         instrumenter: @instrumenter,
       )
 
+      transaction_manager = TransactionManager.new(
+        cluster: cluster,
+        logger: @logger,
+        idempotent: idempotent,
+        transactional: transactional,
+        transactional_id: transactional_id,
+        transactional_timeout: transactional_timeout,
+      )
+
       TopicProducer.new(topic,
-        cluster: initialize_cluster,
+        cluster: cluster,
+        transaction_manager: transaction_manager,
         logger: @logger,
         instrumenter: @instrumenter,
         compressor: compressor,
@@ -56,8 +76,9 @@ module Kafka
   end
 
   class TopicProducer
-    def initialize(topic, cluster:, logger:, instrumenter:, compressor:, ack_timeout:, required_acks:, max_retries:, retry_backoff:, max_buffer_size:, max_buffer_bytesize:)
+    def initialize(topic, cluster:, transaction_manager:, logger:, instrumenter:, compressor:, ack_timeout:, required_acks:, max_retries:, retry_backoff:, max_buffer_size:, max_buffer_bytesize:)
       @cluster = cluster
+      @transaction_manager = transaction_manager
       @logger = logger
       @instrumenter = instrumenter
       @required_acks = required_acks == :all ? -1 : required_acks
@@ -78,18 +99,25 @@ module Kafka
       @pending_message_queue = PendingMessageQueue.new
     end
 
-    def produce(value, key, partition, partition_key)
+    def produce(value, key: nil, partition: nil, partition_key: nil)
       create_time = Time.now
 
       message = PendingMessage.new(
-        value,
-        key,
-        @topic,
-        partition,
-        partition_key,
-        create_time
+        value: value,
+        key: key,
+        headers: EMPTY_HEADER,
+        topic: @topic,
+        partition: partition,
+        partition_key: partition_key,
+        create_time: create_time
       )
 
+      # If the producer is in transactional mode, all the message production
+      # must be used when the producer is currently in transaction
+      if @transaction_manager.transactional? && !@transaction_manager.in_transaction?
+        raise 'You must trigger begin_transaction before producing messages'
+      end
+
       @pending_message_queue.write(message)
 
       nil
@@ -125,10 +153,37 @@ module Kafka
     #
     # @return [nil]
     def shutdown
+      @transaction_manager.close
       @cluster.disconnect
     end
 
-    private
+    def init_transactions
+      @transaction_manager.init_transactions
+    end
+
+    def begin_transaction
+      @transaction_manager.begin_transaction
+    end
+
+    def commit_transaction
+      @transaction_manager.commit_transaction
+    end
+
+    def abort_transaction
+      @transaction_manager.abort_transaction
+    end
+
+    def transaction
+      raise 'This method requires a block' unless block_given?
+      begin_transaction
+      yield
+      commit_transaction
+    rescue Kafka::Producer::AbortTransaction
+      abort_transaction
+    rescue
+      abort_transaction
+      raise
+    end
 
     def deliver_messages_with_retries
       attempt = 0
@@ -137,6 +192,7 @@ module Kafka
 
       operation = ProduceOperation.new(
         cluster: @cluster,
+        transaction_manager: @transaction_manager,
         buffer: @buffer,
         required_acks: @required_acks,
         ack_timeout: @ack_timeout,
@@ -148,7 +204,11 @@ module Kafka
       loop do
         attempt += 1
 
-        @cluster.refresh_metadata_if_necessary!
+        begin
+          @cluster.refresh_metadata_if_necessary!
+        rescue ConnectionError => e
+          raise DeliveryFailed.new(e, buffer_messages)
+        end
 
         assign_partitions!
         operation.execute
@@ -200,6 +260,7 @@ module Kafka
           @buffer.write(
             value: message.value,
             key: message.key,
+            headers: message.headers,
             topic: message.topic,
             partition: partition,
             create_time: message.create_time,
@@ -219,5 +280,29 @@ module Kafka
 
       @pending_message_queue.replace(failed_messages)
     end
+
+    def buffer_messages
+      messages = []
+
+      @pending_message_queue.each do |message|
+        messages << message
+      end
+
+      @buffer.each do |topic, partition, messages_for_partition|
+        messages_for_partition.each do |message|
+          messages << PendingMessage.new(
+            value: message.value,
+            key: message.key,
+            headers: message.headers,
+            topic: topic,
+            partition: partition,
+            partition_key: nil,
+            create_time: message.create_time
+          )
+        end
+      end
+
+      messages
+    end
   end
 end

data/lib/fluent/plugin/out_kafka.rb

@@ -20,6 +20,10 @@ DESC
   config_param :default_partition_key, :string, :default => nil
   config_param :default_partition, :integer, :default => nil
   config_param :client_id, :string, :default => 'kafka'
+  config_param :sasl_over_ssl, :bool, :default => true,
+               :desc => <<-DESC
+Set to false to prevent SSL strict mode when using SASL authentication
+DESC
   config_param :output_data_type, :string, :default => 'json',
                :desc => "Supported format: (json|ltsv|msgpack|attr:<record name>|<formatter name>)"
   config_param :output_include_tag, :bool, :default => false
@@ -104,7 +108,7 @@ DESC
         if @scram_mechanism != nil && @username != nil && @password != nil
           @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
-                             sasl_scram_username: @username, sasl_scram_password: @password, sasl_scram_mechanism: @scram_mechanism)
+                             sasl_scram_username: @username, sasl_scram_password: @password, sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl)
         elsif @username != nil && @password != nil
           @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,

data/lib/fluent/plugin/out_kafka2.rb

@@ -25,6 +25,10 @@ DESC
     config_param :partition_key, :string, :default => 'partition', :desc => "Field for kafka partition"
     config_param :default_partition, :integer, :default => nil
     config_param :client_id, :string, :default => 'fluentd'
+    config_param :sasl_over_ssl, :bool, :default => true,
+               :desc => <<-DESC
+Set to false to prevent SSL strict mode when using SASL authentication
+DESC
     config_param :exclude_partition_key, :bool, :default => false,
                  :desc => 'Set true to remove partition key from data'
     config_param :exclude_partition, :bool, :default => false,
@@ -80,7 +84,7 @@ DESC
         if @scram_mechanism != nil && @username != nil && @password != nil
           @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
-                             sasl_scram_username: @username, sasl_scram_password: @password, sasl_scram_mechanism: @scram_mechanism)
+                             sasl_scram_username: @username, sasl_scram_password: @password, sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl)
         elsif @username != nil && @password != nil
           @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
@@ -212,7 +216,7 @@ DESC
           log.trace { "message will send to #{topic} with partition_key: #{partition_key}, partition: #{partition}, message_key: #{message_key} and value: #{record_buf}." }
           messages += 1
 
-          producer.produce(record_buf, message_key, partition, partition_key)
+          producer.produce(record_buf, key: message_key, partition_key: partition_key, partition: partition)
         }
 
         if messages > 0

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -27,6 +27,10 @@ DESC
   config_param :partition_key, :string, :default => 'partition', :desc => "Field for kafka partition"
   config_param :default_partition, :integer, :default => nil
   config_param :client_id, :string, :default => 'kafka'
+  config_param :sasl_over_ssl, :bool, :default => true,
+               :desc => <<-DESC
+Set to false to prevent SSL strict mode when using SASL authentication
+DESC
   config_param :output_data_type, :string, :default => 'json',
                :desc => <<-DESC
 Supported format: (json|ltsv|msgpack|attr:<record name>|<formatter name>)
@@ -126,7 +130,7 @@ DESC
         if @scram_mechanism != nil && @username != nil && @password != nil
           @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
-                             sasl_scram_username: @username, sasl_scram_password: @password, sasl_scram_mechanism: @scram_mechanism)
+                             sasl_scram_username: @username, sasl_scram_password: @password, sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl)
         elsif @username != nil && @password != nil
           @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
@@ -333,7 +337,7 @@ DESC
         end
         log.trace { "message will send to #{topic} with partition_key: #{partition_key}, partition: #{partition}, message_key: #{message_key} and value: #{record_buf}." }
         messages += 1
-        producer.produce2(record_buf, topic: topic, key: message_key, partition_key: partition_key, partition: partition)
+        producer.produce_for_buffered(record_buf, topic: topic, key: message_key, partition_key: partition_key, partition: partition)
         messages_bytes += record_buf_bytes
 
         records_by_topic[topic] += 1

data/lib/fluent/plugin/out_rdkafka.rb

@@ -0,0 +1,301 @@
+require 'thread'
+require 'fluent/output'
+require 'fluent/plugin/kafka_plugin_util'
+
+require 'rdkafka'
+require 'fluent/plugin/kafka_producer_ext'
+
+class Rdkafka::Producer
+  # return false if producer is forcefully closed, otherwise return true
+  def close(timeout = nil)
+    @closing = true
+    # Wait for the polling thread to finish up
+    # If the broker isn't alive, the thread doesn't exit
+    if timeout
+      thr = @polling_thread.join(timeout)
+      return !!thr
+    else
+      @polling_thread.join
+      return true
+    end
+  end
+end
+
+class Fluent::KafkaOutputBuffered2 < Fluent::BufferedOutput
+  Fluent::Plugin.register_output('rdkafka', self)
+
+  config_param :brokers, :string, :default => 'localhost:9092',
+               :desc => <<-DESC
+Set brokers directly:
+<broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,..
+Brokers: you can choose to use either brokers or zookeeper.
+DESC
+  config_param :default_topic, :string, :default => nil,
+               :desc => "Output topic"
+  config_param :default_message_key, :string, :default => nil
+  config_param :default_partition, :integer, :default => nil
+  config_param :client_id, :string, :default => 'kafka'
+  config_param :output_data_type, :string, :default => 'json',
+               :desc => <<-DESC
+Supported format: (json|ltsv|msgpack|attr:<record name>|<formatter name>)
+DESC
+  config_param :output_include_tag, :bool, :default => false
+  config_param :output_include_time, :bool, :default => false
+  config_param :exclude_partition, :bool, :default => false,
+               :desc => <<-DESC
+Set true to remove partition from data
+DESC
+   config_param :exclude_message_key, :bool, :default => false,
+               :desc => <<-DESC
+Set true to remove partition key from data
+DESC
+   config_param :exclude_topic_key, :bool, :default => false,
+                :desc => <<-DESC
+Set true to remove topic name key from data
+DESC
+  config_param :max_send_retries, :integer, :default => 2,
+               :desc => "Number of times to retry sending of messages to a leader."
+  config_param :required_acks, :integer, :default => -1,
+               :desc => "The number of acks required per request."
+  config_param :ack_timeout, :time, :default => nil,
+               :desc => "How long the producer waits for acks."
+  config_param :compression_codec, :string, :default => nil,
+               :desc => <<-DESC
+The codec the producer uses to compress messages.
+Supported codecs: (gzip|snappy)
+DESC
+
+  config_param :rdkafka_buffering_max_ms, :integer, :default => nil
+  config_param :rdkafka_buffering_max_messages, :integer, :default => nil
+  config_param :rdkafka_message_max_bytes, :integer, :default => nil
+  config_param :rdkafka_message_max_num, :integer, :default => nil
+  config_param :rdkafka_delivery_handle_poll_timeout, :integer, :default => 30
+  config_param :rdkafka_options, :hash, :default => {}
+
+  config_param :max_enqueue_retries, :integer, :default => 3
+  config_param :enqueue_retry_backoff, :integer, :default => 3
+
+  config_param :service_name, :string, :default => nil
+  config_param :ssl_client_cert_key_password, :string, :default => nil
+
+  include Fluent::KafkaPluginUtil::SSLSettings
+  include Fluent::KafkaPluginUtil::SaslSettings
+
+  def initialize
+    super
+    @producers = {}
+    @producers_mutex = Mutex.new
+  end
+
+  def configure(conf)
+    super
+    log.instance_eval {
+      def add(level, &block)
+        if block
+          self.info(block.call)
+        end
+      end
+    }
+    Rdkafka::Config.logger = log
+    config = build_config
+    @rdkafka = Rdkafka::Config.new(config)
+    @formatter_proc = setup_formatter(conf)
+  end
+
+  def build_config
+    config = {
+      :"bootstrap.servers" => @brokers,
+    }
+
+    if @ssl_ca_cert && @ssl_ca_cert[0]
+      ssl = true
+      config[:"ssl.ca.location"] = @ssl_ca_cert[0]
+      config[:"ssl.certificate.location"] = @ssl_client_cert if @ssl_client_cert
+      config[:"ssl.key.location"] = @ssl_client_cert_key if @ssl_client_cert_key
+      config[:"ssl.key.password"] = @ssl_client_cert_key_password if @ssl_client_cert_key_password
+    end
+
+    if @principal
+      sasl = true
+      config[:"sasl.mechanisms"] = "GSSAPI"
+      config[:"sasl.kerberos.principal"] = @principal
+      config[:"sasl.kerberos.service.name"] = @service_name if @service_name
+      config[:"sasl.kerberos.keytab"] = @keytab if @keytab
+    end
+
+    if ssl && sasl
+      security_protocol = "SASL_SSL"
+    elsif ssl && !sasl
+      security_protocol = "SSL"
+    elsif !ssl && sasl
+      security_protocol = "SASL_PLAINTEXT"
+    else
+      security_protocol = "PLAINTEXT"
+    end
+    config[:"security.protocol"] = security_protocol
+
+    config[:"compression.codec"] = @compression_codec if @compression_codec
+    config[:"message.send.max.retries"] = @max_send_retries if @max_send_retries
+    config[:"request.required.acks"] = @required_acks if @required_acks
+    config[:"request.timeout.ms"] = @ack_timeout * 1000 if @ack_timeout
+    config[:"queue.buffering.max.ms"] = @rdkafka_buffering_max_ms if @rdkafka_buffering_max_ms
+    config[:"queue.buffering.max.messages"] = @rdkafka_buffering_max_messages if @rdkafka_buffering_max_messages
+    config[:"message.max.bytes"] = @rdkafka_message_max_bytes if @rdkafka_message_max_bytes
+    config[:"batch.num.messages"] = @rdkafka_message_max_num if @rdkafka_message_max_num
+
+    @rdkafka_options.each { |k, v|
+      config[k.to_sym] = v
+    }
+
+    config
+  end
+
+  def start
+    super
+  end
+
+  def multi_workers_ready?
+    true
+  end
+
+  def shutdown
+    super
+    shutdown_producers
+  end
+
+  def shutdown_producers
+    @producers_mutex.synchronize {
+      shutdown_threads = @producers.map { |key, producer|
+        th = Thread.new {
+          unless producer.close(10)
+            log.warn("Queue is forcefully closed after 10 seconds wait")
+          end
+        }
+        th.abort_on_exception = true
+        th
+      }
+      shutdown_threads.each { |th| th.join }
+      @producers = {}
+    }
+  end
+
+  def get_producer
+    @producers_mutex.synchronize {
+      producer = @producers[Thread.current.object_id]
+      unless producer
+        producer = @rdkafka.producer
+        @producers[Thread.current.object_id] = producer
+      end
+      producer
+    }
+  end
+
+  def emit(tag, es, chain)
+    super(tag, es, chain, tag)
+  end
+
+  def format_stream(tag, es)
+    es.to_msgpack_stream
+  end
+
+  def setup_formatter(conf)
+    if @output_data_type == 'json'
+      begin
+        require 'oj'
+        Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
+        Proc.new { |tag, time, record| Oj.dump(record) }
+      rescue LoadError
+        require 'yajl'
+        Proc.new { |tag, time, record| Yajl::Encoder.encode(record) }
+      end
+    elsif @output_data_type == 'ltsv'
+      require 'ltsv'
+      Proc.new { |tag, time, record| LTSV.dump(record) }
+    elsif @output_data_type == 'msgpack'
+      require 'msgpack'
+      Proc.new { |tag, time, record| record.to_msgpack }
+    elsif @output_data_type =~ /^attr:(.*)$/
+      @custom_attributes = $1.split(',').map(&:strip).reject(&:empty?)
+      @custom_attributes.unshift('time') if @output_include_time
+      @custom_attributes.unshift('tag') if @output_include_tag
+      Proc.new { |tag, time, record|
+        @custom_attributes.map { |attr|
+          record[attr].nil? ? '' : record[attr].to_s
+        }.join(@f_separator)
+      }
+    else
+      @formatter = Fluent::Plugin.new_formatter(@output_data_type)
+      @formatter.configure(conf)
+      @formatter.method(:format)
+    end
+  end
+
+  def write(chunk)
+    tag = chunk.key
+    def_topic = @default_topic || tag
+
+    record_buf = nil
+    record_buf_bytes = nil
+
+    begin
+      chunk.msgpack_each.map { |time, record|
+        begin
+          if @output_include_time
+            if @time_format
+              record['time'.freeze] = Time.at(time).strftime(@time_format)
+            else
+              record['time'.freeze] = time
+            end
+          end
+
+          record['tag'] = tag if @output_include_tag
+          topic = (@exclude_topic_key ? record.delete('topic'.freeze) : record['topic'.freeze]) || def_topic
+          partition = (@exclude_partition ? record.delete('partition'.freeze) : record['partition'.freeze]) || @default_partition
+          message_key = (@exclude_message_key ? record.delete('message_key'.freeze) : record['message_key'.freeze]) || @default_message_key
+
+          record_buf = @formatter_proc.call(tag, time, record)
+          record_buf_bytes = record_buf.bytesize
+          if @max_send_limit_bytes && record_buf_bytes > @max_send_limit_bytes
+            log.warn "record size exceeds max_send_limit_bytes. Skip event:", :time => time, :record => record
+            next
+          end
+        rescue StandardError => e
+          log.warn "unexpected error during format record. Skip broken event:", :error => e.to_s, :error_class => e.class.to_s, :time => time, :record => record
+          next
+        end
+
+        producer = get_producer
+        handler = enqueue_with_retry(producer, topic, record_buf, message_key, partition)
+        handler
+      }.each { |handler|
+        handler.wait(@rdkafka_delivery_handle_poll_timeout) if @rdkafka_delivery_handle_poll_timeout != 0
+      }
+    end
+  rescue Exception => e
+    log.warn "Send exception occurred: #{e} at #{e.backtrace.first}"
+    # Raise exception to retry sendind messages
+    raise e
+  end
+
+  def enqueue_with_retry(producer, topic, record_buf, message_key, partition)
+    attempt = 0
+    loop do
+      begin
+        handler = producer.produce(topic: topic, payload: record_buf, key: message_key, partition: partition)
+        return handler
+      rescue Exception => e
+        if e.code == :queue_full
+          if attempt <= @max_enqueue_retries
+            log.warn "Failed to enqueue message; attempting retry #{attempt} of #{@max_enqueue_retries} after #{@enqueue_retry_backoff}s"
+            sleep @enqueue_retry_backoff
+            attempt += 1
+          else
+            raise "Failed to enqueue message although tried retry #{@max_enqueue_retries} times"
+          end
+        else
+          raise e
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.7.9
+  version: 0.8.0
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-09-12 00:00:00.000000000 Z
+date: 2018-10-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -51,20 +51,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.4.1
+        version: 0.7.1
     - - "<"
       - !ruby/object:Gem::Version
-        version: 0.7.0
+        version: 0.8.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.4.1
+        version: 0.7.1
     - - "<"
       - !ruby/object:Gem::Version
-        version: 0.7.0
+        version: 0.8.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -116,6 +116,7 @@ files:
 - lib/fluent/plugin/out_kafka.rb
 - lib/fluent/plugin/out_kafka2.rb
 - lib/fluent/plugin/out_kafka_buffered.rb
+- lib/fluent/plugin/out_rdkafka.rb
 - test/helper.rb
 - test/plugin/test_out_kafka.rb
 homepage: https://github.com/fluent/fluent-plugin-kafka