fluent-plugin-kafka 0.2.2 → 0.3.0.rc1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 95430357052a2b7305214ffa1f5c0ad426e52914
-  data.tar.gz: 57736e74a0bc9b65441bc41086f7b9875e72a590
+  metadata.gz: 64803e0c72fc4c81fb01fb053436a944c768ba6a
+  data.tar.gz: 602f630699406dfeef4712584cbdb95f1f06483f
 SHA512:
-  metadata.gz: a7f89337ddc1f28e8762de822738d467d16ef454a0bf64570f8bfbd13420dbbc53a2477b4cce836c8234d7aee96726a13d0fbf12ac9e33216f6cf82eea1ee174
-  data.tar.gz: 5568aec86102a4c40aae4e1e8b43c5e1829811e9014589e45981491fc2b1fece473f0042f1b7188e3b09d678eff8cb77e0ac63de7a00d09fe34f7b9bfe80b12f
+  metadata.gz: 9e80898dbac167495ceaf99ef48c8c077974190da51173e97a87c69e7064da9213781d2b9762a68f12f4fd87ba603a5b9fca92f562c2880a109e13104445c890
+  data.tar.gz: 88c81bc3ca6e27bead1010ff6112cf1d80fcd8de665e46c030eada336704d0ea33effc2bc101aa96622463b069bdc8aeffff2c3222395f6321b380bff0e9a8e3

data/fluent-plugin-kafka.gemspec

@@ -12,12 +12,11 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version = '0.2.2'
+  gem.version = '0.3.0.rc1'
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
-  gem.add_dependency 'poseidon_cluster'
   gem.add_dependency 'ltsv'
   gem.add_dependency 'zookeeper'
-  gem.add_dependency 'ruby-kafka', '~> 0.3.9'
+  gem.add_dependency 'ruby-kafka', '~> 0.3.11'
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "test-unit", ">= 3.0.8"
 end

data/lib/fluent/plugin/in_kafka.rb

@@ -1,4 +1,6 @@
 require 'fluent/input'
+require 'fluent/plugin/kafka_plugin_util'
+
 module Fluent
 
 class KafkaInput < Input
@@ -8,10 +10,12 @@ class KafkaInput < Input
                :desc => "Supported format: (json|text|ltsv|msgpack)"
   config_param :message_key, :string, :default => 'message',
                :desc => "For 'text' format only."
-  config_param :host, :string, :default => 'localhost',
+  config_param :host, :string, :default => nil,
                :desc => "Broker host"
-  config_param :port, :integer, :default => 9092,
+  config_param :port, :integer, :default => nil,
                :desc => "Broker port"
+  config_param :brokers, :string, :default => 'localhost:9092',
+               :desc => "List of broker-host:port, separate with comma, must set."
   config_param :interval, :integer, :default => 1, # seconds
                :desc => "Interval (Unit: seconds)"
   config_param :topics, :string, :default => nil,
@@ -30,24 +34,25 @@ class KafkaInput < Input
   config_param :offset_zookeeper, :string, :default => nil
   config_param :offset_zk_root_node, :string, :default => '/fluent-plugin-kafka'
 
-  # poseidon PartitionConsumer options
+  # Kafka#fetch_messages options
   config_param :max_bytes, :integer, :default => nil,
                :desc => "Maximum number of bytes to fetch."
-  config_param :max_wait_ms, :integer, :default => nil,
+  config_param :max_wait_time, :integer, :default => nil,
                :desc => "How long to block until the server sends us data."
   config_param :min_bytes, :integer, :default => nil,
                :desc => "Smallest amount of data the server should send us."
   config_param :socket_timeout_ms, :integer, :default => nil,
                :desc => "How long to wait for reply from server. Should be higher than max_wait_ms."
 
+  include KafkaPluginUtil::SSLSettings
+
   unless method_defined?(:router)
     define_method("router") { Fluent::Engine }
   end
 
   def initialize
     super
-    require 'poseidon'
-    require 'zookeeper'
+    require 'kafka'
   end
 
   def configure(conf)
@@ -73,38 +78,90 @@ class KafkaInput < Input
       raise ConfigError, "kafka: 'topics' or 'topic element' is a require parameter"
     end
 
+    # For backward compatibility
+    @brokers = case
+               when @host && @port
+                 ["#{@host}:#{@port}"]
+               when @host
+                 ["#{@host}:9092"]
+               when @port
+                 ["localhost:#{@port}"]
+               else
+                 @brokers
+               end
+
+    if conf['max_wait_ms']
+      log.warn "'max_wait_ms' parameter is deprecated. Use second unit 'max_wait_time' instead"
+      @max_wait_time = conf['max_wait_ms'].to_i / 1000
+    end
+
+    @max_wait_time = @interval if @max_wait_time.nil?
+
+    require 'zookeeper' if @offset_zookeeper
+
+    @parser_proc = setup_parser
+  end
+
+  def setup_parser
     case @format
     when 'json'
       require 'yajl'
+      Proc.new { |msg, te|
+        r = Yajl::Parser.parse(msg.value)
+        add_offset_in_hash(r, te, msg.offset) if @add_offset_in_record
+        r
+      }
     when 'ltsv'
       require 'ltsv'
+      Proc.new { |msg, te|
+        r = LTSV.parse(msg.value).first
+        add_offset_in_hash(r, te, msg.offset) if @add_offset_in_record
+        r
+      }
     when 'msgpack'
       require 'msgpack'
+      Proc.new { |msg, te|
+        r = MessagePack.unpack(msg.value)
+        add_offset_in_hash(r, te, msg.offset) if @add_offset_in_record
+        r
+      }
+    when 'text'
+      Proc.new { |msg, te|
+        r = {@message_key => msg.value}
+        add_offset_in_hash(r, te, msg.offset) if @add_offset_in_record
+        r
+      }
     end
   end
 
+  def add_offset_in_hash(hash, te, offset)
+    hash['kafka_topic'.freeze] = te.topic
+    hash['kafka_partition'.freeze] = te.partition
+    hash['kafka_offset'.freeze] = offset
+  end
+
   def start
     super
+
     @loop = Coolio::Loop.new
     opt = {}
     opt[:max_bytes] = @max_bytes if @max_bytes
-    opt[:max_wait_ms] = @max_wait_ms if @max_wait_ms
+    opt[:max_wait_time] = @max_wait_time if @max_wait_time
     opt[:min_bytes] = @min_bytes if @min_bytes
-    opt[:socket_timeout_ms] = @socket_timeout_ms if @socket_timeout_ms
 
+    @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id,
+                       ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+                       ssl_client_cert: read_ssl_file(@ssl_client_cert),
+                       ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key))
     @zookeeper = Zookeeper.new(@offset_zookeeper) if @offset_zookeeper
 
     @topic_watchers = @topic_list.map {|topic_entry|
       offset_manager = OffsetManager.new(topic_entry, @zookeeper, @offset_zk_root_node) if @offset_zookeeper
       TopicWatcher.new(
         topic_entry,
-        @host,
-        @port,
-        @client_id,
+        @kafka,
         interval,
-        @format,
-        @message_key,
-        @add_offset_in_record,
+        @parser_proc,
         @add_prefix,
         @add_suffix,
         offset_manager,
@@ -120,26 +177,24 @@ class KafkaInput < Input
   def shutdown
     @loop.stop
     @zookeeper.close! if @zookeeper
+    @thread.join
+    @kafka.close
     super
   end
 
   def run
     @loop.run
-  rescue
-    $log.error "unexpected error", :error=>$!.to_s
+  rescue => e
+    $log.error "unexpected error", :error => e.to_s
     $log.error_backtrace
   end
 
   class TopicWatcher < Coolio::TimerWatcher
-    def initialize(topic_entry, host, port, client_id, interval, format, message_key, add_offset_in_record, add_prefix, add_suffix, offset_manager, router, options={})
+    def initialize(topic_entry, kafka, interval, parser, add_prefix, add_suffix, offset_manager, router, options={})
       @topic_entry = topic_entry
-      @host = host
-      @port = port
-      @client_id = client_id
+      @kafka = kafka
       @callback = method(:consume)
-      @format = format
-      @message_key = message_key
-      @add_offset_in_record = add_offset_in_record
+      @parser = parser
       @add_prefix = add_prefix
       @add_suffix = add_suffix
       @options = options
@@ -150,98 +205,53 @@ class KafkaInput < Input
       if @topic_entry.offset == -1 && offset_manager
         @next_offset = offset_manager.next_offset
       end
-      @consumer = create_consumer(@next_offset)      
+      @fetch_args = {
+        topic: @topic_entry.topic,
+        partition: @topic_entry.partition,
+      }.merge(@options)
 
       super(interval, true)
     end
 
     def on_timer
       @callback.call
-    rescue
-        # TODO log?
-        $log.error $!.to_s
-        $log.error_backtrace
+    rescue => e
+      # TODO log?
+      $log.error e.to_s
+      $log.error_backtrace
     end
 
     def consume
+      offset = @next_offset
+      @fetch_args[:offset] = offset
+      messages = @kafka.fetch_messages(@fetch_args)
+
+      return if messages.size.zero?
+
       es = MultiEventStream.new
       tag = @topic_entry.topic
       tag = @add_prefix + "." + tag if @add_prefix
       tag = tag + "." + @add_suffix if @add_suffix
 
-      if @offset_manager && @consumer.next_offset != @next_offset
-        @consumer = create_consumer(@next_offset)
-      end
-
-      @consumer.fetch.each { |msg|
+      messages.each { |msg|
         begin
-          msg_record = parse_line(msg.value)
-          msg_record = decorate_offset(msg_record, msg.offset) if @add_offset_in_record
-          es.add(Engine.now, msg_record)
-        rescue
-          $log.warn msg_record.to_s, :error=>$!.to_s
+          es.add(Engine.now, @parser.call(msg, @topic_entry))
+        rescue => e
+          $log.warn "parser error in #{@topic_entry.topic}/#{@topic_entry.partition}", :error => e.to_s, :value => msg.value, :offset => msg.offset
           $log.debug_backtrace
         end
       }
+      offset = messages.last.offset + 1
 
       unless es.empty?
         @router.emit_stream(tag, es)
 
         if @offset_manager
-          next_offset = @consumer.next_offset
-          @offset_manager.save_offset(next_offset)
-          @next_offset = next_offset
+          @offset_manager.save_offset(offset)
         end
+        @next_offset = offset
       end
     end
-
-    def create_consumer(offset)
-      @consumer.close if @consumer
-      Poseidon::PartitionConsumer.new(
-        @client_id,             # client_id
-        @host,                  # host
-        @port,                  # port
-        @topic_entry.topic,     # topic
-        @topic_entry.partition, # partition
-        offset,                 # offset
-        @options                # options
-      )
-    end
-
-    def parse_line(record)
-      case @format
-      when 'json'
-        Yajl::Parser.parse(record)
-      when 'ltsv'
-        LTSV.parse(record)
-      when 'msgpack'
-        MessagePack.unpack(record)
-      when 'text'
-        {@message_key => record}
-      end
-    end
-
-    def decorate_offset(record, offset)
-      case @format
-      when 'json'
-        add_offset_in_hash(record, @topic_entry.topic, @topic_entry.partition, offset)
-      when 'ltsv'
-        record.each { |line|
-          add_offset_in_hash(line, @topic_entry.topic, @topic_entry.partition, offset)
-        }
-      when 'msgpack'
-        add_offset_in_hash(record, @topic_entry.topic, @topic_entry.partition, offset)
-      when 'text'
-        add_offset_in_hash(record, @topic_entry.topic, @topic_entry.partition, offset)
-      end
-      record
-    end
-
-    def add_offset_in_hash(hash, topic, partition, offset)
-      hash['kafka_topic'] = topic
-      hash['kafka_partition'] = partition
-      hash['kafka_offset'] = offset
-    end
   end
 
   class TopicEntry

data/lib/fluent/plugin/in_kafka_group.rb

@@ -1,19 +1,17 @@
 require 'fluent/input'
+require 'fluent/plugin/kafka_plugin_util'
+
 module Fluent
 
 class KafkaGroupInput < Input
   Plugin.register_input('kafka_group', self)
 
-  config_param :brokers, :string,
-               :desc => "List of broker-host:port, separate with comma, must set."
-  config_param :zookeepers, :string,
+  config_param :brokers, :string, :default => 'localhost:9092',
                :desc => "List of broker-host:port, separate with comma, must set."
-  config_param :consumer_group, :string, :default => nil,
+  config_param :consumer_group, :string,
                :desc => "Consumer group name, must set."
   config_param :topics, :string,
                :desc => "Listening topics(separate with comma',')."
-  config_param :interval, :integer, :default => 1, # seconds
-               :desc => "Interval (Unit: seconds)"
   config_param :format, :string, :default => 'json',
                :desc => "Supported format: (json|text|ltsv|msgpack)"
   config_param :message_key, :string, :default => 'message',
@@ -23,15 +21,21 @@ class KafkaGroupInput < Input
   config_param :add_suffix, :string, :default => nil,
                :desc => "Tag suffix (Optional)"
 
-  # poseidon PartitionConsumer options
-  config_param :max_bytes, :integer, :default => nil,
-               :desc => "Maximum number of bytes to fetch."
-  config_param :max_wait_ms, :integer, :default => nil,
+  # Kafka consumer options
+  config_param :max_wait_time, :integer, :default => nil,
                :desc => "How long to block until the server sends us data."
   config_param :min_bytes, :integer, :default => nil,
                :desc => "Smallest amount of data the server should send us."
-  config_param :socket_timeout_ms, :integer, :default => nil,
-               :desc => "How long to wait for reply from server. Should be higher than max_wait_ms."
+  config_param :session_timeout, :integer, :default => nil,
+               :desc => "The number of seconds after which, if a client hasn't contacted the Kafka cluster"
+  config_param :offset_commit_interval, :integer, :default => nil,
+               :desc => "The interval between offset commits, in seconds"
+  config_param :offset_commit_threshold, :integer, :default => nil,
+               :desc => "The number of messages that can be processed before their offsets are committed"
+  config_param :start_from_beginning, :bool, :default => true,
+               :desc => "Whether to start from the beginning of the topic or just subscribe to new messages being produced"
+
+  include KafkaPluginUtil::SSLSettings
 
   unless method_defined?(:router)
     define_method("router") { Fluent::Engine }
@@ -39,7 +43,7 @@ class KafkaGroupInput < Input
 
   def initialize
     super
-    require 'poseidon_cluster'
+    require 'kafka'
   end
 
   def _config_to_array(config)
@@ -54,125 +58,89 @@ class KafkaGroupInput < Input
 
   def configure(conf)
     super
-    @broker_list = _config_to_array(@brokers)
-    @zookeeper_list = _config_to_array(@zookeepers)
-    @topic_list = _config_to_array(@topics)
 
-    unless @consumer_group
-      raise ConfigError, "kafka_group: 'consumer_group' is a required parameter"
+    $log.info "Will watch for topics #{@topics} at brokers " \
+              "#{@brokers} and '#{@consumer_group}' group"
+
+    @topics = _config_to_array(@topics)
+
+    if conf['max_wait_ms']
+      log.warn "'max_wait_ms' parameter is deprecated. Use second unit 'max_wait_time' instead"
+      @max_wait_time = conf['max_wait_ms'].to_i / 1000
     end
-    $log.info "Will watch for topics #{@topic_list} at brokers " \
-              "#{@broker_list}, zookeepers #{@zookeeper_list} and group " \
-              "'#{@consumer_group}'"
 
+    @parser_proc = setup_parser
+  end
+
+  def setup_parser
     case @format
     when 'json'
       require 'yajl'
+      Proc.new { |msg| Yajl::Parser.parse(msg.value) }
     when 'ltsv'
       require 'ltsv'
+      Proc.new { |msg| LTSV.parse(msg.value).first }
     when 'msgpack'
       require 'msgpack'
+      Proc.new { |msg| MessagePack.unpack(msg.value) }
+    when 'text'
+      Proc.new { |msg| {@message_key => msg.value} }
     end
   end
 
   def start
     super
-    @loop = Coolio::Loop.new
-    opt = {}
-    opt[:max_bytes] = @max_bytes if @max_bytes
-    opt[:max_wait_ms] = @max_wait_ms if @max_wait_ms
-    opt[:min_bytes] = @min_bytes if @min_bytes
-    opt[:socket_timeout_ms] = @socket_timeout_ms if @socket_timeout_ms
-
-    @topic_watchers = @topic_list.map {|topic|
-      TopicWatcher.new(topic, @broker_list, @zookeeper_list, @consumer_group,
-                       interval, @format, @message_key, @add_prefix,
-                       @add_suffix, router, opt)
-    }
-    @topic_watchers.each {|tw|
-      tw.attach(@loop)
+
+    consumer_opts = {:group_id => @consumer_group}
+    consumer_opts[:session_timeout] = @session_timeout if @session_timeout
+    consumer_opts[:offset_commit_interval] = @offset_commit_interval if @offset_commit_interval
+    consumer_opts[:offset_commit_threshold] = @offset_commit_threshold if @offset_commit_threshold
+
+    @fetch_opts = {}
+    @fetch_opts[:max_wait_time] = @max_wait_time if @max_wait_time
+    @fetch_opts[:min_bytes] = @min_bytes if @min_bytes
+
+    @kafka = Kafka.new(seed_brokers: @brokers,
+                       ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+                       ssl_client_cert: read_ssl_file(@ssl_client_cert),
+                       ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key))
+    @consumer = @kafka.consumer(consumer_opts)
+    @topics.each { |topic|
+      @consumer.subscribe(topic, start_from_beginning: @start_from_beginning)
     }
     @thread = Thread.new(&method(:run))
   end
 
   def shutdown
-    @loop.stop
+    @consumer.stop
+    @thread.join
+    @kafka.close
     super
   end
 
   def run
-    @loop.run
-  rescue
-    $log.error "unexpected error", :error=>$!.to_s
-    $log.error_backtrace
-  end
-
-  class TopicWatcher < Coolio::TimerWatcher
-    def initialize(topic, broker_list, zookeeper_list, consumer_group,
-                   interval, format, message_key, add_prefix, add_suffix,
-                   router, options)
-      @topic = topic
-      @callback = method(:consume)
-      @format = format
-      @message_key = message_key
-      @add_prefix = add_prefix
-      @add_suffix = add_suffix
-      @router = router
-
-      @consumer = Poseidon::ConsumerGroup.new(
-        consumer_group,
-        broker_list,
-        zookeeper_list,
-        topic,
-        options
-      )
-
-      super(interval, true)
-    end
-
-    def on_timer
-      @callback.call
-    rescue
-        # TODO log?
-        $log.error $!.to_s
-        $log.error_backtrace
-    end
-
-    def consume
+    @consumer.each_batch(@fetch_opts) { |batch|
       es = MultiEventStream.new
-      tag = @topic
+      tag = batch.topic
       tag = @add_prefix + "." + tag if @add_prefix
       tag = tag + "." + @add_suffix if @add_suffix
 
-      @consumer.fetch do |partition, bulk|
-        bulk.each do |msg|
-          begin
-            msg_record = parse_line(msg.value)
-            es.add(Engine.now, msg_record)
-          rescue
-            $log.warn msg_record.to_s, :error=>$!.to_s
-            $log.debug_backtrace
-          end
+      batch.messages.each { |msg|
+        begin
+          es.add(Engine.now, @parser_proc.call(msg))
+        rescue => e
+          $log.warn "parser error in #{batch.topic}/#{batch.partition}", :error => e.to_s, :value => msg.value, :offset => msg.offset
+          $log.debug_backtrace
         end
-      end
+      }
 
       unless es.empty?
-        @router.emit_stream(tag, es)
+        router.emit_stream(tag, es)
       end
-    end
-
-    def parse_line(record)
-      case @format
-      when 'json'
-        Yajl::Parser.parse(record)
-      when 'ltsv'
-        LTSV.parse(record)
-      when 'msgpack'
-        MessagePack.unpack(record)
-      when 'text'
-        {@message_key => record}
-      end
-    end
+    }
+  rescue => e
+    $log.error "unexpected error", :error => e.to_s
+    $log.error_backtrace
   end
 end
 

data/lib/fluent/plugin/kafka_plugin_util.rb

@@ -0,0 +1,21 @@
+module Fluent
+  module KafkaPluginUtil
+    module SSLSettings
+      def self.included(klass)
+        klass.instance_eval {
+          config_param :ssl_ca_cert, :string, :default => nil,
+                       :desc => "a PEM encoded CA cert to use with and SSL connection."
+          config_param :ssl_client_cert, :string, :default => nil,
+                       :desc => "a PEM encoded client cert to use with and SSL connection. Must be used in combination with ssl_client_cert_key."
+          config_param :ssl_client_cert_key, :string, :default => nil,
+                       :desc => "a PEM encoded client cert key to use with and SSL connection. Must be used in combination with ssl_client_cert."
+        }
+      end
+
+      def read_ssl_file(path)
+        return nil if path.nil?
+        File.read(path)
+      end
+    end
+  end
+end

data/lib/fluent/plugin/out_kafka.rb

@@ -3,7 +3,10 @@ class Fluent::KafkaOutput < Fluent::Output
 
   def initialize
     super
-    require 'poseidon'
+
+    require 'kafka'
+
+    @kafka = nil
   end
 
   config_param :brokers, :string, :default => 'localhost:9092',
@@ -25,14 +28,22 @@ DESC
   config_param :output_include_tag, :bool, :default => false
   config_param :output_include_time, :bool, :default => false
 
+  # https://github.com/zendesk/ruby-kafka#encryption-and-authentication-using-ssl
+  config_param :ssl_ca_cert, :string, :default => nil,
+               :desc => "a PEM encoded CA cert to use with and SSL connection."
+  config_param :ssl_client_cert, :string, :default => nil,
+               :desc => "a PEM encoded client cert to use with and SSL connection. Must be used in combination with ssl_client_cert_key."
+  config_param :ssl_client_cert_key, :string, :default => nil,
+               :desc => "a PEM encoded client cert key to use with and SSL connection. Must be used in combination with ssl_client_cert."
+
   # poseidon producer options
-  config_param :max_send_retries, :integer, :default => 3,
+  config_param :max_send_retries, :integer, :default => 1,
                :desc => "Number of times to retry sending of messages to a leader."
   config_param :required_acks, :integer, :default => 0,
                :desc => "The number of acks required per request."
-  config_param :ack_timeout_ms, :integer, :default => 1500,
+  config_param :ack_timeout, :integer, :default => nil,
                :desc => "How long the producer waits for acks."
-  config_param :compression_codec, :string, :default => 'none',
+  config_param :compression_codec, :string, :default => nil,
                :desc => "The codec the producer uses to compress messages."
 
   config_param :time_format, :string, :default => nil
@@ -40,13 +51,11 @@ DESC
   attr_accessor :output_data_type
   attr_accessor :field_separator
 
-  @seed_brokers = []
-
   unless method_defined?(:log)
     define_method("log") { $log }
   end
 
-  def refresh_producer()
+  def refresh_client
     if @zookeeper
       @seed_brokers = []
       z = Zookeeper.new(@zookeeper)
@@ -59,8 +68,9 @@ DESC
     end
     begin
       if @seed_brokers.length > 0
-        @producer = Poseidon::Producer.new(@seed_brokers, @client_id, :max_send_retries => @max_send_retries, :required_acks => @required_acks, :ack_timeout_ms => @ack_timeout_ms, :compression_codec => @compression_codec.to_sym)
-        log.info "initialized producer #{@client_id}"
+        @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+                           ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key))
+        log.info "initialized kafka producer: #{@client_id}"
       else
         log.warn "No brokers found on Zookeeper"
       end
@@ -69,25 +79,24 @@ DESC
     end
   end
 
+  def read_ssl_file(path)
+    return nil if path.nil?
+    File.read(path)
+  end
+
   def configure(conf)
     super
+
     if @zookeeper
       require 'zookeeper'
-      require 'yajl'
     else
       @seed_brokers = @brokers.match(",").nil? ? [@brokers] : @brokers.split(",")
       log.info "brokers has been set directly: #{@seed_brokers}"
     end
-    if @compression_codec == 'snappy'
-      require 'snappy'
-    end
-    case @output_data_type
-    when 'json'
-      require 'yajl'
-    when 'ltsv'
-      require 'ltsv'
-    when 'msgpack'
-      require 'msgpack'
+
+    if conf['ack_timeout_ms']
+      log.warn "'ack_timeout_ms' parameter is deprecated. Use second unit 'ack_timeout' instead"
+      @ack_timeout = conf['ack_timeout_ms'].to_i / 1000
     end
 
     @f_separator = case @field_separator
@@ -97,56 +106,57 @@ DESC
                    else "\t"
                    end
 
-    @custom_attributes = if @output_data_type == 'json'
-                           nil
-                         elsif @output_data_type == 'ltsv'
-                           nil
-                         elsif @output_data_type == 'msgpack'
-                           nil
-                         elsif @output_data_type =~ /^attr:(.*)$/
-                           $1.split(',').map(&:strip).reject(&:empty?)
-                         else
-                           @formatter = Fluent::Plugin.new_formatter(@output_data_type)
-                           @formatter.configure(conf)
-                           nil
-                         end
+    @formatter_proc = setup_formatter(conf)
 
+    @producer_opts = {max_retries: @max_send_retries, required_acks: @required_acks}
+    @producer_opts[:ack_timeout] = @ack_timeout if @ack_timeout
+    @producer_opts[:compression_codec] = @compression_codec.to_sym if @compression_codec
   end
 
   def start
     super
-    refresh_producer()
+    refresh_client
   end
 
   def shutdown
     super
+    @kafka = nil
   end
 
-  def parse_record(record)
-    if @custom_attributes.nil?
-      case @output_data_type
-      when 'json'
-        Yajl::Encoder.encode(record)
-      when 'ltsv'
-        LTSV.dump(record)
-      when 'msgpack'
-        record.to_msgpack
-      else
-        record.to_s
-      end
-    else
+  def setup_formatter(conf)
+    if @output_data_type == 'json'
+      require 'yajl'
+      Proc.new { |tag, time, record| Yajl::Encoder.encode(record) }
+    elsif @output_data_type == 'ltsv'
+      require 'ltsv'
+      Proc.new { |tag, time, record| LTSV.dump(record) }
+    elsif @output_data_type == 'msgpack'
+      require 'msgpack'
+      Proc.new { |tag, time, record| record.to_msgpack }
+    elsif @output_data_type =~ /^attr:(.*)$/
+      @custom_attributes = $1.split(',').map(&:strip).reject(&:empty?)
       @custom_attributes.unshift('time') if @output_include_time
       @custom_attributes.unshift('tag') if @output_include_tag
-      @custom_attributes.map { |attr|
-        record[attr].nil? ? '' : record[attr].to_s
-      }.join(@f_separator)
+      Proc.new { |tag, time, record|
+        @custom_attributes.map { |attr|
+          record[attr].nil? ? '' : record[attr].to_s
+        }.join(@f_separator)
+      }
+    else
+      @formatter = Fluent::Plugin.new_formatter(@output_data_type)
+      @formatter.configure(conf)
+      @formatter.method(:format)
     end
   end
 
   def emit(tag, es, chain)
     begin
       chain.next
-      es.each do |time,record|
+
+      # out_kafka is mainly for testing so don't need the performance unlike out_kafka_buffered.
+      producer = @kafka.producer(@producer_opts)
+
+      es.each do |time, record|
         if @output_include_time
           if @time_format
             record['time'] = Time.at(time).strftime(@time_format)
@@ -155,17 +165,20 @@ DESC
           end
         end
         record['tag'] = tag if @output_include_tag
-        topic = record['topic'] || self.default_topic || tag
+        topic = record['topic'] || @default_topic || tag
         partition_key = record['partition_key'] || @default_partition_key
-        value = @formatter.nil? ? parse_record(record) : @formatter.format(tag, time, record)
-        log.trace("message send to #{topic} with key: #{partition_key} and value: #{value}.")
-        message = Poseidon::MessageToSend.new(topic, value, partition_key)
-        @producer.send_messages([message])
+        value = @formatter_proc.call(tag, time, record)
+
+        log.on_trace { log.trace("message send to #{topic} with key: #{partition_key} and value: #{value}.") }
+        producer.produce(value, topic: topic, partition_key: partition_key)
       end
+
+      producer.deliver_messages
+      producer.shutdown
     rescue Exception => e
-      log.warn("Send exception occurred: #{e}")
-      @producer.close if @producer
-      refresh_producer()
+      log.warn "Send exception occurred: #{e}"
+      producer.shutdown if producer
+      refresh_client
       raise e
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.0.rc1
 platform: ruby
 authors:
 - Hidemasa Togashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-21 00:00:00.000000000 Z
+date: 2016-08-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -30,20 +30,6 @@ dependencies:
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
-- !ruby/object:Gem::Dependency
-  name: poseidon_cluster
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: ltsv
   requirement: !ruby/object:Gem::Requirement
@@ -78,14 +64,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.9
+        version: 0.3.11
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.3.9
+        version: 0.3.11
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -130,6 +116,7 @@ files:
 - fluent-plugin-kafka.gemspec
 - lib/fluent/plugin/in_kafka.rb
 - lib/fluent/plugin/in_kafka_group.rb
+- lib/fluent/plugin/kafka_plugin_util.rb
 - lib/fluent/plugin/kafka_producer_ext.rb
 - lib/fluent/plugin/out_kafka.rb
 - lib/fluent/plugin/out_kafka_buffered.rb
@@ -149,9 +136,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.5.1