fluent-plugin-kafka-zendesk 0.1.4

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 37289c076743d4a6760c53274a86604647975e04
+  data.tar.gz: 8a1317287c6d1e4ac03c34f8aa032d1b143320f4
+SHA512:
+  metadata.gz: 28c1cdaf406bef12651ca162f5b3638f7f98eb77c9251848c064a9738d7674b6048d79d96b0e1a2cabe14b22935c1d457e646c4a1f31a586a349843ee058b7d0
+  data.tar.gz: 5e6bbd8cba77e7a9bbe090a36e65f852a6cf46212b585c001854a3260d7a1924e05c5e3d0a2804bce861a2f647f7ce7a69e89d2eaff36ab8b27610bde3cac900

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kafka-poseidon.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2014 htgc
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,196 @@
+# Fluent::Plugin::Kafka
+
+TODO: Write a gem description
+TODO: Also, I need to write tests
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'fluent-plugin-kafka'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fluent-plugin-kafka
+
+## Usage
+
+### Input plugin (@type 'kafka')
+
+    <source>
+      @type  kafka
+      host   <broker host>
+      port   <broker port: default=9092>
+      topics <listening topics(separate with comma',')>
+      format <input text type (text|json|ltsv|msgpack)>
+      message_key <key (Optional, for text format only, default is message)>
+      add_prefix <tag prefix (Optional)>
+      add_suffix <tag suffix (Optional)>
+      max_bytes           (integer)    :default => nil (Use default of Poseidon)
+      max_wait_ms         (integer)    :default => nil (Use default of Poseidon)
+      min_bytes           (integer)    :default => nil (Use default of Poseidon)
+      socket_timeout_ms   (integer)    :default => nil (Use default of Poseidon)
+    </source>
+
+Supports following Poseidon::PartitionConsumer options.
+
+- max_bytes — default: 1048576 (1MB) — Maximum number of bytes to fetch
+- max_wait_ms — default: 100 (100ms) — How long to block until the server sends us data.
+- min_bytes — default: 1 (Send us data as soon as it is ready) — Smallest amount of data the server should send us.
+- socket_timeout_ms - default: 10000 (10s) - How long to wait for reply from server. Should be higher than max_wait_ms.
+
+Supports a start of processing from the assigned offset for specific topics.
+
+    <source>
+      @type  kafka
+      host   <broker host>
+      port   <broker port: default=9092>
+      format <input text type (text|json|ltsv|msgpack)>
+      <topic>
+        topic       <listening topic>
+        partition   <listening partition: default=0>
+        offset      <listening start offset: default=-1>
+      </topic>
+      <topic>
+        topic       <listening topic>
+        partition   <listening partition: default=0>
+        offset      <listening start offset: default=-1>
+      </topic>
+    </source>
+
+See also [Poseidon::PartitionConsumer](http://www.rubydoc.info/github/bpot/poseidon/Poseidon/PartitionConsumer) for more detailed documentation about Poseidon.
+
+### Input plugin (@type 'kafka_group', supports kafka group)
+
+    <source>
+      @type   kafka_group
+      brokers <list of broker-host:port, separate with comma, must set>
+      zookeepers <list of broker-host:port, separate with comma, must set>
+      zookeeper_path <broker path in zookeeper> :default => /brokers/ids # Set path in zookeeper for brokers
+      consumer_group <consumer group name, must set>
+      topics <listening topics(separate with comma',')>
+      format <input text type (text|json|ltsv|msgpack)>
+      message_key <key (Optional, for text format only, default is message)>
+      add_prefix <tag prefix (Optional)>
+      add_suffix <tag suffix (Optional)>
+      max_bytes           (integer)    :default => nil (Use default of Poseidon)
+      max_wait_ms         (integer)    :default => nil (Use default of Poseidon)
+      min_bytes           (integer)    :default => nil (Use default of Poseidon)
+      socket_timeout_ms   (integer)    :default => nil (Use default of Poseidon)
+    </source>
+
+Supports following Poseidon::PartitionConsumer options.
+
+- max_bytes — default: 1048576 (1MB) — Maximum number of bytes to fetch
+- max_wait_ms — default: 100 (100ms) — How long to block until the server sends us data.
+- min_bytes — default: 1 (Send us data as soon as it is ready) — Smallest amount of data the server should send us.
+- socket_timeout_ms - default: 10000 (10s) - How long to wait for reply from server. Should be higher than max_wait_ms.
+
+See also [Poseidon::PartitionConsumer](http://www.rubydoc.info/github/bpot/poseidon/Poseidon/PartitionConsumer) for more detailed documentation about Poseidon.
+
+### Output plugin (non-buffered)
+
+    <match *.**>
+      @type               kafka
+
+      # Brokers: you can choose either brokers or zookeeper.
+      brokers             <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,.. # Set brokers directly
+      zookeeper           <zookeeper_host>:<zookeeper_port> # Set brokers via Zookeeper
+      zookeeper_path      <broker path in zookeeper> :default => /brokers/ids # Set path in zookeeper for kafka
+      default_topic       <output topic>
+      default_partition_key (string)   :default => nil
+      output_data_type    (json|ltsv|msgpack|attr:<record name>|<formatter name>)
+      output_include_tag  (true|false) :default => false
+      output_include_time (true|false) :default => false
+      max_send_retries    (integer)    :default => 3
+      required_acks       (integer)    :default => 0
+      ack_timeout_ms      (integer)    :default => 1500
+      compression_codec   (none|gzip|snappy) :default => none
+    </match>
+
+Supports following Poseidon::Producer options.
+
+- max_send_retries — default: 3 — Number of times to retry sending of messages to a leader.
+- required_acks — default: 0 — The number of acks required per request.
+- ack_timeout_ms — default: 1500 — How long the producer waits for acks.
+- compression_codec - default: none - The codec the producer uses to compress messages.
+
+See also [Poseidon::Producer](http://www.rubydoc.info/github/bpot/poseidon/Poseidon/Producer) for more detailed documentation about Poseidon.
+
+This plugin supports compression codec "snappy" also.
+Install snappy module before you use snappy compression.
+
+    $ gem install snappy
+
+#### Load balancing
+
+Messages will be sent broker in a round-robin manner as default by Poseidon, but you can set `default_partition_key` in config file to route messages to a specific broker.
+If key name `partition_key` exists in a message, this plugin set its value of partition_key as key.
+
+|default_partition_key|partition_key| behavior |
+|-|-|
+|Not set|Not exists| All messages are sent in round-robin |
+|Set| Not exists| All messages are sent to specific broker |
+|Not set| Exists | Messages which have partition_key record are sent to specific broker, others are sent in round-robin|
+|Set| Exists | Messages which have partition_key record are sent to specific broker with parition_key, others are sent to specific broker with default_parition_key|
+
+
+### Buffered output plugin
+
+    <match *.**>
+      @type               kafka_buffered
+
+      # Brokers: you can choose either brokers or zookeeper.
+      brokers             <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,.. # Set brokers directly
+      zookeeper           <zookeeper_host>:<zookeeper_port> # Set brokers via Zookeeper
+      zookeeper_path      <broker path in zookeeper> :default => /brokers/ids # Set path in zookeeper for kafka
+      default_topic       <output topic>
+      default_partition_key (string)   :default => nil
+      flush_interval      <flush interval (sec) :default => 60>
+      buffer_type         (file|memory)
+      output_data_type    (json|ltsv|msgpack|attr:<record name>|<formatter name>)
+      output_include_tag  (true|false) :default => false
+      output_include_time (true|false) :default => false
+      max_send_retries    (integer)    :default => 3
+      required_acks       (integer)    :default => 0
+      ack_timeout_ms      (integer)    :default => 1500
+      compression_codec   (none|gzip|snappy) :default => none
+    </match>
+
+Supports following Poseidon::Producer options.
+
+- max_send_retries — default: 3 — Number of times to retry sending of messages to a leader.
+- required_acks — default: 0 — The number of acks required per request.
+- ack_timeout_ms — default: 1500 — How long the producer waits for acks.
+- compression_codec - default: none - The codec the producer uses to compress messages.
+
+See also [Poseidon::Producer](http://www.rubydoc.info/github/bpot/poseidon/Poseidon/Producer) for more detailed documentation about Poseidon.
+
+This plugin supports compression codec "snappy" also.
+Install snappy module before you use snappy compression.
+
+    $ gem install snappy
+
+#### Load balancing
+
+Messages will be sent broker in a round-robin manner as default by Poseidon, but you can set `default_partition_key` in config file to route messages to a specific broker.
+If key name `partition_key` exists in a message, this plugin set its value of partition_key as key.
+
+|default_partition_key|partition_key| behavior |
+|-|-|
+|Not set|Not exists| All messages are sent in round-robin |
+|Set| Not exists| All messages are sent to specific broker |
+|Not set| Exists | Messages which have partition_key record are sent to specific broker, others are sent in round-robin|
+|Set| Exists | Messages which have partition_key record are sent to specific broker with parition_key, others are sent to specific broker with default_parition_key|
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,11 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test

data/fluent-plugin-kafka.gemspec

@@ -0,0 +1,22 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Hidemasa Togashi"]
+  gem.email         = ["togachiro@gmail.com"]
+  gem.description   = %q{Fluentd plugin for Apache Kafka > 0.8}
+  gem.summary       = %q{Fluentd plugin for Apache Kafka > 0.8}
+  gem.homepage      = "https://github.com/khouse/fluent-plugin-kafka"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "fluent-plugin-kafka-zendesk"
+  gem.require_paths = ["lib"]
+  gem.version = '0.1.4'
+  gem.add_dependency 'fluentd'
+  gem.add_dependency 'poseidon_cluster'
+  gem.add_dependency 'ltsv'
+  gem.add_dependency 'yajl-ruby'
+  gem.add_dependency 'msgpack'
+  gem.add_dependency 'zookeeper'
+end

data/lib/fluent/plugin/in_kafka.rb

@@ -0,0 +1,280 @@
+module Fluent
+
+class KafkaInput < Input
+  Plugin.register_input('kafka', self)
+
+  config_param :format, :string, :default => 'json',
+               :desc => "Supported format: (json|text|ltsv|msgpack)"
+  config_param :message_key, :string, :default => 'message',
+               :desc => "For 'text' format only."
+  config_param :host, :string, :default => 'localhost',
+               :desc => "Broker host"
+  config_param :port, :integer, :default => 9092,
+               :desc => "Broker port"
+  config_param :interval, :integer, :default => 1, # seconds
+               :desc => "Interval (Unit: seconds)"
+  config_param :topics, :string, :default => nil,
+               :desc => "Listening topics(separate with comma',')"
+  config_param :client_id, :string, :default => 'kafka'
+  config_param :partition, :integer, :default => 0,
+               :desc => "Listening partition"
+  config_param :offset, :integer, :default => -1,
+               :desc => "Listening start offset"
+  config_param :add_prefix, :string, :default => nil,
+               :desc => "Tag prefix"
+  config_param :add_suffix, :string, :default => nil,
+               :desc => "tag suffix"
+  config_param :add_offset_in_record, :bool, :default => false
+
+  config_param :offset_zookeeper, :string, :default => nil
+  config_param :offset_zk_root_node, :string, :default => '/fluent-plugin-kafka'
+
+  # poseidon PartitionConsumer options
+  config_param :max_bytes, :integer, :default => nil,
+               :desc => "Maximum number of bytes to fetch."
+  config_param :max_wait_ms, :integer, :default => nil,
+               :desc => "How long to block until the server sends us data."
+  config_param :min_bytes, :integer, :default => nil,
+               :desc => "Smallest amount of data the server should send us."
+  config_param :socket_timeout_ms, :integer, :default => nil,
+               :desc => "How long to wait for reply from server. Should be higher than max_wait_ms."
+
+  unless method_defined?(:router)
+    define_method("router") { Fluent::Engine }
+  end
+
+  def initialize
+    super
+    require 'poseidon'
+    require 'zookeeper'
+  end
+
+  def configure(conf)
+    super
+
+    @topic_list = []
+    if @topics
+      @topic_list = @topics.split(',').map { |topic|
+        TopicEntry.new(topic.strip, @partition, @offset)
+      }
+    else
+      conf.elements.select { |element| element.name == 'topic' }.each do |element|
+        unless element.has_key?('topic')
+          raise ConfigError, "kafka: 'topic' is a require parameter in 'topic element'."
+        end
+        partition = element.has_key?('partition') ? element['partition'].to_i : 0
+        offset = element.has_key?('offset') ? element['offset'].to_i : -1
+        @topic_list.push(TopicEntry.new(element['topic'], partition, offset))
+      end
+    end
+
+    if @topic_list.empty?
+      raise ConfigError, "kafka: 'topics' or 'topic element' is a require parameter"
+    end
+
+    case @format
+    when 'json'
+      require 'yajl'
+    when 'ltsv'
+      require 'ltsv'
+    when 'msgpack'
+      require 'msgpack'
+    end
+  end
+
+  def start
+    @loop = Coolio::Loop.new
+    opt = {}
+    opt[:max_bytes] = @max_bytes if @max_bytes
+    opt[:max_wait_ms] = @max_wait_ms if @max_wait_ms
+    opt[:min_bytes] = @min_bytes if @min_bytes
+    opt[:socket_timeout_ms] = @socket_timeout_ms if @socket_timeout_ms
+
+    @zookeeper = Zookeeper.new(@offset_zookeeper) if @offset_zookeeper
+
+    @topic_watchers = @topic_list.map {|topic_entry|
+      offset_manager = OffsetManager.new(topic_entry, @zookeeper, @offset_zk_root_node) if @offset_zookeeper
+      TopicWatcher.new(
+        topic_entry,
+        @host,
+        @port,
+        @client_id,
+        interval,
+        @format,
+        @message_key,
+        @add_offset_in_record,
+        @add_prefix,
+        @add_suffix,
+        offset_manager,
+        router,
+        opt)
+    }
+    @topic_watchers.each {|tw|
+      tw.attach(@loop)
+    }
+    @thread = Thread.new(&method(:run))
+  end
+
+  def shutdown
+    @loop.stop
+    @zookeeper.close! if @zookeeper
+  end
+
+  def run
+    @loop.run
+  rescue
+    $log.error "unexpected error", :error=>$!.to_s
+    $log.error_backtrace
+  end
+
+  class TopicWatcher < Coolio::TimerWatcher
+    def initialize(topic_entry, host, port, client_id, interval, format, message_key, add_offset_in_record, add_prefix, add_suffix, offset_manager, router, options={})
+      @topic_entry = topic_entry
+      @host = host
+      @port = port
+      @client_id = client_id
+      @callback = method(:consume)
+      @format = format
+      @message_key = message_key
+      @add_offset_in_record = add_offset_in_record
+      @add_prefix = add_prefix
+      @add_suffix = add_suffix
+      @options = options
+      @offset_manager = offset_manager
+      @router = router
+
+      @next_offset = @topic_entry.offset
+      if @topic_entry.offset == -1 && offset_manager
+        @next_offset = offset_manager.next_offset
+      end
+      @consumer = create_consumer(@next_offset)      
+
+      super(interval, true)
+    end
+
+    def on_timer
+      @callback.call
+    rescue
+        # TODO log?
+        $log.error $!.to_s
+        $log.error_backtrace
+    end
+
+    def consume
+      es = MultiEventStream.new
+      tag = @topic_entry.topic
+      tag = @add_prefix + "." + tag if @add_prefix
+      tag = tag + "." + @add_suffix if @add_suffix
+
+      if @offset_manager && @consumer.next_offset != @next_offset
+        @consumer = create_consumer(@next_offset)
+      end
+
+      @consumer.fetch.each { |msg|
+        begin
+          msg_record = parse_line(msg.value)
+          msg_record = decorate_offset(msg_record, msg.offset) if @add_offset_in_record
+          es.add(Engine.now, msg_record)
+        rescue
+          $log.warn msg_record.to_s, :error=>$!.to_s
+          $log.debug_backtrace
+        end
+      }
+
+      unless es.empty?
+        @router.emit_stream(tag, es)
+
+        if @offset_manager
+          next_offset = @consumer.next_offset
+          @offset_manager.save_offset(next_offset)
+          @next_offset = next_offset
+        end
+      end
+    end
+
+    def create_consumer(offset)
+      @consumer.close if @consumer
+      Poseidon::PartitionConsumer.new(
+        @client_id,             # client_id
+        @host,                  # host
+        @port,                  # port
+        @topic_entry.topic,     # topic
+        @topic_entry.partition, # partition
+        offset,                 # offset
+        @options                # options
+      )
+    end
+
+    def parse_line(record)
+      case @format
+      when 'json'
+        Yajl::Parser.parse(record)
+      when 'ltsv'
+        LTSV.parse(record)
+      when 'msgpack'
+        MessagePack.unpack(record)
+      when 'text'
+        {@message_key => record}
+      end
+    end
+
+    def decorate_offset(record, offset)
+      case @format
+      when 'json'
+        add_offset_in_hash(record, @topic_entry.topic, @topic_entry.partition, offset)
+      when 'ltsv'
+        record.each { |line|
+          add_offset_in_hash(line, @topic_entry.topic, @topic_entry.partition, offset)
+        }
+      when 'msgpack'
+        add_offset_in_hash(record, @topic_entry.topic, @topic_entry.partition, offset)
+      when 'text'
+        add_offset_in_hash(record, @topic_entry.topic, @topic_entry.partition, offset)
+      end
+      record
+    end
+
+    def add_offset_in_hash(hash, topic, partition, offset)
+      hash['kafka_topic'] = topic
+      hash['kafka_partition'] = partition
+      hash['kafka_offset'] = offset
+    end
+  end
+
+  class TopicEntry
+    def initialize(topic, partition, offset)
+      @topic = topic
+      @partition = partition
+      @offset = offset
+    end
+    attr_reader :topic, :partition, :offset
+  end
+
+  class OffsetManager
+    def initialize(topic_entry, zookeeper, zk_root_node)
+      @zookeeper = zookeeper
+      @zk_path = "#{zk_root_node}/#{topic_entry.topic}/#{topic_entry.partition}/next_offset"
+      create_node(@zk_path, topic_entry.topic, topic_entry.partition)
+    end
+
+    def create_node(zk_path, topic, partition)
+      path = ""
+      zk_path.split(/(\/[^\/]+)/).reject(&:empty?).each { |dir|
+        path = path + dir
+        @zookeeper.create(:path => "#{path}")
+      }
+      $log.trace "use zk offset node : #{path}"
+    end
+
+    def next_offset
+      @zookeeper.get(:path => @zk_path)[:data].to_i
+    end
+
+    def save_offset(offset)
+      @zookeeper.set(:path => @zk_path, :data => offset.to_s)
+      $log.trace "update zk offset node : #{offset.to_s}"
+    end
+  end
+end
+
+end

data/lib/fluent/plugin/in_kafka_group.rb

@@ -0,0 +1,176 @@
+module Fluent
+
+class KafkaGroupInput < Input
+  Plugin.register_input('kafka_group', self)
+
+  config_param :brokers, :string,
+               :desc => "List of broker-host:port, separate with comma, must set."
+  config_param :zookeepers, :string,
+               :desc => "List of broker-host:port, separate with comma, must set."
+  config_param :consumer_group, :string, :default => nil,
+               :desc => "Consumer group name, must set."
+  config_param :topics, :string,
+               :desc => "Listening topics(separate with comma',')."
+  config_param :interval, :integer, :default => 1, # seconds
+               :desc => "Interval (Unit: seconds)"
+  config_param :format, :string, :default => 'json',
+               :desc => "Supported format: (json|text|ltsv|msgpack)"
+  config_param :message_key, :string, :default => 'message',
+               :desc => "For 'text' format only."
+  config_param :add_prefix, :string, :default => nil,
+               :desc => "Tag prefix (Optional)"
+  config_param :add_suffix, :string, :default => nil,
+               :desc => "Tag suffix (Optional)"
+
+  # poseidon PartitionConsumer options
+  config_param :max_bytes, :integer, :default => nil,
+               :desc => "Maximum number of bytes to fetch."
+  config_param :max_wait_ms, :integer, :default => nil,
+               :desc => "How long to block until the server sends us data."
+  config_param :min_bytes, :integer, :default => nil,
+               :desc => "Smallest amount of data the server should send us."
+  config_param :socket_timeout_ms, :integer, :default => nil,
+               :desc => "How long to wait for reply from server. Should be higher than max_wait_ms."
+
+  unless method_defined?(:router)
+    define_method("router") { Fluent::Engine }
+  end
+
+  def initialize
+    super
+    require 'poseidon_cluster'
+  end
+
+  def _config_to_array(config)
+    config_array = config.split(',').map {|k| k.strip }
+    if config_array.empty?
+      raise ConfigError, "kafka_group: '#{config}' is a required parameter"
+    end
+    config_array
+  end
+
+  private :_config_to_array
+
+  def configure(conf)
+    super
+    @broker_list = _config_to_array(@brokers)
+    @zookeeper_list = _config_to_array(@zookeepers)
+    @topic_list = _config_to_array(@topics)
+
+    unless @consumer_group
+      raise ConfigError, "kafka_group: 'consumer_group' is a required parameter"
+    end
+    $log.info "Will watch for topics #{@topic_list} at brokers " \
+              "#{@broker_list}, zookeepers #{@zookeeper_list} and group " \
+              "'#{@consumer_group}'"
+
+    case @format
+    when 'json'
+      require 'yajl'
+    when 'ltsv'
+      require 'ltsv'
+    when 'msgpack'
+      require 'msgpack'
+    end
+  end
+
+  def start
+    @loop = Coolio::Loop.new
+    opt = {}
+    opt[:max_bytes] = @max_bytes if @max_bytes
+    opt[:max_wait_ms] = @max_wait_ms if @max_wait_ms
+    opt[:min_bytes] = @min_bytes if @min_bytes
+    opt[:socket_timeout_ms] = @socket_timeout_ms if @socket_timeout_ms
+
+    @topic_watchers = @topic_list.map {|topic|
+      TopicWatcher.new(topic, @broker_list, @zookeeper_list, @consumer_group,
+                       interval, @format, @message_key, @add_prefix,
+                       @add_suffix, router, opt)
+    }
+    @topic_watchers.each {|tw|
+      tw.attach(@loop)
+    }
+    @thread = Thread.new(&method(:run))
+  end
+
+  def shutdown
+    @loop.stop
+  end
+
+  def run
+    @loop.run
+  rescue
+    $log.error "unexpected error", :error=>$!.to_s
+    $log.error_backtrace
+  end
+
+  class TopicWatcher < Coolio::TimerWatcher
+    def initialize(topic, broker_list, zookeeper_list, consumer_group,
+                   interval, format, message_key, add_prefix, add_suffix,
+                   router, options)
+      @topic = topic
+      @callback = method(:consume)
+      @format = format
+      @message_key = message_key
+      @add_prefix = add_prefix
+      @add_suffix = add_suffix
+      @router = router
+
+      @consumer = Poseidon::ConsumerGroup.new(
+        consumer_group,
+        broker_list,
+        zookeeper_list,
+        topic,
+        options
+      )
+
+      super(interval, true)
+    end
+
+    def on_timer
+      @callback.call
+    rescue
+        # TODO log?
+        $log.error $!.to_s
+        $log.error_backtrace
+    end
+
+    def consume
+      es = MultiEventStream.new
+      tag = @topic
+      tag = @add_prefix + "." + tag if @add_prefix
+      tag = tag + "." + @add_suffix if @add_suffix
+
+      @consumer.fetch do |partition, bulk|
+        bulk.each do |msg|
+          begin
+            msg_record = parse_line(msg.value)
+            es.add(Engine.now, msg_record)
+          rescue
+            $log.warn msg_record.to_s, :error=>$!.to_s
+            $log.debug_backtrace
+          end
+        end
+      end
+
+      unless es.empty?
+        @router.emit_stream(tag, es)
+      end
+    end
+
+    def parse_line(record)
+      case @format
+      when 'json'
+        Yajl::Parser.parse(record)
+      when 'ltsv'
+        LTSV.parse(record)
+      when 'msgpack'
+        MessagePack.unpack(record)
+      when 'text'
+        {@message_key => record}
+      end
+    end
+  end
+end
+
+end

data/lib/fluent/plugin/out_kafka.rb

@@ -0,0 +1,165 @@
+class Fluent::KafkaOutput < Fluent::Output
+  Fluent::Plugin.register_output('kafka', self)
+
+  def initialize
+    super
+    require 'poseidon'
+  end
+
+  config_param :brokers, :string, :default => 'localhost:9092',
+               :desc => <<-DESC
+Set brokers directly
+<broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,..
+Note that you can choose to use either brokers or zookeeper.
+DESC
+  config_param :zookeeper, :string, :default => nil,
+               :desc => "Set brokers via Zookeeper: <zookeeper_host>:<zookeeper_port>"
+  config_param :zookeeper_path, :string, :default => '/brokers/ids',
+               :desc => "Path in path for Broker id. Default to /brokers/ids"
+  config_param :default_topic, :string, :default => nil,
+               :desc => "Output topic."
+  config_param :default_partition_key, :string, :default => nil
+  config_param :client_id, :string, :default => 'kafka'
+  config_param :output_data_type, :string, :default => 'json',
+               :desc => "Supported format: (json|ltsv|msgpack|attr:<record name>|<formatter name>)"
+  config_param :output_include_tag, :bool, :default => false
+  config_param :output_include_time, :bool, :default => false
+
+  # poseidon producer options
+  config_param :max_send_retries, :integer, :default => 3,
+               :desc => "Number of times to retry sending of messages to a leader."
+  config_param :required_acks, :integer, :default => 0,
+               :desc => "The number of acks required per request."
+  config_param :ack_timeout_ms, :integer, :default => 1500,
+               :desc => "How long the producer waits for acks."
+  config_param :compression_codec, :string, :default => 'none',
+               :desc => "The codec the producer uses to compress messages."
+
+  attr_accessor :output_data_type
+  attr_accessor :field_separator
+
+  @seed_brokers = []
+
+  unless method_defined?(:log)
+    define_method("log") { $log }
+  end
+
+  def refresh_producer()
+    if @zookeeper
+      @seed_brokers = []
+      z = Zookeeper.new(@zookeeper)
+      z.get_children(:path => @zookeeper_path)[:children].each do |id|
+        broker = Yajl.load(z.get(:path => @zookeeper_path + "/#{id}")[:data])
+        @seed_brokers.push("#{broker['host']}:#{broker['port']}")
+      end
+      z.close
+      log.info "brokers has been refreshed via Zookeeper: #{@seed_brokers}"
+    end
+    begin
+      if @seed_brokers.length > 0
+        @producer = Poseidon::Producer.new(@seed_brokers, @client_id, :max_send_retries => @max_send_retries, :required_acks => @required_acks, :ack_timeout_ms => @ack_timeout_ms, :compression_codec => @compression_codec.to_sym)
+        log.info "initialized producer #{@client_id}"
+      else
+        log.warn "No brokers found on Zookeeper"
+      end
+    rescue Exception => e
+      log.error e
+    end
+  end
+
+  def configure(conf)
+    super
+    if @zookeeper
+      require 'zookeeper'
+      require 'yajl'
+    else
+      @seed_brokers = @brokers.match(",").nil? ? [@brokers] : @brokers.split(",")
+      log.info "brokers has been set directly: #{@seed_brokers}"
+    end
+    if @compression_codec == 'snappy'
+      require 'snappy'
+    end
+    case @output_data_type
+    when 'json'
+      require 'yajl'
+    when 'ltsv'
+      require 'ltsv'
+    when 'msgpack'
+      require 'msgpack'
+    end
+
+    @f_separator = case @field_separator
+                   when /SPACE/i then ' '
+                   when /COMMA/i then ','
+                   when /SOH/i then "\x01"
+                   else "\t"
+                   end
+
+    @custom_attributes = if @output_data_type == 'json'
+                           nil
+                         elsif @output_data_type == 'ltsv'
+                           nil
+                         elsif @output_data_type == 'msgpack'
+                           nil
+                         elsif @output_data_type =~ /^attr:(.*)$/
+                           $1.split(',').map(&:strip).reject(&:empty?)
+                         else
+                           @formatter = Fluent::Plugin.new_formatter(@output_data_type)
+                           @formatter.configure(conf)
+                           nil
+                         end
+
+  end
+
+  def start
+    super
+    refresh_producer()
+  end
+
+  def shutdown
+    super
+  end
+
+  def parse_record(record)
+    if @custom_attributes.nil?
+      case @output_data_type
+      when 'json'
+        Yajl::Encoder.encode(record)
+      when 'ltsv'
+        LTSV.dump(record)
+      when 'msgpack'
+        record.to_msgpack
+      else
+        record.to_s
+      end
+    else
+      @custom_attributes.unshift('time') if @output_include_time
+      @custom_attributes.unshift('tag') if @output_include_tag
+      @custom_attributes.map { |attr|
+        record[attr].nil? ? '' : record[attr].to_s
+      }.join(@f_separator)
+    end
+  end
+
+  def emit(tag, es, chain)
+    begin
+      chain.next
+      es.each do |time,record|
+        record['time'] = time if @output_include_time
+        record['tag'] = tag if @output_include_tag
+        topic = record['topic'] || self.default_topic || tag
+        partition_key = record['partition_key'] || @default_partition_key
+        value = @formatter.nil? ? parse_record(record) : @formatter.format(tag, time, record)
+        log.trace("message send to #{topic} with key: #{partition_key} and value: #{value}.")
+        message = Poseidon::MessageToSend.new(topic, value, partition_key)
+        @producer.send_messages([message])
+      end
+    rescue Exception => e
+      log.warn("Send exception occurred: #{e}")
+      @producer.close if @producer
+      refresh_producer()
+      raise e
+    end
+  end
+
+end

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -0,0 +1,187 @@
+# encode: utf-8
+class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
+  Fluent::Plugin.register_output('kafka_buffered', self)
+
+  def initialize
+    super
+    require 'poseidon'
+  end
+
+  config_param :brokers, :string, :default => 'localhost:9092',
+               :desc => <<-DESC
+Set brokers directly:
+<broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,..
+Brokers: you can choose to use either brokers or zookeeper.
+DESC
+  config_param :zookeeper, :string, :default => nil,
+               :desc => <<-DESC
+Set brokers via Zookeeper:
+<zookeeper_host>:<zookeeper_port>
+DESC
+  config_param :zookeeper_path, :string, :default => '/brokers/ids',
+               :desc => "Path in path for Broker id. Default to /brokers/ids"
+  config_param :default_topic, :string, :default => nil,
+               :desc => "Output topic"
+  config_param :default_partition_key, :string, :default => nil
+  config_param :client_id, :string, :default => 'kafka'
+  config_param :output_data_type, :string, :default => 'json',
+               :desc => <<-DESC
+Supported format: (json|ltsv|msgpack|attr:<record name>|<formatter name>)
+DESC
+  config_param :output_include_tag, :bool, :default => false
+  config_param :output_include_time, :bool, :default => false
+  config_param :kafka_agg_max_bytes, :size, :default => 4*1024  #4k
+
+  # poseidon producer options
+  config_param :max_send_retries, :integer, :default => 3,
+               :desc => "Number of times to retry sending of messages to a leader."
+  config_param :required_acks, :integer, :default => 0,
+               :desc => "The number of acks required per request."
+  config_param :ack_timeout_ms, :integer, :default => 1500,
+               :desc => "How long the producer waits for acks."
+  config_param :compression_codec, :string, :default => 'none',
+               :desc => <<-DESC
+The codec the producer uses to compress messages.
+Supported codecs: (none|gzip|snappy)
+DESC
+
+  attr_accessor :output_data_type
+  attr_accessor :field_separator
+
+  unless method_defined?(:log)
+    define_method("log") { $log }
+  end
+
+  @seed_brokers = []
+
+  def refresh_producer()
+    if @zookeeper
+      @seed_brokers = []
+      z = Zookeeper.new(@zookeeper)
+      z.get_children(:path => @zookeeper_path)[:children].each do |id|
+        broker = Yajl.load(z.get(:path => @zookeeper_path + "/#{id}")[:data])
+        @seed_brokers.push("#{broker['host']}:#{broker['port']}")
+      end
+      z.close
+      log.info "brokers has been refreshed via Zookeeper: #{@seed_brokers}"
+    end
+    begin
+      if @seed_brokers.length > 0
+        @producer = Poseidon::Producer.new(@seed_brokers, @client_id, :max_send_retries => @max_send_retries, :required_acks => @required_acks, :ack_timeout_ms => @ack_timeout_ms, :compression_codec => @compression_codec.to_sym)
+        log.info "initialized producer #{@client_id}"
+      else
+        log.warn "No brokers found on Zookeeper"
+      end
+    rescue Exception => e
+      log.error e
+    end
+  end
+
+  def configure(conf)
+    super
+    if @zookeeper
+      require 'zookeeper'
+      require 'yajl'
+    else
+      @seed_brokers = @brokers.match(",").nil? ? [@brokers] : @brokers.split(",")
+      log.info "brokers has been set directly: #{@seed_brokers}"
+    end
+    if @compression_codec == 'snappy'
+      require 'snappy'
+    end
+
+    @f_separator = case @field_separator
+                   when /SPACE/i then ' '
+                   when /COMMA/i then ','
+                   when /SOH/i then "\x01"
+                   else "\t"
+                   end
+
+    @formatter_proc = setup_formatter(conf)
+  end
+
+  def start
+    super
+    refresh_producer()
+  end
+
+  def shutdown
+    super
+  end
+
+  def format(tag, time, record)
+    [tag, time, record].to_msgpack
+  end
+
+  def setup_formatter(conf)
+    if @output_data_type == 'json'
+      require 'yajl'
+      Proc.new { |tag, time, record| Yajl::Encoder.encode(record) }
+    elsif @output_data_type == 'ltsv'
+      require 'ltsv'
+      Proc.new { |tag, time, record| LTSV.dump(record) }
+    elsif @output_data_type == 'msgpack'
+      require 'msgpack'
+      Proc.new { |tag, time, record| record.to_msgpack }
+    elsif @output_data_type =~ /^attr:(.*)$/
+      @custom_attributes = $1.split(',').map(&:strip).reject(&:empty?)
+      @custom_attributes.unshift('time') if @output_include_time
+      @custom_attributes.unshift('tag') if @output_include_tag
+      Proc.new { |tag, time, record|
+        @custom_attributes.map { |attr|
+          record[attr].nil? ? '' : record[attr].to_s
+        }.join(@f_separator)
+      }
+    else
+      @formatter = Fluent::Plugin.new_formatter(@output_data_type)
+      @formatter.configure(conf)
+      Proc.new { |tag, time, record|
+        @formatter.format(tag, time, record)
+      }
+    end
+  end
+
+  def write(chunk)
+    records_by_topic = {}
+    bytes_by_topic = {}
+    messages = []
+    messages_bytes = 0
+    begin
+      chunk.msgpack_each { |tag, time, record|
+        record['time'] = time if @output_include_time
+        record['tag'] = tag if @output_include_tag
+        topic = record['topic'] || @default_topic || tag
+        partition_key = record['partition_key'] || @default_partition_key
+
+        records_by_topic[topic] ||= 0
+        bytes_by_topic[topic] ||= 0
+
+        record_buf = @formatter_proc.call(tag, time, record)
+        record_buf_bytes = record_buf.bytesize
+        if messages.length > 0 and messages_bytes + record_buf_bytes > @kafka_agg_max_bytes
+          log.on_trace { log.trace("#{messages.length} messages send.") }
+          @producer.send_messages(messages)
+          messages = []
+          messages_bytes = 0
+        end
+        log.on_trace { log.trace("message will send to #{topic} with key: #{partition_key} and value: #{record_buf}.") }
+        messages << Poseidon::MessageToSend.new(topic, record_buf, partition_key)
+        messages_bytes += record_buf_bytes
+
+        records_by_topic[topic] += 1
+        bytes_by_topic[topic] += record_buf_bytes
+      }
+      if messages.length > 0
+        log.trace("#{messages.length} messages send.")
+        @producer.send_messages(messages)
+      end
+      log.debug "(records|bytes) (#{records_by_topic}|#{bytes_by_topic})"
+    end
+  rescue Exception => e
+    log.warn "Send exception occurred: #{e}"
+    @producer.close if @producer
+    refresh_producer()
+    # Raise exception to retry sendind messages
+    raise e
+  end
+end

data/test/helper.rb

@@ -0,0 +1,27 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/plugin/out_kafka'
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_out_kafka.rb

@@ -0,0 +1,33 @@
+require 'helper'
+
+class KafkaOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    default_topic kitagawakeiko
+    brokers localhost:9092
+  ]
+
+  def create_driver(conf = CONFIG, tag='test')
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::KafkaOutput, tag).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    assert_equal 'kitagawakeiko', d.instance.default_topic
+    assert_equal 'localhost:9092', d.instance.brokers
+  end
+
+  def test_format
+    d = create_driver
+  end
+
+  def test_write
+    d = create_driver
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit({"a"=>1}, time)
+    d.emit({"a"=>2}, time)
+  end
+end

metadata

@@ -0,0 +1,141 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-kafka-zendesk
+version: !ruby/object:Gem::Version
+  version: 0.1.4
+platform: ruby
+authors:
+- Hidemasa Togashi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-05-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: poseidon_cluster
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ltsv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yajl-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: msgpack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: zookeeper
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluentd plugin for Apache Kafka > 0.8
+email:
+- togachiro@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- fluent-plugin-kafka-0.1.4.gem
+- fluent-plugin-kafka.gemspec
+- lib/fluent/plugin/in_kafka.rb
+- lib/fluent/plugin/in_kafka_group.rb
+- lib/fluent/plugin/out_kafka.rb
+- lib/fluent/plugin/out_kafka_buffered.rb
+- test/helper.rb
+- test/plugin/test_out_kafka.rb
+homepage: https://github.com/khouse/fluent-plugin-kafka
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5.1
+signing_key: 
+specification_version: 4
+summary: Fluentd plugin for Apache Kafka > 0.8
+test_files:
+- test/helper.rb
+- test/plugin/test_out_kafka.rb