fluent-plugin-kafka-enchanced 0.5.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 056609426bfbcfd6ea642db660c8360beaee7f72
+  data.tar.gz: acd3535de138c68109edcaf7ad318ad76cc8dc5d
+SHA512:
+  metadata.gz: 1d25c86afc8ea2513504bd6d2b086751f3bdcc88c3f8f35673188bd3ee5edf7b23fc2d688392e79df3be2d5995b02bc168bc68a5b94564c8f0ebce7ce7e9a19e
+  data.tar.gz: b5c07478cd3595b420330490efa62b1d0878020962a742c9bf72f373186cb4229715ae723335e52fef2256087f17b22550faf849e0454fd66ffb1d1aedf8ee96

data/.gitignore

@@ -0,0 +1,2 @@
+/Gemfile.lock
+*.swp

data/.travis.yml

@@ -0,0 +1,17 @@
+language: ruby
+
+rvm:
+  - 2.1
+  - 2.2
+  - 2.3.1
+  - ruby-head
+
+script:
+  - bundle exec rake test
+
+sudo: false
+
+matrix:
+  allow_failures:
+    - rvm: ruby-head
+

data/ChangeLog

@@ -0,0 +1,49 @@
+Release 0.5.1 - 2017/02/06
+
+	* in_kafka_group: Fix uninitialized constant error
+
+Release 0.5.0 - 2017/01/17
+
+	* output: Add out_kafka2 plugin with v0.14 API
+
+Release 0.4.2 - 2016/12/10
+
+	* input: Add use_record_time and time_format parameters
+	* Update ruby-kafka dependency to 0.3.16.beta2
+
+Release 0.4.1 - 2016/12/01
+
+	* output: Support specifying partition
+
+Release 0.4.0 - 2016/11/08
+
+	* Remove zookeeper dependency
+
+Release 0.3.5 - 2016/10/21
+
+	* output: Support message key and related parameters. #91
+
+Release 0.3.4 - 2016/10/20
+
+	* output: Add exclude_topic_key and exclude_partition_key. #89
+
+Release 0.3.3 - 2016/10/17
+
+	* out_kafka_buffered: Add get_kafka_client_log parameter. #83
+	* out_kafka_buffered: Skip and log invalid record to avoid buffer stuck. #86
+	* in_kafka_group: Add retry_emit_limit to handle BufferQueueLimitError. #87
+
+Release 0.3.2 - 2016/10/06
+
+	* in_kafka_group: Re-fetch events after consumer error. #79
+
+Release 0.3.1 - 2016/08/28
+
+	* output: Change default required_acks to -1. #70
+	* Support ruby version changed to 2.1.0 or later
+
+Release 0.3.0 - 2016/08/24
+
+	* Fully replace poseidon ruby library with ruby-kafka to support latest kafka versions
+
+See git commits for older changes

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kafka.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2014 htgc
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,221 @@
+# fluent-plugin-kafka, a plugin for [Fluentd](http://fluentd.org)
+
+[![Build Status](https://travis-ci.org/htgc/fluent-plugin-kafka.svg?branch=master)](https://travis-ci.org/htgc/fluent-plugin-kafka)
+
+A fluentd plugin to both consume and produce data for Apache Kafka.
+
+TODO: Also, I need to write tests
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'fluent-plugin-kafka'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fluent-plugin-kafka
+
+If you want to use zookeeper related parameters, you also need to install zookeeper gem. zookeeper gem includes native extension, so development tools are needed, e.g. gcc, make and etc.
+
+## Requirements
+
+- Ruby 2.1 or later
+- Input plugins work with kafka v0.9 or later
+- Output plugins work with kafka v0.8 or later
+
+## Usage
+
+### Common parameters
+
+- ssl_ca_cert
+- ssl_client_cert
+- ssl_client_cert_key
+
+Set path to SSL related files. See [Encryption and Authentication using SSL](https://github.com/zendesk/ruby-kafka#encryption-and-authentication-using-ssl) for more detail.
+
+### Input plugin (@type 'kafka')
+
+Consume events by single consumer.
+
+    <source>
+      @type kafka
+
+      brokers <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,..
+      topics <listening topics(separate with comma',')>
+      format <input text type (text|json|ltsv|msgpack)> :default => json
+      message_key <key (Optional, for text format only, default is message)>
+      add_prefix <tag prefix (Optional)>
+      add_suffix <tag suffix (Optional)>
+
+      # Optionally, you can manage topic offset by using zookeeper
+      offset_zookeeper    <zookeer node list (<zookeeper1_host>:<zookeeper1_port>,<zookeeper2_host>:<zookeeper2_port>,..)>
+      offset_zk_root_node <offset path in zookeeper> default => '/fluent-plugin-kafka'
+
+      # ruby-kafka consumer options
+      max_bytes     (integer) :default => nil (Use default of ruby-kafka)
+      max_wait_time (integer) :default => nil (Use default of ruby-kafka)
+      min_bytes     (integer) :default => nil (Use default of ruby-kafka)
+    </source>
+
+Supports a start of processing from the assigned offset for specific topics.
+
+    <source>
+      @type kafka
+
+      brokers <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,..
+      format <input text type (text|json|ltsv|msgpack)>
+      <topic>
+        topic     <listening topic>
+        partition <listening partition: default=0>
+        offset    <listening start offset: default=-1>
+      </topic>
+      <topic>
+        topic     <listening topic>
+        partition <listening partition: default=0>
+        offset    <listening start offset: default=-1>
+      </topic>
+    </source>
+
+See also [ruby-kafka README](https://github.com/zendesk/ruby-kafka#consuming-messages-from-kafka) for more detailed documentation about ruby-kafka.
+
+### Input plugin (@type 'kafka_group', supports kafka group)
+
+Consume events by kafka consumer group features..
+
+    <source>
+      @type kafka_group
+
+      brokers <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,..
+      consumer_group <consumer group name, must set>
+      topics <listening topics(separate with comma',')>
+      format <input text type (text|json|ltsv|msgpack)> :default => json
+      message_key <key (Optional, for text format only, default is message)>
+      add_prefix <tag prefix (Optional)>
+      add_suffix <tag suffix (Optional)>
+      retry_emit_limit <Wait retry_emit_limit x 1s when BuffereQueueLimitError happens. The default is nil and it means waiting until BufferQueueLimitError is resolved>
+      use_record_time <If true, replace event time with contents of 'time' field of fetched record>
+      time_format <string (Optional when use_record_time is used)>
+
+      # ruby-kafka consumer options
+      max_bytes               (integer) :default => 1048576
+      max_wait_time           (integer) :default => nil (Use default of ruby-kafka)
+      min_bytes               (integer) :default => nil (Use default of ruby-kafka)
+      offset_commit_interval  (integer) :default => nil (Use default of ruby-kafka)
+      offset_commit_threshold (integer) :default => nil (Use default of ruby-kafka)
+      start_from_beginning    (bool)    :default => true
+    </source>
+
+See also [ruby-kafka README](https://github.com/zendesk/ruby-kafka#consuming-messages-from-kafka) for more detailed documentation about ruby-kafka options.
+
+### Buffered output plugin
+
+This plugin uses ruby-kafka producer for writing data. This plugin works with recent kafka versions.
+
+    <match *.**>
+      @type kafka_buffered
+
+      # Brokers: you can choose either brokers or zookeeper. If you are not familiar with zookeeper, use brokers parameters.
+      brokers             <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,.. # Set brokers directly
+      zookeeper           <zookeeper_host>:<zookeeper_port> # Set brokers via Zookeeper
+      zookeeper_path      <broker path in zookeeper> :default => /brokers/ids # Set path in zookeeper for kafka
+
+      default_topic         (string) :default => nil
+      default_partition_key (string) :default => nil
+      default_message_key   (string) :default => nil
+      output_data_type      (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
+      output_include_tag    (bool) :default => false
+      output_include_time   (bool) :default => false
+      exclude_topic_key     (bool) :default => false
+      exclude_partition_key (bool) :default => false
+      get_kafka_client_log  (bool) :default => false
+
+      # See fluentd document for buffer related parameters: http://docs.fluentd.org/articles/buffer-plugin-overview
+
+      # ruby-kafka producer options
+      max_send_retries    (integer)     :default => 1
+      required_acks       (integer)     :default => -1
+      ack_timeout         (integer)     :default => nil (Use default of ruby-kafka)
+      compression_codec   (gzip|snappy) :default => nil (No compression)
+    </match>
+
+`<formatter name>` of `output_data_type` uses fluentd's formatter plugins. See [formatter article](http://docs.fluentd.org/articles/formatter-plugin-overview).
+
+ruby-kafka sometimes returns `Kafka::DeliveryFailed` error without good information.
+In this case, `get_kafka_client_log` is useful for identifying the error cause.    
+ruby-kafka's log is routed to fluentd log so you can see ruby-kafka's log in fluentd logs.
+
+Supports following ruby-kafka's producer options.
+
+- max_send_retries - default: 1 - Number of times to retry sending of messages to a leader.
+- required_acks - default: -1 - The number of acks required per request. If you need flush performance, set lower value, e.g. 1, 2.
+- ack_timeout - default: nil - How long the producer waits for acks. The unit is seconds.
+- compression_codec - default: nil - The codec the producer uses to compress messages.
+
+See also [Kafka::Client](http://www.rubydoc.info/gems/ruby-kafka/Kafka/Client) for more detailed documentation about ruby-kafka.
+
+This plugin supports compression codec "snappy" also.
+Install snappy module before you use snappy compression.
+
+    $ gem install snappy
+
+snappy gem uses native extension, so you need to install several packages before.
+On Ubuntu, need development packages and snappy library.
+
+    $ sudo apt-get install build-essential autoconf automake libtool libsnappy-dev
+
+#### Load balancing
+
+Messages will be assigned a partition at random as default by ruby-kafka, but messages with the same partition key will always be assigned to the same partition by setting `default_partition_key` in config file.
+If key name `partition_key` exists in a message, this plugin set its value of partition_key as key.
+
+|default_partition_key|partition_key| behavior |
+| --- | --- | --- |
+|Not set|Not exists| All messages are assigned a partition at random |
+|Set| Not exists| All messages are assigned to the specific partition |
+|Not set| Exists | Messages which have partition_key record are assigned to the specific partition, others are assigned a partition at random |
+|Set| Exists | Messages which have partition_key record are assigned to the specific partition with parition_key, others are assigned to the specific partition with default_parition_key |
+
+If key name `message_key` exists in a message, this plugin publishes the value of message_key to kafka and can be read by consumers. Same message key will be assigned to all messages by setting `default_message_key` in config file. If message_key exists and if partition_key is not set explicitly, messsage_key will be used for partitioning.
+
+### Non-buffered output plugin
+
+This plugin uses ruby-kafka producer for writing data. For performance and reliability concerns, use `kafka_bufferd` output instead. This is mainly for testing.
+
+    <match *.**>
+      @type kafka
+
+      # Brokers: you can choose either brokers or zookeeper.
+      brokers        <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,.. # Set brokers directly
+      zookeeper      <zookeeper_host>:<zookeeper_port> # Set brokers via Zookeeper
+      zookeeper_path <broker path in zookeeper> :default => /brokers/ids # Set path in zookeeper for kafka
+
+      default_topic         (string) :default => nil
+      default_partition_key (string) :default => nil
+      default_message_key   (string) :default => nil
+      output_data_type      (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
+      output_include_tag    (bool) :default => false
+      output_include_time   (bool) :default => false
+      exclude_topic_key     (bool) :default => false
+      exclude_partition_key (bool) :default => false
+
+      # ruby-kafka producer options
+      max_send_retries  (integer)     :default => 1
+      required_acks     (integer)     :default => -1
+      ack_timeout       (integer)     :default => nil (Use default of ruby-kafka)
+      compression_codec (gzip|snappy) :default => nil
+    </match>
+
+This plugin also supports ruby-kafka related parameters. See Buffered output plugin section.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,12 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/**/test_*.rb']
+  test.verbose = true
+end
+
+task :default => [:build]

data/fluent-plugin-kafka.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Paschenko Konstantin"]
+  gem.email         = ["zhr0n4x@gmail.com"]
+  gem.description   = %q{Fluentd plugin for Apache Kafka > 0.8}
+  gem.summary       = %q{Fluentd plugin for Apache Kafka > 0.8}
+  gem.homepage      = "https://github.com/fluent/fluent-plugin-kafka"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "fluent-plugin-kafka-enchanced"
+  gem.require_paths = ["lib"]
+  gem.version       = '0.5.1'
+  gem.required_ruby_version = ">= 2.1.0"
+
+  gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
+  gem.add_dependency 'ltsv'
+  gem.add_dependency 'ruby-kafka', '= 0.3.16.beta2'
+  gem.add_development_dependency "rake", ">= 0.9.2"
+  gem.add_development_dependency "test-unit", ">= 3.0.8"
+end

data/lib/fluent/plugin/in_kafka.rb

@@ -0,0 +1,308 @@
+require 'fluent/input'
+require 'fluent/time'
+require 'fluent/plugin/kafka_plugin_util'
+
+class Fluent::KafkaInput < Fluent::Input
+  Fluent::Plugin.register_input('kafka', self)
+
+  config_param :format, :string, :default => 'json',
+               :desc => "Supported format: (json|text|ltsv|msgpack)"
+  config_param :message_key, :string, :default => 'message',
+               :desc => "For 'text' format only."
+  config_param :host, :string, :default => nil,
+               :desc => "Broker host"
+  config_param :port, :integer, :default => nil,
+               :desc => "Broker port"
+  config_param :brokers, :string, :default => 'localhost:9092',
+               :desc => "List of broker-host:port, separate with comma, must set."
+  config_param :interval, :integer, :default => 1, # seconds
+               :desc => "Interval (Unit: seconds)"
+  config_param :topics, :string, :default => nil,
+               :desc => "Listening topics(separate with comma',')"
+  config_param :client_id, :string, :default => 'kafka'
+  config_param :partition, :integer, :default => 0,
+               :desc => "Listening partition"
+  config_param :offset, :integer, :default => -1,
+               :desc => "Listening start offset"
+  config_param :add_prefix, :string, :default => nil,
+               :desc => "Tag prefix"
+  config_param :add_suffix, :string, :default => nil,
+               :desc => "tag suffix"
+  config_param :add_offset_in_record, :bool, :default => false
+
+  config_param :offset_zookeeper, :string, :default => nil
+  config_param :offset_zk_root_node, :string, :default => '/fluent-plugin-kafka'
+  config_param :use_record_time, :bool, :default => false,
+               :desc => "Replace message timestamp with contents of 'time' field."
+  config_param :time_format, :string, :default => nil,
+               :desc => "Time format to be used to parse 'time' filed."
+
+  # Kafka#fetch_messages options
+  config_param :max_bytes, :integer, :default => nil,
+               :desc => "Maximum number of bytes to fetch."
+  config_param :max_wait_time, :integer, :default => nil,
+               :desc => "How long to block until the server sends us data."
+  config_param :min_bytes, :integer, :default => nil,
+               :desc => "Smallest amount of data the server should send us."
+
+  include Fluent::KafkaPluginUtil::SSLSettings
+
+  unless method_defined?(:router)
+    define_method("router") { Fluent::Engine }
+  end
+
+  def initialize
+    super
+    require 'kafka'
+
+    @time_parser = nil
+  end
+
+  def configure(conf)
+    super
+
+    @topic_list = []
+    if @topics
+      @topic_list = @topics.split(',').map { |topic|
+        TopicEntry.new(topic.strip, @partition, @offset)
+      }
+    else
+      conf.elements.select { |element| element.name == 'topic' }.each do |element|
+        unless element.has_key?('topic')
+          raise Fluent::ConfigError, "kafka: 'topic' is a require parameter in 'topic element'."
+        end
+        partition = element.has_key?('partition') ? element['partition'].to_i : 0
+        offset = element.has_key?('offset') ? element['offset'].to_i : -1
+        @topic_list.push(TopicEntry.new(element['topic'], partition, offset))
+      end
+    end
+
+    if @topic_list.empty?
+      raise Fluent::ConfigError, "kafka: 'topics' or 'topic element' is a require parameter"
+    end
+
+    # For backward compatibility
+    @brokers = case
+               when @host && @port
+                 ["#{@host}:#{@port}"]
+               when @host
+                 ["#{@host}:9092"]
+               when @port
+                 ["localhost:#{@port}"]
+               else
+                 @brokers
+               end
+
+    if conf['max_wait_ms']
+      log.warn "'max_wait_ms' parameter is deprecated. Use second unit 'max_wait_time' instead"
+      @max_wait_time = conf['max_wait_ms'].to_i / 1000
+    end
+
+    @max_wait_time = @interval if @max_wait_time.nil?
+
+    require 'zookeeper' if @offset_zookeeper
+
+    @parser_proc = setup_parser
+
+    if @use_record_time and @time_format
+      @time_parser = Fluent::TextParser::TimeParser.new(@time_format)
+    end
+  end
+
+  def setup_parser
+    case @format
+    when 'json'
+      require 'yajl'
+      Proc.new { |msg, te|
+        r = Yajl::Parser.parse(msg.value)
+        add_offset_in_hash(r, te, msg.offset) if @add_offset_in_record
+        r
+      }
+    when 'ltsv'
+      require 'ltsv'
+      Proc.new { |msg, te|
+        r = LTSV.parse(msg.value).first
+        add_offset_in_hash(r, te, msg.offset) if @add_offset_in_record
+        r
+      }
+    when 'msgpack'
+      require 'msgpack'
+      Proc.new { |msg, te|
+        r = MessagePack.unpack(msg.value)
+        add_offset_in_hash(r, te, msg.offset) if @add_offset_in_record
+        r
+      }
+    when 'text'
+      Proc.new { |msg, te|
+        r = {@message_key => msg.value}
+        add_offset_in_hash(r, te, msg.offset) if @add_offset_in_record
+        r
+      }
+    end
+  end
+
+  def add_offset_in_hash(hash, te, offset)
+    hash['kafka_topic'.freeze] = te.topic
+    hash['kafka_partition'.freeze] = te.partition
+    hash['kafka_offset'.freeze] = offset
+  end
+
+  def start
+    super
+
+    @loop = Coolio::Loop.new
+    opt = {}
+    opt[:max_bytes] = @max_bytes if @max_bytes
+    opt[:max_wait_time] = @max_wait_time if @max_wait_time
+    opt[:min_bytes] = @min_bytes if @min_bytes
+
+    @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id,
+                       ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+                       ssl_client_cert: read_ssl_file(@ssl_client_cert),
+                       ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key))
+    @zookeeper = Zookeeper.new(@offset_zookeeper) if @offset_zookeeper
+
+    @topic_watchers = @topic_list.map {|topic_entry|
+      offset_manager = OffsetManager.new(topic_entry, @zookeeper, @offset_zk_root_node) if @offset_zookeeper
+      TopicWatcher.new(
+        topic_entry,
+        @kafka,
+        interval,
+        @parser_proc,
+        @add_prefix,
+        @add_suffix,
+        offset_manager,
+        router,
+        opt)
+    }
+    @topic_watchers.each {|tw|
+      tw.attach(@loop)
+    }
+    @thread = Thread.new(&method(:run))
+  end
+
+  def shutdown
+    @loop.stop
+    @zookeeper.close! if @zookeeper
+    @thread.join
+    @kafka.close
+    super
+  end
+
+  def run
+    @loop.run
+  rescue => e
+    $log.error "unexpected error", :error => e.to_s
+    $log.error_backtrace
+  end
+
+  class TopicWatcher < Coolio::TimerWatcher
+    def initialize(topic_entry, kafka, interval, parser, add_prefix, add_suffix, offset_manager, router, options={})
+      @topic_entry = topic_entry
+      @kafka = kafka
+      @callback = method(:consume)
+      @parser = parser
+      @add_prefix = add_prefix
+      @add_suffix = add_suffix
+      @options = options
+      @offset_manager = offset_manager
+      @router = router
+
+      @next_offset = @topic_entry.offset
+      if @topic_entry.offset == -1 && offset_manager
+        @next_offset = offset_manager.next_offset
+      end
+      @fetch_args = {
+        topic: @topic_entry.topic,
+        partition: @topic_entry.partition,
+      }.merge(@options)
+
+      super(interval, true)
+    end
+
+    def on_timer
+      @callback.call
+    rescue => e
+      # TODO log?
+      $log.error e.to_s
+      $log.error_backtrace
+    end
+
+    def consume
+      offset = @next_offset
+      @fetch_args[:offset] = offset
+      messages = @kafka.fetch_messages(@fetch_args)
+
+      return if messages.size.zero?
+
+      es = Fluent::MultiEventStream.new
+      tag = @topic_entry.topic
+      tag = @add_prefix + "." + tag if @add_prefix
+      tag = tag + "." + @add_suffix if @add_suffix
+
+      messages.each { |msg|
+        begin
+          record = @parser.call(msg, @topic_entry)
+          if @use_record_time
+            if @time_format
+              record_time = @time_parser.parse(record['time'])
+            else
+              record_time = record['time']
+            end
+          else
+            record_time = Fluent::Engine.now
+          end
+          es.add(record_time, record)
+        rescue => e
+          $log.warn "parser error in #{@topic_entry.topic}/#{@topic_entry.partition}", :error => e.to_s, :value => msg.value, :offset => msg.offset
+          $log.debug_backtrace
+        end
+      }
+      offset = messages.last.offset + 1
+
+      unless es.empty?
+        @router.emit_stream(tag, es)
+
+        if @offset_manager
+          @offset_manager.save_offset(offset)
+        end
+        @next_offset = offset
+      end
+    end
+  end
+
+  class TopicEntry
+    def initialize(topic, partition, offset)
+      @topic = topic
+      @partition = partition
+      @offset = offset
+    end
+    attr_reader :topic, :partition, :offset
+  end
+
+  class OffsetManager
+    def initialize(topic_entry, zookeeper, zk_root_node)
+      @zookeeper = zookeeper
+      @zk_path = "#{zk_root_node}/#{topic_entry.topic}/#{topic_entry.partition}/next_offset"
+      create_node(@zk_path, topic_entry.topic, topic_entry.partition)
+    end
+
+    def create_node(zk_path, topic, partition)
+      path = ""
+      zk_path.split(/(\/[^\/]+)/).reject(&:empty?).each { |dir|
+        path = path + dir
+        @zookeeper.create(:path => "#{path}")
+      }
+      $log.trace "use zk offset node : #{path}"
+    end
+
+    def next_offset
+      @zookeeper.get(:path => @zk_path)[:data].to_i
+    end
+
+    def save_offset(offset)
+      @zookeeper.set(:path => @zk_path, :data => offset.to_s)
+      $log.trace "update zk offset node : #{offset.to_s}"
+    end
+  end
+end