fluent-plugin-kafka 0.0.17 → 0.0.18

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1fff4cfe054606b489a5bba65e1dd904801bd745
-  data.tar.gz: 8450250a2968bd4104c014f2b6ba2e801719a266
+  metadata.gz: 35a323b016500c1df85bc9631d27aceefe63f633
+  data.tar.gz: c7d13cda1a73ad277400c8aa4ff58e27fdf31a49
 SHA512:
-  metadata.gz: 171f15df160c6522191f6ced594e1dfb46f6a84698bba82dcbc3ff7c2a73ea508b68eb19c407b9f673b696f62357a5fd51e6cc6cf326983c8405f68c930bd4ad
-  data.tar.gz: d0d95aa95b9b4542cb371812216f11bb4475c2a27868e7ff248278f8698be68cd6b70d83115201750901f202fc4fdb131cd6d37ebe29e0c1419b9125e5f90aa0
+  metadata.gz: 6524c04bcd2fb424fe0332be575116bf1106eb517f7a25297dd7430da0a46f2901a5417b84c9244a56f626860c02a6ce05ec74303946c5c5b9d5584575bc3c7d
+  data.tar.gz: 0aba3a07b0a558c8d172a861634f7aae517b1827b069041198e78b27bc2988be045b85784d3d9a99f8e9bea8449ec1ab8e1c0fc11f7e05cc6916cce31c3fa2db

data/fluent-plugin-kafka.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version = '0.0.17'
+  gem.version = '0.0.18'
   gem.add_dependency 'fluentd'
   gem.add_dependency 'poseidon'
   gem.add_dependency 'ltsv'

data/lib/fluent/plugin/in_kafka.rb

@@ -16,6 +16,9 @@ class KafkaInput < Input
   config_param :add_suffix, :string, :default => nil
   config_param :add_offset_in_record, :bool, :default => false
 
+  config_param :offset_zookeeper, :string, :default => nil
+  config_param :offset_zk_root_node, :string, :default => '/fluent-plugin-kafka'
+
   # poseidon PartitionConsumer options
   config_param :max_bytes, :integer, :default => nil
   config_param :max_wait_ms, :integer, :default => nil
@@ -25,6 +28,7 @@ class KafkaInput < Input
   def initialize
     super
     require 'poseidon'
+    require 'zookeeper'
   end
 
   def configure(conf)
@@ -68,7 +72,10 @@ class KafkaInput < Input
     opt[:min_bytes] = @min_bytes if @min_bytes
     opt[:socket_timeout_ms] = @socket_timeout_ms if @socket_timeout_ms
 
+    @zookeeper = Zookeeper.new(@offset_zookeeper) if @offset_zookeeper
+
     @topic_watchers = @topic_list.map {|topic_entry|
+      offset_manager = OffsetManager.new(topic_entry, @zookeeper, @offset_zk_root_node) if @offset_zookeeper
       TopicWatcher.new(
         topic_entry,
         @host,
@@ -80,6 +87,7 @@ class KafkaInput < Input
         @add_offset_in_record,
         @add_prefix,
         @add_suffix,
+        offset_manager,
         opt)
     }
     @topic_watchers.each {|tw|
@@ -90,6 +98,7 @@ class KafkaInput < Input
 
   def shutdown
     @loop.stop
+    @zookeeper.close! if @zookeeper
   end
 
   def run
@@ -100,24 +109,26 @@ class KafkaInput < Input
   end
 
   class TopicWatcher < Coolio::TimerWatcher
-    def initialize(topic_entry, host, port, client_id, interval, format, message_key, add_offset_in_record, add_prefix, add_suffix, options={})
+    def initialize(topic_entry, host, port, client_id, interval, format, message_key, add_offset_in_record, add_prefix, add_suffix, offset_manager, options={})
       @topic_entry = topic_entry
-      @add_offset_in_record = add_offset_in_record
+      @host = host
+      @port = port
+      @client_id = client_id
       @callback = method(:consume)
       @format = format
       @message_key = message_key
+      @add_offset_in_record = add_offset_in_record
       @add_prefix = add_prefix
       @add_suffix = add_suffix
-      @consumer = Poseidon::PartitionConsumer.new(
-        client_id,              # client_id
-        host,                   # host
-        port,                   # port
-        topic_entry.topic,      # topic
-        topic_entry.partition,  # partition
-        topic_entry.offset,     # offset
-        options                 # options
-      )
-        
+      @options = options
+      @offset_manager = offset_manager
+
+      @next_offset = @topic_entry.offset
+      if @topic_entry.offset == -1 && offset_manager
+        @next_offset = offset_manager.next_offset
+      end
+      @consumer = create_consumer(@next_offset)      
+
       super(interval, true)
     end
 
@@ -134,6 +145,11 @@ class KafkaInput < Input
       tag = @topic_entry.topic
       tag = @add_prefix + "." + tag if @add_prefix
       tag = tag + "." + @add_suffix if @add_suffix
+
+      if @offset_manager && @consumer.next_offset != @next_offset
+        @consumer = create_consumer(@next_offset)
+      end
+
       @consumer.fetch.each { |msg|
         begin
           msg_record = parse_line(msg.value)
@@ -147,9 +163,28 @@ class KafkaInput < Input
 
       unless es.empty?
         Engine.emit_stream(tag, es)
+
+        if @offset_manager
+          next_offset = @consumer.next_offset
+          @offset_manager.save_offset(next_offset)
+          @next_offset = next_offset
+        end
       end
     end
 
+    def create_consumer(offset)
+      @consumer.close if @consumer
+      Poseidon::PartitionConsumer.new(
+        @client_id,             # client_id
+        @host,                  # host
+        @port,                  # port
+        @topic_entry.topic,     # topic
+        @topic_entry.partition, # partition
+        offset,                 # offset
+        @options                # options
+      )
+    end
+
     def parse_line(record)
       parsed_record = {}
       case @format
@@ -197,6 +232,31 @@ class KafkaInput < Input
     attr_reader :topic, :partition, :offset
   end
 
+  class OffsetManager
+    def initialize(topic_entry, zookeeper, zk_root_node)
+      @zookeeper = zookeeper
+      @zk_path = "#{zk_root_node}/#{topic_entry.topic}/#{topic_entry.partition}/next_offset"
+      create_node(@zk_path, topic_entry.topic, topic_entry.partition)
+    end
+
+    def create_node(zk_path, topic, partition)
+      path = ""
+      zk_path.split(/(\/[^\/]+)/).reject(&:empty?).each { |dir|
+        path = path + dir
+        @zookeeper.create(:path => "#{path}")
+      }
+      $log.trace "use zk offset node : #{path}"
+    end
+
+    def next_offset
+      @zookeeper.get(:path => @zk_path)[:data].to_i
+    end
+
+    def save_offset(offset)
+      @zookeeper.set(:path => @zk_path, :data => offset.to_s)
+      $log.trace "update zk offset node : #{offset.to_s}"
+    end
+  end
 end
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.0.17
+  version: 0.0.18
 platform: ruby
 authors:
 - Hidemasa Togashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-14 00:00:00.000000000 Z
+date: 2015-10-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd