fluent-plugin-kafka 0.0.16 → 0.0.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a72f6359b8397411999429eeda7366e05114e925
-  data.tar.gz: b8dc3a2860a941b43d02f8a98562eec997bd065f
+  metadata.gz: 1fff4cfe054606b489a5bba65e1dd904801bd745
+  data.tar.gz: 8450250a2968bd4104c014f2b6ba2e801719a266
 SHA512:
-  metadata.gz: 6af36d8001d89b0b7b181118f66b6a38f3f4a567e70d4a744e3e69496c00a84939f9272aabba9650bf02ee57fe00113b6ca3782b674a20fa108e8892f8f00e81
-  data.tar.gz: c5a8f2ad75696645cd62a40cc0aad87f471a59119d73fbf04c2be8b0d68d195e33cfb9c2b82afa61312dc2d1160de91182a64686213fec15b37ae725dfbcd043
+  metadata.gz: 171f15df160c6522191f6ced594e1dfb46f6a84698bba82dcbc3ff7c2a73ea508b68eb19c407b9f673b696f62357a5fd51e6cc6cf326983c8405f68c930bd4ad
+  data.tar.gz: d0d95aa95b9b4542cb371812216f11bb4475c2a27868e7ff248278f8698be68cd6b70d83115201750901f202fc4fdb131cd6d37ebe29e0c1419b9125e5f90aa0

data/README.md

@@ -43,6 +43,25 @@ Supports following Poseidon::PartitionConsumer options.
 - min_bytes — default: 1 (Send us data as soon as it is ready) — Smallest amount of data the server should send us.
 - socket_timeout_ms - default: 10000 (10s) - How long to wait for reply from server. Should be higher than max_wait_ms.
 
+Supports a start of processing from the assigned offset for specific topics.
+
+    <source>
+      type   kafka
+      host   <broker host>
+      port   <broker port: default=9092>
+      format <input text type (text|json|ltsv|msgpack)>
+      <topic>
+        topic       <listening topic>
+        partition   <listening partition: default=0>
+        offset      <listening start offset: default=-1>
+      </topic>
+      <topic>
+        topic       <listening topic>
+        partition   <listening partition: default=0>
+        offset      <listening start offset: default=-1>
+      </topic>
+    </source>
+
 See also [Poseidon::PartitionConsumer](http://www.rubydoc.info/github/bpot/poseidon/Poseidon/PartitionConsumer) for more detailed documentation about Poseidon.
 
 ### Output plugin (non-buffered)

data/fluent-plugin-kafka.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version = '0.0.16'
+  gem.version = '0.0.17'
   gem.add_dependency 'fluentd'
   gem.add_dependency 'poseidon'
   gem.add_dependency 'ltsv'

data/lib/fluent/plugin/in_kafka.rb

@@ -8,12 +8,13 @@ class KafkaInput < Input
   config_param :host, :string, :default => 'localhost'
   config_param :port, :integer, :default => 9092
   config_param :interval, :integer, :default => 1 # seconds
-  config_param :topics, :string
+  config_param :topics, :string, :default => nil
   config_param :client_id, :string, :default => 'kafka'
   config_param :partition, :integer, :default => 0
   config_param :offset, :integer, :default => -1
   config_param :add_prefix, :string, :default => nil
   config_param :add_suffix, :string, :default => nil
+  config_param :add_offset_in_record, :bool, :default => false
 
   # poseidon PartitionConsumer options
   config_param :max_bytes, :integer, :default => nil
@@ -28,9 +29,25 @@ class KafkaInput < Input
 
   def configure(conf)
     super
-    @topic_list = @topics.split(',').map {|topic| topic.strip }
+
+    @topic_list = []
+    if @topics
+      @topic_list = @topics.split(',').map { |topic|
+        TopicEntry.new(topic.strip, @partition, @offset)
+      }
+    else
+      conf.elements.select { |element| element.name == 'topic' }.each do |element|
+        unless element.has_key?('topic')
+          raise ConfigError, "kafka: 'topic' is a require parameter in 'topic element'."
+        end
+        partition = element.has_key?('partition') ? element['partition'].to_i : 0
+        offset = element.has_key?('offset') ? element['offset'].to_i : -1
+        @topic_list.push(TopicEntry.new(element['topic'], partition, offset))
+      end
+    end
+
     if @topic_list.empty?
-      raise ConfigError, "kafka: 'topics' is a require parameter"
+      raise ConfigError, "kafka: 'topics' or 'topic element' is a require parameter"
     end
 
     case @format
@@ -51,8 +68,19 @@ class KafkaInput < Input
     opt[:min_bytes] = @min_bytes if @min_bytes
     opt[:socket_timeout_ms] = @socket_timeout_ms if @socket_timeout_ms
 
-    @topic_watchers = @topic_list.map {|topic|
-      TopicWatcher.new(topic, @host, @port, @client_id, @partition, @offset, interval, @format, @message_key, @add_prefix, @add_suffix, opt)
+    @topic_watchers = @topic_list.map {|topic_entry|
+      TopicWatcher.new(
+        topic_entry,
+        @host,
+        @port,
+        @client_id,
+        interval,
+        @format,
+        @message_key,
+        @add_offset_in_record,
+        @add_prefix,
+        @add_suffix,
+        opt)
     }
     @topic_watchers.each {|tw|
       tw.attach(@loop)
@@ -72,21 +100,22 @@ class KafkaInput < Input
   end
 
   class TopicWatcher < Coolio::TimerWatcher
-    def initialize(topic, host, port, client_id, partition, offset, interval, format, message_key, add_prefix, add_suffix, options={})
-      @topic = topic
+    def initialize(topic_entry, host, port, client_id, interval, format, message_key, add_offset_in_record, add_prefix, add_suffix, options={})
+      @topic_entry = topic_entry
+      @add_offset_in_record = add_offset_in_record
       @callback = method(:consume)
       @format = format
       @message_key = message_key
       @add_prefix = add_prefix
       @add_suffix = add_suffix
       @consumer = Poseidon::PartitionConsumer.new(
-        client_id,            # client_id
-        host,                 # host
-        port,                 # port
-        topic,                # topic
-        partition,            # partition
-        offset,               # offset
-        options               # options
+        client_id,              # client_id
+        host,                   # host
+        port,                   # port
+        topic_entry.topic,      # topic
+        topic_entry.partition,  # partition
+        topic_entry.offset,     # offset
+        options                 # options
       )
         
       super(interval, true)
@@ -102,12 +131,13 @@ class KafkaInput < Input
 
     def consume
       es = MultiEventStream.new
-      tag = @topic
+      tag = @topic_entry.topic
       tag = @add_prefix + "." + tag if @add_prefix
       tag = tag + "." + @add_suffix if @add_suffix
       @consumer.fetch.each { |msg|
         begin
           msg_record = parse_line(msg.value)
+          msg_record = decorate_offset(msg_record, msg.offset) if @add_offset_in_record
           es.add(Time.now.to_i, msg_record)
         rescue
           $log.warn msg_record.to_s, :error=>$!.to_s
@@ -134,7 +164,39 @@ class KafkaInput < Input
       end
       parsed_record
     end
+
+    def decorate_offset(record, offset)
+      case @format
+      when 'json'
+        add_offset_in_hash(record, @topic_entry.topic, @topic_entry.partition, offset)
+      when 'ltsv'
+        record.each { |line|
+          add_offset_in_hash(line, @topic_entry.topic, @topic_entry.partition, offset)
+        }
+      when 'msgpack'
+        add_offset_in_hash(record, @topic_entry.topic, @topic_entry.partition, offset)
+      when 'text'
+        add_offset_in_hash(record, @topic_entry.topic, @topic_entry.partition, offset)
+      end
+      record
+    end
+
+    def add_offset_in_hash(hash, topic, partition, offset)
+      hash['kafka_topic'] = topic
+      hash['kafka_partition'] = partition
+      hash['kafka_offset'] = offset
+    end
   end
+
+  class TopicEntry
+    def initialize(topic, partition, offset)
+      @topic = topic
+      @partition = partition
+      @offset = offset
+    end
+    attr_reader :topic, :partition, :offset
+  end
+
 end
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.0.16
+  version: 0.0.17
 platform: ruby
 authors:
 - Hidemasa Togashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-08 00:00:00.000000000 Z
+date: 2015-09-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd