fluent-plugin-kafka 0.0.18 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 35a323b016500c1df85bc9631d27aceefe63f633
-  data.tar.gz: c7d13cda1a73ad277400c8aa4ff58e27fdf31a49
+  metadata.gz: 498665d29438e6b21a3425138b42d0d69fa9bc95
+  data.tar.gz: 20c8b8bc1458035a52aa9492bb83fe4f5beee0d0
 SHA512:
-  metadata.gz: 6524c04bcd2fb424fe0332be575116bf1106eb517f7a25297dd7430da0a46f2901a5417b84c9244a56f626860c02a6ce05ec74303946c5c5b9d5584575bc3c7d
-  data.tar.gz: 0aba3a07b0a558c8d172a861634f7aae517b1827b069041198e78b27bc2988be045b85784d3d9a99f8e9bea8449ec1ab8e1c0fc11f7e05cc6916cce31c3fa2db
+  metadata.gz: c5c1ab553c03c001dd80b471d9c6b30a9356750c12e464b232728e54574e4e68a9e9f6f041ea995b1f6a0c3670d96040c2219df3b02e42e4ab93e285c8a302df
+  data.tar.gz: fbfa49c138e08b3dc5a988b932e20e9a33afc993fc7baa6336fd92412f3136bea11a9b42f8d40176eb8b877387545383fe8701af6d9a28202f75f41aa593504a

data/README.md

@@ -19,7 +19,7 @@ Or install it yourself as:
 
 ## Usage
 
-### Input plugin
+### Input plugin (type 'kafka')
 
     <source>
       type   kafka
@@ -64,6 +64,33 @@ Supports a start of processing from the assigned offset for specific topics.
 
 See also [Poseidon::PartitionConsumer](http://www.rubydoc.info/github/bpot/poseidon/Poseidon/PartitionConsumer) for more detailed documentation about Poseidon.
 
+### Input plugin (type 'kafka_group', supports kafka group)
+
+    <source>
+      type   kafka_group
+      brokers <list of broker-host:port, separate with comma, must set>
+      zookeepers <list of broker-host:port, separate with comma, must set>
+      consumer_group <consumer group name, must set>
+      topics <listening topics(separate with comma',')>
+      format <input text type (text|json|ltsv|msgpack)>
+      message_key <key (Optional, for text format only, default is message)>
+      add_prefix <tag prefix (Optional)>
+      add_suffix <tag suffix (Optional)>
+      max_bytes           (integer)    :default => nil (Use default of Poseidon)
+      max_wait_ms         (integer)    :default => nil (Use default of Poseidon)
+      min_bytes           (integer)    :default => nil (Use default of Poseidon)
+      socket_timeout_ms   (integer)    :default => nil (Use default of Poseidon)
+    </source>
+
+Supports following Poseidon::PartitionConsumer options.
+
+- max_bytes — default: 1048576 (1MB) — Maximum number of bytes to fetch
+- max_wait_ms — default: 100 (100ms) — How long to block until the server sends us data.
+- min_bytes — default: 1 (Send us data as soon as it is ready) — Smallest amount of data the server should send us.
+- socket_timeout_ms - default: 10000 (10s) - How long to wait for reply from server. Should be higher than max_wait_ms.
+
+See also [Poseidon::PartitionConsumer](http://www.rubydoc.info/github/bpot/poseidon/Poseidon/PartitionConsumer) for more detailed documentation about Poseidon.
+
 ### Output plugin (non-buffered)
 
     <match *.**>

data/fluent-plugin-kafka.gemspec

@@ -12,9 +12,9 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version = '0.0.18'
+  gem.version = '0.1.0'
   gem.add_dependency 'fluentd'
-  gem.add_dependency 'poseidon'
+  gem.add_dependency 'poseidon_cluster'
   gem.add_dependency 'ltsv'
   gem.add_dependency 'yajl-ruby'
   gem.add_dependency 'msgpack'

data/lib/fluent/plugin/in_kafka_group.rb

@@ -0,0 +1,160 @@
+module Fluent
+
+class KafkaInput < Input
+  Plugin.register_input('kafka_group', self)
+
+  config_param :brokers, :string
+  config_param :zookeepers, :string
+  config_param :consumer_group, :string, :default => nil
+  config_param :topics, :string
+  config_param :interval, :integer, :default => 1 # seconds
+  config_param :format, :string, :default => 'json' # (json|text|ltsv)
+  config_param :message_key, :string, :default => 'message' # for 'text' format only
+  config_param :add_prefix, :string, :default => nil
+  config_param :add_suffix, :string, :default => nil
+
+  # poseidon PartitionConsumer options
+  config_param :max_bytes, :integer, :default => nil
+  config_param :max_wait_ms, :integer, :default => nil
+  config_param :min_bytes, :integer, :default => nil
+  config_param :socket_timeout_ms, :integer, :default => nil
+
+  def initialize
+    super
+    require 'poseidon_cluster'
+  end
+
+  def _config_to_array(config)
+    config_array = config.split(',').map {|k| k.strip }
+    if config_array.empty?
+      raise ConfigError, "kafka_group: '#{config}' is a required parameter"
+    end
+    config_array
+  end
+
+  private :_config_to_array
+
+  def configure(conf)
+    super
+    @broker_list = _config_to_array(@brokers)
+    @zookeeper_list = _config_to_array(@zookeepers)
+    @topic_list = _config_to_array(@topics)
+
+    unless @consumer_group
+      raise ConfigError, "kafka_group: 'consumer_group' is a required parameter"
+    end
+    $log.info "Will watch for topics #{@topic_list} at brokers " \
+              "#{@broker_list}, zookeepers #{@zookeeper_list} and group " \
+              "'#{@consumer_group}'"
+
+    case @format
+    when 'json'
+      require 'yajl'
+    when 'ltsv'
+      require 'ltsv'
+    when 'msgpack'
+      require 'msgpack'
+    end
+  end
+
+  def start
+    @loop = Coolio::Loop.new
+    opt = {}
+    opt[:max_bytes] = @max_bytes if @max_bytes
+    opt[:max_wait_ms] = @max_wait_ms if @max_wait_ms
+    opt[:min_bytes] = @min_bytes if @min_bytes
+    opt[:socket_timeout_ms] = @socket_timeout_ms if @socket_timeout_ms
+
+    @topic_watchers = @topic_list.map {|topic|
+      TopicWatcher.new(topic, @broker_list, @zookeeper_list, @consumer_group,
+                       interval, @format, @message_key, @add_prefix,
+                       @add_suffix, opt)
+    }
+    @topic_watchers.each {|tw|
+      tw.attach(@loop)
+    }
+    @thread = Thread.new(&method(:run))
+  end
+
+  def shutdown
+    @loop.stop
+  end
+
+  def run
+    @loop.run
+  rescue
+    $log.error "unexpected error", :error=>$!.to_s
+    $log.error_backtrace
+  end
+
+  class TopicWatcher < Coolio::TimerWatcher
+    def initialize(topic, broker_list, zookeeper_list, consumer_group,
+                   interval, format, message_key, add_prefix, add_suffix,
+                   options)
+      @topic = topic
+      @callback = method(:consume)
+      @format = format
+      @message_key = message_key
+      @add_prefix = add_prefix
+      @add_suffix = add_suffix
+
+      @consumer = Poseidon::ConsumerGroup.new(
+        consumer_group,
+        broker_list,
+        zookeeper_list,
+        topic,
+        options
+      )
+
+      super(interval, true)
+    end
+
+    def on_timer
+      @callback.call
+    rescue
+        # TODO log?
+        $log.error $!.to_s
+        $log.error_backtrace
+    end
+
+    def consume
+      es = MultiEventStream.new
+      tag = @topic
+      tag = @add_prefix + "." + tag if @add_prefix
+      tag = tag + "." + @add_suffix if @add_suffix
+
+      @consumer.fetch do |partition, bulk|
+        bulk.each do |msg|
+          begin
+            msg_record = parse_line(msg.value)
+            es.add(Time.now.to_i, msg_record)
+          rescue
+            $log.warn msg_record.to_s, :error=>$!.to_s
+            $log.debug_backtrace
+          end
+        end
+      end
+
+      unless es.empty?
+        Engine.emit_stream(tag, es)
+      end
+    end
+
+    def parse_line(record)
+      parsed_record = {}
+      case @format
+      when 'json'
+        parsed_record = Yajl::Parser.parse(record)
+      when 'ltsv'
+        parsed_record = LTSV.parse(record)
+      when 'msgpack'
+        parsed_record = MessagePack.unpack(record)
+      when 'text'
+        parsed_record[@message_key] = record
+      end
+      parsed_record
+    end
+  end
+end
+
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.0.18
+  version: 0.1.0
 platform: ruby
 authors:
 - Hidemasa Togashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-29 00:00:00.000000000 Z
+date: 2015-11-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -25,7 +25,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: poseidon
+  name: poseidon_cluster
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - '>='
@@ -107,6 +107,7 @@ files:
 - Rakefile
 - fluent-plugin-kafka.gemspec
 - lib/fluent/plugin/in_kafka.rb
+- lib/fluent/plugin/in_kafka_group.rb
 - lib/fluent/plugin/out_kafka.rb
 - lib/fluent/plugin/out_kafka_buffered.rb
 - test/helper.rb