fluent-plugin-kafka 0.0.14 → 0.0.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8cfb2bcd144f0b7025ff278383db1ff29495d7ac
-  data.tar.gz: 630b5829301e2521ea22046702357e96a17776da
+  metadata.gz: ea8e8c26bed79cf08bcf26e11213a6aec0757741
+  data.tar.gz: 40df9fd993cd1f5daf1f7f10331c2370986df549
 SHA512:
-  metadata.gz: fd2e8367a7ccd8e9dcf38fed3b061286af56de6795506963a0eac3157e7ed58b247637a3b3cb25f812cb4ea72e5b48c7f7b1e28120886ac7fc60d0b055edc5e9
-  data.tar.gz: b10b51433c160cd96a97fd11188e3645528f82e93a124e7689ead5ab297fbc2448f20e350b3af2dddaa437f6044c2b6e51447c07a765540a487e9ff58f2f0313
+  metadata.gz: f360beef321eeee7dc4dec739d3386e205bf81878ef8777e1970aee147f681dcf930b345efb46bc5a5bdf037caab07ea1a6f7a0abbec21e70581e44c32fb44b9
+  data.tar.gz: 89b381653212a2a4ac66e69df57033f80206d3e2d6c5c48bc4bb5653e548c4a1f61aa85a7427703a30e544c1de429d0457fd1dfd12ea798a81ae13b824813952

data/README.md

@@ -29,8 +29,21 @@ Or install it yourself as:
       format <input text type (text|json|ltsv|msgpack)>
       add_prefix <tag prefix (Optional)>
       add_suffix <tag suffix (Optional)>
+      max_bytes           (integer)    :default => nil (Use default of Poseidon)
+      max_wait_ms         (integer)    :default => nil (Use default of Poseidon)
+      min_bytes           (integer)    :default => nil (Use default of Poseidon)
+      socket_timeout_ms   (integer)    :default => nil (Use default of Poseidon)
     </source>
 
+Supports following Poseidon::PartitionConsumer options.
+
+- max_bytes — default: 1048576 (1MB) — Maximum number of bytes to fetch
+- max_wait_ms — default: 100 (100ms) — How long to block until the server sends us data.
+- min_bytes — default: 1 (Send us data as soon as it is ready) — Smallest amount of data the server should send us.
+- socket_timeout_ms - default: 10000 (10s) - How long to wait for reply from server. Should be higher than max_wait_ms.
+
+See also [Poseidon::PartitionConsumer](http://www.rubydoc.info/github/bpot/poseidon/Poseidon/PartitionConsumer) for more detailed documentation about Poseidon.
+
 ### Output plugin (non-buffered)
 
     <match *.**>

data/fluent-plugin-kafka.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version = '0.0.14'
+  gem.version = '0.0.15'
   gem.add_dependency 'fluentd'
   gem.add_dependency 'poseidon'
   gem.add_dependency 'ltsv'

data/lib/fluent/plugin/in_kafka.rb

@@ -14,6 +14,12 @@ class KafkaInput < Input
   config_param :add_prefix, :string, :default => nil
   config_param :add_suffix, :string, :default => nil
 
+  # poseidon PartitionConsumer options
+  config_param :max_bytes, :integer, :default => nil
+  config_param :max_wait_ms, :integer, :default => nil
+  config_param :min_bytes, :integer, :default => nil
+  config_param :socket_timeout_ms, :integer, :default => nil
+
   def initialize
     super
     require 'poseidon'
@@ -38,8 +44,14 @@ class KafkaInput < Input
 
   def start
     @loop = Coolio::Loop.new
+    opt = {}
+    opt[:max_bytes] = @max_bytes if @max_bytes
+    opt[:max_wait_ms] = @max_wait_ms if @max_wait_ms
+    opt[:min_bytes] = @min_bytes if @min_bytes
+    opt[:socket_timeout_ms] = @socket_timeout_ms if @socket_timeout_ms
+
     @topic_watchers = @topic_list.map {|topic|
-      TopicWatcher.new(topic, @host, @port, @client_id, @partition, @offset, interval, @format, @add_prefix, @add_suffix)
+      TopicWatcher.new(topic, @host, @port, @client_id, @partition, @offset, interval, @format, @add_prefix, @add_suffix, opt)
     }
     @topic_watchers.each {|tw|
       tw.attach(@loop)
@@ -59,7 +71,7 @@ class KafkaInput < Input
   end
 
   class TopicWatcher < Coolio::TimerWatcher
-    def initialize(topic, host, port, client_id, partition, offset, interval, format, add_prefix, add_suffix)
+    def initialize(topic, host, port, client_id, partition, offset, interval, format, add_prefix, add_suffix, options={})
       @topic = topic
       @callback = method(:consume)
       @format = format
@@ -71,7 +83,8 @@ class KafkaInput < Input
         port,                 # port
         topic,                # topic
         partition,            # partition
-        offset                # offset
+        offset,               # offset
+        options               # options
       )
         
       super(interval, true)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.0.14
+  version: 0.0.15
 platform: ruby
 authors:
 - Hidemasa Togashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-06 00:00:00.000000000 Z
+date: 2015-07-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd