fluent-plugin-kafka 0.0.7 → 0.0.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 65f16e3cb305ed9abe2c9969c929b1f63a73ad9c
-  data.tar.gz: fa75eb12ac1fc0d3596972c9d61c6a2d879e2af3
+  metadata.gz: b1253b7f59126580e11cc6eb1d6f40318f9367bd
+  data.tar.gz: dc75b9c99e96455add399e4645dacf5f26358810
 SHA512:
-  metadata.gz: 74255febdd2152a9cc8665c1d4415bd51b07f3af4089fc82c2de11e2001465f3149e0c8b7fed6bfc5db9a9d729b7f251968a2ac5858bb6c4ce0a435603f8f867
-  data.tar.gz: 6581acbfc5fb1b00df4510f7786e616b8142cc6e168c8bff548752748db629ca3e99df3d3e7b39a429567fb8f41ae9df5bbbf676c12b611561ce1fb2c5f9d061
+  metadata.gz: de391f877d6db11e0ccbd1a35646043274ef5aeab91015ff09981571fc356d24965289c93f599887d12958549114ef6441a4bce4b856728e8e00ca7dc4f1c011
+  data.tar.gz: 8c95e61c6617890c7be10dae81f963b4cdb5257829b8e5618cf4c5dfcda618456429a08d8347bc088520174c0bb63ed674528f41d887e3aa3793334ec9f978f4

data/README.md

@@ -34,27 +34,50 @@ Or install it yourself as:
 ### Output plugin (non-buffered)
 
     <match *.**>
-      type             kafka
-      brokers          <broker1_host>:<broker1_ip>,<broker2_host>:<broker2_ip>,..
-      default_topic    <output topic>
-      output_data_type (json|ltsv|msgpack|attr:<record name>)
+      type                kafka
+      brokers             <broker1_host>:<broker1_ip>,<broker2_host>:<broker2_ip>,..
+      default_topic       <output topic>
+      output_data_type    (json|ltsv|msgpack|attr:<record name>)
       output_include_tag  (true|false) :default => false
       output_include_time (true|false) :default => false
+      max_send_retries    (integer)    :default => 3
+      required_acks       (integer)    :default => 0
+      ack_timeout_ms      (integer)    :default => 1500
     </match>
 
+Supports following Poseidon::Producer options.
+
+- max_send_retries — default: 3 — Number of times to retry sending of messages to a leader.
+- required_acks — default: 0 — The number of acks required per request.
+- ack_timeout_ms — default: 1500 — How long the producer waits for acks.
+
+See also [Poseidon::Producer](http://www.rubydoc.info/github/bpot/poseidon/Poseidon/Producer) for more detailed documentation about Poseidon.
+
 ### Buffered output plugin
 
     <match *.**>
-      type             kafka_buffered
-      brokers          <broker1_host>:<broker1_ip>,<broker2_host>:<broker2_ip>,..
-      default_topic    <output topic>
-      flush_interval   <flush interval (sec) :default => 60>
-      buffer_type      (file|memory)
-      output_data_type (json|ltsv|msgpack|attr:<record name>)
+      type                kafka_buffered
+      brokers             <broker1_host>:<broker1_ip>,<broker2_host>:<broker2_ip>,..
+      default_topic       <output topic>
+      flush_interval      <flush interval (sec) :default => 60>
+      buffer_type         (file|memory)
+      output_data_type    (json|ltsv|msgpack|attr:<record name>)
       output_include_tag  (true|false) :default => false
       output_include_time (true|false) :default => false
+      max_send_retries    (integer)    :default => 3
+      required_acks       (integer)    :default => 0
+      ack_timeout_ms      (integer)    :default => 1500
     </match>
 
+Supports following Poseidon::Producer options.
+
+- max_send_retries — default: 3 — Number of times to retry sending of messages to a leader.
+- required_acks — default: 0 — The number of acks required per request.
+- ack_timeout_ms — default: 1500 — How long the producer waits for acks.
+
+See also [Poseidon::Producer](http://www.rubydoc.info/github/bpot/poseidon/Poseidon/Producer) for more detailed documentation about Poseidon.
+
+
 ## Contributing
 
 1. Fork it

data/fluent-plugin-kafka.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version = '0.0.7'
+  gem.version = '0.0.8'
   gem.add_dependency 'fluentd'
   gem.add_dependency 'poseidon'
   gem.add_dependency 'ltsv'

data/lib/fluent/plugin/out_kafka.rb

@@ -13,6 +13,12 @@ class Fluent::KafkaOutput < Fluent::Output
   config_param :output_data_type, :string, :default => 'json'
   config_param :output_include_tag, :bool, :default => false
   config_param :output_include_time, :bool, :default => false
+
+  # poseidon producer options
+  config_param :max_send_retries, :integer, :default => 3
+  config_param :required_acks, :integer, :default => 0
+  config_param :ack_timeout_ms, :integer, :default => 1500
+
   attr_accessor :output_data_type
   attr_accessor :field_separator
 
@@ -87,7 +93,7 @@ class Fluent::KafkaOutput < Fluent::Output
       topic = record['topic'] || self.default_topic || tag
       partition = record['partition'] || self.default_partition
       message = Poseidon::MessageToSend.new(topic, parse_record(record))
-      @producers[topic] ||= Poseidon::Producer.new(@seed_brokers, self.client_id)
+      @producers[topic] ||= Poseidon::Producer.new(@seed_brokers, self.client_id, :max_send_retries => @max_send_retries, :required_acks => @required_acks, :ack_timeout_ms => @ack_timeout_ms)
       @producers[topic].send_messages([message])
     end
   end

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -14,6 +14,12 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
   config_param :output_data_type, :string, :default => 'json'
   config_param :output_include_tag, :bool, :default => false
   config_param :output_include_time, :bool, :default => false
+
+  # poseidon producer options
+  config_param :max_send_retries, :integer, :default => 3
+  config_param :required_acks, :integer, :default => 0
+  config_param :ack_timeout_ms, :integer, :default => 1500
+
   attr_accessor :output_data_type
   attr_accessor :field_separator
 
@@ -102,7 +108,7 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
     records_by_topic.each { |topic, partitions|
       partitions.each_with_index { |messages, partition|
         next if not messages
-        @producers[topic] ||= Poseidon::Producer.new(@seed_brokers, self.client_id)
+        @producers[topic] ||= Poseidon::Producer.new(@seed_brokers, self.client_id, :max_send_retries => @max_send_retries, :required_acks => @required_acks, :ack_timeout_ms => @ack_timeout_ms)
         @producers[topic].send_messages(messages)
       }
     }

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.8
 platform: ruby
 authors:
 - Hidemasa Togashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-13 00:00:00.000000000 Z
+date: 2014-10-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd