fluent-plugin-kafka 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: eac190de45814d289eb9e82f469f4d5a2af50296
-  data.tar.gz: 1b2217fa4de6d4be5d5ba7e05618fbb9dd0b6c92
+  metadata.gz: f58dd81646c8ff29f5f9e55bb200fda728ef83f1
+  data.tar.gz: 2b47f106dfebc12704d5b751290bfff7a42a44ad
 SHA512:
-  metadata.gz: 840c3ee94d3830a43bc6d503bad1676f340730b0a15b4f47bfec31f838fd09aae7029d093cda3b37e39ec3a1fe82908ed8f9ad49e877acb2d49ebee82d4fe520
-  data.tar.gz: 242b6180a85333ba0536f5e0598db0b121ab03a14dae605188845b1b020542787ce8198a1dbac22286fbd854d7e19b6cf38553574ead496b3edf19b356f73287
+  metadata.gz: 6ecc97a6901090f2f9cd54df11e0af55bc9df901ae6c5186867bd14f26796e762b8d1b3e3140d050827408094adb3d0f99a849bd4171d466e4bbe05b806b2cb8
+  data.tar.gz: 41800c1ddd32cd17cccb0af7cdaff2da11e3f7354f050c4b554ce35aeafb1ddff486f41dcee41806dc9ee002cb0c068025183d9e96996e6c45dba4337c10740b

data/Gemfile

@@ -1,4 +1,4 @@
 source 'https://rubygems.org'
 
-# Specify your gem's dependencies in fluent-plugin-kafka-poseidon.gemspec
+# Specify your gem's dependencies in fluent-plugin-kafka.gemspec
 gemspec

data/README.md

@@ -20,22 +20,30 @@ Or install it yourself as:
 
     $ gem install fluent-plugin-kafka
 
+## Requirements
+
+- Ruby 2.1 or later
+- Input plugins work with kafka v0.9 or later
+- Output plugins work with kafka v0.8 or later
+
 ## Usage
 
 ### Input plugin (@type 'kafka')
 
+Consume events by single consumer.
+
     <source>
       @type kafka
 
       brokers <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,..
       topics <listening topics(separate with comma',')>
-      format <input text type (text|json|ltsv|msgpack)>
+      format <input text type (text|json|ltsv|msgpack)> :default => json
       message_key <key (Optional, for text format only, default is message)>
       add_prefix <tag prefix (Optional)>
       add_suffix <tag suffix (Optional)>
 
       # Optionally, you can manage topic offset by using zookeeper
-      offset_zookeeper <zookeer node list (<zookeeper1_host>:<zookeeper1_port>,<zookeeper2_host>:<zookeeper2_port>,..)>
+      offset_zookeeper    <zookeer node list (<zookeeper1_host>:<zookeeper1_port>,<zookeeper2_host>:<zookeeper2_port>,..)>
       offset_zk_root_node <offset path in zookeeper> default => '/fluent-plugin-kafka'
 
       # ruby-kafka consumer options
@@ -67,13 +75,15 @@ See also [ruby-kafka README](https://github.com/zendesk/ruby-kafka#consuming-mes
 
 ### Input plugin (@type 'kafka_group', supports kafka group)
 
+Consume events by kafka consumer group features..
+
     <source>
       @type kafka_group
 
       brokers <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,..
       consumer_group <consumer group name, must set>
       topics <listening topics(separate with comma',')>
-      format <input text type (text|json|ltsv|msgpack)>
+      format <input text type (text|json|ltsv|msgpack)> :default => json
       message_key <key (Optional, for text format only, default is message)>
       add_prefix <tag prefix (Optional)>
       add_suffix <tag suffix (Optional)>
@@ -109,7 +119,7 @@ This plugin uses ruby-kafka producer for writing data. For performance and relia
 
       # ruby-kafka producer options
       max_send_retries  (integer)     :default => 1
-      required_acks     (integer)     :default => 0
+      required_acks     (integer)     :default => -1
       ack_timeout       (integer)     :default => nil (Use default of ruby-kafka)
       compression_codec (gzip|snappy) :default => nil
     </match>
@@ -117,7 +127,7 @@ This plugin uses ruby-kafka producer for writing data. For performance and relia
 Supports following ruby-kafka::Producer options.
 
 - max_send_retries - default: 1 - Number of times to retry sending of messages to a leader.
-- required_acks - default: 0 - The number of acks required per request.
+- required_acks - default: -1 - The number of acks required per request.
 - ack_timeout - default: nil - How long the producer waits for acks. The unit is seconds.
 - compression_codec - default: nil - The codec the producer uses to compress messages.
 
@@ -130,15 +140,15 @@ Install snappy module before you use snappy compression.
 
 #### Load balancing
 
-Messages will be sent broker in a round-robin manner as default by Poseidon, but you can set `default_partition_key` in config file to route messages to a specific broker.
+Messages will be assigned a partition at random as default by ruby-kafka, but messages with the same partition key will always be assigned to the same partition by setting `default_partition_key` in config file.
 If key name `partition_key` exists in a message, this plugin set its value of partition_key as key.
 
 |default_partition_key|partition_key| behavior |
 | --- | --- | --- |
-|Not set|Not exists| All messages are sent in round-robin |
-|Set| Not exists| All messages are sent to specific broker |
-|Not set| Exists | Messages which have partition_key record are sent to specific broker, others are sent in round-robin|
-|Set| Exists | Messages which have partition_key record are sent to specific broker with parition_key, others are sent to specific broker with default_parition_key|
+|Not set|Not exists| All messages are assigned a partition at random |
+|Set| Not exists| All messages are assigned to the specific partition |
+|Not set| Exists | Messages which have partition_key record are assigned to the specific partition, others are assigned a partition at random |
+|Set| Exists | Messages which have partition_key record are assigned to the specific partition with parition_key, others are assigned to the specific partition with default_parition_key |
 
 
 ### Buffered output plugin
@@ -163,7 +173,7 @@ This plugin uses ruby-kafka producer for writing data. This plugin works with re
 
       # ruby-kafka producer options
       max_send_retries    (integer)     :default => 1
-      required_acks       (integer)     :default => 0
+      required_acks       (integer)     :default => -1
       ack_timeout         (integer)     :default => nil (Use default of ruby-kafka)
       compression_codec   (gzip|snappy) :default => nil (No compression)
     </match>
@@ -171,7 +181,7 @@ This plugin uses ruby-kafka producer for writing data. This plugin works with re
 Supports following ruby-kafka's producer options.
 
 - max_send_retries - default: 1 - Number of times to retry sending of messages to a leader.
-- required_acks - default: 0 - The number of acks required per request.
+- required_acks - default: -1 - The number of acks required per request.
 - ack_timeout - default: nil - How long the producer waits for acks. The unit is seconds.
 - compression_codec - default: nil - The codec the producer uses to compress messages.
 

data/fluent-plugin-kafka.gemspec

@@ -12,7 +12,9 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.3.0'
+  gem.version       = '0.3.1'
+  gem.required_ruby_version = ">= 2.1.0"
+
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
   gem.add_dependency 'ltsv'
   gem.add_dependency 'zookeeper'

data/lib/fluent/plugin/out_kafka.rb

@@ -26,7 +26,7 @@ DESC
   # ruby-kafka producer options
   config_param :max_send_retries, :integer, :default => 1,
                :desc => "Number of times to retry sending of messages to a leader."
-  config_param :required_acks, :integer, :default => 0,
+  config_param :required_acks, :integer, :default => -1,
                :desc => "The number of acks required per request."
   config_param :ack_timeout, :integer, :default => nil,
                :desc => "How long the producer waits for acks."

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -33,7 +33,7 @@ DESC
   # ruby-kafka producer options
   config_param :max_send_retries, :integer, :default => 1,
                :desc => "Number of times to retry sending of messages to a leader."
-  config_param :required_acks, :integer, :default => 0,
+  config_param :required_acks, :integer, :default => -1,
                :desc => "The number of acks required per request."
   config_param :ack_timeout, :time, :default => nil,
                :desc => "How long the producer waits for acks."

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-24 00:00:00.000000000 Z
+date: 2016-08-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -135,7 +135,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="