fluent-plugin-kafka 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d1413ec34a22790b9d9190a1c89500674dd24b99
-  data.tar.gz: 0d63aa90d9ac7420d46baee550eae4b27419d0bf
+  metadata.gz: 1359c3e9e2c824e3ad36cf21eae0eeae3aea80f4
+  data.tar.gz: 0593adb6cac966b2461b52cacdc9b4d7be06a112
 SHA512:
-  metadata.gz: 305f5754a3eaf8220e4b07e1899b7ec2002eeca0b8554b7a998ed72b5ee834ee6cb16f38fe650653bab8a4b3ac4a575f37789a0a5bb81595f147fe1bc810cb2c
-  data.tar.gz: 2093bbb9df3cf5c2d2f7a186b60daf55712d409abaddf10586f8ff486e43187cc715de09558bd6a3c8b6af571f97ccc21b5222a159c9580f9c6541315e4d5e08
+  metadata.gz: d23dcc2070e66fb1eea47ad58318489db7c3ecc9b0e1dbf7fac434266bdfebcd5ae8933fd91208ca71c3422f00a2b354bc71425301a0ce3b8ccc036c774d2dae
+  data.tar.gz: b72d588c40e65e30be3f491743898f8cc0585430191c20b5d40f35eb2e26f283092eb96d6d9350ed89f49173791ae757ed7928a6146fbc63d9ad83cbb19e5d90

data/README.md

@@ -19,10 +19,10 @@ Or install it yourself as:
 
 ## Usage
 
-### Input plugin (type 'kafka')
+### Input plugin (@type 'kafka')
 
     <source>
-      type   kafka
+      @type  kafka
       host   <broker host>
       port   <broker port: default=9092>
       topics <listening topics(separate with comma',')>
@@ -46,7 +46,7 @@ Supports following Poseidon::PartitionConsumer options.
 Supports a start of processing from the assigned offset for specific topics.
 
     <source>
-      type   kafka
+      @type  kafka
       host   <broker host>
       port   <broker port: default=9092>
       format <input text type (text|json|ltsv|msgpack)>
@@ -64,10 +64,10 @@ Supports a start of processing from the assigned offset for specific topics.
 
 See also [Poseidon::PartitionConsumer](http://www.rubydoc.info/github/bpot/poseidon/Poseidon/PartitionConsumer) for more detailed documentation about Poseidon.
 
-### Input plugin (type 'kafka_group', supports kafka group)
+### Input plugin (@type 'kafka_group', supports kafka group)
 
     <source>
-      type   kafka_group
+      @type   kafka_group
       brokers <list of broker-host:port, separate with comma, must set>
       zookeepers <list of broker-host:port, separate with comma, must set>
       consumer_group <consumer group name, must set>
@@ -94,7 +94,7 @@ See also [Poseidon::PartitionConsumer](http://www.rubydoc.info/github/bpot/posei
 ### Output plugin (non-buffered)
 
     <match *.**>
-      type                kafka
+      @type               kafka
 
       # Brokers: you can choose either brokers or zookeeper.
       brokers             <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,.. # Set brokers directly
@@ -141,7 +141,7 @@ If key name `partition_key` exists in a message, this plugin set its value of pa
 ### Buffered output plugin
 
     <match *.**>
-      type                kafka_buffered
+      @type               kafka_buffered
 
       # Brokers: you can choose either brokers or zookeeper.
       brokers             <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,.. # Set brokers directly
@@ -174,17 +174,17 @@ Install snappy module before you use snappy compression.
 
     $ gem install snappy
 
-    #### Load balancing
+#### Load balancing
 
-    Messages will be sent broker in a round-robin manner as default by Poseidon, but you can set `default_partition_key` in config file to route messages to a specific broker.
-    If key name `partition_key` exists in a message, this plugin set its value of partition_key as key.
+Messages will be sent broker in a round-robin manner as default by Poseidon, but you can set `default_partition_key` in config file to route messages to a specific broker.
+If key name `partition_key` exists in a message, this plugin set its value of partition_key as key.
 
-    |default_partition_key|partition_key| behavior |
-    |-|-|
-    |Not set|Not exists| All messages are sent in round-robin |
-    |Set| Not exists| All messages are sent to specific broker |
-    |Not set| Exists | Messages which have partition_key record are sent to specific broker, others are sent in round-robin|
-    |Set| Exists | Messages which have partition_key record are sent to specific broker with parition_key, others are sent to specific broker with default_parition_key|
+|default_partition_key|partition_key| behavior |
+|-|-|
+|Not set|Not exists| All messages are sent in round-robin |
+|Set| Not exists| All messages are sent to specific broker |
+|Not set| Exists | Messages which have partition_key record are sent to specific broker, others are sent in round-robin|
+|Set| Exists | Messages which have partition_key record are sent to specific broker with parition_key, others are sent to specific broker with default_parition_key|
 
 ## Contributing
 

data/fluent-plugin-kafka.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version = '0.1.2'
+  gem.version = '0.1.3'
   gem.add_dependency 'fluentd'
   gem.add_dependency 'poseidon_cluster'
   gem.add_dependency 'ltsv'

data/lib/fluent/plugin/in_kafka.rb

@@ -3,27 +3,41 @@ module Fluent
 class KafkaInput < Input
   Plugin.register_input('kafka', self)
 
-  config_param :format, :string, :default => 'json' # (json|text|ltsv)
-  config_param :message_key, :string, :default => 'message' # for 'text' format only
-  config_param :host, :string, :default => 'localhost'
-  config_param :port, :integer, :default => 9092
-  config_param :interval, :integer, :default => 1 # seconds
-  config_param :topics, :string, :default => nil
+  config_param :format, :string, :default => 'json',
+               :desc => "Supported format: (json|text|ltsv|msgpack)"
+  config_param :message_key, :string, :default => 'message',
+               :desc => "For 'text' format only."
+  config_param :host, :string, :default => 'localhost',
+               :desc => "Broker host"
+  config_param :port, :integer, :default => 9092,
+               :desc => "Broker port"
+  config_param :interval, :integer, :default => 1, # seconds
+               :desc => "Interval (Unit: seconds)"
+  config_param :topics, :string, :default => nil,
+               :desc => "Listening topics(separate with comma',')"
   config_param :client_id, :string, :default => 'kafka'
-  config_param :partition, :integer, :default => 0
-  config_param :offset, :integer, :default => -1
-  config_param :add_prefix, :string, :default => nil
-  config_param :add_suffix, :string, :default => nil
+  config_param :partition, :integer, :default => 0,
+               :desc => "Listening partition"
+  config_param :offset, :integer, :default => -1,
+               :desc => "Listening start offset"
+  config_param :add_prefix, :string, :default => nil,
+               :desc => "Tag prefix"
+  config_param :add_suffix, :string, :default => nil,
+               :desc => "tag suffix"
   config_param :add_offset_in_record, :bool, :default => false
 
   config_param :offset_zookeeper, :string, :default => nil
   config_param :offset_zk_root_node, :string, :default => '/fluent-plugin-kafka'
 
   # poseidon PartitionConsumer options
-  config_param :max_bytes, :integer, :default => nil
-  config_param :max_wait_ms, :integer, :default => nil
-  config_param :min_bytes, :integer, :default => nil
-  config_param :socket_timeout_ms, :integer, :default => nil
+  config_param :max_bytes, :integer, :default => nil,
+               :desc => "Maximum number of bytes to fetch."
+  config_param :max_wait_ms, :integer, :default => nil,
+               :desc => "How long to block until the server sends us data."
+  config_param :min_bytes, :integer, :default => nil,
+               :desc => "Smallest amount of data the server should send us."
+  config_param :socket_timeout_ms, :integer, :default => nil,
+               :desc => "How long to wait for reply from server. Should be higher than max_wait_ms."
 
   unless method_defined?(:router)
     define_method("router") { Fluent::Engine }

data/lib/fluent/plugin/in_kafka_group.rb

@@ -3,21 +3,34 @@ module Fluent
 class KafkaGroupInput < Input
   Plugin.register_input('kafka_group', self)
 
-  config_param :brokers, :string
-  config_param :zookeepers, :string
-  config_param :consumer_group, :string, :default => nil
-  config_param :topics, :string
-  config_param :interval, :integer, :default => 1 # seconds
-  config_param :format, :string, :default => 'json' # (json|text|ltsv)
-  config_param :message_key, :string, :default => 'message' # for 'text' format only
-  config_param :add_prefix, :string, :default => nil
-  config_param :add_suffix, :string, :default => nil
+  config_param :brokers, :string,
+               :desc => "List of broker-host:port, separate with comma, must set."
+  config_param :zookeepers, :string,
+               :desc => "List of broker-host:port, separate with comma, must set."
+  config_param :consumer_group, :string, :default => nil,
+               :desc => "Consumer group name, must set."
+  config_param :topics, :string,
+               :desc => "Listening topics(separate with comma',')."
+  config_param :interval, :integer, :default => 1, # seconds
+               :desc => "Interval (Unit: seconds)"
+  config_param :format, :string, :default => 'json',
+               :desc => "Supported format: (json|text|ltsv|msgpack)"
+  config_param :message_key, :string, :default => 'message',
+               :desc => "For 'text' format only."
+  config_param :add_prefix, :string, :default => nil,
+               :desc => "Tag prefix (Optional)"
+  config_param :add_suffix, :string, :default => nil,
+               :desc => "Tag suffix (Optional)"
 
   # poseidon PartitionConsumer options
-  config_param :max_bytes, :integer, :default => nil
-  config_param :max_wait_ms, :integer, :default => nil
-  config_param :min_bytes, :integer, :default => nil
-  config_param :socket_timeout_ms, :integer, :default => nil
+  config_param :max_bytes, :integer, :default => nil,
+               :desc => "Maximum number of bytes to fetch."
+  config_param :max_wait_ms, :integer, :default => nil,
+               :desc => "How long to block until the server sends us data."
+  config_param :min_bytes, :integer, :default => nil,
+               :desc => "Smallest amount of data the server should send us."
+  config_param :socket_timeout_ms, :integer, :default => nil,
+               :desc => "How long to wait for reply from server. Should be higher than max_wait_ms."
 
   unless method_defined?(:router)
     define_method("router") { Fluent::Engine }

data/lib/fluent/plugin/out_kafka.rb

@@ -6,20 +6,32 @@ class Fluent::KafkaOutput < Fluent::Output
     require 'poseidon'
   end
 
-  config_param :brokers, :string, :default => 'localhost:9092'
-  config_param :zookeeper, :string, :default => nil
-  config_param :default_topic, :string, :default => nil
+  config_param :brokers, :string, :default => 'localhost:9092',
+               :desc => <<-DESC
+Set brokers directly
+<broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,..
+Note that you can choose to use either brokers or zookeeper.
+DESC
+  config_param :zookeeper, :string, :default => nil,
+               :desc => "Set brokers via Zookeeper: <zookeeper_host>:<zookeeper_port>"
+  config_param :default_topic, :string, :default => nil,
+               :desc => "Output topic."
   config_param :default_partition_key, :string, :default => nil
   config_param :client_id, :string, :default => 'kafka'
-  config_param :output_data_type, :string, :default => 'json'
+  config_param :output_data_type, :string, :default => 'json',
+               :desc => "Supported format: (json|ltsv|msgpack|attr:<record name>|<formatter name>)"
   config_param :output_include_tag, :bool, :default => false
   config_param :output_include_time, :bool, :default => false
 
   # poseidon producer options
-  config_param :max_send_retries, :integer, :default => 3
-  config_param :required_acks, :integer, :default => 0
-  config_param :ack_timeout_ms, :integer, :default => 1500
-  config_param :compression_codec, :string, :default => 'none'
+  config_param :max_send_retries, :integer, :default => 3,
+               :desc => "Number of times to retry sending of messages to a leader."
+  config_param :required_acks, :integer, :default => 0,
+               :desc => "The number of acks required per request."
+  config_param :ack_timeout_ms, :integer, :default => 1500,
+               :desc => "How long the producer waits for acks."
+  config_param :compression_codec, :string, :default => 'none',
+               :desc => "The codec the producer uses to compress messages."
 
   attr_accessor :output_data_type
   attr_accessor :field_separator
@@ -38,6 +50,7 @@ class Fluent::KafkaOutput < Fluent::Output
         broker = Yajl.load(z.get(:path => "/brokers/ids/#{id}")[:data])
         @seed_brokers.push("#{broker['host']}:#{broker['port']}")
       end
+      z.close
       log.info "brokers has been refreshed via Zookeeper: #{@seed_brokers}"
     end
     begin
@@ -141,6 +154,7 @@ class Fluent::KafkaOutput < Fluent::Output
       end
     rescue Exception => e
       log.warn("Send exception occurred: #{e}")
+      @producer.close if @producer
       refresh_producer()
       raise e
     end

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -7,21 +7,41 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
     require 'poseidon'
   end
 
-  config_param :brokers, :string, :default => 'localhost:9092'
-  config_param :zookeeper, :string, :default => nil
-  config_param :default_topic, :string, :default => nil
+  config_param :brokers, :string, :default => 'localhost:9092',
+               :desc => <<-DESC
+Set brokers directly:
+<broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,..
+Brokers: you can choose to use either brokers or zookeeper.
+DESC
+  config_param :zookeeper, :string, :default => nil,
+               :desc => <<-DESC
+Set brokers via Zookeeper:
+<zookeeper_host>:<zookeeper_port>
+DESC
+  config_param :default_topic, :string, :default => nil,
+               :desc => "Output topic"
   config_param :default_partition_key, :string, :default => nil
   config_param :client_id, :string, :default => 'kafka'
-  config_param :output_data_type, :string, :default => 'json'
+  config_param :output_data_type, :string, :default => 'json',
+               :desc => <<-DESC
+Supported format: (json|ltsv|msgpack|attr:<record name>|<formatter name>)
+DESC
   config_param :output_include_tag, :bool, :default => false
   config_param :output_include_time, :bool, :default => false
   config_param :kafka_agg_max_bytes, :size, :default => 4*1024  #4k
 
   # poseidon producer options
-  config_param :max_send_retries, :integer, :default => 3
-  config_param :required_acks, :integer, :default => 0
-  config_param :ack_timeout_ms, :integer, :default => 1500
-  config_param :compression_codec, :string, :default => 'none'
+  config_param :max_send_retries, :integer, :default => 3,
+               :desc => "Number of times to retry sending of messages to a leader."
+  config_param :required_acks, :integer, :default => 0,
+               :desc => "The number of acks required per request."
+  config_param :ack_timeout_ms, :integer, :default => 1500,
+               :desc => "How long the producer waits for acks."
+  config_param :compression_codec, :string, :default => 'none',
+               :desc => <<-DESC
+The codec the producer uses to compress messages.
+Supported codecs: (none|gzip|snappy)
+DESC
 
   attr_accessor :output_data_type
   attr_accessor :field_separator
@@ -40,6 +60,7 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
         broker = Yajl.load(z.get(:path => "/brokers/ids/#{id}")[:data])
         @seed_brokers.push("#{broker['host']}:#{broker['port']}")
       end
+      z.close
       log.info "brokers has been refreshed via Zookeeper: #{@seed_brokers}"
     end
     begin
@@ -156,6 +177,7 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
     end
   rescue Exception => e
     log.warn "Send exception occurred: #{e}"
+    @producer.close if @producer
     refresh_producer()
     # Raise exception to retry sendind messages
     raise e

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Hidemasa Togashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-12-11 00:00:00.000000000 Z
+date: 2016-02-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd