fluent-plugin-kafka 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1359c3e9e2c824e3ad36cf21eae0eeae3aea80f4
-  data.tar.gz: 0593adb6cac966b2461b52cacdc9b4d7be06a112
+  metadata.gz: 12a7a6c9b46188b26eddffb85896388788d5b63b
+  data.tar.gz: 851a09fe0932c3ec7001d671cac9fded8a707b67
 SHA512:
-  metadata.gz: d23dcc2070e66fb1eea47ad58318489db7c3ecc9b0e1dbf7fac434266bdfebcd5ae8933fd91208ca71c3422f00a2b354bc71425301a0ce3b8ccc036c774d2dae
-  data.tar.gz: b72d588c40e65e30be3f491743898f8cc0585430191c20b5d40f35eb2e26f283092eb96d6d9350ed89f49173791ae757ed7928a6146fbc63d9ad83cbb19e5d90
+  metadata.gz: 71fa902e08bafcc1f29719246626c0d646d24478b859592b41cd06b38fc3f56d1a72bfd37f5e95891da1397d08cba39d117160aa531763a400186870f0a022d7
+  data.tar.gz: 1bf5eaa93b4cf5899e9c82951abb58e80d104eb074869a9d63d532e623fe51d61e71137b5706ce0ee4eb9e5c5626f29feef250306ad9b364abbe81db17464d5c

data/README.md

@@ -70,6 +70,7 @@ See also [Poseidon::PartitionConsumer](http://www.rubydoc.info/github/bpot/posei
       @type   kafka_group
       brokers <list of broker-host:port, separate with comma, must set>
       zookeepers <list of broker-host:port, separate with comma, must set>
+      zookeeper_path <broker path in zookeeper> :default => /brokers/ids # Set path in zookeeper for brokers
       consumer_group <consumer group name, must set>
       topics <listening topics(separate with comma',')>
       format <input text type (text|json|ltsv|msgpack)>
@@ -99,7 +100,7 @@ See also [Poseidon::PartitionConsumer](http://www.rubydoc.info/github/bpot/posei
       # Brokers: you can choose either brokers or zookeeper.
       brokers             <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,.. # Set brokers directly
       zookeeper           <zookeeper_host>:<zookeeper_port> # Set brokers via Zookeeper
-
+      zookeeper_path      <broker path in zookeeper> :default => /brokers/ids # Set path in zookeeper for kafka
       default_topic       <output topic>
       default_partition_key (string)   :default => nil
       output_data_type    (json|ltsv|msgpack|attr:<record name>|<formatter name>)
@@ -131,7 +132,7 @@ Messages will be sent broker in a round-robin manner as default by Poseidon, but
 If key name `partition_key` exists in a message, this plugin set its value of partition_key as key.
 
 |default_partition_key|partition_key| behavior |
-|-|-|
+| --- | --- | --- |
 |Not set|Not exists| All messages are sent in round-robin |
 |Set| Not exists| All messages are sent to specific broker |
 |Not set| Exists | Messages which have partition_key record are sent to specific broker, others are sent in round-robin|
@@ -146,7 +147,7 @@ If key name `partition_key` exists in a message, this plugin set its value of pa
       # Brokers: you can choose either brokers or zookeeper.
       brokers             <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,.. # Set brokers directly
       zookeeper           <zookeeper_host>:<zookeeper_port> # Set brokers via Zookeeper
-
+      zookeeper_path      <broker path in zookeeper> :default => /brokers/ids # Set path in zookeeper for kafka
       default_topic       <output topic>
       default_partition_key (string)   :default => nil
       flush_interval      <flush interval (sec) :default => 60>
@@ -174,18 +175,6 @@ Install snappy module before you use snappy compression.
 
     $ gem install snappy
 
-#### Load balancing
-
-Messages will be sent broker in a round-robin manner as default by Poseidon, but you can set `default_partition_key` in config file to route messages to a specific broker.
-If key name `partition_key` exists in a message, this plugin set its value of partition_key as key.
-
-|default_partition_key|partition_key| behavior |
-|-|-|
-|Not set|Not exists| All messages are sent in round-robin |
-|Set| Not exists| All messages are sent to specific broker |
-|Not set| Exists | Messages which have partition_key record are sent to specific broker, others are sent in round-robin|
-|Set| Exists | Messages which have partition_key record are sent to specific broker with parition_key, others are sent to specific broker with default_parition_key|
-
 ## Contributing
 
 1. Fork it

data/fluent-plugin-kafka.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version = '0.1.3'
+  gem.version = '0.1.4'
   gem.add_dependency 'fluentd'
   gem.add_dependency 'poseidon_cluster'
   gem.add_dependency 'ltsv'

data/lib/fluent/plugin/out_kafka.rb

@@ -14,6 +14,8 @@ Note that you can choose to use either brokers or zookeeper.
 DESC
   config_param :zookeeper, :string, :default => nil,
                :desc => "Set brokers via Zookeeper: <zookeeper_host>:<zookeeper_port>"
+  config_param :zookeeper_path, :string, :default => '/brokers/ids',
+               :desc => "Path in path for Broker id. Default to /brokers/ids"
   config_param :default_topic, :string, :default => nil,
                :desc => "Output topic."
   config_param :default_partition_key, :string, :default => nil
@@ -33,6 +35,8 @@ DESC
   config_param :compression_codec, :string, :default => 'none',
                :desc => "The codec the producer uses to compress messages."
 
+  config_param :time_format, :string, :default => nil
+
   attr_accessor :output_data_type
   attr_accessor :field_separator
 
@@ -46,8 +50,8 @@ DESC
     if @zookeeper
       @seed_brokers = []
       z = Zookeeper.new(@zookeeper)
-      z.get_children(:path => '/brokers/ids')[:children].each do |id|
-        broker = Yajl.load(z.get(:path => "/brokers/ids/#{id}")[:data])
+      z.get_children(:path => @zookeeper_path)[:children].each do |id|
+        broker = Yajl.load(z.get(:path => @zookeeper_path + "/#{id}")[:data])
         @seed_brokers.push("#{broker['host']}:#{broker['port']}")
       end
       z.close
@@ -143,7 +147,13 @@ DESC
     begin
       chain.next
       es.each do |time,record|
-        record['time'] = time if @output_include_time
+        if @output_include_time
+          if @time_format
+            record['time'] = Time.at(time).strftime(@time_format)
+          else
+            record['time'] = time
+          end
+        end
         record['tag'] = tag if @output_include_tag
         topic = record['topic'] || self.default_topic || tag
         partition_key = record['partition_key'] || @default_partition_key

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -18,6 +18,8 @@ DESC
 Set brokers via Zookeeper:
 <zookeeper_host>:<zookeeper_port>
 DESC
+  config_param :zookeeper_path, :string, :default => '/brokers/ids',
+               :desc => "Path in path for Broker id. Default to /brokers/ids"
   config_param :default_topic, :string, :default => nil,
                :desc => "Output topic"
   config_param :default_partition_key, :string, :default => nil
@@ -43,6 +45,8 @@ The codec the producer uses to compress messages.
 Supported codecs: (none|gzip|snappy)
 DESC
 
+  config_param :time_format, :string, :default => nil
+
   attr_accessor :output_data_type
   attr_accessor :field_separator
 
@@ -56,8 +60,8 @@ DESC
     if @zookeeper
       @seed_brokers = []
       z = Zookeeper.new(@zookeeper)
-      z.get_children(:path => '/brokers/ids')[:children].each do |id|
-        broker = Yajl.load(z.get(:path => "/brokers/ids/#{id}")[:data])
+      z.get_children(:path => @zookeeper_path)[:children].each do |id|
+        broker = Yajl.load(z.get(:path => @zookeeper_path + "/#{id}")[:data])
         @seed_brokers.push("#{broker['host']}:#{broker['port']}")
       end
       z.close
@@ -146,7 +150,14 @@ DESC
     messages_bytes = 0
     begin
       chunk.msgpack_each { |tag, time, record|
-        record['time'] = time if @output_include_time
+        if @output_include_time
+          if @time_format
+            record['time'] = Time.at(time).strftime(@time_format)
+          else
+            record['time'] = time
+          end
+        end
+
         record['tag'] = tag if @output_include_tag
         topic = record['topic'] || @default_topic || tag
         partition_key = record['partition_key'] || @default_partition_key

metadata

@@ -1,97 +1,97 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Hidemasa Togashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-13 00:00:00.000000000 Z
+date: 2016-05-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: poseidon_cluster
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: ltsv
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: msgpack
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: zookeeper
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: Fluentd plugin for Apache Kafka > 0.8
@@ -121,17 +121,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14
+rubygems_version: 2.4.8
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin for Apache Kafka > 0.8