fluent-plugin-kafka 0.0.12 → 0.0.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 69682c218b44b78ebe360adaa07d2defc25c8eb0
-  data.tar.gz: b0e9f47b62c7753988933ac6ab1b4221ab38ef92
+  metadata.gz: 18708524a30b618692c9df1cf9b303a5f40a7625
+  data.tar.gz: cf0f5cb7e2d1deee2736ea6f25391c33766adbfb
 SHA512:
-  metadata.gz: 1187e4ed38b2d93636de7699ba75a1d1759f1dafa0f3aab8b6fa93ff1738230bf7850405809764ec8d1ccabe3d6cf7b6a63d4c730867db442e04facd6081e4f4
-  data.tar.gz: 2fd63a22abb00ded8024596c710393cf797965beb94e22a742800b8fd5329483430c0d3fc16458e993e4bd0851456d3956c5a90593915af9a9cce534e5e42789
+  metadata.gz: 73973698fb730b5f2eb18161c557252cc990cefa4d02ac03dc49ff7523242bb9f78052b54b79f8eb514fd39858a4038547aa318a131825e78c1e81b0e834a993
+  data.tar.gz: 86b72a4927cd364f3b66d58a1fb22abdc78c36bf6ab4560ec5474d6781e2b210d12926f81829a61f8e44448970cf8173a27d49dcbe05c7be719b2205da67efa4

data/README.md

@@ -41,6 +41,7 @@ Or install it yourself as:
       zookeeper           <zookeeper_host>:<zookeeper_port> # Set brokers via Zookeeper
 
       default_topic       <output topic>
+      default_partition_key (string)   :default => nil
       output_data_type    (json|ltsv|msgpack|attr:<record name>|<formatter name>)
       output_include_tag  (true|false) :default => false
       output_include_time (true|false) :default => false
@@ -64,6 +65,18 @@ Install snappy module before you use snappy compression.
 
     $ gem install snappy
 
+#### Load balancing
+
+Messages will be sent broker in a round-robin manner as default by Poseidon, but you can set `default_partition_key` in config file to route messages to a specific broker.
+If key name `partition_key` exists in a message, this plugin set its value of partition_key as key.
+
+|default_partition_key|partition_key| behavior |
+|-|-|
+|Not set|Not exists| All messages are sent in round-robin |
+|Set| Not exists| All messages are sent to specific broker |
+|Not set| Exists | Messages which have partition_key record are sent to specific broker, others are sent in round-robin|
+|Set| Exists | Messages which have partition_key record are sent to specific broker with parition_key, others are sent to specific broker with default_parition_key|
+
 
 ### Buffered output plugin
 
@@ -75,6 +88,7 @@ Install snappy module before you use snappy compression.
       zookeeper           <zookeeper_host>:<zookeeper_port> # Set brokers via Zookeeper
 
       default_topic       <output topic>
+      default_partition_key (string)   :default => nil
       flush_interval      <flush interval (sec) :default => 60>
       buffer_type         (file|memory)
       output_data_type    (json|ltsv|msgpack|attr:<record name>|<formatter name>)
@@ -100,6 +114,17 @@ Install snappy module before you use snappy compression.
 
     $ gem install snappy
 
+    #### Load balancing
+
+    Messages will be sent broker in a round-robin manner as default by Poseidon, but you can set `default_partition_key` in config file to route messages to a specific broker.
+    If key name `partition_key` exists in a message, this plugin set its value of partition_key as key.
+
+    |default_partition_key|partition_key| behavior |
+    |-|-|
+    |Not set|Not exists| All messages are sent in round-robin |
+    |Set| Not exists| All messages are sent to specific broker |
+    |Not set| Exists | Messages which have partition_key record are sent to specific broker, others are sent in round-robin|
+    |Set| Exists | Messages which have partition_key record are sent to specific broker with parition_key, others are sent to specific broker with default_parition_key|
 
 ## Contributing
 

data/fluent-plugin-kafka.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version = '0.0.12'
+  gem.version = '0.0.13'
   gem.add_dependency 'fluentd'
   gem.add_dependency 'poseidon'
   gem.add_dependency 'ltsv'

data/lib/fluent/plugin/out_kafka.rb

@@ -9,7 +9,7 @@ class Fluent::KafkaOutput < Fluent::Output
   config_param :brokers, :string, :default => 'localhost:9092'
   config_param :zookeeper, :string, :default => nil
   config_param :default_topic, :string, :default => nil
-  config_param :default_partition, :integer, :default => 0
+  config_param :default_partition_key, :string, :default => nil
   config_param :client_id, :string, :default => 'kafka'
   config_param :output_data_type, :string, :default => 'json'
   config_param :output_include_tag, :bool, :default => false
@@ -26,6 +26,10 @@ class Fluent::KafkaOutput < Fluent::Output
 
   @seed_brokers = []
 
+  unless method_defined?(:log)
+    define_method("log") { $log }
+  end
+
   def refresh_producer()
     if @zookeeper
       @seed_brokers = []
@@ -129,9 +133,10 @@ class Fluent::KafkaOutput < Fluent::Output
         record['time'] = time if @output_include_time
         record['tag'] = tag if @output_include_tag
         topic = record['topic'] || self.default_topic || tag
-        partition = record['partition'] || self.default_partition
+        partition_key = record['partition_key'] || @default_partition_key
         value = @formatter.nil? ? parse_record(record) : @formatter.format(tag, time, record)
-        message = Poseidon::MessageToSend.new(topic, value)
+        log.trace("message send to #{topic} with key: #{partition_key} and value: #{value}.")
+        message = Poseidon::MessageToSend.new(topic, value, partition_key)
         @producer.send_messages([message])
       end
     rescue Exception => e

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -10,7 +10,7 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
   config_param :brokers, :string, :default => 'localhost:9092'
   config_param :zookeeper, :string, :default => nil
   config_param :default_topic, :string, :default => nil
-  config_param :default_partition, :integer, :default => 0
+  config_param :default_partition_key, :string, :default => nil
   config_param :client_id, :string, :default => 'kafka'
   config_param :output_data_type, :string, :default => 'json'
   config_param :output_include_tag, :bool, :default => false
@@ -141,6 +141,7 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
         record['time'] = time if @output_include_time
         record['tag'] = tag if @output_include_tag
         topic = record['topic'] || @default_topic || tag
+        partition_key = record['partition_key'] || @default_partition_key
 
         records_by_topic[topic] ||= 0
         bytes_by_topic[topic] ||= 0
@@ -148,17 +149,20 @@ class Fluent::KafkaOutputBuffered < Fluent::BufferedOutput
         record_buf = @formatter.nil? ? parse_record(record) : @formatter.format(tag, time, record)
         record_buf_bytes = record_buf.bytesize
         if messages.length > 0 and messages_bytes + record_buf_bytes > @kafka_agg_max_bytes
+          log.trace("#{messages.length} messages send.")
           @producer.send_messages(messages)
           messages = []
           messages_bytes = 0
         end
-        messages << Poseidon::MessageToSend.new(topic, record_buf)
+        log.trace("message will send to #{topic} with key: #{partition_key} and value: #{record_buf}.")
+        messages << Poseidon::MessageToSend.new(topic, record_buf, partition_key)
         messages_bytes += record_buf_bytes
 
         records_by_topic[topic] += 1
         bytes_by_topic[topic] += record_buf_bytes
       }
       if messages.length > 0
+        log.trace("#{messages.length} messages send.")
         @producer.send_messages(messages)
       end
       log.debug "(records|bytes) (#{records_by_topic}|#{bytes_by_topic})"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.0.12
+  version: 0.0.13
 platform: ruby
 authors:
 - Hidemasa Togashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-15 00:00:00.000000000 Z
+date: 2015-06-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd