fluent-plugin-kafka 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f8b27ecbcb564d3fe1cb2961a02c0cf9cd63be9c
-  data.tar.gz: c4513c12341a79e66c739984d27ee9564fe4ff59
+  metadata.gz: 9f3bd136ea663d18398ab38653018906d9294e46
+  data.tar.gz: e927e2731e379645077d2d43006df2da17f7827a
 SHA512:
-  metadata.gz: fe6590aa85fef308e13e1d2f15cd92089c78b57d731b9a847f4c16c0454230ac681d4fb2bb6189691a4c07418474c2ffb0b24c17b2bb6a902615f264e24be4f6
-  data.tar.gz: b95598c0655590551dd815f0fdb8bb2f790ce69d47d4f2d928ea0f2a70cf895260e2965be61b5d38499937e09d59fbdf1a406691057c78ab41dc66606ef6c6fc
+  metadata.gz: 7835b7639afd419afc4a6647bde7d513f0576dbb91a46acee60825d41dc48875a1dbd9f8c8a875df124fc85c216dd3572e25c1e4350b031e2147045679bd45cf
+  data.tar.gz: 3e0b247d8efd4208200aa81398f1afae7d36b32415d9e62addd91e416d6baeb06c3eb930800d0e5f24afb4283357576cb37b3eee29ddd46087ff9dfdece0f2c6

data/ChangeLog

@@ -1,3 +1,11 @@
+Release 0.4.1 - 2016/12/01
+
+	* output: Support specifying partition
+
+Release 0.4.0 - 2016/11/08
+
+	* Remove zookeeper dependency
+
 Release 0.3.5 - 2016/10/21
 
 	* output: Support message key and related parameters. #91

data/README.md

@@ -20,6 +20,8 @@ Or install it yourself as:
 
     $ gem install fluent-plugin-kafka
 
+If you want to use zookeeper related parameters, you also need to install zookeeper gem. zookeeper gem includes native extension, so development tools are needed, e.g. gcc, make and etc.
+
 ## Requirements
 
 - Ruby 2.1 or later

data/fluent-plugin-kafka.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.4.0'
+  gem.version       = '0.4.1'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/out_kafka.rb

@@ -18,6 +18,7 @@ DESC
                :desc => "Output topic."
   config_param :default_message_key, :string, :default => nil
   config_param :default_partition_key, :string, :default => nil
+  config_param :default_partition, :integer, :default => nil
   config_param :client_id, :string, :default => 'kafka'
   config_param :output_data_type, :string, :default => 'json',
                :desc => "Supported format: (json|ltsv|msgpack|attr:<record name>|<formatter name>)"
@@ -27,6 +28,11 @@ DESC
                        :desc => <<-DESC
 Set true to remove partition key from data
 DESC
+  config_param :exclude_partition, :bool, :default => false,
+               :desc => <<-DESC
+Set true to remove partition from data
+DESC
+
   config_param :exclude_message_key, :bool, :default => false,
                          :desc => <<-DESC
 Set true to remove message key from data
@@ -172,12 +178,13 @@ DESC
         record['tag'] = tag if @output_include_tag
         topic = (@exclude_topic_key ? record.delete('topic') : record['topic']) || @default_topic || tag
         partition_key = (@exclude_partition_key ? record.delete('partition_key') : record['partition_key']) || @default_partition_key
+        partition = (@exclude_partition ? record.delete('partition'.freeze) : record['partition'.freeze]) || @default_partition
         message_key = (@exclude_message_key ? record.delete('message_key') : record['message_key']) || @default_message_key
 
         value = @formatter_proc.call(tag, time, record)
 
-        log.on_trace { log.trace("message send to #{topic} with partition_key: #{partition_key}, message_key: #{message_key} and value: #{value}.") }
-        producer.produce(value, topic: topic, key: message_key, partition_key: partition_key)
+        log.on_trace { log.trace("message will send to #{topic} with partition_key: #{partition_key}, partition: #{partition}, message_key: #{message_key} and value: #{record_buf}.") }
+        producer.produce(value, topic: topic, key: message_key, partition: partition, partition_key: partition_key)
       end
 
       producer.deliver_messages

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -22,6 +22,7 @@ DESC
                :desc => "Output topic"
   config_param :default_message_key, :string, :default => nil
   config_param :default_partition_key, :string, :default => nil
+  config_param :default_partition, :integer, :default => nil
   config_param :client_id, :string, :default => 'kafka'
   config_param :output_data_type, :string, :default => 'json',
                :desc => <<-DESC
@@ -32,6 +33,10 @@ DESC
   config_param :exclude_partition_key, :bool, :default => false,
                :desc => <<-DESC
 Set true to remove partition key from data
+DESC
+  config_param :exclude_partition, :bool, :default => false,
+               :desc => <<-DESC
+Set true to remove partition from data
 DESC
    config_param :exclude_message_key, :bool, :default => false,
                :desc => <<-DESC
@@ -229,6 +234,7 @@ DESC
           record['tag'] = tag if @output_include_tag
           topic = (@exclude_topic_key ? record.delete('topic'.freeze) : record['topic'.freeze]) || def_topic
           partition_key = (@exclude_partition_key ? record.delete('partition_key'.freeze) : record['partition_key'.freeze]) || @default_partition_key
+          partition = (@exclude_partition ? record.delete('partition'.freeze) : record['partition'.freeze]) || @default_partition
           message_key = (@exclude_message_key ? record.delete('message_key'.freeze) : record['message_key'.freeze]) || @default_message_key
 
           records_by_topic[topic] ||= 0
@@ -247,9 +253,9 @@ DESC
           messages = 0
           messages_bytes = 0
         end
-        log.on_trace { log.trace("message will send to #{topic} with partition_key: #{partition_key}, message_key: #{message_key} and value: #{record_buf}.") }
+        log.on_trace { log.trace("message will send to #{topic} with partition_key: #{partition_key}, partition: #{partition}, message_key: #{message_key} and value: #{record_buf}.") }
         messages += 1
-        producer.produce2(record_buf, topic: topic, key: message_key, partition_key: partition_key)
+        producer.produce2(record_buf, topic: topic, key: message_key, partition_key: partition_key, partition: partition)
         messages_bytes += record_buf_bytes
 
         records_by_topic[topic] += 1

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-07 00:00:00.000000000 Z
+date: 2016-12-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd