fluent-plugin-kafka 0.3.4 → 0.3.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e2d81baf9816d05b5cc5f5c982aafe722781d978
-  data.tar.gz: cfb1477ba8cd4961424881c173cf0d4dffab999d
+  metadata.gz: c58f69838db9786bbbe80d9fdf6a0c055f11bddd
+  data.tar.gz: 7cafa5f21b85071af41c44570c27fe11518c3f09
 SHA512:
-  metadata.gz: 15b62cb6289281bebf2938172b41fe0a8f7080d6afebfebf672e45a10a1c31779db3ed9b739a28db57300ed5323abd814ae69720e0afc0d3a301ffebfce66e8b
-  data.tar.gz: 70deeef4310ef4ad2674ba21fb2f981147bc0784810d88c14ab32efe837f7b1f7fc41f6bb540223d65bc1df0cae599440ae05eaaae7ded3185c6865a727fedbb
+  metadata.gz: 8b2ec6b3299fcf850de52ad759e88528a493428e4a16d1404c9618bd59ef15aec53e5bd58ff4205c085f16a5ee6c3d6408caa74d5a9dee6bb0979e3129113985
+  data.tar.gz: 0684e0e9659879d7e84d2342637e7d0b0e079783351de7ad7188e0783e63ee439fac20a6d1731903e4fab58c864843379c930d3c9ccc9472a72f53df6a481c52

data/README.md

@@ -113,6 +113,7 @@ This plugin uses ruby-kafka producer for writing data. For performance and relia
 
       default_topic         (string) :default => nil
       default_partition_key (string) :default => nil
+      default_message_key   (string) :default => nil
       output_data_type      (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
       output_include_tag    (bool) :default => false
       output_include_time   (bool) :default => false
@@ -158,6 +159,9 @@ If key name `partition_key` exists in a message, this plugin set its value of pa
 |Set| Exists | Messages which have partition_key record are assigned to the specific partition with parition_key, others are assigned to the specific partition with default_parition_key |
 
 
+If key name `message_key` exists in a message, this plugin publishes the value of message_key to kafka and can be read by consumers. Same message key will be assigned to all messages by setting `default_message_key` in config file. If message_key exists and if partition_key is not set explicitly, messsage_key will be used for partitioning.
+
+
 ### Buffered output plugin
 
 This plugin uses ruby-kafka producer for writing data. This plugin works with recent kafka versions.
@@ -172,6 +176,7 @@ This plugin uses ruby-kafka producer for writing data. This plugin works with re
 
       default_topic         (string) :default => nil
       default_partition_key (string) :default => nil
+      default_message_key   (string) :default => nil
       output_data_type      (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
       output_include_tag    (bool) :default => false
       output_include_time   (bool) :default => false

data/fluent-plugin-kafka.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.3.4'
+  gem.version       = '0.3.5'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/out_kafka.rb

@@ -16,6 +16,7 @@ DESC
                :desc => "Path in path for Broker id. Default to /brokers/ids"
   config_param :default_topic, :string, :default => nil,
                :desc => "Output topic."
+  config_param :default_message_key, :string, :default => nil
   config_param :default_partition_key, :string, :default => nil
   config_param :client_id, :string, :default => 'kafka'
   config_param :output_data_type, :string, :default => 'json',
@@ -25,6 +26,10 @@ DESC
   config_param :exclude_partition_key, :bool, :default => false,
                        :desc => <<-DESC
 Set true to remove partition key from data
+DESC
+  config_param :exclude_message_key, :bool, :default => false,
+                         :desc => <<-DESC
+Set true to remove message key from data
 DESC
   config_param :exclude_topic_key, :bool, :default => false,
                          :desc => <<-DESC
@@ -167,11 +172,12 @@ DESC
         record['tag'] = tag if @output_include_tag
         topic = (@exclude_topic_key ? record.delete('topic') : record['topic']) || @default_topic || tag
         partition_key = (@exclude_partition_key ? record.delete('partition_key') : record['partition_key']) || @default_partition_key
+        message_key = (@exclude_message_key ? record.delete('message_key') : record['message_key']) || @default_message_key
 
         value = @formatter_proc.call(tag, time, record)
 
-        log.on_trace { log.trace("message send to #{topic} with key: #{partition_key} and value: #{value}.") }
-        producer.produce(value, topic: topic, partition_key: partition_key)
+        log.on_trace { log.trace("message send to #{topic} with partition_key: #{partition_key}, message_key: #{message_key} and value: #{value}.") }
+        producer.produce(value, topic: topic, key: message_key, partition_key: partition_key)
       end
 
       producer.deliver_messages

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -20,6 +20,7 @@ DESC
                :desc => "Path in path for Broker id. Default to /brokers/ids"
   config_param :default_topic, :string, :default => nil,
                :desc => "Output topic"
+  config_param :default_message_key, :string, :default => nil
   config_param :default_partition_key, :string, :default => nil
   config_param :client_id, :string, :default => 'kafka'
   config_param :output_data_type, :string, :default => 'json',
@@ -31,6 +32,10 @@ DESC
   config_param :exclude_partition_key, :bool, :default => false,
                :desc => <<-DESC
 Set true to remove partition key from data
+DESC
+   config_param :exclude_message_key, :bool, :default => false,
+               :desc => <<-DESC
+Set true to remove partition key from data
 DESC
    config_param :exclude_topic_key, :bool, :default => false,
                 :desc => <<-DESC
@@ -224,6 +229,7 @@ DESC
           record['tag'] = tag if @output_include_tag
           topic = (@exclude_topic_key ? record.delete('topic'.freeze) : record['topic'.freeze]) || def_topic
           partition_key = (@exclude_partition_key ? record.delete('partition_key'.freeze) : record['partition_key'.freeze]) || @default_partition_key
+          message_key = (@exclude_message_key ? record.delete('message_key'.freeze) : record['message_key'.freeze]) || @default_message_key
 
           records_by_topic[topic] ||= 0
           bytes_by_topic[topic] ||= 0
@@ -241,9 +247,9 @@ DESC
           messages = 0
           messages_bytes = 0
         end
-        log.on_trace { log.trace("message will send to #{topic} with key: #{partition_key} and value: #{record_buf}.") }
+        log.on_trace { log.trace("message will send to #{topic} with partition_key: #{partition_key}, message_key: #{message_key} and value: #{record_buf}.") }
         messages += 1
-        producer.produce2(record_buf, topic: topic, partition_key: partition_key)
+        producer.produce2(record_buf, topic: topic, key: message_key, partition_key: partition_key)
         messages_bytes += record_buf_bytes
 
         records_by_topic[topic] += 1

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.3.4
+  version: 0.3.5
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-10-20 00:00:00.000000000 Z
+date: 2016-10-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd