fluent-plugin-kafka 0.6.5 → 0.6.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1600a5a510be7ebf5665f59f6ed7cae14aabd855
-  data.tar.gz: a8bf174ca989d2f9355d48bd516d5e8fd0c85548
+  metadata.gz: f03db16ea9d9f3e937b8f2e5ee3840ac92929f98
+  data.tar.gz: 8f5db823ac54b6154c38571585e18915b38d0e71
 SHA512:
-  metadata.gz: e22f33f3eddc0fa784e5b16d39349ad75a59ddde246d34c66e66577b01ea865509caf3c31b1c5c4c4ca9e179ee52845c3f3250294726b158be63fd138b87f03d
-  data.tar.gz: 9fe0745ec3f9197259c7222f00afdd650bab4b3a707b1428bb5649759fc4e97669ca7c2ee5df332a0f3fc938f3d68a133f27b635b4665d1706073da2058d3d1a
+  metadata.gz: 5a032b3b1cbabc0ec9678257151cba70d98338999bd028c865419790817259e63eb7172c24a9956db3bfe0ee2b35a6c27615bbb1e6c39e466dfac39cf9282bb0
+  data.tar.gz: fa7da134390f0532f399b5e0d802c377e21d917d4ed606e0ec6d1df1b0a1e81c54e37b7897643fc191cf3d41ba420e1859424170b3fbe07fd7039413300c61c5

data/ChangeLog

@@ -1,3 +1,7 @@
+Release 0.6.6 - 2017/12/25
+
+	* output: Make topic/partition/partition_key/message_key configurable
+
 Release 0.6.5 - 2017/12/14
 
 	* in_kafka_group: Add client_id parameter

data/README.md

@@ -129,7 +129,7 @@ Consuming topic name is used for event tag. So when the target topic name is `ap
 
 This plugin uses ruby-kafka producer for writing data. This plugin works with recent kafka versions.
 
-    <match *.**>
+    <match app.**>
       @type kafka_buffered
 
       # Brokers: you can choose either brokers or zookeeper. If you are not familiar with zookeeper, use brokers parameters.
@@ -137,6 +137,10 @@ This plugin uses ruby-kafka producer for writing data. This plugin works with re
       zookeeper           <zookeeper_host>:<zookeeper_port> # Set brokers via Zookeeper
       zookeeper_path      <broker path in zookeeper> :default => /brokers/ids # Set path in zookeeper for kafka
 
+      topic_key             (string) :default => 'topic'
+      partition_key         (string) :default => 'partition'
+      partition_key_key     (string) :default => 'partition_key'
+      message_key_key       (string) :default => 'message_key'
       default_topic         (string) :default => nil
       default_partition_key (string) :default => nil
       default_message_key   (string) :default => nil
@@ -207,11 +211,52 @@ If key name `partition_key` exists in a message, this plugin set its value of pa
 
 If key name `message_key` exists in a message, this plugin publishes the value of message_key to kafka and can be read by consumers. Same message key will be assigned to all messages by setting `default_message_key` in config file. If message_key exists and if partition_key is not set explicitly, messsage_key will be used for partitioning.
 
+### Output plugin
+
+This plugin is for v1.0. This will be `out_kafka` plugin in the feature.
+
+    <match app.**>
+      @type kafka2
+
+      brokers             <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,.. # Set brokers directly
+
+      topic_key             (string) :default => 'topic'
+      partition_key         (string) :default => 'partition'
+      partition_key_key     (string) :default => 'partition_key'
+      message_key_key       (string) :default => 'message_key'
+      default_topic         (string) :default => nil
+      default_partition_key (string) :default => nil
+      default_message_key   (string) :default => nil
+      exclude_topic_key     (bool) :default => false
+      exclude_partition_key (bool) :default => false
+      get_kafka_client_log  (bool) :default => false
+
+      <format>
+        @type (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
+      </format>
+      <inject>
+        tag_key tag
+        time_key time
+      </inject>
+
+      # See fluentd document for buffer related parameters: http://docs.fluentd.org/articles/buffer-plugin-overview
+      # Buffer chunk key should be same with topic_key. If value is not found in the record, default_topic is used.
+      <buffer topic>
+        flush_interavl 10s
+      </buffer>
+
+      # ruby-kafka producer options
+      max_send_retries             (integer)     :default => 1
+      required_acks                (integer)     :default => -1
+      ack_timeout                  (integer)     :default => nil (Use default of ruby-kafka)
+      compression_codec            (gzip|snappy) :default => nil (No compression)
+    </match>
+
 ### Non-buffered output plugin
 
 This plugin uses ruby-kafka producer for writing data. For performance and reliability concerns, use `kafka_bufferd` output instead. This is mainly for testing.
 
-    <match *.**>
+    <match app.**>
       @type kafka
 
       # Brokers: you can choose either brokers or zookeeper.

data/fluent-plugin-kafka.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.6.5'
+  gem.version       = '0.6.6'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/out_kafka2.rb

@@ -15,10 +15,14 @@ module Fluent::Plugin
 Set brokers directly:
 <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,..
 DESC
+    config_param :topic_key, :string, :default => 'topic', :desc => "Field for kafka topic"
     config_param :default_topic, :string, :default => nil,
                  :desc => "Default output topic when record doesn't have topic field"
+    config_param :message_key_key, :string, :default => 'message_key', :desc => "Field for kafka message key"
     config_param :default_message_key, :string, :default => nil
+    config_param :partition_key_key, :string, :default => 'partition_key', :desc => "Field for kafka partition key"
     config_param :default_partition_key, :string, :default => nil
+    config_param :partition_key, :string, :default => 'partition', :desc => "Field for kafka partition"
     config_param :default_partition, :integer, :default => nil
     config_param :client_id, :string, :default => 'fluentd'
     config_param :exclude_partition_key, :bool, :default => false,
@@ -123,6 +127,8 @@ DESC
           @router.emit("fluent_kafka_stats.#{event.name}", Time.now.to_i, message)
         end
       end
+
+      @topic_key_sym = @topic_key.to_sym
     end
 
     def multi_workers_ready?
@@ -168,7 +174,7 @@ DESC
     # TODO: optimize write performance
     def write(chunk)
       tag = chunk.metadata.tag
-      topic = chunk.metadata.variables[:topic] || @default_topic || tag
+      topic = chunk.metadata.variables[@topic_key_sym] || @default_topic || tag
       producer = @kafka.topic_producer(topic, @producer_opts)
 
       messages = 0
@@ -178,10 +184,10 @@ DESC
         chunk.msgpack_each { |time, record|
           begin
             record = inject_values_to_record(tag, time, record)
-            record.delete('topic'.freeze) if @exclude_topic_key
-            partition_key = (@exclude_partition_key ? record.delete('partition_key'.freeze) : record['partition_key'.freeze]) || @default_partition_key
-            partition = (@exclude_partition ? record.delete('partition'.freeze) : record['partition'.freeze]) || @default_partition
-            message_key = (@exclude_message_key ? record.delete('message_key'.freeze) : record['message_key'.freeze]) || @default_message_key
+            record.delete(@topic_key) if @exclude_topic_key
+            partition_key = (@exclude_partition_key ? record.delete(@partition_key_key) : record[@partition_key_key]) || @default_partition_key
+            partition = (@exclude_partition ? record.delete(@partition_key) : record[@partition_key]) || @default_partition
+            message_key = (@exclude_message_key ? record.delete(@message_key) : record[@message_key]) || @default_message_key
 
             record_buf = @formatter_proc.call(tag, time, record)
           rescue StandardError => e

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -16,12 +16,15 @@ DESC
 Set brokers via Zookeeper:
 <zookeeper_host>:<zookeeper_port>
 DESC
-  config_param :zookeeper_path, :string, :default => '/brokers/ids',
-               :desc => "Path in path for Broker id. Default to /brokers/ids"
-  config_param :default_topic, :string, :default => nil,
-               :desc => "Output topic"
+  config_param :zookeeper_path, :string, :default => '/brokers/ids', :desc => "Path in path for Broker id. Default to /brokers/ids"
+
+  config_param :topic_key, :string, :default => 'topic', :desc => "Field for kafka topic"
+  config_param :default_topic, :string, :default => nil, :desc => "Default output topic when record doesn't have topic field"
+  config_param :message_key_key, :string, :default => 'message_key', :desc => "Field for kafka message key"
   config_param :default_message_key, :string, :default => nil
+  config_param :partition_key_key, :string, :default => 'partition_key', :desc => "Field for kafka partition key"
   config_param :default_partition_key, :string, :default => nil
+  config_param :partition_key, :string, :default => 'partition', :desc => "Field for kafka partition"
   config_param :default_partition, :integer, :default => nil
   config_param :client_id, :string, :default => 'kafka'
   config_param :output_data_type, :string, :default => 'json',
@@ -40,7 +43,7 @@ Set true to remove partition from data
 DESC
    config_param :exclude_message_key, :bool, :default => false,
                :desc => <<-DESC
-Set true to remove partition key from data
+Set true to remove message key from data
 DESC
    config_param :exclude_topic_key, :bool, :default => false,
                 :desc => <<-DESC
@@ -289,10 +292,10 @@ DESC
           end
 
           record['tag'] = tag if @output_include_tag
-          topic = (@exclude_topic_key ? record.delete('topic'.freeze) : record['topic'.freeze]) || def_topic
-          partition_key = (@exclude_partition_key ? record.delete('partition_key'.freeze) : record['partition_key'.freeze]) || @default_partition_key
-          partition = (@exclude_partition ? record.delete('partition'.freeze) : record['partition'.freeze]) || @default_partition
-          message_key = (@exclude_message_key ? record.delete('message_key'.freeze) : record['message_key'.freeze]) || @default_message_key
+          topic = (@exclude_topic_key ? record.delete(@topic_key) : record[@topic_key]) || def_topic
+          partition_key = (@exclude_partition_key ? record.delete(@partition_key_key) : record[@partition_key_key]) || @default_partition_key
+          partition = (@exclude_partition ? record.delete(@partition) : record[@partition]) || @default_partition
+          message_key = (@exclude_message_key ? record.delete(@message_key_key) : record[@message_key_key]) || @default_message_key
 
           records_by_topic[topic] ||= 0
           bytes_by_topic[topic] ||= 0

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.6.5
+  version: 0.6.6
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-12-15 00:00:00.000000000 Z
+date: 2017-12-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd