fluent-plugin-kafka-custom-ruby-version 0.9.3 → 0.9.4.32

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b9e142700a225fe4ab16e9c2084cff87cff2b488
-  data.tar.gz: 93f617d0c3f68eb826e28a57365791e37e0d3251
+  metadata.gz: 75ad0a6363c5f682fc8d5298acbd6e0ac1fe8e36
+  data.tar.gz: 3f296115f6a57bbfebf554ee333ff31b3a46ba18
 SHA512:
-  metadata.gz: ff5e26dc07c48c705e0d63ec13eb5fc30af29d132ae1e0e285a061680c454b0bcb069228b9571124f3c9cb810364b891430c0fbcf629b4a8bff84ac9b1930254
-  data.tar.gz: f7ea83c541088d50872f27f181cca77a49018b6ff2ac7cbed0a55c552bb7298d0919d7190b39e3502fcbefb66cc05383c7a46bbcccb0d8ef024ace61a7cc8c10
+  metadata.gz: 1784282891033222f44c903ac6f18bbfdfb0750fa2605dea54f332afb41cfe18976356fb636f47324ba185120bf899ee8f5edd80747f5185936465bfbc30d515
+  data.tar.gz: 5a5abe1bd6281f1911796e703a46ef646d471ef20e40515b28045cd9ac9e8c7d02370a46091b07358b1e02ed9286587774d85bb256490507f5e22790155be8c0

data/.project

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <projectDescription>
-	<name>fluent-plugin-kafka-master-custom-ruby-version</name>
+	<name>fluent-plugin-kafka-custom</name>
 	<comment></comment>
 	<projects>
 	</projects>

data/ChangeLog

@@ -1,3 +1,12 @@
+Release 0.9.2 - 2019/03/26
+
+	* out_kafka_buffered: Fix typo of partition_key usage
+
+Release 0.9.1 - 2019/03/25
+
+	* output: Support sasl_over_ssl parameter
+	* Support ruby-kafka 0.7.6
+
 Release 0.9.0 - 2019/02/22
 
 	* Add v1 API based rdkafka2 output plugin

data/Gemfile

@@ -1,4 +1,4 @@
-source 'https://rubygems.org'
-
-# Specify your gem's dependencies in fluent-plugin-kafka-custom-ruby-version.gemspec
-gemspec
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kafka-custom-ruby-version.gemspec
+gemspec

data/README.md

@@ -1,333 +1,334 @@
-# fluent-plugin-kafka, a plugin for [Fluentd](http://fluentd.org)
-
-[![Build Status](https://travis-ci.org/fluent/fluent-plugin-kafka.svg?branch=master)](https://travis-ci.org/fluent/fluent-plugin-kafka)
-
-A fluentd plugin to both consume and produce data for Apache Kafka.
-
-TODO: Also, I need to write tests
-
-## Installation
-
-Add this line to your application's Gemfile:
-
-    gem 'fluent-plugin-kafka'
-
-And then execute:
-
-    $ bundle
-
-Or install it yourself as:
-
-    $ gem install fluent-plugin-kafka --no-document
-
-If you want to use zookeeper related parameters, you also need to install zookeeper gem. zookeeper gem includes native extension, so development tools are needed, e.g. gcc, make and etc.
-
-## Requirements
-
-- Ruby 2.1 or later
-- Input plugins work with kafka v0.9 or later
-- Output plugins work with kafka v0.8 or later
-
-## Usage
-
-### Common parameters
-
-#### SSL authentication
-
-- ssl_ca_cert
-- ssl_client_cert
-- ssl_client_cert_key
-- ssl_ca_certs_from_system
-
-Set path to SSL related files. See [Encryption and Authentication using SSL](https://github.com/zendesk/ruby-kafka#encryption-and-authentication-using-ssl) for more detail.
-
-#### SASL authentication 
-
-##### with GSSAPI
-
-- principal
-- keytab
-
-Set principal and path to keytab for SASL/GSSAPI authentication. 
-See [Authentication using SASL](https://github.com/zendesk/ruby-kafka#authentication-using-sasl) for more details.
-
-##### with Plain/SCRAM
-
-- username
-- password
-- scram_mechanism
-- sasl_over_ssl
-
-Set username, password, scram_mechanism and sasl_over_ssl for SASL/Plain or Scram authentication. 
-See [Authentication using SASL](https://github.com/zendesk/ruby-kafka#authentication-using-sasl) for more details.
-
-### Input plugin (@type 'kafka')
-
-Consume events by single consumer.
-
-    <source>
-      @type kafka
-
-      brokers <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,..
-      topics <listening topics(separate with comma',')>
-      format <input text type (text|json|ltsv|msgpack)> :default => json
-      message_key <key (Optional, for text format only, default is message)>
-      add_prefix <tag prefix (Optional)>
-      add_suffix <tag suffix (Optional)>
-
-      # Optionally, you can manage topic offset by using zookeeper
-      offset_zookeeper    <zookeer node list (<zookeeper1_host>:<zookeeper1_port>,<zookeeper2_host>:<zookeeper2_port>,..)>
-      offset_zk_root_node <offset path in zookeeper> default => '/fluent-plugin-kafka'
-
-      # ruby-kafka consumer options
-      max_bytes     (integer) :default => nil (Use default of ruby-kafka)
-      max_wait_time (integer) :default => nil (Use default of ruby-kafka)
-      min_bytes     (integer) :default => nil (Use default of ruby-kafka)
-    </source>
-
-Supports a start of processing from the assigned offset for specific topics.
-
-    <source>
-      @type kafka
-
-      brokers <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,..
-      format <input text type (text|json|ltsv|msgpack)>
-      <topic>
-        topic     <listening topic>
-        partition <listening partition: default=0>
-        offset    <listening start offset: default=-1>
-      </topic>
-      <topic>
-        topic     <listening topic>
-        partition <listening partition: default=0>
-        offset    <listening start offset: default=-1>
-      </topic>
-    </source>
-
-See also [ruby-kafka README](https://github.com/zendesk/ruby-kafka#consuming-messages-from-kafka) for more detailed documentation about ruby-kafka.
-
-Consuming topic name is used for event tag. So when the target topic name is `app_event`, the tag is `app_event`. If you want to modify tag, use `add_prefix` or `add_suffix` parameters. With `add_prefix kafka`, the tag is `kafka.app_event`.
-
-### Input plugin (@type 'kafka_group', supports kafka group)
-
-Consume events by kafka consumer group features..
-
-    <source>
-      @type kafka_group
-
-      brokers <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,..
-      consumer_group <consumer group name, must set>
-      topics <listening topics(separate with comma',')>
-      format <input text type (text|json|ltsv|msgpack)> :default => json
-      message_key <key (Optional, for text format only, default is message)>
-      add_prefix <tag prefix (Optional)>
-      add_suffix <tag suffix (Optional)>
-      retry_emit_limit <Wait retry_emit_limit x 1s when BuffereQueueLimitError happens. The default is nil and it means waiting until BufferQueueLimitError is resolved>
-      use_record_time <If true, replace event time with contents of 'time' field of fetched record>
-      time_format <string (Optional when use_record_time is used)>
-
-      # ruby-kafka consumer options
-      max_bytes               (integer) :default => 1048576
-      max_wait_time           (integer) :default => nil (Use default of ruby-kafka)
-      min_bytes               (integer) :default => nil (Use default of ruby-kafka)
-      offset_commit_interval  (integer) :default => nil (Use default of ruby-kafka)
-      offset_commit_threshold (integer) :default => nil (Use default of ruby-kafka)
-      fetcher_max_queue_size  (integer) :default => nil (Use default of ruby-kafka)
-      start_from_beginning    (bool)    :default => true
-    </source>
-
-See also [ruby-kafka README](https://github.com/zendesk/ruby-kafka#consuming-messages-from-kafka) for more detailed documentation about ruby-kafka options.
-
-Consuming topic name is used for event tag. So when the target topic name is `app_event`, the tag is `app_event`. If you want to modify tag, use `add_prefix` or `add_suffix` parameter. With `add_prefix kafka`, the tag is `kafka.app_event`.
-
-### Buffered output plugin
-
-This plugin uses ruby-kafka producer for writing data. This plugin works with recent kafka versions.
-
-    <match app.**>
-      @type kafka_buffered
-
-      # Brokers: you can choose either brokers or zookeeper. If you are not familiar with zookeeper, use brokers parameters.
-      brokers             <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,.. # Set brokers directly
-      zookeeper           <zookeeper_host>:<zookeeper_port> # Set brokers via Zookeeper
-      zookeeper_path      <broker path in zookeeper> :default => /brokers/ids # Set path in zookeeper for kafka
-
-      topic_key             (string) :default => 'topic'
-      partition_key         (string) :default => 'partition'
-      partition_key_key     (string) :default => 'partition_key'
-      message_key_key       (string) :default => 'message_key'
-      default_topic         (string) :default => nil
-      default_partition_key (string) :default => nil
-      default_message_key   (string) :default => nil
-      output_data_type      (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
-      output_include_tag    (bool) :default => false
-      output_include_time   (bool) :default => false
-      exclude_topic_key     (bool) :default => false
-      exclude_partition_key (bool) :default => false
-      get_kafka_client_log  (bool) :default => false
-
-      # See fluentd document for buffer related parameters: http://docs.fluentd.org/articles/buffer-plugin-overview
-
-      # ruby-kafka producer options
-      max_send_retries             (integer)     :default => 1
-      required_acks                (integer)     :default => -1
-      ack_timeout                  (integer)     :default => nil (Use default of ruby-kafka)
-      compression_codec            (gzip|snappy) :default => nil (No compression)
-      kafka_agg_max_bytes          (integer)     :default => 4096
-      kafka_agg_max_messages       (integer)     :default => nil (No limit)
-      max_send_limit_bytes         (integer)     :default => nil (No drop)
-      discard_kafka_delivery_failed   (bool)     :default => false (No discard)
-      monitoring_list              (array)       :default => []
-    </match>
-
-`<formatter name>` of `output_data_type` uses fluentd's formatter plugins. See [formatter article](http://docs.fluentd.org/articles/formatter-plugin-overview).
-
-ruby-kafka sometimes returns `Kafka::DeliveryFailed` error without good information.
-In this case, `get_kafka_client_log` is useful for identifying the error cause.
-ruby-kafka's log is routed to fluentd log so you can see ruby-kafka's log in fluentd logs.
-
-Supports following ruby-kafka's producer options.
-
-- max_send_retries - default: 1 - Number of times to retry sending of messages to a leader.
-- required_acks - default: -1 - The number of acks required per request. If you need flush performance, set lower value, e.g. 1, 2.
-- ack_timeout - default: nil - How long the producer waits for acks. The unit is seconds.
-- compression_codec - default: nil - The codec the producer uses to compress messages.
-- kafka_agg_max_bytes - default: 4096 - Maximum value of total message size to be included in one batch transmission.
-- kafka_agg_max_messages - default: nil - Maximum number of messages to include in one batch transmission.
-- max_send_limit_bytes - default: nil - Max byte size to send message to avoid MessageSizeTooLarge. For example, if you set 1000000(message.max.bytes in kafka), Message more than 1000000 byes will be dropped.
-- discard_kafka_delivery_failed - default: false - discard the record where [Kafka::DeliveryFailed](http://www.rubydoc.info/gems/ruby-kafka/Kafka/DeliveryFailed) occurred
-- monitoring_list - default: [] - library to be used to monitor. statsd and datadog are supported
-
-If you want to know about detail of monitoring, see also https://github.com/zendesk/ruby-kafka#monitoring
-
-See also [Kafka::Client](http://www.rubydoc.info/gems/ruby-kafka/Kafka/Client) for more detailed documentation about ruby-kafka.
-
-This plugin supports compression codec "snappy" also.
-Install snappy module before you use snappy compression.
-
-    $ gem install snappy --no-document
-
-snappy gem uses native extension, so you need to install several packages before.
-On Ubuntu, need development packages and snappy library.
-
-    $ sudo apt-get install build-essential autoconf automake libtool libsnappy-dev
-
-On CentOS 7 installation is also necessary.
-
-    $ sudo yum install gcc autoconf automake libtool snappy-devel
-
-#### Load balancing
-
-Messages will be assigned a partition at random as default by ruby-kafka, but messages with the same partition key will always be assigned to the same partition by setting `default_partition_key` in config file.
-If key name `partition_key` exists in a message, this plugin set its value of partition_key as key.
-
-|default_partition_key|partition_key| behavior |
-| --- | --- | --- |
-|Not set|Not exists| All messages are assigned a partition at random |
-|Set| Not exists| All messages are assigned to the specific partition |
-|Not set| Exists | Messages which have partition_key record are assigned to the specific partition, others are assigned a partition at random |
-|Set| Exists | Messages which have partition_key record are assigned to the specific partition with parition_key, others are assigned to the specific partition with default_parition_key |
-
-If key name `message_key` exists in a message, this plugin publishes the value of message_key to kafka and can be read by consumers. Same message key will be assigned to all messages by setting `default_message_key` in config file. If message_key exists and if partition_key is not set explicitly, messsage_key will be used for partitioning.
-
-### Output plugin
-
-This plugin is for fluentd v1.0 or later. This will be `out_kafka` plugin in the future.
-
-    <match app.**>
-      @type kafka2
-
-      brokers             <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,.. # Set brokers directly
-
-      topic_key             (string) :default => 'topic'
-      partition_key         (string) :default => 'partition'
-      partition_key_key     (string) :default => 'partition_key'
-      message_key_key       (string) :default => 'message_key'
-      default_topic         (string) :default => nil
-      default_partition_key (string) :default => nil
-      default_message_key   (string) :default => nil
-      exclude_topic_key     (bool) :default => false
-      exclude_partition_key (bool) :default => false
-      get_kafka_client_log  (bool) :default => false
-      use_default_for_unknown_topic (bool) :default => false
-
-      <format>
-        @type (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
-      </format>
-      <inject>
-        tag_key tag
-        time_key time
-      </inject>
-
-      # See fluentd document for buffer related parameters: http://docs.fluentd.org/articles/buffer-plugin-overview
-      # Buffer chunk key should be same with topic_key. If value is not found in the record, default_topic is used.
-      <buffer topic>
-        flush_interval 10s
-      </buffer>
-
-      # ruby-kafka producer options
-      max_send_retries             (integer)     :default => 1
-      required_acks                (integer)     :default => -1
-      ack_timeout                  (integer)     :default => nil (Use default of ruby-kafka)
-      compression_codec            (gzip|snappy) :default => nil (No compression)
-    </match>
-
-### Non-buffered output plugin
-
-This plugin uses ruby-kafka producer for writing data. For performance and reliability concerns, use `kafka_bufferd` output instead. This is mainly for testing.
-
-    <match app.**>
-      @type kafka
-
-      # Brokers: you can choose either brokers or zookeeper.
-      brokers        <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,.. # Set brokers directly
-      zookeeper      <zookeeper_host>:<zookeeper_port> # Set brokers via Zookeeper
-      zookeeper_path <broker path in zookeeper> :default => /brokers/ids # Set path in zookeeper for kafka
-
-      default_topic         (string) :default => nil
-      default_partition_key (string) :default => nil
-      default_message_key   (string) :default => nil
-      output_data_type      (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
-      output_include_tag    (bool) :default => false
-      output_include_time   (bool) :default => false
-      exclude_topic_key     (bool) :default => false
-      exclude_partition_key (bool) :default => false
-
-      # ruby-kafka producer options
-      max_send_retries    (integer)     :default => 1
-      required_acks       (integer)     :default => -1
-      ack_timeout         (integer)     :default => nil (Use default of ruby-kafka)
-      compression_codec   (gzip|snappy) :default => nil
-      max_buffer_size     (integer)     :default => nil (Use default of ruby-kafka)
-      max_buffer_bytesize (integer)     :default => nil (Use default of ruby-kafka)
-    </match>
-
-This plugin also supports ruby-kafka related parameters. See Buffered output plugin section.
-
-### rdkafka based output plugin
-
-This plugin uses `rdkafka` instead of `ruby-kafka` for ruby client.
-You need to install rdkafka gem.
-
-    # rdkafka is C extension library so need development tools like ruby-devel, gcc and etc
-    $ gem install rdkafka --no-document
-
-    <match kafka.**>
-      @type rdkafka
-
-      default_topic kafka
-      flush_interval 1s
-      output_data_type json
-
-      rdkafka_options {
-        "log_level" : 7
-      }
-    </match>
-
-## Contributing
-
-1. Fork it
-2. Create your feature branch (`git checkout -b my-new-feature`)
-3. Commit your changes (`git commit -am 'Added some feature'`)
-4. Push to the branch (`git push origin my-new-feature`)
-5. Create new Pull Request
+# for support kafka 1.x
+# fluent-plugin-kafka, a plugin for [Fluentd](http://fluentd.org)
+
+[![Build Status](https://travis-ci.org/fluent/fluent-plugin-kafka.svg?branch=master)](https://travis-ci.org/fluent/fluent-plugin-kafka)
+
+A fluentd plugin to both consume and produce data for Apache Kafka.
+
+TODO: Also, I need to write tests
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'fluent-plugin-kafka'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fluent-plugin-kafka --no-document
+
+If you want to use zookeeper related parameters, you also need to install zookeeper gem. zookeeper gem includes native extension, so development tools are needed, e.g. gcc, make and etc.
+
+## Requirements
+
+- Ruby 2.1 or later
+- Input plugins work with kafka v0.9 or later
+- Output plugins work with kafka v0.8 or later
+
+## Usage
+
+### Common parameters
+
+#### SSL authentication
+
+- ssl_ca_cert
+- ssl_client_cert
+- ssl_client_cert_key
+- ssl_ca_certs_from_system
+
+Set path to SSL related files. See [Encryption and Authentication using SSL](https://github.com/zendesk/ruby-kafka#encryption-and-authentication-using-ssl) for more detail.
+
+#### SASL authentication 
+
+##### with GSSAPI
+
+- principal
+- keytab
+
+Set principal and path to keytab for SASL/GSSAPI authentication. 
+See [Authentication using SASL](https://github.com/zendesk/ruby-kafka#authentication-using-sasl) for more details.
+
+##### with Plain/SCRAM
+
+- username
+- password
+- scram_mechanism
+- sasl_over_ssl
+
+Set username, password, scram_mechanism and sasl_over_ssl for SASL/Plain or Scram authentication. 
+See [Authentication using SASL](https://github.com/zendesk/ruby-kafka#authentication-using-sasl) for more details.
+
+### Input plugin (@type 'kafka')
+
+Consume events by single consumer.
+
+    <source>
+      @type kafka
+
+      brokers <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,..
+      topics <listening topics(separate with comma',')>
+      format <input text type (text|json|ltsv|msgpack)> :default => json
+      message_key <key (Optional, for text format only, default is message)>
+      add_prefix <tag prefix (Optional)>
+      add_suffix <tag suffix (Optional)>
+
+      # Optionally, you can manage topic offset by using zookeeper
+      offset_zookeeper    <zookeer node list (<zookeeper1_host>:<zookeeper1_port>,<zookeeper2_host>:<zookeeper2_port>,..)>
+      offset_zk_root_node <offset path in zookeeper> default => '/fluent-plugin-kafka'
+
+      # ruby-kafka consumer options
+      max_bytes     (integer) :default => nil (Use default of ruby-kafka)
+      max_wait_time (integer) :default => nil (Use default of ruby-kafka)
+      min_bytes     (integer) :default => nil (Use default of ruby-kafka)
+    </source>
+
+Supports a start of processing from the assigned offset for specific topics.
+
+    <source>
+      @type kafka
+
+      brokers <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,..
+      format <input text type (text|json|ltsv|msgpack)>
+      <topic>
+        topic     <listening topic>
+        partition <listening partition: default=0>
+        offset    <listening start offset: default=-1>
+      </topic>
+      <topic>
+        topic     <listening topic>
+        partition <listening partition: default=0>
+        offset    <listening start offset: default=-1>
+      </topic>
+    </source>
+
+See also [ruby-kafka README](https://github.com/zendesk/ruby-kafka#consuming-messages-from-kafka) for more detailed documentation about ruby-kafka.
+
+Consuming topic name is used for event tag. So when the target topic name is `app_event`, the tag is `app_event`. If you want to modify tag, use `add_prefix` or `add_suffix` parameters. With `add_prefix kafka`, the tag is `kafka.app_event`.
+
+### Input plugin (@type 'kafka_group', supports kafka group)
+
+Consume events by kafka consumer group features..
+
+    <source>
+      @type kafka_group
+
+      brokers <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,..
+      consumer_group <consumer group name, must set>
+      topics <listening topics(separate with comma',')>
+      format <input text type (text|json|ltsv|msgpack)> :default => json
+      message_key <key (Optional, for text format only, default is message)>
+      add_prefix <tag prefix (Optional)>
+      add_suffix <tag suffix (Optional)>
+      retry_emit_limit <Wait retry_emit_limit x 1s when BuffereQueueLimitError happens. The default is nil and it means waiting until BufferQueueLimitError is resolved>
+      use_record_time <If true, replace event time with contents of 'time' field of fetched record>
+      time_format <string (Optional when use_record_time is used)>
+
+      # ruby-kafka consumer options
+      max_bytes               (integer) :default => 1048576
+      max_wait_time           (integer) :default => nil (Use default of ruby-kafka)
+      min_bytes               (integer) :default => nil (Use default of ruby-kafka)
+      offset_commit_interval  (integer) :default => nil (Use default of ruby-kafka)
+      offset_commit_threshold (integer) :default => nil (Use default of ruby-kafka)
+      fetcher_max_queue_size  (integer) :default => nil (Use default of ruby-kafka)
+      start_from_beginning    (bool)    :default => true
+    </source>
+
+See also [ruby-kafka README](https://github.com/zendesk/ruby-kafka#consuming-messages-from-kafka) for more detailed documentation about ruby-kafka options.
+
+Consuming topic name is used for event tag. So when the target topic name is `app_event`, the tag is `app_event`. If you want to modify tag, use `add_prefix` or `add_suffix` parameter. With `add_prefix kafka`, the tag is `kafka.app_event`.
+
+### Buffered output plugin
+
+This plugin uses ruby-kafka producer for writing data. This plugin works with recent kafka versions.
+
+    <match app.**>
+      @type kafka_buffered
+
+      # Brokers: you can choose either brokers or zookeeper. If you are not familiar with zookeeper, use brokers parameters.
+      brokers             <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,.. # Set brokers directly
+      zookeeper           <zookeeper_host>:<zookeeper_port> # Set brokers via Zookeeper
+      zookeeper_path      <broker path in zookeeper> :default => /brokers/ids # Set path in zookeeper for kafka
+
+      topic_key             (string) :default => 'topic'
+      partition_key         (string) :default => 'partition'
+      partition_key_key     (string) :default => 'partition_key'
+      message_key_key       (string) :default => 'message_key'
+      default_topic         (string) :default => nil
+      default_partition_key (string) :default => nil
+      default_message_key   (string) :default => nil
+      output_data_type      (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
+      output_include_tag    (bool) :default => false
+      output_include_time   (bool) :default => false
+      exclude_topic_key     (bool) :default => false
+      exclude_partition_key (bool) :default => false
+      get_kafka_client_log  (bool) :default => false
+
+      # See fluentd document for buffer related parameters: http://docs.fluentd.org/articles/buffer-plugin-overview
+
+      # ruby-kafka producer options
+      max_send_retries             (integer)     :default => 1
+      required_acks                (integer)     :default => -1
+      ack_timeout                  (integer)     :default => nil (Use default of ruby-kafka)
+      compression_codec            (gzip|snappy) :default => nil (No compression)
+      kafka_agg_max_bytes          (integer)     :default => 4096
+      kafka_agg_max_messages       (integer)     :default => nil (No limit)
+      max_send_limit_bytes         (integer)     :default => nil (No drop)
+      discard_kafka_delivery_failed   (bool)     :default => false (No discard)
+      monitoring_list              (array)       :default => []
+    </match>
+
+`<formatter name>` of `output_data_type` uses fluentd's formatter plugins. See [formatter article](http://docs.fluentd.org/articles/formatter-plugin-overview).
+
+ruby-kafka sometimes returns `Kafka::DeliveryFailed` error without good information.
+In this case, `get_kafka_client_log` is useful for identifying the error cause.
+ruby-kafka's log is routed to fluentd log so you can see ruby-kafka's log in fluentd logs.
+
+Supports following ruby-kafka's producer options.
+
+- max_send_retries - default: 1 - Number of times to retry sending of messages to a leader.
+- required_acks - default: -1 - The number of acks required per request. If you need flush performance, set lower value, e.g. 1, 2.
+- ack_timeout - default: nil - How long the producer waits for acks. The unit is seconds.
+- compression_codec - default: nil - The codec the producer uses to compress messages.
+- kafka_agg_max_bytes - default: 4096 - Maximum value of total message size to be included in one batch transmission.
+- kafka_agg_max_messages - default: nil - Maximum number of messages to include in one batch transmission.
+- max_send_limit_bytes - default: nil - Max byte size to send message to avoid MessageSizeTooLarge. For example, if you set 1000000(message.max.bytes in kafka), Message more than 1000000 byes will be dropped.
+- discard_kafka_delivery_failed - default: false - discard the record where [Kafka::DeliveryFailed](http://www.rubydoc.info/gems/ruby-kafka/Kafka/DeliveryFailed) occurred
+- monitoring_list - default: [] - library to be used to monitor. statsd and datadog are supported
+
+If you want to know about detail of monitoring, see also https://github.com/zendesk/ruby-kafka#monitoring
+
+See also [Kafka::Client](http://www.rubydoc.info/gems/ruby-kafka/Kafka/Client) for more detailed documentation about ruby-kafka.
+
+This plugin supports compression codec "snappy" also.
+Install snappy module before you use snappy compression.
+
+    $ gem install snappy --no-document
+
+snappy gem uses native extension, so you need to install several packages before.
+On Ubuntu, need development packages and snappy library.
+
+    $ sudo apt-get install build-essential autoconf automake libtool libsnappy-dev
+
+On CentOS 7 installation is also necessary.
+
+    $ sudo yum install gcc autoconf automake libtool snappy-devel
+
+#### Load balancing
+
+Messages will be assigned a partition at random as default by ruby-kafka, but messages with the same partition key will always be assigned to the same partition by setting `default_partition_key` in config file.
+If key name `partition_key` exists in a message, this plugin set its value of partition_key as key.
+
+|default_partition_key|partition_key| behavior |
+| --- | --- | --- |
+|Not set|Not exists| All messages are assigned a partition at random |
+|Set| Not exists| All messages are assigned to the specific partition |
+|Not set| Exists | Messages which have partition_key record are assigned to the specific partition, others are assigned a partition at random |
+|Set| Exists | Messages which have partition_key record are assigned to the specific partition with parition_key, others are assigned to the specific partition with default_parition_key |
+
+If key name `message_key` exists in a message, this plugin publishes the value of message_key to kafka and can be read by consumers. Same message key will be assigned to all messages by setting `default_message_key` in config file. If message_key exists and if partition_key is not set explicitly, messsage_key will be used for partitioning.
+
+### Output plugin
+
+This plugin is for fluentd v1.0 or later. This will be `out_kafka` plugin in the future.
+
+    <match app.**>
+      @type kafka2
+
+      brokers             <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,.. # Set brokers directly
+
+      topic_key             (string) :default => 'topic'
+      partition_key         (string) :default => 'partition'
+      partition_key_key     (string) :default => 'partition_key'
+      message_key_key       (string) :default => 'message_key'
+      default_topic         (string) :default => nil
+      default_partition_key (string) :default => nil
+      default_message_key   (string) :default => nil
+      exclude_topic_key     (bool) :default => false
+      exclude_partition_key (bool) :default => false
+      get_kafka_client_log  (bool) :default => false
+      use_default_for_unknown_topic (bool) :default => false
+
+      <format>
+        @type (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
+      </format>
+      <inject>
+        tag_key tag
+        time_key time
+      </inject>
+
+      # See fluentd document for buffer related parameters: http://docs.fluentd.org/articles/buffer-plugin-overview
+      # Buffer chunk key should be same with topic_key. If value is not found in the record, default_topic is used.
+      <buffer topic>
+        flush_interval 10s
+      </buffer>
+
+      # ruby-kafka producer options
+      max_send_retries             (integer)     :default => 1
+      required_acks                (integer)     :default => -1
+      ack_timeout                  (integer)     :default => nil (Use default of ruby-kafka)
+      compression_codec            (gzip|snappy) :default => nil (No compression)
+    </match>
+
+### Non-buffered output plugin
+
+This plugin uses ruby-kafka producer for writing data. For performance and reliability concerns, use `kafka_bufferd` output instead. This is mainly for testing.
+
+    <match app.**>
+      @type kafka
+
+      # Brokers: you can choose either brokers or zookeeper.
+      brokers        <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,.. # Set brokers directly
+      zookeeper      <zookeeper_host>:<zookeeper_port> # Set brokers via Zookeeper
+      zookeeper_path <broker path in zookeeper> :default => /brokers/ids # Set path in zookeeper for kafka
+
+      default_topic         (string) :default => nil
+      default_partition_key (string) :default => nil
+      default_message_key   (string) :default => nil
+      output_data_type      (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
+      output_include_tag    (bool) :default => false
+      output_include_time   (bool) :default => false
+      exclude_topic_key     (bool) :default => false
+      exclude_partition_key (bool) :default => false
+
+      # ruby-kafka producer options
+      max_send_retries    (integer)     :default => 1
+      required_acks       (integer)     :default => -1
+      ack_timeout         (integer)     :default => nil (Use default of ruby-kafka)
+      compression_codec   (gzip|snappy) :default => nil
+      max_buffer_size     (integer)     :default => nil (Use default of ruby-kafka)
+      max_buffer_bytesize (integer)     :default => nil (Use default of ruby-kafka)
+    </match>
+
+This plugin also supports ruby-kafka related parameters. See Buffered output plugin section.
+
+### rdkafka based output plugin
+
+This plugin uses `rdkafka` instead of `ruby-kafka` for ruby client.
+You need to install rdkafka gem.
+
+    # rdkafka is C extension library so need development tools like ruby-devel, gcc and etc
+    $ gem install rdkafka --no-document
+
+    <match kafka.**>
+      @type rdkafka
+
+      default_topic kafka
+      flush_interval 1s
+      output_data_type json
+
+      rdkafka_options {
+        "log_level" : 7
+      }
+    </match>
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/buildclean_gem.sh

@@ -0,0 +1,4 @@
+bundle clean --force
+gem clean
+gem build fluent-plugin-kafka.gemspec
+gem install fluent-plugin-kafka-custom-ruby-version-$1.gem

data/fluent-plugin-kafka.gemspec

@@ -1,24 +1,24 @@
-# -*- encoding: utf-8 -*-
-
-Gem::Specification.new do |gem|
-  gem.authors       = ["Hidemasa Togashi", "Masahiro Nakagawa"]
-  gem.email         = ["togachiro@gmail.com", "repeatedly@gmail.com"]
-  gem.description   = %q{Fluentd plugin for Apache Kafka > 0.8}
-  gem.summary       = %q{Fluentd plugin for Apache Kafka > 0.8}
-  gem.homepage      = "https://github.com/gozzip2009/fluent-plugin-kafka-custom-ruby-version"
-  gem.license       = "Apache-2.0"
-
-  gem.files         = `git ls-files`.split($\)
-  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
-  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
-  gem.name          = "fluent-plugin-kafka-custom-ruby-version"
-  gem.require_paths = ["lib"]
-  gem.version       = '0.9.3'
-  gem.required_ruby_version = ">= 2.1.0"
-
-  gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
-  gem.add_dependency 'ltsv'
-  gem.add_dependency 'ruby-kafka', '0.6.7'
-  gem.add_development_dependency "rake", ">= 0.9.2"
-  gem.add_development_dependency "test-unit", ">= 3.0.8"
-end
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Hidemasa Togashi", "Masahiro Nakagawa"]
+  gem.email         = ["togachiro@gmail.com", "repeatedly@gmail.com"]
+  gem.description   = %q{Fluentd plugin for Apache Kafka > 0.8}
+  gem.summary       = %q{Fluentd plugin for Apache Kafka > 0.8}
+  gem.homepage      = "https://github.com/gozzip2009/fluent-plugin-kafka-custom"
+  gem.license       = "Apache-2.0"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "fluent-plugin-kafka-custom-ruby-version"
+  gem.require_paths = ["lib"]
+  gem.version       = '0.9.4.32'
+  gem.required_ruby_version = ">= 2.1.0"
+
+  gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
+  gem.add_dependency 'ltsv'
+  gem.add_dependency "ruby-kafka-custom"
+  gem.add_development_dependency "rake", ">= 0.9.2"
+  gem.add_development_dependency "test-unit", ">= 3.0.8"
+end

data/lib/fluent/plugin/in_kafka.rb

@@ -33,7 +33,7 @@ class Fluent::KafkaInput < Fluent::Input
   config_param :add_offset_in_record, :bool, :default => false
 
   config_param :offset_zookeeper, :string, :default => nil
-  config_param :offset_zk_root_node, :string, :default => '/fluent-plugin-kafka-custom-ruby-version'
+  config_param :offset_zk_root_node, :string, :default => '/fluent-plugin-kafka'
   config_param :use_record_time, :bool, :default => false,
                :desc => "Replace message timestamp with contents of 'time' field."
   config_param :time_format, :string, :default => nil,

data/lib/fluent/plugin/kafka_plugin_util.rb

@@ -17,6 +17,19 @@ module Fluent
         }
       end
 
+      DummyFormatter = Object.new
+
+      def start
+        super
+
+        # This is bad point here but easy to fix for all kafka plugins
+        unless log.respond_to?(:formatter)
+          def log.formatter
+            Fluent::KafkaPluginUtil::SSLSettings::DummyFormatter
+          end
+        end
+      end
+
       def read_ssl_file(path)
         return nil if path.nil?
 

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -313,7 +313,7 @@ DESC
           record['tag'] = tag if @output_include_tag
           topic = (@exclude_topic_key ? record.delete(@topic_key) : record[@topic_key]) || def_topic
           partition_key = (@exclude_partition_key ? record.delete(@partition_key_key) : record[@partition_key_key]) || @default_partition_key
-          partition = (@exclude_partition ? record.delete(@partition) : record[@partition]) || @default_partition
+          partition = (@exclude_partition ? record.delete(@partition_key) : record[@partition_key]) || @default_partition
           message_key = (@exclude_message_key ? record.delete(@message_key_key) : record[@message_key_key]) || @default_message_key
 
           records_by_topic[topic] ||= 0

data/test/helper.rb

@@ -21,7 +21,7 @@ unless ENV.has_key?('VERBOSE')
   $log = nulllogger
 end
 
-require 'fluent/plugin/out_kafka'
+require 'fluent/plugin/in_kafka_group'
 
 class Test::Unit::TestCase
 end

data/test/plugin/test_in_kafka.rb

@@ -0,0 +1,37 @@
+require 'fluent/input'
+require 'fluent/plugin/in_kafka_group'
+require 'test/unit'
+
+class KafkaInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+          brokers 172.16.2.114:9092,172.16.2.115:9092,172.16.2.116:9092
+          format json
+          consumer_group journey-playground
+          topics journey-playground
+          kafka_message_key message_key
+          start_from_beginning true
+
+          principal journey@KAFKA.SECURE
+          keytab E:\\doc_true\\kafka_client\\journey.user.service.keytab
+          sasl_over_ssl false
+
+          ssl_ca_cert E:\\doc_true\\kafka_client\\kafka.client.cert.pem
+    ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::MyInput).configure(conf)
+  end
+
+  def test_read
+    d = create_driver(CONFIG)
+    d.run(timeout: 10)
+    
+    d.events.each do |tag, time, record|
+      print record
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka-custom-ruby-version
 version: !ruby/object:Gem::Version
-  version: 0.9.3
+  version: 0.9.4.32
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-02-28 00:00:00.000000000 Z
+date: 2019-04-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -46,19 +46,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: ruby-kafka
+  name: ruby-kafka-custom
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.6.7
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 0.6.7
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -103,6 +103,7 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- buildclean_gem.sh
 - fluent-plugin-kafka.gemspec
 - lib/fluent/plugin/in_kafka.rb
 - lib/fluent/plugin/in_kafka_group.rb
@@ -114,8 +115,9 @@ files:
 - lib/fluent/plugin/out_rdkafka.rb
 - lib/fluent/plugin/out_rdkafka2.rb
 - test/helper.rb
+- test/plugin/test_in_kafka.rb
 - test/plugin/test_out_kafka.rb
-homepage: https://github.com/gozzip2009/fluent-plugin-kafka-custom-ruby-version
+homepage: https://github.com/gozzip2009/fluent-plugin-kafka-custom
 licenses:
 - Apache-2.0
 metadata: {}
@@ -141,4 +143,5 @@ specification_version: 4
 summary: Fluentd plugin for Apache Kafka > 0.8
 test_files:
 - test/helper.rb
+- test/plugin/test_in_kafka.rb
 - test/plugin/test_out_kafka.rb