sk-fluent-plugin-kafka 0.8.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 86ed6fb9afbe05c446370bc1c434ee727dde5dd1aa544640e5d535009fb29569
+  data.tar.gz: fc1030930eb993b1b4a828a4b2e81cbb95fb9c0e67a7832ade312a84e2c8cfb4
+SHA512:
+  metadata.gz: 6134b73d32ab965833fb2607c646be42859b9f311f3aeb1d942683795a6cf61761e66473a23fcf9a196c4ba7c48fdd715d5950144d7a2035f261c45a3d6c25ae
+  data.tar.gz: 37d5bd24956e1ab3d6ce383d8e29f98f6df670268b24b92f1f6100073e247ff72afc952b074cbcb2003a2980f8dc121d532e99be83fee52a1e755c6fbe4fc062

data/.gitignore

@@ -0,0 +1,2 @@
+/Gemfile.lock
+*.swp

data/.travis.yml

@@ -0,0 +1,25 @@
+language: ruby
+
+rvm:
+  - 2.1
+  - 2.2
+  - 2.3.1
+  - 2.4.1
+  - 2.5.0
+  - ruby-head
+
+before_install:
+  - gem update --system
+script:
+  - bundle exec rake test
+
+sudo: false
+
+matrix:
+  allow_failures:
+    - rvm: ruby-head
+deploy:
+  provider: rubygems
+  api_key: "bc5cc687adff822e861d4df96c0986e4"
+  on:
+    tags: true

data/ChangeLog

@@ -0,0 +1,161 @@
+Release 0.8.0 - 2018/10/18
+
+	* output: Support SASL without SSL
+	* output: Add rdkafka based output
+	* Update ruby-kafka dependency to v0.7 or later
+
+Release 0.7.9 - 2018/09/11
+
+	* in_kafka_group: Add fetcher_max_queue_size parameter
+
+Release 0.7.8 - 2018/08/29
+
+	* input: Fix regression of kafka initialization
+
+Release 0.7.7 - 2018/08/27
+
+	* in_kafka_group: Add retry_limit and disable_retry_limit parameters
+	* input: Support SSL parameters
+	* Add ssl_ca_certs_from_system parameter
+	* Make password parameter secret
+
+Release 0.7.6 - 2018/08/15
+
+	* out_kafka2: Add ignore_exceptions and exception_backup parameters
+
+Release 0.7.5 - 2018/08/14
+
+	* out_kafka2: Fix `message_key_key` parameter handling
+	* Limit ruby-kafka version('< 0.7.0') to avoid runtime error
+
+Release 0.7.4 - 2018/07/11
+
+	* out_kafka2: Need event_emitter helper for notification
+
+Release 0.7.3 - 2018/05/30
+
+	* output: Use SSL endpoint when ssl_client_cert is true
+
+Release 0.7.2 - 2018/03/12
+
+	* output: Fix the regression of get_kafka_client_log parameter
+
+Release 0.7.1 - 2018/03/05
+
+	* out_kafka2: Fix the regression of kafka client initialization
+
+Release 0.7.0 - 2018/02/27
+
+	* output: SASL PLAIN/SCRAM support
+	* input: Fix TimeParser location bug in v1
+	* out_kafka2: Fix warning condition for tag chunk key
+
+Release 0.6.6 - 2017/12/25
+
+	* output: Make topic/partition/partition_key/message_key configurable
+
+Release 0.6.5 - 2017/12/14
+
+	* in_kafka_group: Add client_id parameter
+
+Release 0.6.4 - 2017/11/23
+
+	* Multi worker support for v0.14
+	* Add kafka_message_key parameter to input plugins
+	* Relax ruby-kafka version for 0.5 or later
+	* Use oj for json when it is installed
+
+Release 0.6.3 - 2017/11/14
+
+	* in_kafka_group: re-create consumer when error happens during event fetch
+
+Release 0.6.2 - 2017/11/1
+
+	* Fix ltsv parsing issue which generates symbol keys
+
+Release 0.6.1 - 2017/08/30
+
+	* Add stats and datadog monitoring support
+	* ssl_ca_certs now accepts multiple paths
+	* Fix bug by ruby-kafka 0.4.1 changes
+	* Update ruby-kafka dependency to v0.4.1
+
+Release 0.6.0 - 2017/07/25
+
+	* Add principal and keytab parameters for SASL support
+
+Release 0.5.7 - 2017/07/13
+
+	* out_kafka_buffered: Add kafka_agg_max_messages parameter
+
+Release 0.5.6 - 2017/07/10
+
+	* output: Add ActiveSupport notification support
+
+Release 0.5.5 - 2017/04/19
+
+	* output: Some trace log level changed to debug
+	* out_kafka_buffered: Add discard_kafka_delivery_failed parameter
+
+Release 0.5.4 - 2017/04/12
+
+	* out_kafka_buffered: Add max_send_limit_bytes parameter
+	* out_kafka: Improve buffer overflow handling of ruby-kafka
+
+Release 0.5.3 - 2017/02/13
+
+	* Relax ruby-kafka dependency
+
+Release 0.5.2 - 2017/02/13
+
+	* in_kafka_group: Add max_bytes parameter
+
+Release 0.5.1 - 2017/02/06
+
+	* in_kafka_group: Fix uninitialized constant error
+
+Release 0.5.0 - 2017/01/17
+
+	* output: Add out_kafka2 plugin with v0.14 API
+
+Release 0.4.2 - 2016/12/10
+
+	* input: Add use_record_time and time_format parameters
+	* Update ruby-kafka dependency to 0.3.16.beta2
+
+Release 0.4.1 - 2016/12/01
+
+	* output: Support specifying partition
+
+Release 0.4.0 - 2016/11/08
+
+	* Remove zookeeper dependency
+
+Release 0.3.5 - 2016/10/21
+
+	* output: Support message key and related parameters. #91
+
+Release 0.3.4 - 2016/10/20
+
+	* output: Add exclude_topic_key and exclude_partition_key. #89
+
+Release 0.3.3 - 2016/10/17
+
+	* out_kafka_buffered: Add get_kafka_client_log parameter. #83
+	* out_kafka_buffered: Skip and log invalid record to avoid buffer stuck. #86
+	* in_kafka_group: Add retry_emit_limit to handle BufferQueueLimitError. #87
+
+Release 0.3.2 - 2016/10/06
+
+	* in_kafka_group: Re-fetch events after consumer error. #79
+
+Release 0.3.1 - 2016/08/28
+
+	* output: Change default required_acks to -1. #70
+	* Support ruby version changed to 2.1.0 or later
+
+Release 0.3.0 - 2016/08/24
+
+	* Fully replace poseidon ruby library with ruby-kafka to support latest kafka versions
+
+See git commits for older changes

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in fluent-plugin-kafka.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,14 @@
+Copyright (C) 2014 htgc
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+

data/README.md

@@ -0,0 +1,319 @@
+# fluent-plugin-kafka, a plugin for [Fluentd](http://fluentd.org)
+
+[![Build Status](https://travis-ci.org/fluent/fluent-plugin-kafka.svg?branch=master)](https://travis-ci.org/fluent/fluent-plugin-kafka)
+
+A fluentd plugin to both consume and produce data for Apache Kafka.
+
+TODO: Also, I need to write tests
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'fluent-plugin-kafka'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fluent-plugin-kafka --no-document
+
+If you want to use zookeeper related parameters, you also need to install zookeeper gem. zookeeper gem includes native extension, so development tools are needed, e.g. gcc, make and etc.
+
+## Requirements
+
+- Ruby 2.1 or later
+- Input plugins work with kafka v0.9 or later
+- Output plugins work with kafka v0.8 or later
+
+## Usage
+
+### Common parameters
+
+#### SSL authentication
+
+- ssl_ca_cert
+- ssl_client_cert
+- ssl_client_cert_key
+- ssl_ca_certs_from_system
+
+Set path to SSL related files. See [Encryption and Authentication using SSL](https://github.com/zendesk/ruby-kafka#encryption-and-authentication-using-ssl) for more detail.
+
+#### SASL authentication
+
+- principal
+- keytab
+
+Set principal and path to keytab for SASL/GSSAPI authentication. See [Authentication using SASL](https://github.com/zendesk/ruby-kafka#authentication-using-sasl) for more details.
+
+### Input plugin (@type 'kafka')
+
+Consume events by single consumer.
+
+    <source>
+      @type kafka
+
+      brokers <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,..
+      topics <listening topics(separate with comma',')>
+      format <input text type (text|json|ltsv|msgpack)> :default => json
+      message_key <key (Optional, for text format only, default is message)>
+      add_prefix <tag prefix (Optional)>
+      add_suffix <tag suffix (Optional)>
+
+      # Optionally, you can manage topic offset by using zookeeper
+      offset_zookeeper    <zookeer node list (<zookeeper1_host>:<zookeeper1_port>,<zookeeper2_host>:<zookeeper2_port>,..)>
+      offset_zk_root_node <offset path in zookeeper> default => '/fluent-plugin-kafka'
+
+      # ruby-kafka consumer options
+      max_bytes     (integer) :default => nil (Use default of ruby-kafka)
+      max_wait_time (integer) :default => nil (Use default of ruby-kafka)
+      min_bytes     (integer) :default => nil (Use default of ruby-kafka)
+    </source>
+
+Supports a start of processing from the assigned offset for specific topics.
+
+    <source>
+      @type kafka
+
+      brokers <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,..
+      format <input text type (text|json|ltsv|msgpack)>
+      <topic>
+        topic     <listening topic>
+        partition <listening partition: default=0>
+        offset    <listening start offset: default=-1>
+      </topic>
+      <topic>
+        topic     <listening topic>
+        partition <listening partition: default=0>
+        offset    <listening start offset: default=-1>
+      </topic>
+    </source>
+
+See also [ruby-kafka README](https://github.com/zendesk/ruby-kafka#consuming-messages-from-kafka) for more detailed documentation about ruby-kafka.
+
+Consuming topic name is used for event tag. So when the target topic name is `app_event`, the tag is `app_event`. If you want to modify tag, use `add_prefix` or `add_suffix` parameters. With `add_prefix kafka`, the tag is `kafka.app_event`.
+
+### Input plugin (@type 'kafka_group', supports kafka group)
+
+Consume events by kafka consumer group features..
+
+    <source>
+      @type kafka_group
+
+      brokers <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,..
+      consumer_group <consumer group name, must set>
+      topics <listening topics(separate with comma',')>
+      format <input text type (text|json|ltsv|msgpack)> :default => json
+      message_key <key (Optional, for text format only, default is message)>
+      add_prefix <tag prefix (Optional)>
+      add_suffix <tag suffix (Optional)>
+      retry_emit_limit <Wait retry_emit_limit x 1s when BuffereQueueLimitError happens. The default is nil and it means waiting until BufferQueueLimitError is resolved>
+      use_record_time <If true, replace event time with contents of 'time' field of fetched record>
+      time_format <string (Optional when use_record_time is used)>
+
+      # ruby-kafka consumer options
+      max_bytes               (integer) :default => 1048576
+      max_wait_time           (integer) :default => nil (Use default of ruby-kafka)
+      min_bytes               (integer) :default => nil (Use default of ruby-kafka)
+      offset_commit_interval  (integer) :default => nil (Use default of ruby-kafka)
+      offset_commit_threshold (integer) :default => nil (Use default of ruby-kafka)
+      fetcher_max_queue_size  (integer) :default => nil (Use default of ruby-kafka)
+      start_from_beginning    (bool)    :default => true
+    </source>
+
+See also [ruby-kafka README](https://github.com/zendesk/ruby-kafka#consuming-messages-from-kafka) for more detailed documentation about ruby-kafka options.
+
+Consuming topic name is used for event tag. So when the target topic name is `app_event`, the tag is `app_event`. If you want to modify tag, use `add_prefix` or `add_suffix` parameter. With `add_prefix kafka`, the tag is `kafka.app_event`.
+
+### Buffered output plugin
+
+This plugin uses ruby-kafka producer for writing data. This plugin works with recent kafka versions.
+
+    <match app.**>
+      @type kafka_buffered
+
+      # Brokers: you can choose either brokers or zookeeper. If you are not familiar with zookeeper, use brokers parameters.
+      brokers             <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,.. # Set brokers directly
+      zookeeper           <zookeeper_host>:<zookeeper_port> # Set brokers via Zookeeper
+      zookeeper_path      <broker path in zookeeper> :default => /brokers/ids # Set path in zookeeper for kafka
+
+      topic_key             (string) :default => 'topic'
+      partition_key         (string) :default => 'partition'
+      partition_key_key     (string) :default => 'partition_key'
+      message_key_key       (string) :default => 'message_key'
+      default_topic         (string) :default => nil
+      default_partition_key (string) :default => nil
+      default_message_key   (string) :default => nil
+      output_data_type      (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
+      output_include_tag    (bool) :default => false
+      output_include_time   (bool) :default => false
+      exclude_topic_key     (bool) :default => false
+      exclude_partition_key (bool) :default => false
+      get_kafka_client_log  (bool) :default => false
+
+      # See fluentd document for buffer related parameters: http://docs.fluentd.org/articles/buffer-plugin-overview
+
+      # ruby-kafka producer options
+      max_send_retries             (integer)     :default => 1
+      required_acks                (integer)     :default => -1
+      ack_timeout                  (integer)     :default => nil (Use default of ruby-kafka)
+      compression_codec            (gzip|snappy) :default => nil (No compression)
+      kafka_agg_max_bytes          (integer)     :default => 4096
+      kafka_agg_max_messages       (integer)     :default => nil (No limit)
+      max_send_limit_bytes         (integer)     :default => nil (No drop)
+      discard_kafka_delivery_failed   (bool)     :default => false (No discard)
+      monitoring_list              (array)       :default => []
+    </match>
+
+`<formatter name>` of `output_data_type` uses fluentd's formatter plugins. See [formatter article](http://docs.fluentd.org/articles/formatter-plugin-overview).
+
+ruby-kafka sometimes returns `Kafka::DeliveryFailed` error without good information.
+In this case, `get_kafka_client_log` is useful for identifying the error cause.
+ruby-kafka's log is routed to fluentd log so you can see ruby-kafka's log in fluentd logs.
+
+Supports following ruby-kafka's producer options.
+
+- max_send_retries - default: 1 - Number of times to retry sending of messages to a leader.
+- required_acks - default: -1 - The number of acks required per request. If you need flush performance, set lower value, e.g. 1, 2.
+- ack_timeout - default: nil - How long the producer waits for acks. The unit is seconds.
+- compression_codec - default: nil - The codec the producer uses to compress messages.
+- kafka_agg_max_bytes - default: 4096 - Maximum value of total message size to be included in one batch transmission.
+- kafka_agg_max_messages - default: nil - Maximum number of messages to include in one batch transmission.
+- max_send_limit_bytes - default: nil - Max byte size to send message to avoid MessageSizeTooLarge. For example, if you set 1000000(message.max.bytes in kafka), Message more than 1000000 byes will be dropped.
+- discard_kafka_delivery_failed - default: false - discard the record where [Kafka::DeliveryFailed](http://www.rubydoc.info/gems/ruby-kafka/Kafka/DeliveryFailed) occurred
+- monitoring_list - default: [] - library to be used to monitor. statsd and datadog are supported
+
+If you want to know about detail of monitoring, see also https://github.com/zendesk/ruby-kafka#monitoring
+
+See also [Kafka::Client](http://www.rubydoc.info/gems/ruby-kafka/Kafka/Client) for more detailed documentation about ruby-kafka.
+
+This plugin supports compression codec "snappy" also.
+Install snappy module before you use snappy compression.
+
+    $ gem install snappy --no-document
+
+snappy gem uses native extension, so you need to install several packages before.
+On Ubuntu, need development packages and snappy library.
+
+    $ sudo apt-get install build-essential autoconf automake libtool libsnappy-dev
+
+On CentOS 7 installation is also necessary.
+
+    $ sudo yum install gcc autoconf automake libtool snappy-devel
+
+#### Load balancing
+
+Messages will be assigned a partition at random as default by ruby-kafka, but messages with the same partition key will always be assigned to the same partition by setting `default_partition_key` in config file.
+If key name `partition_key` exists in a message, this plugin set its value of partition_key as key.
+
+|default_partition_key|partition_key| behavior |
+| --- | --- | --- |
+|Not set|Not exists| All messages are assigned a partition at random |
+|Set| Not exists| All messages are assigned to the specific partition |
+|Not set| Exists | Messages which have partition_key record are assigned to the specific partition, others are assigned a partition at random |
+|Set| Exists | Messages which have partition_key record are assigned to the specific partition with parition_key, others are assigned to the specific partition with default_parition_key |
+
+If key name `message_key` exists in a message, this plugin publishes the value of message_key to kafka and can be read by consumers. Same message key will be assigned to all messages by setting `default_message_key` in config file. If message_key exists and if partition_key is not set explicitly, messsage_key will be used for partitioning.
+
+### Output plugin
+
+This plugin is for fluentd v1.0 or later. This will be `out_kafka` plugin in the future.
+
+    <match app.**>
+      @type kafka2
+
+      brokers             <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,.. # Set brokers directly
+
+      topic_key             (string) :default => 'topic'
+      partition_key         (string) :default => 'partition'
+      partition_key_key     (string) :default => 'partition_key'
+      message_key_key       (string) :default => 'message_key'
+      default_topic         (string) :default => nil
+      default_partition_key (string) :default => nil
+      default_message_key   (string) :default => nil
+      exclude_topic_key     (bool) :default => false
+      exclude_partition_key (bool) :default => false
+      get_kafka_client_log  (bool) :default => false
+
+      <format>
+        @type (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
+      </format>
+      <inject>
+        tag_key tag
+        time_key time
+      </inject>
+
+      # See fluentd document for buffer related parameters: http://docs.fluentd.org/articles/buffer-plugin-overview
+      # Buffer chunk key should be same with topic_key. If value is not found in the record, default_topic is used.
+      <buffer topic>
+        flush_interval 10s
+      </buffer>
+
+      # ruby-kafka producer options
+      max_send_retries             (integer)     :default => 1
+      required_acks                (integer)     :default => -1
+      ack_timeout                  (integer)     :default => nil (Use default of ruby-kafka)
+      compression_codec            (gzip|snappy) :default => nil (No compression)
+    </match>
+
+### Non-buffered output plugin
+
+This plugin uses ruby-kafka producer for writing data. For performance and reliability concerns, use `kafka_bufferd` output instead. This is mainly for testing.
+
+    <match app.**>
+      @type kafka
+
+      # Brokers: you can choose either brokers or zookeeper.
+      brokers        <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,.. # Set brokers directly
+      zookeeper      <zookeeper_host>:<zookeeper_port> # Set brokers via Zookeeper
+      zookeeper_path <broker path in zookeeper> :default => /brokers/ids # Set path in zookeeper for kafka
+
+      default_topic         (string) :default => nil
+      default_partition_key (string) :default => nil
+      default_message_key   (string) :default => nil
+      output_data_type      (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
+      output_include_tag    (bool) :default => false
+      output_include_time   (bool) :default => false
+      exclude_topic_key     (bool) :default => false
+      exclude_partition_key (bool) :default => false
+
+      # ruby-kafka producer options
+      max_send_retries    (integer)     :default => 1
+      required_acks       (integer)     :default => -1
+      ack_timeout         (integer)     :default => nil (Use default of ruby-kafka)
+      compression_codec   (gzip|snappy) :default => nil
+      max_buffer_size     (integer)     :default => nil (Use default of ruby-kafka)
+      max_buffer_bytesize (integer)     :default => nil (Use default of ruby-kafka)
+    </match>
+
+This plugin also supports ruby-kafka related parameters. See Buffered output plugin section.
+
+### rdkafka based output plugin
+
+This plugin uses `rdkafka` instead of `ruby-kafka` for ruby client.
+You need to install rdkafka gem.
+
+    # rdkafka is C extension library so need development tools like ruby-devel, gcc and etc
+    $ gem install rdkafka --no-document
+
+    <match kafka.**>
+      @type rdkafka
+
+      default_topic kafka
+      flush_interval 1s
+      output_data_type json
+
+      rdkafka_options {
+        "log_level" : 7
+      }
+    </match>
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request