fluent-plugin-kafka 0.5.0 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a09a4933e7d0f7a30094cd98900a03a80dac3c9a
-  data.tar.gz: 9421658f52091e37e39e32ae10e7e93132d9394f
+  metadata.gz: 3b6b3346c69250a4e8500843ed0dc689a8b63058
+  data.tar.gz: 3da518e3b7f82cdac6cc951d2b2ad33aa1242ef6
 SHA512:
-  metadata.gz: e3a72bb6fecbe2dd0204e8bc84234d3d11c44017699eb7ce88ee4b154d04966f939487d85ef90b4811510c490fb733a922ba1f70012dc6fbf9490afebf843ed7
-  data.tar.gz: 5b633c21eadd8797a5a6672191a391468a5b8c8e2b4c7968f74b67d7e3edc2978f3db0df2c157b835a7dcd75e899f4b26baf1d62c02ac7cf47f55943dd72dba8
+  metadata.gz: 188796327c6d23264052082b6c0838acf3515fd045d087017d6384c00463be18f130ce44439992069b2cb5efcdbafa83d7740f02a47ab72e5e9c4d2e67f0b1aa
+  data.tar.gz: 3cea790e8c4532d4887d5c97ce4806705a59596f493d9af340286c6b1db2d061d48eb448180f4dee51edb44c68a5776cba5aa100e9c47c8d4d0856f752805fd4

data/ChangeLog

@@ -1,3 +1,7 @@
+Release 0.5.1 - 2017/02/06
+
+	* in_kafka_group: Fix uninitialized constant error
+
 Release 0.5.0 - 2017/01/17
 
 	* output: Add out_kafka2 plugin with v0.14 API

data/README.md

@@ -30,6 +30,14 @@ If you want to use zookeeper related parameters, you also need to install zookee
 
 ## Usage
 
+### Common parameters
+
+- ssl_ca_cert
+- ssl_client_cert
+- ssl_client_cert_key
+
+Set path to SSL related files. See [Encryption and Authentication using SSL](https://github.com/zendesk/ruby-kafka#encryption-and-authentication-using-ssl) for more detail.
+
 ### Input plugin (@type 'kafka')
 
 Consume events by single consumer.
@@ -101,17 +109,17 @@ Consume events by kafka consumer group features..
 
 See also [ruby-kafka README](https://github.com/zendesk/ruby-kafka#consuming-messages-from-kafka) for more detailed documentation about ruby-kafka options.
 
-### Output plugin (non-buffered)
+### Buffered output plugin
 
-This plugin uses ruby-kafka producer for writing data. For performance and reliability concerns, use `kafka_bufferd` output instead.
+This plugin uses ruby-kafka producer for writing data. This plugin works with recent kafka versions.
 
     <match *.**>
-      @type kafka
+      @type kafka_buffered
 
-      # Brokers: you can choose either brokers or zookeeper.
-      brokers        <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,.. # Set brokers directly
-      zookeeper      <zookeeper_host>:<zookeeper_port> # Set brokers via Zookeeper
-      zookeeper_path <broker path in zookeeper> :default => /brokers/ids # Set path in zookeeper for kafka
+      # Brokers: you can choose either brokers or zookeeper. If you are not familiar with zookeeper, use brokers parameters.
+      brokers             <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,.. # Set brokers directly
+      zookeeper           <zookeeper_host>:<zookeeper_port> # Set brokers via Zookeeper
+      zookeeper_path      <broker path in zookeeper> :default => /brokers/ids # Set path in zookeeper for kafka
 
       default_topic         (string) :default => nil
       default_partition_key (string) :default => nil
@@ -121,18 +129,27 @@ This plugin uses ruby-kafka producer for writing data. For performance and relia
       output_include_time   (bool) :default => false
       exclude_topic_key     (bool) :default => false
       exclude_partition_key (bool) :default => false
+      get_kafka_client_log  (bool) :default => false
+
+      # See fluentd document for buffer related parameters: http://docs.fluentd.org/articles/buffer-plugin-overview
 
       # ruby-kafka producer options
-      max_send_retries  (integer)     :default => 1
-      required_acks     (integer)     :default => -1
-      ack_timeout       (integer)     :default => nil (Use default of ruby-kafka)
-      compression_codec (gzip|snappy) :default => nil
+      max_send_retries    (integer)     :default => 1
+      required_acks       (integer)     :default => -1
+      ack_timeout         (integer)     :default => nil (Use default of ruby-kafka)
+      compression_codec   (gzip|snappy) :default => nil (No compression)
     </match>
 
-Supports following ruby-kafka::Producer options.
+`<formatter name>` of `output_data_type` uses fluentd's formatter plugins. See [formatter article](http://docs.fluentd.org/articles/formatter-plugin-overview).
+
+ruby-kafka sometimes returns `Kafka::DeliveryFailed` error without good information.
+In this case, `get_kafka_client_log` is useful for identifying the error cause.    
+ruby-kafka's log is routed to fluentd log so you can see ruby-kafka's log in fluentd logs.
+
+Supports following ruby-kafka's producer options.
 
 - max_send_retries - default: 1 - Number of times to retry sending of messages to a leader.
-- required_acks - default: -1 - The number of acks required per request. Default is waiting all acks for safety. If you need flush performance, set lower value, e.g. 1, 2.
+- required_acks - default: -1 - The number of acks required per request. If you need flush performance, set lower value, e.g. 1, 2.
 - ack_timeout - default: nil - How long the producer waits for acks. The unit is seconds.
 - compression_codec - default: nil - The codec the producer uses to compress messages.
 
@@ -160,21 +177,19 @@ If key name `partition_key` exists in a message, this plugin set its value of pa
 |Not set| Exists | Messages which have partition_key record are assigned to the specific partition, others are assigned a partition at random |
 |Set| Exists | Messages which have partition_key record are assigned to the specific partition with parition_key, others are assigned to the specific partition with default_parition_key |
 
-
 If key name `message_key` exists in a message, this plugin publishes the value of message_key to kafka and can be read by consumers. Same message key will be assigned to all messages by setting `default_message_key` in config file. If message_key exists and if partition_key is not set explicitly, messsage_key will be used for partitioning.
 
+### Non-buffered output plugin
 
-### Buffered output plugin
-
-This plugin uses ruby-kafka producer for writing data. This plugin works with recent kafka versions.
+This plugin uses ruby-kafka producer for writing data. For performance and reliability concerns, use `kafka_bufferd` output instead. This is mainly for testing.
 
     <match *.**>
-      @type kafka_buffered
+      @type kafka
 
       # Brokers: you can choose either brokers or zookeeper.
-      brokers             <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,.. # Set brokers directly
-      zookeeper           <zookeeper_host>:<zookeeper_port> # Set brokers via Zookeeper
-      zookeeper_path      <broker path in zookeeper> :default => /brokers/ids # Set path in zookeeper for kafka
+      brokers        <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,.. # Set brokers directly
+      zookeeper      <zookeeper_host>:<zookeeper_port> # Set brokers via Zookeeper
+      zookeeper_path <broker path in zookeeper> :default => /brokers/ids # Set path in zookeeper for kafka
 
       default_topic         (string) :default => nil
       default_partition_key (string) :default => nil
@@ -184,37 +199,15 @@ This plugin uses ruby-kafka producer for writing data. This plugin works with re
       output_include_time   (bool) :default => false
       exclude_topic_key     (bool) :default => false
       exclude_partition_key (bool) :default => false
-      get_kafka_client_log  (bool) :default => false
-
-      # See fluentd document for buffer related parameters: http://docs.fluentd.org/articles/buffer-plugin-overview
 
       # ruby-kafka producer options
-      max_send_retries    (integer)     :default => 1
-      required_acks       (integer)     :default => -1
-      ack_timeout         (integer)     :default => nil (Use default of ruby-kafka)
-      compression_codec   (gzip|snappy) :default => nil (No compression)
+      max_send_retries  (integer)     :default => 1
+      required_acks     (integer)     :default => -1
+      ack_timeout       (integer)     :default => nil (Use default of ruby-kafka)
+      compression_codec (gzip|snappy) :default => nil
     </match>
 
-`<formatter name>` of `output_data_type` uses fluentd's formatter plugins. See [formatter article](http://docs.fluentd.org/articles/formatter-plugin-overview).
-
-ruby-kafka sometimes returns `Kafka::DeliveryFailed` error without good information.
-In this case, `get_kafka_client_log` is useful for identifying the error cause.    
-ruby-kafka's log is routed to fluentd log so you can see ruby-kafka's log in fluentd logs.
-
-Supports following ruby-kafka's producer options.
-
-- max_send_retries - default: 1 - Number of times to retry sending of messages to a leader.
-- required_acks - default: -1 - The number of acks required per request.
-- ack_timeout - default: nil - How long the producer waits for acks. The unit is seconds.
-- compression_codec - default: nil - The codec the producer uses to compress messages.
-
-See also [Kafka::Client](http://www.rubydoc.info/gems/ruby-kafka/Kafka/Client) for more detailed documentation about ruby-kafka.
-
-
-This plugin supports compression codec "snappy" also.
-Install snappy module before you use snappy compression.
-
-    $ gem install snappy
+This plugin also supports ruby-kafka related parameters. See Buffered output plugin section.
 
 ## Contributing
 

data/fluent-plugin-kafka.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.5.0'
+  gem.version       = '0.5.1'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/in_kafka_group.rb

@@ -45,6 +45,12 @@ class Fluent::KafkaGroupInput < Fluent::Input
   class ForShutdown < StandardError
   end
 
+  BufferError = if defined?(Fluent::Plugin::Buffer::BufferOverflowError)
+                  Fluent::Plugin::Buffer::BufferOverflowError
+                else
+                  Fluent::BufferQueueLimitError
+                end
+
   unless method_defined?(:router)
     define_method("router") { Fluent::Engine }
   end
@@ -190,7 +196,7 @@ class Fluent::KafkaGroupInput < Fluent::Input
     retries = 0
     begin
       router.emit_stream(tag, es)
-    rescue Fluent::BufferQueueLimitError
+    rescue BufferError
       raise ForShutdown if @consumer.nil?
 
       if @retry_emit_limit.nil?

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.5.1
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-01-17 00:00:00.000000000 Z
+date: 2017-02-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -131,7 +131,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.8
+rubygems_version: 2.5.2
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin for Apache Kafka > 0.8