fluent-plugin-kafka 0.9.2 → 0.9.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 19ca96347c691e54c3e17b448f35cd854ce47631
-  data.tar.gz: 71ffa3bf2f49fbc7b8e6524a35a42753d1fbb916
+  metadata.gz: 42992f4c6af7ffc02905273a11f9522d581d397e
+  data.tar.gz: e057965b640edc5b5aab0f2aa0250286effae66d
 SHA512:
-  metadata.gz: b81ecd0cd9d0a8373df4de46165342b36caf67e214df542b6e790bc1363a9bd3f01662710c2076f027c618aacd2552fc1468716bf50f1e677f8e383d07b9e595
-  data.tar.gz: dc60575fc8913313c68774b9148eeb7e6eab1bd6f2ac47570f068333ce597ad058f558c4e3540b4f27022c098ec824bb8c0cb8825b48592ac4d7a2c090363479
+  metadata.gz: 35880b0157f34ee5c0a8e2809c5338b417bef35d8c1e65c88964de1eb492279a4ec0c971fe0f7f760769787689067864f46fe26029954dc5b79c0e62908ecd21
+  data.tar.gz: e2c493ae1978845b9bb319ad8dceff26b5fb45ae1f9127e544841a9b4f1df4849a57d5abf688c322fae3a2e7366653f20c0de617ed0145028bb2e69b8f03444c

data/ChangeLog

@@ -1,3 +1,7 @@
+Release 0.9.3 - 2019/05/04
+
+	* in_kafka_group: Add connect_timeout and socket_timeout parameters
+
 Release 0.9.2 - 2019/03/26
 
 	* out_kafka_buffered: Fix typo of partition_key usage

data/README.md

@@ -20,7 +20,7 @@ Or install it yourself as:
 
     $ gem install fluent-plugin-kafka --no-document
 
-If you want to use zookeeper related parameters, you also need to install zookeeper gem. zookeeper gem includes native extension, so development tools are needed, e.g. gcc, make and etc.
+If you want to use zookeeper related parameters, you also need to install zookeeper gem. zookeeper gem includes native extension, so development tools are needed, e.g. ruby-devel, gcc, make and etc.
 
 ## Requirements
 
@@ -142,7 +142,7 @@ Consuming topic name is used for event tag. So when the target topic name is `ap
 
 ### Buffered output plugin
 
-This plugin uses ruby-kafka producer for writing data. This plugin works with recent kafka versions.
+This plugin uses ruby-kafka producer for writing data. This plugin works with recent kafka versions. This plugin is for v0.12. If you use v1, see `kafka2`.
 
     <match app.**>
       @type kafka_buffered
@@ -254,6 +254,8 @@ This plugin is for fluentd v1.0 or later. This will be `out_kafka` plugin in the
       <format>
         @type (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
       </format>
+
+      # Optional. See https://docs.fluentd.org/v1.0/articles/inject-section
       <inject>
         tag_key tag
         time_key time
@@ -312,6 +314,48 @@ You need to install rdkafka gem.
     # rdkafka is C extension library so need development tools like ruby-devel, gcc and etc
     $ gem install rdkafka --no-document
 
+`rdkafka2` is for fluentd v1.0 or later.
+
+    <match app.**>
+      @type rdkafka2
+
+      brokers <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,.. # Set brokers directly
+
+      topic_key             (string) :default => 'topic'
+      default_topic         (string) :default => nil
+      partition_key         (string) :default => 'partition'
+      partition_key_key     (string) :default => 'partition_key'
+      message_key_key       (string) :default => 'message_key'
+      default_topic         (string) :default => nil
+      default_partition_key (string) :default => nil
+      default_message_key   (string) :default => nil
+      exclude_topic_key     (bool) :default => false
+      exclude_partition_key (bool) :default => false
+
+      <format>
+        @type (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
+      </format>
+
+      # Optional. See https://docs.fluentd.org/v1.0/articles/inject-section
+      <inject>
+        tag_key tag
+        time_key time
+      </inject>
+
+      # See fluentd document for buffer section parameters: https://docs.fluentd.org/v1.0/articles/buffer-section
+      # Buffer chunk key should be same with topic_key. If value is not found in the record, default_topic is used.
+      <buffer topic>
+        flush_interval 10s
+      </buffer>
+
+      # You can set any rdkafka configuration via this parameter: https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
+      rdkafka_options {
+        "log_level" : 7
+      }
+    </match>
+
+If you use v0.12, use `rdkafka` instead.
+
     <match kafka.**>
       @type rdkafka
 

data/fluent-plugin-kafka.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.9.2'
+  gem.version       = '0.9.3'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/in_kafka_group.rb

@@ -30,6 +30,10 @@ class Fluent::KafkaGroupInput < Fluent::Input
                :desc => "Time format to be used to parse 'time' filed."
   config_param :kafka_message_key, :string, :default => nil,
                :desc => "Set kafka's message key to this field"
+  config_param :connect_timeout, :integer, :default => nil,
+               :desc => "[Integer, nil] the timeout setting for connecting to brokers"
+  config_param :socket_timeout, :integer, :default => nil,
+               :desc => "[Integer, nil] the timeout setting for socket connection"
 
   config_param :retry_wait_seconds, :integer, :default => 30
   config_param :disable_retry_limit, :bool, :default => false,
@@ -152,16 +156,16 @@ class Fluent::KafkaGroupInput < Fluent::Input
     super
 
     if @scram_mechanism != nil && @username != nil && @password != nil
-      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: log, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: log, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_scram_username: @username, sasl_scram_password: @password,
                          sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl)
     elsif @username != nil && @password != nil
-      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: log, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: log, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_plain_username: @username, sasl_plain_password: @password)
     else
-      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: log, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: log, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab)
     end

data/lib/fluent/plugin/out_rdkafka2.rb

@@ -39,7 +39,6 @@ DESC
     config_param :default_message_key, :string, :default => nil
     config_param :partition_key, :string, :default => 'partition', :desc => "Field for kafka partition"
     config_param :default_partition, :integer, :default => nil
-    config_param :client_id, :string, :default => 'kafka'
     config_param :output_data_type, :string, :default => 'json', :obsoleted => "Use <format> section instead"
     config_param :output_include_tag, :bool, :default => false, :obsoleted => "Use <inject> section instead"
     config_param :output_include_time, :bool, :default => false, :obsoleted => "Use <inject> section instead"
@@ -49,36 +48,36 @@ Set true to remove partition from data
 DESC
     config_param :exclude_message_key, :bool, :default => false,
                  :desc => <<-DESC
-Set true to remove partition key from data
+Set true to remove message_key from data
 DESC
     config_param :exclude_topic_key, :bool, :default => false,
                  :desc => <<-DESC
-Set true to remove topic name key from data
+Set true to remove topic key from data
 DESC
     config_param :max_send_retries, :integer, :default => 2,
-                 :desc => "Number of times to retry sending of messages to a leader."
+                 :desc => "Number of times to retry sending of messages to a leader. Used for message.send.max.retries"
     config_param :required_acks, :integer, :default => -1,
-                 :desc => "The number of acks required per request."
+                 :desc => "The number of acks required per request. Used for request.required.acks"
     config_param :ack_timeout, :time, :default => nil,
-                 :desc => "How long the producer waits for acks."
+                 :desc => "How long the producer waits for acks. Used for request.timeout.ms"
     config_param :compression_codec, :string, :default => nil,
                  :desc => <<-DESC
-The codec the producer uses to compress messages.
+The codec the producer uses to compress messages. Used for compression.codec
 Supported codecs: (gzip|snappy)
 DESC
 
-    config_param :rdkafka_buffering_max_ms, :integer, :default => nil
-    config_param :rdkafka_buffering_max_messages, :integer, :default => nil
-    config_param :rdkafka_message_max_bytes, :integer, :default => nil
-    config_param :rdkafka_message_max_num, :integer, :default => nil
-    config_param :rdkafka_delivery_handle_poll_timeout, :integer, :default => 30
-    config_param :rdkafka_options, :hash, :default => {}
+    config_param :rdkafka_buffering_max_ms, :integer, :default => nil, :desc => 'Used for queue.buffering.max.ms'
+    config_param :rdkafka_buffering_max_messages, :integer, :default => nil, :desc => 'Used for queue.buffering.max.messages'
+    config_param :rdkafka_message_max_bytes, :integer, :default => nil, :desc => 'Used for message.max.bytes'
+    config_param :rdkafka_message_max_num, :integer, :default => nil, :desc => 'Used for batch.num.messages'
+    config_param :rdkafka_delivery_handle_poll_timeout, :integer, :default => 30, :desc => 'Timeout for polling message wait'
+    config_param :rdkafka_options, :hash, :default => {}, :desc => 'Set any rdkafka configuration. See https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md'
 
     config_param :max_enqueue_retries, :integer, :default => 3
     config_param :enqueue_retry_backoff, :integer, :default => 3
 
-    config_param :service_name, :string, :default => nil
-    config_param :ssl_client_cert_key_password, :string, :default => nil
+    config_param :service_name, :string, :default => nil, :desc => 'Used for sasl.kerberos.service.name'
+    config_param :ssl_client_cert_key_password, :string, :default => nil, :desc => 'Used for ssl.key.password'
 
     config_section :buffer do
       config_set_default :chunk_keys, ["topic"]

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.9.2
+  version: 0.9.3
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-26 00:00:00.000000000 Z
+date: 2019-05-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd