fluent-plugin-kafka 0.9.6 → 0.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 283d567ceb1e9bbca30164b40f335f015466abafddb0a0ca45f9d12023270d5d
-  data.tar.gz: 417f0d18dc4043e04c133ae98daace0a7d368df81cf7b4ef26f8702ffac452ec
+  metadata.gz: e2c8a3efaae4c3daecbc360c8092aa17a46a0bbcd2e533243f6f7ed375eb3aa4
+  data.tar.gz: e2ae435c6923d71ea46e4fe3a5bfdd599cfb08aeea309da461ab15196cebdc07
 SHA512:
-  metadata.gz: 7665925660938965d03f649a660a15d5c5abb420c7897fa62a28c3818b13e34cc453f663bbb59be8cba5cac481acbb85618e7ae323ea7d4e765562079adf4fad
-  data.tar.gz: 77fa999ff6ce9fb1efe18d5d1d47e8b9daaa1e06d8aad9dafd627d97834adb5aba834e0832e5782161547b42bb55786e420a1a4423fd063af2b8614af51a65b7
+  metadata.gz: 233159921bb7c2cd699b2e10939d26bd90e3280bfae56b9e5146dc7ac75619e3d91256e1f404c5ab412726bcf74e980f61b0e52600290b0b927579969aa21937
+  data.tar.gz: f20ce22e379eb1785667d19ff95089b70c9cd33bbe0517f52e4c67b733cebc8128e6096b7376468fe15c1aea4aea3c9347f3413ad1c8535be58579f16f409583

data/ChangeLog

@@ -1,3 +1,10 @@
+Release 0.10.0 - 2019/07/03
+
+	* output: Fixed max_send_limit_bytes parameter to be in all kafka outputs
+	* output: Add ssl_verify_hostname parameter
+	* rdkafka output: Fix exceptiona handling
+	* Update ruby-kafka version to v0.7.8 or later
+
 Release 0.9.6 - 2019/05/28
 
 	* out_kafka2: Add metadata nil check

data/README.md

@@ -140,18 +140,14 @@ See also [ruby-kafka README](https://github.com/zendesk/ruby-kafka#consuming-mes
 
 Consuming topic name is used for event tag. So when the target topic name is `app_event`, the tag is `app_event`. If you want to modify tag, use `add_prefix` or `add_suffix` parameter. With `add_prefix kafka`, the tag is `kafka.app_event`.
 
-### Buffered output plugin
+### Output plugin
 
-This plugin uses ruby-kafka producer for writing data. This plugin works with recent kafka versions. This plugin is for v0.12. If you use v1, see `kafka2`.
+This plugin is for fluentd v1.0 or later. This will be `out_kafka` plugin in the future.
 
     <match app.**>
-      @type kafka_buffered
-
-      # Brokers: you can choose either brokers or zookeeper. If you are not familiar with zookeeper, use brokers parameters.
-      brokers             <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,.. # Set brokers directly
-      zookeeper           <zookeeper_host>:<zookeeper_port> # Set brokers via Zookeeper
-      zookeeper_path      <broker path in zookeeper> :default => /brokers/ids # Set path in zookeeper for kafka
+      @type kafka2
 
+      brokers               <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,.. # Set brokers directly
       topic_key             (string) :default => 'topic'
       partition_key         (string) :default => 'partition'
       partition_key_key     (string) :default => 'partition_key'
@@ -159,28 +155,37 @@ This plugin uses ruby-kafka producer for writing data. This plugin works with re
       default_topic         (string) :default => nil
       default_partition_key (string) :default => nil
       default_message_key   (string) :default => nil
-      output_data_type      (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
-      output_include_tag    (bool) :default => false
-      output_include_time   (bool) :default => false
-      exclude_topic_key     (bool) :default => false
-      exclude_partition_key (bool) :default => false
-      get_kafka_client_log  (bool) :default => false
+      exclude_topic_key     (bool)   :default => false
+      exclude_partition_key (bool)   :default => false
+      get_kafka_client_log  (bool)   :default => false
+      use_default_for_unknown_topic (bool) :default => false
+
+      <format>
+        @type (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
+      </format>
 
-      # See fluentd document for buffer related parameters: http://docs.fluentd.org/articles/buffer-plugin-overview
+      # Optional. See https://docs.fluentd.org/v/1.0/configuration/inject-section
+      <inject>
+        tag_key tag
+        time_key time
+      </inject>
+
+      # See fluentd document for buffer related parameters: https://docs.fluentd.org/v/1.0/configuration/buffer-section
+      # Buffer chunk key should be same with topic_key. If value is not found in the record, default_topic is used.
+      <buffer topic>
+        flush_interval 10s
+      </buffer>
 
       # ruby-kafka producer options
-      max_send_retries             (integer)     :default => 1
-      required_acks                (integer)     :default => -1
-      ack_timeout                  (integer)     :default => nil (Use default of ruby-kafka)
-      compression_codec            (gzip|snappy) :default => nil (No compression)
-      kafka_agg_max_bytes          (integer)     :default => 4096
-      kafka_agg_max_messages       (integer)     :default => nil (No limit)
-      max_send_limit_bytes         (integer)     :default => nil (No drop)
-      discard_kafka_delivery_failed   (bool)     :default => false (No discard)
-      monitoring_list              (array)       :default => []
+      idempotent        (bool)    :default => false
+      sasl_over_ssl     (bool)    :default => false
+      max_send_retries  (integer) :default => 1
+      required_acks     (integer) :default => -1
+      ack_timeout       (integer) :default => nil (Use default of ruby-kafka)
+      compression_codec (string)  :default => nil (No compression. Depends on ruby-kafka: https://github.com/zendesk/ruby-kafka#compression)
     </match>
 
-`<formatter name>` of `output_data_type` uses fluentd's formatter plugins. See [formatter article](http://docs.fluentd.org/articles/formatter-plugin-overview).
+The `<formatter name>` in `<format>` uses fluentd's formatter plugins. See [formatter article](https://docs.fluentd.org/v/1.0/formatter).
 
 ruby-kafka sometimes returns `Kafka::DeliveryFailed` error without good information.
 In this case, `get_kafka_client_log` is useful for identifying the error cause.
@@ -216,6 +221,16 @@ On CentOS 7 installation is also necessary.
 
     $ sudo yum install gcc autoconf automake libtool snappy-devel
 
+This plugin supports compression codec "lz4" also.
+Install extlz4 module before you use lz4 compression.
+
+    $ gem install extlz4 --no-document
+
+This plugin supports compression codec "zstd" also.
+Install zstd-ruby module before you use zstd compression.
+
+    $ gem install zstd-ruby --no-document
+
 #### Load balancing
 
 Messages will be assigned a partition at random as default by ruby-kafka, but messages with the same partition key will always be assigned to the same partition by setting `default_partition_key` in config file.
@@ -230,14 +245,17 @@ If key name `partition_key_key` exists in a message, this plugin set the value o
 
 If key name `message_key_key` exists in a message, this plugin publishes the value of message_key_key to kafka and can be read by consumers. Same message key will be assigned to all messages by setting `default_message_key` in config file. If message_key_key exists and if partition_key_key is not set explicitly, messsage_key_key will be used for partitioning.
 
-### Output plugin
+### Buffered output plugin
 
-This plugin is for fluentd v1.0 or later. This will be `out_kafka` plugin in the future.
+This plugin uses ruby-kafka producer for writing data. This plugin works with recent kafka versions. This plugin is for v0.12. If you use v1, see `kafka2`.
 
     <match app.**>
-      @type kafka2
+      @type kafka_buffered
 
+      # Brokers: you can choose either brokers or zookeeper. If you are not familiar with zookeeper, use brokers parameters.
       brokers             <broker1_host>:<broker1_port>,<broker2_host>:<broker2_port>,.. # Set brokers directly
+      zookeeper           <zookeeper_host>:<zookeeper_port> # Set brokers via Zookeeper
+      zookeeper_path      <broker path in zookeeper> :default => /brokers/ids # Set path in zookeeper for kafka
 
       topic_key             (string) :default => 'topic'
       partition_key         (string) :default => 'partition'
@@ -246,32 +264,27 @@ This plugin is for fluentd v1.0 or later. This will be `out_kafka` plugin in the
       default_topic         (string) :default => nil
       default_partition_key (string) :default => nil
       default_message_key   (string) :default => nil
+      output_data_type      (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
+      output_include_tag    (bool) :default => false
+      output_include_time   (bool) :default => false
       exclude_topic_key     (bool) :default => false
       exclude_partition_key (bool) :default => false
       get_kafka_client_log  (bool) :default => false
-      use_default_for_unknown_topic (bool) :default => false
-
-      <format>
-        @type (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
-      </format>
 
-      # Optional. See https://docs.fluentd.org/v1.0/articles/inject-section
-      <inject>
-        tag_key tag
-        time_key time
-      </inject>
-
-      # See fluentd document for buffer related parameters: http://docs.fluentd.org/articles/buffer-plugin-overview
-      # Buffer chunk key should be same with topic_key. If value is not found in the record, default_topic is used.
-      <buffer topic>
-        flush_interval 10s
-      </buffer>
+      # See fluentd document for buffer related parameters: https://docs.fluentd.org/v/0.12/buffer
 
       # ruby-kafka producer options
-      max_send_retries             (integer)     :default => 1
-      required_acks                (integer)     :default => -1
-      ack_timeout                  (integer)     :default => nil (Use default of ruby-kafka)
-      compression_codec            (gzip|snappy) :default => nil (No compression)
+      idempotent                   (bool)    :default => false
+      sasl_over_ssl                (bool)    :default => false
+      max_send_retries             (integer) :default => 1
+      required_acks                (integer) :default => -1
+      ack_timeout                  (integer) :default => nil (Use default of ruby-kafka)
+      compression_codec            (string)  :default => nil (No compression. Depends on ruby-kafka: https://github.com/zendesk/ruby-kafka#compression)
+      kafka_agg_max_bytes          (integer) :default => 4096
+      kafka_agg_max_messages       (integer) :default => nil (No limit)
+      max_send_limit_bytes         (integer) :default => nil (No drop)
+      discard_kafka_delivery_failed   (bool) :default => false (No discard)
+      monitoring_list              (array)   :default => []
     </match>
 
 ### Non-buffered output plugin
@@ -296,12 +309,12 @@ This plugin uses ruby-kafka producer for writing data. For performance and relia
       exclude_partition_key (bool) :default => false
 
       # ruby-kafka producer options
-      max_send_retries    (integer)     :default => 1
-      required_acks       (integer)     :default => -1
-      ack_timeout         (integer)     :default => nil (Use default of ruby-kafka)
-      compression_codec   (gzip|snappy) :default => nil
-      max_buffer_size     (integer)     :default => nil (Use default of ruby-kafka)
-      max_buffer_bytesize (integer)     :default => nil (Use default of ruby-kafka)
+      max_send_retries    (integer) :default => 1
+      required_acks       (integer) :default => -1
+      ack_timeout         (integer) :default => nil (Use default of ruby-kafka)
+      compression_codec   (string)  :default => nil (No compression. Depends on ruby-kafka: https://github.com/zendesk/ruby-kafka#compression)
+      max_buffer_size     (integer) :default => nil (Use default of ruby-kafka)
+      max_buffer_bytesize (integer) :default => nil (Use default of ruby-kafka)
     </match>
 
 This plugin also supports ruby-kafka related parameters. See Buffered output plugin section.
@@ -336,13 +349,13 @@ You need to install rdkafka gem.
         @type (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
       </format>
 
-      # Optional. See https://docs.fluentd.org/v1.0/articles/inject-section
+      # Optional. See https://docs.fluentd.org/v/1.0/configuration/inject-section
       <inject>
         tag_key tag
         time_key time
       </inject>
 
-      # See fluentd document for buffer section parameters: https://docs.fluentd.org/v1.0/articles/buffer-section
+      # See fluentd document for buffer section parameters: https://docs.fluentd.org/v/1.0/configuration/buffer-section
       # Buffer chunk key should be same with topic_key. If value is not found in the record, default_topic is used.
       <buffer topic>
         flush_interval 10s

data/fluent-plugin-kafka.gemspec

@@ -13,12 +13,12 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.9.6'
+  gem.version       = '0.10.0'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
   gem.add_dependency 'ltsv'
-  gem.add_dependency 'ruby-kafka', '>= 0.7.1', '< 0.8.0'
+  gem.add_dependency 'ruby-kafka', '>= 0.7.8', '< 0.8.0'
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "test-unit", ">= 3.0.8"
 end

data/lib/fluent/plugin/kafka_plugin_util.rb

@@ -14,6 +14,8 @@ module Fluent
                        :desc => "an extra PEM encoded cert to use with and SSL connection."
           config_param :ssl_ca_certs_from_system, :bool, :default => false,
                        :desc => "this configures the store to look up CA certificates from the system default certificate store on an as needed basis. The location of the store can usually be determined by: OpenSSL::X509::DEFAULT_CERT_FILE."
+          config_param :ssl_verify_hostname, :bool, :default => true,
+                       :desc => "this configures whether hostname of certificate should be verified or not."
         }
       end
 

data/lib/fluent/plugin/out_kafka.rb

@@ -55,7 +55,7 @@ DESC
                :desc => "How long the producer waits for acks."
   config_param :compression_codec, :string, :default => nil,
                :desc => "The codec the producer uses to compress messages."
-
+  config_param :max_send_limit_bytes, :size, :default => nil
   config_param :time_format, :string, :default => nil
 
   config_param :max_buffer_size, :integer, :default => nil,
@@ -108,15 +108,16 @@ DESC
         if @scram_mechanism != nil && @username != nil && @password != nil
           @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
-                             sasl_scram_username: @username, sasl_scram_password: @password, sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl)
+                             sasl_scram_username: @username, sasl_scram_password: @password, sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl,
+                             ssl_verify_hostname: @ssl_verify_hostname)
         elsif @username != nil && @password != nil
           @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
-                             sasl_plain_username: @username, sasl_plain_password: @password, sasl_over_ssl: @sasl_over_ssl)
+                             sasl_plain_username: @username, sasl_plain_password: @password, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
         else
           @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
-                             sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab, sasl_over_ssl: @sasl_over_ssl)
+                             sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
         end
         log.info "initialized kafka producer: #{@client_id}"
       else
@@ -228,17 +229,21 @@ DESC
         partition = (@exclude_partition ? record.delete('partition'.freeze) : record['partition'.freeze]) || @default_partition
         message_key = (@exclude_message_key ? record.delete('message_key') : record['message_key']) || @default_message_key
 
-        value = @formatter_proc.call(tag, time, record)
-
-        log.trace { "message will send to #{topic} with partition_key: #{partition_key}, partition: #{partition}, message_key: #{message_key} and value: #{value}." }
-	begin
-          producer.produce(value, topic: topic, key: message_key, partition: partition, partition_key: partition_key)
-	rescue Kafka::BufferOverflow => e
-	  log.warn "BufferOverflow occurred: #{e}"
-	  log.info "Trying to deliver the messages to prevent the buffer from overflowing again."
-	  producer.deliver_messages
-	  log.info "Recovered from BufferOverflow successfully`"
-	end
+        record_buf = @formatter_proc.call(tag, time, record)
+        record_buf_bytes = record_buf.bytesize
+        if @max_send_limit_bytes && record_buf_bytes > @max_send_limit_bytes
+          log.warn "record size exceeds max_send_limit_bytes. Skip event:", :time => time, :record => record
+          next
+        end
+        log.trace { "message will send to #{topic} with partition_key: #{partition_key}, partition: #{partition}, message_key: #{message_key} and value: #{record_buf}." }
+        begin
+          producer.produce(record_buf, topic: topic, key: message_key, partition: partition, partition_key: partition_key)
+        rescue Kafka::BufferOverflow => e
+          log.warn "BufferOverflow occurred: #{e}"
+          log.info "Trying to deliver the messages to prevent the buffer from overflowing again."
+          producer.deliver_messages
+          log.info "Recovered from BufferOverflow successfully`"
+        end
       end
 
       producer.deliver_messages

data/lib/fluent/plugin/out_kafka2.rb

@@ -55,9 +55,9 @@ DESC
     config_param :compression_codec, :string, :default => nil,
                  :desc => <<-DESC
 The codec the producer uses to compress messages.
-Supported codecs: (gzip|snappy)
+Supported codecs depends on ruby-kafka: https://github.com/zendesk/ruby-kafka#compression
 DESC
-
+    config_param :max_send_limit_bytes, :size, :default => nil
     config_param :active_support_notification_regex, :string, :default => nil,
                  :desc => <<-DESC
 Add a regular expression to capture ActiveSupport notifications from the Kafka client
@@ -87,15 +87,17 @@ DESC
           @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_client_cert_chain: read_ssl_file(@ssl_client_cert_chain),
                              ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_scram_username: @username, sasl_scram_password: @password,
-                             sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl)
+                             sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
         elsif @username != nil && @password != nil
           @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_client_cert_chain: read_ssl_file(@ssl_client_cert_chain),
-                             ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_plain_username: @username, sasl_plain_password: @password, sasl_over_ssl: @sasl_over_ssl)
+                             ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_plain_username: @username, sasl_plain_password: @password, sasl_over_ssl: @sasl_over_ssl,
+                             ssl_verify_hostname: @ssl_verify_hostname)
         else
           @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_client_cert_chain: read_ssl_file(@ssl_client_cert_chain),
-                             ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab, sasl_over_ssl: @sasl_over_ssl)
+                             ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab, sasl_over_ssl: @sasl_over_ssl,
+                             ssl_verify_hostname: @ssl_verify_hostname)
         end
         log.info "initialized kafka producer: #{@client_id}"
       rescue Exception => e
@@ -214,6 +216,11 @@ DESC
             message_key = (@exclude_message_key ? record.delete(@message_key_key) : record[@message_key_key]) || @default_message_key
 
             record_buf = @formatter_proc.call(tag, time, record)
+            record_buf_bytes = record_buf.bytesize
+            if @max_send_limit_bytes && record_buf_bytes > @max_send_limit_bytes
+              log.warn "record size exceeds max_send_limit_bytes. Skip event:", :time => time, :record => record
+              next
+            end
           rescue StandardError => e
             log.warn "unexpected error during format record. Skip broken event:", :error => e.to_s, :error_class => e.class.to_s, :time => time, :record => record
             next

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -69,7 +69,7 @@ DESC
   config_param :compression_codec, :string, :default => nil,
                :desc => <<-DESC
 The codec the producer uses to compress messages.
-Supported codecs: (gzip|snappy)
+Supported codecs depends on ruby-kafka: https://github.com/zendesk/ruby-kafka#compression
 DESC
   config_param :max_send_limit_bytes, :size, :default => nil
   config_param :discard_kafka_delivery_failed, :bool, :default => false
@@ -131,15 +131,16 @@ DESC
         if @scram_mechanism != nil && @username != nil && @password != nil
           @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
-                             sasl_scram_username: @username, sasl_scram_password: @password, sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl)
+                             sasl_scram_username: @username, sasl_scram_password: @password, sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl,
+                             ssl_verify_hostname: @ssl_verify_hostname)
         elsif @username != nil && @password != nil
           @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
-                             sasl_plain_username: @username, sasl_plain_password: @password, sasl_over_ssl: @sasl_over_ssl)
+                             sasl_plain_username: @username, sasl_plain_password: @password, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
         else
           @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
-                             sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab, sasl_over_ssl: @sasl_over_ssl)
+                             sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab, sasl_over_ssl: @sasl_over_ssl,  ssl_verify_hostname: @ssl_verify_hostname)
         end
         log.info "initialized kafka producer: #{@client_id}"
       else

data/lib/fluent/plugin/out_rdkafka.rb

@@ -64,7 +64,7 @@ DESC
 The codec the producer uses to compress messages.
 Supported codecs: (gzip|snappy)
 DESC
-
+  config_param :max_send_limit_bytes, :size, :default => nil
   config_param :rdkafka_buffering_max_ms, :integer, :default => nil
   config_param :rdkafka_buffering_max_messages, :integer, :default => nil
   config_param :rdkafka_message_max_bytes, :integer, :default => nil
@@ -284,7 +284,7 @@ DESC
         handler = producer.produce(topic: topic, payload: record_buf, key: message_key, partition: partition)
         return handler
       rescue Exception => e
-        if e.code == :queue_full
+        if e.respond_to?(:code) && e.code == :queue_full
           if attempt <= @max_enqueue_retries
             log.warn "Failed to enqueue message; attempting retry #{attempt} of #{@max_enqueue_retries} after #{@enqueue_retry_backoff}s"
             sleep @enqueue_retry_backoff

data/lib/fluent/plugin/out_rdkafka2.rb

@@ -65,7 +65,7 @@ DESC
 The codec the producer uses to compress messages. Used for compression.codec
 Supported codecs: (gzip|snappy)
 DESC
-
+    config_param :max_send_limit_bytes, :size, :default => nil
     config_param :rdkafka_buffering_max_ms, :integer, :default => nil, :desc => 'Used for queue.buffering.max.ms'
     config_param :rdkafka_buffering_max_messages, :integer, :default => nil, :desc => 'Used for queue.buffering.max.messages'
     config_param :rdkafka_message_max_bytes, :integer, :default => nil, :desc => 'Used for message.max.bytes'
@@ -274,7 +274,7 @@ DESC
         begin
           return producer.produce(topic: topic, payload: record_buf, key: message_key, partition: partition)
         rescue Exception => e
-          if e.code == :queue_full
+          if e.respond_to?(:code) && e.code == :queue_full
             if attempt <= @max_enqueue_retries
               log.warn "Failed to enqueue message; attempting retry #{attempt} of #{@max_enqueue_retries} after #{@enqueue_retry_backoff}s"
               sleep @enqueue_retry_backoff

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.9.6
+  version: 0.10.0
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-05-29 00:00:00.000000000 Z
+date: 2019-07-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -51,7 +51,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.7.1
+        version: 0.7.8
     - - "<"
       - !ruby/object:Gem::Version
         version: 0.8.0
@@ -61,7 +61,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.7.1
+        version: 0.7.8
     - - "<"
       - !ruby/object:Gem::Version
         version: 0.8.0