fluent-plugin-kafka 0.19.7 → 0.19.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2a709cad3615bca0c45f0606644142278acbb75b8e43f766dd4a1821a624aa75
-  data.tar.gz: 269bfdaa2ef8fb7b6cbb254971cc728769aba7215a472a73b34af2c04d735f6f
+  metadata.gz: f3e5a24c5a5c936b9d502631d6382faa9315a54f0813e5b636cf025ecb64212c
+  data.tar.gz: 43c1b088840c92c330c70c456d92e3849d9de326b6b772c55bae63affea9f3e3
 SHA512:
-  metadata.gz: b15df023bd526fb22752da54b460ed12ce9a2d3c35e3ad9959c86d1ba8ac06fb9915799da95135fec3c6911ae776375ee8bc13f4803cad6663e2b60351b302c1
-  data.tar.gz: f9bfe163282d53f4f583ac905bffd50cab6eeaaaf5da2b9bf7511c7dad064c3aff458252af784ff8a52a3b6b5b63c80d47fccc80677692f58dc8a9fed814e6f4
+  metadata.gz: 608800269cb2f306911e4a1a6a6aeea8b7e0e69fde4b34b5a165f67d4a306e82226de890151ffff813a9ba91ef4399d87aeb531243cb63cdf625a023cf92d3e7
+  data.tar.gz: e818390d3f8946d0449a80d8c8ddd612e4a96632b2d6765fdcbf3beeb394c0cdc7c95e0cdce8c0780a05e66e39812328ddb48cd32135476cf5a85dac244485ef

data/.github/dependabot.yml

@@ -4,3 +4,20 @@ updates:
     directory: '/'
     schedule:
       interval: 'monthly'
+    groups:
+      # PR: "Security update [package] from [old] to [new]"
+      # This PR should be merged in hurry
+      security-updates:
+        applies-to: security-updates
+        patterns:
+          - '*'
+
+      # PR: "Bump [package] from [old] to [new]"
+      # No need to be merged this PR in hurry. It is enough to merge
+      # once in a month.
+      monthly-updates:
+        applies-to: version-updates
+        patterns:
+          - '*'
+    # Allow to create PR both of security and normal updates.
+    open-pull-requests-limit: 1

data/.github/workflows/linux.yml

@@ -46,8 +46,8 @@ jobs:
             rdkafka_versions: { min: '>= 0.12.0', max: '= 0.15.2' }
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }} with rdkafka gem version (min ${{ matrix.rdkafka_versions.min }} max ${{ matrix.rdkafka_versions.max }})
     steps:
-    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-    - uses: ruby/setup-ruby@afeafc3d1ab54a631816aba4c914a0081c12ff2f # v1.310.0
+    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+    - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0
       with:
         ruby-version: ${{ matrix.ruby }}
     - name: Install confluent-kafka

data/.github/workflows/stale-actions.yml

@@ -10,7 +10,7 @@ jobs:
       issues: write
       pull-requests: write
     steps:
-    - uses: actions/stale@eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899 # v10.3.0
+    - uses: actions/stale@4391f3da665fdf50b6810c1a66712fb9ba21aa93 # v11.0.0
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         days-before-stale: 90

data/ChangeLog

@@ -1,3 +1,56 @@
+Release 0.19.8 - 2026/09/04
+	* in_kafka: Emit each record with its own tag when `tag_source record` is
+				set. The whole fetch batch used to be emitted with the tag of the
+				last parsed record, so `tag_source record` never routed anything.
+				A record whose tag field is not a non-empty String is now skipped
+				with a warning, and the offset advances even when every message of
+				a batch is skipped. (#581)
+	* in_rdkafka_group: Apply SSL and SASL parameters to the consumer.
+				`ssl_ca_cert`, `ssl_client_cert`, `username`, `password`,
+				`scram_mechanism` and `principal` were accepted and then discarded,
+				so a consumer configured for SSL connected in the clear. NOTE: these
+				consumers start negotiating TLS after this change, and passing SASL
+				credentials through `kafka_configs` with `security.protocol
+				SASL_PLAINTEXT` is now rejected at startup unless `sasl_over_ssl
+				false` is set. (#580)
+	* out_rdkafka2: Apply `unrecoverable_error_codes` to delivery report
+				errors. The check only ran for errors raised by `produce`, so even the
+				default `msg_size_too_large` kept the chunk retrying until
+				`retry_timeout`. NOTE: such a chunk is now passed to the secondary
+				output, or written to the backup directory. (#579)
+	* out_rdkafka2: Fix the rdkafka patch version dispatch. The 0_16_0 patch
+				had been unreachable since it was added, and rdkafka-ruby 0.12.1
+				through 0.13.x got no patch at all, where `producer.close(10)` raises
+				ArgumentError on shutdown. Add a patch for 0.13.x. (#578)
+	* out_kafka2, out_rdkafka2: Skip an event whose `partition` is not a
+				decimal integer within the int32 range, and coerce `message_key` with
+				`to_s`. Such an event used to raise outside the per-event guard and
+				Fluentd retried the same chunk forever, stalling every source feeding
+				the output. NOTE: `partition` is now read as base 10, so "010" means
+				10 rather than 8. (#576, #577)
+	* out_rdkafka2: Fix `scram_mechanism` being ignored. SCRAM was silently
+				downgraded to PLAIN. `scram_mechanism` is now an enum of the
+				mechanisms ruby-kafka accepts, so an unknown value is rejected at
+				startup instead of falling back. (#574, #575)
+	* out_rdkafka, out_rdkafka2: Fix `ssl_verify_hostname` being ignored, and
+				set the SSL flag for any of `ssl_ca_cert`, `ssl_ca_certs_from_system`,
+				`ssl_client_cert` and `ssl_client_cert_key`. A mutual TLS setup
+				without `ssl_ca_cert` used to connect in the clear. NOTE: these
+				configurations start negotiating TLS after this change. (#572, #573)
+	* Validate that `ssl_client_cert`, `ssl_client_cert_key`,
+				`ssl_client_cert_chain` and `ssl_client_cert_key_password` are set
+				together, in every plugin that accepts them. librdkafka used to accept
+				a certificate without a key and connect with no client identity.
+				NOTE: such a configuration is now rejected at startup. (#571)
+	* out_rdkafka2: Add `sasl_over_ssl` parameter and reject SASL credentials
+				over a plaintext connection. NOTE: a configuration that sends
+				`username` and `password` over `PLAINTEXT` or `SASL_PLAINTEXT` now
+				fails at startup. Set `sasl_over_ssl false` only if you deliberately
+				keep sending them in the clear. (#570)
+	* Replace yajl-ruby with the json gem for JSON handling, and keep
+				accepting duplicate keys so that json 3.0 does not raise
+				`JSON::ParserError`. (#566, #568)
+
 Release 0.19.7 - 2026/06/02
     * in_rdkafka_group: support regexp pattern in topics (#541)
 

data/README.md

@@ -166,6 +166,12 @@ With the introduction of the rdkafka-ruby based input plugin we hope to support
       time_source <source for message timestamp (now|kafka|record)> :default => now
       time_format <string (Optional when use_record_time is used)>
 
+      # The SSL and SASL parameters in "Common parameters" are mapped to librdkafka
+      # properties. Keys given in kafka_configs take precedence over the mapped values.
+      # ssl_ca_cert uses only its first entry, and ssl_client_cert_chain is not mapped.
+      # When true, SASL authentication requires an SSL connection
+      sasl_over_ssl (bool) :default => true
+
       # kafka consumer options
       max_wait_time_ms 500
       max_batch_size 10000
@@ -523,6 +529,8 @@ You need to install rdkafka gem.
       discard_kafka_delivery_failed (bool) :default => false (No discard)
       discard_kafka_delivery_failed_regex (regexp) :default => nil (No discard)
       use_event_time        (bool) :default => false
+      # When true, SASL authentication requires an SSL connection
+      sasl_over_ssl         (bool) :default => true
 
       # same with kafka2
       headers               (hash) :default => {}
@@ -569,6 +577,11 @@ You need to install rdkafka gem.
 `rdkafka2` supports `discard_kafka_delivery_failed_regex` parameter:
 - `discard_kafka_delivery_failed_regex` - default: nil - discard the record where the Kafka::DeliveryFailed occurred and the emitted message matches the given regex pattern, such as `/unknown_topic/`. 
 
+`rdkafka2` supports `unrecoverable_error_codes` parameter:
+- `unrecoverable_error_codes` - default: `["topic_authorization_failed", "msg_size_too_large"]` - librdkafka error codes to treat as unrecoverable errors. When a listed code is returned, `Fluent::UnrecoverableError` is raised, so Fluentd hands the whole chunk to the secondary output or to the backup directory instead of retrying it until `retry_timeout` expires. Both the produce call and the delivery report are covered, the latter only while `rdkafka_delivery_handle_poll_timeout` is not 0. Set the parameter to an empty value keeping retrying every error, as this plugin did before v0.19.x.
+- A code is spelled the way `Rdkafka::RdkafkaError#code` reports it, that is the librdkafka name lowercased with the leading underscore removed, so `RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION` is written as `unknown_partition`. A name that matches no error code is accepted and never matches anything.
+- `discard_kafka_delivery_failed` and `discard_kafka_delivery_failed_regex` take precedence over this parameter. The regex is matched against the librdkafka message, such as `Broker: Message size too large (msg_size_too_large)`.
+
 If you use v0.12, use `rdkafka` instead.
 
     <match kafka.**>

data/fluent-plugin-kafka.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.19.7'
+  gem.version       = '0.19.8'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/in_kafka.rb

@@ -143,9 +143,9 @@ class Fluent::KafkaInput < Fluent::Input
           r
         }
       rescue LoadError
-        require 'yajl'
+        require 'json'
         Proc.new { |msg, te|
-          r = Yajl::Parser.parse(msg.value)
+          r = JSON.parse(msg.value, allow_duplicate_key: true)
           add_offset_in_hash(r, te, msg.offset) if @add_offset_in_record
           r
         }
@@ -302,18 +302,24 @@ class Fluent::KafkaInput < Fluent::Input
 
       return if messages.size.zero?
 
-      es = Fluent::MultiEventStream.new
-      tag = @topic_entry.topic
-      tag = @add_prefix + "." + tag if @add_prefix
-      tag = tag + "." + @add_suffix if @add_suffix
+      event_streams = {}
+      topic_tag = @topic_entry.topic
+      topic_tag = @add_prefix + "." + topic_tag if @add_prefix
+      topic_tag = topic_tag + "." + @add_suffix if @add_suffix
 
       messages.each { |msg|
         begin
           record = @parser.call(msg, @topic_entry)
           if @tag_source == :record
             tag = record[@record_tag_key]
+            unless tag.is_a?(String) && !tag.empty?
+              $log.warn "skipped record with invalid tag in #{@topic_entry.topic}/#{@topic_entry.partition}", :key => @record_tag_key, :value => tag, :offset => msg.offset
+              next
+            end
             tag = @add_prefix + "." + tag if @add_prefix
             tag = tag + "." + @add_suffix if @add_suffix
+          else
+            tag = topic_tag
           end
           case @time_source
           when :kafka
@@ -332,7 +338,8 @@ class Fluent::KafkaInput < Fluent::Input
           if @kafka_message_key
             record[@kafka_message_key] = msg.key
           end
-          es.add(record_time, record)
+          event_streams[tag] ||= Fluent::MultiEventStream.new
+          event_streams[tag].add(record_time, record)
         rescue => e
           $log.warn "parser error in #{@topic_entry.topic}/#{@topic_entry.partition}", :error => e.to_s, :value => msg.value, :offset => msg.offset
           $log.debug_backtrace
@@ -340,14 +347,14 @@ class Fluent::KafkaInput < Fluent::Input
       }
       offset = messages.last.offset + 1
 
-      unless es.empty?
+      event_streams.each { |tag, es|
         @router.emit_stream(tag, es)
+      }
 
-        if @offset_manager
-          @offset_manager.save_offset(offset)
-        end
-        @next_offset = offset
+      if @offset_manager
+        @offset_manager.save_offset(offset)
       end
+      @next_offset = offset
     end
   end
 

data/lib/fluent/plugin/in_kafka_group.rb

@@ -160,8 +160,8 @@ class Fluent::KafkaGroupInput < Fluent::Input
         Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
         Proc.new { |msg| Oj.load(msg.value) }
       rescue LoadError
-        require 'yajl'
-        Proc.new { |msg| Yajl::Parser.parse(msg.value) }
+        require 'json'
+        Proc.new { |msg| JSON.parse(msg.value, allow_duplicate_key: true) }
       end
     when 'ltsv'
       require 'ltsv'
@@ -265,7 +265,7 @@ class Fluent::KafkaGroupInput < Fluent::Input
   end
 
   def process_batch_with_record_tag(batch)
-    es = {} 
+    es = {}
     batch.messages.each { |msg|
       begin
         record = @parser_proc.call(msg)
@@ -361,7 +361,7 @@ class Fluent::KafkaGroupInput < Fluent::Input
           if @tag_source == :record
             process_batch_with_record_tag(batch)
           else
-            process_batch(batch) 
+            process_batch(batch)
           end
         }
       rescue ForShutdown

data/lib/fluent/plugin/in_rdkafka_group.rb

@@ -56,6 +56,7 @@ class Fluent::Plugin::RdKafkaGroupInput < Fluent::Plugin::Input
 
   include Fluent::KafkaPluginUtil::SSLSettings
   include Fluent::KafkaPluginUtil::SaslSettings
+  include Fluent::KafkaPluginUtil::RdkafkaSecuritySettings
 
   class ForShutdown < StandardError
   end
@@ -126,6 +127,8 @@ class Fluent::Plugin::RdKafkaGroupInput < Fluent::Plugin::Input
     if @time_source == :record and @time_format
       @time_parser = Fluent::TimeParser.new(@time_format)
     end
+
+    @rdkafka_config = build_config
   end
 
   def setup_parser(parser_conf)
@@ -137,8 +140,8 @@ class Fluent::Plugin::RdKafkaGroupInput < Fluent::Plugin::Input
         Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
         Proc.new { |msg| Oj.load(msg.payload) }
       rescue LoadError
-        require 'yajl'
-        Proc.new { |msg| Yajl::Parser.parse(msg.payload) }
+        require 'json'
+        Proc.new { |msg| JSON.parse(msg.payload, allow_duplicate_key: true) }
       end
     when 'ltsv'
       require 'ltsv'
@@ -177,8 +180,17 @@ class Fluent::Plugin::RdKafkaGroupInput < Fluent::Plugin::Input
     super
   end
 
+  def build_config
+    config = rdkafka_security_config
+    @kafka_configs.each { |k, v|
+      config[k.to_sym] = v
+    }
+    validate_sasl_over_ssl(config)
+    config
+  end
+
   def setup_consumer
-    consumer = Rdkafka::Config.new(@kafka_configs).consumer
+    consumer = Rdkafka::Config.new(@rdkafka_config).consumer
     consumer.subscribe(*@topics)
     consumer
   end

data/lib/fluent/plugin/kafka_plugin_util.rb

@@ -1,5 +1,24 @@
 module Fluent
   module KafkaPluginUtil
+    module PartitionSettings
+      # -1 is the unassigned partition; the Kafka protocol takes the index as int32
+      PARTITION_RANGE = -1..(2**31 - 1)
+
+      def coerce_partition(partition)
+        partition = case partition
+                    when Integer
+                      partition
+                    when String
+                      Integer(partition, 10)
+                    else
+                      raise TypeError, "partition must be an Integer or a decimal String, got #{partition.class}"
+                    end
+        raise RangeError, "partition #{partition} is out of range" unless PARTITION_RANGE.cover?(partition)
+
+        partition
+      end
+    end
+
     module AwsIamSettings
       def self.included(klass)
         klass.instance_eval do
@@ -36,6 +55,20 @@ module Fluent
 
       DummyFormatter = Object.new
 
+      def configure(conf)
+        super
+
+        if @ssl_client_cert && !@ssl_client_cert_key
+          raise Fluent::ConfigError, "ssl_client_cert is set but ssl_client_cert_key is not. Please provide both."
+        elsif !@ssl_client_cert && @ssl_client_cert_key
+          raise Fluent::ConfigError, "ssl_client_cert_key is set but ssl_client_cert is not. Please provide both."
+        elsif @ssl_client_cert_chain && !@ssl_client_cert
+          raise Fluent::ConfigError, "ssl_client_cert_chain is set but ssl_client_cert is not. Please provide cert, key and chain."
+        elsif @ssl_client_cert_key_password && !@ssl_client_cert_key
+          raise Fluent::ConfigError, "ssl_client_cert_key_password is set but ssl_client_cert_key is not. Please provide both."
+        end
+      end
+
       def start
         super
 
@@ -75,10 +108,82 @@ module Fluent
                        :desc => "a username when using PLAIN/SCRAM SASL authentication"
           config_param :password, :string, :default => nil, secret: true,
                        :desc => "a password when using PLAIN/SCRAM SASL authentication"
-          config_param :scram_mechanism, :string, :default => nil,
+          config_param :scram_mechanism, :enum, :list => [:sha256, :sha512], :default => nil,
                        :desc => "if set, use SCRAM authentication with specified mechanism. When unset, default to PLAIN authentication"
         }
       end
+
+      def configure(conf)
+        super
+
+        @scram_mechanism = @scram_mechanism.to_s if @scram_mechanism
+      end
+    end
+
+    module RdkafkaSecuritySettings
+      SCRAM_MECHANISMS = {
+        "sha256" => "SCRAM-SHA-256",
+        "sha512" => "SCRAM-SHA-512",
+      }.freeze
+
+      def self.included(klass)
+        klass.instance_eval {
+          config_param :service_name, :string, :default => nil, :desc => 'Used for sasl.kerberos.service.name'
+          config_param :sasl_over_ssl, :bool, :default => true,
+                       :desc => 'When true, SASL authentication requires an SSL connection'
+        }
+      end
+
+      def rdkafka_security_config
+        config = {}
+
+        if (@ssl_ca_cert && @ssl_ca_cert[0]) || @ssl_ca_certs_from_system || @ssl_client_cert || @ssl_client_cert_key
+          ssl = true
+          config[:"ssl.ca.location"] = @ssl_ca_cert[0] if @ssl_ca_cert && @ssl_ca_cert[0]
+          config[:"ssl.certificate.location"] = @ssl_client_cert if @ssl_client_cert
+          config[:"ssl.key.location"] = @ssl_client_cert_key if @ssl_client_cert_key
+          config[:"ssl.key.password"] = @ssl_client_cert_key_password if @ssl_client_cert_key_password
+          config[:"ssl.endpoint.identification.algorithm"] = @ssl_verify_hostname ? "https" : "none"
+          config[:"enable.ssl.certificate.verification"] = true
+        end
+
+        if @principal
+          sasl = true
+          config[:"sasl.mechanisms"] = "GSSAPI"
+          config[:"sasl.kerberos.principal"] = @principal
+          config[:"sasl.kerberos.service.name"] = @service_name if @service_name
+          config[:"sasl.kerberos.keytab"] = @keytab if @keytab
+        end
+
+        if @username && @password
+          sasl = true
+          config[:"sasl.mechanisms"] = SCRAM_MECHANISMS.fetch(@scram_mechanism, 'PLAIN')
+        elsif @scram_mechanism
+          log.warn "scram_mechanism is ignored because username and password are not set"
+        end
+
+        if ssl && sasl
+          security_protocol = "SASL_SSL"
+        elsif ssl && !sasl
+          security_protocol = "SSL"
+        elsif !ssl && sasl
+          security_protocol = "SASL_PLAINTEXT"
+        else
+          security_protocol = "PLAINTEXT"
+        end
+        config[:"security.protocol"] = security_protocol
+
+        config[:"sasl.username"] = @username if @username
+        config[:"sasl.password"] = @password if @password
+
+        config
+      end
+
+      def validate_sasl_over_ssl(config)
+        if @sasl_over_ssl && config[:"sasl.password"] && config[:"security.protocol"].to_s.upcase == "SASL_PLAINTEXT"
+          raise Fluent::ConfigError, "SASL authentication requires that SSL is configured. Set 'sasl_over_ssl false' to send SASL credentials over a plaintext connection"
+        end
+      end
     end
   end
 end

data/lib/fluent/plugin/out_kafka.rb

@@ -96,7 +96,7 @@ DESC
       @seed_brokers = []
       z = Zookeeper.new(@zookeeper)
       z.get_children(:path => @zookeeper_path)[:children].each do |id|
-        broker = Yajl.load(z.get(:path => @zookeeper_path + "/#{id}")[:data])
+        broker = JSON.parse(z.get(:path => @zookeeper_path + "/#{id}")[:data], allow_duplicate_key: true)
         if @ssl_client_cert
           @seed_brokers.push(pickup_ssl_endpoint(broker))
         else
@@ -192,8 +192,8 @@ DESC
 
   def setup_formatter(conf)
     if @output_data_type == 'json'
-      require 'yajl'
-      Proc.new { |tag, time, record| Yajl::Encoder.encode(record) }
+      require 'json'
+      Proc.new { |tag, time, record| JSON.generate(record) }
     elsif @output_data_type == 'ltsv'
       require 'ltsv'
       Proc.new { |tag, time, record| LTSV.dump(record) }

data/lib/fluent/plugin/out_kafka2.rb

@@ -98,6 +98,7 @@ DESC
     include Fluent::KafkaPluginUtil::AwsIamSettings
     include Fluent::KafkaPluginUtil::SSLSettings
     include Fluent::KafkaPluginUtil::SaslSettings
+    include Fluent::KafkaPluginUtil::PartitionSettings
 
     def initialize
       super
@@ -314,8 +315,8 @@ DESC
           Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
           Proc.new { |tag, time, record| Oj.dump(record) }
         rescue LoadError
-          require 'yajl'
-          Proc.new { |tag, time, record| Yajl::Encoder.encode(record) }
+          require 'json'
+          Proc.new { |tag, time, record| JSON.generate(record) }
         end
       when 'ltsv'
         require 'ltsv'
@@ -351,6 +352,7 @@ DESC
             partition_key = (@exclude_partition_key ? record.delete(@partition_key_key) : record[@partition_key_key]) || @default_partition_key
             partition = (@exclude_partition ? record.delete(@partition_key) : record[@partition_key]) || @default_partition
             message_key = (@exclude_message_key ? record.delete(@message_key_key) : record[@message_key_key]) || @default_message_key
+            partition = coerce_partition(partition) unless partition.nil?
 
             if mutate_headers
               headers = base_headers.clone

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -119,7 +119,7 @@ DESC
       @seed_brokers = []
       z = Zookeeper.new(@zookeeper)
       z.get_children(:path => @zookeeper_path)[:children].each do |id|
-        broker = Yajl.load(z.get(:path => @zookeeper_path + "/#{id}")[:data])
+        broker = JSON.parse(z.get(:path => @zookeeper_path + "/#{id}")[:data], allow_duplicate_key: true)
         if @ssl_client_cert
           @seed_brokers.push(pickup_ssl_endpoint(broker))
         else
@@ -259,8 +259,8 @@ DESC
         Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
         Proc.new { |tag, time, record| Oj.dump(record) }
       rescue LoadError
-        require 'yajl'
-        Proc.new { |tag, time, record| Yajl::Encoder.encode(record) }
+        require 'json'
+        Proc.new { |tag, time, record| JSON.generate(record) }
       end
     elsif @output_data_type == 'ltsv'
       require 'ltsv'

data/lib/fluent/plugin/out_rdkafka.rb

@@ -130,12 +130,14 @@ DESC
       :"bootstrap.servers" => @brokers,
     }
 
-    if @ssl_ca_cert && @ssl_ca_cert[0]
+    if (@ssl_ca_cert && @ssl_ca_cert[0]) || @ssl_ca_certs_from_system || @ssl_client_cert || @ssl_client_cert_key
       ssl = true
-      config[:"ssl.ca.location"] = @ssl_ca_cert[0]
+      config[:"ssl.ca.location"] = @ssl_ca_cert[0] if @ssl_ca_cert && @ssl_ca_cert[0]
       config[:"ssl.certificate.location"] = @ssl_client_cert if @ssl_client_cert
       config[:"ssl.key.location"] = @ssl_client_cert_key if @ssl_client_cert_key
       config[:"ssl.key.password"] = @ssl_client_cert_key_password if @ssl_client_cert_key_password
+      config[:"ssl.endpoint.identification.algorithm"] = @ssl_verify_hostname ? "https" : "none"
+      config[:"enable.ssl.certificate.verification"] = true
     end
 
     if @principal
@@ -228,8 +230,8 @@ DESC
         Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
         Proc.new { |tag, time, record| Oj.dump(record) }
       rescue LoadError
-        require 'yajl'
-        Proc.new { |tag, time, record| Yajl::Encoder.encode(record) }
+        require 'json'
+        Proc.new { |tag, time, record| JSON.generate(record) }
       end
     elsif @output_data_type == 'ltsv'
       require 'ltsv'

data/lib/fluent/plugin/out_rdkafka2.rb

@@ -7,14 +7,16 @@ require 'rdkafka'
 
 begin
   rdkafka_version = Gem::Version::create(Rdkafka::VERSION)
-  if rdkafka_version < Gem::Version.create('0.12.0')
-    require_relative 'rdkafka_patch/0_11_0'
-  elsif rdkafka_version == Gem::Version.create('0.12.0')
-    require_relative 'rdkafka_patch/0_12_0'
+  if rdkafka_version >= Gem::Version.create('0.16.0')
+    require_relative 'rdkafka_patch/0_16_0'
   elsif rdkafka_version >= Gem::Version.create('0.14.0')
     require_relative 'rdkafka_patch/0_14_0'
-  elsif rdkafka_version >= Gem::Version.create('0.16.0')
-    require_relative 'rdkafka_patch/0_16_0'
+  elsif rdkafka_version >= Gem::Version.create('0.13.0')
+    require_relative 'rdkafka_patch/0_13_0'
+  elsif rdkafka_version >= Gem::Version.create('0.12.0')
+    require_relative 'rdkafka_patch/0_12_0'
+  else
+    require_relative 'rdkafka_patch/0_11_0'
   end
 rescue LoadError, NameError
   raise "unable to patch rdkafka."
@@ -97,7 +99,6 @@ DESC
     config_param :enqueue_retry_backoff, :integer, :default => 3
     config_param :max_enqueue_bytes_per_second, :size, :default => nil, :desc => 'The maximum number of enqueueing bytes per second'
 
-    config_param :service_name, :string, :default => nil, :desc => 'Used for sasl.kerberos.service.name'
     config_param :unrecoverable_error_codes, :array, :default => ["topic_authorization_failed", "msg_size_too_large"],
                  :desc => 'Handle some of the error codes should be unrecoverable if specified'
 
@@ -111,6 +112,8 @@ DESC
 
     include Fluent::KafkaPluginUtil::SSLSettings
     include Fluent::KafkaPluginUtil::SaslSettings
+    include Fluent::KafkaPluginUtil::PartitionSettings
+    include Fluent::KafkaPluginUtil::RdkafkaSecuritySettings
 
     class EnqueueRate
       class LimitExceeded < StandardError
@@ -250,39 +253,7 @@ DESC
     end
 
     def build_config
-      config = {:"bootstrap.servers" => @brokers}
-
-      if @ssl_ca_cert && @ssl_ca_cert[0]
-        ssl = true
-        config[:"ssl.ca.location"] = @ssl_ca_cert[0]
-        config[:"ssl.certificate.location"] = @ssl_client_cert if @ssl_client_cert
-        config[:"ssl.key.location"] = @ssl_client_cert_key if @ssl_client_cert_key
-        config[:"ssl.key.password"] = @ssl_client_cert_key_password if @ssl_client_cert_key_password
-      end
-
-      if @principal
-        sasl = true
-        config[:"sasl.mechanisms"] = "GSSAPI"
-        config[:"sasl.kerberos.principal"] = @principal
-        config[:"sasl.kerberos.service.name"] = @service_name if @service_name
-        config[:"sasl.kerberos.keytab"] = @keytab if @keytab
-      end
-
-      if @username && @password
-        sasl = true
-        config[:"sasl.mechanisms"] = 'PLAIN'
-      end
-
-      if ssl && sasl
-        security_protocol = "SASL_SSL"
-      elsif ssl && !sasl
-        security_protocol = "SSL"
-      elsif !ssl && sasl
-        security_protocol = "SASL_PLAINTEXT"
-      else
-        security_protocol = "PLAINTEXT"
-      end
-      config[:"security.protocol"] = security_protocol
+      config = {:"bootstrap.servers" => @brokers}.merge(rdkafka_security_config)
 
       config[:"compression.codec"] = @compression_codec if @compression_codec
       config[:"message.send.max.retries"] = @max_send_retries if @max_send_retries
@@ -292,14 +263,14 @@ DESC
       config[:"queue.buffering.max.messages"] = @rdkafka_buffering_max_messages if @rdkafka_buffering_max_messages
       config[:"message.max.bytes"] = @rdkafka_message_max_bytes if @rdkafka_message_max_bytes
       config[:"batch.num.messages"] = @rdkafka_message_max_num if @rdkafka_message_max_num
-      config[:"sasl.username"] = @username if @username
-      config[:"sasl.password"] = @password if @password
       config[:"enable.idempotence"] = @idempotent if @idempotent
 
       @rdkafka_options.each { |k, v|
         config[k.to_sym] = v
       }
 
+      validate_sasl_over_ssl(config)
+
       config
     end
 
@@ -406,6 +377,8 @@ DESC
             record.delete(@topic_key) if @exclude_topic_key
             partition = (@exclude_partition ? record.delete(@partition_key) : record[@partition_key]) || @default_partition
             message_key = (@exclude_message_key ? record.delete(@message_key_key) : record[@message_key_key]) || @default_message_key
+            partition = coerce_partition(partition) unless partition.nil?
+            message_key = message_key.to_s unless message_key.nil?
 
             @headers_from_record_accessors.each do |key, header_accessor|
               headers[key] = header_accessor.call(record)
@@ -451,6 +424,7 @@ DESC
           log.warn "Delivery failed and matched regexp pattern #{@discard_kafka_delivery_failed_regex}. Discard events:", :error => e.to_s, :error_class => e.class.to_s, :tag => tag
 	else
 	  log.warn "Send exception occurred: #{e} at #{e.backtrace.first}"
+          raise Fluent::UnrecoverableError, "Rejected due to #{e}" if unrecoverable_error?(e)
           # Raise exception to retry sendind messages
           raise e
 	end
@@ -507,15 +481,14 @@ DESC
 
             raise e
           else
-            if unrecoverable_error_codes.include?(e.code.to_s)
-              # some of the errors should be handled as an unrecoverable error
-              raise Fluent::UnrecoverableError, "Rejected due to #{e}"
-            else
-              raise e
-            end
+            raise e
           end
         end
       end
     end
+
+    def unrecoverable_error?(e)
+      e.is_a?(Rdkafka::RdkafkaError) && unrecoverable_error_codes.include?(e.code.to_s)
+    end
   end
 end