fluent-plugin-kafka 0.19.6 → 0.19.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c01dcccf05f4c04f7dc5eb08c7f2da880ea7f892242931e95a009fe8cb18c270
-  data.tar.gz: d446556bb8e8c65b7a2b16460aac3fab05720f444a5551bb7da486e5941e8c98
+  metadata.gz: f3e5a24c5a5c936b9d502631d6382faa9315a54f0813e5b636cf025ecb64212c
+  data.tar.gz: 43c1b088840c92c330c70c456d92e3849d9de326b6b772c55bae63affea9f3e3
 SHA512:
-  metadata.gz: efe8546b9cdd28016b8872cdcbf91402b3cfd11b50ef1a63e38215b46128399bd05ed0d211cf03e8cbee1122b5316f15f95c3bc906937d2244340e4a9b68cdd8
-  data.tar.gz: 100e538e93d736579a5886db60e3dc769807efc0ac516f352def62b126f7d636e594161ea72c490fffcf3c8a4f0bd22ada8fc59be40873aa407f9aa034e20e75
+  metadata.gz: 608800269cb2f306911e4a1a6a6aeea8b7e0e69fde4b34b5a165f67d4a306e82226de890151ffff813a9ba91ef4399d87aeb531243cb63cdf625a023cf92d3e7
+  data.tar.gz: e818390d3f8946d0449a80d8c8ddd612e4a96632b2d6765fdcbf3beeb394c0cdc7c95e0cdce8c0780a05e66e39812328ddb48cd32135476cf5a85dac244485ef

data/.github/dependabot.yml

@@ -3,4 +3,21 @@ updates:
   - package-ecosystem: 'github-actions'
     directory: '/'
     schedule:
-      interval: 'weekly'
+      interval: 'monthly'
+    groups:
+      # PR: "Security update [package] from [old] to [new]"
+      # This PR should be merged in hurry
+      security-updates:
+        applies-to: security-updates
+        patterns:
+          - '*'
+
+      # PR: "Bump [package] from [old] to [new]"
+      # No need to be merged this PR in hurry. It is enough to merge
+      # once in a month.
+      monthly-updates:
+        applies-to: version-updates
+        patterns:
+          - '*'
+    # Allow to create PR both of security and normal updates.
+    open-pull-requests-limit: 1

data/.github/workflows/add-to-project.yml

@@ -10,8 +10,15 @@ jobs:
     name: Add issue to project
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/add-to-project@v1.0.2
+      - name: Generate token
+        id: generate_token
+        uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0
+        with:
+          app-id: ${{ secrets.PROJECT_APP_ID }}
+          private-key: ${{ secrets.PROJECT_APP_PRIVATE_KEY }}
+          owner: fluent
+      - uses: actions/add-to-project@5afcf98fcd03f1c2f92c3c83f58ae24323cc57fd # v2.0.0
         with:
           project-url: https://github.com/orgs/fluent/projects/4
-          github-token: ${{ secrets.ADD_TO_PROJECT_PAT }}
+          github-token: ${{ steps.generate_token.outputs.token }}
           labeled: waiting-for-triage

data/.github/workflows/linux.yml

@@ -4,15 +4,23 @@ on:
     branches: [master]
   pull_request:
     branches: [master]
+  schedule:
+    - cron: '0 0 1 * *'
 jobs:
+  ruby-versions:
+    uses: ruby/actions/.github/workflows/ruby_versions.yml@master
+    with:
+      engine: cruby
+      min_version: 3.0
   build:
+    needs: ruby-versions
     runs-on: ${{ matrix.os }}
     env:
       USE_RDKAFKA: 1
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '3.4', '3.3', '3.2', '3.1', '3.0' ]
+        ruby: ${{ fromJson(needs.ruby-versions.outputs.versions) }}
         os:
           - ubuntu-latest
         rdkafka_versions:
@@ -23,6 +31,7 @@ jobs:
           - { min: '>= 0.25.0', max: '>= 0.25.0' }
         # rdkafka 0.25.0 supports Ruby 3.2+, so we exclude it for Ruby 3.0 and 3.1
         exclude:
+          - ruby: head
           - ruby: '3.1'
             rdkafka_versions: { min: '>= 0.25.0', max: '>= 0.25.0' }
           - ruby: '3.0'
@@ -37,17 +46,17 @@ jobs:
             rdkafka_versions: { min: '>= 0.12.0', max: '= 0.15.2' }
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }} with rdkafka gem version (min ${{ matrix.rdkafka_versions.min }} max ${{ matrix.rdkafka_versions.max }})
     steps:
-    - uses: actions/checkout@v6
-    - uses: ruby/setup-ruby@v1
+    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+    - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0
       with:
         ruby-version: ${{ matrix.ruby }}
     - name: Install confluent-kafka
       run: |
         sudo apt install -V -y gnupg2 wget
-        wget https://packages.confluent.io/deb/6.0/archive.key
+        wget https://packages.confluent.io/deb/7.9/archive.key
         sudo gpg2 --homedir /tmp --no-default-keyring --keyring gnupg-ring:/usr/share/keyrings/confluent-archive-keyring.gpg --import archive.key
         sudo chmod 644 /usr/share/keyrings/confluent-archive-keyring.gpg
-        sudo sh -c 'echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/confluent-archive-keyring.gpg] https://packages.confluent.io/deb/6.0 stable main" > /etc/apt/sources.list.d/confluent.list'
+        sudo sh -c 'echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/confluent-archive-keyring.gpg] https://packages.confluent.io/deb/7.9 stable main" > /etc/apt/sources.list.d/confluent.list'
         sudo apt update
         sudo apt install -y confluent-community-2.13 openjdk-11-jre netcat-openbsd
     - name: unit testing
@@ -57,6 +66,5 @@ jobs:
         RDKAFKA_VERSION_MAX_RANGE: ${{ matrix.rdkafka_versions.max }}
       run: |
         sudo ./ci/prepare-kafka-server.sh
-        gem install rake
         bundle install --jobs 4 --retry 3
         bundle exec rake test

data/.github/workflows/stale-actions.yml

@@ -6,8 +6,11 @@ on:
 jobs:
   stale:
     runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
     steps:
-    - uses: actions/stale@v10
+    - uses: actions/stale@4391f3da665fdf50b6810c1a66712fb9ba21aa93 # v11.0.0
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         days-before-stale: 90

data/ChangeLog

@@ -1,3 +1,59 @@
+Release 0.19.8 - 2026/09/04
+	* in_kafka: Emit each record with its own tag when `tag_source record` is
+				set. The whole fetch batch used to be emitted with the tag of the
+				last parsed record, so `tag_source record` never routed anything.
+				A record whose tag field is not a non-empty String is now skipped
+				with a warning, and the offset advances even when every message of
+				a batch is skipped. (#581)
+	* in_rdkafka_group: Apply SSL and SASL parameters to the consumer.
+				`ssl_ca_cert`, `ssl_client_cert`, `username`, `password`,
+				`scram_mechanism` and `principal` were accepted and then discarded,
+				so a consumer configured for SSL connected in the clear. NOTE: these
+				consumers start negotiating TLS after this change, and passing SASL
+				credentials through `kafka_configs` with `security.protocol
+				SASL_PLAINTEXT` is now rejected at startup unless `sasl_over_ssl
+				false` is set. (#580)
+	* out_rdkafka2: Apply `unrecoverable_error_codes` to delivery report
+				errors. The check only ran for errors raised by `produce`, so even the
+				default `msg_size_too_large` kept the chunk retrying until
+				`retry_timeout`. NOTE: such a chunk is now passed to the secondary
+				output, or written to the backup directory. (#579)
+	* out_rdkafka2: Fix the rdkafka patch version dispatch. The 0_16_0 patch
+				had been unreachable since it was added, and rdkafka-ruby 0.12.1
+				through 0.13.x got no patch at all, where `producer.close(10)` raises
+				ArgumentError on shutdown. Add a patch for 0.13.x. (#578)
+	* out_kafka2, out_rdkafka2: Skip an event whose `partition` is not a
+				decimal integer within the int32 range, and coerce `message_key` with
+				`to_s`. Such an event used to raise outside the per-event guard and
+				Fluentd retried the same chunk forever, stalling every source feeding
+				the output. NOTE: `partition` is now read as base 10, so "010" means
+				10 rather than 8. (#576, #577)
+	* out_rdkafka2: Fix `scram_mechanism` being ignored. SCRAM was silently
+				downgraded to PLAIN. `scram_mechanism` is now an enum of the
+				mechanisms ruby-kafka accepts, so an unknown value is rejected at
+				startup instead of falling back. (#574, #575)
+	* out_rdkafka, out_rdkafka2: Fix `ssl_verify_hostname` being ignored, and
+				set the SSL flag for any of `ssl_ca_cert`, `ssl_ca_certs_from_system`,
+				`ssl_client_cert` and `ssl_client_cert_key`. A mutual TLS setup
+				without `ssl_ca_cert` used to connect in the clear. NOTE: these
+				configurations start negotiating TLS after this change. (#572, #573)
+	* Validate that `ssl_client_cert`, `ssl_client_cert_key`,
+				`ssl_client_cert_chain` and `ssl_client_cert_key_password` are set
+				together, in every plugin that accepts them. librdkafka used to accept
+				a certificate without a key and connect with no client identity.
+				NOTE: such a configuration is now rejected at startup. (#571)
+	* out_rdkafka2: Add `sasl_over_ssl` parameter and reject SASL credentials
+				over a plaintext connection. NOTE: a configuration that sends
+				`username` and `password` over `PLAINTEXT` or `SASL_PLAINTEXT` now
+				fails at startup. Set `sasl_over_ssl false` only if you deliberately
+				keep sending them in the clear. (#570)
+	* Replace yajl-ruby with the json gem for JSON handling, and keep
+				accepting duplicate keys so that json 3.0 does not raise
+				`JSON::ParserError`. (#566, #568)
+
+Release 0.19.7 - 2026/06/02
+    * in_rdkafka_group: support regexp pattern in topics (#541)
+
 Release 0.19.6 - 2026/03/04
 	* out_rdkafka2: Auto-configure SASL PLAIN when username and password are set (#547)
 	* out_rdkafka2: Fix NoMethodError during graceful reload (#546)

data/README.md

@@ -166,6 +166,12 @@ With the introduction of the rdkafka-ruby based input plugin we hope to support
       time_source <source for message timestamp (now|kafka|record)> :default => now
       time_format <string (Optional when use_record_time is used)>
 
+      # The SSL and SASL parameters in "Common parameters" are mapped to librdkafka
+      # properties. Keys given in kafka_configs take precedence over the mapped values.
+      # ssl_ca_cert uses only its first entry, and ssl_client_cert_chain is not mapped.
+      # When true, SASL authentication requires an SSL connection
+      sasl_over_ssl (bool) :default => true
+
       # kafka consumer options
       max_wait_time_ms 500
       max_batch_size 10000
@@ -177,6 +183,10 @@ With the introduction of the rdkafka-ruby based input plugin we hope to support
 
 See also [rdkafka-ruby](https://github.com/appsignal/rdkafka-ruby) and [librdkafka](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md) for more detailed documentation about Kafka consumer options.
 
+`topics` supports POSIX Extended Regular Expression pattern (not Ruby regex syntax) since v0.19.6. If you want to use regex pattern, use `/pattern/` like `/foo.*/`.
+
+**Note**: A caret (`^`) is automatically added to the beginning of the pattern.
+
 Consuming topic name is used for event tag. So when the target topic name is `app_event`, the tag is `app_event`. If you want to modify tag, use `add_prefix` or `add_suffix` parameter. With `add_prefix kafka`, the tag is `kafka.app_event`.
 
 ### Output plugin
@@ -519,6 +529,8 @@ You need to install rdkafka gem.
       discard_kafka_delivery_failed (bool) :default => false (No discard)
       discard_kafka_delivery_failed_regex (regexp) :default => nil (No discard)
       use_event_time        (bool) :default => false
+      # When true, SASL authentication requires an SSL connection
+      sasl_over_ssl         (bool) :default => true
 
       # same with kafka2
       headers               (hash) :default => {}
@@ -565,6 +577,11 @@ You need to install rdkafka gem.
 `rdkafka2` supports `discard_kafka_delivery_failed_regex` parameter:
 - `discard_kafka_delivery_failed_regex` - default: nil - discard the record where the Kafka::DeliveryFailed occurred and the emitted message matches the given regex pattern, such as `/unknown_topic/`. 
 
+`rdkafka2` supports `unrecoverable_error_codes` parameter:
+- `unrecoverable_error_codes` - default: `["topic_authorization_failed", "msg_size_too_large"]` - librdkafka error codes to treat as unrecoverable errors. When a listed code is returned, `Fluent::UnrecoverableError` is raised, so Fluentd hands the whole chunk to the secondary output or to the backup directory instead of retrying it until `retry_timeout` expires. Both the produce call and the delivery report are covered, the latter only while `rdkafka_delivery_handle_poll_timeout` is not 0. Set the parameter to an empty value keeping retrying every error, as this plugin did before v0.19.x.
+- A code is spelled the way `Rdkafka::RdkafkaError#code` reports it, that is the librdkafka name lowercased with the leading underscore removed, so `RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION` is written as `unknown_partition`. A name that matches no error code is accepted and never matches anything.
+- `discard_kafka_delivery_failed` and `discard_kafka_delivery_failed_regex` take precedence over this parameter. The regex is matched against the librdkafka message, such as `Broker: Message size too large (msg_size_too_large)`.
+
 If you use v0.12, use `rdkafka` instead.
 
     <match kafka.**>

data/ci/prepare-kafka-server.sh

@@ -1,5 +1,7 @@
 #!/bin/sh
 
+set -x
+
 export KAFKA_OPTS=-Dzookeeper.4lw.commands.whitelist=ruok
 /usr/bin/zookeeper-server-start /etc/kafka/zookeeper.properties  &
 N_POLLING=30
@@ -30,4 +32,14 @@ while true ; do
 	exit 1
     fi
 done
-/usr/bin/kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test
+# kafka-topics --version fails to connect zookeeper here.
+case "$(java -cp "/usr/share/java/kafka/*" kafka.Kafka --version 2>/dev/null)" in
+    7.*)
+        /usr/bin/kafka-topics --create --bootstrap-server localhost:9092 --replication-factor 1 --partitions 1 --topic test
+        ;;
+    *)
+        # Deprecated zookeeper
+        /usr/bin/kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test
+        ;;
+esac
+

data/fluent-plugin-kafka.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.19.6'
+  gem.version       = '0.19.8'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/in_kafka.rb

@@ -143,9 +143,9 @@ class Fluent::KafkaInput < Fluent::Input
           r
         }
       rescue LoadError
-        require 'yajl'
+        require 'json'
         Proc.new { |msg, te|
-          r = Yajl::Parser.parse(msg.value)
+          r = JSON.parse(msg.value, allow_duplicate_key: true)
           add_offset_in_hash(r, te, msg.offset) if @add_offset_in_record
           r
         }
@@ -302,18 +302,24 @@ class Fluent::KafkaInput < Fluent::Input
 
       return if messages.size.zero?
 
-      es = Fluent::MultiEventStream.new
-      tag = @topic_entry.topic
-      tag = @add_prefix + "." + tag if @add_prefix
-      tag = tag + "." + @add_suffix if @add_suffix
+      event_streams = {}
+      topic_tag = @topic_entry.topic
+      topic_tag = @add_prefix + "." + topic_tag if @add_prefix
+      topic_tag = topic_tag + "." + @add_suffix if @add_suffix
 
       messages.each { |msg|
         begin
           record = @parser.call(msg, @topic_entry)
           if @tag_source == :record
             tag = record[@record_tag_key]
+            unless tag.is_a?(String) && !tag.empty?
+              $log.warn "skipped record with invalid tag in #{@topic_entry.topic}/#{@topic_entry.partition}", :key => @record_tag_key, :value => tag, :offset => msg.offset
+              next
+            end
             tag = @add_prefix + "." + tag if @add_prefix
             tag = tag + "." + @add_suffix if @add_suffix
+          else
+            tag = topic_tag
           end
           case @time_source
           when :kafka
@@ -332,7 +338,8 @@ class Fluent::KafkaInput < Fluent::Input
           if @kafka_message_key
             record[@kafka_message_key] = msg.key
           end
-          es.add(record_time, record)
+          event_streams[tag] ||= Fluent::MultiEventStream.new
+          event_streams[tag].add(record_time, record)
         rescue => e
           $log.warn "parser error in #{@topic_entry.topic}/#{@topic_entry.partition}", :error => e.to_s, :value => msg.value, :offset => msg.offset
           $log.debug_backtrace
@@ -340,14 +347,14 @@ class Fluent::KafkaInput < Fluent::Input
       }
       offset = messages.last.offset + 1
 
-      unless es.empty?
+      event_streams.each { |tag, es|
         @router.emit_stream(tag, es)
+      }
 
-        if @offset_manager
-          @offset_manager.save_offset(offset)
-        end
-        @next_offset = offset
+      if @offset_manager
+        @offset_manager.save_offset(offset)
       end
+      @next_offset = offset
     end
   end
 

data/lib/fluent/plugin/in_kafka_group.rb

@@ -160,8 +160,8 @@ class Fluent::KafkaGroupInput < Fluent::Input
         Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
         Proc.new { |msg| Oj.load(msg.value) }
       rescue LoadError
-        require 'yajl'
-        Proc.new { |msg| Yajl::Parser.parse(msg.value) }
+        require 'json'
+        Proc.new { |msg| JSON.parse(msg.value, allow_duplicate_key: true) }
       end
     when 'ltsv'
       require 'ltsv'
@@ -265,7 +265,7 @@ class Fluent::KafkaGroupInput < Fluent::Input
   end
 
   def process_batch_with_record_tag(batch)
-    es = {} 
+    es = {}
     batch.messages.each { |msg|
       begin
         record = @parser_proc.call(msg)
@@ -361,7 +361,7 @@ class Fluent::KafkaGroupInput < Fluent::Input
           if @tag_source == :record
             process_batch_with_record_tag(batch)
           else
-            process_batch(batch) 
+            process_batch(batch)
           end
         }
       rescue ForShutdown

data/lib/fluent/plugin/in_rdkafka_group.rb

@@ -56,6 +56,7 @@ class Fluent::Plugin::RdKafkaGroupInput < Fluent::Plugin::Input
 
   include Fluent::KafkaPluginUtil::SSLSettings
   include Fluent::KafkaPluginUtil::SaslSettings
+  include Fluent::KafkaPluginUtil::RdkafkaSecuritySettings
 
   class ForShutdown < StandardError
   end
@@ -76,13 +77,25 @@ class Fluent::Plugin::RdKafkaGroupInput < Fluent::Plugin::Input
     end
     config_array
   end
+  private :_config_to_array
+
+  def regex_pattern?(topic)
+    topic.start_with?('/') && topic.end_with?('/') ? true : false
+  end
+  private :regex_pattern?
+
+  def _config_regex_pattern(topic)
+    # librdkafka recognizes string as regex pattern if the topic name starts with '^'.
+    # https://github.com/confluentinc/librdkafka/blob/570c785e9e35812db8f50254bd2f7e0cf47def39/src/rdkafka.h#L4148
+    # https://github.com/confluentinc/librdkafka/blob/e1db7eaa517f0a6438bc846a9c49ede73b9ea211/src/rdkafka_topic.c#L2064
+    [topic[1..-2].prepend('^')]
+  end
+  private :_config_regex_pattern
 
   def multi_workers_ready?
     true
   end
 
-  private :_config_to_array
-
   def configure(conf)
     compat_parameters_convert(conf, :parser)
 
@@ -93,7 +106,12 @@ class Fluent::Plugin::RdKafkaGroupInput < Fluent::Plugin::Input
     log.info "Will watch for topics #{@topics} at brokers " \
               "#{@kafka_configs["bootstrap.servers"]} and '#{@kafka_configs["group.id"]}' group"
 
-    @topics = _config_to_array(@topics)
+    topics = @topics.strip
+    if regex_pattern?(topics)
+      @topics = _config_regex_pattern(topics)
+    else
+      @topics = _config_to_array(topics)
+    end
 
     parser_conf = conf.elements('parse').first
     unless parser_conf
@@ -109,6 +127,8 @@ class Fluent::Plugin::RdKafkaGroupInput < Fluent::Plugin::Input
     if @time_source == :record and @time_format
       @time_parser = Fluent::TimeParser.new(@time_format)
     end
+
+    @rdkafka_config = build_config
   end
 
   def setup_parser(parser_conf)
@@ -120,8 +140,8 @@ class Fluent::Plugin::RdKafkaGroupInput < Fluent::Plugin::Input
         Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
         Proc.new { |msg| Oj.load(msg.payload) }
       rescue LoadError
-        require 'yajl'
-        Proc.new { |msg| Yajl::Parser.parse(msg.payload) }
+        require 'json'
+        Proc.new { |msg| JSON.parse(msg.payload, allow_duplicate_key: true) }
       end
     when 'ltsv'
       require 'ltsv'
@@ -160,8 +180,17 @@ class Fluent::Plugin::RdKafkaGroupInput < Fluent::Plugin::Input
     super
   end
 
+  def build_config
+    config = rdkafka_security_config
+    @kafka_configs.each { |k, v|
+      config[k.to_sym] = v
+    }
+    validate_sasl_over_ssl(config)
+    config
+  end
+
   def setup_consumer
-    consumer = Rdkafka::Config.new(@kafka_configs).consumer
+    consumer = Rdkafka::Config.new(@rdkafka_config).consumer
     consumer.subscribe(*@topics)
     consumer
   end

data/lib/fluent/plugin/kafka_plugin_util.rb

@@ -1,5 +1,24 @@
 module Fluent
   module KafkaPluginUtil
+    module PartitionSettings
+      # -1 is the unassigned partition; the Kafka protocol takes the index as int32
+      PARTITION_RANGE = -1..(2**31 - 1)
+
+      def coerce_partition(partition)
+        partition = case partition
+                    when Integer
+                      partition
+                    when String
+                      Integer(partition, 10)
+                    else
+                      raise TypeError, "partition must be an Integer or a decimal String, got #{partition.class}"
+                    end
+        raise RangeError, "partition #{partition} is out of range" unless PARTITION_RANGE.cover?(partition)
+
+        partition
+      end
+    end
+
     module AwsIamSettings
       def self.included(klass)
         klass.instance_eval do
@@ -36,6 +55,20 @@ module Fluent
 
       DummyFormatter = Object.new
 
+      def configure(conf)
+        super
+
+        if @ssl_client_cert && !@ssl_client_cert_key
+          raise Fluent::ConfigError, "ssl_client_cert is set but ssl_client_cert_key is not. Please provide both."
+        elsif !@ssl_client_cert && @ssl_client_cert_key
+          raise Fluent::ConfigError, "ssl_client_cert_key is set but ssl_client_cert is not. Please provide both."
+        elsif @ssl_client_cert_chain && !@ssl_client_cert
+          raise Fluent::ConfigError, "ssl_client_cert_chain is set but ssl_client_cert is not. Please provide cert, key and chain."
+        elsif @ssl_client_cert_key_password && !@ssl_client_cert_key
+          raise Fluent::ConfigError, "ssl_client_cert_key_password is set but ssl_client_cert_key is not. Please provide both."
+        end
+      end
+
       def start
         super
 
@@ -75,10 +108,82 @@ module Fluent
                        :desc => "a username when using PLAIN/SCRAM SASL authentication"
           config_param :password, :string, :default => nil, secret: true,
                        :desc => "a password when using PLAIN/SCRAM SASL authentication"
-          config_param :scram_mechanism, :string, :default => nil,
+          config_param :scram_mechanism, :enum, :list => [:sha256, :sha512], :default => nil,
                        :desc => "if set, use SCRAM authentication with specified mechanism. When unset, default to PLAIN authentication"
         }
       end
+
+      def configure(conf)
+        super
+
+        @scram_mechanism = @scram_mechanism.to_s if @scram_mechanism
+      end
+    end
+
+    module RdkafkaSecuritySettings
+      SCRAM_MECHANISMS = {
+        "sha256" => "SCRAM-SHA-256",
+        "sha512" => "SCRAM-SHA-512",
+      }.freeze
+
+      def self.included(klass)
+        klass.instance_eval {
+          config_param :service_name, :string, :default => nil, :desc => 'Used for sasl.kerberos.service.name'
+          config_param :sasl_over_ssl, :bool, :default => true,
+                       :desc => 'When true, SASL authentication requires an SSL connection'
+        }
+      end
+
+      def rdkafka_security_config
+        config = {}
+
+        if (@ssl_ca_cert && @ssl_ca_cert[0]) || @ssl_ca_certs_from_system || @ssl_client_cert || @ssl_client_cert_key
+          ssl = true
+          config[:"ssl.ca.location"] = @ssl_ca_cert[0] if @ssl_ca_cert && @ssl_ca_cert[0]
+          config[:"ssl.certificate.location"] = @ssl_client_cert if @ssl_client_cert
+          config[:"ssl.key.location"] = @ssl_client_cert_key if @ssl_client_cert_key
+          config[:"ssl.key.password"] = @ssl_client_cert_key_password if @ssl_client_cert_key_password
+          config[:"ssl.endpoint.identification.algorithm"] = @ssl_verify_hostname ? "https" : "none"
+          config[:"enable.ssl.certificate.verification"] = true
+        end
+
+        if @principal
+          sasl = true
+          config[:"sasl.mechanisms"] = "GSSAPI"
+          config[:"sasl.kerberos.principal"] = @principal
+          config[:"sasl.kerberos.service.name"] = @service_name if @service_name
+          config[:"sasl.kerberos.keytab"] = @keytab if @keytab
+        end
+
+        if @username && @password
+          sasl = true
+          config[:"sasl.mechanisms"] = SCRAM_MECHANISMS.fetch(@scram_mechanism, 'PLAIN')
+        elsif @scram_mechanism
+          log.warn "scram_mechanism is ignored because username and password are not set"
+        end
+
+        if ssl && sasl
+          security_protocol = "SASL_SSL"
+        elsif ssl && !sasl
+          security_protocol = "SSL"
+        elsif !ssl && sasl
+          security_protocol = "SASL_PLAINTEXT"
+        else
+          security_protocol = "PLAINTEXT"
+        end
+        config[:"security.protocol"] = security_protocol
+
+        config[:"sasl.username"] = @username if @username
+        config[:"sasl.password"] = @password if @password
+
+        config
+      end
+
+      def validate_sasl_over_ssl(config)
+        if @sasl_over_ssl && config[:"sasl.password"] && config[:"security.protocol"].to_s.upcase == "SASL_PLAINTEXT"
+          raise Fluent::ConfigError, "SASL authentication requires that SSL is configured. Set 'sasl_over_ssl false' to send SASL credentials over a plaintext connection"
+        end
+      end
     end
   end
 end

data/lib/fluent/plugin/out_kafka.rb

@@ -96,7 +96,7 @@ DESC
       @seed_brokers = []
       z = Zookeeper.new(@zookeeper)
       z.get_children(:path => @zookeeper_path)[:children].each do |id|
-        broker = Yajl.load(z.get(:path => @zookeeper_path + "/#{id}")[:data])
+        broker = JSON.parse(z.get(:path => @zookeeper_path + "/#{id}")[:data], allow_duplicate_key: true)
         if @ssl_client_cert
           @seed_brokers.push(pickup_ssl_endpoint(broker))
         else
@@ -192,8 +192,8 @@ DESC
 
   def setup_formatter(conf)
     if @output_data_type == 'json'
-      require 'yajl'
-      Proc.new { |tag, time, record| Yajl::Encoder.encode(record) }
+      require 'json'
+      Proc.new { |tag, time, record| JSON.generate(record) }
     elsif @output_data_type == 'ltsv'
       require 'ltsv'
       Proc.new { |tag, time, record| LTSV.dump(record) }

data/lib/fluent/plugin/out_kafka2.rb

@@ -98,6 +98,7 @@ DESC
     include Fluent::KafkaPluginUtil::AwsIamSettings
     include Fluent::KafkaPluginUtil::SSLSettings
     include Fluent::KafkaPluginUtil::SaslSettings
+    include Fluent::KafkaPluginUtil::PartitionSettings
 
     def initialize
       super
@@ -314,8 +315,8 @@ DESC
           Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
           Proc.new { |tag, time, record| Oj.dump(record) }
         rescue LoadError
-          require 'yajl'
-          Proc.new { |tag, time, record| Yajl::Encoder.encode(record) }
+          require 'json'
+          Proc.new { |tag, time, record| JSON.generate(record) }
         end
       when 'ltsv'
         require 'ltsv'
@@ -351,6 +352,7 @@ DESC
             partition_key = (@exclude_partition_key ? record.delete(@partition_key_key) : record[@partition_key_key]) || @default_partition_key
             partition = (@exclude_partition ? record.delete(@partition_key) : record[@partition_key]) || @default_partition
             message_key = (@exclude_message_key ? record.delete(@message_key_key) : record[@message_key_key]) || @default_message_key
+            partition = coerce_partition(partition) unless partition.nil?
 
             if mutate_headers
               headers = base_headers.clone

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -119,7 +119,7 @@ DESC
       @seed_brokers = []
       z = Zookeeper.new(@zookeeper)
       z.get_children(:path => @zookeeper_path)[:children].each do |id|
-        broker = Yajl.load(z.get(:path => @zookeeper_path + "/#{id}")[:data])
+        broker = JSON.parse(z.get(:path => @zookeeper_path + "/#{id}")[:data], allow_duplicate_key: true)
         if @ssl_client_cert
           @seed_brokers.push(pickup_ssl_endpoint(broker))
         else
@@ -259,8 +259,8 @@ DESC
         Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
         Proc.new { |tag, time, record| Oj.dump(record) }
       rescue LoadError
-        require 'yajl'
-        Proc.new { |tag, time, record| Yajl::Encoder.encode(record) }
+        require 'json'
+        Proc.new { |tag, time, record| JSON.generate(record) }
       end
     elsif @output_data_type == 'ltsv'
       require 'ltsv'