fluent-plugin-kafka 0.19.7 → 0.19.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2a709cad3615bca0c45f0606644142278acbb75b8e43f766dd4a1821a624aa75
-  data.tar.gz: 269bfdaa2ef8fb7b6cbb254971cc728769aba7215a472a73b34af2c04d735f6f
+  metadata.gz: 2eeb3b37a48db8cde6fda0eb9e413f507076c231abd06456a35540a10eed5667
+  data.tar.gz: 90b95493d3445829123ee569af66f581b109dcea7f91cf3d9aee6b2b6b9caeab
 SHA512:
-  metadata.gz: b15df023bd526fb22752da54b460ed12ce9a2d3c35e3ad9959c86d1ba8ac06fb9915799da95135fec3c6911ae776375ee8bc13f4803cad6663e2b60351b302c1
-  data.tar.gz: f9bfe163282d53f4f583ac905bffd50cab6eeaaaf5da2b9bf7511c7dad064c3aff458252af784ff8a52a3b6b5b63c80d47fccc80677692f58dc8a9fed814e6f4
+  metadata.gz: 3abf77244f8a7b239e92f63c798f0dc70b0d078a3b75de8f17ddc9e7cb17a8108088cc4f4bc3e44005496c68c89e960ca727def67cb7418f8e001bb713c938b4
+  data.tar.gz: 892d0c09e810b9fe78f87917a8468ac392b206991e99c4080f755e80953db79d8637e41717ef3ae0acbf13baa26e2fd1fe032bef7650ca70bc62df44b1dead7a

data/.github/dependabot.yml

@@ -4,3 +4,20 @@ updates:
     directory: '/'
     schedule:
       interval: 'monthly'
+    groups:
+      # PR: "Security update [package] from [old] to [new]"
+      # This PR should be merged in hurry
+      security-updates:
+        applies-to: security-updates
+        patterns:
+          - '*'
+
+      # PR: "Bump [package] from [old] to [new]"
+      # No need to be merged this PR in hurry. It is enough to merge
+      # once in a month.
+      monthly-updates:
+        applies-to: version-updates
+        patterns:
+          - '*'
+    # Allow to create PR both of security and normal updates.
+    open-pull-requests-limit: 1

data/.github/workflows/linux.yml

@@ -11,7 +11,7 @@ jobs:
     uses: ruby/actions/.github/workflows/ruby_versions.yml@master
     with:
       engine: cruby
-      min_version: 3.0
+      min_version: 3.2
   build:
     needs: ruby-versions
     runs-on: ${{ matrix.os }}
@@ -29,25 +29,12 @@ jobs:
           - { min: '>= 0.14.0', max: '< 0.16.0' }
           - { min: '>= 0.16.0', max: '< 0.25.0' }
           - { min: '>= 0.25.0', max: '>= 0.25.0' }
-        # rdkafka 0.25.0 supports Ruby 3.2+, so we exclude it for Ruby 3.0 and 3.1
         exclude:
           - ruby: head
-          - ruby: '3.1'
-            rdkafka_versions: { min: '>= 0.25.0', max: '>= 0.25.0' }
-          - ruby: '3.0'
-            rdkafka_versions: { min: '>= 0.25.0', max: '>= 0.25.0' }
-        # rdkafka 0.15.2 is the last version which supports Ruby 2.7
-        include:
-          - ruby: '2.7'
-            os: ubuntu-latest
-            rdkafka_versions: { min: '>= 0.6.0', max: '< 0.12.0' }
-          - ruby: '2.7'
-            os: ubuntu-latest
-            rdkafka_versions: { min: '>= 0.12.0', max: '= 0.15.2' }
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }} with rdkafka gem version (min ${{ matrix.rdkafka_versions.min }} max ${{ matrix.rdkafka_versions.max }})
     steps:
-    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
-    - uses: ruby/setup-ruby@afeafc3d1ab54a631816aba4c914a0081c12ff2f # v1.310.0
+    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+    - uses: ruby/setup-ruby@14594264cd68ce8a2345dd349bc3d138a4ef85c8 # v1.327.0
       with:
         ruby-version: ${{ matrix.ruby }}
     - name: Install confluent-kafka

data/.github/workflows/stale-actions.yml

@@ -10,7 +10,7 @@ jobs:
       issues: write
       pull-requests: write
     steps:
-    - uses: actions/stale@eb5cf3af3ac0a1aa4c9c45633dd1ae542a27a899 # v10.3.0
+    - uses: actions/stale@4391f3da665fdf50b6810c1a66712fb9ba21aa93 # v11.0.0
       with:
         repo-token: ${{ secrets.GITHUB_TOKEN }}
         days-before-stale: 90

data/ChangeLog

@@ -1,3 +1,78 @@
+Release 0.19.9 - 2026/10/30
+	* out_kafka2: Fix duplicated records and failing writes with
+				`share_producer true`. All flush threads used one ruby-kafka producer
+				at the same time, although it is not thread safe. NOTE: only one flush
+				thread at a time now uses the shared producer, so chunks are no longer
+				sent in parallel even if `flush_thread_count` is 2 or more. Keep
+				`share_producer false` to send them in parallel. (#589)
+	* out_kafka2: Fix later writes failing until restart after a delivery
+				error. Since 0.18.0, the messages of a failed delivery stayed in the
+				producer, so later writes could keep failing with
+				`Kafka::MessageSizeTooLarge`. The producer is now discarded after a
+				failure and created again on the next write. (#588, #589)
+	* in_kafka: Fix endless re-fetching of a batch with a record larger than
+				`chunk_limit_size`. The offset did not advance, and the records that
+				did fit were emitted again on every `interval`. NOTE: such a record is
+				now dropped with a warning. Use `<label @ERROR>` to keep it. (#584)
+	* in_kafka_group: Fix endless consumer restarts on a record larger than
+				`chunk_limit_size`. The consumer restarted without committing offsets,
+				so the same batch was fetched forever, and the records that did fit
+				were emitted again each time. NOTE: such a record is now dropped with a
+				warning. Use `<label @ERROR>` to keep it. (#583)
+
+Release 0.19.8 - 2026/09/04
+	* in_kafka: Emit each record with its own tag when `tag_source record` is
+				set. The whole fetch batch used to be emitted with the tag of the
+				last parsed record, so `tag_source record` never routed anything.
+				A record whose tag field is not a non-empty String is now skipped
+				with a warning, and the offset advances even when every message of
+				a batch is skipped. (#581)
+	* in_rdkafka_group: Apply SSL and SASL parameters to the consumer.
+				`ssl_ca_cert`, `ssl_client_cert`, `username`, `password`,
+				`scram_mechanism` and `principal` were accepted and then discarded,
+				so a consumer configured for SSL connected in the clear. NOTE: these
+				consumers start negotiating TLS after this change, and passing SASL
+				credentials through `kafka_configs` with `security.protocol
+				SASL_PLAINTEXT` is now rejected at startup unless `sasl_over_ssl
+				false` is set. (#580)
+	* out_rdkafka2: Apply `unrecoverable_error_codes` to delivery report
+				errors. The check only ran for errors raised by `produce`, so even the
+				default `msg_size_too_large` kept the chunk retrying until
+				`retry_timeout`. NOTE: such a chunk is now passed to the secondary
+				output, or written to the backup directory. (#579)
+	* out_rdkafka2: Fix the rdkafka patch version dispatch. The 0_16_0 patch
+				had been unreachable since it was added, and rdkafka-ruby 0.12.1
+				through 0.13.x got no patch at all, where `producer.close(10)` raises
+				ArgumentError on shutdown. Add a patch for 0.13.x. (#578)
+	* out_kafka2, out_rdkafka2: Skip an event whose `partition` is not a
+				decimal integer within the int32 range, and coerce `message_key` with
+				`to_s`. Such an event used to raise outside the per-event guard and
+				Fluentd retried the same chunk forever, stalling every source feeding
+				the output. NOTE: `partition` is now read as base 10, so "010" means
+				10 rather than 8. (#576, #577)
+	* out_rdkafka2: Fix `scram_mechanism` being ignored. SCRAM was silently
+				downgraded to PLAIN. `scram_mechanism` is now an enum of the
+				mechanisms ruby-kafka accepts, so an unknown value is rejected at
+				startup instead of falling back. (#574, #575)
+	* out_rdkafka, out_rdkafka2: Fix `ssl_verify_hostname` being ignored, and
+				set the SSL flag for any of `ssl_ca_cert`, `ssl_ca_certs_from_system`,
+				`ssl_client_cert` and `ssl_client_cert_key`. A mutual TLS setup
+				without `ssl_ca_cert` used to connect in the clear. NOTE: these
+				configurations start negotiating TLS after this change. (#572, #573)
+	* Validate that `ssl_client_cert`, `ssl_client_cert_key`,
+				`ssl_client_cert_chain` and `ssl_client_cert_key_password` are set
+				together, in every plugin that accepts them. librdkafka used to accept
+				a certificate without a key and connect with no client identity.
+				NOTE: such a configuration is now rejected at startup. (#571)
+	* out_rdkafka2: Add `sasl_over_ssl` parameter and reject SASL credentials
+				over a plaintext connection. NOTE: a configuration that sends
+				`username` and `password` over `PLAINTEXT` or `SASL_PLAINTEXT` now
+				fails at startup. Set `sasl_over_ssl false` only if you deliberately
+				keep sending them in the clear. (#570)
+	* Replace yajl-ruby with the json gem for JSON handling, and keep
+				accepting duplicate keys so that json 3.0 does not raise
+				`JSON::ParserError`. (#566, #568)
+
 Release 0.19.7 - 2026/06/02
     * in_rdkafka_group: support regexp pattern in topics (#541)
 

data/README.md

@@ -166,6 +166,12 @@ With the introduction of the rdkafka-ruby based input plugin we hope to support
       time_source <source for message timestamp (now|kafka|record)> :default => now
       time_format <string (Optional when use_record_time is used)>
 
+      # The SSL and SASL parameters in "Common parameters" are mapped to librdkafka
+      # properties. Keys given in kafka_configs take precedence over the mapped values.
+      # ssl_ca_cert uses only its first entry, and ssl_client_cert_chain is not mapped.
+      # When true, SASL authentication requires an SSL connection
+      sasl_over_ssl (bool) :default => true
+
       # kafka consumer options
       max_wait_time_ms 500
       max_batch_size 10000
@@ -257,6 +263,8 @@ The `<formatter name>` in `<format>` uses fluentd's formatter plugins. See [form
 
 **Note:** Java based Kafka client uses `murmur2` as partitioner function by default. If you want to use same partitioning behavior with fluent-plugin-kafka, change it to `murmur2` instead of `crc32`. Note that for using `murmur2` hash partitioner function, you must install `digest-murmurhash` gem.
 
+**Note:** `share_producer true` makes all flush threads use one ruby-kafka producer. This reduces the connections to the brokers. However, the ruby-kafka producer is not thread safe, so the flush threads send chunks one at a time. To send chunks in parallel with `flush_thread_count`, keep `share_producer` false. `rdkafka2` can share its producer without this limit, because librdkafka is thread safe.
+
 ruby-kafka sometimes returns `Kafka::DeliveryFailed` error without good information.
 In this case, `get_kafka_client_log` is useful for identifying the error cause.
 ruby-kafka's log is routed to fluentd log so you can see ruby-kafka's log in fluentd logs.
@@ -267,7 +275,7 @@ Supports following ruby-kafka's producer options.
 - required_acks - default: -1 - The number of acks required per request. If you need flush performance, set lower value, e.g. 1, 2.
 - ack_timeout - default: nil - How long the producer waits for acks. The unit is seconds.
 - compression_codec - default: nil - The codec the producer uses to compress messages.
-- max_send_limit_bytes - default: nil - Max byte size to send message to avoid MessageSizeTooLarge. For example, if you set 1000000(message.max.bytes in kafka), Message more than 1000000 byes will be dropped.
+- max_send_limit_bytes - default: nil - Max byte size of one record after formatting. A larger record is dropped with a warning, and the other records in the chunk are sent. For example, if you set 1000000, a record larger than 1000000 bytes is dropped. This does not limit the size of a record batch, so it does not prevent MessageSizeTooLarge by itself. See [FAQ](#why-does-fluentd-fail-with-kafkamessagesizetoolarge-or-msg_size_too_large).
 - discard_kafka_delivery_failed - default: false - discard the record where [Kafka::DeliveryFailed](http://www.rubydoc.info/gems/ruby-kafka/Kafka/DeliveryFailed) occurred
 
 If you want to know about detail of monitoring, see also https://github.com/zendesk/ruby-kafka#monitoring
@@ -445,7 +453,7 @@ Support of fluentd v0.12 has ended. `kafka_buffered` will be an alias of `kafka2
 - required_acks - default: -1 - The number of acks required per request. If you need flush performance, set lower value, e.g. 1, 2.
 - ack_timeout - default: nil - How long the producer waits for acks. The unit is seconds.
 - compression_codec - default: nil - The codec the producer uses to compress messages.
-- max_send_limit_bytes - default: nil - Max byte size to send message to avoid MessageSizeTooLarge. For example, if you set 1000000(message.max.bytes in kafka), Message more than 1000000 byes will be dropped.
+- max_send_limit_bytes - default: nil - Max byte size of one record after formatting. A larger record is dropped with a warning, and the other records in the chunk are sent. For example, if you set 1000000, a record larger than 1000000 bytes is dropped. This does not limit the size of a record batch, so it does not prevent MessageSizeTooLarge by itself. See [FAQ](#why-does-fluentd-fail-with-kafkamessagesizetoolarge-or-msg_size_too_large).
 - discard_kafka_delivery_failed - default: false - discard the record where [Kafka::DeliveryFailed](http://www.rubydoc.info/gems/ruby-kafka/Kafka/DeliveryFailed) occurred
 - monitoring_list - default: [] - library to be used to monitor. statsd and datadog are supported
 
@@ -523,6 +531,8 @@ You need to install rdkafka gem.
       discard_kafka_delivery_failed (bool) :default => false (No discard)
       discard_kafka_delivery_failed_regex (regexp) :default => nil (No discard)
       use_event_time        (bool) :default => false
+      # When true, SASL authentication requires an SSL connection
+      sasl_over_ssl         (bool) :default => true
 
       # same with kafka2
       headers               (hash) :default => {}
@@ -556,7 +566,8 @@ You need to install rdkafka gem.
       share_producer (bool) :default => false
       # Timeout for polling message wait. If 0, no wait.
       rdkafka_delivery_handle_poll_timeout (integer) :default => 30
-      # If the record size is larger than this value, such records are ignored. Default is no limit
+      # Max byte size of one record after formatting. Larger records are dropped.
+      # This does not limit the size of a record batch. Default is no limit
       max_send_limit_bytes (integer) :default => nil
       # The maximum number of enqueueing bytes per second. It can reduce the
       # load of both Fluentd and Kafka when excessive messages are attempted
@@ -569,6 +580,11 @@ You need to install rdkafka gem.
 `rdkafka2` supports `discard_kafka_delivery_failed_regex` parameter:
 - `discard_kafka_delivery_failed_regex` - default: nil - discard the record where the Kafka::DeliveryFailed occurred and the emitted message matches the given regex pattern, such as `/unknown_topic/`. 
 
+`rdkafka2` supports `unrecoverable_error_codes` parameter:
+- `unrecoverable_error_codes` - default: `["topic_authorization_failed", "msg_size_too_large"]` - librdkafka error codes to treat as unrecoverable errors. When a listed code is returned, `Fluent::UnrecoverableError` is raised, so Fluentd hands the whole chunk to the secondary output or to the backup directory instead of retrying it until `retry_timeout` expires. Both the produce call and the delivery report are covered, the latter only while `rdkafka_delivery_handle_poll_timeout` is not 0. Set the parameter to an empty value keeping retrying every error, as this plugin did before v0.19.x.
+- A code is spelled the way `Rdkafka::RdkafkaError#code` reports it, that is the librdkafka name lowercased with the leading underscore removed, so `RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION` is written as `unknown_partition`. A name that matches no error code is accepted and never matches anything.
+- `discard_kafka_delivery_failed` and `discard_kafka_delivery_failed_regex` take precedence over this parameter. The regex is matched against the librdkafka message, such as `Broker: Message size too large (msg_size_too_large)`.
+
 If you use v0.12, use `rdkafka` instead.
 
     <match kafka.**>
@@ -595,6 +611,40 @@ To avoid the problem, there are 2 approaches:
 - Upgrade your kafka cluster to latest version. This is better because recent version is faster and robust.
 - Downgrade ruby-kafka/fluent-plugin-kafka to work with your older kafka.
 
+### Why does Fluentd fail with Kafka::MessageSizeTooLarge (or msg_size_too_large)?
+
+Kafka rejects a record batch that is larger than the size limit of the topic. The limit is `max.message.bytes` of the topic if it is set, and `message.max.bytes` of the broker otherwise. The topic value can be larger or smaller than the broker value. Below, "the topic limit" means this value. If one output writes to several topics, use the smallest one.
+
+`kafka2` sends all records of a buffer chunk at once, and the records for the same partition go into one record batch. In the worst case, the whole chunk becomes one record batch. To make the batches smaller, lower `chunk_limit_size` in `<buffer>`.
+
+`chunk_limit_size` is the size of a chunk in the buffer, where records are kept as msgpack. The records sent to Kafka can be larger, because of `<format>`, `<inject>` and the overhead of each Kafka record. Start with about half of the topic limit, and lower it if the error continues. If `<buffer>` has `compress gzip` or `compress zstd`, `chunk_limit_size` limits the compressed size, and a chunk can hold several times more data. In that case, divide it further by the compression ratio of your data. `compression_codec` works the other way. Kafka checks the size of a record batch after compression, so compressed batches leave more room. How much depends on how well your data compresses.
+
+`max_send_limit_bytes` does not make the batch smaller. It checks only the formatted value of each record, and drops the records larger than this setting. The message key, headers and the overhead of the batch are not counted. To drop a single record that Kafka would reject, set it below the topic limit with some room for them.
+
+The following example assumes a topic limit of 1MB.
+
+    <match app.**>
+      @type kafka2
+      brokers <broker1_host>:<broker1_port>
+      default_topic app
+
+      max_send_limit_bytes 1000000
+
+      <format>
+        @type json
+      </format>
+
+      <buffer topic>
+        chunk_limit_size 512k
+      </buffer>
+    </match>
+
+Fluentd may log `chunk bytes limit exceeds for an emitted event stream`. This warning comes from the Fluentd buffer. It means that one event stream emitted to the output was larger than `chunk_limit_size`, and Fluentd splits it into several chunks. The number in the warning is the size of that event stream in the buffer, not the growth by formatting. If a single record is larger than `chunk_limit_size`, that record cannot be stored, and Fluentd raises `BufferChunkOverflowError`.
+
+`kafka_buffered` splits a chunk into smaller deliveries by `kafka_agg_max_bytes` (default: 4096) and `kafka_agg_max_messages`, so `chunk_limit_size` does not decide its batch size.
+
+`rdkafka2` passes records to librdkafka one by one, and librdkafka builds the record batches within its own `batch.size` and `message.max.bytes`. You can set them in `rdkafka_options`. Keep them below the topic limit. `msg_size_too_large` is in the default of `unrecoverable_error_codes`, so Fluentd does not retry a chunk that hits it.
+
 ## Contributing
 
 1. Fork it

data/fluent-plugin-kafka.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.19.7'
+  gem.version       = '0.19.9'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/in_kafka.rb

@@ -62,6 +62,12 @@ class Fluent::KafkaInput < Fluent::Input
   include Fluent::KafkaPluginUtil::SSLSettings
   include Fluent::KafkaPluginUtil::SaslSettings
 
+  BufferChunkOverflowError = if defined?(Fluent::Plugin::Buffer::BufferChunkOverflowError)
+                               Fluent::Plugin::Buffer::BufferChunkOverflowError
+                             else
+                               Fluent::BufferChunkLimitError
+                             end
+
   unless method_defined?(:router)
     define_method("router") { Fluent::Engine }
   end
@@ -143,9 +149,9 @@ class Fluent::KafkaInput < Fluent::Input
           r
         }
       rescue LoadError
-        require 'yajl'
+        require 'json'
         Proc.new { |msg, te|
-          r = Yajl::Parser.parse(msg.value)
+          r = JSON.parse(msg.value, allow_duplicate_key: true)
           add_offset_in_hash(r, te, msg.offset) if @add_offset_in_record
           r
         }
@@ -302,18 +308,24 @@ class Fluent::KafkaInput < Fluent::Input
 
       return if messages.size.zero?
 
-      es = Fluent::MultiEventStream.new
-      tag = @topic_entry.topic
-      tag = @add_prefix + "." + tag if @add_prefix
-      tag = tag + "." + @add_suffix if @add_suffix
+      event_streams = {}
+      topic_tag = @topic_entry.topic
+      topic_tag = @add_prefix + "." + topic_tag if @add_prefix
+      topic_tag = topic_tag + "." + @add_suffix if @add_suffix
 
       messages.each { |msg|
         begin
           record = @parser.call(msg, @topic_entry)
           if @tag_source == :record
             tag = record[@record_tag_key]
+            unless tag.is_a?(String) && !tag.empty?
+              $log.warn "skipped record with invalid tag in #{@topic_entry.topic}/#{@topic_entry.partition}", :key => @record_tag_key, :value => tag, :offset => msg.offset
+              next
+            end
             tag = @add_prefix + "." + tag if @add_prefix
             tag = tag + "." + @add_suffix if @add_suffix
+          else
+            tag = topic_tag
           end
           case @time_source
           when :kafka
@@ -332,7 +344,8 @@ class Fluent::KafkaInput < Fluent::Input
           if @kafka_message_key
             record[@kafka_message_key] = msg.key
           end
-          es.add(record_time, record)
+          event_streams[tag] ||= Fluent::MultiEventStream.new
+          event_streams[tag].add(record_time, record)
         rescue => e
           $log.warn "parser error in #{@topic_entry.topic}/#{@topic_entry.partition}", :error => e.to_s, :value => msg.value, :offset => msg.offset
           $log.debug_backtrace
@@ -340,14 +353,18 @@ class Fluent::KafkaInput < Fluent::Input
       }
       offset = messages.last.offset + 1
 
-      unless es.empty?
-        @router.emit_stream(tag, es)
-
-        if @offset_manager
-          @offset_manager.save_offset(offset)
+      event_streams.each { |tag, es|
+        begin
+          @router.emit_stream(tag, es)
+        rescue BufferChunkOverflowError => e
+          $log.warn "Skipped records larger than the buffer chunk limit size in #{@topic_entry.topic}/#{@topic_entry.partition}", :tag => tag, :error => e.to_s
         end
-        @next_offset = offset
+      }
+
+      if @offset_manager
+        @offset_manager.save_offset(offset)
       end
+      @next_offset = offset
     end
   end
 

data/lib/fluent/plugin/in_kafka_group.rb

@@ -85,6 +85,12 @@ class Fluent::KafkaGroupInput < Fluent::Input
                   Fluent::BufferQueueLimitError
                 end
 
+  BufferChunkOverflowError = if defined?(Fluent::Plugin::Buffer::BufferChunkOverflowError)
+                               Fluent::Plugin::Buffer::BufferChunkOverflowError
+                             else
+                               Fluent::BufferChunkLimitError
+                             end
+
   unless method_defined?(:router)
     define_method("router") { Fluent::Engine }
   end
@@ -160,8 +166,8 @@ class Fluent::KafkaGroupInput < Fluent::Input
         Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
         Proc.new { |msg| Oj.load(msg.value) }
       rescue LoadError
-        require 'yajl'
-        Proc.new { |msg| Yajl::Parser.parse(msg.value) }
+        require 'json'
+        Proc.new { |msg| JSON.parse(msg.value, allow_duplicate_key: true) }
       end
     when 'ltsv'
       require 'ltsv'
@@ -265,7 +271,7 @@ class Fluent::KafkaGroupInput < Fluent::Input
   end
 
   def process_batch_with_record_tag(batch)
-    es = {} 
+    es = {}
     batch.messages.each { |msg|
       begin
         record = @parser_proc.call(msg)
@@ -361,7 +367,7 @@ class Fluent::KafkaGroupInput < Fluent::Input
           if @tag_source == :record
             process_batch_with_record_tag(batch)
           else
-            process_batch(batch) 
+            process_batch(batch)
           end
         }
       rescue ForShutdown
@@ -380,6 +386,8 @@ class Fluent::KafkaGroupInput < Fluent::Input
     retries = 0
     begin
       router.emit_stream(tag, es)
+    rescue BufferChunkOverflowError => e
+      log.warn "Skipped records larger than the buffer chunk limit size", :tag => tag, :error => e.to_s
     rescue BufferError
       raise ForShutdown if @consumer.nil?
 

data/lib/fluent/plugin/in_rdkafka_group.rb

@@ -56,6 +56,7 @@ class Fluent::Plugin::RdKafkaGroupInput < Fluent::Plugin::Input
 
   include Fluent::KafkaPluginUtil::SSLSettings
   include Fluent::KafkaPluginUtil::SaslSettings
+  include Fluent::KafkaPluginUtil::RdkafkaSecuritySettings
 
   class ForShutdown < StandardError
   end
@@ -126,6 +127,8 @@ class Fluent::Plugin::RdKafkaGroupInput < Fluent::Plugin::Input
     if @time_source == :record and @time_format
       @time_parser = Fluent::TimeParser.new(@time_format)
     end
+
+    @rdkafka_config = build_config
   end
 
   def setup_parser(parser_conf)
@@ -137,8 +140,8 @@ class Fluent::Plugin::RdKafkaGroupInput < Fluent::Plugin::Input
         Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
         Proc.new { |msg| Oj.load(msg.payload) }
       rescue LoadError
-        require 'yajl'
-        Proc.new { |msg| Yajl::Parser.parse(msg.payload) }
+        require 'json'
+        Proc.new { |msg| JSON.parse(msg.payload, allow_duplicate_key: true) }
       end
     when 'ltsv'
       require 'ltsv'
@@ -177,8 +180,17 @@ class Fluent::Plugin::RdKafkaGroupInput < Fluent::Plugin::Input
     super
   end
 
+  def build_config
+    config = rdkafka_security_config
+    @kafka_configs.each { |k, v|
+      config[k.to_sym] = v
+    }
+    validate_sasl_over_ssl(config)
+    config
+  end
+
   def setup_consumer
-    consumer = Rdkafka::Config.new(@kafka_configs).consumer
+    consumer = Rdkafka::Config.new(@rdkafka_config).consumer
     consumer.subscribe(*@topics)
     consumer
   end

data/lib/fluent/plugin/kafka_plugin_util.rb

@@ -1,5 +1,24 @@
 module Fluent
   module KafkaPluginUtil
+    module PartitionSettings
+      # -1 is the unassigned partition; the Kafka protocol takes the index as int32
+      PARTITION_RANGE = -1..(2**31 - 1)
+
+      def coerce_partition(partition)
+        partition = case partition
+                    when Integer
+                      partition
+                    when String
+                      Integer(partition, 10)
+                    else
+                      raise TypeError, "partition must be an Integer or a decimal String, got #{partition.class}"
+                    end
+        raise RangeError, "partition #{partition} is out of range" unless PARTITION_RANGE.cover?(partition)
+
+        partition
+      end
+    end
+
     module AwsIamSettings
       def self.included(klass)
         klass.instance_eval do
@@ -36,6 +55,20 @@ module Fluent
 
       DummyFormatter = Object.new
 
+      def configure(conf)
+        super
+
+        if @ssl_client_cert && !@ssl_client_cert_key
+          raise Fluent::ConfigError, "ssl_client_cert is set but ssl_client_cert_key is not. Please provide both."
+        elsif !@ssl_client_cert && @ssl_client_cert_key
+          raise Fluent::ConfigError, "ssl_client_cert_key is set but ssl_client_cert is not. Please provide both."
+        elsif @ssl_client_cert_chain && !@ssl_client_cert
+          raise Fluent::ConfigError, "ssl_client_cert_chain is set but ssl_client_cert is not. Please provide cert, key and chain."
+        elsif @ssl_client_cert_key_password && !@ssl_client_cert_key
+          raise Fluent::ConfigError, "ssl_client_cert_key_password is set but ssl_client_cert_key is not. Please provide both."
+        end
+      end
+
       def start
         super
 
@@ -75,10 +108,82 @@ module Fluent
                        :desc => "a username when using PLAIN/SCRAM SASL authentication"
           config_param :password, :string, :default => nil, secret: true,
                        :desc => "a password when using PLAIN/SCRAM SASL authentication"
-          config_param :scram_mechanism, :string, :default => nil,
+          config_param :scram_mechanism, :enum, :list => [:sha256, :sha512], :default => nil,
                        :desc => "if set, use SCRAM authentication with specified mechanism. When unset, default to PLAIN authentication"
         }
       end
+
+      def configure(conf)
+        super
+
+        @scram_mechanism = @scram_mechanism.to_s if @scram_mechanism
+      end
+    end
+
+    module RdkafkaSecuritySettings
+      SCRAM_MECHANISMS = {
+        "sha256" => "SCRAM-SHA-256",
+        "sha512" => "SCRAM-SHA-512",
+      }.freeze
+
+      def self.included(klass)
+        klass.instance_eval {
+          config_param :service_name, :string, :default => nil, :desc => 'Used for sasl.kerberos.service.name'
+          config_param :sasl_over_ssl, :bool, :default => true,
+                       :desc => 'When true, SASL authentication requires an SSL connection'
+        }
+      end
+
+      def rdkafka_security_config
+        config = {}
+
+        if (@ssl_ca_cert && @ssl_ca_cert[0]) || @ssl_ca_certs_from_system || @ssl_client_cert || @ssl_client_cert_key
+          ssl = true
+          config[:"ssl.ca.location"] = @ssl_ca_cert[0] if @ssl_ca_cert && @ssl_ca_cert[0]
+          config[:"ssl.certificate.location"] = @ssl_client_cert if @ssl_client_cert
+          config[:"ssl.key.location"] = @ssl_client_cert_key if @ssl_client_cert_key
+          config[:"ssl.key.password"] = @ssl_client_cert_key_password if @ssl_client_cert_key_password
+          config[:"ssl.endpoint.identification.algorithm"] = @ssl_verify_hostname ? "https" : "none"
+          config[:"enable.ssl.certificate.verification"] = true
+        end
+
+        if @principal
+          sasl = true
+          config[:"sasl.mechanisms"] = "GSSAPI"
+          config[:"sasl.kerberos.principal"] = @principal
+          config[:"sasl.kerberos.service.name"] = @service_name if @service_name
+          config[:"sasl.kerberos.keytab"] = @keytab if @keytab
+        end
+
+        if @username && @password
+          sasl = true
+          config[:"sasl.mechanisms"] = SCRAM_MECHANISMS.fetch(@scram_mechanism, 'PLAIN')
+        elsif @scram_mechanism
+          log.warn "scram_mechanism is ignored because username and password are not set"
+        end
+
+        if ssl && sasl
+          security_protocol = "SASL_SSL"
+        elsif ssl && !sasl
+          security_protocol = "SSL"
+        elsif !ssl && sasl
+          security_protocol = "SASL_PLAINTEXT"
+        else
+          security_protocol = "PLAINTEXT"
+        end
+        config[:"security.protocol"] = security_protocol
+
+        config[:"sasl.username"] = @username if @username
+        config[:"sasl.password"] = @password if @password
+
+        config
+      end
+
+      def validate_sasl_over_ssl(config)
+        if @sasl_over_ssl && config[:"sasl.password"] && config[:"security.protocol"].to_s.upcase == "SASL_PLAINTEXT"
+          raise Fluent::ConfigError, "SASL authentication requires that SSL is configured. Set 'sasl_over_ssl false' to send SASL credentials over a plaintext connection"
+        end
+      end
     end
   end
 end

data/lib/fluent/plugin/out_kafka.rb

@@ -96,7 +96,7 @@ DESC
       @seed_brokers = []
       z = Zookeeper.new(@zookeeper)
       z.get_children(:path => @zookeeper_path)[:children].each do |id|
-        broker = Yajl.load(z.get(:path => @zookeeper_path + "/#{id}")[:data])
+        broker = JSON.parse(z.get(:path => @zookeeper_path + "/#{id}")[:data], allow_duplicate_key: true)
         if @ssl_client_cert
           @seed_brokers.push(pickup_ssl_endpoint(broker))
         else
@@ -192,8 +192,8 @@ DESC
 
   def setup_formatter(conf)
     if @output_data_type == 'json'
-      require 'yajl'
-      Proc.new { |tag, time, record| Yajl::Encoder.encode(record) }
+      require 'json'
+      Proc.new { |tag, time, record| JSON.generate(record) }
     elsif @output_data_type == 'ltsv'
       require 'ltsv'
       Proc.new { |tag, time, record| LTSV.dump(record) }