fluent-plugin-kafka 0.16.3 → 0.17.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1cc628766df718a6fff17debcf2cffdcc041419cb96c1c9b6ce1fee532807b58
-  data.tar.gz: 543ccbd91345f45ee75c3e5e84ef5b3414420a0e2557c76a40afc8e308d70735
+  metadata.gz: 380a588c3128af581568815d5fdec9edea856c702832de8be87a9e83a012a8b7
+  data.tar.gz: afd5d2e7ba2b89a6948637cbb325b3b0683e0c23ac37a591f717902a78a5ba1b
 SHA512:
-  metadata.gz: 7cb27a4fe28ccc0f31e36449046faf8d4d3389be7820f7cfce8f612aad84b3239234663a71b50523cb08a13e145f78fdc86e93cf9cec28d35d82765b96c9dddc
-  data.tar.gz: 98f52a5d84fb348178f9df6ede6592412ddccd43891c95e9d6937622bff8b02bb4cbb3231c7ca7d709a3ae6cdbfb90d3f00074eefe4cc0703379315098424809
+  metadata.gz: fe4f49213a66d2e58bb4177f888ab3c0f3cf0fefe3ca2fa5972bc6394ee46c34c7f054088efd035c10e3beb4d277a5368f0f17ff9aa81954d0340e0962e31902
+  data.tar.gz: 1c59d58e36828d61eb8d42b9c60578850dd8a439b21faadc0261d7f6b873a7d3874797923b341b9fc6fc6ffce59e1bff4ac90433e8b329580df2031b41d4005a

data/.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -0,0 +1,71 @@
+name: Bug Report
+description: Create a report with a procedure for reproducing the bug
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Check [README](https://github.com/fluent/fluent-plugin-kafka/#faq) first and here is the list to help us investigate the problem.
+  - type: textarea
+    id: description
+    attributes:
+      label: Describe the bug
+      description: A clear and concise description of what the bug is
+    validations:
+      required: true
+  - type: textarea
+    id: reproduce
+    attributes:
+      label: To Reproduce
+      description: Steps to reproduce the behavior
+    validations:
+      required: true
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected behavior
+      description: A clear and concise description of what you expected to happen
+    validations:
+      required: true
+  - type: textarea
+    id: environment
+    attributes:
+      label: Your Environment
+      description: |
+        - Fluentd or td-agent version: `fluentd --version` or `td-agent --version`
+        - Operating system: `cat /etc/os-release`
+        - Kernel version: `uname -r`
+
+        Tip: If you hit the problem with older fluentd version, try latest version first.
+      value: |
+        - Fluentd version:
+        - TD Agent version:
+        - fluent-plugin-kafka version:
+        - ruby-kafka version:
+        - Operating system:
+        - Kernel version:
+      render: markdown
+    validations:
+      required: true
+  - type: textarea
+    id: configuration
+    attributes:
+      label: Your Configuration
+      description: |
+        Write your configuration here. Minimum reproducible fluentd.conf is recommended.
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Your Error Log
+      description: Write your ALL error log here
+      render: shell
+    validations:
+      required: true
+  - type: textarea
+    id: addtional-context
+    attributes:
+      label: Additional context
+      description: Add any other context about the problem here.
+    validations:
+      required: false

data/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Ask a Question
+    url: https://discuss.fluentd.org/
+    about: I have questions about fluent-plugin-kafka. Please ask and answer questions at https://discuss.fluentd.org/.

data/.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -0,0 +1,38 @@
+name: Feature request
+description: Suggest an idea for this project
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Check [README.md](https://github.com/fluent/fluent-plugin-kafka/blob/master/README.md) first and here is the list to help us investigate the problem.
+  - type: textarea
+    id: description
+    attributes:
+      label: Is your feature request related to a problem? Please describe.
+      description: |
+        A clear and concise description of what the problem is.
+        Ex. I'm always frustrated when [...]
+    validations:
+      required: true
+  - type: textarea
+    id: solution
+    attributes:
+      label: Describe the solution you'd like
+      description: A clear and concise description of what you want to happen.
+    validations:
+      required: true
+  - type: textarea
+    id: alternative
+    attributes:
+      label: Describe alternatives you've considered
+      description: A clear and concise description of any alternative solutions or features you've considered.
+    validations:
+      required: true
+  - type: textarea
+    id: addtional-context
+    attributes:
+      label: Additional context
+      description: Add any other context or screenshots about the feature request here.
+    validations:
+      required: false
+

data/.github/workflows/linux.yml

@@ -5,6 +5,8 @@ on:
 jobs:
   build:
     runs-on: ${{ matrix.os }}
+    env:
+      USE_RDKAFKA: 1
     strategy:
       fail-fast: false
       matrix:

data/.github/workflows/stale-actions.yml

@@ -0,0 +1,22 @@
+name: "Mark or close stale issues and PRs"
+on:
+  schedule:
+  - cron: "00 10 * * *"
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/stale@v3
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        days-before-stale: 90
+        days-before-close: 30
+        stale-issue-message: "This issue has been automatically marked as stale because it has been open 90 days with no activity. Remove stale label or comment or this issue will be closed in 30 days"
+        stale-pr-message: "This PR has been automatically marked as stale because it has been open 90 days with no activity. Remove stale label or comment or this PR will be closed in 30 days"
+        close-issue-message: "This issue was automatically closed because of stale in 30 days"
+        close-pr-message: "This PR was automatically closed because of stale in 30 days"
+        stale-pr-label: "stale"
+        stale-issue-label: "stale"
+        exempt-issue-labels: "bug,enhancement,help wanted"
+        exempt-pr-labels: "bug,enhancement,help wanted"

data/ChangeLog

@@ -1,3 +1,21 @@
+Release 0.17.3 - 2021/11/26
+	* output: Suppress large warning logs for events skipped by `max_send_limit_bytes`
+
+Release 0.17.2 - 2021/10/14
+	* out_rdkafka2: Add `max_enqueue_bytes_per_second` parameter
+	* out_rdkafka2: Support `use_event_time` parameter
+	* out_rdkafka2: Fix a potential bug that the plugin might exit without receiving responses from Kafka.
+
+Release 0.17.1 - 2021/09/24
+	* out_rdkafka/out_rdkafka2: Support rdkafka 0.9.0 or later
+	* out_rdkafka/out_rdkafka2: Add `exclude_fields` parameter
+	* out_kafka2.rb: Fix one more Ruby 3.0 keyword arguments issue
+
+Release 0.17.0 - 2021/08/30
+	* out_kafka/out_kafka_buffered/out_kafka2: Provide murmur2 partitioner hash function choice
+	* in_kafka/in_kafka_group/out_kafka/out_kafka_buffered/out_kafka2: Use Ruby Kafka's ssl_ca_cert_file_path parameter to feed the CA certs
+	* out_kafka/out_kafka2: fix description for `exclude_message_key` option
+
 Release 0.16.3 - 2021/05/17
 	* in_kafka_group: Fix one more Ruby 3.0 keyword arguments issue
 

data/Gemfile

@@ -2,3 +2,5 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kafka.gemspec
 gemspec
+
+gem 'rdkafka', '>= 0.6.0' if ENV["USE_RDKAFKA"]

data/README.md

@@ -40,14 +40,14 @@ If you want to use zookeeper related parameters, you also need to install zookee
 
 Set path to SSL related files. See [Encryption and Authentication using SSL](https://github.com/zendesk/ruby-kafka#encryption-and-authentication-using-ssl) for more detail.
 
-#### SASL authentication 
+#### SASL authentication
 
 ##### with GSSAPI
 
 - principal
 - keytab
 
-Set principal and path to keytab for SASL/GSSAPI authentication. 
+Set principal and path to keytab for SASL/GSSAPI authentication.
 See [Authentication using SASL](https://github.com/zendesk/ruby-kafka#authentication-using-sasl) for more details.
 
 ##### with Plain/SCRAM
@@ -57,7 +57,7 @@ See [Authentication using SASL](https://github.com/zendesk/ruby-kafka#authentica
 - scram_mechanism
 - sasl_over_ssl
 
-Set username, password, scram_mechanism and sasl_over_ssl for SASL/Plain or Scram authentication. 
+Set username, password, scram_mechanism and sasl_over_ssl for SASL/Plain or Scram authentication.
 See [Authentication using SASL](https://github.com/zendesk/ruby-kafka#authentication-using-sasl) for more details.
 
 ### Input plugin (@type 'kafka')
@@ -119,7 +119,7 @@ Consume events by kafka consumer group features..
       topics <listening topics(separate with comma',')>
       format <input text type (text|json|ltsv|msgpack)> :default => json
       message_key <key (Optional, for text format only, default is message)>
-      kafka_mesasge_key <key (Optional, If specified, set kafka's message key to this key)>
+      kafka_message_key <key (Optional, If specified, set kafka's message key to this key)>
       add_headers <If true, add kafka's message headers to record>
       add_prefix <tag prefix (Optional)>
       add_suffix <tag suffix (Optional)>
@@ -155,7 +155,7 @@ With the introduction of the rdkafka-ruby based input plugin we hope to support
       topics <listening topics(separate with comma',')>
       format <input text type (text|json|ltsv|msgpack)> :default => json
       message_key <key (Optional, for text format only, default is message)>
-      kafka_mesasge_key <key (Optional, If specified, set kafka's message key to this key)>
+      kafka_message_key <key (Optional, If specified, set kafka's message key to this key)>
       add_headers <If true, add kafka's message headers to record>
       add_prefix <tag prefix (Optional)>
       add_suffix <tag suffix (Optional)>
@@ -200,8 +200,10 @@ If `ruby-kafka` doesn't fit your kafka environment, check `rdkafka2` plugin inst
       get_kafka_client_log  (bool)   :default => false
       headers               (hash)   :default => {}
       headers_from_record   (hash)   :default => {}
+      use_event_time        (bool)   :default => false
       use_default_for_unknown_topic (bool) :default => false
       discard_kafka_delivery_failed (bool) :default => false (No discard)
+      partitioner_hash_function (enum) (crc32|murmur2) :default => 'crc32'
 
       <format>
         @type (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
@@ -230,6 +232,8 @@ If `ruby-kafka` doesn't fit your kafka environment, check `rdkafka2` plugin inst
 
 The `<formatter name>` in `<format>` uses fluentd's formatter plugins. See [formatter article](https://docs.fluentd.org/v/1.0/formatter).
 
+**Note:** Java based Kafka client uses `murmur2` as partitioner function by default. If you want to use same partitioning behavior with fluent-plugin-kafka, change it to `murmur2` instead of `crc32`. Note that for using `murmur2` hash partitioner function, you must install `digest-murmurhash` gem.
+
 ruby-kafka sometimes returns `Kafka::DeliveryFailed` error without good information.
 In this case, `get_kafka_client_log` is useful for identifying the error cause.
 ruby-kafka's log is routed to fluentd log so you can see ruby-kafka's log in fluentd logs.
@@ -313,6 +317,23 @@ The Kafka message will have a header of source_ip=12.7.0.0.1.
 
 The configuration format is jsonpath. It is descibed in https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-record_accessor
 
+#### Excluding fields
+Fields can be excluded from output data. Only works for kafka2 and rdkafka2 output plugin.
+
+Fields must be specified using an array of dot notation `$.`, for example:
+
+    <match app.**>
+      @type kafka2
+      [...]
+      exclude_fields $.source.ip,$.HTTP_FOO
+    <match>
+
+This config can be used to remove fields used on another configs.
+
+For example, `$.source.ip` can be extracted with config `headers_from_record` and excluded from message payload.
+
+> Using this config to remove unused fields is discouraged. A [filter plugin](https://docs.fluentd.org/v/0.12/filter) can be used for this purpose.
+
 ### Buffered output plugin
 
 This plugin uses ruby-kafka producer for writing data. This plugin is for v0.12. If you use v1, see `kafka2`.
@@ -343,6 +364,8 @@ Support of fluentd v0.12 has ended. `kafka_buffered` will be an alias of `kafka2
       exclude_topic_key     (bool) :default => false
       exclude_partition_key (bool) :default => false
       get_kafka_client_log  (bool) :default => false
+      use_event_time        (bool) :default => false
+      partitioner_hash_function (enum) (crc32|murmur2) :default => 'crc32'
 
       # See fluentd document for buffer related parameters: https://docs.fluentd.org/v/0.12/buffer
 
@@ -365,6 +388,8 @@ Support of fluentd v0.12 has ended. `kafka_buffered` will be an alias of `kafka2
 - kafka_agg_max_bytes - default: 4096 - Maximum value of total message size to be included in one batch transmission.
 - kafka_agg_max_messages - default: nil - Maximum number of messages to include in one batch transmission.
 
+**Note:** Java based Kafka client uses `murmur2` as partitioner function by default. If you want to use same partitioning behavior with fluent-plugin-kafka, change it to `murmur2` instead of `crc32`. Note that for using `murmur2` hash partitioner function, you must install `digest-murmurhash` gem.
+
 ### Non-buffered output plugin
 
 This plugin uses ruby-kafka producer for writing data. For performance and reliability concerns, use `kafka_bufferd` output instead. This is mainly for testing.
@@ -385,6 +410,7 @@ This plugin uses ruby-kafka producer for writing data. For performance and relia
       output_include_time   (bool) :default => false
       exclude_topic_key     (bool) :default => false
       exclude_partition_key (bool) :default => false
+      partitioner_hash_function (enum) (crc32|murmur2) :default => 'crc32'
 
       # ruby-kafka producer options
       max_send_retries    (integer) :default => 1
@@ -397,6 +423,8 @@ This plugin uses ruby-kafka producer for writing data. For performance and relia
 
 This plugin also supports ruby-kafka related parameters. See Buffered output plugin section.
 
+**Note:** Java based Kafka client uses `murmur2` as partitioner function by default. If you want to use same partitioning behavior with fluent-plugin-kafka, change it to `murmur2` instead of `crc32`. Note that for using `murmur2` hash partitioner function, you must install `digest-murmurhash` gem.
+
 ### rdkafka based output plugin
 
 This plugin uses `rdkafka` instead of `ruby-kafka` for kafka client.
@@ -426,6 +454,7 @@ You need to install rdkafka gem.
       exclude_topic_key     (bool) :default => false
       exclude_partition_key (bool) :default => false
       discard_kafka_delivery_failed (bool) :default => false (No discard)
+      use_event_time        (bool) :default => false
 
       # same with kafka2
       headers               (hash) :default => {}
@@ -460,6 +489,10 @@ You need to install rdkafka gem.
       rdkafka_delivery_handle_poll_timeout (integer) :default => 30
       # If the record size is larger than this value, such records are ignored. Default is no limit
       max_send_limit_bytes (integer) :default => nil
+      # The maximum number of enqueueing bytes per second. It can reduce the
+      # load of both Fluentd and Kafka when excessive messages are attempted
+      # to send. Default is no limit.
+      max_enqueue_bytes_per_second (integer) :default => nil
     </match>
 
 If you use v0.12, use `rdkafka` instead.

data/fluent-plugin-kafka.gemspec

@@ -13,13 +13,15 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.16.3'
+  gem.version       = '0.17.3'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
   gem.add_dependency 'ltsv'
-  gem.add_dependency 'ruby-kafka', '>= 1.2.0', '< 2'
+  gem.add_dependency 'ruby-kafka', '>= 1.4.0', '< 2'
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "test-unit", ">= 3.0.8"
+  gem.add_development_dependency "test-unit-rr", "~> 1.0"
   gem.add_development_dependency "webrick"
+  gem.add_development_dependency "digest-murmurhash"
 end

data/lib/fluent/plugin/in_kafka.rb

@@ -71,6 +71,7 @@ class Fluent::KafkaInput < Fluent::Input
     require 'kafka'
 
     @time_parser = nil
+    @zookeeper = nil
   end
 
   def configure(conf)
@@ -197,17 +198,17 @@ class Fluent::KafkaInput < Fluent::Input
 
     logger = @get_kafka_client_log ? log : nil
     if @scram_mechanism != nil && @username != nil && @password != nil
-      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert_file_path: @ssl_ca_cert,
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_scram_username: @username, sasl_scram_password: @password,
                          sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
     elsif @username != nil && @password != nil
-      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert_file_path: @ssl_ca_cert,
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system,sasl_plain_username: @username, sasl_plain_password: @password,
                          sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
     else
-      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert_file_path: @ssl_ca_cert,
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab,
                          ssl_verify_hostname: @ssl_verify_hostname)

data/lib/fluent/plugin/in_kafka_group.rb

@@ -183,17 +183,17 @@ class Fluent::KafkaGroupInput < Fluent::Input
 
     logger = @get_kafka_client_log ? log : nil
     if @scram_mechanism != nil && @username != nil && @password != nil
-      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert_file_path: @ssl_ca_cert,
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_scram_username: @username, sasl_scram_password: @password,
                          sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
     elsif @username != nil && @password != nil
-      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert_file_path: @ssl_ca_cert,
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_plain_username: @username, sasl_plain_password: @password,
                          sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
     else
-      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert_file_path: @ssl_ca_cert,
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab,
                          ssl_verify_hostname: @ssl_verify_hostname)

data/lib/fluent/plugin/out_kafka.rb

@@ -19,6 +19,8 @@ DESC
   config_param :default_message_key, :string, :default => nil
   config_param :default_partition_key, :string, :default => nil
   config_param :default_partition, :integer, :default => nil
+  config_param :partitioner_hash_function, :enum, list: [:crc32, :murmur2], :default => :crc32,
+               :desc => "Specify kafka patrtitioner hash algorithm"
   config_param :client_id, :string, :default => 'kafka'
   config_param :sasl_over_ssl, :bool, :default => true,
                :desc => <<-DESC
@@ -86,6 +88,7 @@ DESC
     require 'kafka'
 
     @kafka = nil
+    @field_separator = nil
   end
 
   def refresh_client
@@ -106,18 +109,21 @@ DESC
     begin
       if @seed_brokers.length > 0
         if @scram_mechanism != nil && @username != nil && @password != nil
-          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert_file_path: @ssl_ca_cert,
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
                              sasl_scram_username: @username, sasl_scram_password: @password, sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl,
-                             ssl_verify_hostname: @ssl_verify_hostname)
+                             ssl_verify_hostname: @ssl_verify_hostname,
+                             partitioner: Kafka::Partitioner.new(hash_function: @partitioner_hash_function))
         elsif @username != nil && @password != nil
-          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert_file_path: @ssl_ca_cert,
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
-                             sasl_plain_username: @username, sasl_plain_password: @password, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
+                             sasl_plain_username: @username, sasl_plain_password: @password, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname,
+                             partitioner: Kafka::Partitioner.new(hash_function: @partitioner_hash_function))
         else
-          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert_file_path: @ssl_ca_cert,
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
-                             sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
+                             sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname,
+                             partitioner: Kafka::Partitioner.new(hash_function: @partitioner_hash_function))
         end
         log.info "initialized kafka producer: #{@client_id}"
       else
@@ -234,7 +240,8 @@ DESC
         record_buf = @formatter_proc.call(tag, time, record)
         record_buf_bytes = record_buf.bytesize
         if @max_send_limit_bytes && record_buf_bytes > @max_send_limit_bytes
-          log.warn "record size exceeds max_send_limit_bytes. Skip event:", :time => time, :record => record
+          log.warn "record size exceeds max_send_limit_bytes. Skip event:", :time => time, :record_size => record_buf_bytes
+          log.debug "Skipped event:", :record => record
           next
         end
         log.trace { "message will send to #{topic} with partition_key: #{partition_key}, partition: #{partition}, message_key: #{message_key} and value: #{record_buf}." }

data/lib/fluent/plugin/out_kafka2.rb

@@ -24,6 +24,8 @@ DESC
     config_param :partition_key_key, :string, :default => 'partition_key', :desc => "Field for kafka partition key"
     config_param :default_partition_key, :string, :default => nil
     config_param :partition_key, :string, :default => 'partition', :desc => "Field for kafka partition"
+    config_param :partitioner_hash_function, :enum, list: [:crc32, :murmur2], :default => :crc32,
+                 :desc => "Specify kafka patrtitioner hash algorithm"
     config_param :default_partition, :integer, :default => nil
     config_param :use_default_for_unknown_topic, :bool, :default => false, :desc => "If true, default_topic is used when topic not found"
     config_param :client_id, :string, :default => 'fluentd'
@@ -37,9 +39,11 @@ DESC
     config_param :exclude_partition, :bool, :default => false,
                  :desc => 'Set true to remove partition from data'
     config_param :exclude_message_key, :bool, :default => false,
-                 :desc => 'Set true to remove partition key from data'
+                 :desc => 'Set true to remove message key from data'
     config_param :exclude_topic_key, :bool, :default => false,
                  :desc => 'Set true to remove topic name key from data'
+    config_param :exclude_fields, :array, :default => [], value_type: :string,
+                 :desc => 'Fields to remove from data where the value is a jsonpath to a record value'
     config_param :use_event_time, :bool, :default => false, :desc => 'Use fluentd event time for kafka create_time'
     config_param :headers, :hash, default: {}, symbolize_keys: true, value_type: :string,
                  :desc => 'Kafka message headers'
@@ -96,20 +100,23 @@ DESC
       begin
         logger = @get_kafka_client_log ? log : nil
         if @scram_mechanism != nil && @username != nil && @password != nil
-          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert_file_path: @ssl_ca_cert,
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_client_cert_chain: read_ssl_file(@ssl_client_cert_chain),
                              ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_scram_username: @username, sasl_scram_password: @password,
-                             sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
+                             sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname,
+                             partitioner: Kafka::Partitioner.new(hash_function: @partitioner_hash_function))
         elsif @username != nil && @password != nil
-          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert_file_path: @ssl_ca_cert,
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_client_cert_chain: read_ssl_file(@ssl_client_cert_chain),
                              ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_plain_username: @username, sasl_plain_password: @password, sasl_over_ssl: @sasl_over_ssl,
-                             ssl_verify_hostname: @ssl_verify_hostname)
+                             ssl_verify_hostname: @ssl_verify_hostname,
+                             partitioner: Kafka::Partitioner.new(hash_function: @partitioner_hash_function))
         else
-          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert_file_path: @ssl_ca_cert,
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_client_cert_chain: read_ssl_file(@ssl_client_cert_chain),
                              ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab, sasl_over_ssl: @sasl_over_ssl,
-                             ssl_verify_hostname: @ssl_verify_hostname)
+                             ssl_verify_hostname: @ssl_verify_hostname,
+                             partitioner: Kafka::Partitioner.new(hash_function: @partitioner_hash_function))
         end
         log.info "initialized kafka producer: #{@client_id}"
       rescue Exception => e
@@ -172,6 +179,10 @@ DESC
       @headers_from_record.each do |key, value|
         @headers_from_record_accessors[key] = record_accessor_create(value)
       end
+
+      @exclude_field_accessors = @exclude_fields.map do |field|
+        record_accessor_create(field)
+      end
     end
 
     def multi_workers_ready?
@@ -230,7 +241,7 @@ DESC
       mutate_headers = !@headers_from_record_accessors.empty?
 
       begin
-        producer = @kafka.topic_producer(topic, @producer_opts)
+        producer = @kafka.topic_producer(topic, **@producer_opts)
         chunk.msgpack_each { |time, record|
           begin
             record = inject_values_to_record(tag, time, record)
@@ -248,10 +259,17 @@ DESC
               headers = base_headers
             end
 
+            unless @exclude_fields.empty?
+              @exclude_field_accessors.each do |exclude_field_accessor|
+                exclude_field_accessor.delete(record)
+              end
+            end
+
             record_buf = @formatter_proc.call(tag, time, record)
             record_buf_bytes = record_buf.bytesize
             if @max_send_limit_bytes && record_buf_bytes > @max_send_limit_bytes
-              log.warn "record size exceeds max_send_limit_bytes. Skip event:", :time => time, :record => record
+              log.warn "record size exceeds max_send_limit_bytes. Skip event:", :time => time, :record_size => record_buf_bytes
+              log.debug "Skipped event:", :record => record
               next
             end
           rescue StandardError => e

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -26,6 +26,8 @@ DESC
   config_param :default_partition_key, :string, :default => nil
   config_param :partition_key, :string, :default => 'partition', :desc => "Field for kafka partition"
   config_param :default_partition, :integer, :default => nil
+  config_param :partitioner_hash_function, :enum, list: [:crc32, :murmur2], :default => :crc32,
+               :desc => "Specify kafka patrtitioner hash algorithm"
   config_param :client_id, :string, :default => 'kafka'
   config_param :idempotent, :bool, :default => false, :desc => 'Enable idempotent producer'
   config_param :sasl_over_ssl, :bool, :default => true,
@@ -105,6 +107,7 @@ DESC
     @kafka = nil
     @producers = {}
     @producers_mutex = Mutex.new
+    @field_separator = nil
   end
 
   def multi_workers_ready?
@@ -130,18 +133,21 @@ DESC
       if @seed_brokers.length > 0
         logger = @get_kafka_client_log ? log : nil
         if @scram_mechanism != nil && @username != nil && @password != nil
-          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert_file_path: @ssl_ca_cert,
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
                              sasl_scram_username: @username, sasl_scram_password: @password, sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl,
-                             ssl_verify_hostname: @ssl_verify_hostname)
+                             ssl_verify_hostname: @ssl_verify_hostname,
+                             partitioner: Kafka::Partitioner.new(hash_function: @partitioner_hash_function))
         elsif @username != nil && @password != nil
-          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert_file_path: @ssl_ca_cert,
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
-                             sasl_plain_username: @username, sasl_plain_password: @password, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
+                             sasl_plain_username: @username, sasl_plain_password: @password, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname,
+                             partitioner: Kafka::Partitioner.new(hash_function: @partitioner_hash_function))
         else
-          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert_file_path: @ssl_ca_cert,
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
-                             sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab, sasl_over_ssl: @sasl_over_ssl,  ssl_verify_hostname: @ssl_verify_hostname)
+                             sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab, sasl_over_ssl: @sasl_over_ssl,  ssl_verify_hostname: @ssl_verify_hostname,
+                             partitioner: Kafka::Partitioner.new(hash_function: @partitioner_hash_function))
         end
         log.info "initialized kafka producer: #{@client_id}"
       else
@@ -326,7 +332,8 @@ DESC
           record_buf = @formatter_proc.call(tag, time, record)
           record_buf_bytes = record_buf.bytesize
           if @max_send_limit_bytes && record_buf_bytes > @max_send_limit_bytes
-            log.warn "record size exceeds max_send_limit_bytes. Skip event:", :time => time, :record => record
+            log.warn "record size exceeds max_send_limit_bytes. Skip event:", :time => time, :record_size => record_buf_bytes
+            log.debug "Skipped event:", :record => record
             next
           end
         rescue StandardError => e