fluent-plugin-kafka 0.16.2 → 0.17.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1f388f79dd4b9bf95418b71cd78a9e2f712fc01aa791dab8e0f5e22cd899c2b6
-  data.tar.gz: 00cee257a09cbfe3773d8e077b651668e9115d94f813b2fe575917789248a966
+  metadata.gz: 16968c0e56c22f64225e8e41e905294defc7240df6054813151d3904a79a4107
+  data.tar.gz: 71b13953b11048f201c8a8a275350e983a51377b45e84440196b701063e317c4
 SHA512:
-  metadata.gz: 5e23cb803cef76ac8446007b1a60e091868c99c62b839d040093232ae90456306e1eb6bef9630f4c8230e9662ac3a0039eca1728640bccd906f4ed099f5fed4a
-  data.tar.gz: bc2c9f55199a5227d4ce6d5ea56c91fc31ca03afc0d45b23966d81f3d3a9f1eeea356b2af161d851b5e800fe4492ca6d7df7c9d07c81942e1602768d64a21156
+  metadata.gz: aedae36f4b7a29408bc96838da3e158a98ddac136987408dcd4bc347068e3d27839e844d1d3d4a239e4f175e023f19d9ecd82e14d4ab089fd43625f60dcca17c
+  data.tar.gz: 4a72ea62a754b689944d3f2de81cc495c9ef92df97c127b80b46ace6824aca00df5b4cc499eb3993d0edbd3f156f140c6c8ba718fb82f65b5328170adf26f622

data/.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -0,0 +1,71 @@
+name: Bug Report
+description: Create a report with a procedure for reproducing the bug
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Check [README](https://github.com/fluent/fluent-plugin-kafka/#faq) first and here is the list to help us investigate the problem.
+  - type: textarea
+    id: description
+    attributes:
+      label: Describe the bug
+      description: A clear and concise description of what the bug is
+    validations:
+      required: true
+  - type: textarea
+    id: reproduce
+    attributes:
+      label: To Reproduce
+      description: Steps to reproduce the behavior
+    validations:
+      required: true
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected behavior
+      description: A clear and concise description of what you expected to happen
+    validations:
+      required: true
+  - type: textarea
+    id: environment
+    attributes:
+      label: Your Environment
+      description: |
+        - Fluentd or td-agent version: `fluentd --version` or `td-agent --version`
+        - Operating system: `cat /etc/os-release`
+        - Kernel version: `uname -r`
+
+        Tip: If you hit the problem with older fluentd version, try latest version first.
+      value: |
+        - Fluentd version:
+        - TD Agent version:
+        - fluent-plugin-kafka version:
+        - ruby-kafka version:
+        - Operating system:
+        - Kernel version:
+      render: markdown
+    validations:
+      required: true
+  - type: textarea
+    id: configuration
+    attributes:
+      label: Your Configuration
+      description: |
+        Write your configuration here. Minimum reproducible fluentd.conf is recommended.
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Your Error Log
+      description: Write your ALL error log here
+      render: shell
+    validations:
+      required: true
+  - type: textarea
+    id: addtional-context
+    attributes:
+      label: Additional context
+      description: Add any other context about the problem here.
+    validations:
+      required: false

data/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Ask a Question
+    url: https://discuss.fluentd.org/
+    about: I have questions about fluent-plugin-kafka. Please ask and answer questions at https://discuss.fluentd.org/.

data/.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -0,0 +1,38 @@
+name: Feature request
+description: Suggest an idea for this project
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Check [README.md](https://github.com/fluent/fluent-plugin-kafka/blob/master/README.md) first and here is the list to help us investigate the problem.
+  - type: textarea
+    id: description
+    attributes:
+      label: Is your feature request related to a problem? Please describe.
+      description: |
+        A clear and concise description of what the problem is.
+        Ex. I'm always frustrated when [...]
+    validations:
+      required: true
+  - type: textarea
+    id: solution
+    attributes:
+      label: Describe the solution you'd like
+      description: A clear and concise description of what you want to happen.
+    validations:
+      required: true
+  - type: textarea
+    id: alternative
+    attributes:
+      label: Describe alternatives you've considered
+      description: A clear and concise description of any alternative solutions or features you've considered.
+    validations:
+      required: true
+  - type: textarea
+    id: addtional-context
+    attributes:
+      label: Additional context
+      description: Add any other context or screenshots about the feature request here.
+    validations:
+      required: false
+

data/.github/workflows/linux.yml

@@ -5,6 +5,8 @@ on:
 jobs:
   build:
     runs-on: ${{ matrix.os }}
+    env:
+      USE_RDKAFKA: 1
     strategy:
       fail-fast: false
       matrix:
@@ -17,10 +19,20 @@ jobs:
     - uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}
+    - name: Install confluent-kafka
+      run: |
+        sudo apt install -V -y gnupg2 wget
+        wget https://packages.confluent.io/deb/6.0/archive.key
+        sudo gpg2 --homedir /tmp --no-default-keyring --keyring gnupg-ring:/usr/share/keyrings/confluent-archive-keyring.gpg --import archive.key
+        sudo chmod 644 /usr/share/keyrings/confluent-archive-keyring.gpg
+        sudo sh -c 'echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/confluent-archive-keyring.gpg] https://packages.confluent.io/deb/6.0 stable main" > /etc/apt/sources.list.d/confluent.list'
+        sudo apt update
+        sudo apt install -y confluent-community-2.13 openjdk-11-jre netcat-openbsd
     - name: unit testing
       env:
         CI: true
       run: |
+        sudo ./ci/prepare-kafka-server.sh
         gem install bundler rake
         bundle install --jobs 4 --retry 3
         bundle exec rake test

data/.github/workflows/stale-actions.yml

@@ -0,0 +1,22 @@
+name: "Mark or close stale issues and PRs"
+on:
+  schedule:
+  - cron: "00 10 * * *"
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/stale@v3
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        days-before-stale: 90
+        days-before-close: 30
+        stale-issue-message: "This issue has been automatically marked as stale because it has been open 90 days with no activity. Remove stale label or comment or this issue will be closed in 30 days"
+        stale-pr-message: "This PR has been automatically marked as stale because it has been open 90 days with no activity. Remove stale label or comment or this PR will be closed in 30 days"
+        close-issue-message: "This issue was automatically closed because of stale in 30 days"
+        close-pr-message: "This PR was automatically closed because of stale in 30 days"
+        stale-pr-label: "stale"
+        stale-issue-label: "stale"
+        exempt-issue-labels: "bug,enhancement,help wanted"
+        exempt-pr-labels: "bug,enhancement,help wanted"

data/ChangeLog

@@ -1,3 +1,21 @@
+Release 0.17.2 - 2021/10/14
+	* out_rdkafka2: Add `max_enqueue_bytes_per_second` parameter
+	* out_rdkafka2: Support `use_event_time` parameter
+	* out_rdkafka2: Fix a potential bug that the plugin might exit without receiving responses from Kafka.
+
+Release 0.17.1 - 2021/09/24
+	* out_rdkafka/out_rdkafka2: Support rdkafka 0.9.0 or later
+	* out_rdkafka/out_rdkafka2: Add `exclude_fields` parameter
+	* out_kafka2.rb: Fix one more Ruby 3.0 keyword arguments issue
+
+Release 0.17.0 - 2021/08/30
+	* out_kafka/out_kafka_buffered/out_kafka2: Provide murmur2 partitioner hash function choice
+	* in_kafka/in_kafka_group/out_kafka/out_kafka_buffered/out_kafka2: Use Ruby Kafka's ssl_ca_cert_file_path parameter to feed the CA certs
+	* out_kafka/out_kafka2: fix description for `exclude_message_key` option
+
+Release 0.16.3 - 2021/05/17
+	* in_kafka_group: Fix one more Ruby 3.0 keyword arguments issue
+
 Release 0.16.2 - 2021/05/17
 	* in_kafka, in_kafka_group: Support Ruby 3.0 keyword arguments interop
 

data/Gemfile

@@ -2,3 +2,5 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kafka.gemspec
 gemspec
+
+gem 'rdkafka', '>= 0.6.0' if ENV["USE_RDKAFKA"]

data/README.md

@@ -40,14 +40,14 @@ If you want to use zookeeper related parameters, you also need to install zookee
 
 Set path to SSL related files. See [Encryption and Authentication using SSL](https://github.com/zendesk/ruby-kafka#encryption-and-authentication-using-ssl) for more detail.
 
-#### SASL authentication 
+#### SASL authentication
 
 ##### with GSSAPI
 
 - principal
 - keytab
 
-Set principal and path to keytab for SASL/GSSAPI authentication. 
+Set principal and path to keytab for SASL/GSSAPI authentication.
 See [Authentication using SASL](https://github.com/zendesk/ruby-kafka#authentication-using-sasl) for more details.
 
 ##### with Plain/SCRAM
@@ -57,7 +57,7 @@ See [Authentication using SASL](https://github.com/zendesk/ruby-kafka#authentica
 - scram_mechanism
 - sasl_over_ssl
 
-Set username, password, scram_mechanism and sasl_over_ssl for SASL/Plain or Scram authentication. 
+Set username, password, scram_mechanism and sasl_over_ssl for SASL/Plain or Scram authentication.
 See [Authentication using SASL](https://github.com/zendesk/ruby-kafka#authentication-using-sasl) for more details.
 
 ### Input plugin (@type 'kafka')
@@ -200,8 +200,10 @@ If `ruby-kafka` doesn't fit your kafka environment, check `rdkafka2` plugin inst
       get_kafka_client_log  (bool)   :default => false
       headers               (hash)   :default => {}
       headers_from_record   (hash)   :default => {}
+      use_event_time        (bool)   :default => false
       use_default_for_unknown_topic (bool) :default => false
       discard_kafka_delivery_failed (bool) :default => false (No discard)
+      partitioner_hash_function (enum) (crc32|murmur2) :default => 'crc32'
 
       <format>
         @type (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
@@ -230,6 +232,8 @@ If `ruby-kafka` doesn't fit your kafka environment, check `rdkafka2` plugin inst
 
 The `<formatter name>` in `<format>` uses fluentd's formatter plugins. See [formatter article](https://docs.fluentd.org/v/1.0/formatter).
 
+**Note:** Java based Kafka client uses `murmur2` as partitioner function by default. If you want to use same partitioning behavior with fluent-plugin-kafka, change it to `murmur2` instead of `crc32`. Note that for using `murmur2` hash partitioner function, you must install `digest-murmurhash` gem.
+
 ruby-kafka sometimes returns `Kafka::DeliveryFailed` error without good information.
 In this case, `get_kafka_client_log` is useful for identifying the error cause.
 ruby-kafka's log is routed to fluentd log so you can see ruby-kafka's log in fluentd logs.
@@ -313,6 +317,23 @@ The Kafka message will have a header of source_ip=12.7.0.0.1.
 
 The configuration format is jsonpath. It is descibed in https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-record_accessor
 
+#### Excluding fields
+Fields can be excluded from output data. Only works for kafka2 and rdkafka2 output plugin.
+
+Fields must be specified using an array of dot notation `$.`, for example:
+
+    <match app.**>
+      @type kafka2
+      [...]
+      exclude_fields $.source.ip,$.HTTP_FOO
+    <match>
+
+This config can be used to remove fields used on another configs.
+
+For example, `$.source.ip` can be extracted with config `headers_from_record` and excluded from message payload.
+
+> Using this config to remove unused fields is discouraged. A [filter plugin](https://docs.fluentd.org/v/0.12/filter) can be used for this purpose.
+
 ### Buffered output plugin
 
 This plugin uses ruby-kafka producer for writing data. This plugin is for v0.12. If you use v1, see `kafka2`.
@@ -343,6 +364,8 @@ Support of fluentd v0.12 has ended. `kafka_buffered` will be an alias of `kafka2
       exclude_topic_key     (bool) :default => false
       exclude_partition_key (bool) :default => false
       get_kafka_client_log  (bool) :default => false
+      use_event_time        (bool) :default => false
+      partitioner_hash_function (enum) (crc32|murmur2) :default => 'crc32'
 
       # See fluentd document for buffer related parameters: https://docs.fluentd.org/v/0.12/buffer
 
@@ -365,6 +388,8 @@ Support of fluentd v0.12 has ended. `kafka_buffered` will be an alias of `kafka2
 - kafka_agg_max_bytes - default: 4096 - Maximum value of total message size to be included in one batch transmission.
 - kafka_agg_max_messages - default: nil - Maximum number of messages to include in one batch transmission.
 
+**Note:** Java based Kafka client uses `murmur2` as partitioner function by default. If you want to use same partitioning behavior with fluent-plugin-kafka, change it to `murmur2` instead of `crc32`. Note that for using `murmur2` hash partitioner function, you must install `digest-murmurhash` gem.
+
 ### Non-buffered output plugin
 
 This plugin uses ruby-kafka producer for writing data. For performance and reliability concerns, use `kafka_bufferd` output instead. This is mainly for testing.
@@ -385,6 +410,7 @@ This plugin uses ruby-kafka producer for writing data. For performance and relia
       output_include_time   (bool) :default => false
       exclude_topic_key     (bool) :default => false
       exclude_partition_key (bool) :default => false
+      partitioner_hash_function (enum) (crc32|murmur2) :default => 'crc32'
 
       # ruby-kafka producer options
       max_send_retries    (integer) :default => 1
@@ -397,6 +423,8 @@ This plugin uses ruby-kafka producer for writing data. For performance and relia
 
 This plugin also supports ruby-kafka related parameters. See Buffered output plugin section.
 
+**Note:** Java based Kafka client uses `murmur2` as partitioner function by default. If you want to use same partitioning behavior with fluent-plugin-kafka, change it to `murmur2` instead of `crc32`. Note that for using `murmur2` hash partitioner function, you must install `digest-murmurhash` gem.
+
 ### rdkafka based output plugin
 
 This plugin uses `rdkafka` instead of `ruby-kafka` for kafka client.
@@ -426,6 +454,7 @@ You need to install rdkafka gem.
       exclude_topic_key     (bool) :default => false
       exclude_partition_key (bool) :default => false
       discard_kafka_delivery_failed (bool) :default => false (No discard)
+      use_event_time        (bool) :default => false
 
       # same with kafka2
       headers               (hash) :default => {}
@@ -460,6 +489,10 @@ You need to install rdkafka gem.
       rdkafka_delivery_handle_poll_timeout (integer) :default => 30
       # If the record size is larger than this value, such records are ignored. Default is no limit
       max_send_limit_bytes (integer) :default => nil
+      # The maximum number of enqueueing bytes per second. It can reduce the
+      # load of both Fluentd and Kafka when excessive messages are attempted
+      # to send. Default is no limit.
+      max_enqueue_bytes_per_second (integer) :default => nil
     </match>
 
 If you use v0.12, use `rdkafka` instead.

data/ci/prepare-kafka-server.sh

@@ -0,0 +1,33 @@
+#!/bin/sh
+
+export KAFKA_OPTS=-Dzookeeper.4lw.commands.whitelist=ruok
+/usr/bin/zookeeper-server-start /etc/kafka/zookeeper.properties  &
+N_POLLING=30
+n=1
+while true ; do
+    sleep 1
+    status=$(echo ruok | nc localhost 2181)
+    if [ "$status" = "imok" ]; then
+	break
+    fi
+    n=$((n + 1))
+    if [ $n -ge $N_POLLING ]; then
+	echo "failed to get response from zookeeper-server"
+	exit 1
+    fi
+done
+/usr/bin/kafka-server-start /etc/kafka/server.properties &
+n=1
+while true ; do
+    sleep 1
+    status=$(/usr/bin/zookeeper-shell localhost:2181 ls /brokers/ids | sed -n 6p)
+    if [ "$status" = "[0]" ]; then
+	break
+    fi
+    n=$((n + 1))
+    if [ $n -ge $N_POLLING ]; then
+	echo "failed to get response from kafka-server"
+	exit 1
+    fi
+done
+/usr/bin/kafka-topics --create --zookeeper localhost:2181 --replication-factor 1 --partitions 1 --topic test

data/fluent-plugin-kafka.gemspec

@@ -13,13 +13,15 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.16.2'
+  gem.version       = '0.17.2'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
   gem.add_dependency 'ltsv'
-  gem.add_dependency 'ruby-kafka', '>= 1.2.0', '< 2'
+  gem.add_dependency 'ruby-kafka', '>= 1.4.0', '< 2'
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "test-unit", ">= 3.0.8"
+  gem.add_development_dependency "test-unit-rr", "~> 1.0"
   gem.add_development_dependency "webrick"
+  gem.add_development_dependency "digest-murmurhash"
 end

data/lib/fluent/plugin/in_kafka.rb

@@ -71,6 +71,7 @@ class Fluent::KafkaInput < Fluent::Input
     require 'kafka'
 
     @time_parser = nil
+    @zookeeper = nil
   end
 
   def configure(conf)
@@ -197,17 +198,17 @@ class Fluent::KafkaInput < Fluent::Input
 
     logger = @get_kafka_client_log ? log : nil
     if @scram_mechanism != nil && @username != nil && @password != nil
-      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert_file_path: @ssl_ca_cert,
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_scram_username: @username, sasl_scram_password: @password,
                          sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
     elsif @username != nil && @password != nil
-      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert_file_path: @ssl_ca_cert,
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system,sasl_plain_username: @username, sasl_plain_password: @password,
                          sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
     else
-      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert_file_path: @ssl_ca_cert,
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab,
                          ssl_verify_hostname: @ssl_verify_hostname)

data/lib/fluent/plugin/in_kafka_group.rb

@@ -183,17 +183,17 @@ class Fluent::KafkaGroupInput < Fluent::Input
 
     logger = @get_kafka_client_log ? log : nil
     if @scram_mechanism != nil && @username != nil && @password != nil
-      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert_file_path: @ssl_ca_cert,
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_scram_username: @username, sasl_scram_password: @password,
                          sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
     elsif @username != nil && @password != nil
-      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert_file_path: @ssl_ca_cert,
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_plain_username: @username, sasl_plain_password: @password,
                          sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
     else
-      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert_file_path: @ssl_ca_cert,
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab,
                          ssl_verify_hostname: @ssl_verify_hostname)
@@ -345,7 +345,7 @@ class Fluent::KafkaGroupInput < Fluent::Input
   def run
     while @consumer
       begin
-        @consumer.each_batch(@fetch_opts) { |batch|
+        @consumer.each_batch(**@fetch_opts) { |batch|
           if @tag_source == :record
             process_batch_with_record_tag(batch)
           else

data/lib/fluent/plugin/out_kafka.rb

@@ -19,6 +19,8 @@ DESC
   config_param :default_message_key, :string, :default => nil
   config_param :default_partition_key, :string, :default => nil
   config_param :default_partition, :integer, :default => nil
+  config_param :partitioner_hash_function, :enum, list: [:crc32, :murmur2], :default => :crc32,
+               :desc => "Specify kafka patrtitioner hash algorithm"
   config_param :client_id, :string, :default => 'kafka'
   config_param :sasl_over_ssl, :bool, :default => true,
                :desc => <<-DESC
@@ -86,6 +88,7 @@ DESC
     require 'kafka'
 
     @kafka = nil
+    @field_separator = nil
   end
 
   def refresh_client
@@ -106,18 +109,21 @@ DESC
     begin
       if @seed_brokers.length > 0
         if @scram_mechanism != nil && @username != nil && @password != nil
-          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert_file_path: @ssl_ca_cert,
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
                              sasl_scram_username: @username, sasl_scram_password: @password, sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl,
-                             ssl_verify_hostname: @ssl_verify_hostname)
+                             ssl_verify_hostname: @ssl_verify_hostname,
+                             partitioner: Kafka::Partitioner.new(hash_function: @partitioner_hash_function))
         elsif @username != nil && @password != nil
-          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert_file_path: @ssl_ca_cert,
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
-                             sasl_plain_username: @username, sasl_plain_password: @password, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
+                             sasl_plain_username: @username, sasl_plain_password: @password, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname,
+                             partitioner: Kafka::Partitioner.new(hash_function: @partitioner_hash_function))
         else
-          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert_file_path: @ssl_ca_cert,
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
-                             sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
+                             sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname,
+                             partitioner: Kafka::Partitioner.new(hash_function: @partitioner_hash_function))
         end
         log.info "initialized kafka producer: #{@client_id}"
       else

data/lib/fluent/plugin/out_kafka2.rb

@@ -24,6 +24,8 @@ DESC
     config_param :partition_key_key, :string, :default => 'partition_key', :desc => "Field for kafka partition key"
     config_param :default_partition_key, :string, :default => nil
     config_param :partition_key, :string, :default => 'partition', :desc => "Field for kafka partition"
+    config_param :partitioner_hash_function, :enum, list: [:crc32, :murmur2], :default => :crc32,
+                 :desc => "Specify kafka patrtitioner hash algorithm"
     config_param :default_partition, :integer, :default => nil
     config_param :use_default_for_unknown_topic, :bool, :default => false, :desc => "If true, default_topic is used when topic not found"
     config_param :client_id, :string, :default => 'fluentd'
@@ -37,9 +39,11 @@ DESC
     config_param :exclude_partition, :bool, :default => false,
                  :desc => 'Set true to remove partition from data'
     config_param :exclude_message_key, :bool, :default => false,
-                 :desc => 'Set true to remove partition key from data'
+                 :desc => 'Set true to remove message key from data'
     config_param :exclude_topic_key, :bool, :default => false,
                  :desc => 'Set true to remove topic name key from data'
+    config_param :exclude_fields, :array, :default => [], value_type: :string,
+                 :desc => 'Fields to remove from data where the value is a jsonpath to a record value'
     config_param :use_event_time, :bool, :default => false, :desc => 'Use fluentd event time for kafka create_time'
     config_param :headers, :hash, default: {}, symbolize_keys: true, value_type: :string,
                  :desc => 'Kafka message headers'
@@ -96,20 +100,23 @@ DESC
       begin
         logger = @get_kafka_client_log ? log : nil
         if @scram_mechanism != nil && @username != nil && @password != nil
-          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert_file_path: @ssl_ca_cert,
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_client_cert_chain: read_ssl_file(@ssl_client_cert_chain),
                              ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_scram_username: @username, sasl_scram_password: @password,
-                             sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
+                             sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname,
+                             partitioner: Kafka::Partitioner.new(hash_function: @partitioner_hash_function))
         elsif @username != nil && @password != nil
-          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert_file_path: @ssl_ca_cert,
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_client_cert_chain: read_ssl_file(@ssl_client_cert_chain),
                              ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_plain_username: @username, sasl_plain_password: @password, sasl_over_ssl: @sasl_over_ssl,
-                             ssl_verify_hostname: @ssl_verify_hostname)
+                             ssl_verify_hostname: @ssl_verify_hostname,
+                             partitioner: Kafka::Partitioner.new(hash_function: @partitioner_hash_function))
         else
-          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert_file_path: @ssl_ca_cert,
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_client_cert_chain: read_ssl_file(@ssl_client_cert_chain),
                              ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab, sasl_over_ssl: @sasl_over_ssl,
-                             ssl_verify_hostname: @ssl_verify_hostname)
+                             ssl_verify_hostname: @ssl_verify_hostname,
+                             partitioner: Kafka::Partitioner.new(hash_function: @partitioner_hash_function))
         end
         log.info "initialized kafka producer: #{@client_id}"
       rescue Exception => e
@@ -172,6 +179,10 @@ DESC
       @headers_from_record.each do |key, value|
         @headers_from_record_accessors[key] = record_accessor_create(value)
       end
+
+      @exclude_field_accessors = @exclude_fields.map do |field|
+        record_accessor_create(field)
+      end
     end
 
     def multi_workers_ready?
@@ -230,7 +241,7 @@ DESC
       mutate_headers = !@headers_from_record_accessors.empty?
 
       begin
-        producer = @kafka.topic_producer(topic, @producer_opts)
+        producer = @kafka.topic_producer(topic, **@producer_opts)
         chunk.msgpack_each { |time, record|
           begin
             record = inject_values_to_record(tag, time, record)
@@ -248,6 +259,12 @@ DESC
               headers = base_headers
             end
 
+            unless @exclude_fields.empty?
+              @exclude_field_accessors.each do |exclude_field_accessor|
+                exclude_field_accessor.delete(record)
+              end
+            end
+
             record_buf = @formatter_proc.call(tag, time, record)
             record_buf_bytes = record_buf.bytesize
             if @max_send_limit_bytes && record_buf_bytes > @max_send_limit_bytes

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -26,6 +26,8 @@ DESC
   config_param :default_partition_key, :string, :default => nil
   config_param :partition_key, :string, :default => 'partition', :desc => "Field for kafka partition"
   config_param :default_partition, :integer, :default => nil
+  config_param :partitioner_hash_function, :enum, list: [:crc32, :murmur2], :default => :crc32,
+               :desc => "Specify kafka patrtitioner hash algorithm"
   config_param :client_id, :string, :default => 'kafka'
   config_param :idempotent, :bool, :default => false, :desc => 'Enable idempotent producer'
   config_param :sasl_over_ssl, :bool, :default => true,
@@ -105,6 +107,7 @@ DESC
     @kafka = nil
     @producers = {}
     @producers_mutex = Mutex.new
+    @field_separator = nil
   end
 
   def multi_workers_ready?
@@ -130,18 +133,21 @@ DESC
       if @seed_brokers.length > 0
         logger = @get_kafka_client_log ? log : nil
         if @scram_mechanism != nil && @username != nil && @password != nil
-          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert_file_path: @ssl_ca_cert,
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
                              sasl_scram_username: @username, sasl_scram_password: @password, sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl,
-                             ssl_verify_hostname: @ssl_verify_hostname)
+                             ssl_verify_hostname: @ssl_verify_hostname,
+                             partitioner: Kafka::Partitioner.new(hash_function: @partitioner_hash_function))
         elsif @username != nil && @password != nil
-          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert_file_path: @ssl_ca_cert,
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
-                             sasl_plain_username: @username, sasl_plain_password: @password, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
+                             sasl_plain_username: @username, sasl_plain_password: @password, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname,
+                             partitioner: Kafka::Partitioner.new(hash_function: @partitioner_hash_function))
         else
-          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert_file_path: @ssl_ca_cert,
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
-                             sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab, sasl_over_ssl: @sasl_over_ssl,  ssl_verify_hostname: @ssl_verify_hostname)
+                             sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab, sasl_over_ssl: @sasl_over_ssl,  ssl_verify_hostname: @ssl_verify_hostname,
+                             partitioner: Kafka::Partitioner.new(hash_function: @partitioner_hash_function))
         end
         log.info "initialized kafka producer: #{@client_id}"
       else