fluent-plugin-kafka 0.16.3 → 0.17.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1cc628766df718a6fff17debcf2cffdcc041419cb96c1c9b6ce1fee532807b58
-  data.tar.gz: 543ccbd91345f45ee75c3e5e84ef5b3414420a0e2557c76a40afc8e308d70735
+  metadata.gz: 15805411e4029813123b9b636b6faadb937cf38c9841adcd9d998a0f54d8b687
+  data.tar.gz: 75fef11595c86beb4a54d2d2ff659f77075e328426463f7ef830982c9724ff16
 SHA512:
-  metadata.gz: 7cb27a4fe28ccc0f31e36449046faf8d4d3389be7820f7cfce8f612aad84b3239234663a71b50523cb08a13e145f78fdc86e93cf9cec28d35d82765b96c9dddc
-  data.tar.gz: 98f52a5d84fb348178f9df6ede6592412ddccd43891c95e9d6937622bff8b02bb4cbb3231c7ca7d709a3ae6cdbfb90d3f00074eefe4cc0703379315098424809
+  metadata.gz: db08ffbde4fe36ce38abe6eebf83e6e0dc157bc5d8dd95141ef911d0c5b59074a208c3a8f836e258ffddc34f73fda303006eaef1cd8fc12ae4dd4c79d101d0c7
+  data.tar.gz: 9b3b098b1bc58654924d50ac8d685af007763942c1cc68187bbd6d39510a27ca31ae795df0de312d97ad67463ae0c428882118640fce052119288e27e3fb0df5

data/.github/ISSUE_TEMPLATE/bug_report.yaml

@@ -0,0 +1,71 @@
+name: Bug Report
+description: Create a report with a procedure for reproducing the bug
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Check [README](https://github.com/fluent/fluent-plugin-kafka/#faq) first and here is the list to help us investigate the problem.
+  - type: textarea
+    id: description
+    attributes:
+      label: Describe the bug
+      description: A clear and concise description of what the bug is
+    validations:
+      required: true
+  - type: textarea
+    id: reproduce
+    attributes:
+      label: To Reproduce
+      description: Steps to reproduce the behavior
+    validations:
+      required: true
+  - type: textarea
+    id: expected
+    attributes:
+      label: Expected behavior
+      description: A clear and concise description of what you expected to happen
+    validations:
+      required: true
+  - type: textarea
+    id: environment
+    attributes:
+      label: Your Environment
+      description: |
+        - Fluentd or td-agent version: `fluentd --version` or `td-agent --version`
+        - Operating system: `cat /etc/os-release`
+        - Kernel version: `uname -r`
+
+        Tip: If you hit the problem with older fluentd version, try latest version first.
+      value: |
+        - Fluentd version:
+        - TD Agent version:
+        - fluent-plugin-kafka version:
+        - ruby-kafka version:
+        - Operating system:
+        - Kernel version:
+      render: markdown
+    validations:
+      required: true
+  - type: textarea
+    id: configuration
+    attributes:
+      label: Your Configuration
+      description: |
+        Write your configuration here. Minimum reproducible fluentd.conf is recommended.
+    validations:
+      required: true
+  - type: textarea
+    id: logs
+    attributes:
+      label: Your Error Log
+      description: Write your ALL error log here
+      render: shell
+    validations:
+      required: true
+  - type: textarea
+    id: addtional-context
+    attributes:
+      label: Additional context
+      description: Add any other context about the problem here.
+    validations:
+      required: false

data/.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,5 @@
+blank_issues_enabled: false
+contact_links:
+  - name: Ask a Question
+    url: https://discuss.fluentd.org/
+    about: I have questions about fluent-plugin-kafka. Please ask and answer questions at https://discuss.fluentd.org/.

data/.github/ISSUE_TEMPLATE/feature_request.yaml

@@ -0,0 +1,38 @@
+name: Feature request
+description: Suggest an idea for this project
+body:
+  - type: markdown
+    attributes:
+      value: |
+        Check [README.md](https://github.com/fluent/fluent-plugin-kafka/blob/master/README.md) first and here is the list to help us investigate the problem.
+  - type: textarea
+    id: description
+    attributes:
+      label: Is your feature request related to a problem? Please describe.
+      description: |
+        A clear and concise description of what the problem is.
+        Ex. I'm always frustrated when [...]
+    validations:
+      required: true
+  - type: textarea
+    id: solution
+    attributes:
+      label: Describe the solution you'd like
+      description: A clear and concise description of what you want to happen.
+    validations:
+      required: true
+  - type: textarea
+    id: alternative
+    attributes:
+      label: Describe alternatives you've considered
+      description: A clear and concise description of any alternative solutions or features you've considered.
+    validations:
+      required: true
+  - type: textarea
+    id: addtional-context
+    attributes:
+      label: Additional context
+      description: Add any other context or screenshots about the feature request here.
+    validations:
+      required: false
+

data/.github/workflows/stale-actions.yml

@@ -0,0 +1,22 @@
+name: "Mark or close stale issues and PRs"
+on:
+  schedule:
+  - cron: "00 10 * * *"
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/stale@v3
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        days-before-stale: 90
+        days-before-close: 30
+        stale-issue-message: "This issue has been automatically marked as stale because it has been open 90 days with no activity. Remove stale label or comment or this issue will be closed in 30 days"
+        stale-pr-message: "This PR has been automatically marked as stale because it has been open 90 days with no activity. Remove stale label or comment or this PR will be closed in 30 days"
+        close-issue-message: "This issue was automatically closed because of stale in 30 days"
+        close-pr-message: "This PR was automatically closed because of stale in 30 days"
+        stale-pr-label: "stale"
+        stale-issue-label: "stale"
+        exempt-issue-labels: "bug,enhancement,help wanted"
+        exempt-pr-labels: "bug,enhancement,help wanted"

data/ChangeLog

@@ -1,3 +1,8 @@
+Release 0.17.0 - 2021/08/30
+	* out_kafka/out_kafka_buffered/out_kafka2: Provide murmur2 partitioner hash function choice
+	* in_kafka/in_kafka_group/out_kafka/out_kafka_buffered/out_kafka2: Use Ruby Kafka's ssl_ca_cert_file_path parameter to feed the CA certs
+	* out_kafka/out_kafka2: fix description for `exclude_message_key` option
+
 Release 0.16.3 - 2021/05/17
 	* in_kafka_group: Fix one more Ruby 3.0 keyword arguments issue
 

data/README.md

@@ -202,6 +202,7 @@ If `ruby-kafka` doesn't fit your kafka environment, check `rdkafka2` plugin inst
       headers_from_record   (hash)   :default => {}
       use_default_for_unknown_topic (bool) :default => false
       discard_kafka_delivery_failed (bool) :default => false (No discard)
+      partitioner_hash_function (enum) (crc32|murmur2) :default => 'crc32'
 
       <format>
         @type (json|ltsv|msgpack|attr:<record name>|<formatter name>) :default => json
@@ -230,6 +231,8 @@ If `ruby-kafka` doesn't fit your kafka environment, check `rdkafka2` plugin inst
 
 The `<formatter name>` in `<format>` uses fluentd's formatter plugins. See [formatter article](https://docs.fluentd.org/v/1.0/formatter).
 
+**Note:** Java based Kafka client uses `murmur2` as partitioner function by default. If you want to use same partitioning behavior with fluent-plugin-kafka, change it to `murmur2` instead of `crc32`. Note that for using `murmur2` hash partitioner function, you must install `digest-murmurhash` gem.
+
 ruby-kafka sometimes returns `Kafka::DeliveryFailed` error without good information.
 In this case, `get_kafka_client_log` is useful for identifying the error cause.
 ruby-kafka's log is routed to fluentd log so you can see ruby-kafka's log in fluentd logs.
@@ -343,6 +346,7 @@ Support of fluentd v0.12 has ended. `kafka_buffered` will be an alias of `kafka2
       exclude_topic_key     (bool) :default => false
       exclude_partition_key (bool) :default => false
       get_kafka_client_log  (bool) :default => false
+      partitioner_hash_function (enum) (crc32|murmur2) :default => 'crc32'
 
       # See fluentd document for buffer related parameters: https://docs.fluentd.org/v/0.12/buffer
 
@@ -365,6 +369,8 @@ Support of fluentd v0.12 has ended. `kafka_buffered` will be an alias of `kafka2
 - kafka_agg_max_bytes - default: 4096 - Maximum value of total message size to be included in one batch transmission.
 - kafka_agg_max_messages - default: nil - Maximum number of messages to include in one batch transmission.
 
+**Note:** Java based Kafka client uses `murmur2` as partitioner function by default. If you want to use same partitioning behavior with fluent-plugin-kafka, change it to `murmur2` instead of `crc32`. Note that for using `murmur2` hash partitioner function, you must install `digest-murmurhash` gem.
+
 ### Non-buffered output plugin
 
 This plugin uses ruby-kafka producer for writing data. For performance and reliability concerns, use `kafka_bufferd` output instead. This is mainly for testing.
@@ -385,6 +391,7 @@ This plugin uses ruby-kafka producer for writing data. For performance and relia
       output_include_time   (bool) :default => false
       exclude_topic_key     (bool) :default => false
       exclude_partition_key (bool) :default => false
+      partitioner_hash_function (enum) (crc32|murmur2) :default => 'crc32'
 
       # ruby-kafka producer options
       max_send_retries    (integer) :default => 1
@@ -397,6 +404,8 @@ This plugin uses ruby-kafka producer for writing data. For performance and relia
 
 This plugin also supports ruby-kafka related parameters. See Buffered output plugin section.
 
+**Note:** Java based Kafka client uses `murmur2` as partitioner function by default. If you want to use same partitioning behavior with fluent-plugin-kafka, change it to `murmur2` instead of `crc32`. Note that for using `murmur2` hash partitioner function, you must install `digest-murmurhash` gem.
+
 ### rdkafka based output plugin
 
 This plugin uses `rdkafka` instead of `ruby-kafka` for kafka client.

data/fluent-plugin-kafka.gemspec

@@ -13,13 +13,15 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.16.3'
+  gem.version       = '0.17.0'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
   gem.add_dependency 'ltsv'
-  gem.add_dependency 'ruby-kafka', '>= 1.2.0', '< 2'
+  gem.add_dependency 'ruby-kafka', '>= 1.4.0', '< 2'
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "test-unit", ">= 3.0.8"
+  gem.add_development_dependency "test-unit-rr", "~> 1.0"
   gem.add_development_dependency "webrick"
+  gem.add_development_dependency "digest-murmurhash"
 end

data/lib/fluent/plugin/in_kafka.rb

@@ -197,17 +197,17 @@ class Fluent::KafkaInput < Fluent::Input
 
     logger = @get_kafka_client_log ? log : nil
     if @scram_mechanism != nil && @username != nil && @password != nil
-      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert_file_path: @ssl_ca_cert,
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_scram_username: @username, sasl_scram_password: @password,
                          sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
     elsif @username != nil && @password != nil
-      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert_file_path: @ssl_ca_cert,
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system,sasl_plain_username: @username, sasl_plain_password: @password,
                          sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
     else
-      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, ssl_ca_cert_file_path: @ssl_ca_cert,
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab,
                          ssl_verify_hostname: @ssl_verify_hostname)

data/lib/fluent/plugin/in_kafka_group.rb

@@ -183,17 +183,17 @@ class Fluent::KafkaGroupInput < Fluent::Input
 
     logger = @get_kafka_client_log ? log : nil
     if @scram_mechanism != nil && @username != nil && @password != nil
-      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert_file_path: @ssl_ca_cert,
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_scram_username: @username, sasl_scram_password: @password,
                          sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
     elsif @username != nil && @password != nil
-      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert_file_path: @ssl_ca_cert,
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_plain_username: @username, sasl_plain_password: @password,
                          sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
     else
-      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+      @kafka = Kafka.new(seed_brokers: @brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert_file_path: @ssl_ca_cert,
                          ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key),
                          ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab,
                          ssl_verify_hostname: @ssl_verify_hostname)

data/lib/fluent/plugin/out_kafka.rb

@@ -19,6 +19,8 @@ DESC
   config_param :default_message_key, :string, :default => nil
   config_param :default_partition_key, :string, :default => nil
   config_param :default_partition, :integer, :default => nil
+  config_param :partitioner_hash_function, :enum, list: [:crc32, :murmur2], :default => :crc32,
+               :desc => "Specify kafka patrtitioner hash algorithm"
   config_param :client_id, :string, :default => 'kafka'
   config_param :sasl_over_ssl, :bool, :default => true,
                :desc => <<-DESC
@@ -106,18 +108,21 @@ DESC
     begin
       if @seed_brokers.length > 0
         if @scram_mechanism != nil && @username != nil && @password != nil
-          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert_file_path: @ssl_ca_cert,
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
                              sasl_scram_username: @username, sasl_scram_password: @password, sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl,
-                             ssl_verify_hostname: @ssl_verify_hostname)
+                             ssl_verify_hostname: @ssl_verify_hostname,
+                             partitioner: Kafka::Partitioner.new(hash_function: @partitioner_hash_function))
         elsif @username != nil && @password != nil
-          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert_file_path: @ssl_ca_cert,
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
-                             sasl_plain_username: @username, sasl_plain_password: @password, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
+                             sasl_plain_username: @username, sasl_plain_password: @password, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname,
+                             partitioner: Kafka::Partitioner.new(hash_function: @partitioner_hash_function))
         else
-          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, ssl_ca_cert_file_path: @ssl_ca_cert,
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
-                             sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
+                             sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname,
+                             partitioner: Kafka::Partitioner.new(hash_function: @partitioner_hash_function))
         end
         log.info "initialized kafka producer: #{@client_id}"
       else

data/lib/fluent/plugin/out_kafka2.rb

@@ -24,6 +24,8 @@ DESC
     config_param :partition_key_key, :string, :default => 'partition_key', :desc => "Field for kafka partition key"
     config_param :default_partition_key, :string, :default => nil
     config_param :partition_key, :string, :default => 'partition', :desc => "Field for kafka partition"
+    config_param :partitioner_hash_function, :enum, list: [:crc32, :murmur2], :default => :crc32,
+                 :desc => "Specify kafka patrtitioner hash algorithm"
     config_param :default_partition, :integer, :default => nil
     config_param :use_default_for_unknown_topic, :bool, :default => false, :desc => "If true, default_topic is used when topic not found"
     config_param :client_id, :string, :default => 'fluentd'
@@ -37,7 +39,7 @@ DESC
     config_param :exclude_partition, :bool, :default => false,
                  :desc => 'Set true to remove partition from data'
     config_param :exclude_message_key, :bool, :default => false,
-                 :desc => 'Set true to remove partition key from data'
+                 :desc => 'Set true to remove message key from data'
     config_param :exclude_topic_key, :bool, :default => false,
                  :desc => 'Set true to remove topic name key from data'
     config_param :use_event_time, :bool, :default => false, :desc => 'Use fluentd event time for kafka create_time'
@@ -96,20 +98,23 @@ DESC
       begin
         logger = @get_kafka_client_log ? log : nil
         if @scram_mechanism != nil && @username != nil && @password != nil
-          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert_file_path: @ssl_ca_cert,
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_client_cert_chain: read_ssl_file(@ssl_client_cert_chain),
                              ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_scram_username: @username, sasl_scram_password: @password,
-                             sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
+                             sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname,
+                             partitioner: Kafka::Partitioner.new(hash_function: @partitioner_hash_function))
         elsif @username != nil && @password != nil
-          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert_file_path: @ssl_ca_cert,
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_client_cert_chain: read_ssl_file(@ssl_client_cert_chain),
                              ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_plain_username: @username, sasl_plain_password: @password, sasl_over_ssl: @sasl_over_ssl,
-                             ssl_verify_hostname: @ssl_verify_hostname)
+                             ssl_verify_hostname: @ssl_verify_hostname,
+                             partitioner: Kafka::Partitioner.new(hash_function: @partitioner_hash_function))
         else
-          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert_file_path: @ssl_ca_cert,
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_client_cert_chain: read_ssl_file(@ssl_client_cert_chain),
                              ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab, sasl_over_ssl: @sasl_over_ssl,
-                             ssl_verify_hostname: @ssl_verify_hostname)
+                             ssl_verify_hostname: @ssl_verify_hostname,
+                             partitioner: Kafka::Partitioner.new(hash_function: @partitioner_hash_function))
         end
         log.info "initialized kafka producer: #{@client_id}"
       rescue Exception => e

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -26,6 +26,8 @@ DESC
   config_param :default_partition_key, :string, :default => nil
   config_param :partition_key, :string, :default => 'partition', :desc => "Field for kafka partition"
   config_param :default_partition, :integer, :default => nil
+  config_param :partitioner_hash_function, :enum, list: [:crc32, :murmur2], :default => :crc32,
+               :desc => "Specify kafka patrtitioner hash algorithm"
   config_param :client_id, :string, :default => 'kafka'
   config_param :idempotent, :bool, :default => false, :desc => 'Enable idempotent producer'
   config_param :sasl_over_ssl, :bool, :default => true,
@@ -130,18 +132,21 @@ DESC
       if @seed_brokers.length > 0
         logger = @get_kafka_client_log ? log : nil
         if @scram_mechanism != nil && @username != nil && @password != nil
-          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert_file_path: @ssl_ca_cert,
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
                              sasl_scram_username: @username, sasl_scram_password: @password, sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl,
-                             ssl_verify_hostname: @ssl_verify_hostname)
+                             ssl_verify_hostname: @ssl_verify_hostname,
+                             partitioner: Kafka::Partitioner.new(hash_function: @partitioner_hash_function))
         elsif @username != nil && @password != nil
-          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert_file_path: @ssl_ca_cert,
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
-                             sasl_plain_username: @username, sasl_plain_password: @password, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname)
+                             sasl_plain_username: @username, sasl_plain_password: @password, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname,
+                             partitioner: Kafka::Partitioner.new(hash_function: @partitioner_hash_function))
         else
-          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert: read_ssl_file(@ssl_ca_cert),
+          @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, ssl_ca_cert_file_path: @ssl_ca_cert,
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_ca_certs_from_system: @ssl_ca_certs_from_system,
-                             sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab, sasl_over_ssl: @sasl_over_ssl,  ssl_verify_hostname: @ssl_verify_hostname)
+                             sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab, sasl_over_ssl: @sasl_over_ssl,  ssl_verify_hostname: @ssl_verify_hostname,
+                             partitioner: Kafka::Partitioner.new(hash_function: @partitioner_hash_function))
         end
         log.info "initialized kafka producer: #{@client_id}"
       else

data/lib/fluent/plugin/out_rdkafka.rb

@@ -48,7 +48,7 @@ Set true to remove partition from data
 DESC
    config_param :exclude_message_key, :bool, :default => false,
                :desc => <<-DESC
-Set true to remove partition key from data
+Set true to remove message key from data
 DESC
    config_param :exclude_topic_key, :bool, :default => false,
                 :desc => <<-DESC

data/test/helper.rb

@@ -8,6 +8,7 @@ rescue Bundler::BundlerError => e
   exit e.status_code
 end
 require 'test/unit'
+require 'test/unit/rr'
 
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
@@ -22,8 +23,12 @@ unless ENV.has_key?('VERBOSE')
 end
 
 require 'fluent/plugin/out_kafka'
+require 'fluent/plugin/out_kafka_buffered'
+require 'fluent/plugin/out_kafka2'
 require 'fluent/plugin/in_kafka'
 require 'fluent/plugin/in_kafka_group'
 
+require "fluent/test/driver/output"
+
 class Test::Unit::TestCase
 end

data/test/plugin/test_kafka_plugin_util.rb

@@ -1,14 +1,8 @@
 require 'helper'
 require 'fluent/plugin/kafka_plugin_util'
 
-class File
-    def File::read(path)
-        path
-    end
-end
-
 class KafkaPluginUtilTest < Test::Unit::TestCase
-    
+
     def self.config_param(name, type, options)
     end
     include Fluent::KafkaPluginUtil::SSLSettings
@@ -20,19 +14,31 @@ class KafkaPluginUtilTest < Test::Unit::TestCase
     end
 
     def test_read_ssl_file_when_nil
-        assert_equal(nil, read_ssl_file(nil))
+      stub(File).read(anything) do |path|
+        path
+      end
+      assert_equal(nil, read_ssl_file(nil))
     end
 
     def test_read_ssl_file_when_empty_string
-        assert_equal(nil, read_ssl_file(""))
+      stub(File).read(anything) do |path|
+        path
+      end
+      assert_equal(nil, read_ssl_file(""))
     end
 
     def test_read_ssl_file_when_non_empty_path
-        assert_equal("path", read_ssl_file("path"))
+      stub(File).read(anything) do |path|
+        path
+      end
+      assert_equal("path", read_ssl_file("path"))
     end
 
     def test_read_ssl_file_when_non_empty_array
-        assert_equal(["a","b"], read_ssl_file(["a","b"]))
+      stub(File).read(anything) do |path|
+        path
+      end
+      assert_equal(["a","b"], read_ssl_file(["a","b"]))
     end
 
-end
+end

data/test/plugin/test_out_kafka.rb

@@ -43,6 +43,16 @@ class KafkaOutputTest < Test::Unit::TestCase
     d = create_driver
   end
 
+  data("crc32" => "crc32",
+      "murmur2" => "murmur2")
+  def test_partitioner_hash_function(data)
+    hash_type = data
+    d = create_driver(CONFIG + %[partitioner_hash_function #{hash_type}])
+    assert_nothing_raised do
+      d.instance.refresh_client
+    end
+  end
+
   def test_mutli_worker_support
     d = create_driver
     assert_equal true, d.instance.multi_workers_ready?

data/test/plugin/test_out_kafka2.rb

@@ -0,0 +1,60 @@
+require 'helper'
+require 'fluent/test/helpers'
+require 'fluent/output'
+
+class Kafka2OutputTest < Test::Unit::TestCase
+  include Fluent::Test::Helpers
+
+  def setup
+    Fluent::Test.setup
+  end
+
+  def base_config
+    config_element('ROOT', '', {"@type" => "kafka2"}, [
+                     config_element('format', "", {"@type" => "json"})
+                   ])
+  end
+
+  def config
+    base_config + config_element('ROOT', '', {"default_topic" => "kitagawakeiko",
+                                              "brokers" => "localhost:9092"}, [
+                                 ])
+  end
+
+  def create_driver(conf = config, tag='test')
+    Fluent::Test::Driver::Output.new(Fluent::Kafka2Output).configure(conf)
+  end
+
+  def test_configure
+    assert_nothing_raised(Fluent::ConfigError) {
+      create_driver(base_config)
+    }
+
+    assert_nothing_raised(Fluent::ConfigError) {
+      create_driver(config)
+    }
+
+    assert_nothing_raised(Fluent::ConfigError) {
+      create_driver(config + config_element('buffer', "", {"@type" => "memory"}))
+    }
+
+    d = create_driver
+    assert_equal 'kitagawakeiko', d.instance.default_topic
+    assert_equal ['localhost:9092'], d.instance.brokers
+  end
+
+  data("crc32" => "crc32",
+       "murmur2" => "murmur2")
+  def test_partitioner_hash_function(data)
+    hash_type = data
+    d = create_driver(config + config_element('ROOT', '', {"partitioner_hash_function" => hash_type}))
+    assert_nothing_raised do
+      d.instance.refresh_client
+    end
+  end
+
+  def test_mutli_worker_support
+    d = create_driver
+    assert_equal true, d.instance.multi_workers_ready?
+  end
+end

data/test/plugin/test_out_kafka_buffered.rb

@@ -0,0 +1,68 @@
+require 'helper'
+require 'fluent/output'
+
+class KafkaBufferedOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  BASE_CONFIG = %[
+    type kafka_buffered
+  ]
+
+  CONFIG = BASE_CONFIG + %[
+    default_topic kitagawakeiko
+    brokers localhost:9092
+  ]
+
+  def create_driver(conf = CONFIG, tag='test')
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::KafkaOutputBuffered, tag).configure(conf)
+  end
+
+  def test_configure
+    assert_nothing_raised(Fluent::ConfigError) {
+      create_driver(BASE_CONFIG)
+    }
+
+    assert_nothing_raised(Fluent::ConfigError) {
+      create_driver(CONFIG)
+    }
+
+    assert_nothing_raised(Fluent::ConfigError) {
+      create_driver(CONFIG + %[
+        buffer_type memory
+      ])
+    }
+
+    d = create_driver
+    assert_equal 'kitagawakeiko', d.instance.default_topic
+    assert_equal 'localhost:9092', d.instance.brokers
+  end
+
+  def test_format
+    d = create_driver
+  end
+
+  data("crc32" => "crc32",
+      "murmur2" => "murmur2")
+  def test_partitioner_hash_function(data)
+    hash_type = data
+    d = create_driver(CONFIG + %[partitioner_hash_function #{hash_type}])
+    assert_nothing_raised do
+      d.instance.refresh_client
+    end
+  end
+
+  def test_mutli_worker_support
+    d = create_driver
+    assert_equal true, d.instance.multi_workers_ready?
+
+  end
+
+  def test_write
+    d = create_driver
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit({"a"=>1}, time)
+    d.emit({"a"=>2}, time)
+  end
+end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.16.3
+  version: 0.17.0
 platform: ruby
 authors:
 - Hidemasa Togashi
 - Masahiro Nakagawa
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-05-17 00:00:00.000000000 Z
+date: 2021-08-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -51,7 +51,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.2.0
+        version: 1.4.0
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -61,7 +61,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.2.0
+        version: 1.4.0
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -93,6 +93,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.0.8
+- !ruby/object:Gem::Dependency
+  name: test-unit-rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: webrick
   requirement: !ruby/object:Gem::Requirement
@@ -107,6 +121,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: digest-murmurhash
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Fluentd plugin for Apache Kafka > 0.8
 email:
 - togachiro@gmail.com
@@ -115,7 +143,11 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/ISSUE_TEMPLATE/bug_report.yaml"
+- ".github/ISSUE_TEMPLATE/config.yml"
+- ".github/ISSUE_TEMPLATE/feature_request.yaml"
 - ".github/workflows/linux.yml"
+- ".github/workflows/stale-actions.yml"
 - ".gitignore"
 - ChangeLog
 - Gemfile
@@ -139,11 +171,13 @@ files:
 - test/plugin/test_in_kafka_group.rb
 - test/plugin/test_kafka_plugin_util.rb
 - test/plugin/test_out_kafka.rb
+- test/plugin/test_out_kafka2.rb
+- test/plugin/test_out_kafka_buffered.rb
 homepage: https://github.com/fluent/fluent-plugin-kafka
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -158,8 +192,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.5
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: Fluentd plugin for Apache Kafka > 0.8
 test_files:
@@ -168,3 +202,5 @@ test_files:
 - test/plugin/test_in_kafka_group.rb
 - test/plugin/test_kafka_plugin_util.rb
 - test/plugin/test_out_kafka.rb
+- test/plugin/test_out_kafka2.rb
+- test/plugin/test_out_kafka_buffered.rb