fluent-plugin-kafka 0.17.2 → 0.17.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 16968c0e56c22f64225e8e41e905294defc7240df6054813151d3904a79a4107
-  data.tar.gz: 71b13953b11048f201c8a8a275350e983a51377b45e84440196b701063e317c4
+  metadata.gz: eb7f693666ff7fd4bdb43b249d3e60e882ebf99e80402de2a5310b61668ef9e7
+  data.tar.gz: d4b141409cf83402ad1e28b9579317c672b20c9760b8f58b8d54800e8c46313b
 SHA512:
-  metadata.gz: aedae36f4b7a29408bc96838da3e158a98ddac136987408dcd4bc347068e3d27839e844d1d3d4a239e4f175e023f19d9ecd82e14d4ab089fd43625f60dcca17c
-  data.tar.gz: 4a72ea62a754b689944d3f2de81cc495c9ef92df97c127b80b46ace6824aca00df5b4cc499eb3993d0edbd3f156f140c6c8ba718fb82f65b5328170adf26f622
+  metadata.gz: 8f86c7c82fbf5db63ef0c92737ba4935a3a60534d434dfb9552c2ad624ec1e28fea9876c689a29ca6b16c16abc3818fdfb24bd7540495f980daa8a5509b29ca0
+  data.tar.gz: 8b1dfba11d40e12f9b0b1fbf4ff947678fb5ef2ab3fbd3969e363935c8e895c29f497273ddb86c80c4e1ebc9d254045275dc7e08b07788d4573799c97ce7a393

data/.github/workflows/linux.yml

@@ -1,7 +1,9 @@
 name: linux
 on:
-  - push
-  - pull_request
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
 jobs:
   build:
     runs-on: ${{ matrix.os }}
@@ -10,7 +12,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.4', '2.5', '2.6', '2.7', '3.0' ]
+        ruby: [ '3.1', '3.0', '2.7', '2.6' ]
         os:
           - ubuntu-latest
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}

data/ChangeLog

@@ -1,3 +1,12 @@
+Release 0.17.5 - 2022/03/18
+	* out_kafka2: Add `resolve_seed_brokers` parameter
+
+Release 0.17.4 - 2022/01/25
+	* in_kafka_group: Add `refresh_topic_interval` parameter
+
+Release 0.17.3 - 2021/11/26
+	* output: Suppress large warning logs for events skipped by `max_send_limit_bytes`
+
 Release 0.17.2 - 2021/10/14
 	* out_rdkafka2: Add `max_enqueue_bytes_per_second` parameter
 	* out_rdkafka2: Support `use_event_time` parameter

data/README.md

@@ -119,7 +119,7 @@ Consume events by kafka consumer group features..
       topics <listening topics(separate with comma',')>
       format <input text type (text|json|ltsv|msgpack)> :default => json
       message_key <key (Optional, for text format only, default is message)>
-      kafka_mesasge_key <key (Optional, If specified, set kafka's message key to this key)>
+      kafka_message_key <key (Optional, If specified, set kafka's message key to this key)>
       add_headers <If true, add kafka's message headers to record>
       add_prefix <tag prefix (Optional)>
       add_suffix <tag suffix (Optional)>
@@ -135,6 +135,7 @@ Consume events by kafka consumer group features..
       offset_commit_interval  (integer) :default => nil (Use default of ruby-kafka)
       offset_commit_threshold (integer) :default => nil (Use default of ruby-kafka)
       fetcher_max_queue_size  (integer) :default => nil (Use default of ruby-kafka)
+      refresh_topic_interval  (integer) :default => nil (Use default of ruby-kafka)
       start_from_beginning    (bool)    :default => true
     </source>
 
@@ -155,7 +156,7 @@ With the introduction of the rdkafka-ruby based input plugin we hope to support
       topics <listening topics(separate with comma',')>
       format <input text type (text|json|ltsv|msgpack)> :default => json
       message_key <key (Optional, for text format only, default is message)>
-      kafka_mesasge_key <key (Optional, If specified, set kafka's message key to this key)>
+      kafka_message_key <key (Optional, If specified, set kafka's message key to this key)>
       add_headers <If true, add kafka's message headers to record>
       add_prefix <tag prefix (Optional)>
       add_suffix <tag suffix (Optional)>

data/fluent-plugin-kafka.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.17.2'
+  gem.version       = '0.17.5'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/in_kafka_group.rb

@@ -67,6 +67,8 @@ class Fluent::KafkaGroupInput < Fluent::Input
                :desc => "The number of messages that can be processed before their offsets are committed"
   config_param :fetcher_max_queue_size, :integer, :default => nil,
                :desc => "The number of fetched messages per partition that are queued in fetcher queue"
+  config_param :refresh_topic_interval, :integer, :default => nil,
+               :desc => "The interval of refreshing the topic list in seconds. Zero or unset disables this"
   config_param :start_from_beginning, :bool, :default => true,
                :desc => "Whether to start from the beginning of the topic or just subscribe to new messages being produced"
 
@@ -128,6 +130,7 @@ class Fluent::KafkaGroupInput < Fluent::Input
     @consumer_opts[:offset_commit_interval] = @offset_commit_interval if @offset_commit_interval
     @consumer_opts[:offset_commit_threshold] = @offset_commit_threshold if @offset_commit_threshold
     @consumer_opts[:fetcher_max_queue_size] = @fetcher_max_queue_size if @fetcher_max_queue_size
+    @consumer_opts[:refresh_topic_interval] = @refresh_topic_interval if @refresh_topic_interval
 
     @fetch_opts = {}
     @fetch_opts[:max_wait_time] = @max_wait_time if @max_wait_time

data/lib/fluent/plugin/out_kafka.rb

@@ -240,7 +240,8 @@ DESC
         record_buf = @formatter_proc.call(tag, time, record)
         record_buf_bytes = record_buf.bytesize
         if @max_send_limit_bytes && record_buf_bytes > @max_send_limit_bytes
-          log.warn "record size exceeds max_send_limit_bytes. Skip event:", :time => time, :record => record
+          log.warn "record size exceeds max_send_limit_bytes. Skip event:", :time => time, :record_size => record_buf_bytes
+          log.debug "Skipped event:", :record => record
           next
         end
         log.trace { "message will send to #{topic} with partition_key: #{partition_key}, partition: #{partition}, message_key: #{message_key} and value: #{record_buf}." }

data/lib/fluent/plugin/out_kafka2.rb

@@ -49,6 +49,8 @@ DESC
                  :desc => 'Kafka message headers'
     config_param :headers_from_record, :hash, default: {}, symbolize_keys: true, value_type: :string,
                  :desc => 'Kafka message headers where the header value is a jsonpath to a record value'
+    config_param :resolve_seed_brokers, :bool, :default => false,
+                 :desc => "support brokers' hostname with multiple addresses"
 
     config_param :get_kafka_client_log, :bool, :default => false
 
@@ -103,19 +105,19 @@ DESC
           @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert_file_path: @ssl_ca_cert,
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_client_cert_chain: read_ssl_file(@ssl_client_cert_chain),
                              ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_scram_username: @username, sasl_scram_password: @password,
-                             sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname,
+                             sasl_scram_mechanism: @scram_mechanism, sasl_over_ssl: @sasl_over_ssl, ssl_verify_hostname: @ssl_verify_hostname, resolve_seed_brokers: @resolve_seed_brokers,
                              partitioner: Kafka::Partitioner.new(hash_function: @partitioner_hash_function))
         elsif @username != nil && @password != nil
           @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert_file_path: @ssl_ca_cert,
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_client_cert_chain: read_ssl_file(@ssl_client_cert_chain),
                              ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_plain_username: @username, sasl_plain_password: @password, sasl_over_ssl: @sasl_over_ssl,
-                             ssl_verify_hostname: @ssl_verify_hostname,
+                             ssl_verify_hostname: @ssl_verify_hostname, resolve_seed_brokers: @resolve_seed_brokers,
                              partitioner: Kafka::Partitioner.new(hash_function: @partitioner_hash_function))
         else
           @kafka = Kafka.new(seed_brokers: @seed_brokers, client_id: @client_id, logger: logger, connect_timeout: @connect_timeout, socket_timeout: @socket_timeout, ssl_ca_cert_file_path: @ssl_ca_cert,
                              ssl_client_cert: read_ssl_file(@ssl_client_cert), ssl_client_cert_key: read_ssl_file(@ssl_client_cert_key), ssl_client_cert_chain: read_ssl_file(@ssl_client_cert_chain),
                              ssl_ca_certs_from_system: @ssl_ca_certs_from_system, sasl_gssapi_principal: @principal, sasl_gssapi_keytab: @keytab, sasl_over_ssl: @sasl_over_ssl,
-                             ssl_verify_hostname: @ssl_verify_hostname,
+                             ssl_verify_hostname: @ssl_verify_hostname, resolve_seed_brokers: @resolve_seed_brokers,
                              partitioner: Kafka::Partitioner.new(hash_function: @partitioner_hash_function))
         end
         log.info "initialized kafka producer: #{@client_id}"
@@ -268,7 +270,8 @@ DESC
             record_buf = @formatter_proc.call(tag, time, record)
             record_buf_bytes = record_buf.bytesize
             if @max_send_limit_bytes && record_buf_bytes > @max_send_limit_bytes
-              log.warn "record size exceeds max_send_limit_bytes. Skip event:", :time => time, :record => record
+              log.warn "record size exceeds max_send_limit_bytes. Skip event:", :time => time, :record_size => record_buf_bytes
+              log.debug "Skipped event:", :record => record
               next
             end
           rescue StandardError => e

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -332,7 +332,8 @@ DESC
           record_buf = @formatter_proc.call(tag, time, record)
           record_buf_bytes = record_buf.bytesize
           if @max_send_limit_bytes && record_buf_bytes > @max_send_limit_bytes
-            log.warn "record size exceeds max_send_limit_bytes. Skip event:", :time => time, :record => record
+            log.warn "record size exceeds max_send_limit_bytes. Skip event:", :time => time, :record_size => record_buf_bytes
+            log.debug "Skipped event:", :record => record
             next
           end
         rescue StandardError => e

data/lib/fluent/plugin/out_rdkafka.rb

@@ -278,7 +278,8 @@ DESC
           record_buf = @formatter_proc.call(tag, time, record)
           record_buf_bytes = record_buf.bytesize
           if @max_send_limit_bytes && record_buf_bytes > @max_send_limit_bytes
-            log.warn "record size exceeds max_send_limit_bytes. Skip event:", :time => time, :record => record
+            log.warn "record size exceeds max_send_limit_bytes. Skip event:", :time => time, :record_size => record_buf_bytes
+            log.debug "Skipped event:", :record => record
             next
           end
         rescue StandardError => e

data/lib/fluent/plugin/out_rdkafka2.rb

@@ -398,7 +398,8 @@ DESC
             record_buf = @formatter_proc.call(tag, time, record)
             record_buf_bytes = record_buf.bytesize
             if @max_send_limit_bytes && record_buf_bytes > @max_send_limit_bytes
-              log.warn "record size exceeds max_send_limit_bytes. Skip event:", :time => time, :record => record
+              log.warn "record size exceeds max_send_limit_bytes. Skip event:", :time => time, :record_size => record_buf_bytes
+              log.debug "Skipped event:", :record => record
               next
             end
           rescue StandardError => e

data/test/plugin/test_in_kafka_group.rb

@@ -14,6 +14,7 @@ class KafkaGroupInputTest < Test::Unit::TestCase
     brokers localhost:9092
     consumer_group fluentd
     format text
+    refresh_topic_interval 0
     @label @kafka
     topics #{TOPIC_NAME}
   ]
@@ -52,6 +53,7 @@ class KafkaGroupInputTest < Test::Unit::TestCase
         brokers localhost:9092
         format text
         @label @kafka
+        refresh_topic_interval 0
         topics #{TOPIC_NAME}
       ]
       d = create_driver

data/test/plugin/test_out_kafka2.rb

@@ -60,6 +60,13 @@ class Kafka2OutputTest < Test::Unit::TestCase
     assert_equal true, d.instance.multi_workers_ready?
   end
 
+  def test_resolve_seed_brokers
+    d = create_driver(config + config_element('ROOT', '', {"resolve_seed_brokers" => true}))
+    assert_nothing_raised do
+      d.instance.refresh_client
+    end
+  end
+
   class WriteTest < self
     TOPIC_NAME = "kafka-output-#{SecureRandom.uuid}"
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.17.2
+  version: 0.17.5
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-10-14 00:00:00.000000000 Z
+date: 2022-03-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd