fluent-plugin-kafka 0.19.0 → 0.19.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b4a8c37b041fedc3f95046620413e5f7da437557fc19294439585fa89fd5b244
-  data.tar.gz: fe4c8cc8df6b8b5b105fbf709044e10752d8b64f321b5a4e290cb7e316e10fe1
+  metadata.gz: 62d114017cabc47f58cd39db1ea21dd8cfb6d76126c4804741e25766a5b4576c
+  data.tar.gz: f5e4ea9adb96a8f2f46077c0ac89b51e1c177ea0933242ed6cbb6472e40577a2
 SHA512:
-  metadata.gz: 2ff333ee092e0ffd653ab476acf8b2656b4ca59ea32d6dcc846eb8a174f69d98811272e8a735bfa3bdfac3d5b3753ce499adb5c8a215b3197310b1f4d822364e
-  data.tar.gz: ebf6cafbde9635cfc886ee4dce84495353b2d389e447e6188f6a88eb4ca11b19086ce5a12565bac550ef26936f424e4ec5a0fe908fad4b39bea984758ac44720
+  metadata.gz: 80f4d2ce4d23d95837d3b344841339fc517581bc3347e17e5352a135b088756317b970b559eb66f05e0216ce8841c9e374c5ed0fd09e48d22346249ec9aec0b9
+  data.tar.gz: 8d89ff55bf6ac92e5d2c7a3942882a6b307e6fe19ede620bf299d54feb6f2f1d70fcdece731008324a993c96c69f44456a71798c9af879666de9d017b38e39c3

data/.github/workflows/linux.yml

@@ -20,7 +20,7 @@ jobs:
           - { min: '>= 0.12.0', max: '>= 0.12.0' }
     name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }} with rdkafka gem version (min ${{ matrix.rdkafka_versions.min }} max ${{ matrix.rdkafka_versions.max }})
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}

data/ChangeLog

@@ -1,3 +1,9 @@
+Release 0.19.2 - 2023/10/13
+	* out_rdkafka2: Add `discard_kafka_delivery_failed_regex`
+
+Release 0.19.1 - 2023/09/20
+	* out_rdkafka2: Add `use_default_for_unknown_topic` & `use_default_for_unknown_partition_error`
+
 Release 0.19.0 - 2023/04/26
 	* out_kafka2: Add support for AWS IAM authentication
 	* in_kafka, in_kafka_group, out_kafka2: Add support for ssl client cert key password

data/README.md

@@ -510,11 +510,14 @@ You need to install rdkafka gem.
       partition_key_key     (string) :default => 'partition_key'
       message_key_key       (string) :default => 'message_key'
       default_topic         (string) :default => nil
+      use_default_for_unknown_topic           (bool) :default => false
+      use_default_for_unknown_partition_error (bool) :default => false
       default_partition_key (string) :default => nil
       default_message_key   (string) :default => nil
       exclude_topic_key     (bool) :default => false
       exclude_partition_key (bool) :default => false
       discard_kafka_delivery_failed (bool) :default => false (No discard)
+      discard_kafka_delivery_failed_regex (regexp) :default => nil (No discard)
       use_event_time        (bool) :default => false
 
       # same with kafka2
@@ -557,6 +560,9 @@ You need to install rdkafka gem.
       max_enqueue_bytes_per_second (integer) :default => nil
     </match>
 
+`rdkafka2` supports `discard_kafka_delivery_failed_regex` parameter:
+- `discard_kafka_delivery_failed_regex` - default: nil - discard the record where the Kafka::DeliveryFailed occurred and the emitted message matches the given regex pattern, such as `/unknown_topic/`. 
+
 If you use v0.12, use `rdkafka` instead.
 
     <match kafka.**>

data/fluent-plugin-kafka.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.19.0'
+  gem.version       = '0.19.2'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/out_rdkafka2.rb

@@ -65,6 +65,8 @@ DESC
     config_param :topic_key, :string, :default => 'topic', :desc => "Field for kafka topic"
     config_param :default_topic, :string, :default => nil,
                  :desc => "Default output topic when record doesn't have topic field"
+    config_param :use_default_for_unknown_topic, :bool, :default => false, :desc => "If true, default_topic is used when topic not found"
+    config_param :use_default_for_unknown_partition_error, :bool, :default => false, :desc => "If true, default_topic is used when received unknown_partition error"
     config_param :message_key_key, :string, :default => 'message_key', :desc => "Field for kafka message key"
     config_param :default_message_key, :string, :default => nil
     config_param :partition_key, :string, :default => 'partition', :desc => "Field for kafka partition"
@@ -110,6 +112,7 @@ DESC
     config_param :use_event_time, :bool, :default => false, :desc => 'Use fluentd event time for rdkafka timestamp'
     config_param :max_send_limit_bytes, :size, :default => nil
     config_param :discard_kafka_delivery_failed, :bool, :default => false
+    config_param :discard_kafka_delivery_failed_regex, :regexp, :default => nil
     config_param :rdkafka_buffering_max_ms, :integer, :default => nil, :desc => 'Used for queue.buffering.max.ms'
     config_param :rdkafka_buffering_max_messages, :integer, :default => nil, :desc => 'Used for queue.buffering.max.messages'
     config_param :rdkafka_message_max_bytes, :integer, :default => nil, :desc => 'Used for message.max.bytes'
@@ -233,6 +236,9 @@ DESC
       @rdkafka = Rdkafka::Config.new(config)
 
       if @default_topic.nil?
+        if @use_default_for_unknown_topic || @use_default_for_unknown_partition_error
+          raise Fluent::ConfigError, "default_topic must be set when use_default_for_unknown_topic or use_default_for_unknown_partition_error is true"
+        end
         if @chunk_keys.include?(@topic_key) && !@chunk_key_tag
           log.warn "Use '#{@topic_key}' field of event record for topic but no fallback. Recommend to set default_topic or set 'tag' in buffer chunk keys like <buffer #{@topic_key},tag>"
         end
@@ -456,9 +462,13 @@ DESC
       if @discard_kafka_delivery_failed
         log.warn "Delivery failed. Discard events:", :error => e.to_s, :error_class => e.class.to_s, :tag => tag
       else
-        log.warn "Send exception occurred: #{e} at #{e.backtrace.first}"
-        # Raise exception to retry sendind messages
-        raise e
+	if @discard_kafka_delivery_failed_regex != nil && @discard_kafka_delivery_failed_regex.match?(e.to_s)
+          log.warn "Delivery failed and matched regexp pattern #{@discard_kafka_delivery_failed_regex}. Discard events:", :error => e.to_s, :error_class => e.class.to_s, :tag => tag
+	else
+	  log.warn "Send exception occurred: #{e} at #{e.backtrace.first}"
+          # Raise exception to retry sendind messages
+          raise e
+	end
       end
     ensure
       @writing_threads_mutex.synchronize { @writing_threads.delete(Thread.current) }
@@ -466,17 +476,25 @@ DESC
 
     def enqueue_with_retry(producer, topic, record_buf, message_key, partition, headers, time)
       attempt = 0
+      actual_topic = topic
+
       loop do
         begin
           @enqueue_rate.raise_if_limit_exceeded(record_buf.bytesize) if @enqueue_rate
-          return producer.produce(topic: topic, payload: record_buf, key: message_key, partition: partition, headers: headers, timestamp: @use_event_time ? Time.at(time) : nil)
+          return producer.produce(topic: actual_topic, payload: record_buf, key: message_key, partition: partition, headers: headers, timestamp: @use_event_time ? Time.at(time) : nil)
         rescue EnqueueRate::LimitExceeded => e
           @enqueue_rate.revert if @enqueue_rate
           duration = e.next_retry_clock - Fluent::Clock.now
           sleep(duration) if duration > 0.0
         rescue Exception => e
           @enqueue_rate.revert if @enqueue_rate
-          if e.respond_to?(:code) && e.code == :queue_full
+
+          if !e.respond_to?(:code)
+            raise e
+          end
+
+          case e.code
+          when :queue_full
             if attempt <= @max_enqueue_retries
               log.warn "Failed to enqueue message; attempting retry #{attempt} of #{@max_enqueue_retries} after #{@enqueue_retry_backoff}s"
               sleep @enqueue_retry_backoff
@@ -484,6 +502,25 @@ DESC
             else
               raise "Failed to enqueue message although tried retry #{@max_enqueue_retries} times"
             end
+          # https://github.com/confluentinc/librdkafka/blob/c282ba2423b2694052393c8edb0399a5ef471b3f/src/rdkafka.h#LL309C9-L309C41
+          # RD_KAFKA_RESP_ERR__UNKNOWN_TOPIC
+          when :unknown_topic
+            if @use_default_for_unknown_topic && actual_topic != @default_topic
+              log.debug "'#{actual_topic}' topic not found. Retry with '#{@default_topic}' topic"
+              actual_topic = @default_topic
+              retry
+            end
+            raise e
+          # https://github.com/confluentinc/librdkafka/blob/c282ba2423b2694052393c8edb0399a5ef471b3f/src/rdkafka.h#L305
+          # RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION
+          when :unknown_partition
+            if @use_default_for_unknown_partition_error && actual_topic != @default_topic
+              log.debug "failed writing to topic '#{actual_topic}' with error '#{e.to_s}'. Writing message to topic '#{@default_topic}'"
+              actual_topic = @default_topic
+              retry
+            end
+
+            raise e
           else
             raise e
           end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.19.0
+  version: 0.19.2
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-04-26 00:00:00.000000000 Z
+date: 2023-10-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd