fluent-plugin-kafka 0.19.6 → 0.19.8

data/test/plugin/test_kafka_plugin_util.rb

@@ -6,6 +6,7 @@ class KafkaPluginUtilTest < Test::Unit::TestCase
     def self.config_param(name, type, options)
     end
     include Fluent::KafkaPluginUtil::SSLSettings
+    include Fluent::KafkaPluginUtil::PartitionSettings
 
     def config_param
     end
@@ -13,6 +14,31 @@ class KafkaPluginUtilTest < Test::Unit::TestCase
         Fluent::Test.setup
     end
 
+    data("integer" => [3, 3],
+         "decimal string" => ["3", 3],
+         "zero padded string" => ["010", 10],
+         "unassigned partition" => [-1, -1],
+         "max int32" => [2**31 - 1, 2**31 - 1])
+    def test_coerce_partition(data)
+      given, expected = data
+      assert_equal(expected, coerce_partition(given))
+    end
+
+    data("non numeric string" => ["not-a-number", ArgumentError],
+         "empty string" => ["", ArgumentError],
+         "hexadecimal string" => ["0x10", ArgumentError],
+         "float string" => ["3.5", ArgumentError],
+         "float" => [3.9, TypeError],
+         "nil" => [nil, TypeError],
+         "negative partition" => [-2, RangeError],
+         "too big for int32" => [2**31, RangeError])
+    def test_coerce_partition_rejects_invalid_value(data)
+      given, expected = data
+      assert_raise(expected) do
+        coerce_partition(given)
+      end
+    end
+
     def test_read_ssl_file_when_nil
       stub(File).read(anything) do |path|
         path

data/test/plugin/test_out_kafka2.rb

@@ -27,6 +27,32 @@ class Kafka2OutputTest < Test::Unit::TestCase
     Fluent::Test::Driver::Output.new(Fluent::Kafka2Output).configure(conf)
   end
 
+  class DummyProducer
+    attr_reader :produced
+
+    # Coerces like ruby-kafka's Producer#produce, which is where an invalid
+    # partition raises today
+    def produce(value, key:, partition_key:, partition:, headers:, create_time:, topic:)
+      @produced ||= []
+      @produced << {value: value && value.to_s, key: key && key.to_s,
+                    partition: partition && Integer(partition)}
+    end
+
+    def deliver_messages
+    end
+
+    def clear_buffer
+    end
+  end
+
+  def create_chunk(records, tag: 'test')
+    metadata = Fluent::Plugin::Buffer::Metadata.new(nil, tag, nil)
+    chunk = Fluent::Plugin::Buffer::MemoryChunk.new(metadata)
+    chunk.extend(Fluent::ChunkMessagePackEventStreamer)
+    chunk.append(records.map { |record| [event_time, record].to_msgpack })
+    chunk
+  end
+
   def test_configure
     assert_nothing_raised(Fluent::ConfigError) {
       create_driver(base_config)
@@ -45,6 +71,79 @@ class Kafka2OutputTest < Test::Unit::TestCase
     assert_equal ['localhost:9092'], d.instance.brokers
   end
 
+  data("cert without key" => {"ssl_client_cert" => "/path/to/cert.pem"},
+       "key without cert" => {"ssl_client_cert_key" => "/path/to/key.pem"},
+       "chain without cert" => {"ssl_client_cert_chain" => "/path/to/chain.pem"},
+       "key password without key" => {"ssl_client_cert_key_password" => "secret"})
+  def test_configure_incomplete_ssl_client_cert(params)
+    conf = config + config_element('ROOT', '', params, [])
+
+    assert_raise(Fluent::ConfigError) {
+      create_driver(conf)
+    }
+  end
+
+  def test_configure_ssl_client_cert_with_key
+    conf = config + config_element('ROOT', '', {"ssl_client_cert" => "/path/to/cert.pem",
+                                                "ssl_client_cert_key" => "/path/to/key.pem"}, [])
+
+    assert_nothing_raised(Fluent::ConfigError) {
+      create_driver(conf)
+    }
+  end
+
+  data("sha256" => "sha256",
+       "sha512" => "sha512")
+  def test_configure_scram_mechanism(mechanism)
+    conf = config + config_element('ROOT', '', {"username" => "testuser",
+                                                "password" => "testpass",
+                                                "scram_mechanism" => mechanism,
+                                                "ssl_ca_certs_from_system" => "true"}, [])
+    d = create_driver(conf)
+
+    assert_equal mechanism, d.instance.scram_mechanism
+
+    assert_nothing_raised {
+      d.instance.refresh_client
+    }
+  end
+
+  def test_configure_unsupported_scram_mechanism
+    conf = config + config_element('ROOT', '', {"username" => "testuser",
+                                                "password" => "testpass",
+                                                "scram_mechanism" => "sha1"}, [])
+
+    assert_raise(Fluent::ConfigError) {
+      create_driver(conf)
+    }
+  end
+
+  data("non numeric partition" => "not-a-number",
+       "hash partition" => {"x" => 1},
+       "negative partition" => -2,
+       "out of int32 range" => 2**31)
+  def test_write_skips_event_with_invalid_partition(partition)
+    d = create_driver
+    producer = DummyProducer.new
+    stub(d.instance).get_producer { producer }
+
+    assert_nothing_raised {
+      d.instance.write(create_chunk([{"a" => "b"}, {"a" => "c", "partition" => partition}, {"a" => "d"}]))
+    }
+
+    assert_equal ['{"a":"b"}', '{"a":"d"}'], producer.produced.collect { |message| message[:value] }
+  end
+
+  def test_write_keeps_event_with_zero_padded_partition
+    d = create_driver
+    producer = DummyProducer.new
+    stub(d.instance).get_producer { producer }
+
+    d.instance.write(create_chunk([{"a" => "b", "partition" => "010"}]))
+
+    assert_equal [10], producer.produced.collect { |message| message[:partition] }
+  end
+
   data("crc32" => "crc32",
        "murmur2" => "murmur2")
   def test_partitioner_hash_function(data)

data/test/plugin/test_out_rdkafka.rb

@@ -0,0 +1,105 @@
+require 'helper'
+require 'fluent/test/helpers'
+require 'fluent/test/driver/output'
+
+class RdkafkaOutputTest < Test::Unit::TestCase
+  include Fluent::Test::Helpers
+
+  def have_rdkafka
+    begin
+      require 'fluent/plugin/out_rdkafka'
+      true
+    rescue LoadError
+      false
+    end
+  end
+
+  def setup
+    omit_unless(have_rdkafka, "rdkafka isn't installed")
+    Fluent::Test.setup
+  end
+
+  def base_config(params = {})
+    config_element('ROOT', '', {"@type" => "rdkafka",
+                                "brokers" => "localhost:9092"}.merge(params), [])
+  end
+
+  def create_driver(conf = base_config)
+    Fluent::Test::Driver::Output.new(Fluent::KafkaOutputBuffered2).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+
+    assert_equal 'localhost:9092', d.instance.brokers
+  end
+
+  def test_configure_ssl_ca_cert
+    d = create_driver(base_config("ssl_ca_cert" => "/path/to/ca_cert.pem"))
+
+    config = d.instance.build_config
+
+    assert_equal 'SSL', config[:"security.protocol"]
+    assert_equal '/path/to/ca_cert.pem', config[:"ssl.ca.location"]
+  end
+
+  def test_configure_ssl_ca_certs_from_system
+    d = create_driver(base_config("ssl_ca_certs_from_system" => "true"))
+
+    config = d.instance.build_config
+
+    assert_equal 'SSL', config[:"security.protocol"]
+    assert_nil config[:"ssl.ca.location"]
+  end
+
+  def test_configure_ssl_client_cert_without_ca_cert
+    d = create_driver(base_config("ssl_client_cert" => "/path/to/cert.pem",
+                                  "ssl_client_cert_key" => "/path/to/key.pem"))
+
+    config = d.instance.build_config
+
+    assert_equal 'SSL', config[:"security.protocol"]
+    assert_equal '/path/to/cert.pem', config[:"ssl.certificate.location"]
+    assert_equal '/path/to/key.pem', config[:"ssl.key.location"]
+    assert_nil config[:"ssl.ca.location"]
+  end
+
+  def test_configure_ssl_verify_hostname_default
+    d = create_driver(base_config("ssl_ca_cert" => "/path/to/ca_cert.pem"))
+
+    config = d.instance.build_config
+
+    assert_equal 'SSL', config[:"security.protocol"]
+    assert_equal 'https', config[:"ssl.endpoint.identification.algorithm"]
+    assert_equal true, config[:"enable.ssl.certificate.verification"]
+  end
+
+  def test_configure_ssl_verify_hostname_false
+    d = create_driver(base_config("ssl_ca_cert" => "/path/to/ca_cert.pem",
+                                  "ssl_verify_hostname" => "false"))
+
+    config = d.instance.build_config
+
+    assert_equal 'none', config[:"ssl.endpoint.identification.algorithm"]
+    assert_equal true, config[:"enable.ssl.certificate.verification"]
+  end
+
+  def test_configure_without_ssl_has_no_endpoint_identification
+    config = create_driver.instance.build_config
+
+    assert_equal 'PLAINTEXT', config[:"security.protocol"]
+    assert_nil config[:"ssl.endpoint.identification.algorithm"]
+    assert_nil config[:"enable.ssl.certificate.verification"]
+  end
+
+  def test_configure_sasl_gssapi_over_ssl
+    d = create_driver(base_config("principal" => "testuser@EXAMPLE.COM",
+                                  "ssl_client_cert" => "/path/to/cert.pem",
+                                  "ssl_client_cert_key" => "/path/to/key.pem"))
+
+    config = d.instance.build_config
+
+    assert_equal 'SASL_SSL', config[:"security.protocol"]
+    assert_equal 'GSSAPI', config[:"sasl.mechanisms"]
+  end
+end

data/test/plugin/test_out_rdkafka2.rb

@@ -37,6 +37,80 @@ class Rdkafka2OutputTest < Test::Unit::TestCase
     Fluent::Test::Driver::Output.new(Fluent::Rdkafka2Output).configure(conf)
   end
 
+  class DummyDeliveryHandle
+    def initialize(error = nil)
+      @error = error
+    end
+
+    def wait(**)
+      raise @error if @error
+    end
+  end
+
+  class DummyProducer
+    attr_reader :produced
+
+    def initialize(produce_error: nil, delivery_error: nil)
+      @produced = []
+      @produce_error = produce_error
+      @delivery_error = delivery_error
+    end
+
+    # Raises what the rdkafka FFI binding raises: the partition must be an
+    # int32 and the key must respond to #bytesize
+    def produce(topic:, payload:, key:, partition:, headers:, timestamp:)
+      unless partition.nil?
+        raise TypeError, "no implicit conversion of #{partition.class} into Integer" unless partition.is_a?(Integer)
+        raise RangeError, "integer #{partition} too big to convert to 'int'" unless (-2**31..2**31 - 1).cover?(partition)
+      end
+      key.bytesize unless key.nil?
+      raise @produce_error if @produce_error
+
+      @produced << {payload: payload, key: key, partition: partition}
+      DummyDeliveryHandle.new(@delivery_error)
+    end
+  end
+
+  RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION = -190
+  RD_KAFKA_RESP_ERR_MSG_SIZE_TOO_LARGE = 10
+
+  class DummyErrorWithCode < StandardError
+    def code
+      :msg_size_too_large
+    end
+  end
+
+  def create_chunk(records, tag: 'test')
+    metadata = Fluent::Plugin::Buffer::Metadata.new(nil, tag, nil)
+    chunk = Fluent::Plugin::Buffer::MemoryChunk.new(metadata)
+    chunk.extend(Fluent::ChunkMessagePackEventStreamer)
+    chunk.append(records.map { |record| [event_time, record].to_msgpack })
+    chunk
+  end
+
+  def test_rdkafka_patch_is_applied
+    version = Gem::Version.create(Rdkafka::VERSION)
+    expected = if version >= Gem::Version.create('0.16.0')
+                 '0_16_0.rb'
+               elsif version >= Gem::Version.create('0.14.0')
+                 '0_14_0.rb'
+               elsif version >= Gem::Version.create('0.13.0')
+                 '0_13_0.rb'
+               elsif version >= Gem::Version.create('0.12.0')
+                 '0_12_0.rb'
+               else
+                 '0_11_0.rb'
+               end
+
+    assert_equal expected, File.basename(Rdkafka::Producer.instance_method(:close).source_location.first)
+  end
+
+  def test_producer_close_accepts_timeout
+    producer = Rdkafka::Config.new("bootstrap.servers" => "localhost:9092").producer
+
+    assert_true producer.close(10)
+  end
+
   def test_configure
     assert_nothing_raised(Fluent::ConfigError) {
       create_driver(base_config)
@@ -55,8 +129,30 @@ class Rdkafka2OutputTest < Test::Unit::TestCase
     assert_equal 'localhost:9092', d.instance.brokers
   end
 
-  def test_configure_sasl_plain
+  def test_configure_sasl_plain_over_ssl
+    conf = base_config + config_element('ROOT', '', {"username" => "testuser", "password" => "testpass",
+                                                     "ssl_ca_cert" => "/path/to/ca_cert.pem"}, [])
+    d = create_driver(conf)
+
+    config = d.instance.build_config
+
+    assert_equal 'PLAIN', config[:"sasl.mechanisms"]
+    assert_equal 'SASL_SSL', config[:"security.protocol"]
+    assert_equal 'testuser', config[:"sasl.username"]
+    assert_equal 'testpass', config[:"sasl.password"]
+  end
+
+  def test_configure_sasl_plain_without_ssl
     conf = base_config + config_element('ROOT', '', {"username" => "testuser", "password" => "testpass"}, [])
+
+    assert_raise(Fluent::ConfigError) {
+      create_driver(conf)
+    }
+  end
+
+  def test_configure_sasl_plain_without_ssl_allowed_by_sasl_over_ssl
+    conf = base_config + config_element('ROOT', '', {"username" => "testuser", "password" => "testpass",
+                                                     "sasl_over_ssl" => "false"}, [])
     d = create_driver(conf)
 
     config = d.instance.build_config
@@ -67,6 +163,250 @@ class Rdkafka2OutputTest < Test::Unit::TestCase
     assert_equal 'testpass', config[:"sasl.password"]
   end
 
+  def test_configure_sasl_plain_with_security_protocol_from_rdkafka_options
+    conf = base_config + config_element('ROOT', '', {"username" => "testuser", "password" => "testpass",
+                                                     "rdkafka_options" => '{"security.protocol": "SASL_SSL"}'}, [])
+    d = create_driver(conf)
+
+    config = d.instance.build_config
+
+    assert_equal 'SASL_SSL', config[:"security.protocol"]
+  end
+
+  def test_configure_sasl_plain_with_lowercase_security_protocol_from_rdkafka_options
+    conf = base_config + config_element('ROOT', '', {"username" => "testuser", "password" => "testpass",
+                                                     "rdkafka_options" => '{"security.protocol": "sasl_plaintext"}'}, [])
+
+    assert_raise(Fluent::ConfigError) {
+      create_driver(conf)
+    }
+  end
+
+  data("sha256" => ["sha256", "SCRAM-SHA-256"],
+       "sha512" => ["sha512", "SCRAM-SHA-512"])
+  def test_configure_sasl_scram(data)
+    mechanism, expected = data
+    conf = base_config + config_element('ROOT', '', {"username" => "testuser", "password" => "testpass",
+                                                     "scram_mechanism" => mechanism,
+                                                     "ssl_ca_cert" => "/path/to/ca_cert.pem"}, [])
+    d = create_driver(conf)
+
+    config = d.instance.build_config
+
+    assert_equal expected, config[:"sasl.mechanisms"]
+    assert_equal 'SASL_SSL', config[:"security.protocol"]
+    assert_equal 'testuser', config[:"sasl.username"]
+    assert_equal 'testpass', config[:"sasl.password"]
+  end
+
+  def test_configure_ssl_ca_certs_from_system
+    conf = base_config + config_element('ROOT', '', {"ssl_ca_certs_from_system" => "true"}, [])
+    d = create_driver(conf)
+
+    config = d.instance.build_config
+
+    assert_equal 'SSL', config[:"security.protocol"]
+    assert_nil config[:"ssl.ca.location"]
+  end
+
+  def test_configure_ssl_client_cert_without_ca_cert
+    conf = base_config + config_element('ROOT', '', {"ssl_client_cert" => "/path/to/cert.pem",
+                                                     "ssl_client_cert_key" => "/path/to/key.pem"}, [])
+    d = create_driver(conf)
+
+    config = d.instance.build_config
+
+    assert_equal 'SSL', config[:"security.protocol"]
+    assert_equal '/path/to/cert.pem', config[:"ssl.certificate.location"]
+    assert_equal '/path/to/key.pem', config[:"ssl.key.location"]
+    assert_nil config[:"ssl.ca.location"]
+  end
+
+  def test_configure_sasl_plain_over_ssl_ca_certs_from_system
+    conf = base_config + config_element('ROOT', '', {"username" => "testuser", "password" => "testpass",
+                                                     "ssl_ca_certs_from_system" => "true"}, [])
+    d = create_driver(conf)
+
+    config = d.instance.build_config
+
+    assert_equal 'SASL_SSL', config[:"security.protocol"]
+    assert_equal 'testpass', config[:"sasl.password"]
+  end
+
+  def test_configure_ssl_verify_hostname_default
+    conf = base_config + config_element('ROOT', '', {"ssl_ca_cert" => "/path/to/ca_cert.pem"}, [])
+    d = create_driver(conf)
+
+    config = d.instance.build_config
+
+    assert_equal 'SSL', config[:"security.protocol"]
+    assert_equal 'https', config[:"ssl.endpoint.identification.algorithm"]
+    assert_equal true, config[:"enable.ssl.certificate.verification"]
+  end
+
+  def test_configure_ssl_verify_hostname_false
+    conf = base_config + config_element('ROOT', '', {"ssl_ca_cert" => "/path/to/ca_cert.pem",
+                                                     "ssl_verify_hostname" => "false"}, [])
+    d = create_driver(conf)
+
+    config = d.instance.build_config
+
+    assert_equal 'none', config[:"ssl.endpoint.identification.algorithm"]
+    assert_equal true, config[:"enable.ssl.certificate.verification"]
+  end
+
+  def test_configure_without_ssl_has_no_endpoint_identification
+    d = create_driver
+
+    config = d.instance.build_config
+
+    assert_equal 'PLAINTEXT', config[:"security.protocol"]
+    assert_nil config[:"ssl.endpoint.identification.algorithm"]
+    assert_nil config[:"enable.ssl.certificate.verification"]
+  end
+
+  data("non numeric partition" => "not-a-number",
+       "hexadecimal partition" => "0x10",
+       "float partition" => 3.9,
+       "negative partition" => -2,
+       "out of int32 range" => 2**31)
+  def test_write_skips_event_with_invalid_partition(partition)
+    d = create_driver
+    producer = DummyProducer.new
+    stub(d.instance).get_producer { producer }
+
+    assert_nothing_raised {
+      d.instance.write(create_chunk([{"a" => "b"}, {"a" => "c", "partition" => partition}, {"a" => "d"}]))
+    }
+
+    assert_equal ['{"a":"b"}', '{"a":"d"}'], producer.produced.collect { |message| message[:payload] }
+  end
+
+  def test_write_keeps_event_with_zero_padded_partition
+    d = create_driver
+    producer = DummyProducer.new
+    stub(d.instance).get_producer { producer }
+
+    d.instance.write(create_chunk([{"a" => "b", "partition" => "010"}]))
+
+    assert_equal [10], producer.produced.collect { |message| message[:partition] }
+  end
+
+  def test_write_converts_non_string_message_key
+    d = create_driver
+    producer = DummyProducer.new
+    stub(d.instance).get_producer { producer }
+
+    d.instance.write(create_chunk([{"a" => "b", "message_key" => 12345}]))
+
+    assert_equal ["12345"], producer.produced.collect { |message| message[:key] }
+  end
+
+  def test_rdkafka_error_codes_under_test
+    assert_equal [:unknown_partition, :msg_size_too_large],
+                 [Rdkafka::RdkafkaError.new(RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION).code,
+                  Rdkafka::RdkafkaError.new(RD_KAFKA_RESP_ERR_MSG_SIZE_TOO_LARGE).code]
+  end
+
+  data("produce" => :produce_error,
+       "delivery report" => :delivery_error)
+  def test_write_raises_unrecoverable_error_for_listed_error_code(path)
+    d = create_driver
+    producer = DummyProducer.new(path => Rdkafka::RdkafkaError.new(RD_KAFKA_RESP_ERR_MSG_SIZE_TOO_LARGE))
+    stub(d.instance).get_producer { producer }
+
+    assert_raise(Fluent::UnrecoverableError) {
+      d.instance.write(create_chunk([{"a" => "b"}]))
+    }
+  end
+
+  data("produce" => :produce_error,
+       "delivery report" => :delivery_error)
+  def test_write_reraises_unlisted_error_code(path)
+    d = create_driver
+    producer = DummyProducer.new(path => Rdkafka::RdkafkaError.new(RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION))
+    stub(d.instance).get_producer { producer }
+
+    assert_raise(Rdkafka::RdkafkaError) {
+      d.instance.write(create_chunk([{"a" => "b"}]))
+    }
+  end
+
+  data("produce" => :produce_error,
+       "delivery report" => :delivery_error)
+  def test_write_raises_unrecoverable_error_for_unknown_partition_when_listed(path)
+    d = create_driver(config + config_element('ROOT', '', {"unrecoverable_error_codes" => "unknown_partition"}))
+    producer = DummyProducer.new(path => Rdkafka::RdkafkaError.new(RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION))
+    stub(d.instance).get_producer { producer }
+
+    assert_raise(Fluent::UnrecoverableError) {
+      d.instance.write(create_chunk([{"a" => "b"}]))
+    }
+  end
+
+  data("produce" => :produce_error,
+       "delivery report" => :delivery_error)
+  def test_write_discards_listed_error_code_when_discard_kafka_delivery_failed(path)
+    d = create_driver(config + config_element('ROOT', '', {"discard_kafka_delivery_failed" => "true"}))
+    producer = DummyProducer.new(path => Rdkafka::RdkafkaError.new(RD_KAFKA_RESP_ERR_MSG_SIZE_TOO_LARGE))
+    stub(d.instance).get_producer { producer }
+
+    assert_nothing_raised {
+      d.instance.write(create_chunk([{"a" => "b"}]))
+    }
+    assert_true d.logs.any? { |log| log.include?("Delivery failed. Discard events") }
+  end
+
+  data("produce" => :produce_error,
+       "delivery report" => :delivery_error)
+  def test_write_reraises_error_that_is_not_a_kafka_error(path)
+    d = create_driver
+    producer = DummyProducer.new(path => DummyErrorWithCode.new)
+    stub(d.instance).get_producer { producer }
+
+    assert_raise(DummyErrorWithCode) {
+      d.instance.write(create_chunk([{"a" => "b"}]))
+    }
+  end
+
+  data("produce" => :produce_error,
+       "delivery report" => :delivery_error)
+  def test_write_discards_listed_error_code_matching_discard_kafka_delivery_failed_regex(path)
+    d = create_driver(config + config_element('ROOT', '', {"discard_kafka_delivery_failed_regex" => "/msg_size_too_large/"}))
+    producer = DummyProducer.new(path => Rdkafka::RdkafkaError.new(RD_KAFKA_RESP_ERR_MSG_SIZE_TOO_LARGE))
+    stub(d.instance).get_producer { producer }
+
+    assert_nothing_raised {
+      d.instance.write(create_chunk([{"a" => "b"}]))
+    }
+    assert_true d.logs.any? { |log| log.include?("Delivery failed and matched regexp pattern") }
+  end
+
+  def test_write_produces_every_record_before_raising_unrecoverable_error
+    d = create_driver
+    producer = DummyProducer.new(delivery_error: Rdkafka::RdkafkaError.new(RD_KAFKA_RESP_ERR_MSG_SIZE_TOO_LARGE))
+    stub(d.instance).get_producer { producer }
+
+    assert_raise(Fluent::UnrecoverableError) {
+      d.instance.write(create_chunk([{"a" => "b"}, {"a" => "c"}, {"a" => "d"}]))
+    }
+
+    assert_equal ['{"a":"b"}', '{"a":"c"}', '{"a":"d"}'], producer.produced.collect { |message| message[:payload] }
+  end
+
+  data("produce" => :produce_error,
+       "delivery report" => :delivery_error)
+  def test_write_reraises_listed_error_code_when_unrecoverable_error_codes_is_empty(path)
+    d = create_driver(config + config_element('ROOT', '', {"unrecoverable_error_codes" => ""}))
+    producer = DummyProducer.new(path => Rdkafka::RdkafkaError.new(RD_KAFKA_RESP_ERR_MSG_SIZE_TOO_LARGE))
+    stub(d.instance).get_producer { producer }
+
+    assert_equal [], d.instance.unrecoverable_error_codes
+    assert_raise(Rdkafka::RdkafkaError) {
+      d.instance.write(create_chunk([{"a" => "b"}]))
+    }
+  end
+
   def test_mutli_worker_support
     d = create_driver
     assert_equal true, d.instance.multi_workers_ready?

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.19.6
+  version: 0.19.8
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -187,15 +187,18 @@ files:
 - lib/fluent/plugin/out_rdkafka2.rb
 - lib/fluent/plugin/rdkafka_patch/0_11_0.rb
 - lib/fluent/plugin/rdkafka_patch/0_12_0.rb
+- lib/fluent/plugin/rdkafka_patch/0_13_0.rb
 - lib/fluent/plugin/rdkafka_patch/0_14_0.rb
 - lib/fluent/plugin/rdkafka_patch/0_16_0.rb
 - test/helper.rb
 - test/plugin/test_in_kafka.rb
 - test/plugin/test_in_kafka_group.rb
+- test/plugin/test_in_rdkafka_group.rb
 - test/plugin/test_kafka_plugin_util.rb
 - test/plugin/test_out_kafka.rb
 - test/plugin/test_out_kafka2.rb
 - test/plugin/test_out_kafka_buffered.rb
+- test/plugin/test_out_rdkafka.rb
 - test/plugin/test_out_rdkafka2.rb
 homepage: https://github.com/fluent/fluent-plugin-kafka
 licenses:
@@ -215,15 +218,17 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.6
+rubygems_version: 4.0.18
 specification_version: 4
 summary: Fluentd plugin for Apache Kafka > 0.8
 test_files:
 - test/helper.rb
 - test/plugin/test_in_kafka.rb
 - test/plugin/test_in_kafka_group.rb
+- test/plugin/test_in_rdkafka_group.rb
 - test/plugin/test_kafka_plugin_util.rb
 - test/plugin/test_out_kafka.rb
 - test/plugin/test_out_kafka2.rb
 - test/plugin/test_out_kafka_buffered.rb
+- test/plugin/test_out_rdkafka.rb
 - test/plugin/test_out_rdkafka2.rb