fluent-plugin-kafka 0.19.7 → 0.19.8

data/test/plugin/test_out_rdkafka2.rb

@@ -37,6 +37,80 @@ class Rdkafka2OutputTest < Test::Unit::TestCase
     Fluent::Test::Driver::Output.new(Fluent::Rdkafka2Output).configure(conf)
   end
 
+  class DummyDeliveryHandle
+    def initialize(error = nil)
+      @error = error
+    end
+
+    def wait(**)
+      raise @error if @error
+    end
+  end
+
+  class DummyProducer
+    attr_reader :produced
+
+    def initialize(produce_error: nil, delivery_error: nil)
+      @produced = []
+      @produce_error = produce_error
+      @delivery_error = delivery_error
+    end
+
+    # Raises what the rdkafka FFI binding raises: the partition must be an
+    # int32 and the key must respond to #bytesize
+    def produce(topic:, payload:, key:, partition:, headers:, timestamp:)
+      unless partition.nil?
+        raise TypeError, "no implicit conversion of #{partition.class} into Integer" unless partition.is_a?(Integer)
+        raise RangeError, "integer #{partition} too big to convert to 'int'" unless (-2**31..2**31 - 1).cover?(partition)
+      end
+      key.bytesize unless key.nil?
+      raise @produce_error if @produce_error
+
+      @produced << {payload: payload, key: key, partition: partition}
+      DummyDeliveryHandle.new(@delivery_error)
+    end
+  end
+
+  RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION = -190
+  RD_KAFKA_RESP_ERR_MSG_SIZE_TOO_LARGE = 10
+
+  class DummyErrorWithCode < StandardError
+    def code
+      :msg_size_too_large
+    end
+  end
+
+  def create_chunk(records, tag: 'test')
+    metadata = Fluent::Plugin::Buffer::Metadata.new(nil, tag, nil)
+    chunk = Fluent::Plugin::Buffer::MemoryChunk.new(metadata)
+    chunk.extend(Fluent::ChunkMessagePackEventStreamer)
+    chunk.append(records.map { |record| [event_time, record].to_msgpack })
+    chunk
+  end
+
+  def test_rdkafka_patch_is_applied
+    version = Gem::Version.create(Rdkafka::VERSION)
+    expected = if version >= Gem::Version.create('0.16.0')
+                 '0_16_0.rb'
+               elsif version >= Gem::Version.create('0.14.0')
+                 '0_14_0.rb'
+               elsif version >= Gem::Version.create('0.13.0')
+                 '0_13_0.rb'
+               elsif version >= Gem::Version.create('0.12.0')
+                 '0_12_0.rb'
+               else
+                 '0_11_0.rb'
+               end
+
+    assert_equal expected, File.basename(Rdkafka::Producer.instance_method(:close).source_location.first)
+  end
+
+  def test_producer_close_accepts_timeout
+    producer = Rdkafka::Config.new("bootstrap.servers" => "localhost:9092").producer
+
+    assert_true producer.close(10)
+  end
+
   def test_configure
     assert_nothing_raised(Fluent::ConfigError) {
       create_driver(base_config)
@@ -55,8 +129,30 @@ class Rdkafka2OutputTest < Test::Unit::TestCase
     assert_equal 'localhost:9092', d.instance.brokers
   end
 
-  def test_configure_sasl_plain
+  def test_configure_sasl_plain_over_ssl
+    conf = base_config + config_element('ROOT', '', {"username" => "testuser", "password" => "testpass",
+                                                     "ssl_ca_cert" => "/path/to/ca_cert.pem"}, [])
+    d = create_driver(conf)
+
+    config = d.instance.build_config
+
+    assert_equal 'PLAIN', config[:"sasl.mechanisms"]
+    assert_equal 'SASL_SSL', config[:"security.protocol"]
+    assert_equal 'testuser', config[:"sasl.username"]
+    assert_equal 'testpass', config[:"sasl.password"]
+  end
+
+  def test_configure_sasl_plain_without_ssl
     conf = base_config + config_element('ROOT', '', {"username" => "testuser", "password" => "testpass"}, [])
+
+    assert_raise(Fluent::ConfigError) {
+      create_driver(conf)
+    }
+  end
+
+  def test_configure_sasl_plain_without_ssl_allowed_by_sasl_over_ssl
+    conf = base_config + config_element('ROOT', '', {"username" => "testuser", "password" => "testpass",
+                                                     "sasl_over_ssl" => "false"}, [])
     d = create_driver(conf)
 
     config = d.instance.build_config
@@ -67,6 +163,250 @@ class Rdkafka2OutputTest < Test::Unit::TestCase
     assert_equal 'testpass', config[:"sasl.password"]
   end
 
+  def test_configure_sasl_plain_with_security_protocol_from_rdkafka_options
+    conf = base_config + config_element('ROOT', '', {"username" => "testuser", "password" => "testpass",
+                                                     "rdkafka_options" => '{"security.protocol": "SASL_SSL"}'}, [])
+    d = create_driver(conf)
+
+    config = d.instance.build_config
+
+    assert_equal 'SASL_SSL', config[:"security.protocol"]
+  end
+
+  def test_configure_sasl_plain_with_lowercase_security_protocol_from_rdkafka_options
+    conf = base_config + config_element('ROOT', '', {"username" => "testuser", "password" => "testpass",
+                                                     "rdkafka_options" => '{"security.protocol": "sasl_plaintext"}'}, [])
+
+    assert_raise(Fluent::ConfigError) {
+      create_driver(conf)
+    }
+  end
+
+  data("sha256" => ["sha256", "SCRAM-SHA-256"],
+       "sha512" => ["sha512", "SCRAM-SHA-512"])
+  def test_configure_sasl_scram(data)
+    mechanism, expected = data
+    conf = base_config + config_element('ROOT', '', {"username" => "testuser", "password" => "testpass",
+                                                     "scram_mechanism" => mechanism,
+                                                     "ssl_ca_cert" => "/path/to/ca_cert.pem"}, [])
+    d = create_driver(conf)
+
+    config = d.instance.build_config
+
+    assert_equal expected, config[:"sasl.mechanisms"]
+    assert_equal 'SASL_SSL', config[:"security.protocol"]
+    assert_equal 'testuser', config[:"sasl.username"]
+    assert_equal 'testpass', config[:"sasl.password"]
+  end
+
+  def test_configure_ssl_ca_certs_from_system
+    conf = base_config + config_element('ROOT', '', {"ssl_ca_certs_from_system" => "true"}, [])
+    d = create_driver(conf)
+
+    config = d.instance.build_config
+
+    assert_equal 'SSL', config[:"security.protocol"]
+    assert_nil config[:"ssl.ca.location"]
+  end
+
+  def test_configure_ssl_client_cert_without_ca_cert
+    conf = base_config + config_element('ROOT', '', {"ssl_client_cert" => "/path/to/cert.pem",
+                                                     "ssl_client_cert_key" => "/path/to/key.pem"}, [])
+    d = create_driver(conf)
+
+    config = d.instance.build_config
+
+    assert_equal 'SSL', config[:"security.protocol"]
+    assert_equal '/path/to/cert.pem', config[:"ssl.certificate.location"]
+    assert_equal '/path/to/key.pem', config[:"ssl.key.location"]
+    assert_nil config[:"ssl.ca.location"]
+  end
+
+  def test_configure_sasl_plain_over_ssl_ca_certs_from_system
+    conf = base_config + config_element('ROOT', '', {"username" => "testuser", "password" => "testpass",
+                                                     "ssl_ca_certs_from_system" => "true"}, [])
+    d = create_driver(conf)
+
+    config = d.instance.build_config
+
+    assert_equal 'SASL_SSL', config[:"security.protocol"]
+    assert_equal 'testpass', config[:"sasl.password"]
+  end
+
+  def test_configure_ssl_verify_hostname_default
+    conf = base_config + config_element('ROOT', '', {"ssl_ca_cert" => "/path/to/ca_cert.pem"}, [])
+    d = create_driver(conf)
+
+    config = d.instance.build_config
+
+    assert_equal 'SSL', config[:"security.protocol"]
+    assert_equal 'https', config[:"ssl.endpoint.identification.algorithm"]
+    assert_equal true, config[:"enable.ssl.certificate.verification"]
+  end
+
+  def test_configure_ssl_verify_hostname_false
+    conf = base_config + config_element('ROOT', '', {"ssl_ca_cert" => "/path/to/ca_cert.pem",
+                                                     "ssl_verify_hostname" => "false"}, [])
+    d = create_driver(conf)
+
+    config = d.instance.build_config
+
+    assert_equal 'none', config[:"ssl.endpoint.identification.algorithm"]
+    assert_equal true, config[:"enable.ssl.certificate.verification"]
+  end
+
+  def test_configure_without_ssl_has_no_endpoint_identification
+    d = create_driver
+
+    config = d.instance.build_config
+
+    assert_equal 'PLAINTEXT', config[:"security.protocol"]
+    assert_nil config[:"ssl.endpoint.identification.algorithm"]
+    assert_nil config[:"enable.ssl.certificate.verification"]
+  end
+
+  data("non numeric partition" => "not-a-number",
+       "hexadecimal partition" => "0x10",
+       "float partition" => 3.9,
+       "negative partition" => -2,
+       "out of int32 range" => 2**31)
+  def test_write_skips_event_with_invalid_partition(partition)
+    d = create_driver
+    producer = DummyProducer.new
+    stub(d.instance).get_producer { producer }
+
+    assert_nothing_raised {
+      d.instance.write(create_chunk([{"a" => "b"}, {"a" => "c", "partition" => partition}, {"a" => "d"}]))
+    }
+
+    assert_equal ['{"a":"b"}', '{"a":"d"}'], producer.produced.collect { |message| message[:payload] }
+  end
+
+  def test_write_keeps_event_with_zero_padded_partition
+    d = create_driver
+    producer = DummyProducer.new
+    stub(d.instance).get_producer { producer }
+
+    d.instance.write(create_chunk([{"a" => "b", "partition" => "010"}]))
+
+    assert_equal [10], producer.produced.collect { |message| message[:partition] }
+  end
+
+  def test_write_converts_non_string_message_key
+    d = create_driver
+    producer = DummyProducer.new
+    stub(d.instance).get_producer { producer }
+
+    d.instance.write(create_chunk([{"a" => "b", "message_key" => 12345}]))
+
+    assert_equal ["12345"], producer.produced.collect { |message| message[:key] }
+  end
+
+  def test_rdkafka_error_codes_under_test
+    assert_equal [:unknown_partition, :msg_size_too_large],
+                 [Rdkafka::RdkafkaError.new(RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION).code,
+                  Rdkafka::RdkafkaError.new(RD_KAFKA_RESP_ERR_MSG_SIZE_TOO_LARGE).code]
+  end
+
+  data("produce" => :produce_error,
+       "delivery report" => :delivery_error)
+  def test_write_raises_unrecoverable_error_for_listed_error_code(path)
+    d = create_driver
+    producer = DummyProducer.new(path => Rdkafka::RdkafkaError.new(RD_KAFKA_RESP_ERR_MSG_SIZE_TOO_LARGE))
+    stub(d.instance).get_producer { producer }
+
+    assert_raise(Fluent::UnrecoverableError) {
+      d.instance.write(create_chunk([{"a" => "b"}]))
+    }
+  end
+
+  data("produce" => :produce_error,
+       "delivery report" => :delivery_error)
+  def test_write_reraises_unlisted_error_code(path)
+    d = create_driver
+    producer = DummyProducer.new(path => Rdkafka::RdkafkaError.new(RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION))
+    stub(d.instance).get_producer { producer }
+
+    assert_raise(Rdkafka::RdkafkaError) {
+      d.instance.write(create_chunk([{"a" => "b"}]))
+    }
+  end
+
+  data("produce" => :produce_error,
+       "delivery report" => :delivery_error)
+  def test_write_raises_unrecoverable_error_for_unknown_partition_when_listed(path)
+    d = create_driver(config + config_element('ROOT', '', {"unrecoverable_error_codes" => "unknown_partition"}))
+    producer = DummyProducer.new(path => Rdkafka::RdkafkaError.new(RD_KAFKA_RESP_ERR__UNKNOWN_PARTITION))
+    stub(d.instance).get_producer { producer }
+
+    assert_raise(Fluent::UnrecoverableError) {
+      d.instance.write(create_chunk([{"a" => "b"}]))
+    }
+  end
+
+  data("produce" => :produce_error,
+       "delivery report" => :delivery_error)
+  def test_write_discards_listed_error_code_when_discard_kafka_delivery_failed(path)
+    d = create_driver(config + config_element('ROOT', '', {"discard_kafka_delivery_failed" => "true"}))
+    producer = DummyProducer.new(path => Rdkafka::RdkafkaError.new(RD_KAFKA_RESP_ERR_MSG_SIZE_TOO_LARGE))
+    stub(d.instance).get_producer { producer }
+
+    assert_nothing_raised {
+      d.instance.write(create_chunk([{"a" => "b"}]))
+    }
+    assert_true d.logs.any? { |log| log.include?("Delivery failed. Discard events") }
+  end
+
+  data("produce" => :produce_error,
+       "delivery report" => :delivery_error)
+  def test_write_reraises_error_that_is_not_a_kafka_error(path)
+    d = create_driver
+    producer = DummyProducer.new(path => DummyErrorWithCode.new)
+    stub(d.instance).get_producer { producer }
+
+    assert_raise(DummyErrorWithCode) {
+      d.instance.write(create_chunk([{"a" => "b"}]))
+    }
+  end
+
+  data("produce" => :produce_error,
+       "delivery report" => :delivery_error)
+  def test_write_discards_listed_error_code_matching_discard_kafka_delivery_failed_regex(path)
+    d = create_driver(config + config_element('ROOT', '', {"discard_kafka_delivery_failed_regex" => "/msg_size_too_large/"}))
+    producer = DummyProducer.new(path => Rdkafka::RdkafkaError.new(RD_KAFKA_RESP_ERR_MSG_SIZE_TOO_LARGE))
+    stub(d.instance).get_producer { producer }
+
+    assert_nothing_raised {
+      d.instance.write(create_chunk([{"a" => "b"}]))
+    }
+    assert_true d.logs.any? { |log| log.include?("Delivery failed and matched regexp pattern") }
+  end
+
+  def test_write_produces_every_record_before_raising_unrecoverable_error
+    d = create_driver
+    producer = DummyProducer.new(delivery_error: Rdkafka::RdkafkaError.new(RD_KAFKA_RESP_ERR_MSG_SIZE_TOO_LARGE))
+    stub(d.instance).get_producer { producer }
+
+    assert_raise(Fluent::UnrecoverableError) {
+      d.instance.write(create_chunk([{"a" => "b"}, {"a" => "c"}, {"a" => "d"}]))
+    }
+
+    assert_equal ['{"a":"b"}', '{"a":"c"}', '{"a":"d"}'], producer.produced.collect { |message| message[:payload] }
+  end
+
+  data("produce" => :produce_error,
+       "delivery report" => :delivery_error)
+  def test_write_reraises_listed_error_code_when_unrecoverable_error_codes_is_empty(path)
+    d = create_driver(config + config_element('ROOT', '', {"unrecoverable_error_codes" => ""}))
+    producer = DummyProducer.new(path => Rdkafka::RdkafkaError.new(RD_KAFKA_RESP_ERR_MSG_SIZE_TOO_LARGE))
+    stub(d.instance).get_producer { producer }
+
+    assert_equal [], d.instance.unrecoverable_error_codes
+    assert_raise(Rdkafka::RdkafkaError) {
+      d.instance.write(create_chunk([{"a" => "b"}]))
+    }
+  end
+
   def test_mutli_worker_support
     d = create_driver
     assert_equal true, d.instance.multi_workers_ready?

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.19.7
+  version: 0.19.8
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -187,6 +187,7 @@ files:
 - lib/fluent/plugin/out_rdkafka2.rb
 - lib/fluent/plugin/rdkafka_patch/0_11_0.rb
 - lib/fluent/plugin/rdkafka_patch/0_12_0.rb
+- lib/fluent/plugin/rdkafka_patch/0_13_0.rb
 - lib/fluent/plugin/rdkafka_patch/0_14_0.rb
 - lib/fluent/plugin/rdkafka_patch/0_16_0.rb
 - test/helper.rb
@@ -197,6 +198,7 @@ files:
 - test/plugin/test_out_kafka.rb
 - test/plugin/test_out_kafka2.rb
 - test/plugin/test_out_kafka_buffered.rb
+- test/plugin/test_out_rdkafka.rb
 - test/plugin/test_out_rdkafka2.rb
 homepage: https://github.com/fluent/fluent-plugin-kafka
 licenses:
@@ -216,7 +218,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.10
+rubygems_version: 4.0.18
 specification_version: 4
 summary: Fluentd plugin for Apache Kafka > 0.8
 test_files:
@@ -228,4 +230,5 @@ test_files:
 - test/plugin/test_out_kafka.rb
 - test/plugin/test_out_kafka2.rb
 - test/plugin/test_out_kafka_buffered.rb
+- test/plugin/test_out_rdkafka.rb
 - test/plugin/test_out_rdkafka2.rb