fluent-plugin-kafka 0.19.7 → 0.19.9

data/test/plugin/test_in_rdkafka_group.rb

@@ -44,6 +44,204 @@ class RdkafkaGroupInputTest < Test::Unit::TestCase
     assert_true d.instance.multi_workers_ready?
   end
 
+  def test_build_config_without_security_parameters
+    d = create_driver
+    config = d.instance.build_config
+
+    assert_equal 'localhost:9092', config[:"bootstrap.servers"]
+    assert_equal 'test_group', config[:"group.id"]
+    assert_equal 'PLAINTEXT', config[:"security.protocol"]
+    assert_nil config[:"ssl.endpoint.identification.algorithm"]
+    assert_nil config[:"sasl.mechanisms"]
+  end
+
+  def test_build_config_ssl_ca_certs_from_system
+    d = create_driver(CONFIG + %[
+      ssl_ca_certs_from_system true
+    ])
+    config = d.instance.build_config
+
+    assert_equal 'SSL', config[:"security.protocol"]
+    assert_equal 'https', config[:"ssl.endpoint.identification.algorithm"]
+    assert_equal true, config[:"enable.ssl.certificate.verification"]
+    assert_nil config[:"ssl.ca.location"]
+  end
+
+  def test_build_config_ssl_client_cert
+    d = create_driver(CONFIG + %[
+      ssl_ca_cert /path/to/ca_cert.pem
+      ssl_client_cert /path/to/cert.pem
+      ssl_client_cert_key /path/to/key.pem
+      ssl_client_cert_key_password secret
+    ])
+    config = d.instance.build_config
+
+    assert_equal 'SSL', config[:"security.protocol"]
+    assert_equal '/path/to/ca_cert.pem', config[:"ssl.ca.location"]
+    assert_equal '/path/to/cert.pem', config[:"ssl.certificate.location"]
+    assert_equal '/path/to/key.pem', config[:"ssl.key.location"]
+    assert_equal 'secret', config[:"ssl.key.password"]
+  end
+
+  def test_build_config_ssl_verify_hostname_false
+    d = create_driver(CONFIG + %[
+      ssl_ca_certs_from_system true
+      ssl_verify_hostname false
+    ])
+    config = d.instance.build_config
+
+    assert_equal 'none', config[:"ssl.endpoint.identification.algorithm"]
+    assert_equal true, config[:"enable.ssl.certificate.verification"]
+  end
+
+  def test_build_config_sasl_plain_over_ssl
+    d = create_driver(CONFIG + %[
+      username testuser
+      password testpass
+      ssl_ca_certs_from_system true
+    ])
+    config = d.instance.build_config
+
+    assert_equal 'SASL_SSL', config[:"security.protocol"]
+    assert_equal 'PLAIN', config[:"sasl.mechanisms"]
+    assert_equal 'testuser', config[:"sasl.username"]
+    assert_equal 'testpass', config[:"sasl.password"]
+  end
+
+  def test_configure_sasl_plain_without_ssl_raises
+    assert_raise(Fluent::ConfigError) {
+      create_driver(CONFIG + %[
+        username testuser
+        password testpass
+      ])
+    }
+  end
+
+  def test_build_config_sasl_plain_without_ssl_allowed_by_sasl_over_ssl
+    d = create_driver(CONFIG + %[
+      username testuser
+      password testpass
+      sasl_over_ssl false
+    ])
+    config = d.instance.build_config
+
+    assert_equal 'SASL_PLAINTEXT', config[:"security.protocol"]
+    assert_equal 'testpass', config[:"sasl.password"]
+  end
+
+  data("sha256" => ["sha256", "SCRAM-SHA-256"],
+       "sha512" => ["sha512", "SCRAM-SHA-512"])
+  def test_build_config_sasl_scram(data)
+    mechanism, expected = data
+    d = create_driver(CONFIG + %[
+      username testuser
+      password testpass
+      scram_mechanism #{mechanism}
+      ssl_ca_certs_from_system true
+    ])
+    config = d.instance.build_config
+
+    assert_equal expected, config[:"sasl.mechanisms"]
+    assert_equal 'SASL_SSL', config[:"security.protocol"]
+  end
+
+  def test_build_config_sasl_scram_without_credentials_warns
+    d = create_driver(CONFIG + %[
+      scram_mechanism sha256
+      ssl_ca_certs_from_system true
+    ])
+    config = d.instance.build_config
+
+    assert_equal 'SSL', config[:"security.protocol"]
+    assert_nil config[:"sasl.mechanisms"]
+    assert_true d.logs.any? { |log| log.include?("scram_mechanism is ignored") }
+  end
+
+  def test_build_config_sasl_gssapi
+    d = create_driver(CONFIG + %[
+      principal kafka/host@REALM
+      keytab /path/to/kafka.keytab
+      service_name kafka
+      ssl_ca_certs_from_system true
+    ])
+    config = d.instance.build_config
+
+    assert_equal 'SASL_SSL', config[:"security.protocol"]
+    assert_equal 'GSSAPI', config[:"sasl.mechanisms"]
+    assert_equal 'kafka/host@REALM', config[:"sasl.kerberos.principal"]
+    assert_equal '/path/to/kafka.keytab', config[:"sasl.kerberos.keytab"]
+    assert_equal 'kafka', config[:"sasl.kerberos.service.name"]
+  end
+
+  def test_build_config_kafka_configs_take_precedence
+    conf = %[
+      topics #{TOPIC_NAME}
+      ssl_ca_certs_from_system true
+      kafka_configs {"bootstrap.servers": "localhost:9092", "group.id": "test_group", "security.protocol": "SASL_SSL", "ssl.endpoint.identification.algorithm": "none"}
+      <parse>
+        @type none
+      </parse>
+    ]
+    d = create_driver(conf)
+    config = d.instance.build_config
+
+    assert_equal 'SASL_SSL', config[:"security.protocol"]
+    assert_equal 'none', config[:"ssl.endpoint.identification.algorithm"]
+    assert_nil config["security.protocol"]
+  end
+
+  def test_configure_sasl_plaintext_in_kafka_configs_raises
+    conf = %[
+      topics #{TOPIC_NAME}
+      kafka_configs {"bootstrap.servers": "localhost:9092", "group.id": "test_group", "security.protocol": "SASL_PLAINTEXT", "sasl.mechanisms": "PLAIN", "sasl.username": "testuser", "sasl.password": "testpass"}
+      <parse>
+        @type none
+      </parse>
+    ]
+
+    assert_raise(Fluent::ConfigError) {
+      create_driver(conf)
+    }
+    assert_nothing_raised {
+      create_driver(conf + "sasl_over_ssl false\n")
+    }
+  end
+
+  data("uppercase" => "SASL_PLAINTEXT",
+       "lowercase" => "sasl_plaintext")
+  def test_configure_sasl_plaintext_in_kafka_configs_raises_regardless_of_case(protocol)
+    conf = %[
+      topics #{TOPIC_NAME}
+      username testuser
+      password testpass
+      kafka_configs {"bootstrap.servers": "localhost:9092", "group.id": "test_group", "security.protocol": "#{protocol}"}
+      <parse>
+        @type none
+      </parse>
+    ]
+
+    assert_raise(Fluent::ConfigError) {
+      create_driver(conf)
+    }
+  end
+
+  def test_setup_consumer_uses_build_config
+    d = create_driver(CONFIG + %[
+      ssl_ca_certs_from_system true
+    ])
+    consumer = Object.new
+    stub(consumer).subscribe
+    rdkafka_config = Object.new
+    stub(rdkafka_config).consumer { consumer }
+    passed = nil
+    stub(Rdkafka::Config).new { |config| passed = config; rdkafka_config }
+
+    d.instance.setup_consumer
+
+    assert_equal 'localhost:9092', passed[:"bootstrap.servers"]
+    assert_equal 'SSL', passed[:"security.protocol"]
+  end
+
   class ConsumeTest < self
     TOPIC_NAME = "kafka-input-#{SecureRandom.uuid}"
 

data/test/plugin/test_kafka_plugin_util.rb

@@ -6,6 +6,7 @@ class KafkaPluginUtilTest < Test::Unit::TestCase
     def self.config_param(name, type, options)
     end
     include Fluent::KafkaPluginUtil::SSLSettings
+    include Fluent::KafkaPluginUtil::PartitionSettings
 
     def config_param
     end
@@ -13,6 +14,31 @@ class KafkaPluginUtilTest < Test::Unit::TestCase
         Fluent::Test.setup
     end
 
+    data("integer" => [3, 3],
+         "decimal string" => ["3", 3],
+         "zero padded string" => ["010", 10],
+         "unassigned partition" => [-1, -1],
+         "max int32" => [2**31 - 1, 2**31 - 1])
+    def test_coerce_partition(data)
+      given, expected = data
+      assert_equal(expected, coerce_partition(given))
+    end
+
+    data("non numeric string" => ["not-a-number", ArgumentError],
+         "empty string" => ["", ArgumentError],
+         "hexadecimal string" => ["0x10", ArgumentError],
+         "float string" => ["3.5", ArgumentError],
+         "float" => [3.9, TypeError],
+         "nil" => [nil, TypeError],
+         "negative partition" => [-2, RangeError],
+         "too big for int32" => [2**31, RangeError])
+    def test_coerce_partition_rejects_invalid_value(data)
+      given, expected = data
+      assert_raise(expected) do
+        coerce_partition(given)
+      end
+    end
+
     def test_read_ssl_file_when_nil
       stub(File).read(anything) do |path|
         path

data/test/plugin/test_out_kafka2.rb

@@ -27,6 +27,32 @@ class Kafka2OutputTest < Test::Unit::TestCase
     Fluent::Test::Driver::Output.new(Fluent::Kafka2Output).configure(conf)
   end
 
+  class DummyProducer
+    attr_reader :produced
+
+    # Coerces like ruby-kafka's Producer#produce, which is where an invalid
+    # partition raises today
+    def produce(value, key:, partition_key:, partition:, headers:, create_time:, topic:)
+      @produced ||= []
+      @produced << {value: value && value.to_s, key: key && key.to_s,
+                    partition: partition && Integer(partition)}
+    end
+
+    def deliver_messages
+    end
+
+    def clear_buffer
+    end
+  end
+
+  def create_chunk(records, tag: 'test')
+    metadata = Fluent::Plugin::Buffer::Metadata.new(nil, tag, nil)
+    chunk = Fluent::Plugin::Buffer::MemoryChunk.new(metadata)
+    chunk.extend(Fluent::ChunkMessagePackEventStreamer)
+    chunk.append(records.map { |record| [event_time, record].to_msgpack })
+    chunk
+  end
+
   def test_configure
     assert_nothing_raised(Fluent::ConfigError) {
       create_driver(base_config)
@@ -45,6 +71,152 @@ class Kafka2OutputTest < Test::Unit::TestCase
     assert_equal ['localhost:9092'], d.instance.brokers
   end
 
+  data("cert without key" => {"ssl_client_cert" => "/path/to/cert.pem"},
+       "key without cert" => {"ssl_client_cert_key" => "/path/to/key.pem"},
+       "chain without cert" => {"ssl_client_cert_chain" => "/path/to/chain.pem"},
+       "key password without key" => {"ssl_client_cert_key_password" => "secret"})
+  def test_configure_incomplete_ssl_client_cert(params)
+    conf = config + config_element('ROOT', '', params, [])
+
+    assert_raise(Fluent::ConfigError) {
+      create_driver(conf)
+    }
+  end
+
+  def test_configure_ssl_client_cert_with_key
+    conf = config + config_element('ROOT', '', {"ssl_client_cert" => "/path/to/cert.pem",
+                                                "ssl_client_cert_key" => "/path/to/key.pem"}, [])
+
+    assert_nothing_raised(Fluent::ConfigError) {
+      create_driver(conf)
+    }
+  end
+
+  data("sha256" => "sha256",
+       "sha512" => "sha512")
+  def test_configure_scram_mechanism(mechanism)
+    conf = config + config_element('ROOT', '', {"username" => "testuser",
+                                                "password" => "testpass",
+                                                "scram_mechanism" => mechanism,
+                                                "ssl_ca_certs_from_system" => "true"}, [])
+    d = create_driver(conf)
+
+    assert_equal mechanism, d.instance.scram_mechanism
+
+    assert_nothing_raised {
+      d.instance.refresh_client
+    }
+  end
+
+  def test_configure_unsupported_scram_mechanism
+    conf = config + config_element('ROOT', '', {"username" => "testuser",
+                                                "password" => "testpass",
+                                                "scram_mechanism" => "sha1"}, [])
+
+    assert_raise(Fluent::ConfigError) {
+      create_driver(conf)
+    }
+  end
+
+  data("non numeric partition" => "not-a-number",
+       "hash partition" => {"x" => 1},
+       "negative partition" => -2,
+       "out of int32 range" => 2**31)
+  def test_write_skips_event_with_invalid_partition(partition)
+    d = create_driver
+    producer = DummyProducer.new
+    stub(d.instance).get_producer { producer }
+
+    assert_nothing_raised {
+      d.instance.write(create_chunk([{"a" => "b"}, {"a" => "c", "partition" => partition}, {"a" => "d"}]))
+    }
+
+    assert_equal ['{"a":"b"}', '{"a":"d"}'], producer.produced.collect { |message| message[:value] }
+  end
+
+  def test_write_keeps_event_with_zero_padded_partition
+    d = create_driver
+    producer = DummyProducer.new
+    stub(d.instance).get_producer { producer }
+
+    d.instance.write(create_chunk([{"a" => "b", "partition" => "010"}]))
+
+    assert_equal [10], producer.produced.collect { |message| message[:partition] }
+  end
+
+  data("per thread producer" => false,
+       "shared producer" => true)
+  def test_write_failure_replaces_producer(share_producer)
+    d = create_driver(base_config + config_element('ROOT', '', {"default_topic" => "kitagawakeiko",
+                                                                "brokers" => "localhost:1",
+                                                                "share_producer" => share_producer}))
+    d.instance_start
+    producer = d.instance.get_producer
+    mock.proxy(producer).shutdown
+
+    assert_raise(Kafka::ConnectionError) {
+      d.instance.write(create_chunk([{"a" => "b"}]))
+    }
+
+    assert_not_same producer, d.instance.get_producer
+  ensure
+    d.instance_shutdown if d
+  end
+
+  data("per thread producer" => false,
+       "shared producer" => true)
+  def test_write_failure_keeps_error_when_producer_shutdown_fails(share_producer)
+    d = create_driver(base_config + config_element('ROOT', '', {"default_topic" => "kitagawakeiko",
+                                                                "brokers" => "localhost:1",
+                                                                "share_producer" => share_producer}))
+    d.instance_start
+    producer = d.instance.get_producer
+    stub(producer).shutdown { raise IOError }
+
+    assert_raise(Kafka::ConnectionError) {
+      d.instance.write(create_chunk([{"a" => "b"}]))
+    }
+  ensure
+    d.instance_shutdown if d
+  end
+
+  class SlowProducer < DummyProducer
+    attr_reader :max_concurrent_deliveries
+
+    def initialize
+      @mutex = Mutex.new
+      @deliveries = 0
+      @max_concurrent_deliveries = 0
+    end
+
+    def deliver_messages
+      @mutex.synchronize {
+        @deliveries += 1
+        @max_concurrent_deliveries = [@max_concurrent_deliveries, @deliveries].max
+      }
+      sleep 0.1
+      @mutex.synchronize { @deliveries -= 1 }
+    end
+
+    def shutdown
+    end
+  end
+
+  def test_shared_producer_delivers_one_chunk_at_a_time
+    d = create_driver(config + config_element('ROOT', '', {"share_producer" => true}))
+    producer = SlowProducer.new
+    stub(d.instance).create_producer { producer }
+    d.instance_start
+
+    2.times.map {
+      Thread.new { d.instance.write(create_chunk([{"a" => "b"}])) }
+    }.each(&:join)
+
+    assert_equal 1, producer.max_concurrent_deliveries
+  ensure
+    d.instance_shutdown if d
+  end
+
   data("crc32" => "crc32",
        "murmur2" => "murmur2")
   def test_partitioner_hash_function(data)

data/test/plugin/test_out_rdkafka.rb

@@ -0,0 +1,105 @@
+require 'helper'
+require 'fluent/test/helpers'
+require 'fluent/test/driver/output'
+
+class RdkafkaOutputTest < Test::Unit::TestCase
+  include Fluent::Test::Helpers
+
+  def have_rdkafka
+    begin
+      require 'fluent/plugin/out_rdkafka'
+      true
+    rescue LoadError
+      false
+    end
+  end
+
+  def setup
+    omit_unless(have_rdkafka, "rdkafka isn't installed")
+    Fluent::Test.setup
+  end
+
+  def base_config(params = {})
+    config_element('ROOT', '', {"@type" => "rdkafka",
+                                "brokers" => "localhost:9092"}.merge(params), [])
+  end
+
+  def create_driver(conf = base_config)
+    Fluent::Test::Driver::Output.new(Fluent::KafkaOutputBuffered2).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+
+    assert_equal 'localhost:9092', d.instance.brokers
+  end
+
+  def test_configure_ssl_ca_cert
+    d = create_driver(base_config("ssl_ca_cert" => "/path/to/ca_cert.pem"))
+
+    config = d.instance.build_config
+
+    assert_equal 'SSL', config[:"security.protocol"]
+    assert_equal '/path/to/ca_cert.pem', config[:"ssl.ca.location"]
+  end
+
+  def test_configure_ssl_ca_certs_from_system
+    d = create_driver(base_config("ssl_ca_certs_from_system" => "true"))
+
+    config = d.instance.build_config
+
+    assert_equal 'SSL', config[:"security.protocol"]
+    assert_nil config[:"ssl.ca.location"]
+  end
+
+  def test_configure_ssl_client_cert_without_ca_cert
+    d = create_driver(base_config("ssl_client_cert" => "/path/to/cert.pem",
+                                  "ssl_client_cert_key" => "/path/to/key.pem"))
+
+    config = d.instance.build_config
+
+    assert_equal 'SSL', config[:"security.protocol"]
+    assert_equal '/path/to/cert.pem', config[:"ssl.certificate.location"]
+    assert_equal '/path/to/key.pem', config[:"ssl.key.location"]
+    assert_nil config[:"ssl.ca.location"]
+  end
+
+  def test_configure_ssl_verify_hostname_default
+    d = create_driver(base_config("ssl_ca_cert" => "/path/to/ca_cert.pem"))
+
+    config = d.instance.build_config
+
+    assert_equal 'SSL', config[:"security.protocol"]
+    assert_equal 'https', config[:"ssl.endpoint.identification.algorithm"]
+    assert_equal true, config[:"enable.ssl.certificate.verification"]
+  end
+
+  def test_configure_ssl_verify_hostname_false
+    d = create_driver(base_config("ssl_ca_cert" => "/path/to/ca_cert.pem",
+                                  "ssl_verify_hostname" => "false"))
+
+    config = d.instance.build_config
+
+    assert_equal 'none', config[:"ssl.endpoint.identification.algorithm"]
+    assert_equal true, config[:"enable.ssl.certificate.verification"]
+  end
+
+  def test_configure_without_ssl_has_no_endpoint_identification
+    config = create_driver.instance.build_config
+
+    assert_equal 'PLAINTEXT', config[:"security.protocol"]
+    assert_nil config[:"ssl.endpoint.identification.algorithm"]
+    assert_nil config[:"enable.ssl.certificate.verification"]
+  end
+
+  def test_configure_sasl_gssapi_over_ssl
+    d = create_driver(base_config("principal" => "testuser@EXAMPLE.COM",
+                                  "ssl_client_cert" => "/path/to/cert.pem",
+                                  "ssl_client_cert_key" => "/path/to/key.pem"))
+
+    config = d.instance.build_config
+
+    assert_equal 'SASL_SSL', config[:"security.protocol"]
+    assert_equal 'GSSAPI', config[:"sasl.mechanisms"]
+  end
+end