fluent-plugin-kafka 0.19.6 → 0.19.8

data/lib/fluent/plugin/out_rdkafka.rb

@@ -130,12 +130,14 @@ DESC
       :"bootstrap.servers" => @brokers,
     }
 
-    if @ssl_ca_cert && @ssl_ca_cert[0]
+    if (@ssl_ca_cert && @ssl_ca_cert[0]) || @ssl_ca_certs_from_system || @ssl_client_cert || @ssl_client_cert_key
       ssl = true
-      config[:"ssl.ca.location"] = @ssl_ca_cert[0]
+      config[:"ssl.ca.location"] = @ssl_ca_cert[0] if @ssl_ca_cert && @ssl_ca_cert[0]
       config[:"ssl.certificate.location"] = @ssl_client_cert if @ssl_client_cert
       config[:"ssl.key.location"] = @ssl_client_cert_key if @ssl_client_cert_key
       config[:"ssl.key.password"] = @ssl_client_cert_key_password if @ssl_client_cert_key_password
+      config[:"ssl.endpoint.identification.algorithm"] = @ssl_verify_hostname ? "https" : "none"
+      config[:"enable.ssl.certificate.verification"] = true
     end
 
     if @principal
@@ -228,8 +230,8 @@ DESC
         Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
         Proc.new { |tag, time, record| Oj.dump(record) }
       rescue LoadError
-        require 'yajl'
-        Proc.new { |tag, time, record| Yajl::Encoder.encode(record) }
+        require 'json'
+        Proc.new { |tag, time, record| JSON.generate(record) }
       end
     elsif @output_data_type == 'ltsv'
       require 'ltsv'

data/lib/fluent/plugin/out_rdkafka2.rb

@@ -7,14 +7,16 @@ require 'rdkafka'
 
 begin
   rdkafka_version = Gem::Version::create(Rdkafka::VERSION)
-  if rdkafka_version < Gem::Version.create('0.12.0')
-    require_relative 'rdkafka_patch/0_11_0'
-  elsif rdkafka_version == Gem::Version.create('0.12.0')
-    require_relative 'rdkafka_patch/0_12_0'
+  if rdkafka_version >= Gem::Version.create('0.16.0')
+    require_relative 'rdkafka_patch/0_16_0'
   elsif rdkafka_version >= Gem::Version.create('0.14.0')
     require_relative 'rdkafka_patch/0_14_0'
-  elsif rdkafka_version >= Gem::Version.create('0.16.0')
-    require_relative 'rdkafka_patch/0_16_0'
+  elsif rdkafka_version >= Gem::Version.create('0.13.0')
+    require_relative 'rdkafka_patch/0_13_0'
+  elsif rdkafka_version >= Gem::Version.create('0.12.0')
+    require_relative 'rdkafka_patch/0_12_0'
+  else
+    require_relative 'rdkafka_patch/0_11_0'
   end
 rescue LoadError, NameError
   raise "unable to patch rdkafka."
@@ -97,7 +99,6 @@ DESC
     config_param :enqueue_retry_backoff, :integer, :default => 3
     config_param :max_enqueue_bytes_per_second, :size, :default => nil, :desc => 'The maximum number of enqueueing bytes per second'
 
-    config_param :service_name, :string, :default => nil, :desc => 'Used for sasl.kerberos.service.name'
     config_param :unrecoverable_error_codes, :array, :default => ["topic_authorization_failed", "msg_size_too_large"],
                  :desc => 'Handle some of the error codes should be unrecoverable if specified'
 
@@ -111,6 +112,8 @@ DESC
 
     include Fluent::KafkaPluginUtil::SSLSettings
     include Fluent::KafkaPluginUtil::SaslSettings
+    include Fluent::KafkaPluginUtil::PartitionSettings
+    include Fluent::KafkaPluginUtil::RdkafkaSecuritySettings
 
     class EnqueueRate
       class LimitExceeded < StandardError
@@ -250,39 +253,7 @@ DESC
     end
 
     def build_config
-      config = {:"bootstrap.servers" => @brokers}
-
-      if @ssl_ca_cert && @ssl_ca_cert[0]
-        ssl = true
-        config[:"ssl.ca.location"] = @ssl_ca_cert[0]
-        config[:"ssl.certificate.location"] = @ssl_client_cert if @ssl_client_cert
-        config[:"ssl.key.location"] = @ssl_client_cert_key if @ssl_client_cert_key
-        config[:"ssl.key.password"] = @ssl_client_cert_key_password if @ssl_client_cert_key_password
-      end
-
-      if @principal
-        sasl = true
-        config[:"sasl.mechanisms"] = "GSSAPI"
-        config[:"sasl.kerberos.principal"] = @principal
-        config[:"sasl.kerberos.service.name"] = @service_name if @service_name
-        config[:"sasl.kerberos.keytab"] = @keytab if @keytab
-      end
-
-      if @username && @password
-        sasl = true
-        config[:"sasl.mechanisms"] = 'PLAIN'
-      end
-
-      if ssl && sasl
-        security_protocol = "SASL_SSL"
-      elsif ssl && !sasl
-        security_protocol = "SSL"
-      elsif !ssl && sasl
-        security_protocol = "SASL_PLAINTEXT"
-      else
-        security_protocol = "PLAINTEXT"
-      end
-      config[:"security.protocol"] = security_protocol
+      config = {:"bootstrap.servers" => @brokers}.merge(rdkafka_security_config)
 
       config[:"compression.codec"] = @compression_codec if @compression_codec
       config[:"message.send.max.retries"] = @max_send_retries if @max_send_retries
@@ -292,14 +263,14 @@ DESC
       config[:"queue.buffering.max.messages"] = @rdkafka_buffering_max_messages if @rdkafka_buffering_max_messages
       config[:"message.max.bytes"] = @rdkafka_message_max_bytes if @rdkafka_message_max_bytes
       config[:"batch.num.messages"] = @rdkafka_message_max_num if @rdkafka_message_max_num
-      config[:"sasl.username"] = @username if @username
-      config[:"sasl.password"] = @password if @password
       config[:"enable.idempotence"] = @idempotent if @idempotent
 
       @rdkafka_options.each { |k, v|
         config[k.to_sym] = v
       }
 
+      validate_sasl_over_ssl(config)
+
       config
     end
 
@@ -406,6 +377,8 @@ DESC
             record.delete(@topic_key) if @exclude_topic_key
             partition = (@exclude_partition ? record.delete(@partition_key) : record[@partition_key]) || @default_partition
             message_key = (@exclude_message_key ? record.delete(@message_key_key) : record[@message_key_key]) || @default_message_key
+            partition = coerce_partition(partition) unless partition.nil?
+            message_key = message_key.to_s unless message_key.nil?
 
             @headers_from_record_accessors.each do |key, header_accessor|
               headers[key] = header_accessor.call(record)
@@ -451,6 +424,7 @@ DESC
           log.warn "Delivery failed and matched regexp pattern #{@discard_kafka_delivery_failed_regex}. Discard events:", :error => e.to_s, :error_class => e.class.to_s, :tag => tag
 	else
 	  log.warn "Send exception occurred: #{e} at #{e.backtrace.first}"
+          raise Fluent::UnrecoverableError, "Rejected due to #{e}" if unrecoverable_error?(e)
           # Raise exception to retry sendind messages
           raise e
 	end
@@ -507,15 +481,14 @@ DESC
 
             raise e
           else
-            if unrecoverable_error_codes.include?(e.code.to_s)
-              # some of the errors should be handled as an unrecoverable error
-              raise Fluent::UnrecoverableError, "Rejected due to #{e}"
-            else
-              raise e
-            end
+            raise e
           end
         end
       end
     end
+
+    def unrecoverable_error?(e)
+      e.is_a?(Rdkafka::RdkafkaError) && unrecoverable_error_codes.include?(e.code.to_s)
+    end
   end
 end

data/lib/fluent/plugin/rdkafka_patch/0_13_0.rb

@@ -0,0 +1,43 @@
+class Rdkafka::NativeKafka
+  # return false if producer is forcefully closed, otherwise return true
+  def close(timeout=nil, object_id=nil)
+    return true if closed?
+
+    synchronize do
+      # Indicate to the outside world that we are closing
+      @closing = true
+
+      thread_status = :unknown
+      if @polling_thread
+        # Indicate to polling thread that we're closing
+        @polling_thread[:closing] = true
+
+        # Wait for the polling thread to finish up,
+        # this can be aborted in practice if this
+        # code runs from a finalizer.
+        thread_status = @polling_thread.join(timeout)
+      end
+
+      # Destroy the client after locking both mutexes
+      @poll_mutex.lock
+
+      # This check prevents a race condition, where we would enter the close in two threads
+      # and after unlocking the primary one that hold the lock but finished, ours would be unlocked
+      # and would continue to run, trying to destroy inner twice
+      if @inner
+        Rdkafka::Bindings.rd_kafka_destroy(@inner)
+        @inner = nil
+      end
+
+      !thread_status.nil?
+    end
+  end
+end
+
+class Rdkafka::Producer
+  def close(timeout = nil)
+    return true if closed?
+    ObjectSpace.undefine_finalizer(self)
+    @native_kafka.close(timeout)
+  end
+end

data/lib/fluent/plugin/rdkafka_patch/0_16_0.rb

@@ -24,7 +24,7 @@ class Rdkafka::NativeKafka
       # This check prevents a race condition, where we would enter the close in two threads
       # and after unlocking the primary one that hold the lock but finished, ours would be unlocked
       # and would continue to run, trying to destroy inner twice
-      retun unless @inner
+      return unless @inner
 
       Rdkafka::Bindings.rd_kafka_destroy(@inner)
       @inner = nil
@@ -40,7 +40,7 @@ class Rdkafka::Producer
     return true if closed?
     ObjectSpace.undefine_finalizer(self)
 
-    @native_kafka.close(timeout) do
+    closed = @native_kafka.close(timeout) do
       # We need to remove the topics references objects before we destroy the producer,
       # otherwise they would leak out
       @topics_refs_map.each_value do |refs|
@@ -51,5 +51,6 @@ class Rdkafka::Producer
     end
 
     @topics_refs_map.clear
+    closed
   end
 end

data/test/plugin/test_in_kafka.rb

@@ -1,6 +1,7 @@
 require 'helper'
 require 'fluent/test/driver/input'
 require 'securerandom'
+require 'json'
 
 class KafkaInputTest < Test::Unit::TestCase
   def setup
@@ -34,6 +35,132 @@ class KafkaInputTest < Test::Unit::TestCase
     assert_false d.instance.multi_workers_ready?
   end
 
+  class TopicWatcherTest < self
+    FakeMessage = Struct.new(:value, :key, :offset, :create_time)
+
+    class FakeRouter
+      attr_reader :emitted
+
+      def initialize
+        @emitted = []
+      end
+
+      def emit_stream(tag, es)
+        records = []
+        es.each { |time, record| records << record }
+        @emitted << [tag, records]
+      end
+    end
+
+    def create_topic_watcher(messages, router, tag_source: :record, add_prefix: nil, add_suffix: nil)
+      kafka = Object.new
+      kafka.define_singleton_method(:fetch_messages) { |**args| messages }
+      parser = Proc.new { |msg, te| JSON.parse(msg.value) }
+
+      Fluent::KafkaInput::TopicWatcher.new(
+        Fluent::KafkaInput::TopicEntry.new(TOPIC_NAME, 0, 0),
+        kafka,
+        1,
+        parser,
+        add_prefix,
+        add_suffix,
+        nil,
+        router,
+        nil,
+        :now,
+        'time',
+        tag_source,
+        'tag')
+    end
+
+    def tagged_message(tag, message, offset)
+      FakeMessage.new({'tag' => tag, 'message' => message}.to_json, nil, offset, Time.now)
+    end
+
+    def test_consume_with_tag_source_record_emits_stream_per_tag
+      messages = [
+        tagged_message('app.trusted', 'record 1', 0),
+        tagged_message('app.trusted', 'record 2', 1),
+        tagged_message('attacker.controlled', 'record 3', 2),
+      ]
+      router = FakeRouter.new
+      create_topic_watcher(messages, router).consume
+
+      assert_equal([['app.trusted', ['record 1', 'record 2']],
+                    ['attacker.controlled', ['record 3']]],
+                   router.emitted.map { |tag, records| [tag, records.map { |r| r['message'] }] })
+    end
+
+    def test_consume_with_tag_source_record_applies_prefix_and_suffix_per_tag
+      messages = [
+        tagged_message('app.trusted', 'record 1', 0),
+        tagged_message('attacker.controlled', 'record 2', 1),
+      ]
+      router = FakeRouter.new
+      create_topic_watcher(messages, router, add_prefix: 'prefix', add_suffix: 'suffix').consume
+
+      assert_equal(['prefix.app.trusted.suffix', 'prefix.attacker.controlled.suffix'],
+                   router.emitted.map(&:first))
+    end
+
+    def test_consume_with_tag_source_record_skips_unparsable_message
+      messages = [
+        tagged_message('app.trusted', 'record 1', 0),
+        FakeMessage.new('this is not json', nil, 1, Time.now),
+        tagged_message('attacker.controlled', 'record 2', 2),
+      ]
+      router = FakeRouter.new
+      create_topic_watcher(messages, router).consume
+
+      assert_equal([['app.trusted', ['record 1']],
+                    ['attacker.controlled', ['record 2']]],
+                   router.emitted.map { |tag, records| [tag, records.map { |r| r['message'] }] })
+    end
+
+    def test_consume_with_tag_source_record_skips_invalid_tag
+      messages = [
+        tagged_message('app.trusted', 'record 1', 0),
+        tagged_message(7, 'record 2', 1),
+        FakeMessage.new({'message' => 'record 3'}.to_json, nil, 2, Time.now),
+        tagged_message('', 'record 4', 3),
+        tagged_message('attacker.controlled', 'record 5', 4),
+      ]
+      router = FakeRouter.new
+      watcher = create_topic_watcher(messages, router)
+      watcher.consume
+
+      assert_equal([['app.trusted', ['record 1']],
+                    ['attacker.controlled', ['record 5']]],
+                   router.emitted.map { |tag, records| [tag, records.map { |r| r['message'] }] })
+      assert_equal(5, watcher.instance_variable_get(:@next_offset))
+    end
+
+    def test_consume_advances_offset_when_every_message_is_skipped
+      messages = [
+        FakeMessage.new('this is not json', nil, 0, Time.now),
+        tagged_message(nil, 'record 1', 1),
+      ]
+      router = FakeRouter.new
+      watcher = create_topic_watcher(messages, router)
+      watcher.consume
+
+      assert_equal([], router.emitted)
+      assert_equal(2, watcher.instance_variable_get(:@next_offset))
+    end
+
+    def test_consume_with_tag_source_topic_emits_single_stream
+      messages = [
+        tagged_message('app.trusted', 'record 1', 0),
+        tagged_message('attacker.controlled', 'record 2', 1),
+      ]
+      router = FakeRouter.new
+      create_topic_watcher(messages, router, tag_source: :topic).consume
+
+      assert_equal([[TOPIC_NAME, ['record 1', 'record 2']]],
+                   router.emitted.map { |tag, records| [tag, records.map { |r| r['message'] }] })
+    end
+  end
+
   class ConsumeTest < self
     def setup
       @kafka = Kafka.new(["localhost:9092"], client_id: 'kafka')

data/test/plugin/test_in_rdkafka_group.rb

@@ -0,0 +1,326 @@
+require 'helper'
+require 'fluent/test/driver/input'
+require 'securerandom'
+
+class RdkafkaGroupInputTest < Test::Unit::TestCase
+
+  def have_rdkafka
+    begin
+      require 'fluent/plugin/in_rdkafka_group'
+      true
+    rescue LoadError
+      false
+    end
+  end
+
+  def setup
+    omit_unless(have_rdkafka, "rdkafka isn't installed")
+    Fluent::Test.setup
+  end
+
+  TOPIC_NAME = "kafka-input-#{SecureRandom.uuid}"
+
+  CONFIG = %[
+    topics #{TOPIC_NAME}
+    kafka_configs {"bootstrap.servers": "localhost:9092", "group.id": "test_group"}
+    <parse>
+      @type none
+    </parse>
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::RdKafkaGroupInput).configure(conf)
+  end
+
+
+  def test_configure
+    d = create_driver
+    assert_equal [TOPIC_NAME], d.instance.topics
+    assert_equal 'localhost:9092', d.instance.kafka_configs['bootstrap.servers']
+  end
+
+  def test_multi_worker_support
+    d = create_driver
+    assert_true d.instance.multi_workers_ready?
+  end
+
+  def test_build_config_without_security_parameters
+    d = create_driver
+    config = d.instance.build_config
+
+    assert_equal 'localhost:9092', config[:"bootstrap.servers"]
+    assert_equal 'test_group', config[:"group.id"]
+    assert_equal 'PLAINTEXT', config[:"security.protocol"]
+    assert_nil config[:"ssl.endpoint.identification.algorithm"]
+    assert_nil config[:"sasl.mechanisms"]
+  end
+
+  def test_build_config_ssl_ca_certs_from_system
+    d = create_driver(CONFIG + %[
+      ssl_ca_certs_from_system true
+    ])
+    config = d.instance.build_config
+
+    assert_equal 'SSL', config[:"security.protocol"]
+    assert_equal 'https', config[:"ssl.endpoint.identification.algorithm"]
+    assert_equal true, config[:"enable.ssl.certificate.verification"]
+    assert_nil config[:"ssl.ca.location"]
+  end
+
+  def test_build_config_ssl_client_cert
+    d = create_driver(CONFIG + %[
+      ssl_ca_cert /path/to/ca_cert.pem
+      ssl_client_cert /path/to/cert.pem
+      ssl_client_cert_key /path/to/key.pem
+      ssl_client_cert_key_password secret
+    ])
+    config = d.instance.build_config
+
+    assert_equal 'SSL', config[:"security.protocol"]
+    assert_equal '/path/to/ca_cert.pem', config[:"ssl.ca.location"]
+    assert_equal '/path/to/cert.pem', config[:"ssl.certificate.location"]
+    assert_equal '/path/to/key.pem', config[:"ssl.key.location"]
+    assert_equal 'secret', config[:"ssl.key.password"]
+  end
+
+  def test_build_config_ssl_verify_hostname_false
+    d = create_driver(CONFIG + %[
+      ssl_ca_certs_from_system true
+      ssl_verify_hostname false
+    ])
+    config = d.instance.build_config
+
+    assert_equal 'none', config[:"ssl.endpoint.identification.algorithm"]
+    assert_equal true, config[:"enable.ssl.certificate.verification"]
+  end
+
+  def test_build_config_sasl_plain_over_ssl
+    d = create_driver(CONFIG + %[
+      username testuser
+      password testpass
+      ssl_ca_certs_from_system true
+    ])
+    config = d.instance.build_config
+
+    assert_equal 'SASL_SSL', config[:"security.protocol"]
+    assert_equal 'PLAIN', config[:"sasl.mechanisms"]
+    assert_equal 'testuser', config[:"sasl.username"]
+    assert_equal 'testpass', config[:"sasl.password"]
+  end
+
+  def test_configure_sasl_plain_without_ssl_raises
+    assert_raise(Fluent::ConfigError) {
+      create_driver(CONFIG + %[
+        username testuser
+        password testpass
+      ])
+    }
+  end
+
+  def test_build_config_sasl_plain_without_ssl_allowed_by_sasl_over_ssl
+    d = create_driver(CONFIG + %[
+      username testuser
+      password testpass
+      sasl_over_ssl false
+    ])
+    config = d.instance.build_config
+
+    assert_equal 'SASL_PLAINTEXT', config[:"security.protocol"]
+    assert_equal 'testpass', config[:"sasl.password"]
+  end
+
+  data("sha256" => ["sha256", "SCRAM-SHA-256"],
+       "sha512" => ["sha512", "SCRAM-SHA-512"])
+  def test_build_config_sasl_scram(data)
+    mechanism, expected = data
+    d = create_driver(CONFIG + %[
+      username testuser
+      password testpass
+      scram_mechanism #{mechanism}
+      ssl_ca_certs_from_system true
+    ])
+    config = d.instance.build_config
+
+    assert_equal expected, config[:"sasl.mechanisms"]
+    assert_equal 'SASL_SSL', config[:"security.protocol"]
+  end
+
+  def test_build_config_sasl_scram_without_credentials_warns
+    d = create_driver(CONFIG + %[
+      scram_mechanism sha256
+      ssl_ca_certs_from_system true
+    ])
+    config = d.instance.build_config
+
+    assert_equal 'SSL', config[:"security.protocol"]
+    assert_nil config[:"sasl.mechanisms"]
+    assert_true d.logs.any? { |log| log.include?("scram_mechanism is ignored") }
+  end
+
+  def test_build_config_sasl_gssapi
+    d = create_driver(CONFIG + %[
+      principal kafka/host@REALM
+      keytab /path/to/kafka.keytab
+      service_name kafka
+      ssl_ca_certs_from_system true
+    ])
+    config = d.instance.build_config
+
+    assert_equal 'SASL_SSL', config[:"security.protocol"]
+    assert_equal 'GSSAPI', config[:"sasl.mechanisms"]
+    assert_equal 'kafka/host@REALM', config[:"sasl.kerberos.principal"]
+    assert_equal '/path/to/kafka.keytab', config[:"sasl.kerberos.keytab"]
+    assert_equal 'kafka', config[:"sasl.kerberos.service.name"]
+  end
+
+  def test_build_config_kafka_configs_take_precedence
+    conf = %[
+      topics #{TOPIC_NAME}
+      ssl_ca_certs_from_system true
+      kafka_configs {"bootstrap.servers": "localhost:9092", "group.id": "test_group", "security.protocol": "SASL_SSL", "ssl.endpoint.identification.algorithm": "none"}
+      <parse>
+        @type none
+      </parse>
+    ]
+    d = create_driver(conf)
+    config = d.instance.build_config
+
+    assert_equal 'SASL_SSL', config[:"security.protocol"]
+    assert_equal 'none', config[:"ssl.endpoint.identification.algorithm"]
+    assert_nil config["security.protocol"]
+  end
+
+  def test_configure_sasl_plaintext_in_kafka_configs_raises
+    conf = %[
+      topics #{TOPIC_NAME}
+      kafka_configs {"bootstrap.servers": "localhost:9092", "group.id": "test_group", "security.protocol": "SASL_PLAINTEXT", "sasl.mechanisms": "PLAIN", "sasl.username": "testuser", "sasl.password": "testpass"}
+      <parse>
+        @type none
+      </parse>
+    ]
+
+    assert_raise(Fluent::ConfigError) {
+      create_driver(conf)
+    }
+    assert_nothing_raised {
+      create_driver(conf + "sasl_over_ssl false\n")
+    }
+  end
+
+  data("uppercase" => "SASL_PLAINTEXT",
+       "lowercase" => "sasl_plaintext")
+  def test_configure_sasl_plaintext_in_kafka_configs_raises_regardless_of_case(protocol)
+    conf = %[
+      topics #{TOPIC_NAME}
+      username testuser
+      password testpass
+      kafka_configs {"bootstrap.servers": "localhost:9092", "group.id": "test_group", "security.protocol": "#{protocol}"}
+      <parse>
+        @type none
+      </parse>
+    ]
+
+    assert_raise(Fluent::ConfigError) {
+      create_driver(conf)
+    }
+  end
+
+  def test_setup_consumer_uses_build_config
+    d = create_driver(CONFIG + %[
+      ssl_ca_certs_from_system true
+    ])
+    consumer = Object.new
+    stub(consumer).subscribe
+    rdkafka_config = Object.new
+    stub(rdkafka_config).consumer { consumer }
+    passed = nil
+    stub(Rdkafka::Config).new { |config| passed = config; rdkafka_config }
+
+    d.instance.setup_consumer
+
+    assert_equal 'localhost:9092', passed[:"bootstrap.servers"]
+    assert_equal 'SSL', passed[:"security.protocol"]
+  end
+
+  class ConsumeTest < self
+    TOPIC_NAME = "kafka-input-#{SecureRandom.uuid}"
+
+    def setup
+      @kafka = Kafka.new(["localhost:9092"], client_id: 'kafka')
+      @producer = @kafka.producer
+      @kafka.create_topic(TOPIC_NAME)
+    end
+
+    def teardown
+      @kafka.delete_topic(TOPIC_NAME)
+      @kafka.close
+    end
+
+    def test_consume
+      conf = %[
+        topics #{TOPIC_NAME}
+        kafka_configs {"bootstrap.servers": "localhost:9092", "group.id": "test_group"}
+        <parse>
+          @type none
+        </parse>
+      ]
+
+      d = create_driver(conf)
+
+      d.run(expect_records: 1, timeout: 10) do
+        sleep 0.1
+        @producer.produce("Hello, fluent-plugin-kafka!", topic: TOPIC_NAME)
+        @producer.deliver_messages
+      end
+
+      expected = {'message'  => 'Hello, fluent-plugin-kafka!'}
+      assert_equal expected, d.events[0][2]
+    end
+  end
+
+  class ConsumeTopicWithRegexpTest < self
+    TOPIC_NAME1       = "kafka-input-1-#{SecureRandom.uuid}"
+    TOPIC_NAME2       = "kafka-input-22-#{SecureRandom.uuid}"
+    UNMATCHED_TOPIC   = "kafka-input-333-#{SecureRandom.uuid}"
+
+    TOPIC_NAME_REGEXP = "/kafka-input-[0-9]{1,2}-.*/"
+
+    def setup
+      @kafka = Kafka.new(["localhost:9092"], client_id: 'kafka')
+      @producer = @kafka.producer
+      @kafka.create_topic(TOPIC_NAME1)
+      @kafka.create_topic(TOPIC_NAME2)
+      @kafka.create_topic(UNMATCHED_TOPIC)
+    end
+
+    def teardown
+      @kafka.delete_topic(TOPIC_NAME1)
+      @kafka.delete_topic(TOPIC_NAME2)
+      @kafka.delete_topic(UNMATCHED_TOPIC)
+      @kafka.close
+    end
+
+    def test_consume_with_regexp
+      conf = %[
+        topics #{TOPIC_NAME_REGEXP}
+        kafka_configs {"bootstrap.servers": "localhost:9092", "group.id": "test_group"}
+        <parse>
+          @type none
+        </parse>
+      ]
+      d = create_driver(conf)
+
+      d.run(expect_records: 2, timeout: 10) do
+        sleep 0.1
+        @producer.produce("Hello, fluent-plugin-kafka! in topic 1", topic: TOPIC_NAME1)
+        @producer.produce("Hello, fluent-plugin-kafka! in topic 2", topic: TOPIC_NAME2)
+        @producer.produce("Should be ignored", topic: UNMATCHED_TOPIC)
+        @producer.deliver_messages
+      end
+      expected_message_pattern = /Hello, fluent-plugin-kafka! in topic [12]/
+      assert_equal 2, d.events.size
+      assert_match(expected_message_pattern, d.events[0][2]['message'])
+      assert_match(expected_message_pattern, d.events[1][2]['message'])
+    end
+  end
+end