fluent-plugin-kafka 0.16.1 → 0.17.1

data/lib/fluent/plugin/out_rdkafka.rb

@@ -48,7 +48,7 @@ Set true to remove partition from data
 DESC
    config_param :exclude_message_key, :bool, :default => false,
                :desc => <<-DESC
-Set true to remove partition key from data
+Set true to remove message key from data
 DESC
    config_param :exclude_topic_key, :bool, :default => false,
                 :desc => <<-DESC
@@ -91,23 +91,29 @@ DESC
   def configure(conf)
     super
     log.instance_eval {
-      def add(level, &block)
-        return unless block
+      def add(level, message = nil)
+        if message.nil?
+          if block_given?
+            message = yield
+          else
+            return
+          end
+        end
 
         # Follow rdkakfa's log level. See also rdkafka-ruby's bindings.rb: https://github.com/appsignal/rdkafka-ruby/blob/e5c7261e3f2637554a5c12b924be297d7dca1328/lib/rdkafka/bindings.rb#L117
         case level
         when Logger::FATAL
-          self.fatal(block.call)
+          self.fatal(message)
         when Logger::ERROR
-          self.error(block.call)
+          self.error(message)
         when Logger::WARN
-          self.warn(block.call)
+          self.warn(message)
         when Logger::INFO
-          self.info(block.call)
+          self.info(message)
         when Logger::DEBUG
-          self.debug(block.call)
+          self.debug(message)
         else
-          self.trace(block.call)
+          self.trace(message)
         end
       end
     }

data/lib/fluent/plugin/out_rdkafka2.rb

@@ -56,6 +56,8 @@ DESC
                  :desc => <<-DESC
 Set true to remove topic key from data
 DESC
+    config_param :exclude_fields, :array, :default => [], value_type: :string,
+                 :desc => 'Fields to remove from data where the value is a jsonpath to a record value'
     config_param :headers, :hash, default: {}, symbolize_keys: true, value_type: :string,
                  :desc => 'Kafka message headers'
     config_param :headers_from_record, :hash, default: {}, symbolize_keys: true, value_type: :string,
@@ -110,23 +112,29 @@ DESC
     def configure(conf)
       super
       log.instance_eval {
-        def add(level, &block)
-          return unless block
+        def add(level, message = nil)
+          if message.nil?
+            if block_given?
+              message = yield
+            else
+              return
+            end
+          end
 
           # Follow rdkakfa's log level. See also rdkafka-ruby's bindings.rb: https://github.com/appsignal/rdkafka-ruby/blob/e5c7261e3f2637554a5c12b924be297d7dca1328/lib/rdkafka/bindings.rb#L117
           case level
           when Logger::FATAL
-            self.fatal(block.call)
+            self.fatal(message)
           when Logger::ERROR
-            self.error(block.call)
+            self.error(message)
           when Logger::WARN
-            self.warn(block.call)
+            self.warn(message)
           when Logger::INFO
-            self.info(block.call)
+            self.info(message)
           when Logger::DEBUG
-            self.debug(block.call)
+            self.debug(message)
           else
-            self.trace(block.call)
+            self.trace(message)
           end
         end
       }
@@ -158,6 +166,10 @@ DESC
       @headers_from_record.each do |key, value|
         @headers_from_record_accessors[key] = record_accessor_create(value)
       end
+
+      @exclude_field_accessors = @exclude_fields.map do |field|
+        record_accessor_create(field)
+      end
     end
 
     def build_config
@@ -305,6 +317,12 @@ DESC
               headers[key] = header_accessor.call(record)
             end
 
+            unless @exclude_fields.empty?
+              @exclude_field_accessors.each do |exclude_field_acessor|
+                exclude_field_acessor.delete(record)
+              end
+            end
+
             record_buf = @formatter_proc.call(tag, time, record)
             record_buf_bytes = record_buf.bytesize
             if @max_send_limit_bytes && record_buf_bytes > @max_send_limit_bytes

data/test/helper.rb

@@ -8,6 +8,7 @@ rescue Bundler::BundlerError => e
   exit e.status_code
 end
 require 'test/unit'
+require 'test/unit/rr'
 
 $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
 $LOAD_PATH.unshift(File.dirname(__FILE__))
@@ -22,6 +23,12 @@ unless ENV.has_key?('VERBOSE')
 end
 
 require 'fluent/plugin/out_kafka'
+require 'fluent/plugin/out_kafka_buffered'
+require 'fluent/plugin/out_kafka2'
+require 'fluent/plugin/in_kafka'
+require 'fluent/plugin/in_kafka_group'
+
+require "fluent/test/driver/output"
 
 class Test::Unit::TestCase
 end

data/test/plugin/test_in_kafka.rb

@@ -0,0 +1,66 @@
+require 'helper'
+require 'fluent/test/driver/input'
+require 'securerandom'
+
+class KafkaInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  TOPIC_NAME = "kafka-input-#{SecureRandom.uuid}"
+
+  CONFIG = %[
+    @type kafka
+    brokers localhost:9092
+    format text
+    @label @kafka
+    topics #{TOPIC_NAME}
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Input.new(Fluent::KafkaInput).configure(conf)
+  end
+
+
+  def test_configure
+    d = create_driver
+    assert_equal TOPIC_NAME, d.instance.topics
+    assert_equal 'text', d.instance.format
+    assert_equal 'localhost:9092', d.instance.brokers
+  end
+
+  def test_multi_worker_support
+    d = create_driver
+    assert_false d.instance.multi_workers_ready?
+  end
+
+  class ConsumeTest < self
+    def setup
+      @kafka = Kafka.new(["localhost:9092"], client_id: 'kafka')
+      @producer = @kafka.producer
+    end
+
+    def teardown
+      @kafka.delete_topic(TOPIC_NAME)
+      @kafka.close
+    end
+
+    def test_consume
+      conf = %[
+        @type kafka
+        brokers localhost:9092
+        format text
+        @label @kafka
+        topics #{TOPIC_NAME}
+      ]
+      d = create_driver
+
+      d.run(expect_records: 1, timeout: 10) do
+        @producer.produce("Hello, fluent-plugin-kafka!", topic: TOPIC_NAME)
+        @producer.deliver_messages
+      end
+      expected = {'message'  => 'Hello, fluent-plugin-kafka!'}
+      assert_equal expected, d.events[0][2]
+    end
+  end
+end

data/test/plugin/test_in_kafka_group.rb

@@ -0,0 +1,67 @@
+require 'helper'
+require 'fluent/test/driver/input'
+require 'securerandom'
+
+class KafkaGroupInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  TOPIC_NAME = "kafka-input-#{SecureRandom.uuid}"
+
+  CONFIG = %[
+    @type kafka
+    brokers localhost:9092
+    consumer_group fluentd
+    format text
+    @label @kafka
+    topics #{TOPIC_NAME}
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Input.new(Fluent::KafkaGroupInput).configure(conf)
+  end
+
+
+  def test_configure
+    d = create_driver
+    assert_equal [TOPIC_NAME], d.instance.topics
+    assert_equal 'text', d.instance.format
+    assert_equal 'localhost:9092', d.instance.brokers
+  end
+
+  def test_multi_worker_support
+    d = create_driver
+    assert_true d.instance.multi_workers_ready?
+  end
+
+  class ConsumeTest < self
+    def setup
+      @kafka = Kafka.new(["localhost:9092"], client_id: 'kafka')
+      @producer = @kafka.producer
+    end
+
+    def teardown
+      @kafka.delete_topic(TOPIC_NAME)
+      @kafka.close
+    end
+
+    def test_consume
+      conf = %[
+        @type kafka
+        brokers localhost:9092
+        format text
+        @label @kafka
+        topics #{TOPIC_NAME}
+      ]
+      d = create_driver
+
+      d.run(expect_records: 1, timeout: 10) do
+        @producer.produce("Hello, fluent-plugin-kafka!", topic: TOPIC_NAME)
+        @producer.deliver_messages
+      end
+      expected = {'message'  => 'Hello, fluent-plugin-kafka!'}
+      assert_equal expected, d.events[0][2]
+    end
+  end
+end

data/test/plugin/test_kafka_plugin_util.rb

@@ -1,14 +1,8 @@
 require 'helper'
 require 'fluent/plugin/kafka_plugin_util'
 
-class File
-    def File::read(path)
-        path
-    end
-end
-
 class KafkaPluginUtilTest < Test::Unit::TestCase
-    
+
     def self.config_param(name, type, options)
     end
     include Fluent::KafkaPluginUtil::SSLSettings
@@ -20,19 +14,31 @@ class KafkaPluginUtilTest < Test::Unit::TestCase
     end
 
     def test_read_ssl_file_when_nil
-        assert_equal(nil, read_ssl_file(nil))
+      stub(File).read(anything) do |path|
+        path
+      end
+      assert_equal(nil, read_ssl_file(nil))
     end
 
     def test_read_ssl_file_when_empty_string
-        assert_equal(nil, read_ssl_file(""))
+      stub(File).read(anything) do |path|
+        path
+      end
+      assert_equal(nil, read_ssl_file(""))
     end
 
     def test_read_ssl_file_when_non_empty_path
-        assert_equal("path", read_ssl_file("path"))
+      stub(File).read(anything) do |path|
+        path
+      end
+      assert_equal("path", read_ssl_file("path"))
     end
 
     def test_read_ssl_file_when_non_empty_array
-        assert_equal(["a","b"], read_ssl_file(["a","b"]))
+      stub(File).read(anything) do |path|
+        path
+      end
+      assert_equal(["a","b"], read_ssl_file(["a","b"]))
     end
 
-end
+end

data/test/plugin/test_out_kafka.rb

@@ -43,6 +43,16 @@ class KafkaOutputTest < Test::Unit::TestCase
     d = create_driver
   end
 
+  data("crc32" => "crc32",
+      "murmur2" => "murmur2")
+  def test_partitioner_hash_function(data)
+    hash_type = data
+    d = create_driver(CONFIG + %[partitioner_hash_function #{hash_type}])
+    assert_nothing_raised do
+      d.instance.refresh_client
+    end
+  end
+
   def test_mutli_worker_support
     d = create_driver
     assert_equal true, d.instance.multi_workers_ready?

data/test/plugin/test_out_kafka2.rb

@@ -0,0 +1,116 @@
+require 'helper'
+require 'fluent/test/helpers'
+require 'fluent/test/driver/input'
+require 'fluent/test/driver/output'
+require 'securerandom'
+
+class Kafka2OutputTest < Test::Unit::TestCase
+  include Fluent::Test::Helpers
+
+  def setup
+    Fluent::Test.setup
+  end
+
+  def base_config
+    config_element('ROOT', '', {"@type" => "kafka2"}, [
+                     config_element('format', "", {"@type" => "json"})
+                   ])
+  end
+
+  def config(default_topic: "kitagawakeiko")
+    base_config + config_element('ROOT', '', {"default_topic" => default_topic,
+                                              "brokers" => "localhost:9092"}, [
+                                 ])
+  end
+
+  def create_driver(conf = config, tag='test')
+    Fluent::Test::Driver::Output.new(Fluent::Kafka2Output).configure(conf)
+  end
+
+  def test_configure
+    assert_nothing_raised(Fluent::ConfigError) {
+      create_driver(base_config)
+    }
+
+    assert_nothing_raised(Fluent::ConfigError) {
+      create_driver(config)
+    }
+
+    assert_nothing_raised(Fluent::ConfigError) {
+      create_driver(config + config_element('buffer', "", {"@type" => "memory"}))
+    }
+
+    d = create_driver
+    assert_equal 'kitagawakeiko', d.instance.default_topic
+    assert_equal ['localhost:9092'], d.instance.brokers
+  end
+
+  data("crc32" => "crc32",
+       "murmur2" => "murmur2")
+  def test_partitioner_hash_function(data)
+    hash_type = data
+    d = create_driver(config + config_element('ROOT', '', {"partitioner_hash_function" => hash_type}))
+    assert_nothing_raised do
+      d.instance.refresh_client
+    end
+  end
+
+  def test_mutli_worker_support
+    d = create_driver
+    assert_equal true, d.instance.multi_workers_ready?
+  end
+
+  class WriteTest < self
+    TOPIC_NAME = "kafka-output-#{SecureRandom.uuid}"
+
+    INPUT_CONFIG = %[
+      @type kafka
+      brokers localhost:9092
+      format json
+      @label @kafka
+      topics #{TOPIC_NAME}
+    ]
+
+    def create_target_driver(conf = INPUT_CONFIG)
+      Fluent::Test::Driver::Input.new(Fluent::KafkaInput).configure(conf)
+    end
+
+    def setup
+      @kafka = Kafka.new(["localhost:9092"], client_id: 'kafka')
+    end
+
+    def teardown
+      @kafka.delete_topic(TOPIC_NAME)
+      @kafka.close
+    end
+
+    def test_write
+      target_driver = create_target_driver
+      expected_message = {"a" => 2}
+      target_driver.run(expect_records: 1, timeout: 5) do
+        sleep 2
+        d = create_driver(config(default_topic: TOPIC_NAME))
+        d.run do
+          d.feed("test", event_time, expected_message)
+        end
+      end
+      actual_messages = target_driver.events.collect { |event| event[2] }
+      assert_equal([expected_message], actual_messages)
+    end
+
+    def test_exclude_fields
+      conf = config(default_topic: TOPIC_NAME) +
+             config_element('ROOT', '', {"exclude_fields" => "$.foo"}, [])
+      target_driver = create_target_driver
+      target_driver.run(expect_records: 1, timeout: 5) do
+        sleep 2
+        d = create_driver(conf)
+        d.run do
+          d.feed('test', event_time, {'a' => 'b', 'foo' => 'bar', 'message' => 'test'})
+        end
+      end
+      actual_messages = target_driver.events.collect { |event| event[2] }
+      assert_equal([{'a' => 'b', 'message' => 'test'}], actual_messages)
+    end
+  end
+end

data/test/plugin/test_out_kafka_buffered.rb

@@ -0,0 +1,68 @@
+require 'helper'
+require 'fluent/output'
+
+class KafkaBufferedOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  BASE_CONFIG = %[
+    type kafka_buffered
+  ]
+
+  CONFIG = BASE_CONFIG + %[
+    default_topic kitagawakeiko
+    brokers localhost:9092
+  ]
+
+  def create_driver(conf = CONFIG, tag='test')
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::KafkaOutputBuffered, tag).configure(conf)
+  end
+
+  def test_configure
+    assert_nothing_raised(Fluent::ConfigError) {
+      create_driver(BASE_CONFIG)
+    }
+
+    assert_nothing_raised(Fluent::ConfigError) {
+      create_driver(CONFIG)
+    }
+
+    assert_nothing_raised(Fluent::ConfigError) {
+      create_driver(CONFIG + %[
+        buffer_type memory
+      ])
+    }
+
+    d = create_driver
+    assert_equal 'kitagawakeiko', d.instance.default_topic
+    assert_equal 'localhost:9092', d.instance.brokers
+  end
+
+  def test_format
+    d = create_driver
+  end
+
+  data("crc32" => "crc32",
+      "murmur2" => "murmur2")
+  def test_partitioner_hash_function(data)
+    hash_type = data
+    d = create_driver(CONFIG + %[partitioner_hash_function #{hash_type}])
+    assert_nothing_raised do
+      d.instance.refresh_client
+    end
+  end
+
+  def test_mutli_worker_support
+    d = create_driver
+    assert_equal true, d.instance.multi_workers_ready?
+
+  end
+
+  def test_write
+    d = create_driver
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit({"a"=>1}, time)
+    d.emit({"a"=>2}, time)
+  end
+end

data/test/plugin/test_out_rdkafka2.rb

@@ -0,0 +1,120 @@
+require 'helper'
+require 'fluent/test/helpers'
+require 'fluent/test/driver/input'
+require 'fluent/test/driver/output'
+require 'securerandom'
+
+class Rdkafka2OutputTest < Test::Unit::TestCase
+  include Fluent::Test::Helpers
+
+  def have_rdkafka
+    begin
+      require 'fluent/plugin/out_rdkafka2'
+      true
+    rescue LoadError
+      false
+    end
+  end
+
+  def setup
+    omit_unless(have_rdkafka, "rdkafka isn't installed")
+    Fluent::Test.setup
+  end
+
+  def base_config
+    config_element('ROOT', '', {"@type" => "rdkafka2"}, [
+                     config_element('format', "", {"@type" => "json"})
+                   ])
+  end
+
+  def config(default_topic: "kitagawakeiko")
+    base_config + config_element('ROOT', '', {"default_topic" => default_topic,
+                                              "brokers" => "localhost:9092"}, [
+                                 ])
+  end
+
+  def create_driver(conf = config, tag='test')
+    Fluent::Test::Driver::Output.new(Fluent::Rdkafka2Output).configure(conf)
+  end
+
+  def test_configure
+    assert_nothing_raised(Fluent::ConfigError) {
+      create_driver(base_config)
+    }
+
+    assert_nothing_raised(Fluent::ConfigError) {
+      create_driver(config)
+    }
+
+    assert_nothing_raised(Fluent::ConfigError) {
+      create_driver(config + config_element('buffer', "", {"@type" => "memory"}))
+    }
+
+    d = create_driver
+    assert_equal 'kitagawakeiko', d.instance.default_topic
+    assert_equal 'localhost:9092', d.instance.brokers
+  end
+
+  def test_mutli_worker_support
+    d = create_driver
+    assert_equal true, d.instance.multi_workers_ready?
+  end
+
+  class WriteTest < self
+    TOPIC_NAME = "kafka-output-#{SecureRandom.uuid}"
+
+    INPUT_CONFIG = %[
+      @type kafka
+      brokers localhost:9092
+      format json
+      @label @kafka
+      topics #{TOPIC_NAME}
+    ]
+
+    def create_target_driver(conf = INPUT_CONFIG)
+      Fluent::Test::Driver::Input.new(Fluent::KafkaInput).configure(conf)
+    end
+
+    def setup
+      @kafka = nil
+      omit_unless(have_rdkafka, "rdkafka isn't installed")
+      @kafka = Kafka.new(["localhost:9092"], client_id: 'kafka')
+    end
+
+    def teardown
+      if @kafka
+        @kafka.delete_topic(TOPIC_NAME)
+        @kafka.close
+      end
+    end
+
+    def test_write
+      target_driver = create_target_driver
+      expected_message = {"a" => 2}
+      target_driver.run(expect_records: 1, timeout: 5) do
+        sleep 2
+        d = create_driver(config(default_topic: TOPIC_NAME))
+        d.run do
+          d.feed("test", event_time, expected_message)
+        end
+      end
+      actual_messages = target_driver.events.collect { |event| event[2] }
+      assert_equal([expected_message], actual_messages)
+    end
+
+    def test_exclude_fields
+      conf = config(default_topic: TOPIC_NAME) +
+             config_element('ROOT', '', {"exclude_fields" => "$.foo"}, [])
+      target_driver = create_target_driver
+      target_driver.run(expect_records: 1, timeout: 5) do
+        sleep 2
+        d = create_driver(conf)
+        d.run do
+          d.feed('test', event_time, {'a' => 'b', 'foo' => 'bar', 'message' => 'test'})
+        end
+      end
+      actual_messages = target_driver.events.collect { |event| event[2] }
+      assert_equal([{'a' => 'b', 'message' => 'test'}], actual_messages)
+    end
+  end
+end