fluent-plugin-kafka 0.16.2 → 0.17.2

data/test/plugin/test_out_kafka_buffered.rb

@@ -0,0 +1,68 @@
+require 'helper'
+require 'fluent/output'
+
+class KafkaBufferedOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  BASE_CONFIG = %[
+    type kafka_buffered
+  ]
+
+  CONFIG = BASE_CONFIG + %[
+    default_topic kitagawakeiko
+    brokers localhost:9092
+  ]
+
+  def create_driver(conf = CONFIG, tag='test')
+    Fluent::Test::BufferedOutputTestDriver.new(Fluent::KafkaOutputBuffered, tag).configure(conf)
+  end
+
+  def test_configure
+    assert_nothing_raised(Fluent::ConfigError) {
+      create_driver(BASE_CONFIG)
+    }
+
+    assert_nothing_raised(Fluent::ConfigError) {
+      create_driver(CONFIG)
+    }
+
+    assert_nothing_raised(Fluent::ConfigError) {
+      create_driver(CONFIG + %[
+        buffer_type memory
+      ])
+    }
+
+    d = create_driver
+    assert_equal 'kitagawakeiko', d.instance.default_topic
+    assert_equal 'localhost:9092', d.instance.brokers
+  end
+
+  def test_format
+    d = create_driver
+  end
+
+  data("crc32" => "crc32",
+      "murmur2" => "murmur2")
+  def test_partitioner_hash_function(data)
+    hash_type = data
+    d = create_driver(CONFIG + %[partitioner_hash_function #{hash_type}])
+    assert_nothing_raised do
+      d.instance.refresh_client
+    end
+  end
+
+  def test_mutli_worker_support
+    d = create_driver
+    assert_equal true, d.instance.multi_workers_ready?
+
+  end
+
+  def test_write
+    d = create_driver
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    d.emit({"a"=>1}, time)
+    d.emit({"a"=>2}, time)
+  end
+end

data/test/plugin/test_out_rdkafka2.rb

@@ -0,0 +1,167 @@
+require 'helper'
+require 'fluent/test/helpers'
+require 'fluent/test/driver/input'
+require 'fluent/test/driver/output'
+require 'securerandom'
+
+class Rdkafka2OutputTest < Test::Unit::TestCase
+  include Fluent::Test::Helpers
+
+  def have_rdkafka
+    begin
+      require 'fluent/plugin/out_rdkafka2'
+      true
+    rescue LoadError
+      false
+    end
+  end
+
+  def setup
+    omit_unless(have_rdkafka, "rdkafka isn't installed")
+    Fluent::Test.setup
+  end
+
+  def base_config
+    config_element('ROOT', '', {"@type" => "rdkafka2"}, [
+                     config_element('format', "", {"@type" => "json"})
+                   ])
+  end
+
+  def config(default_topic: "kitagawakeiko")
+    base_config + config_element('ROOT', '', {"default_topic" => default_topic,
+                                              "brokers" => "localhost:9092"}, [
+                                 ])
+  end
+
+  def create_driver(conf = config, tag='test')
+    Fluent::Test::Driver::Output.new(Fluent::Rdkafka2Output).configure(conf)
+  end
+
+  def test_configure
+    assert_nothing_raised(Fluent::ConfigError) {
+      create_driver(base_config)
+    }
+
+    assert_nothing_raised(Fluent::ConfigError) {
+      create_driver(config)
+    }
+
+    assert_nothing_raised(Fluent::ConfigError) {
+      create_driver(config + config_element('buffer', "", {"@type" => "memory"}))
+    }
+
+    d = create_driver
+    assert_equal 'kitagawakeiko', d.instance.default_topic
+    assert_equal 'localhost:9092', d.instance.brokers
+  end
+
+  def test_mutli_worker_support
+    d = create_driver
+    assert_equal true, d.instance.multi_workers_ready?
+  end
+
+  class WriteTest < self
+    TOPIC_NAME = "kafka-output-#{SecureRandom.uuid}"
+
+    INPUT_CONFIG = %[
+      @type kafka
+      brokers localhost:9092
+      format json
+      @label @kafka
+      topics #{TOPIC_NAME}
+    ]
+
+    def create_target_driver(conf = INPUT_CONFIG)
+      Fluent::Test::Driver::Input.new(Fluent::KafkaInput).configure(conf)
+    end
+
+    def setup
+      @kafka = nil
+      omit_unless(have_rdkafka, "rdkafka isn't installed")
+      @kafka = Kafka.new(["localhost:9092"], client_id: 'kafka')
+    end
+
+    def teardown
+      if @kafka
+        @kafka.delete_topic(TOPIC_NAME)
+        @kafka.close
+      end
+    end
+
+    def test_write
+      target_driver = create_target_driver
+      expected_message = {"a" => 2}
+      target_driver.run(expect_records: 1, timeout: 5) do
+        sleep 2
+        d = create_driver(config(default_topic: TOPIC_NAME))
+        d.run do
+          d.feed("test", event_time, expected_message)
+        end
+      end
+      actual_messages = target_driver.events.collect { |event| event[2] }
+      assert_equal([expected_message], actual_messages)
+    end
+
+    def test_write_with_use_event_time
+      input_config = %[
+        @type kafka
+        brokers localhost:9092
+        format json
+        @label @kafka
+        topics #{TOPIC_NAME}
+        time_source kafka
+      ]
+      target_driver = create_target_driver(input_config)
+      expected_message = {"a" => 2}
+      now = event_time
+      target_driver.run(expect_records: 1, timeout: 5) do
+        sleep 2
+        d = create_driver(config(default_topic: TOPIC_NAME) + config_element('ROOT', '', {"use_event_time" => true}))
+        d.run do
+          d.feed("test", now, expected_message)
+        end
+      end
+      actual_time = target_driver.events.collect { |event| event[1] }.last
+      assert_in_delta(actual_time, now, 0.001) # expects millseconds precision
+      actual_messages = target_driver.events.collect { |event| event[2] }
+      assert_equal([expected_message], actual_messages)
+    end
+
+    def test_exclude_fields
+      conf = config(default_topic: TOPIC_NAME) +
+             config_element('ROOT', '', {"exclude_fields" => "$.foo"}, [])
+      target_driver = create_target_driver
+      target_driver.run(expect_records: 1, timeout: 5) do
+        sleep 2
+        d = create_driver(conf)
+        d.run do
+          d.feed('test', event_time, {'a' => 'b', 'foo' => 'bar', 'message' => 'test'})
+        end
+      end
+      actual_messages = target_driver.events.collect { |event| event[2] }
+      assert_equal([{'a' => 'b', 'message' => 'test'}], actual_messages)
+    end
+
+    def test_max_enqueue_bytes_per_second
+      conf = config(default_topic: TOPIC_NAME) +
+             config_element('ROOT', '', {"max_enqueue_bytes_per_second" => 32 * 3}, [])
+      target_driver = create_target_driver
+      expected_messages = []
+      target_driver.run(expect_records: 9, timeout: 10) do
+        sleep 2
+        d = create_driver(conf)
+        start_time = Fluent::Clock.now
+        d.run do
+          9.times do |i|
+            message = {"message" => "32bytes message: #{i}"}
+            d.feed("test", event_time, message)
+            expected_messages << message
+          end
+        end
+        assert_in_delta(2.0, Fluent::Clock.now - start_time, 0.5)
+      end
+      actual_messages = target_driver.events.collect { |event| event[2] }
+      assert_equal(expected_messages, actual_messages)
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.16.2
+  version: 0.17.2
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-05-17 00:00:00.000000000 Z
+date: 2021-10-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -51,7 +51,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.2.0
+        version: 1.4.0
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -61,7 +61,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.2.0
+        version: 1.4.0
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -93,6 +93,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.0.8
+- !ruby/object:Gem::Dependency
+  name: test-unit-rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: webrick
   requirement: !ruby/object:Gem::Requirement
@@ -107,6 +121,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: digest-murmurhash
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Fluentd plugin for Apache Kafka > 0.8
 email:
 - togachiro@gmail.com
@@ -115,13 +143,18 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/ISSUE_TEMPLATE/bug_report.yaml"
+- ".github/ISSUE_TEMPLATE/config.yml"
+- ".github/ISSUE_TEMPLATE/feature_request.yaml"
 - ".github/workflows/linux.yml"
+- ".github/workflows/stale-actions.yml"
 - ".gitignore"
 - ChangeLog
 - Gemfile
 - LICENSE
 - README.md
 - Rakefile
+- ci/prepare-kafka-server.sh
 - fluent-plugin-kafka.gemspec
 - lib/fluent/plugin/in_kafka.rb
 - lib/fluent/plugin/in_kafka_group.rb
@@ -134,8 +167,13 @@ files:
 - lib/fluent/plugin/out_rdkafka.rb
 - lib/fluent/plugin/out_rdkafka2.rb
 - test/helper.rb
+- test/plugin/test_in_kafka.rb
+- test/plugin/test_in_kafka_group.rb
 - test/plugin/test_kafka_plugin_util.rb
 - test/plugin/test_out_kafka.rb
+- test/plugin/test_out_kafka2.rb
+- test/plugin/test_out_kafka_buffered.rb
+- test/plugin/test_out_rdkafka2.rb
 homepage: https://github.com/fluent/fluent-plugin-kafka
 licenses:
 - Apache-2.0
@@ -161,5 +199,10 @@ specification_version: 4
 summary: Fluentd plugin for Apache Kafka > 0.8
 test_files:
 - test/helper.rb
+- test/plugin/test_in_kafka.rb
+- test/plugin/test_in_kafka_group.rb
 - test/plugin/test_kafka_plugin_util.rb
 - test/plugin/test_out_kafka.rb
+- test/plugin/test_out_kafka2.rb
+- test/plugin/test_out_kafka_buffered.rb
+- test/plugin/test_out_rdkafka2.rb