fluent-plugin-kafka 0.17.0 → 0.17.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 15805411e4029813123b9b636b6faadb937cf38c9841adcd9d998a0f54d8b687
-  data.tar.gz: 75fef11595c86beb4a54d2d2ff659f77075e328426463f7ef830982c9724ff16
+  metadata.gz: 230266adf4ba3d77b8b2fd743863377c4dd2532297f45e4489df62bfcbee1db5
+  data.tar.gz: 4b82d2f33bbbb3294f547a7af154783c432c6316f1155a0e3a822361232743e4
 SHA512:
-  metadata.gz: db08ffbde4fe36ce38abe6eebf83e6e0dc157bc5d8dd95141ef911d0c5b59074a208c3a8f836e258ffddc34f73fda303006eaef1cd8fc12ae4dd4c79d101d0c7
-  data.tar.gz: 9b3b098b1bc58654924d50ac8d685af007763942c1cc68187bbd6d39510a27ca31ae795df0de312d97ad67463ae0c428882118640fce052119288e27e3fb0df5
+  metadata.gz: 9321cd8bd10dcd603b653c101e6695b7071d7b9dc61fb49bc79248b22cbe9f1db46416c40132243c9862e9c4a3888c818f7a8906906ccf0dda545032f2ac53fd
+  data.tar.gz: 832517ac39c4f775d95775454b07bc0f3ecc2f27b1c0bd4c3f8025e795dfbe869d3e349084f868bdb54cfa72188d4cbd96173192f728fae4281f7f84e5d973aa

data/.github/workflows/linux.yml

@@ -5,6 +5,8 @@ on:
 jobs:
   build:
     runs-on: ${{ matrix.os }}
+    env:
+      USE_RDKAFKA: 1
     strategy:
       fail-fast: false
       matrix:

data/ChangeLog

@@ -1,3 +1,8 @@
+Release 0.17.1 - 2021/09/24
+	* out_rdkafka/out_rdkafka2: Support rdkafka 0.9.0 or later
+	* out_rdkafka/out_rdkafka2: Add `exclude_fields` parameter
+	* out_kafka2.rb: Fix one more Ruby 3.0 keyword arguments issue
+
 Release 0.17.0 - 2021/08/30
 	* out_kafka/out_kafka_buffered/out_kafka2: Provide murmur2 partitioner hash function choice
 	* in_kafka/in_kafka_group/out_kafka/out_kafka_buffered/out_kafka2: Use Ruby Kafka's ssl_ca_cert_file_path parameter to feed the CA certs

data/Gemfile

@@ -2,3 +2,5 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in fluent-plugin-kafka.gemspec
 gemspec
+
+gem 'rdkafka', '>= 0.6.0' if ENV["USE_RDKAFKA"]

data/README.md

@@ -40,14 +40,14 @@ If you want to use zookeeper related parameters, you also need to install zookee
 
 Set path to SSL related files. See [Encryption and Authentication using SSL](https://github.com/zendesk/ruby-kafka#encryption-and-authentication-using-ssl) for more detail.
 
-#### SASL authentication 
+#### SASL authentication
 
 ##### with GSSAPI
 
 - principal
 - keytab
 
-Set principal and path to keytab for SASL/GSSAPI authentication. 
+Set principal and path to keytab for SASL/GSSAPI authentication.
 See [Authentication using SASL](https://github.com/zendesk/ruby-kafka#authentication-using-sasl) for more details.
 
 ##### with Plain/SCRAM
@@ -57,7 +57,7 @@ See [Authentication using SASL](https://github.com/zendesk/ruby-kafka#authentica
 - scram_mechanism
 - sasl_over_ssl
 
-Set username, password, scram_mechanism and sasl_over_ssl for SASL/Plain or Scram authentication. 
+Set username, password, scram_mechanism and sasl_over_ssl for SASL/Plain or Scram authentication.
 See [Authentication using SASL](https://github.com/zendesk/ruby-kafka#authentication-using-sasl) for more details.
 
 ### Input plugin (@type 'kafka')
@@ -316,6 +316,23 @@ The Kafka message will have a header of source_ip=12.7.0.0.1.
 
 The configuration format is jsonpath. It is descibed in https://docs.fluentd.org/plugin-helper-overview/api-plugin-helper-record_accessor
 
+#### Excluding fields
+Fields can be excluded from output data. Only works for kafka2 and rdkafka2 output plugin.
+
+Fields must be specified using an array of dot notation `$.`, for example:
+
+    <match app.**>
+      @type kafka2
+      [...]
+      exclude_fields $.source.ip,$.HTTP_FOO
+    <match>
+
+This config can be used to remove fields used on another configs.
+
+For example, `$.source.ip` can be extracted with config `headers_from_record` and excluded from message payload.
+
+> Using this config to remove unused fields is discouraged. A [filter plugin](https://docs.fluentd.org/v/0.12/filter) can be used for this purpose.
+
 ### Buffered output plugin
 
 This plugin uses ruby-kafka producer for writing data. This plugin is for v0.12. If you use v1, see `kafka2`.

data/fluent-plugin-kafka.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.17.0'
+  gem.version       = '0.17.1'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/in_kafka.rb

@@ -71,6 +71,7 @@ class Fluent::KafkaInput < Fluent::Input
     require 'kafka'
 
     @time_parser = nil
+    @zookeeper = nil
   end
 
   def configure(conf)

data/lib/fluent/plugin/out_kafka.rb

@@ -88,6 +88,7 @@ DESC
     require 'kafka'
 
     @kafka = nil
+    @field_separator = nil
   end
 
   def refresh_client

data/lib/fluent/plugin/out_kafka2.rb

@@ -42,6 +42,8 @@ DESC
                  :desc => 'Set true to remove message key from data'
     config_param :exclude_topic_key, :bool, :default => false,
                  :desc => 'Set true to remove topic name key from data'
+    config_param :exclude_fields, :array, :default => [], value_type: :string,
+                 :desc => 'Fields to remove from data where the value is a jsonpath to a record value'
     config_param :use_event_time, :bool, :default => false, :desc => 'Use fluentd event time for kafka create_time'
     config_param :headers, :hash, default: {}, symbolize_keys: true, value_type: :string,
                  :desc => 'Kafka message headers'
@@ -177,6 +179,10 @@ DESC
       @headers_from_record.each do |key, value|
         @headers_from_record_accessors[key] = record_accessor_create(value)
       end
+
+      @exclude_field_accessors = @exclude_fields.map do |field|
+        record_accessor_create(field)
+      end
     end
 
     def multi_workers_ready?
@@ -235,7 +241,7 @@ DESC
       mutate_headers = !@headers_from_record_accessors.empty?
 
       begin
-        producer = @kafka.topic_producer(topic, @producer_opts)
+        producer = @kafka.topic_producer(topic, **@producer_opts)
         chunk.msgpack_each { |time, record|
           begin
             record = inject_values_to_record(tag, time, record)
@@ -253,6 +259,12 @@ DESC
               headers = base_headers
             end
 
+            unless @exclude_fields.empty?
+              @exclude_field_accessors.each do |exclude_field_accessor|
+                exclude_field_accessor.delete(record)
+              end
+            end
+
             record_buf = @formatter_proc.call(tag, time, record)
             record_buf_bytes = record_buf.bytesize
             if @max_send_limit_bytes && record_buf_bytes > @max_send_limit_bytes

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -107,6 +107,7 @@ DESC
     @kafka = nil
     @producers = {}
     @producers_mutex = Mutex.new
+    @field_separator = nil
   end
 
   def multi_workers_ready?

data/lib/fluent/plugin/out_rdkafka.rb

@@ -91,23 +91,29 @@ DESC
   def configure(conf)
     super
     log.instance_eval {
-      def add(level, &block)
-        return unless block
+      def add(level, message = nil)
+        if message.nil?
+          if block_given?
+            message = yield
+          else
+            return
+          end
+        end
 
         # Follow rdkakfa's log level. See also rdkafka-ruby's bindings.rb: https://github.com/appsignal/rdkafka-ruby/blob/e5c7261e3f2637554a5c12b924be297d7dca1328/lib/rdkafka/bindings.rb#L117
         case level
         when Logger::FATAL
-          self.fatal(block.call)
+          self.fatal(message)
         when Logger::ERROR
-          self.error(block.call)
+          self.error(message)
         when Logger::WARN
-          self.warn(block.call)
+          self.warn(message)
         when Logger::INFO
-          self.info(block.call)
+          self.info(message)
         when Logger::DEBUG
-          self.debug(block.call)
+          self.debug(message)
         else
-          self.trace(block.call)
+          self.trace(message)
         end
       end
     }

data/lib/fluent/plugin/out_rdkafka2.rb

@@ -56,6 +56,8 @@ DESC
                  :desc => <<-DESC
 Set true to remove topic key from data
 DESC
+    config_param :exclude_fields, :array, :default => [], value_type: :string,
+                 :desc => 'Fields to remove from data where the value is a jsonpath to a record value'
     config_param :headers, :hash, default: {}, symbolize_keys: true, value_type: :string,
                  :desc => 'Kafka message headers'
     config_param :headers_from_record, :hash, default: {}, symbolize_keys: true, value_type: :string,
@@ -110,23 +112,29 @@ DESC
     def configure(conf)
       super
       log.instance_eval {
-        def add(level, &block)
-          return unless block
+        def add(level, message = nil)
+          if message.nil?
+            if block_given?
+              message = yield
+            else
+              return
+            end
+          end
 
           # Follow rdkakfa's log level. See also rdkafka-ruby's bindings.rb: https://github.com/appsignal/rdkafka-ruby/blob/e5c7261e3f2637554a5c12b924be297d7dca1328/lib/rdkafka/bindings.rb#L117
           case level
           when Logger::FATAL
-            self.fatal(block.call)
+            self.fatal(message)
           when Logger::ERROR
-            self.error(block.call)
+            self.error(message)
           when Logger::WARN
-            self.warn(block.call)
+            self.warn(message)
           when Logger::INFO
-            self.info(block.call)
+            self.info(message)
           when Logger::DEBUG
-            self.debug(block.call)
+            self.debug(message)
           else
-            self.trace(block.call)
+            self.trace(message)
           end
         end
       }
@@ -158,6 +166,10 @@ DESC
       @headers_from_record.each do |key, value|
         @headers_from_record_accessors[key] = record_accessor_create(value)
       end
+
+      @exclude_field_accessors = @exclude_fields.map do |field|
+        record_accessor_create(field)
+      end
     end
 
     def build_config
@@ -305,6 +317,12 @@ DESC
               headers[key] = header_accessor.call(record)
             end
 
+            unless @exclude_fields.empty?
+              @exclude_field_accessors.each do |exclude_field_acessor|
+                exclude_field_acessor.delete(record)
+              end
+            end
+
             record_buf = @formatter_proc.call(tag, time, record)
             record_buf_bytes = record_buf.bytesize
             if @max_send_limit_bytes && record_buf_bytes > @max_send_limit_bytes

data/test/plugin/test_out_kafka2.rb

@@ -1,6 +1,8 @@
 require 'helper'
 require 'fluent/test/helpers'
-require 'fluent/output'
+require 'fluent/test/driver/input'
+require 'fluent/test/driver/output'
+require 'securerandom'
 
 class Kafka2OutputTest < Test::Unit::TestCase
   include Fluent::Test::Helpers
@@ -15,8 +17,8 @@ class Kafka2OutputTest < Test::Unit::TestCase
                    ])
   end
 
-  def config
-    base_config + config_element('ROOT', '', {"default_topic" => "kitagawakeiko",
+  def config(default_topic: "kitagawakeiko")
+    base_config + config_element('ROOT', '', {"default_topic" => default_topic,
                                               "brokers" => "localhost:9092"}, [
                                  ])
   end
@@ -57,4 +59,58 @@ class Kafka2OutputTest < Test::Unit::TestCase
     d = create_driver
     assert_equal true, d.instance.multi_workers_ready?
   end
+
+  class WriteTest < self
+    TOPIC_NAME = "kafka-output-#{SecureRandom.uuid}"
+
+    INPUT_CONFIG = %[
+      @type kafka
+      brokers localhost:9092
+      format json
+      @label @kafka
+      topics #{TOPIC_NAME}
+    ]
+
+    def create_target_driver(conf = INPUT_CONFIG)
+      Fluent::Test::Driver::Input.new(Fluent::KafkaInput).configure(conf)
+    end
+
+    def setup
+      @kafka = Kafka.new(["localhost:9092"], client_id: 'kafka')
+    end
+
+    def teardown
+      @kafka.delete_topic(TOPIC_NAME)
+      @kafka.close
+    end
+
+    def test_write
+      target_driver = create_target_driver
+      expected_message = {"a" => 2}
+      target_driver.run(expect_records: 1, timeout: 5) do
+        sleep 2
+        d = create_driver(config(default_topic: TOPIC_NAME))
+        d.run do
+          d.feed("test", event_time, expected_message)
+        end
+      end
+      actual_messages = target_driver.events.collect { |event| event[2] }
+      assert_equal([expected_message], actual_messages)
+    end
+
+    def test_exclude_fields
+      conf = config(default_topic: TOPIC_NAME) +
+             config_element('ROOT', '', {"exclude_fields" => "$.foo"}, [])
+      target_driver = create_target_driver
+      target_driver.run(expect_records: 1, timeout: 5) do
+        sleep 2
+        d = create_driver(conf)
+        d.run do
+          d.feed('test', event_time, {'a' => 'b', 'foo' => 'bar', 'message' => 'test'})
+        end
+      end
+      actual_messages = target_driver.events.collect { |event| event[2] }
+      assert_equal([{'a' => 'b', 'message' => 'test'}], actual_messages)
+    end
+  end
 end

data/test/plugin/test_out_rdkafka2.rb

@@ -0,0 +1,120 @@
+require 'helper'
+require 'fluent/test/helpers'
+require 'fluent/test/driver/input'
+require 'fluent/test/driver/output'
+require 'securerandom'
+
+class Rdkafka2OutputTest < Test::Unit::TestCase
+  include Fluent::Test::Helpers
+
+  def have_rdkafka
+    begin
+      require 'fluent/plugin/out_rdkafka2'
+      true
+    rescue LoadError
+      false
+    end
+  end
+
+  def setup
+    omit_unless(have_rdkafka, "rdkafka isn't installed")
+    Fluent::Test.setup
+  end
+
+  def base_config
+    config_element('ROOT', '', {"@type" => "rdkafka2"}, [
+                     config_element('format', "", {"@type" => "json"})
+                   ])
+  end
+
+  def config(default_topic: "kitagawakeiko")
+    base_config + config_element('ROOT', '', {"default_topic" => default_topic,
+                                              "brokers" => "localhost:9092"}, [
+                                 ])
+  end
+
+  def create_driver(conf = config, tag='test')
+    Fluent::Test::Driver::Output.new(Fluent::Rdkafka2Output).configure(conf)
+  end
+
+  def test_configure
+    assert_nothing_raised(Fluent::ConfigError) {
+      create_driver(base_config)
+    }
+
+    assert_nothing_raised(Fluent::ConfigError) {
+      create_driver(config)
+    }
+
+    assert_nothing_raised(Fluent::ConfigError) {
+      create_driver(config + config_element('buffer', "", {"@type" => "memory"}))
+    }
+
+    d = create_driver
+    assert_equal 'kitagawakeiko', d.instance.default_topic
+    assert_equal 'localhost:9092', d.instance.brokers
+  end
+
+  def test_mutli_worker_support
+    d = create_driver
+    assert_equal true, d.instance.multi_workers_ready?
+  end
+
+  class WriteTest < self
+    TOPIC_NAME = "kafka-output-#{SecureRandom.uuid}"
+
+    INPUT_CONFIG = %[
+      @type kafka
+      brokers localhost:9092
+      format json
+      @label @kafka
+      topics #{TOPIC_NAME}
+    ]
+
+    def create_target_driver(conf = INPUT_CONFIG)
+      Fluent::Test::Driver::Input.new(Fluent::KafkaInput).configure(conf)
+    end
+
+    def setup
+      @kafka = nil
+      omit_unless(have_rdkafka, "rdkafka isn't installed")
+      @kafka = Kafka.new(["localhost:9092"], client_id: 'kafka')
+    end
+
+    def teardown
+      if @kafka
+        @kafka.delete_topic(TOPIC_NAME)
+        @kafka.close
+      end
+    end
+
+    def test_write
+      target_driver = create_target_driver
+      expected_message = {"a" => 2}
+      target_driver.run(expect_records: 1, timeout: 5) do
+        sleep 2
+        d = create_driver(config(default_topic: TOPIC_NAME))
+        d.run do
+          d.feed("test", event_time, expected_message)
+        end
+      end
+      actual_messages = target_driver.events.collect { |event| event[2] }
+      assert_equal([expected_message], actual_messages)
+    end
+
+    def test_exclude_fields
+      conf = config(default_topic: TOPIC_NAME) +
+             config_element('ROOT', '', {"exclude_fields" => "$.foo"}, [])
+      target_driver = create_target_driver
+      target_driver.run(expect_records: 1, timeout: 5) do
+        sleep 2
+        d = create_driver(conf)
+        d.run do
+          d.feed('test', event_time, {'a' => 'b', 'foo' => 'bar', 'message' => 'test'})
+        end
+      end
+      actual_messages = target_driver.events.collect { |event| event[2] }
+      assert_equal([{'a' => 'b', 'message' => 'test'}], actual_messages)
+    end
+  end
+end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.17.0
+  version: 0.17.1
 platform: ruby
 authors:
 - Hidemasa Togashi
 - Masahiro Nakagawa
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-30 00:00:00.000000000 Z
+date: 2021-09-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -173,11 +173,12 @@ files:
 - test/plugin/test_out_kafka.rb
 - test/plugin/test_out_kafka2.rb
 - test/plugin/test_out_kafka_buffered.rb
+- test/plugin/test_out_rdkafka2.rb
 homepage: https://github.com/fluent/fluent-plugin-kafka
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -192,8 +193,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
-signing_key:
+rubygems_version: 3.2.5
+signing_key: 
 specification_version: 4
 summary: Fluentd plugin for Apache Kafka > 0.8
 test_files:
@@ -204,3 +205,4 @@ test_files:
 - test/plugin/test_out_kafka.rb
 - test/plugin/test_out_kafka2.rb
 - test/plugin/test_out_kafka_buffered.rb
+- test/plugin/test_out_rdkafka2.rb