fluent-plugin-kafka 0.15.0 → 0.16.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 43c3a759f4636168c932c33f45c38105ebb522b5ea8222f1b1a7eceb53343348
-  data.tar.gz: c64a103244e721fa2de124f466f2480c960daafc713fd16f685ea4dd4a545a3d
+  metadata.gz: 5f03efe1dee8bff74ba9e0588956bc919244ac6ddf24ff40821d072602dab916
+  data.tar.gz: b081913e347f5dd28d1339a1eca3f063661217fa4f1738787161cb825c3485fc
 SHA512:
-  metadata.gz: 707d92f2a23041b53daf6410d3fadb0e84053c4eb250b20c6dd3c72a15969273d2279b71950334187d156767bf6646a0af468a0f84e85ca683a34c127e47e363
-  data.tar.gz: 978883c8a72152bb6b9262ccea4e6b65b91bca1a3907ea43a7930cf7b4d414f1a9f47cb593d420738a48bd47d86451f376fe4cc6e7dec6b4f2c4e81ad5213d00
+  metadata.gz: 03e89037f5982ca8aa121c1c1672160e95fb488ee7e9ca6bd8038e8d9b0cc9d99348ab91a2e1896e174fed4fa2b4c25fb9e65ee72b9a4ba74062c631953ca345
+  data.tar.gz: 5ca6f6093a1f46ab89b4b268416c16d9758102b76fcdb2bba8d2f235daa7b88e9914e48a0e4d0397df13ea9159a21ba1bbe525186670fd7b7622cfdd3d2cac2f

data/.github/workflows/linux.yml

@@ -0,0 +1,26 @@
+name: linux
+on:
+  - push
+  - pull_request
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ '2.4', '2.5', '2.6', '2.7', '3.0' ]
+        os:
+          - ubuntu-latest
+    name: Ruby ${{ matrix.ruby }} unit testing on ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+    - name: unit testing
+      env:
+        CI: true
+      run: |
+        gem install bundler rake
+        bundle install --jobs 4 --retry 3
+        bundle exec rake test

data/ChangeLog

@@ -1,3 +1,24 @@
+Release 0.16.1 - 2021/04/14
+	* out_kafka/out_kafka_buffered: Support Ruby 3.0.0 keyword arguments interop
+	* kafka_plugin_util: Treat empty string in read_ssl_file as nil
+
+Release 0.16.0 - 2021/01/25
+
+	* input: Add `tag_source` and `record_tag_key` parameters for using record field as tag
+	* in_kafka_group: Use NumericParser for floating point
+
+Release 0.15.3 - 2020/12/08
+
+	* in_kafka: Fix `record_time_key` parameter not working
+
+Release 0.15.2 - 2020/09/30
+
+	* input: Support 3rd party parser
+
+Release 0.15.1 - 2020/09/17
+
+	* out_kafka2: Fix wrong class name for configuration error
+
 Release 0.15.0 - 2020/09/14
 
 	* Add experimental `in_rdkafka_group`
@@ -20,6 +41,7 @@ Release 0.14.0 - 2020/08/07
 Release 0.13.1 - 2020/07/17
 
 	* in_kafka_group: Support ssl_verify_hostname parameter
+	* in_kafka_group: Support regex based topics
 	* out_kafka2/out_rdkafka2: Support topic parameter with placeholders
 
 Release 0.13.0 - 2020/03/09

data/README.md

@@ -139,6 +139,8 @@ Consume events by kafka consumer group features..
 
 See also [ruby-kafka README](https://github.com/zendesk/ruby-kafka#consuming-messages-from-kafka) for more detailed documentation about ruby-kafka options.
 
+`topics` supports regex pattern since v0.13.1. If you want to use regex pattern, use `/pattern/` like `/foo.*/`.
+
 Consuming topic name is used for event tag. So when the target topic name is `app_event`, the tag is `app_event`. If you want to modify tag, use `add_prefix` or `add_suffix` parameter. With `add_prefix kafka`, the tag is `kafka.app_event`.
 
 ### Input plugin (@type 'rdkafka_group', supports kafka consumer groups, uses rdkafka-ruby)

data/fluent-plugin-kafka.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.15.0'
+  gem.version       = '0.16.1'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
@@ -21,4 +21,5 @@ Gem::Specification.new do |gem|
   gem.add_dependency 'ruby-kafka', '>= 1.2.0', '< 2'
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "test-unit", ">= 3.0.8"
+  gem.add_development_dependency "webrick"
 end

data/lib/fluent/plugin/in_kafka.rb

@@ -31,6 +31,10 @@ class Fluent::KafkaInput < Fluent::Input
   config_param :add_suffix, :string, :default => nil,
                :desc => "tag suffix"
   config_param :add_offset_in_record, :bool, :default => false
+  config_param :tag_source, :enum, :list => [:topic, :record], :default => :topic,
+               :desc => "Source for the fluentd event tag"
+  config_param :record_tag_key, :string, :default => 'tag',
+               :desc => "Tag field when tag_source is 'record'"
 
   config_param :offset_zookeeper, :string, :default => nil
   config_param :offset_zk_root_node, :string, :default => '/fluent-plugin-kafka'
@@ -113,7 +117,7 @@ class Fluent::KafkaInput < Fluent::Input
 
     require 'zookeeper' if @offset_zookeeper
 
-    @parser_proc = setup_parser
+    @parser_proc = setup_parser(conf)
 
     @time_source = :record if @use_record_time
 
@@ -126,7 +130,7 @@ class Fluent::KafkaInput < Fluent::Input
     end
   end
 
-  def setup_parser
+  def setup_parser(conf)
     case @format
     when 'json'
       begin
@@ -165,6 +169,14 @@ class Fluent::KafkaInput < Fluent::Input
         add_offset_in_hash(r, te, msg.offset) if @add_offset_in_record
         r
       }
+    else
+      @custom_parser = Fluent::Plugin.new_parser(conf['format'])
+      @custom_parser.configure(conf)
+      Proc.new { |msg|
+        @custom_parser.parse(msg.value) {|_time, record|
+          record
+        }
+      }
     end
   end
 
@@ -216,6 +228,9 @@ class Fluent::KafkaInput < Fluent::Input
         router,
         @kafka_message_key,
         @time_source,
+        @record_time_key,
+        @tag_source,
+        @record_tag_key,
         opt)
     }
     @topic_watchers.each {|tw|
@@ -240,7 +255,7 @@ class Fluent::KafkaInput < Fluent::Input
   end
 
   class TopicWatcher < Coolio::TimerWatcher
-    def initialize(topic_entry, kafka, interval, parser, add_prefix, add_suffix, offset_manager, router, kafka_message_key, time_source, options={})
+    def initialize(topic_entry, kafka, interval, parser, add_prefix, add_suffix, offset_manager, router, kafka_message_key, time_source, record_time_key, tag_source, record_tag_key, options={})
       @topic_entry = topic_entry
       @kafka = kafka
       @callback = method(:consume)
@@ -252,6 +267,9 @@ class Fluent::KafkaInput < Fluent::Input
       @router = router
       @kafka_message_key = kafka_message_key
       @time_source = time_source
+      @record_time_key = record_time_key
+      @tag_source = tag_source
+      @record_tag_key = record_tag_key
 
       @next_offset = @topic_entry.offset
       if @topic_entry.offset == -1 && offset_manager
@@ -288,6 +306,11 @@ class Fluent::KafkaInput < Fluent::Input
       messages.each { |msg|
         begin
           record = @parser.call(msg, @topic_entry)
+          if @tag_source == :record
+            tag = record[@record_tag_key]
+            tag = @add_prefix + "." + tag if @add_prefix
+            tag = tag + "." + @add_suffix if @add_suffix
+          end
           case @time_source
           when :kafka
             record_time = Fluent::EventTime.from_time(msg.create_time)

data/lib/fluent/plugin/in_kafka_group.rb

@@ -36,6 +36,10 @@ class Fluent::KafkaGroupInput < Fluent::Input
   config_param :get_kafka_client_log, :bool, :default => false
   config_param :time_format, :string, :default => nil,
                :desc => "Time format to be used to parse 'time' field."
+  config_param :tag_source, :enum, :list => [:topic, :record], :default => :topic,
+               :desc => "Source for the fluentd event tag"
+  config_param :record_tag_key, :string, :default => 'tag',
+               :desc => "Tag field when tag_source is 'record'"
   config_param :kafka_message_key, :string, :default => nil,
                :desc => "Set kafka's message key to this field"
   config_param :connect_timeout, :integer, :default => nil,
@@ -117,7 +121,7 @@ class Fluent::KafkaGroupInput < Fluent::Input
       @max_wait_time = conf['max_wait_ms'].to_i / 1000
     end
 
-    @parser_proc = setup_parser
+    @parser_proc = setup_parser(conf)
 
     @consumer_opts = {:group_id => @consumer_group}
     @consumer_opts[:session_timeout] = @session_timeout if @session_timeout
@@ -138,9 +142,13 @@ class Fluent::KafkaGroupInput < Fluent::Input
         @time_parser = Fluent::TextParser::TimeParser.new(@time_format)
       end
     end
+
+    if @time_source == :record && defined?(Fluent::NumericTimeParser)
+      @float_numeric_parse = Fluent::NumericTimeParser.new(:float)
+    end
   end
 
-  def setup_parser
+  def setup_parser(conf)
     case @format
     when 'json'
       begin
@@ -159,6 +167,14 @@ class Fluent::KafkaGroupInput < Fluent::Input
       Proc.new { |msg| MessagePack.unpack(msg.value) }
     when 'text'
       Proc.new { |msg| {@message_key => msg.value} }
+    else
+      @custom_parser = Fluent::Plugin.new_parser(conf['format'])
+      @custom_parser.configure(conf)
+      Proc.new { |msg|
+        @custom_parser.parse(msg.value) {|_time, record|
+          record
+        }
+      }
     end
   end
 
@@ -236,49 +252,104 @@ class Fluent::KafkaGroupInput < Fluent::Input
     end
   end
 
+  def process_batch_with_record_tag(batch)
+    es = {} 
+    batch.messages.each { |msg|
+      begin
+        record = @parser_proc.call(msg)
+        tag = record[@record_tag_key]
+        tag = @add_prefix + "." + tag if @add_prefix
+        tag = tag + "." + @add_suffix if @add_suffix
+        es[tag] ||= Fluent::MultiEventStream.new
+        case @time_source
+        when :kafka
+          record_time = Fluent::EventTime.from_time(msg.create_time)
+        when :now
+          record_time = Fluent::Engine.now
+        when :record
+          if @time_format
+            record_time = @time_parser.parse(record[@record_time_key].to_s)
+          else
+            record_time = record[@record_time_key]
+          end
+        else
+          log.fatal "BUG: invalid time_source: #{@time_source}"
+        end
+        if @kafka_message_key
+          record[@kafka_message_key] = msg.key
+        end
+        if @add_headers
+          msg.headers.each_pair { |k, v|
+            record[k] = v
+          }
+        end
+        es[tag].add(record_time, record)
+      rescue => e
+        log.warn "parser error in #{batch.topic}/#{batch.partition}", :error => e.to_s, :value => msg.value, :offset => msg.offset
+        log.debug_backtrace
+      end
+    }
+
+    unless es.empty?
+      es.each { |tag,es|
+        emit_events(tag, es)
+      }
+    end
+  end
+
+  def process_batch(batch)
+    es = Fluent::MultiEventStream.new
+    tag = batch.topic
+    tag = @add_prefix + "." + tag if @add_prefix
+    tag = tag + "." + @add_suffix if @add_suffix
+
+    batch.messages.each { |msg|
+      begin
+        record = @parser_proc.call(msg)
+        case @time_source
+        when :kafka
+          record_time = Fluent::EventTime.from_time(msg.create_time)
+        when :now
+          record_time = Fluent::Engine.now
+        when :record
+          record_time = record[@record_time_key]
+
+          if @time_format
+            record_time = @time_parser.parse(record_time.to_s)
+          elsif record_time.is_a?(Float) && @float_numeric_parse
+            record_time = @float_numeric_parse.parse(record_time)
+          end
+        else
+          log.fatal "BUG: invalid time_source: #{@time_source}"
+        end
+        if @kafka_message_key
+          record[@kafka_message_key] = msg.key
+        end
+        if @add_headers
+          msg.headers.each_pair { |k, v|
+            record[k] = v
+          }
+        end
+        es.add(record_time, record)
+      rescue => e
+        log.warn "parser error in #{batch.topic}/#{batch.partition}", :error => e.to_s, :value => msg.value, :offset => msg.offset
+        log.debug_backtrace
+      end
+    }
+
+    unless es.empty?
+      emit_events(tag, es)
+    end
+  end
+
   def run
     while @consumer
       begin
         @consumer.each_batch(@fetch_opts) { |batch|
-          es = Fluent::MultiEventStream.new
-          tag = batch.topic
-          tag = @add_prefix + "." + tag if @add_prefix
-          tag = tag + "." + @add_suffix if @add_suffix
-
-          batch.messages.each { |msg|
-            begin
-              record = @parser_proc.call(msg)
-              case @time_source
-              when :kafka
-                record_time = Fluent::EventTime.from_time(msg.create_time)
-              when :now
-                record_time = Fluent::Engine.now
-              when :record
-                if @time_format
-                  record_time = @time_parser.parse(record[@record_time_key].to_s)
-                else
-                  record_time = record[@record_time_key]
-                end
-              else
-                log.fatal "BUG: invalid time_source: #{@time_source}"
-              end
-              if @kafka_message_key
-                record[@kafka_message_key] = msg.key
-              end
-              if @add_headers
-                msg.headers.each_pair { |k, v|
-                  record[k] = v
-                }
-              end
-              es.add(record_time, record)
-            rescue => e
-              log.warn "parser error in #{batch.topic}/#{batch.partition}", :error => e.to_s, :value => msg.value, :offset => msg.offset
-              log.debug_backtrace
-            end
-          }
-
-          unless es.empty?
-            emit_events(tag, es)
+          if @tag_source == :record
+            process_batch_with_record_tag(batch)
+          else
+            process_batch(batch) 
           end
         }
       rescue ForShutdown

data/lib/fluent/plugin/in_rdkafka_group.rb

@@ -7,7 +7,7 @@ require 'rdkafka'
 class Fluent::Plugin::RdKafkaGroupInput < Fluent::Plugin::Input
   Fluent::Plugin.register_input('rdkafka_group', self)
 
-  helpers :thread
+  helpers :thread, :parser, :compat_parameters
 
   config_param :topics, :string,
                :desc => "Listening topics(separate with comma',')."
@@ -41,15 +41,19 @@ class Fluent::Plugin::RdKafkaGroupInput < Fluent::Plugin::Input
                :desc => "If set true, it disables retry_limit and make Fluentd retry indefinitely (default: false)"
   config_param :retry_limit, :integer, :default => 10,
                :desc => "The maximum number of retries for connecting kafka (default: 10)"
- 
+
   config_param :max_wait_time_ms, :integer, :default => 250,
                :desc => "How long to block polls in milliseconds until the server sends us data."
   config_param :max_batch_size, :integer, :default => 10000,
                :desc => "Maximum number of log lines emitted in a single batch."
- 
+
   config_param :kafka_configs, :hash, :default => {},
                :desc => "Kafka configuration properties as desribed in https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md"
 
+  config_section :parse do
+    config_set_default :@type, 'json'
+  end
+
   include Fluent::KafkaPluginUtil::SSLSettings
   include Fluent::KafkaPluginUtil::SaslSettings
 
@@ -80,6 +84,8 @@ class Fluent::Plugin::RdKafkaGroupInput < Fluent::Plugin::Input
   private :_config_to_array
 
   def configure(conf)
+    compat_parameters_convert(conf, :parser)
+
     super
 
     log.warn "The in_rdkafka_group consumer was not yet tested under heavy production load. Use it at your own risk!"
@@ -89,7 +95,14 @@ class Fluent::Plugin::RdKafkaGroupInput < Fluent::Plugin::Input
 
     @topics = _config_to_array(@topics)
 
-    @parser_proc = setup_parser
+    parser_conf = conf.elements('parse').first
+    unless parser_conf
+      raise Fluent::ConfigError, "<parse> section or format parameter is required."
+    end
+    unless parser_conf["@type"]
+      raise Fluent::ConfigError, "parse/@type is required."
+    end
+    @parser_proc = setup_parser(parser_conf)
 
     @time_source = :record if @use_record_time
 
@@ -98,8 +111,9 @@ class Fluent::Plugin::RdKafkaGroupInput < Fluent::Plugin::Input
     end
   end
 
-  def setup_parser
-    case @format
+  def setup_parser(parser_conf)
+    format = parser_conf["@type"]
+    case format
     when 'json'
       begin
         require 'oj'
@@ -117,6 +131,13 @@ class Fluent::Plugin::RdKafkaGroupInput < Fluent::Plugin::Input
       Proc.new { |msg| MessagePack.unpack(msg.payload) }
     when 'text'
       Proc.new { |msg| {@message_key => msg.payload} }
+    else
+      @custom_parser = parser_create(usage: 'in-rdkafka-plugin', conf: parser_conf)
+      Proc.new { |msg|
+        @custom_parser.parse(msg.payload) {|_time, record|
+          record
+        }
+      }
     end
   end
 

data/lib/fluent/plugin/kafka_plugin_util.rb

@@ -33,7 +33,7 @@ module Fluent
       end
 
       def read_ssl_file(path)
-        return nil if path.nil?
+        return nil if path.nil? || path.respond_to?(:strip) && path.strip.empty?
 
         if path.is_a?(Array)
           path.map { |fp| File.read(fp) }

data/lib/fluent/plugin/out_kafka.rb

@@ -215,7 +215,7 @@ DESC
       chain.next
 
       # out_kafka is mainly for testing so don't need the performance unlike out_kafka_buffered.
-      producer = @kafka.producer(@producer_opts)
+      producer = @kafka.producer(**@producer_opts)
 
       es.each do |time, record|
         if @output_include_time

data/lib/fluent/plugin/out_kafka2.rb

@@ -128,7 +128,7 @@ DESC
         @seed_brokers = @brokers
         log.info "brokers has been set: #{@seed_brokers}"
       else
-        raise Fluent::Config, 'No brokers specified. Need one broker at least.'
+        raise Fluent::ConfigError, 'No brokers specified. Need one broker at least.'
       end
 
       formatter_conf = conf.elements('format').first

data/lib/fluent/plugin/out_kafka_buffered.rb

@@ -239,7 +239,7 @@ DESC
     @producers_mutex.synchronize {
       producer = @producers[Thread.current.object_id]
       unless producer
-        producer = @kafka.producer(@producer_opts)
+        producer = @kafka.producer(**@producer_opts)
         @producers[Thread.current.object_id] = producer
       end
       producer

data/test/plugin/test_kafka_plugin_util.rb

@@ -0,0 +1,38 @@
+require 'helper'
+require 'fluent/plugin/kafka_plugin_util'
+
+class File
+    def File::read(path)
+        path
+    end
+end
+
+class KafkaPluginUtilTest < Test::Unit::TestCase
+    
+    def self.config_param(name, type, options)
+    end
+    include Fluent::KafkaPluginUtil::SSLSettings
+
+    def config_param
+    end
+    def setup
+        Fluent::Test.setup
+    end
+
+    def test_read_ssl_file_when_nil
+        assert_equal(nil, read_ssl_file(nil))
+    end
+
+    def test_read_ssl_file_when_empty_string
+        assert_equal(nil, read_ssl_file(""))
+    end
+
+    def test_read_ssl_file_when_non_empty_path
+        assert_equal("path", read_ssl_file("path"))
+    end
+
+    def test_read_ssl_file_when_non_empty_array
+        assert_equal(["a","b"], read_ssl_file(["a","b"]))
+    end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.15.0
+  version: 0.16.1
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-09-14 00:00:00.000000000 Z
+date: 2021-04-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -93,6 +93,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.0.8
+- !ruby/object:Gem::Dependency
+  name: webrick
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Fluentd plugin for Apache Kafka > 0.8
 email:
 - togachiro@gmail.com
@@ -101,6 +115,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/linux.yml"
 - ".gitignore"
 - ".travis.yml"
 - ChangeLog
@@ -120,6 +135,7 @@ files:
 - lib/fluent/plugin/out_rdkafka.rb
 - lib/fluent/plugin/out_rdkafka2.rb
 - test/helper.rb
+- test/plugin/test_kafka_plugin_util.rb
 - test/plugin/test_out_kafka.rb
 homepage: https://github.com/fluent/fluent-plugin-kafka
 licenses:
@@ -140,10 +156,11 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin for Apache Kafka > 0.8
 test_files:
 - test/helper.rb
+- test/plugin/test_kafka_plugin_util.rb
 - test/plugin/test_out_kafka.rb