fluent-plugin-kafka 0.15.1 → 0.15.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 56e2a0fc884e2ef670d81e61b180de9ccc91eb6b67b9f0edb8015af4212e76ce
-  data.tar.gz: 1fa73921dba4cc833e4032f5636287659e874ab98c210b20ee5a71a0194bc40a
+  metadata.gz: 866fb421d7097ccbac1bde2e279d9975bce4de086dab74609cadbe50429bb312
+  data.tar.gz: 0e763fc4276177949f6cec2b4839ad5168892cd4d346d633f81449fdb8df84d7
 SHA512:
-  metadata.gz: f7ec3be524feb670cd1823a948dc37b600142107c9949a596cc5b4aa4948e1a83387113ccb5319d1df3484bbcdf7d0df8981f1fbcf7281053f1a0419f80cc5e8
-  data.tar.gz: 3c28d90f1863317f3da9efde6fc217f8754bbea942f1d94714cb4cce3eac210691416f8aab9f1e76c50ab2a372365358390e283c53e83adc24ead06468b81542
+  metadata.gz: b41f5cb35d1c4dea3743e513b505e11f634dfcbc33e339f188f2ac4c2b710ed1357c00779e24873c5f4a0bdd5326f8c1a731b4f2a4c323dc6fde8b85bc78ef28
+  data.tar.gz: df4061316f692fbe264b2344fd74f7ba1d15174bb91a617c09f5d9d3de6d50a5f0b6c8aa702ce9322d438ee6ed83b09cf620e71537e96e13db73882f01e291cb

data/ChangeLog

@@ -1,3 +1,7 @@
+Release 0.15.2 - 2020/09/30
+
+	* input: Support 3rd party parser
+
 Release 0.15.1 - 2020/09/17
 
 	* out_kafka2: Fix wrong class name for configuration error

data/fluent-plugin-kafka.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.15.1'
+  gem.version       = '0.15.2'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/in_kafka.rb

@@ -113,7 +113,7 @@ class Fluent::KafkaInput < Fluent::Input
 
     require 'zookeeper' if @offset_zookeeper
 
-    @parser_proc = setup_parser
+    @parser_proc = setup_parser(conf)
 
     @time_source = :record if @use_record_time
 
@@ -126,7 +126,7 @@ class Fluent::KafkaInput < Fluent::Input
     end
   end
 
-  def setup_parser
+  def setup_parser(conf)
     case @format
     when 'json'
       begin
@@ -165,6 +165,14 @@ class Fluent::KafkaInput < Fluent::Input
         add_offset_in_hash(r, te, msg.offset) if @add_offset_in_record
         r
       }
+    else
+      @custom_parser = Fluent::Plugin.new_parser(conf['format'])
+      @custom_parser.configure(conf)
+      Proc.new { |msg|
+        @custom_parser.parse(msg.value) {|_time, record|
+          record
+        }
+      }
     end
   end
 

data/lib/fluent/plugin/in_kafka_group.rb

@@ -117,7 +117,7 @@ class Fluent::KafkaGroupInput < Fluent::Input
       @max_wait_time = conf['max_wait_ms'].to_i / 1000
     end
 
-    @parser_proc = setup_parser
+    @parser_proc = setup_parser(conf)
 
     @consumer_opts = {:group_id => @consumer_group}
     @consumer_opts[:session_timeout] = @session_timeout if @session_timeout
@@ -140,7 +140,7 @@ class Fluent::KafkaGroupInput < Fluent::Input
     end
   end
 
-  def setup_parser
+  def setup_parser(conf)
     case @format
     when 'json'
       begin
@@ -159,6 +159,14 @@ class Fluent::KafkaGroupInput < Fluent::Input
       Proc.new { |msg| MessagePack.unpack(msg.value) }
     when 'text'
       Proc.new { |msg| {@message_key => msg.value} }
+    else
+      @custom_parser = Fluent::Plugin.new_parser(conf['format'])
+      @custom_parser.configure(conf)
+      Proc.new { |msg|
+        @custom_parser.parse(msg.value) {|_time, record|
+          record
+        }
+      }
     end
   end
 

data/lib/fluent/plugin/in_rdkafka_group.rb

@@ -7,7 +7,7 @@ require 'rdkafka'
 class Fluent::Plugin::RdKafkaGroupInput < Fluent::Plugin::Input
   Fluent::Plugin.register_input('rdkafka_group', self)
 
-  helpers :thread
+  helpers :thread, :parser, :compat_parameters
 
   config_param :topics, :string,
                :desc => "Listening topics(separate with comma',')."
@@ -41,15 +41,19 @@ class Fluent::Plugin::RdKafkaGroupInput < Fluent::Plugin::Input
                :desc => "If set true, it disables retry_limit and make Fluentd retry indefinitely (default: false)"
   config_param :retry_limit, :integer, :default => 10,
                :desc => "The maximum number of retries for connecting kafka (default: 10)"
- 
+
   config_param :max_wait_time_ms, :integer, :default => 250,
                :desc => "How long to block polls in milliseconds until the server sends us data."
   config_param :max_batch_size, :integer, :default => 10000,
                :desc => "Maximum number of log lines emitted in a single batch."
- 
+
   config_param :kafka_configs, :hash, :default => {},
                :desc => "Kafka configuration properties as desribed in https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md"
 
+  config_section :parse do
+    config_set_default :@type, 'json'
+  end
+
   include Fluent::KafkaPluginUtil::SSLSettings
   include Fluent::KafkaPluginUtil::SaslSettings
 
@@ -80,6 +84,8 @@ class Fluent::Plugin::RdKafkaGroupInput < Fluent::Plugin::Input
   private :_config_to_array
 
   def configure(conf)
+    compat_parameters_convert(conf, :parser)
+
     super
 
     log.warn "The in_rdkafka_group consumer was not yet tested under heavy production load. Use it at your own risk!"
@@ -89,7 +95,14 @@ class Fluent::Plugin::RdKafkaGroupInput < Fluent::Plugin::Input
 
     @topics = _config_to_array(@topics)
 
-    @parser_proc = setup_parser
+    parser_conf = conf.elements('parse').first
+    unless parser_conf
+      raise Fluent::ConfigError, "<parse> section or format parameter is required."
+    end
+    unless parser_conf["@type"]
+      raise Fluent::ConfigError, "parse/@type is required."
+    end
+    @parser_proc = setup_parser(parser_conf)
 
     @time_source = :record if @use_record_time
 
@@ -98,8 +111,9 @@ class Fluent::Plugin::RdKafkaGroupInput < Fluent::Plugin::Input
     end
   end
 
-  def setup_parser
-    case @format
+  def setup_parser(parser_conf)
+    format = parser_conf["@type"]
+    case format
     when 'json'
       begin
         require 'oj'
@@ -117,6 +131,13 @@ class Fluent::Plugin::RdKafkaGroupInput < Fluent::Plugin::Input
       Proc.new { |msg| MessagePack.unpack(msg.payload) }
     when 'text'
       Proc.new { |msg| {@message_key => msg.payload} }
+    else
+      @custom_parser = parser_create(usage: 'in-rdkafka-plugin', conf: parser_conf)
+      Proc.new { |msg|
+        @custom_parser.parse(msg.payload) {|_time, record|
+          record
+        }
+      }
     end
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.15.1
+  version: 0.15.2
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-09-17 00:00:00.000000000 Z
+date: 2020-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd