fluent-plugin-kafka 0.12.0 → 0.12.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 76970655d31ac19d5088faf9e2d6afe0d4bd4f239f7fea5373a9e1b90e292e1f
-  data.tar.gz: 50c1999dc3f5196eaceb3c8a65ca041e83ce4e85e0d190229340a711bd9ea96d
+  metadata.gz: 497c5a450bc4f55ddaf1b46454ed80f028c283775666e2a38ee0d91f16d391ef
+  data.tar.gz: 5f40c26e75f06e4ee73cc0b3ba32ef8438da48bea465549fb95f4bcb89e98819
 SHA512:
-  metadata.gz: 99464865d2cc92905645e656e38380b1dcfde57027666bde9d31eb8e0c109e086053ecdb2b1347118c6e2f0805f6acbcb3591fff044a7a92985ed7328af4869b
-  data.tar.gz: '01296913c6807deec4e33bd067517c8641ddf9e59ac8ea410ad1861ce6ab64b69aa8ac308c505b8ec44e94408b6d49d8bf63754c9ec4a55a589e291c12741550'
+  metadata.gz: dcac3281832905427ad5799b6b1d7c9dab9b13562c19ba8283fa4eaf2ea1b88e7c782714be7cec9546878e26ab223725b0cbc06259bc8e375eff5e407575bb3f
+  data.tar.gz: deef97208222254e9fc4b0279f0dd9045c177354fa1a8dcca3a304d3b2a5d1090e1e7970b5cd9066d7ff917e8020b798d4dd674623285c52041c173fe37d48f5

data/ChangeLog

@@ -1,3 +1,7 @@
+Release 0.12.1 - 2019/10/14
+
+	* input: Add time_source parameter to replace use_record_time
+
 Release 0.12.0 - 2019/09/27
 
 	* Support rdkafka-ruby v0.7. Drop v0.6 support.

data/fluent-plugin-kafka.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.12.0'
+  gem.version       = '0.12.1'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/in_kafka.rb

@@ -35,10 +35,13 @@ class Fluent::KafkaInput < Fluent::Input
   config_param :offset_zookeeper, :string, :default => nil
   config_param :offset_zk_root_node, :string, :default => '/fluent-plugin-kafka'
   config_param :use_record_time, :bool, :default => false,
-               :desc => "Replace message timestamp with contents of 'time' field."
+               :desc => "Replace message timestamp with contents of 'time' field.",
+               :deprecated => "Use 'time_source record' instead."
+  config_param :time_source, :enum, :list => [:now, :kafka, :record], :default => :now,
+               :desc => "Source for message timestamp."
   config_param :get_kafka_client_log, :bool, :default => false
   config_param :time_format, :string, :default => nil,
-               :desc => "Time format to be used to parse 'time' filed."
+               :desc => "Time format to be used to parse 'time' field."
   config_param :kafka_message_key, :string, :default => nil,
                :desc => "Set kafka's message key to this field"
 
@@ -110,7 +113,9 @@ class Fluent::KafkaInput < Fluent::Input
 
     @parser_proc = setup_parser
 
-    if @use_record_time and @time_format
+    @time_source = :record if @use_record_time
+
+    if @time_source == :record and @time_format
       if defined?(Fluent::TimeParser)
         @time_parser = Fluent::TimeParser.new(@time_format)
       else
@@ -206,6 +211,7 @@ class Fluent::KafkaInput < Fluent::Input
         offset_manager,
         router,
         @kafka_message_key,
+        @time_source,
         opt)
     }
     @topic_watchers.each {|tw|
@@ -230,7 +236,7 @@ class Fluent::KafkaInput < Fluent::Input
   end
 
   class TopicWatcher < Coolio::TimerWatcher
-    def initialize(topic_entry, kafka, interval, parser, add_prefix, add_suffix, offset_manager, router, kafka_message_key, options={})
+    def initialize(topic_entry, kafka, interval, parser, add_prefix, add_suffix, offset_manager, router, kafka_message_key, time_source, options={})
       @topic_entry = topic_entry
       @kafka = kafka
       @callback = method(:consume)
@@ -241,6 +247,7 @@ class Fluent::KafkaInput < Fluent::Input
       @offset_manager = offset_manager
       @router = router
       @kafka_message_key = kafka_message_key
+      @time_source = time_source
 
       @next_offset = @topic_entry.offset
       if @topic_entry.offset == -1 && offset_manager
@@ -277,14 +284,19 @@ class Fluent::KafkaInput < Fluent::Input
       messages.each { |msg|
         begin
           record = @parser.call(msg, @topic_entry)
-          if @use_record_time
+          case @time_source
+          when :kafka
+            record_time = Fluent::EventTime.from_time(msg.create_time)
+          when :now
+            record_time = Fluent::Engine.now
+          when :record
             if @time_format
               record_time = @time_parser.parse(record['time'])
             else
               record_time = record['time']
             end
           else
-            record_time = Fluent::Engine.now
+            $log.fatal "BUG: invalid time_source: #{@time_source}"
           end
           if @kafka_message_key
             record[@kafka_message_key] = msg.key

data/lib/fluent/plugin/in_kafka_group.rb

@@ -25,10 +25,13 @@ class Fluent::KafkaGroupInput < Fluent::Input
   config_param :retry_emit_limit, :integer, :default => nil,
                :desc => "How long to stop event consuming when BufferQueueLimitError happens. Wait retry_emit_limit x 1s. The default is waiting until BufferQueueLimitError is resolved"
   config_param :use_record_time, :bool, :default => false,
-               :desc => "Replace message timestamp with contents of 'time' field."
+               :desc => "Replace message timestamp with contents of 'time' field.",
+               :deprecated => "Use 'time_source record' instead."
+  config_param :time_source, :enum, :list => [:now, :kafka, :record], :default => :now,
+               :desc => "Source for message timestamp."
   config_param :get_kafka_client_log, :bool, :default => false
   config_param :time_format, :string, :default => nil,
-               :desc => "Time format to be used to parse 'time' filed."
+               :desc => "Time format to be used to parse 'time' field."
   config_param :kafka_message_key, :string, :default => nil,
                :desc => "Set kafka's message key to this field"
   config_param :connect_timeout, :integer, :default => nil,
@@ -122,7 +125,9 @@ class Fluent::KafkaGroupInput < Fluent::Input
     @fetch_opts[:max_wait_time] = @max_wait_time if @max_wait_time
     @fetch_opts[:min_bytes] = @min_bytes if @min_bytes
 
-    if @use_record_time and @time_format
+    @time_source = :record if @use_record_time
+
+    if @time_source == :record and @time_format
       if defined?(Fluent::TimeParser)
         @time_parser = Fluent::TimeParser.new(@time_format)
       else
@@ -230,14 +235,19 @@ class Fluent::KafkaGroupInput < Fluent::Input
           batch.messages.each { |msg|
             begin
               record = @parser_proc.call(msg)
-              if @use_record_time
+              case @time_source
+              when :kafka
+                record_time = Fluent::EventTime.from_time(msg.create_time)
+              when :now
+                record_time = Fluent::Engine.now
+              when :record
                 if @time_format
                   record_time = @time_parser.parse(record['time'].to_s)
                 else
                   record_time = record['time']
                 end
               else
-                record_time = Fluent::Engine.now
+                log.fatal "BUG: invalid time_source: #{@time_source}"
               end
               if @kafka_message_key
                 record[@kafka_message_key] = msg.key

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.12.0
+  version: 0.12.1
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-09-27 00:00:00.000000000 Z
+date: 2019-10-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd