fluent-plugin-kafka 0.4.2.rc1 → 0.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 52c06cbf0456782d2a59f18dcc66b3e648efa5cd
-  data.tar.gz: f441973708ffe14e5bde106a1b0976d68d508344
+  metadata.gz: 3022fed18061233c956a0b13292cbebb9da8a79f
+  data.tar.gz: 5e8f4ecf691f72b620b21358a60f8d8994b9e359
 SHA512:
-  metadata.gz: 67da7b3ac7246a8d25c761a9e746d6d6830b5fd8a2cc2c5be8a958d6e0b498bb5277532400cbd37aece628fc159cb1d6e9bf9795d5129a13f42f3d7f374c9514
-  data.tar.gz: c8629214c3834372acd04e0979559d6b11015e80e799adf478842fdd533bb649934b2470a4e087dc1913d9e221e1238fb20912ddb31b0a8d6fc2ff94ebe51061
+  metadata.gz: fb824c849ef4feb963a1c6675717aadcaea78969ed30285b1d871f9e2a8f62d026489ca660a849a79c4ac8040673ec816b870037647fdde1dc19b7cf772c4f4f
+  data.tar.gz: e1c6ff1982adcac8ce67f0c7f87f94c6b1b497b92f994a4c5c50ac8d55151c01aa2f798dcf7c873ea5158069b835ea92255bc177da972df83b07c641a67c1aaf

data/ChangeLog

@@ -1,3 +1,8 @@
+Release 0.4.2 - 2016012/10
+
+	* input: Add use_record_time and time_format parameters
+	* Update ruby-kafka dependency to 0.3.16.beta2
+
 Release 0.4.1 - 2016/12/01
 
 	* output: Support specifying partition

data/fluent-plugin-kafka.gemspec

@@ -12,7 +12,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "fluent-plugin-kafka"
   gem.require_paths = ["lib"]
-  gem.version       = '0.4.2.rc1'
+  gem.version       = '0.4.2'
   gem.required_ruby_version = ">= 2.1.0"
 
   gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]

data/lib/fluent/plugin/in_kafka.rb

@@ -1,4 +1,5 @@
 require 'fluent/input'
+require 'fluent/time'
 require 'fluent/plugin/kafka_plugin_util'
 
 class Fluent::KafkaInput < Fluent::Input
@@ -31,6 +32,10 @@ class Fluent::KafkaInput < Fluent::Input
 
   config_param :offset_zookeeper, :string, :default => nil
   config_param :offset_zk_root_node, :string, :default => '/fluent-plugin-kafka'
+  config_param :use_record_time, :bool, :default => false,
+               :desc => "Replace message timestamp with contents of 'time' field."
+  config_param :time_format, :string, :default => nil,
+               :desc => "Time format to be used to parse 'time' filed."
 
   # Kafka#fetch_messages options
   config_param :max_bytes, :integer, :default => nil,
@@ -49,6 +54,8 @@ class Fluent::KafkaInput < Fluent::Input
   def initialize
     super
     require 'kafka'
+
+    @time_parser = nil
   end
 
   def configure(conf)
@@ -96,6 +103,10 @@ class Fluent::KafkaInput < Fluent::Input
     require 'zookeeper' if @offset_zookeeper
 
     @parser_proc = setup_parser
+
+    if @use_record_time and @time_format
+      @time_parser = Fluent::TextParser::TimeParser.new(@time_format)
+    end
   end
 
   def setup_parser
@@ -231,7 +242,17 @@ class Fluent::KafkaInput < Fluent::Input
 
       messages.each { |msg|
         begin
-          es.add(Fluent::Engine.now, @parser.call(msg, @topic_entry))
+          record = @parser.call(msg, @topic_entry)
+          if @use_record_time
+            if @time_format
+              record_time = @time_parser.parse(record['time'])
+            else
+              record_time = record['time']
+            end
+          else
+            record_time = Fluent::Engine.now
+          end
+          es.add(record_time, record)
         rescue => e
           $log.warn "parser error in #{@topic_entry.topic}/#{@topic_entry.partition}", :error => e.to_s, :value => msg.value, :offset => msg.offset
           $log.debug_backtrace

data/lib/fluent/plugin/in_kafka_group.rb

@@ -1,4 +1,5 @@
 require 'fluent/input'
+require 'fluent/time'
 require 'fluent/plugin/kafka_plugin_util'
 
 class Fluent::KafkaGroupInput < Fluent::Input
@@ -20,6 +21,10 @@ class Fluent::KafkaGroupInput < Fluent::Input
                :desc => "Tag suffix (Optional)"
   config_param :retry_emit_limit, :integer, :default => nil,
                :desc => "How long to stop event consuming when BufferQueueLimitError happens. Wait retry_emit_limit x 1s. The default is waiting until BufferQueueLimitError is resolved"
+  config_param :use_record_time, :bool, :default => false,
+               :desc => "Replace message timestamp with contents of 'time' field."
+  config_param :time_format, :string, :default => nil,
+               :desc => "Time format to be used to parse 'time' filed."
 
   # Kafka consumer options
   config_param :max_wait_time, :integer, :default => nil,
@@ -47,6 +52,8 @@ class Fluent::KafkaGroupInput < Fluent::Input
   def initialize
     super
     require 'kafka'
+
+    @time_parser = nil
   end
 
   def _config_to_array(config)
@@ -82,6 +89,10 @@ class Fluent::KafkaGroupInput < Fluent::Input
     @fetch_opts = {}
     @fetch_opts[:max_wait_time] = @max_wait_time if @max_wait_time
     @fetch_opts[:min_bytes] = @min_bytes if @min_bytes
+
+    if @use_record_time and @time_format
+      @time_parser = Fluent::TextParser::TimeParser.new(@time_format)
+    end
   end
 
   def setup_parser
@@ -143,7 +154,17 @@ class Fluent::KafkaGroupInput < Fluent::Input
 
           batch.messages.each { |msg|
             begin
-              es.add(Fluent::Engine.now, @parser_proc.call(msg))
+              record = @parser_proc.call(msg)
+              if @use_record_time
+                if @time_format
+                  record_time = @time_parser.parse(record['time'])
+                else
+                  record_time = record['time']
+                end
+              else
+                record_time = Fluent::Engine.now
+              end
+              es.add(record_time, record)
             rescue => e
               log.warn "parser error in #{batch.topic}/#{batch.partition}", :error => e.to_s, :value => msg.value, :offset => msg.offset
               log.debug_backtrace

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-kafka
 version: !ruby/object:Gem::Version
-  version: 0.4.2.rc1
+  version: 0.4.2
 platform: ruby
 authors:
 - Hidemasa Togashi
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-20 00:00:00.000000000 Z
+date: 2017-01-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -125,9 +125,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: 2.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.5.1