fluent-plugin-elasticsearch 3.4.0 → 3.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4e8eed03eaeddf3765e1ac07f5038cd026bacbc7f86d63b286af4da186c7d04f
-  data.tar.gz: 17d90e2717897e2a021629b4414c475954ff0100c16b9d4db2526e8bd4ffc084
+  metadata.gz: 0b1ff650a7dd92dbe450e0e4bc6ee1c8b4db2e104d3b50b56aee885e1e06fa4a
+  data.tar.gz: 653c6a1d031fc876a43774336cd2eb5388c4c540f88874fd281640279e003e39
 SHA512:
-  metadata.gz: 94d36c2a7804887d9f974e26d55a2abbd8ade8e042126f5f6b7bbc586db0ff63fe6195feb8a51d5c610e8ffbc9274d76a886290f49af10ce0ff005d72758fb81
-  data.tar.gz: 0b3bfaf992cfebd919cdc526c52e5b3200cc70cc424bce51a197cc7915890e9b495904ce367cdb329df69b182c6928675d6872f9ce4e579d55cb2f9e5b53304d
+  metadata.gz: 1ba9e19a4c955eb8f3b85f0d07523c8ad2d21a041de16aaaa916e6761d5d653077c607e69374c1ea61e87eb804a9322ed76371adbd8b90fbd0f0f8c5d0b91b28
+  data.tar.gz: 449d5f8db19a1e01cc77af21384d003a4d59d33c204de61b2d6dfb3a9476c0721d20c22780fd26428b01cc411738ca4860b2d3cb47a73cc4a404d3d9c5c710d0

data/History.md

@@ -2,6 +2,9 @@
 
 ### [Unreleased]
 
+### 3.4.1
+- Handle non-String value on parse_time (#570)
+
 ### 3.4.0
 - Check exclusive feature on #configure (#569)
 - Modify FAQ for highly load k8s EFK stack (#566)

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '3.4.0'
+  s.version       = '3.4.1'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -12,6 +12,7 @@ end
 require 'fluent/plugin/output'
 require 'fluent/event'
 require 'fluent/error'
+require 'fluent/time'
 require_relative 'elasticsearch_constants'
 require_relative 'elasticsearch_error'
 require_relative 'elasticsearch_error_handler'
@@ -358,18 +359,32 @@ EOC
           # Strptime doesn't support all formats, but for those it does it's
           # blazingly fast.
           strptime = Strptime.new(@time_key_format)
-          Proc.new { |value| strptime.exec(value).to_datetime }
+          Proc.new { |value|
+            value = convert_numeric_time_into_string(value, @time_key_format) if value.is_a?(Numeric)
+            strptime.exec(value).to_datetime
+          }
         rescue
           # Can happen if Strptime doesn't recognize the format; or
           # if strptime couldn't be required (because it's not installed -- it's
           # ruby 2 only)
-          Proc.new { |value| DateTime.strptime(value, @time_key_format) }
+          Proc.new { |value|
+            value = convert_numeric_time_into_string(value, @time_key_format) if value.is_a?(Numeric)
+            DateTime.strptime(value, @time_key_format)
+          }
         end
       else
-        Proc.new { |value| DateTime.parse(value) }
+        Proc.new { |value|
+          value = convert_numeric_time_into_string(value) if value.is_a?(Numeric)
+          DateTime.parse(value)
+        }
       end
     end
 
+    def convert_numeric_time_into_string(numeric_time, time_key_format = "%Y-%m-%d %H:%M:%S.%N%z")
+      numeric_time_parser = Fluent::NumericTimeParser.new(:float)
+      Time.at(numeric_time_parser.parse(numeric_time).to_r).strftime(time_key_format)
+    end
+
     def parse_time(value, event_time, tag)
       @time_parser.call(value)
     rescue => e

data/test/plugin/test_out_elasticsearch.rb

@@ -1711,6 +1711,20 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal(index_cmds[1]['@timestamp'], ts)
   end
 
+  def test_uses_custom_time_key_with_float_record
+    driver.configure("logstash_format true
+                      time_precision 3
+                      time_key vtm\n")
+    stub_elastic
+    time = Time.now
+    float_time = time.to_f
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record.merge!('vtm' => float_time))
+    end
+    assert(index_cmds[1].has_key? '@timestamp')
+    assert_equal(index_cmds[1]['@timestamp'], time.to_datetime.iso8601(3))
+  end
+
   def test_uses_custom_time_key_with_format
     driver.configure("logstash_format true
                       time_key_format %Y-%m-%d %H:%M:%S.%N%z
@@ -1725,6 +1739,22 @@ class ElasticsearchOutput < Test::Unit::TestCase
     assert_equal("logstash-2001.02.03", index_cmds[0]['index']['_index'])
   end
 
+  def test_uses_custom_time_key_with_float_record_and_format
+    driver.configure("logstash_format true
+                      time_key_format %Y-%m-%d %H:%M:%S.%N%z
+                      time_key vtm\n")
+    stub_elastic
+    ts = "2001-02-03 13:14:01.673+02:00"
+    time = Time.parse(ts)
+    current_zone_offset = Time.now.to_datetime.offset
+    float_time = time.to_f
+    driver.run(default_tag: 'test') do
+      driver.feed(sample_record.merge!('vtm' => float_time))
+    end
+    assert(index_cmds[1].has_key? '@timestamp')
+    assert_equal(index_cmds[1]['@timestamp'], DateTime.parse(ts).new_offset(current_zone_offset).iso8601(9))
+  end
+
   def test_uses_custom_time_key_with_format_without_logstash
     driver.configure("include_timestamp true
                       index_name test

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 3.4.0
+  version: 3.4.1
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-28 00:00:00.000000000 Z
+date: 2019-04-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd