fluent-plugin-elasticsearch 2.5.0 → 2.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9d722819bc4644442e84bd13acbd6d1fe7845d6fc01119d3a4c8475321d1195d
-  data.tar.gz: a4956fa749f46343aa3609cc319cff6c5e28797b50384ece1284566ee773045f
+  metadata.gz: 3427ecaceac8a69b1fc2bf12bef3f54e10fe5d88c4de2ed314e4f5eaf370b213
+  data.tar.gz: 284c78599965eda36d1c0927e9f291db9eddf08d07d586f8e333e530c7ca3cdf
 SHA512:
-  metadata.gz: 55cb3916853e0a3ad24bc450aecd577f13cbcb20f8d9a4b446ffa4beb7dce6df5fcd8ca4583846f53921b69404421f3bd8a714dedaac596745600a4b7084744e
-  data.tar.gz: e46f86ba7109bce5f51628f3209497c3a5ec706641f25cb267391a173384acc26c626bce2f1280d1d72f1cebb7e9188f76afa6baf1246e2b01f6f5e3c913b3f5
+  metadata.gz: a7cec3fc89d1a48543052096072e34495992c950deaa56b982be29d89062533e90844b3edd9bf7eef46c74e787eda51395751041384c948ff0bff5dbfa15c598
+  data.tar.gz: bf4140896391b928c970bcb3cf877fb0380326a182c9b0980793b9ab0522be8e5fd789247e66d4313bace64e430d8f8719a40b42a40bad958b9d339add490ce7

data/.travis.yml

@@ -3,7 +3,9 @@ language: ruby
 rvm:
   - 2.1
   - 2.2
-  - 2.3.1
+  - 2.3.6
+  - 2.4.3
+  - 2.5.0
 
 gemfile:
  - Gemfile

data/History.md

@@ -4,8 +4,11 @@
 - Log ES response errors (#230)
 - Use latest elasticsearch-ruby (#240)
 
+### 2.6.0
+- Handle high precision time format when using custom time_key (#360)
+
 ### 2.5.0
-- Using nested record in `id_key`, `parent_key`, and `routing_key` (#351
+- Using nested record in `id_key`, `parent_key`, and `routing_key` (#351)
 - Fix inverted case of a proper noun "Elasticsearch" (#349)
 
 ### 2.4.1

data/README.md

@@ -73,6 +73,8 @@ Current maintainers: @cosmo0920
 
 NOTE: For v0.12 version, you should use 1.x.y version. Please send patch into v0.12 branch if you encountered 1.x version's bug.
 
+NOTE: This documentation is for fluent-plugin-elasticsearch 2.x or later. For 1.x documentation, please see [v0.12 branch](https://github.com/uken/fluent-plugin-elasticsearch/tree/v0.12).
+
 ## Installation
 
 ```sh

data/fluent-plugin-elasticsearch.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name          = 'fluent-plugin-elasticsearch'
-  s.version       = '2.5.0'
+  s.version       = '2.6.0'
   s.authors       = ['diogo', 'pitr']
   s.email         = ['pitr.vern@gmail.com', 'me@diogoterror.com']
   s.description   = %q{Elasticsearch output plugin for Fluent event collector}

data/lib/fluent/plugin/out_elasticsearch.rb

@@ -376,7 +376,7 @@ module Fluent::Plugin
           elsif record.has_key?(@time_key)
             rts = record[@time_key]
             dt = parse_time(rts, time, tag)
-            record[TIMESTAMP_FIELD] = rts unless @time_key_exclude_timestamp
+            record[TIMESTAMP_FIELD] = dt.iso8601(@time_precision) unless @time_key_exclude_timestamp
           else
             dt = Time.at(time).to_datetime
             record[TIMESTAMP_FIELD] = dt.iso8601(@time_precision)

data/test/plugin/test_out_elasticsearch.rb

@@ -706,7 +706,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
                        ))
       stub_elastic_ping
       stub_elastic
-      time = Time.parse Date.today.to_s
+      time = Time.parse Date.today.iso8601
       driver.run(default_tag: 'test') do
         driver.feed(time.to_i, sample_record)
       end
@@ -722,7 +722,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
                            Fluent::Config::Element.new('buffer', 'tag,pipeline_id', {}, [])
                          ]
                        ))
-      time = Time.parse Date.today.to_s
+      time = Time.parse Date.today.iso8601
       pipeline_id = "mypipeline"
       logstash_index = "myindex.#{pipeline_id}"
       stub_elastic_ping
@@ -761,7 +761,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
   def test_writes_to_target_index_key_logstash
     driver.configure("target_index_key @target_index
                       logstash_format true")
-    time = Time.parse Date.today.to_s
+    time = Time.parse Date.today.iso8601
     stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
@@ -773,7 +773,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
    def test_writes_to_target_index_key_logstash_uppercase
     driver.configure("target_index_key @target_index
                       logstash_format true")
-    time = Time.parse Date.today.to_s
+    time = Time.parse Date.today.iso8601
     stub_elastic_ping
     stub_elastic
     driver.run(default_tag: 'test') do
@@ -808,7 +808,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
   def test_writes_to_target_index_key_fallack_logstash
     driver.configure("target_index_key @target_index\n
                       logstash_format true")
-    time = Time.parse Date.today.to_s
+    time = Time.parse Date.today.iso8601
     logstash_index = "logstash-#{time.getutc.strftime("%Y.%m.%d")}"
     stub_elastic_ping
     stub_elastic
@@ -1040,7 +1040,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
   def test_writes_to_logstash_index_with_specified_prefix
     driver.configure("logstash_format true
                       logstash_prefix myprefix")
-    time = Time.parse Date.today.to_s
+    time = Time.parse Date.today.iso8601
     logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m.%d")}"
     stub_elastic_ping
     stub_elastic
@@ -1055,7 +1055,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.configure("logstash_format true
                       logstash_prefix_separator #{separator}
                       logstash_prefix myprefix")
-    time = Time.parse Date.today.to_s
+    time = Time.parse Date.today.iso8601
     logstash_index = "myprefix#{separator}#{time.getutc.strftime("%Y.%m.%d")}"
     stub_elastic_ping
     stub_elastic
@@ -1069,7 +1069,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
     def test_writes_to_logstash_index_with_specified_prefix_and_tag_placeholder
       driver.configure("logstash_format true
                       logstash_prefix myprefix-${tag}")
-      time = Time.parse Date.today.to_s
+      time = Time.parse Date.today.iso8601
       logstash_index = "myprefix-test-#{time.getutc.strftime("%Y.%m.%d")}"
       stub_elastic_ping
       stub_elastic
@@ -1092,7 +1092,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
                                                        }, [])
                          ]
                        ))
-      time = Time.parse Date.today.to_s
+      time = Time.parse Date.today.iso8601
       logstash_index = "myprefix-#{time.getutc.strftime("%H")}-#{time.getutc.strftime("%Y.%m.%d")}"
       stub_elastic_ping
       stub_elastic
@@ -1112,7 +1112,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
                            Fluent::Config::Element.new('buffer', 'tag,pipeline_id', {}, [])
                          ]
                        ))
-      time = Time.parse Date.today.to_s
+      time = Time.parse Date.today.iso8601
       pipeline_id = "mypipeline"
       logstash_index = "myprefix-#{pipeline_id}-#{time.getutc.strftime("%Y.%m.%d")}"
       stub_elastic_ping
@@ -1127,7 +1127,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
   def test_writes_to_logstash_index_with_specified_prefix_uppercase
     driver.configure("logstash_format true
                       logstash_prefix MyPrefix")
-    time = Time.parse Date.today.to_s
+    time = Time.parse Date.today.iso8601
     logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m.%d")}"
     stub_elastic_ping
     stub_elastic
@@ -1142,7 +1142,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
     def test_writes_to_logstash_index_with_specified_dateformat
     driver.configure("logstash_format true
                       logstash_dateformat %Y.%m")
-    time = Time.parse Date.today.to_s
+    time = Time.parse Date.today.iso8601
     logstash_index = "logstash-#{time.getutc.strftime("%Y.%m")}"
     stub_elastic_ping
     stub_elastic
@@ -1156,7 +1156,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.configure("logstash_format true
                       logstash_prefix myprefix
                       logstash_dateformat %Y.%m")
-    time = Time.parse Date.today.to_s
+    time = Time.parse Date.today.iso8601
     logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m")}"
     stub_elastic_ping
     stub_elastic
@@ -1202,10 +1202,8 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.run(default_tag: 'test') do
       driver.feed(time, sample_record)
     end
-    tf = "%Y-%m-%dT%H:%M:%S%:z"
-    timef = Fluent::TimeFormatter.new(tf, true, ENV["TZ"])
     assert(index_cmds[1].has_key? '@timestamp')
-    assert_equal(timef.format(Time.parse(index_cmds[1]['@timestamp'])).to_s, ts.to_s)
+    assert_equal(index_cmds[1]['@timestamp'], ts.iso8601(9))
   end
 
   def test_adds_timestamp_when_include_timestamp
@@ -1217,17 +1215,15 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.run(default_tag: 'test') do
       driver.feed(time, sample_record)
     end
-    tf = "%Y-%m-%dT%H:%M:%S%:z"
-    timef = Fluent::TimeFormatter.new(tf, true, ENV["TZ"])
     assert(index_cmds[1].has_key? '@timestamp')
-    assert_equal(timef.format(Time.parse(index_cmds[1]['@timestamp'])).to_s, ts.to_s)
+    assert_equal(index_cmds[1]['@timestamp'], ts.iso8601(9))
   end
 
   def test_uses_custom_timestamp_when_included_in_record
     driver.configure("logstash_format true\n")
     stub_elastic_ping
     stub_elastic
-    ts = DateTime.new(2001,2,3).to_s
+    ts = DateTime.new(2001,2,3).iso8601
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge!('@timestamp' => ts))
     end
@@ -1239,7 +1235,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
     driver.configure("include_timestamp true\n")
     stub_elastic_ping
     stub_elastic
-    ts = DateTime.new(2001,2,3).to_s
+    ts = DateTime.new(2001,2,3).iso8601
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge!('@timestamp' => ts))
     end
@@ -1252,7 +1248,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
                       time_key vtm\n")
     stub_elastic_ping
     stub_elastic
-    ts = DateTime.new(2001,2,3).to_s
+    ts = DateTime.new(2001,2,3).iso8601(9)
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge!('vtm' => ts))
     end
@@ -1271,7 +1267,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
       driver.feed(sample_record.merge!('vtm' => ts))
     end
     assert(index_cmds[1].has_key? '@timestamp')
-    assert_equal(index_cmds[1]['@timestamp'], ts)
+    assert_equal(index_cmds[1]['@timestamp'], DateTime.parse(ts).iso8601(9))
     assert_equal("logstash-2001.02.03", index_cmds[0]['index']['_index'])
   end
 
@@ -1287,7 +1283,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
       driver.feed(sample_record.merge!('vtm' => ts))
     end
     assert(index_cmds[1].has_key? '@timestamp')
-    assert_equal(index_cmds[1]['@timestamp'], ts)
+    assert_equal(index_cmds[1]['@timestamp'], DateTime.parse(ts).iso8601(9))
     assert_equal("test", index_cmds[0]['index']['_index'])
   end
 
@@ -1297,7 +1293,7 @@ class ElasticsearchOutput < Test::Unit::TestCase
                       time_key_exclude_timestamp true\n")
     stub_elastic_ping
     stub_elastic
-    ts = DateTime.new(2001,2,3).to_s
+    ts = DateTime.new(2001,2,3).iso8601
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge!('vtm' => ts))
     end

data/test/plugin/test_out_elasticsearch_dynamic.rb

@@ -369,7 +369,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
 
   def test_writes_to_logstash_index
     driver.configure("logstash_format true\n")
-    time = Time.parse Date.today.to_s
+    time = Time.parse Date.today.iso8601
     logstash_index = "logstash-#{time.getutc.strftime("%Y.%m.%d")}"
     stub_elastic_ping
     stub_elastic
@@ -382,7 +382,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   def test_writes_to_logstash_utc_index
     driver.configure("logstash_format true
                       utc_index false")
-    time = Time.parse Date.today.to_s
+    time = Time.parse Date.today.iso8601
     utc_index = "logstash-#{time.strftime("%Y.%m.%d")}"
     stub_elastic_ping
     stub_elastic
@@ -395,7 +395,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   def test_writes_to_logstash_index_with_specified_prefix
     driver.configure("logstash_format true
                       logstash_prefix myprefix")
-    time = Time.parse Date.today.to_s
+    time = Time.parse Date.today.iso8601
     logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m.%d")}"
     stub_elastic_ping
     stub_elastic
@@ -408,7 +408,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
   def test_writes_to_logstash_index_with_specified_prefix_uppercase
     driver.configure("logstash_format true
                       logstash_prefix MyPrefix")
-    time = Time.parse Date.today.to_s
+    time = Time.parse Date.today.iso8601
     logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m.%d")}"
     stub_elastic_ping
     stub_elastic
@@ -421,7 +421,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     def test_writes_to_logstash_index_with_specified_dateformat
     driver.configure("logstash_format true
                       logstash_dateformat %Y.%m")
-    time = Time.parse Date.today.to_s
+    time = Time.parse Date.today.iso8601
     logstash_index = "logstash-#{time.getutc.strftime("%Y.%m")}"
     stub_elastic_ping
     stub_elastic
@@ -435,7 +435,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     driver.configure("logstash_format true
                       logstash_prefix myprefix
                       logstash_dateformat %Y.%m")
-    time = Time.parse Date.today.to_s
+    time = Time.parse Date.today.iso8601
     logstash_index = "myprefix-#{time.getutc.strftime("%Y.%m")}"
     stub_elastic_ping
     stub_elastic
@@ -464,14 +464,14 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
       driver.feed(time, sample_record)
     end
     assert(index_cmds[1].has_key? '@timestamp')
-    assert_equal(index_cmds[1]['@timestamp'], ts.to_s)
+    assert_equal(index_cmds[1]['@timestamp'], ts.iso8601)
   end
 
   def test_uses_custom_timestamp_when_included_in_record
     driver.configure("include_timestamp true\n")
     stub_elastic_ping
     stub_elastic
-    ts = DateTime.new(2001,2,3).to_s
+    ts = DateTime.new(2001,2,3).iso8601
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge!('@timestamp' => ts))
     end
@@ -483,7 +483,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
     driver.configure("logstash_format true\n")
     stub_elastic_ping
     stub_elastic
-    ts = DateTime.new(2001,2,3).to_s
+    ts = DateTime.new(2001,2,3).iso8601
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge!('@timestamp' => ts))
     end
@@ -496,7 +496,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
                       time_key vtm\n")
     stub_elastic_ping
     stub_elastic
-    ts = DateTime.new(2001,2,3).to_s
+    ts = DateTime.new(2001,2,3).iso8601
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge!('vtm' => ts))
     end
@@ -509,7 +509,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
                       time_key vtm\n")
     stub_elastic_ping
     stub_elastic
-    ts = DateTime.new(2001,2,3).to_s
+    ts = DateTime.new(2001,2,3).iso8601
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge!('vtm' => ts))
     end
@@ -523,7 +523,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
                       time_key vtm\n")
     stub_elastic_ping
     stub_elastic
-    ts = DateTime.new(2001,2,3).to_s
+    ts = DateTime.new(2001,2,3).iso8601
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge!('vtm' => ts))
     end
@@ -538,7 +538,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
                       time_key_exclude_timestamp true\n")
     stub_elastic_ping
     stub_elastic
-    ts = DateTime.new(2001,2,3).to_s
+    ts = DateTime.new(2001,2,3).iso8601
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge!('vtm' => ts))
     end
@@ -551,7 +551,7 @@ class ElasticsearchOutputDynamic < Test::Unit::TestCase
                       time_key_exclude_timestamp true\n")
     stub_elastic_ping
     stub_elastic
-    ts = DateTime.new(2001,2,3).to_s
+    ts = DateTime.new(2001,2,3).iso8601
     driver.run(default_tag: 'test') do
       driver.feed(sample_record.merge!('vtm' => ts))
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch
 version: !ruby/object:Gem::Version
-  version: 2.5.0
+  version: 2.6.0
 platform: ruby
 authors:
 - diogo
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-01-25 00:00:00.000000000 Z
+date: 2018-02-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd