fluent-plugin-elasticsearch-timestamp-check 0.2.6 → 0.2.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 5cd7b06f7edb4ce3f72d1c6dccf21929046ceb51
-  data.tar.gz: 9ef81ffc2b011bd4e8a06fbdf66b38562c59711f
+  metadata.gz: 507e9c8bcb7590f680c4c72c39b26a86ef9029d3
+  data.tar.gz: c709c47aa550498ffef8ab50f8493dda0ed154a9
 SHA512:
-  metadata.gz: a091f77db169760a13a1838c2d6e2a68b50f97274902f4506b467568f3c279f9b5e0763be181a9259ff7a5051a0ffd9ae8db5e06ef08534d85f20ecaa09a5fe8
-  data.tar.gz: 0da8c759f7c0d29b5970ccd0d3c06ac2317e2d411cd159321c178cea88bd8e9c538b80442a7c8cf83a9cf7b903ba9620bf43b28a83126cdf89e0bf0e8f45ea08
+  metadata.gz: '082a9b6262d7bdd1d0eff963c4b3d30a4e0983012a802192e5c2b8bd2a50241ac65dd2795890c3a3fdfe760624139fa28e14273142165925bc8f2fce67b95d1b'
+  data.tar.gz: 02c0c8ce935610f5024f2a39d69670e85ef8474690454f2e5c171a027ce8b6358534662cce83068c039de9d2045ee110430ade8671d47fbf50d47301e94dc4c9

data/README.md

@@ -13,23 +13,39 @@ The purpose of this filter is to make sure the @timestamp field exists in the
 record which is necessary for the record to be indexed properly by
 elasticsearch.
 
-* If *@timestamp* field already exists, it will ensure the format is correct
+* If `@timestamp` field already exists, it will ensure the format is correct
   by parse and convert to format '%Y-%m-%dT%H:%M:%S%z'. **As of version 0.2.4, it
   will support epoch second / epoch millis format as a valid timestamp value. If
   such value is detected, it will be converted to iso8601 format for easier
   consumption of elasticsearch when dynamic mapping is used.**
 
-* If a field named *timestamp* or *time* or *syslog_timestamp* exists, it will
-  parse that field and conver it to format '%Y-%m-%dT%H:%M:%S%z' then store it
-  in *@timestamp* field.
+* If a field named `timestamp` or `time` or `syslog_timestamp` exists, it will
+  parse that field and conver it to format '%Y-%m-%dT%H:%M:%S.%L%z' then store it
+  in `@timestamp` field. In addition, a field `fluent_converted_timestamp`
+  is added to the object with the same value.
 
 * If none of the above field exists, it will insert current time in
-  '%Y-%m-%dT%H:%M:%S%z' format as the *@timestamp* field.
+  '%Y-%m-%dT%H:%M:%S.%L%z' format as the `@timestamp` field. A field
+  `fluent_added_timestamp` is added to the object with same value.
+
+## (>=0.2.6) Subsecond Precision
+
+`subsecond_precision` controls the subsecond precision during the conversion.
+Default value is set to `3` (millisecond).
+
+Other `subsecond_precision` sample values are:
+
+* `6` (microsecond)
+* `9` (nanosecond)
+* `12` (picosecond)
+
+and more high precision is also supported.
 
 ## Usage
 
 ```
 <filter **>
   type elasticsearch_timestamp_check
+  subsecond_precision 3
 </filter>
 ```

data/fluent-plugin-elasticsearch-timestamp-check.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-elasticsearch-timestamp-check"
-  spec.version       = "0.2.6"
+  spec.version       = "0.2.7"
   spec.authors       = ["Richard Li"]
   spec.email         = ["evilcat@wisewolfsolutions.com"]
   spec.description   = %q{fluent filter plugin to ensure @timestamp is in proper format}

data/lib/fluent/plugin/filter_elasticsearch_timestamp_check.rb

@@ -2,6 +2,8 @@ require 'fluent/plugin/filter'
 
 module Fluent::Plugin
   class ElasticsearchTimestampCheckFilter < Filter
+    attr_reader :timestamp_digits
+
     Fluent::Plugin.register_filter('elasticsearch_timestamp_check', self)
 
     config_param :subsecond_precision, :integer, default: 3
@@ -11,6 +13,15 @@ module Fluent::Plugin
       require 'date'
       raise Fluent::ConfigError, "specify 1 or bigger number." if subsecond_precision < 1
       @strftime_format = "%Y-%m-%dT%H:%M:%S.%#{@subsecond_precision}N%z".freeze
+      @timestamp_digits = configure_digits
+    end
+
+    def configure_digits
+      subepoch_precision = 10 + @subsecond_precision
+      timestamp_digits = [10, 13]
+      timestamp_digits << subepoch_precision
+      timestamp_digits.uniq!
+      timestamp_digits
     end
 
     def start
@@ -29,10 +40,11 @@ module Fluent::Plugin
           # all digit entry would be treated as epoch seconds or epoch millis
           if !!(timestamp =~ /\A[-+]?\d+\z/)
             num = timestamp.to_f
-            # epoch second or epoch millis should be either 10 or 13 digits
+            # By default, epoch second or epoch millis should be either 10 or 13 digits
             # other length should be considered invalid (until the next digit
             # rollover at 2286-11-20  17:46:40 Z
-            next unless [10, 13].include?(Math.log10(num).to_i + 1)
+            # For user-defined precision also should handle here.
+            next unless @timestamp_digits.include?(Math.log10(num).to_i + 1)
             record['@timestamp'] = record['fluent_converted_timestamp'] =
               Time.at(
                 num / (10 ** ((Math.log10(num).to_i + 1) - 10))

data/test/plugin/test_filter_elasticsearch_timestamp_check.rb

@@ -64,6 +64,30 @@ class TestElasticsearchTimestampCheckFilter < Test::Unit::TestCase
     formatted_time = Time.at(
       num / (10 ** ((Math.log10(num).to_i + 1) - 10))
     ).strftime('%Y-%m-%dT%H:%M:%S.%3N%z')
+    assert_equal([10, 13], d.instance.timestamp_digits)
+    assert_true(filtered.key?("@timestamp"))
+    assert_true(filtered.key?("fluent_converted_timestamp"))
+    assert_equal(formatted_time, filtered["fluent_converted_timestamp"])
+  end
+
+  data('@timestamp'       => '@timestamp',
+       'timestamp'        => 'timestamp',
+       'time'             => 'time',
+       'syslog_timestamp' => 'syslog_timestamp')
+  def test_timestamp_with_digit_and_micro_precision(data)
+    timekey = data
+    precision = 6
+    d = create_driver(%[subsecond_precision #{precision}])
+    timestamp = '1517878172013770'
+    d.run(default_tag: 'test') do
+      d.feed({'test' => 'notime'}.merge(timekey => timestamp))
+    end
+    filtered = d.filtered.map{|e| e.last}.first
+    num = timestamp.to_f
+    formatted_time = Time.at(
+      num / (10 ** ((Math.log10(num).to_i + 1) - 10))
+    ).strftime("%Y-%m-%dT%H:%M:%S.%#{precision}N%z")
+    assert_equal([10, 13, 10 + precision], d.instance.timestamp_digits)
     assert_true(filtered.key?("@timestamp"))
     assert_true(filtered.key?("fluent_converted_timestamp"))
     assert_equal(formatted_time, filtered["fluent_converted_timestamp"])
@@ -77,7 +101,7 @@ class TestElasticsearchTimestampCheckFilter < Test::Unit::TestCase
     timekey = data
     precision = 9
     d = create_driver(%[subsecond_precision #{precision}])
-    timestamp = '1505800348899'
+    timestamp = '1517878172013770000'
     d.run(default_tag: 'test') do
       d.feed({'test' => 'notime'}.merge(timekey => timestamp))
     end
@@ -86,6 +110,7 @@ class TestElasticsearchTimestampCheckFilter < Test::Unit::TestCase
     formatted_time = Time.at(
       num / (10 ** ((Math.log10(num).to_i + 1) - 10))
     ).strftime("%Y-%m-%dT%H:%M:%S.%#{precision}N%z")
+    assert_equal([10, 13, 10 + precision], d.instance.timestamp_digits)
     assert_true(filtered.key?("@timestamp"))
     assert_true(filtered.key?("fluent_converted_timestamp"))
     assert_equal(formatted_time, filtered["fluent_converted_timestamp"])

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch-timestamp-check
 version: !ruby/object:Gem::Version
-  version: 0.2.6
+  version: 0.2.7
 platform: ruby
 authors:
 - Richard Li
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-02-05 00:00:00.000000000 Z
+date: 2018-02-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd