fluent-plugin-elasticsearch-timestamp-check 0.2.4 → 0.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: caf11889f096c3937908340a59ff69af610e80a9
-  data.tar.gz: 5fec5fb248ae90cae895942b8d1fb6544e5a233c
+  metadata.gz: dddb0ad42d2491532c832ae7d7e1a447c65431cd
+  data.tar.gz: 09b846178a4d199a69cced307fc4b9a489c790f7
 SHA512:
-  metadata.gz: 4904d97b3143926d00c313bef06a1e3e2f79413f6c867ca2e3cde2bbbf6db46b68960bab98abd0e28c702e849377bd15b5ee2c56a0c5640ece68f77204138c6d
-  data.tar.gz: 3d3e8575835bdb4a9d9e28e70c9abbb7a58ea8cee7a6fc0e97dff92b3ecfb09afd728c928a47b3dbd115d54a5661e1a821ec55ef9c603169a3d2dae7d30bae76
+  metadata.gz: 1d0d3fcb7ff42f8da5c38e3c2fc264a09de5a42178af961a06ed83c817608e08dfce337b1eea453c6a0ffcc1a87602536e9bdafaf9844ba0ec1d330e1427a3b2
+  data.tar.gz: 92fd025a3111678ea4d6351e0d1707f5eb6a85c585662fdfc32888033bf494640455c7ab9d657014d0c4d92a1c8eececdd155c71c5391e62dc4cc3f4d4548653

data/Gemfile

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+source "https://rubygems.org"
+
+gemspec

data/README.md

@@ -14,7 +14,10 @@ record which is necessary for the record to be indexed properly by
 elasticsearch.
 
 * If *@timestamp* field already exists, it will ensure the format is correct
-  by parse and convert to format '%Y-%m-%dT%H:%M:%S%z'.
+  by parse and convert to format '%Y-%m-%dT%H:%M:%S%z'. **As of version 0.2.4, it
+  will support epoch second / epoch millis format as a valid timestamp value. If
+  such value is detected, it will be converted to iso8601 format for easier
+  consumption of elasticsearch when dynamic mapping is used.**
 
 * If a field named *timestamp* or *time* or *syslog_timestamp* exists, it will
   parse that field and conver it to format '%Y-%m-%dT%H:%M:%S%z' then store it

data/Rakefile

@@ -0,0 +1,11 @@
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+  test.warning = false
+end
+
+task :default => :test

data/fluent-plugin-elasticsearch-timestamp-check.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-elasticsearch-timestamp-check"
-  spec.version       = "0.2.4"
+  spec.version       = "0.2.5"
   spec.authors       = ["Richard Li"]
   spec.email         = ["evilcat@wisewolfsolutions.com"]
   spec.description   = %q{fluent filter plugin to ensure @timestamp is in proper format}
@@ -13,5 +13,8 @@ Gem::Specification.new do |spec|
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
+  spec.add_runtime_dependency "fluentd", [">= 0.14.0", "< 2"]
+  spec.add_development_dependency "rake", "~> 11.0"
   spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "test-unit", "~> 3.2"
 end

data/lib/fluent/plugin/filter_elasticsearch_timestamp_check.rb

@@ -24,14 +24,14 @@ module Fluent::Plugin
         begin
           # all digit entry would be treated as epoch seconds or epoch millis
           if !!(timestamp =~ /\A[-+]?\d+\z/)
-            num = timestamp.to_i
+            num = timestamp.to_f
             # epoch second or epoch millis should be either 10 or 13 digits
             # other length should be considered invalid (until the next digit
             # rollover at 2286-11-20  17:46:40 Z
-            next unless [10, 13].include?(num.to_s.length)
+            next unless [10, 13].include?(Math.log10(num).to_i + 1)
             record['@timestamp'] = record['fluent_converted_timestamp'] =
               Time.at(
-                num / (10 ** (num.to_s.length - 10))
+                num / (10 ** ((Math.log10(num).to_i + 1) - 10))
               ).strftime('%Y-%m-%dT%H:%M:%S.%L%z')
             break
           end

data/test/helper.rb

@@ -0,0 +1,8 @@
+require 'fluent/test'
+require 'fluent/test/helpers'
+require 'fluent/test/driver/filter'
+require 'fluent/plugin/filter_elasticsearch_timestamp_check'
+
+class Test::Unit::TestCase
+  include Fluent::Test::Helpers
+end

data/test/plugin/test_filter_elasticsearch_timestamp_check.rb

@@ -0,0 +1,60 @@
+require 'helper'
+
+class TestElasticsearchTimestampCheckFilter < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf='')
+    Fluent::Test::Driver::Filter.new(Fluent::Plugin::ElasticsearchTimestampCheckFilter).configure(conf)
+  end
+
+  def test_added_timestamp
+    d = create_driver
+    d.run(default_tag: 'test') do
+      d.feed({'test' => 'notime'})
+    end
+    filtered = d.filtered.map {|e| e.last}.first
+    assert_true(filtered.key?("@timestamp"))
+    assert_true(filtered.key?("fluent_added_timestamp"))
+  end
+
+  data('@timestamp'       => '@timestamp',
+       'timestamp'        => 'timestamp',
+       'time'             => 'time',
+       'syslog_timestamp' => 'syslog_timestamp')
+  def test_timestamp_with_normal(data)
+    timekey = data
+    d = create_driver
+    timestamp = '2017-09-19T14:40:08.321+0900'
+    d.run(default_tag: 'test') do
+      d.feed({'test' => 'notime'}.merge(timekey => timestamp))
+    end
+    filtered = d.filtered.map{|e| e.last}.first
+    assert_true(filtered.key?("@timestamp"))
+    assert_true(filtered.key?("fluent_converted_timestamp"))
+    assert_equal(timestamp, filtered["fluent_converted_timestamp"])
+  end
+
+  data('@timestamp'       => '@timestamp',
+       'timestamp'        => 'timestamp',
+       'time'             => 'time',
+       'syslog_timestamp' => 'syslog_timestamp')
+  def test_timestamp_with_digit(data)
+    timekey = data
+    d = create_driver
+    timestamp = '1505800348899'
+    d.run(default_tag: 'test') do
+      d.feed({'test' => 'notime'}.merge(timekey => timestamp))
+    end
+    filtered = d.filtered.map{|e| e.last}.first
+    num = timestamp.to_f
+    formatted_time = Time.at(
+      num / (10 ** ((Math.log10(num).to_i + 1) - 10))
+    ).strftime('%Y-%m-%dT%H:%M:%S.%L%z')
+    assert_true(filtered.key?("@timestamp"))
+    assert_true(filtered.key?("fluent_converted_timestamp"))
+    assert_equal(formatted_time, filtered["fluent_converted_timestamp"])
+  end
+
+end

metadata

@@ -1,15 +1,49 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-elasticsearch-timestamp-check
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.2.5
 platform: ruby
 authors:
 - Richard Li
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-06-20 00:00:00.000000000 Z
+date: 2017-09-20 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '11.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '11.0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -24,6 +58,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
 description: fluent filter plugin to ensure @timestamp is in proper format
 email:
 - evilcat@wisewolfsolutions.com
@@ -32,10 +80,14 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- Gemfile
 - LICENSE
 - README.md
+- Rakefile
 - fluent-plugin-elasticsearch-timestamp-check.gemspec
 - lib/fluent/plugin/filter_elasticsearch_timestamp_check.rb
+- test/helper.rb
+- test/plugin/test_filter_elasticsearch_timestamp_check.rb
 homepage: https://github.com/ecwws/fluent-plugin-elasticsearch-timestamp-check
 licenses:
 - MIT
@@ -56,9 +108,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.2.5
 signing_key: 
 specification_version: 4
 summary: fluent timestamp checker filter
-test_files: []
-has_rdoc: 
+test_files:
+- test/helper.rb
+- test/plugin/test_filter_elasticsearch_timestamp_check.rb