fluent-plugin-better-timestamp-timekey 0.1.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d77280f829ab1d3a679b571af8fd882e839840e6
+  data.tar.gz: 1334762f5f1d33684c8c7d1980561e6e79cf644c
+SHA512:
+  metadata.gz: 270893a0d605c2f4a5ad681acd9678f27a689b77b4febc428fb28de1be1ee75022adb28fc433f13cfcdad496bfbd6abfcd2fabc2082ce2a7d792e5850cea5d35
+  data.tar.gz: 43540b5dbb8e8e1b649684bd357ba02dc10f28326446cbd54a2e0257257dd78aaec8ee80e54ab5e18d07b601fddd0f7510dda2a1d422bfb4e2ef179ab8fedef0

data/.gitignore

@@ -0,0 +1 @@
+pkg

data/ChangeLog

@@ -0,0 +1,3 @@
+Release 0.1.0 - 2014/11/16
+
+  * First release

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/README.md

@@ -0,0 +1,45 @@
+fluent-plugin-better-timestamp
+==============================
+
+fluentd plugin to store timestamp with millisecond to elasticsearch
+
+### fluent-plugin-elasticsearch
+fluentd uses unixtime as 'time' and fluent-plugin-elasticsearch inserts the 'time' to elasticsearch as @timestamp. Therefore, it's not easy to store more detailed time data like millisec. But fluentd-plugin-elasticsearch provides another way to store timestamp to elasticsearch. if record has '@timestamp', that will be  stored in elasticsearh.
+
+### fluent-plugin-better-timestamp
+this plugin works as a filter that creates '@timestamp' with proper format from 'time' and one more field which have 'msec' data
+
+## install
+```
+gem install fluent-plugin-better-timestamp
+```
+
+### configuration
+'better-timestamp' plugin has only 3 config parameters
+
+ * tag
+ * msec_key - specifies the key name for msec (default:  msec)
+ * timestamp_key - specifies the key name for output (default: @timestamp)
+
+### example
+
+```
+<source>
+  type tail
+  format
+  format /^(?<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}).(?<msec>\d{3}) .....
+  ...
+  tag log
+</source>
+<match log>
+  type better_timestamp
+  tag log.with_msec
+</match>
+<match tag log.with_msec>
+  type elasticsearch
+  ...
+</match
+```
+
+
+

data/Rakefile

@@ -0,0 +1,14 @@
+
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/*.rb']
+  test.verbose = true
+end
+
+task :default => [:build]
+

data/VERSION

@@ -0,0 +1 @@
+0.1.1

data/fluent-plugin-better-timestamp.gemspec

@@ -0,0 +1,23 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-better-timestamp-timekey"
+  gem.description = "Output filter plugin which put timestamp with configurable time_key"
+  gem.homepage    = "https://github.com/spynode/fluent-plugin-better-timestamp"
+  gem.summary     = gem.description
+  gem.version     = File.read("VERSION").strip
+  gem.authors     = ["Kenichi Otsuka"]
+  gem.email       = "shivaken@gmail.com"
+  gem.has_rdoc    = false
+  #gem.platform    = Gem::Platform::RUBY
+  gem.license     = 'MIT'
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "fluentd", "~> 0.10.17"
+  gem.add_dependency "fluent-mixin-config-placeholders", ">= 0.3.0"
+  gem.add_development_dependency "rake", ">= 0.9.2"
+end

data/lib/fluent/plugin/out_better_timestamp.rb

@@ -0,0 +1,49 @@
+require 'fluent/mixin/config_placeholders'
+
+module Fluent
+  class BetterTimestampOutput < Output
+    Fluent::Plugin.register_output('better_timestamp', self)
+
+    config_param :tag, :string
+    config_param :time_key, :string, :default => 'time'
+    config_param :msec_key, :string, :default => 'msec'
+    config_param :timestamp_key, :string, :default => '@timestamp'
+
+    include SetTagKeyMixin
+    include Fluent::Mixin::ConfigPlaceholders
+
+    BUILTIN_CONFIGURATIONS = %W(type tag timestamp_key msec_key)
+
+    def configure(conf)
+      super
+
+      @map = {}
+      conf.each_pair { |k, v|
+        unless BUILTIN_CONFIGURATIONS.include?(k)
+          conf.has_key?(k)
+          @map[k] = v
+        end
+      }
+
+    end
+
+    def emit(tag, es, chain)
+      es.each { |time, record|
+        filter_record(tag, time, record)
+        Engine.emit(@tag, time, modify_record(time, record))
+      }
+
+      chain.next
+    end
+
+    private
+
+    def modify_record(time, record)
+      if record[@msec_key] then
+        record[@timestamp_key] = Time.at(record[@time_key], record[@msec_key].to_i * 1000).strftime("%Y-%m-%dT%H:%M:%S.%L%z")
+        record.delete(@msec_key)
+      end
+      record
+    end
+  end
+end

data/test/out_better_timestamp.rb

@@ -0,0 +1,42 @@
+require 'fluent/test'
+require 'fluent/plugin/out_better_timestamp'
+
+class BetterTimestampOutputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  CONFIG = %[
+    type better_timestamp
+    tag foo.filtered
+    msec_key msec
+    timestamp_key
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::OutputTestDriver.new(Fluent::BetterTimestampOutput, tag='test_tag').configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    map = d.instance.instance_variable_get(:@map)
+
+    #assert_equal 'msec', map['msec_key']
+  end
+
+  def test_remove_one_key
+    d = create_driver %[
+      type record_modifier
+      tag foo.filtered
+      msec_key msec
+    ]
+
+    mapped = {}
+
+    d.run do
+      d.emit("msec" => '1', "k1" => 'v')
+    end
+
+    assert d.records[0]['@timestamp']
+  end
+end

metadata

@@ -0,0 +1,96 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-better-timestamp-timekey
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- Kenichi Otsuka
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-05-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.17
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.17
+- !ruby/object:Gem::Dependency
+  name: fluent-mixin-config-placeholders
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+description: Output filter plugin which put timestamp with configurable time_key
+email: shivaken@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ChangeLog
+- Gemfile
+- README.md
+- Rakefile
+- VERSION
+- fluent-plugin-better-timestamp.gemspec
+- lib/fluent/plugin/out_better_timestamp.rb
+- pkg/fluent-plugin-better-timestamp-0.1.0.gem
+- test/out_better_timestamp.rb
+homepage: https://github.com/spynode/fluent-plugin-better-timestamp
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Output filter plugin which put timestamp with configurable time_key
+test_files:
+- test/out_better_timestamp.rb