fluent-plugin-tai64n_parser 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d71dccb155099860504f8f0c7ed61a60afb462e6
+  data.tar.gz: 8ebf64e76a9e2c03b48946d1acfb87843cf182bd
+SHA512:
+  metadata.gz: b829d4ffb506122ab704986834efc918d5a55224dcf2fe11b46bd3de6c6c0c52d0b4fac25df7dfb0ff888556627756c52a7a21c3866c4ac9b83af8940510eff7
+  data.tar.gz: 220286072d8ea0f22ed55b61fd83add6a15c25cc591b0cb0c079643e54e72ced3d9a6d1495226cfe39685495c10a119b87b253f2c6032b61a996aadc67fb4b9f

data/.gitignore

@@ -0,0 +1,4 @@
+.bundle
+*.swp
+.DS_Store
+Gemfile.lock

data/.travis.yml

@@ -0,0 +1,18 @@
+# https://github.com/travis-ci/travis-ci/wiki/.travis.yml-options
+language: ruby
+rvm:
+  - 1.9.3
+  - 2.0.0
+
+before_install:
+  - gem update bundler
+
+matrie:
+  allow_failures:
+    - rvm: 1.9.3
+    - rvm: 2.0.0
+
+env:
+  - TZ="Asia/Tokyo"
+
+script: bundle exec rake test

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2014- Akira Meda
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,56 @@
+# fluent-plugin-tai64n_parser
+
+[![Build Status](https://travis-ci.org/glidenote/fluent-plugin-tai64n_parser.png?branch=master)](https://travis-ci.org/glidenote/fluent-plugin-tai64n_parser)
+
+Fluentd plugin to parse [TAI64N format](http://cr.yp.to/libtai/tai64.html#tai64n).
+
+## Installation
+
+Use RubyGems:
+
+```
+gem install fluent-plugin-tai64n_parser
+```
+
+## Configuration
+
+Exampe:
+
+```
+<match test.**>
+  type             tai64n_parser
+
+  key              tai64n
+  parsed_time_tag  parsed_time
+  add_tag_prefix   parsed.
+</match>
+```
+
+Assume following input is coming (indented):
+
+``` js
+"test" => {
+  "tai64n"      => "@4000000052f88ea32489532c"
+}
+```
+
+then output becomes as below (indented):
+
+``` js
+"parsed.test" => {
+  "tai64n"      => "@4000000052f88ea32489532c"
+  "parsed_time" => "2014-02-10 17:32:25.612979500",
+}
+```
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+### Copyright
+
+Copyright (c) 2014- Akira Maeda. See [LICENSE](LICENSE) for details.

data/Rakefile

@@ -0,0 +1,9 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end

data/fluent-plugin-tai64n_parser.gemspec

@@ -0,0 +1,20 @@
+Gem::Specification.new do |gem|
+  gem.name          = 'fluent-plugin-tai64n_parser'
+  gem.version       = '0.0.1'
+  gem.authors       = ['Akira Maeda']
+  gem.email         = ['glidenote+github@gmail.com']
+  gem.homepage      = ''
+  gem.description   = %q{Fluentd plugin to parse the tai64n format log.}
+  gem.summary       = %q{Fluentd plugin to parse the tai64n format log.}
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ['lib']
+
+  gem.add_development_dependency 'rake'
+  gem.add_development_dependency 'fluentd'
+  gem.add_development_dependency 'test-unit'
+
+  gem.add_runtime_dependency     'fluentd'
+end

data/lib/fluent/plugin/out_tai64n_parser.rb

@@ -0,0 +1,61 @@
+module Fluent
+  class Tai64nParserOutput < Output
+    include Fluent::HandleTagNameMixin
+    Fluent::Plugin.register_output('tai64n_parser', self)
+
+    config_param :key, :string, :default => 'tai64n'
+    config_param :parsed_time_tag, :string, :default => 'parsed_time'
+
+    def configure(conf)
+      super
+      if (
+          !remove_tag_prefix &&
+          !remove_tag_suffix &&
+          !add_tag_prefix    &&
+          !add_tag_suffix
+      )
+        raise ConfigError, "out_tai64n_parser: At least one of remove_tag_prefix/remove_tag_suffix/add_tag_prefix/add_tag_suffix is required to be set."
+      end
+    end
+
+    def start
+      super
+    end
+
+    def shutdown
+      super
+    end
+
+    def emit(tag, es, chain)
+      es.each {|time,record|
+        t = tag.dup
+        filter_record(t, time, record)
+        Engine.emit(t, time, record)
+      }
+      chain.next
+    end
+
+    def filter_record(tag, time, record)
+      begin
+        # @4000000052f88ea32489532c
+        # 0123456789012345678901234
+        # 0         1         2
+        #   |-------------||------|
+        if record[key][0,2] == '@4' then
+          ts = record[key][2,15].hex
+          tf = record[key][17,8].hex
+          t = Time.at(ts-10,tf/1000.0)
+          record_time = t.strftime("%Y-%m-%d %X.%9N")
+        else
+          record_time = nil
+        end
+
+        record[parsed_time_tag] = record_time
+
+      rescue ArgumentError => error
+        $log.warn("out_tai64n_parser: #{error.message}")
+      end
+      super(tag, time, record)
+    end
+  end
+end

data/test/plugin/test_out_tai64n_parser.rb

@@ -0,0 +1,112 @@
+# -*- encoding: utf-8 -*-
+require 'test_helper'
+
+class Tai64nParserOutputTest < Test::Unit::TestCase
+
+  TAI64N_TIME = "@4000000052f88ea32489532c"
+  BAD_TAI64N_TIME = "4000000052f88ea32489"
+
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf, tag = 'test')
+    Fluent::Test::OutputTestDriver.new(
+      Fluent::Tai64nParserOutput, tag
+    ).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver(%[
+      key            test
+      add_tag_prefix parsed.
+    ])
+    assert_equal 'test', d.instance.key
+    assert_equal 'parsed.', d.instance.add_tag_prefix
+
+    #Default Key
+    d = create_driver(%[
+      add_tag_prefix parsed.
+    ])
+    assert_equal 'tai64n', d.instance.key
+    assert_equal 'parsed.', d.instance.add_tag_prefix
+  end
+
+  def test_filter_record
+    d = create_driver(%[
+      key            tai64n
+      add_tag_prefix parsed.
+    ])
+    tag    = 'test'
+    record = {'tai64n' => TAI64N_TIME}
+    d.instance.filter_record('test', Time.now, record)
+
+    assert_equal record['tai64n'], TAI64N_TIME
+    assert_equal record['parsed_time'], '2014-02-10 17:32:25.612979500'
+  end
+
+  def test_filter_record_bad_parameters
+    d = create_driver(%[
+      key            tai64n
+      add_tag_prefix parsed.
+    ])
+    tag    = 'test'
+    record = {'tai64n' => BAD_TAI64N_TIME}
+
+    d.instance.filter_record('test', Time.now, record)
+    assert_equal record['parsed_time'], nil
+
+    record = {'tai64n' => "this is not a date"}
+    d.instance.filter_record('test', Time.now, record)
+    assert_equal record['parsed_time'], nil
+  end
+
+  def test_emit
+    d = create_driver(%[
+      key            tai64n
+      add_tag_prefix parsed.
+    ])
+
+    d.run { d.emit('tai64n' => TAI64N_TIME) }
+    emits = d.emits
+
+    assert_equal 1, emits.count
+    assert_equal 'parsed.test', emits[0][0]
+    assert_equal TAI64N_TIME, emits[0][2]['tai64n']
+  end
+
+  def test_emit_multi
+    d = create_driver(%[
+      key            tai64n
+      add_tag_prefix parsed.
+    ])
+
+    d.run do
+      d.emit('tai64n' => TAI64N_TIME)
+      d.emit('tai64n' => TAI64N_TIME)
+      d.emit('tai64n' => TAI64N_TIME)
+    end
+    emits = d.emits
+
+    assert_equal 3, emits.count
+    0.upto(2) do |i|
+      assert_equal 'parsed.test', emits[i][0]
+      assert_equal TAI64N_TIME, emits[i][2]['tai64n']
+    end
+  end
+
+  def test_emit_with_invalid_tai64n
+    d = create_driver(%[
+      key            tai64n
+      add_tag_prefix parsed.
+    ])
+    wrong_time = 'wrong time'
+    d.run { d.emit('tai64n' => wrong_time) }
+    emits = d.emits
+
+    assert_equal 1, emits.count
+    assert_equal 'parsed.test', emits[0][0]
+    assert_equal wrong_time, emits[0][2]['tai64n']
+  end
+
+end

data/test/test_helper.rb

@@ -0,0 +1,21 @@
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'fluent/test'
+
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/plugin/out_tai64n_parser'
+
+class Test::Unit::TestCase
+end

metadata

@@ -0,0 +1,111 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-tai64n_parser
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Akira Maeda
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-02-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluentd plugin to parse the tai64n format log.
+email:
+- glidenote+github@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- fluent-plugin-tai64n_parser.gemspec
+- lib/fluent/plugin/out_tai64n_parser.rb
+- test/plugin/test_out_tai64n_parser.rb
+- test/test_helper.rb
+homepage: ''
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.14
+signing_key: 
+specification_version: 4
+summary: Fluentd plugin to parse the tai64n format log.
+test_files:
+- test/plugin/test_out_tai64n_parser.rb
+- test/test_helper.rb