fluent-plugin-tai64n_parser 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1f47b272961f4be0457ff8ef982c0f29fa396f74
-  data.tar.gz: 0863699cd861ef18b4b999681cfeb7383c100841
+  metadata.gz: 98eeadeabea7c5c15b15b840c1d8d6b9ade095ce
+  data.tar.gz: 0a03ab28e47f559559422470cd0f07573a9694a7
 SHA512:
-  metadata.gz: 0225747aa31a264174f99c84e63372e37227a28a3a2863f09b2d34030497b3ca32baa703170b6487838a9eb688fbcf2aa2c793d1ba25c82c1e016e3469598def
-  data.tar.gz: d7506a97b2d672c990e9ea9701e831e9b20684199d286220eda0de7b8ae78c0e1e90f72d86720ffce5054bfcf7af115143feb8d87e40b0447197f6e5abe39a3c
+  metadata.gz: ee877722ddb16ce7162959e1b12bc3a475324f9244adbeaade7e6584fbf112ff1a16bf87067ea54ce14f8bf56ce4c37b04fa1c133ee48402a3e477a811b148f2
+  data.tar.gz: 204d311a6c71145c83aa716376956dd9d4e7a8f090e8ab1f717bcb1cb43d2483df4681efde2a4f47ab33b538c28e33a446d7c09f3ae247db67c4a3d2ea5410d8

data/.travis.yml

@@ -1,18 +1,15 @@
 # https://github.com/travis-ci/travis-ci/wiki/.travis.yml-options
 language: ruby
 rvm:
-  - 1.9.3
-  - 2.0.0
-
+  - 2.1.*
+  - 2.2.*
+  - 2.3.*
+gemfile:
+  - Gemfile
+  - Gemfile.fluentd.v0.12
+  - Gemfile.fluentd.v0.10
 before_install:
   - gem update bundler
-
-matrie:
-  allow_failures:
-    - rvm: 1.9.3
-    - rvm: 2.0.0
-
 env:
   - TZ="Asia/Tokyo"
-
 script: bundle exec rake test

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+# 0.2.0 (2016-09-24)
+
+Enhancements
+
+* Add filter plugin

data/Gemfile

@@ -1,2 +1,3 @@
 source 'https://rubygems.org'
+
 gemspec

data/Gemfile.fluentd.v0.10

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+gem 'fluentd', '~> 0.10.43'
+gemspec

data/Gemfile.fluentd.v0.12

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+gem 'fluentd', '~> 0.12.0'
+gemspec

data/README.md

@@ -14,15 +14,39 @@ gem install fluent-plugin-tai64n_parser
 
 ## Configuration
 
-### Basic Example:
+### Filter Plugin
 
 ```
-<match test.**>
-  type             tai64n_parser
+<filter test.**>
+  @type tai64n_parser
+  key   message
+</match>
+```
 
-  key              tai64n
-  output_key       parsed_time
-  add_tag_prefix   parsed.
+Assume following input is coming (indented):
+
+``` js
+"test" => {
+  "message" => "@4000000052f88ea32489532c [FATAL] blah blah"
+}
+```
+
+then output becomes as below (indented):
+
+``` js
+"test" => {
+  "message" => "2014-02-10 17:32:25.612979500 [FATAL] blah blah"
+}
+```
+
+### Output Plugin (Old Style)
+
+```
+<match test.**>
+  @type           tai64n_parser
+  
+  key             message
+  add_tag_prefix  parsed.
 </match>
 ```
 
@@ -30,7 +54,7 @@ Assume following input is coming (indented):
 
 ``` js
 "test" => {
-  "tai64n"      => "@4000000052f88ea32489532c"
+  "message" => "@4000000052f88ea32489532c [FATAL] blah blah"
 }
 ```
 
@@ -38,8 +62,7 @@ then output becomes as below (indented):
 
 ``` js
 "parsed.test" => {
-  "tai64n"      => "@4000000052f88ea32489532c"
-  "parsed_time" => "2014-02-10 17:32:25.612979500",
+  "message" => "2014-02-10 17:32:25.612979500 [FATAL] blah blah"
 }
 ```
 
@@ -93,6 +116,52 @@ delivery_id":"9","parsed_time":"2014-02-12 13:50:11.947211500"}
 2014-02-12T13:50:11+09:00       parsed.qmail.sent       {"tai64n":"@4000000052fafd8d387554bc","message":"end msg 3890","key":"3890","parsed_time":"2014-02-12 13:50:11.947213500"}
 ```
 
+## Parameters
+
+### Filter Plugin
+
+- key *field\_key*
+
+    The target field key to parse tai64n. The first 25 characters are tried to parse.
+    If the characters are of tail64n format, convert to the time format as of `tai64nlocal` command.
+    If the characters are not of tai64n format, just pass through the input with info level messages on the log.
+
+- output\_key *field\_key*
+
+    The target field to output the parsed result. The default is equivalent with `key` which means to overwrite the `key` field. 
+
+### Output Plugin (Old Style)
+
+- key *field\_key*
+
+    The target field key to parse tai64n. The first 25 characters are tried to parse.
+    If the characters are of tail64n format, convert to the time format as of `tai64nlocal` command.
+    If the characters are not of tai64n format, just pass through the input with info level messages on the log.
+
+- output\_key *field\_key*
+
+    The target field to output the parsed result. The default is equivalent with `key` which means to overwrite the `key` field. 
+
+- add_tag_prefix
+
+    Add tag prefix for output message
+
+- remove_tag_prefix
+
+    Remove tag prefix for output message
+
+- add_tag_suffix
+
+    Add tag suffix for output message
+
+- remove_tag_suffix
+
+    Remove tag suffix for output message
+
+## ChangeLog
+
+[CHANGELOG.md](./CHANGELOG.md)
+
 ## Contributing
 
 1. Fork it

data/Rakefile

@@ -6,4 +6,5 @@ Rake::TestTask.new(:test) do |test|
   test.libs << 'lib' << 'test'
   test.pattern = 'test/**/test_*.rb'
   test.verbose = true
+  test.warning = false
 end

data/fluent-plugin-tai64n_parser.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |gem|
   gem.name          = 'fluent-plugin-tai64n_parser'
-  gem.version       = '0.1.0'
+  gem.version       = '0.2.0'
   gem.authors       = ['Akira Maeda','Naotoshi Seo']
   gem.email         = ['glidenote+github@gmail.com','sonots@gmail.com']
   gem.homepage      = 'https://github.com/glidenote/fluent-plugin-tai64n_parser'
@@ -12,9 +12,8 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ['lib']
 
+  gem.add_runtime_dependency     'fluentd'
+
   gem.add_development_dependency 'rake'
-  gem.add_development_dependency 'fluentd'
   gem.add_development_dependency 'test-unit'
-
-  gem.add_runtime_dependency     'fluentd'
 end

data/lib/fluent/plugin/filter_tai64n_parser.rb

@@ -0,0 +1,35 @@
+require_relative 'tai64n_parser_mixin'
+
+module Fluent
+  class Tai64nParserFilter < Filter
+    include Fluent::Tai64nParserMixin
+    Fluent::Plugin.register_filter('tai64n_parser', self)
+
+    config_param :key, :string, :default => 'tai64n'
+    config_param :output_key, :string, :default => nil
+
+    def configure(conf)
+      super
+      @output_key ||= @key
+    end
+
+    def start
+      super
+    end
+
+    def shutdown
+      super
+    end
+
+    def filter(tag, time, record)
+      begin
+        record[output_key] = try_replace_tai64n(record[key])
+      rescue => e
+        log.warn("filter_tai64n_parser: #{e.class} #{e.message}")
+        log.warn_backtrace
+      end
+      record
+    end
+
+  end
+end if defined?(Fluent::Filter) # Support only >= v0.12

data/lib/fluent/plugin/out_tai64n_parser.rb

@@ -1,6 +1,9 @@
+require_relative 'tai64n_parser_mixin'
+
 module Fluent
   class Tai64nParserOutput < Output
     include Fluent::HandleTagNameMixin
+    include Fluent::Tai64nParserMixin
     Fluent::Plugin.register_output('tai64n_parser', self)
 
     # Define `log` method for v0.10.42 or earlier
@@ -8,6 +11,11 @@ module Fluent
       define_method("log") { $log }
     end
 
+    # Define `router` method of v0.12 to support v0.10 or earlier
+    unless method_defined?(:router)
+      define_method("router") { Fluent::Engine }
+    end
+
     config_param :key, :string, :default => 'tai64n'
     config_param :output_key, :string, :default => nil
 
@@ -36,41 +44,20 @@ module Fluent
       es.each {|time,record|
         t = tag.dup
         filter_record(t, time, record)
-        Engine.emit(t, time, record)
+        router.emit(t, time, record)
       }
       chain.next
     end
 
     def filter_record(tag, time, record)
       begin
-        record[output_key] = replace_tai64n(record[key])
-      rescue => error
-        log.warn("out_tai64n_parser: #{error.class} #{error.message} #{error.backtrace.first}")
+        record[output_key] = try_replace_tai64n(record[key])
+      rescue => e
+        log.warn("out_tai64n_parser: #{e.class} #{e.message}")
+        log.warn_backtrace
       end
-      super(tag, time, record)
+      super(tag, time, record) # HandleTagNameMixin
     end
 
-    def replace_tai64n(str)
-      tai64n, rest = str[0,25], str[25..-1]
-      parsed = parse_tai64n(tai64n)
-      if parsed
-        "#{parsed}#{rest}"
-      else
-        log.info("out_tai64n_parser: record['#{key}']='#{str}' does not start with valid tai64n")
-        str
-      end
-    end
-
-    def parse_tai64n(tai64n)
-      # @4000000052f88ea32489532c
-      # 0123456789012345678901234
-      # 0         1         2
-      #   |-------------||------|
-      return nil unless tai64n[0,2] == '@4'
-      ts = tai64n[2,15].hex
-      tf = tai64n[17,8].hex
-      t = Time.at(ts-10,tf/1000.0)
-      t.strftime("%Y-%m-%d %X.%9N")
-    end
   end
 end

data/lib/fluent/plugin/tai64n_parser_mixin.rb

@@ -0,0 +1,26 @@
+module Fluent
+  module Tai64nParserMixin
+    def try_replace_tai64n(str)
+      tai64n, rest = str[0,25], str[25..-1]
+      parsed = parse_tai64n(tai64n)
+      if parsed
+        "#{parsed}#{rest}"
+      else
+        log.info("tai64n_parser: record['#{key}']='#{str}' does not start with valid tai64n")
+        str
+      end
+    end
+
+    def parse_tai64n(tai64n)
+      # @4000000052f88ea32489532c
+      # 0123456789012345678901234
+      # 0         1         2
+      #   |-------------||------|
+      return nil unless tai64n[0,2] == '@4'
+      ts = tai64n[2,15].hex
+      tf = tai64n[17,8].hex
+      t = Time.at(ts-10,tf/1000.0)
+      t.strftime("%Y-%m-%d %X.%9N")
+    end
+  end
+end

data/test/plugin/test_filter_tai64n_parser.rb

@@ -0,0 +1,58 @@
+require_relative '../test_helper'
+require 'fluent/plugin/filter_tai64n_parser'
+
+class Tai64nParserFilterTest < Test::Unit::TestCase
+
+  TAI64N_TIME = "@4000000052f88ea32489532c"
+  PARSED_TIME = "2014-02-10 17:32:25.612979500"
+
+  def setup
+    Fluent::Test.setup
+    @time = Fluent::Engine.now
+  end
+
+  def create_driver(conf)
+    Fluent::Test::FilterTestDriver.new(
+      Fluent::Tai64nParserFilter
+    ).configure(conf)
+  end
+
+  def filter(config, msgs)
+    d = create_driver(config)
+    d.run {
+      msgs.each {|msg|
+        d.filter(msg, @time)
+      }
+    }
+    filtered = d.filtered_as_array
+    filtered.map {|m| m[2] }
+  end
+
+  def test_configure
+    d = create_driver(%[
+      key        test
+      output_key parsed_time
+    ])
+    assert_equal 'test', d.instance.key
+    assert_equal 'parsed_time', d.instance.output_key
+
+    #Default Key
+    d = create_driver(%[
+    ])
+    assert_equal 'tai64n', d.instance.key
+    assert_equal 'tai64n', d.instance.output_key
+  end
+
+  def test_filter
+    d = create_driver(%[
+      key        tai64n
+      output_key parsed_time
+    ])
+    record = {'tai64n' => TAI64N_TIME}
+    d.run { d.filter(record, @time) }
+
+    filtered_record = d.filtered_as_array.first[2]
+    assert_equal PARSED_TIME, filtered_record['parsed_time']
+  end
+
+end if defined?(Fluent::Filter)

data/test/plugin/test_out_tai64n_parser.rb

@@ -1,11 +1,10 @@
-# -*- encoding: utf-8 -*-
-require 'test_helper'
+require_relative '../test_helper'
+require 'fluent/plugin/out_tai64n_parser'
 
 class Tai64nParserOutputTest < Test::Unit::TestCase
 
   TAI64N_TIME = "@4000000052f88ea32489532c"
   PARSED_TIME = "2014-02-10 17:32:25.612979500"
-  BAD_TAI64N_TIME = "4000000052f88ea32489"
 
   def setup
     Fluent::Test.setup
@@ -50,23 +49,6 @@ class Tai64nParserOutputTest < Test::Unit::TestCase
     assert_equal record['parsed_time'], '2014-02-10 17:32:25.612979500'
   end
 
-  def test_filter_record_bad_parameters
-    d = create_driver(%[
-      key            tai64n
-      output_key     parsed_time
-      add_tag_prefix parsed.
-    ])
-    tag    = 'test'
-    record = {'tai64n' => BAD_TAI64N_TIME}
-
-    d.instance.filter_record('test', Time.now, record)
-    assert_equal record['parsed_time'], BAD_TAI64N_TIME
-
-    record = {'tai64n' => "this is not a date"}
-    d.instance.filter_record('test', Time.now, record)
-    assert_equal record['parsed_time'], "this is not a date"
-  end
-
   def test_emit
     d = create_driver(%[
       key            tai64n
@@ -103,34 +85,4 @@ class Tai64nParserOutputTest < Test::Unit::TestCase
     end
   end
 
-  def test_emit_with_invalid_tai64n
-    d = create_driver(%[
-      key            tai64n
-      output_key     parsed_time
-      add_tag_prefix parsed.
-    ])
-    wrong_time = 'wrong time'
-    d.run { d.emit('tai64n' => wrong_time) }
-    emits = d.emits
-
-    assert_equal 1, emits.count
-    assert_equal 'parsed.test', emits[0][0]
-    assert_equal wrong_time, emits[0][2]['tai64n']
-  end
-
-  def test_emit_with_replace
-    d = create_driver(%[
-      key             message
-      parsed_time_tag message
-      add_tag_prefix parsed.
-    ])
-
-    d.run { d.emit('message' => "#{TAI64N_TIME}foobar") }
-    emits = d.emits
-
-    assert_equal 1, emits.count
-    assert_equal 'parsed.test', emits[0][0]
-    assert_equal "#{PARSED_TIME}foobar", emits[0][2]['message']
-  end
-
 end

data/test/plugin/test_tai64n_parser_mixin.rb

@@ -0,0 +1,38 @@
+require_relative '../test_helper'
+require 'fluent/plugin/tai64n_parser_mixin'
+require 'logger'
+
+class Tai64nParserTest < Test::Unit::TestCase
+
+  class Tai64nParser
+    include Fluent::Tai64nParserMixin
+
+    def log
+      @log ||= Logger.new(logdev)
+    end
+
+    def logdev
+      @logdev ||= StringIO.new
+    end
+
+    def key
+      'foo'
+    end
+  end
+
+  def parser
+    @parser ||= Tai64nParser.new
+  end
+
+  def test_try_replace_tai64n
+    good_tai64n = "@4000000052f88ea32489532c"
+    expected_time = "2014-02-10 17:32:25.612979500"
+    assert_equal expected_time, parser.try_replace_tai64n(good_tai64n)
+    assert_equal "#{expected_time} foo", parser.try_replace_tai64n("#{good_tai64n} foo")
+
+    bad_tai64n = "4000000052f88ea32489"
+    assert_equal bad_tai64n, parser.try_replace_tai64n(bad_tai64n)
+    assert_true parser.logdev.tap {|d| d.rewind }.read.include?('does not start with valid tai64n')
+  end
+
+end

data/test/test_helper.rb

@@ -15,7 +15,5 @@ unless ENV.has_key?('VERBOSE')
   $log = nulllogger
 end
 
-require 'fluent/plugin/out_tai64n_parser'
-
 class Test::Unit::TestCase
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-tai64n_parser
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Akira Maeda
@@ -9,62 +9,48 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-19 00:00:00.000000000 Z
+date: 2016-09-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: fluentd
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: test-unit
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-- !ruby/object:Gem::Dependency
-  name: fluentd
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: Fluentd plugin to parse the tai64n format log.
@@ -75,15 +61,22 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .travis.yml
+- ".gitignore"
+- ".travis.yml"
+- CHANGELOG.md
 - Gemfile
+- Gemfile.fluentd.v0.10
+- Gemfile.fluentd.v0.12
 - LICENSE
 - README.md
 - Rakefile
 - fluent-plugin-tai64n_parser.gemspec
+- lib/fluent/plugin/filter_tai64n_parser.rb
 - lib/fluent/plugin/out_tai64n_parser.rb
+- lib/fluent/plugin/tai64n_parser_mixin.rb
+- test/plugin/test_filter_tai64n_parser.rb
 - test/plugin/test_out_tai64n_parser.rb
+- test/plugin/test_tai64n_parser_mixin.rb
 - test/test_helper.rb
 homepage: https://github.com/glidenote/fluent-plugin-tai64n_parser
 licenses: []
@@ -94,20 +87,22 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Fluentd plugin to parse the tai64n format log.
 test_files:
+- test/plugin/test_filter_tai64n_parser.rb
 - test/plugin/test_out_tai64n_parser.rb
+- test/plugin/test_tai64n_parser_mixin.rb
 - test/test_helper.rb