fluent-plugin-tai64n_parser 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +4 -0
- data/.travis.yml +18 -0
- data/Gemfile +2 -0
- data/LICENSE +22 -0
- data/README.md +56 -0
- data/Rakefile +9 -0
- data/fluent-plugin-tai64n_parser.gemspec +20 -0
- data/lib/fluent/plugin/out_tai64n_parser.rb +61 -0
- data/test/plugin/test_out_tai64n_parser.rb +112 -0
- data/test/test_helper.rb +21 -0
- metadata +111 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: d71dccb155099860504f8f0c7ed61a60afb462e6
|
4
|
+
data.tar.gz: 8ebf64e76a9e2c03b48946d1acfb87843cf182bd
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: b829d4ffb506122ab704986834efc918d5a55224dcf2fe11b46bd3de6c6c0c52d0b4fac25df7dfb0ff888556627756c52a7a21c3866c4ac9b83af8940510eff7
|
7
|
+
data.tar.gz: 220286072d8ea0f22ed55b61fd83add6a15c25cc591b0cb0c079643e54e72ced3d9a6d1495226cfe39685495c10a119b87b253f2c6032b61a996aadc67fb4b9f
|
data/.gitignore
ADDED
data/.travis.yml
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
# https://github.com/travis-ci/travis-ci/wiki/.travis.yml-options
|
2
|
+
language: ruby
|
3
|
+
rvm:
|
4
|
+
- 1.9.3
|
5
|
+
- 2.0.0
|
6
|
+
|
7
|
+
before_install:
|
8
|
+
- gem update bundler
|
9
|
+
|
10
|
+
matrie:
|
11
|
+
allow_failures:
|
12
|
+
- rvm: 1.9.3
|
13
|
+
- rvm: 2.0.0
|
14
|
+
|
15
|
+
env:
|
16
|
+
- TZ="Asia/Tokyo"
|
17
|
+
|
18
|
+
script: bundle exec rake test
|
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2014- Akira Meda
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,56 @@
|
|
1
|
+
# fluent-plugin-tai64n_parser
|
2
|
+
|
3
|
+
[](https://travis-ci.org/glidenote/fluent-plugin-tai64n_parser)
|
4
|
+
|
5
|
+
Fluentd plugin to parse [TAI64N format](http://cr.yp.to/libtai/tai64.html#tai64n).
|
6
|
+
|
7
|
+
## Installation
|
8
|
+
|
9
|
+
Use RubyGems:
|
10
|
+
|
11
|
+
```
|
12
|
+
gem install fluent-plugin-tai64n_parser
|
13
|
+
```
|
14
|
+
|
15
|
+
## Configuration
|
16
|
+
|
17
|
+
Exampe:
|
18
|
+
|
19
|
+
```
|
20
|
+
<match test.**>
|
21
|
+
type tai64n_parser
|
22
|
+
|
23
|
+
key tai64n
|
24
|
+
parsed_time_tag parsed_time
|
25
|
+
add_tag_prefix parsed.
|
26
|
+
</match>
|
27
|
+
```
|
28
|
+
|
29
|
+
Assume following input is coming (indented):
|
30
|
+
|
31
|
+
``` js
|
32
|
+
"test" => {
|
33
|
+
"tai64n" => "@4000000052f88ea32489532c"
|
34
|
+
}
|
35
|
+
```
|
36
|
+
|
37
|
+
then output becomes as below (indented):
|
38
|
+
|
39
|
+
``` js
|
40
|
+
"parsed.test" => {
|
41
|
+
"tai64n" => "@4000000052f88ea32489532c"
|
42
|
+
"parsed_time" => "2014-02-10 17:32:25.612979500",
|
43
|
+
}
|
44
|
+
```
|
45
|
+
|
46
|
+
## Contributing
|
47
|
+
|
48
|
+
1. Fork it
|
49
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
50
|
+
3. Commit your changes (`git commit -am 'Added some feature'`)
|
51
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
52
|
+
5. Create new Pull Request
|
53
|
+
|
54
|
+
### Copyright
|
55
|
+
|
56
|
+
Copyright (c) 2014- Akira Maeda. See [LICENSE](LICENSE) for details.
|
data/Rakefile
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Gem::Specification.new do |gem|
|
2
|
+
gem.name = 'fluent-plugin-tai64n_parser'
|
3
|
+
gem.version = '0.0.1'
|
4
|
+
gem.authors = ['Akira Maeda']
|
5
|
+
gem.email = ['glidenote+github@gmail.com']
|
6
|
+
gem.homepage = ''
|
7
|
+
gem.description = %q{Fluentd plugin to parse the tai64n format log.}
|
8
|
+
gem.summary = %q{Fluentd plugin to parse the tai64n format log.}
|
9
|
+
|
10
|
+
gem.files = `git ls-files`.split($\)
|
11
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
12
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
13
|
+
gem.require_paths = ['lib']
|
14
|
+
|
15
|
+
gem.add_development_dependency 'rake'
|
16
|
+
gem.add_development_dependency 'fluentd'
|
17
|
+
gem.add_development_dependency 'test-unit'
|
18
|
+
|
19
|
+
gem.add_runtime_dependency 'fluentd'
|
20
|
+
end
|
@@ -0,0 +1,61 @@
|
|
1
|
+
module Fluent
|
2
|
+
class Tai64nParserOutput < Output
|
3
|
+
include Fluent::HandleTagNameMixin
|
4
|
+
Fluent::Plugin.register_output('tai64n_parser', self)
|
5
|
+
|
6
|
+
config_param :key, :string, :default => 'tai64n'
|
7
|
+
config_param :parsed_time_tag, :string, :default => 'parsed_time'
|
8
|
+
|
9
|
+
def configure(conf)
|
10
|
+
super
|
11
|
+
if (
|
12
|
+
!remove_tag_prefix &&
|
13
|
+
!remove_tag_suffix &&
|
14
|
+
!add_tag_prefix &&
|
15
|
+
!add_tag_suffix
|
16
|
+
)
|
17
|
+
raise ConfigError, "out_tai64n_parser: At least one of remove_tag_prefix/remove_tag_suffix/add_tag_prefix/add_tag_suffix is required to be set."
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def start
|
22
|
+
super
|
23
|
+
end
|
24
|
+
|
25
|
+
def shutdown
|
26
|
+
super
|
27
|
+
end
|
28
|
+
|
29
|
+
def emit(tag, es, chain)
|
30
|
+
es.each {|time,record|
|
31
|
+
t = tag.dup
|
32
|
+
filter_record(t, time, record)
|
33
|
+
Engine.emit(t, time, record)
|
34
|
+
}
|
35
|
+
chain.next
|
36
|
+
end
|
37
|
+
|
38
|
+
def filter_record(tag, time, record)
|
39
|
+
begin
|
40
|
+
# @4000000052f88ea32489532c
|
41
|
+
# 0123456789012345678901234
|
42
|
+
# 0 1 2
|
43
|
+
# |-------------||------|
|
44
|
+
if record[key][0,2] == '@4' then
|
45
|
+
ts = record[key][2,15].hex
|
46
|
+
tf = record[key][17,8].hex
|
47
|
+
t = Time.at(ts-10,tf/1000.0)
|
48
|
+
record_time = t.strftime("%Y-%m-%d %X.%9N")
|
49
|
+
else
|
50
|
+
record_time = nil
|
51
|
+
end
|
52
|
+
|
53
|
+
record[parsed_time_tag] = record_time
|
54
|
+
|
55
|
+
rescue ArgumentError => error
|
56
|
+
$log.warn("out_tai64n_parser: #{error.message}")
|
57
|
+
end
|
58
|
+
super(tag, time, record)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
@@ -0,0 +1,112 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
require 'test_helper'
|
3
|
+
|
4
|
+
class Tai64nParserOutputTest < Test::Unit::TestCase
|
5
|
+
|
6
|
+
TAI64N_TIME = "@4000000052f88ea32489532c"
|
7
|
+
BAD_TAI64N_TIME = "4000000052f88ea32489"
|
8
|
+
|
9
|
+
def setup
|
10
|
+
Fluent::Test.setup
|
11
|
+
end
|
12
|
+
|
13
|
+
def create_driver(conf, tag = 'test')
|
14
|
+
Fluent::Test::OutputTestDriver.new(
|
15
|
+
Fluent::Tai64nParserOutput, tag
|
16
|
+
).configure(conf)
|
17
|
+
end
|
18
|
+
|
19
|
+
def test_configure
|
20
|
+
d = create_driver(%[
|
21
|
+
key test
|
22
|
+
add_tag_prefix parsed.
|
23
|
+
])
|
24
|
+
assert_equal 'test', d.instance.key
|
25
|
+
assert_equal 'parsed.', d.instance.add_tag_prefix
|
26
|
+
|
27
|
+
#Default Key
|
28
|
+
d = create_driver(%[
|
29
|
+
add_tag_prefix parsed.
|
30
|
+
])
|
31
|
+
assert_equal 'tai64n', d.instance.key
|
32
|
+
assert_equal 'parsed.', d.instance.add_tag_prefix
|
33
|
+
end
|
34
|
+
|
35
|
+
def test_filter_record
|
36
|
+
d = create_driver(%[
|
37
|
+
key tai64n
|
38
|
+
add_tag_prefix parsed.
|
39
|
+
])
|
40
|
+
tag = 'test'
|
41
|
+
record = {'tai64n' => TAI64N_TIME}
|
42
|
+
d.instance.filter_record('test', Time.now, record)
|
43
|
+
|
44
|
+
assert_equal record['tai64n'], TAI64N_TIME
|
45
|
+
assert_equal record['parsed_time'], '2014-02-10 17:32:25.612979500'
|
46
|
+
end
|
47
|
+
|
48
|
+
def test_filter_record_bad_parameters
|
49
|
+
d = create_driver(%[
|
50
|
+
key tai64n
|
51
|
+
add_tag_prefix parsed.
|
52
|
+
])
|
53
|
+
tag = 'test'
|
54
|
+
record = {'tai64n' => BAD_TAI64N_TIME}
|
55
|
+
|
56
|
+
d.instance.filter_record('test', Time.now, record)
|
57
|
+
assert_equal record['parsed_time'], nil
|
58
|
+
|
59
|
+
record = {'tai64n' => "this is not a date"}
|
60
|
+
d.instance.filter_record('test', Time.now, record)
|
61
|
+
assert_equal record['parsed_time'], nil
|
62
|
+
end
|
63
|
+
|
64
|
+
def test_emit
|
65
|
+
d = create_driver(%[
|
66
|
+
key tai64n
|
67
|
+
add_tag_prefix parsed.
|
68
|
+
])
|
69
|
+
|
70
|
+
d.run { d.emit('tai64n' => TAI64N_TIME) }
|
71
|
+
emits = d.emits
|
72
|
+
|
73
|
+
assert_equal 1, emits.count
|
74
|
+
assert_equal 'parsed.test', emits[0][0]
|
75
|
+
assert_equal TAI64N_TIME, emits[0][2]['tai64n']
|
76
|
+
end
|
77
|
+
|
78
|
+
def test_emit_multi
|
79
|
+
d = create_driver(%[
|
80
|
+
key tai64n
|
81
|
+
add_tag_prefix parsed.
|
82
|
+
])
|
83
|
+
|
84
|
+
d.run do
|
85
|
+
d.emit('tai64n' => TAI64N_TIME)
|
86
|
+
d.emit('tai64n' => TAI64N_TIME)
|
87
|
+
d.emit('tai64n' => TAI64N_TIME)
|
88
|
+
end
|
89
|
+
emits = d.emits
|
90
|
+
|
91
|
+
assert_equal 3, emits.count
|
92
|
+
0.upto(2) do |i|
|
93
|
+
assert_equal 'parsed.test', emits[i][0]
|
94
|
+
assert_equal TAI64N_TIME, emits[i][2]['tai64n']
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
def test_emit_with_invalid_tai64n
|
99
|
+
d = create_driver(%[
|
100
|
+
key tai64n
|
101
|
+
add_tag_prefix parsed.
|
102
|
+
])
|
103
|
+
wrong_time = 'wrong time'
|
104
|
+
d.run { d.emit('tai64n' => wrong_time) }
|
105
|
+
emits = d.emits
|
106
|
+
|
107
|
+
assert_equal 1, emits.count
|
108
|
+
assert_equal 'parsed.test', emits[0][0]
|
109
|
+
assert_equal wrong_time, emits[0][2]['tai64n']
|
110
|
+
end
|
111
|
+
|
112
|
+
end
|
data/test/test_helper.rb
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'test/unit'
|
2
|
+
|
3
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
4
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
5
|
+
|
6
|
+
require 'fluent/test'
|
7
|
+
|
8
|
+
unless ENV.has_key?('VERBOSE')
|
9
|
+
nulllogger = Object.new
|
10
|
+
nulllogger.instance_eval {|obj|
|
11
|
+
def method_missing(method, *args)
|
12
|
+
# pass
|
13
|
+
end
|
14
|
+
}
|
15
|
+
$log = nulllogger
|
16
|
+
end
|
17
|
+
|
18
|
+
require 'fluent/plugin/out_tai64n_parser'
|
19
|
+
|
20
|
+
class Test::Unit::TestCase
|
21
|
+
end
|
metadata
ADDED
@@ -0,0 +1,111 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-tai64n_parser
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Akira Maeda
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2014-02-12 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: rake
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - '>='
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - '>='
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: fluentd
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - '>='
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - '>='
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: test-unit
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - '>='
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: fluentd
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - '>='
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0'
|
69
|
+
description: Fluentd plugin to parse the tai64n format log.
|
70
|
+
email:
|
71
|
+
- glidenote+github@gmail.com
|
72
|
+
executables: []
|
73
|
+
extensions: []
|
74
|
+
extra_rdoc_files: []
|
75
|
+
files:
|
76
|
+
- .gitignore
|
77
|
+
- .travis.yml
|
78
|
+
- Gemfile
|
79
|
+
- LICENSE
|
80
|
+
- README.md
|
81
|
+
- Rakefile
|
82
|
+
- fluent-plugin-tai64n_parser.gemspec
|
83
|
+
- lib/fluent/plugin/out_tai64n_parser.rb
|
84
|
+
- test/plugin/test_out_tai64n_parser.rb
|
85
|
+
- test/test_helper.rb
|
86
|
+
homepage: ''
|
87
|
+
licenses: []
|
88
|
+
metadata: {}
|
89
|
+
post_install_message:
|
90
|
+
rdoc_options: []
|
91
|
+
require_paths:
|
92
|
+
- lib
|
93
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
94
|
+
requirements:
|
95
|
+
- - '>='
|
96
|
+
- !ruby/object:Gem::Version
|
97
|
+
version: '0'
|
98
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
99
|
+
requirements:
|
100
|
+
- - '>='
|
101
|
+
- !ruby/object:Gem::Version
|
102
|
+
version: '0'
|
103
|
+
requirements: []
|
104
|
+
rubyforge_project:
|
105
|
+
rubygems_version: 2.0.14
|
106
|
+
signing_key:
|
107
|
+
specification_version: 4
|
108
|
+
summary: Fluentd plugin to parse the tai64n format log.
|
109
|
+
test_files:
|
110
|
+
- test/plugin/test_out_tai64n_parser.rb
|
111
|
+
- test/test_helper.rb
|