fluent-plugin-tai64n_parser 0.0.1 → 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (6) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +52 -2
  3. data/fluent-plugin-tai64n_parser.gemspec +4 -4
  4. data/lib/fluent/plugin/out_tai64n_parser.rb +33 -18
  5. data/test/plugin/test_out_tai64n_parser.rb +26 -2
  6. metadata +5 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: d71dccb155099860504f8f0c7ed61a60afb462e6
4
- data.tar.gz: 8ebf64e76a9e2c03b48946d1acfb87843cf182bd
3
+ metadata.gz: 1f47b272961f4be0457ff8ef982c0f29fa396f74
4
+ data.tar.gz: 0863699cd861ef18b4b999681cfeb7383c100841
5
5
  SHA512:
6
- metadata.gz: b829d4ffb506122ab704986834efc918d5a55224dcf2fe11b46bd3de6c6c0c52d0b4fac25df7dfb0ff888556627756c52a7a21c3866c4ac9b83af8940510eff7
7
- data.tar.gz: 220286072d8ea0f22ed55b61fd83add6a15c25cc591b0cb0c079643e54e72ced3d9a6d1495226cfe39685495c10a119b87b253f2c6032b61a996aadc67fb4b9f
6
+ metadata.gz: 0225747aa31a264174f99c84e63372e37227a28a3a2863f09b2d34030497b3ca32baa703170b6487838a9eb688fbcf2aa2c793d1ba25c82c1e016e3469598def
7
+ data.tar.gz: d7506a97b2d672c990e9ea9701e831e9b20684199d286220eda0de7b8ae78c0e1e90f72d86720ffce5054bfcf7af115143feb8d87e40b0447197f6e5abe39a3c
data/README.md CHANGED
@@ -14,14 +14,14 @@ gem install fluent-plugin-tai64n_parser
14
14
 
15
15
  ## Configuration
16
16
 
17
- Exampe:
17
+ ### Basic Example:
18
18
 
19
19
  ```
20
20
  <match test.**>
21
21
  type tai64n_parser
22
22
 
23
23
  key tai64n
24
- parsed_time_tag parsed_time
24
+ output_key parsed_time
25
25
  add_tag_prefix parsed.
26
26
  </match>
27
27
  ```
@@ -43,6 +43,56 @@ then output becomes as below (indented):
43
43
  }
44
44
  ```
45
45
 
46
+ ### Parse qmail log example:
47
+
48
+ ```
49
+ <match raw.qmail.sent>
50
+ type parser
51
+ remove_prefix raw
52
+ format /^(?<tai64n>[^ ]+) (?<message>(((?:new|end) msg (?<key>[0-9]+)|info msg (?<key>[0-9]+): bytes (?:\d+) from <(?<address>[^>]*)> |starting delivery (?<delivery_id>[0-9]+): msg (?<key>[0-9]+) to (?:local|remote) (?<address>.+)|delivery (?<delivery_id>[0-9]+))?.*))$/
53
+ key_name message
54
+ suppress_parse_error_log true
55
+ </match>
56
+
57
+ <match qmail.sent>
58
+ type tai64n_parser
59
+
60
+ key tai64n
61
+ output_key parsed_time
62
+ add_tag_prefix parsed.
63
+ </match>
64
+
65
+ <match parsed.qmail.sent>
66
+ type file
67
+ path /var/log/td-agent/qmail_tai64n_parsed.log
68
+ </match>
69
+ ```
70
+
71
+ Assume following input is coming:
72
+
73
+ ```
74
+ @4000000052fafd8d3298434c new msg 3890
75
+ @4000000052fafd8d32984b1c info msg 3890: bytes 372 from <root@**********.pb> qp 31835 uid 0
76
+ @4000000052fafd8d373b5dbc starting delivery 9: msg 3890 to remote glidenote@********.co.jp
77
+ @4000000052fafd8d373b6974 status: local 0/120 remote 1/60
78
+ @4000000052fafd8d38754cec delivery 9: success: ***.***.***.***_accepted_message./Remote_host_said:_250_ok_1392180611_qp_10394/
79
+ @4000000052fafd8d387554bc status: local 0/120 remote 0/60
80
+ @4000000052fafd8d387554bc end msg 3890
81
+ ```
82
+
83
+ then output becomes as below:
84
+
85
+ ```
86
+ 2014-02-12T13:50:11+09:00 parsed.qmail.sent {"tai64n":"@4000000052fafd8d3298434c","message":"new msg 3890","key":"3890","parsed_time":"2014-02-12 13:50:11.848839500"}
87
+ 2014-02-12T13:50:11+09:00 parsed.qmail.sent {"tai64n":"@4000000052fafd8d32984b1c","message":"info msg 3890: bytes 372 from <root@**********.pb> qp 31835 uid 0","key":"3890","address":"root@**********.pb","parsed_time":"2014-02-12 13:50:11.848841500"}
88
+ 2014-02-12T13:50:11+09:00 parsed.qmail.sent {"tai64n":"@4000000052fafd8d373b5dbc","message":"starting delivery 9: msg 3890 to remote glidenote@**********.co.jp","key":"3890","address":"glidenote@**********.co.jp","delivery_id":"9","parsed_time":"2014-02-12 13:50:11.926637500"}
89
+ 2014-02-12T13:50:11+09:00 parsed.qmail.sent {"tai64n":"@4000000052fafd8d373b6974","message":"status: local 0/120 remote 1/60","parsed_time":"2014-02-12 13:50:11.926640500"}
90
+ 2014-02-12T13:50:11+09:00 parsed.qmail.sent {"tai64n":"@4000000052fafd8d38754cec","message":"delivery 9: success: ***.***.***.***_accepted_message./Remote_host_said:_250_ok_1392180611_qp_10394/","
91
+ delivery_id":"9","parsed_time":"2014-02-12 13:50:11.947211500"}
92
+ 2014-02-12T13:50:11+09:00 parsed.qmail.sent {"tai64n":"@4000000052fafd8d387554bc","message":"status: local 0/120 remote 0/60","parsed_time":"2014-02-12 13:50:11.947213500"}
93
+ 2014-02-12T13:50:11+09:00 parsed.qmail.sent {"tai64n":"@4000000052fafd8d387554bc","message":"end msg 3890","key":"3890","parsed_time":"2014-02-12 13:50:11.947213500"}
94
+ ```
95
+
46
96
  ## Contributing
47
97
 
48
98
  1. Fork it
data/fluent-plugin-tai64n_parser.gemspec CHANGED
@@ -1,9 +1,9 @@
1
1
  Gem::Specification.new do |gem|
2
2
  gem.name = 'fluent-plugin-tai64n_parser'
3
- gem.version = '0.0.1'
4
- gem.authors = ['Akira Maeda']
5
- gem.email = ['glidenote+github@gmail.com']
6
- gem.homepage = ''
3
+ gem.version = '0.1.0'
4
+ gem.authors = ['Akira Maeda','Naotoshi Seo']
5
+ gem.email = ['glidenote+github@gmail.com','sonots@gmail.com']
6
+ gem.homepage = 'https://github.com/glidenote/fluent-plugin-tai64n_parser'
7
7
  gem.description = %q{Fluentd plugin to parse the tai64n format log.}
8
8
  gem.summary = %q{Fluentd plugin to parse the tai64n format log.}
9
9
 
data/lib/fluent/plugin/out_tai64n_parser.rb CHANGED
@@ -3,8 +3,13 @@ module Fluent
3
3
  include Fluent::HandleTagNameMixin
4
4
  Fluent::Plugin.register_output('tai64n_parser', self)
5
5
 
6
+ # Define `log` method for v0.10.42 or earlier
7
+ unless method_defined?(:log)
8
+ define_method("log") { $log }
9
+ end
10
+
6
11
  config_param :key, :string, :default => 'tai64n'
7
- config_param :parsed_time_tag, :string, :default => 'parsed_time'
12
+ config_param :output_key, :string, :default => nil
8
13
 
9
14
  def configure(conf)
10
15
  super
@@ -16,6 +21,7 @@ module Fluent
16
21
  )
17
22
  raise ConfigError, "out_tai64n_parser: At least one of remove_tag_prefix/remove_tag_suffix/add_tag_prefix/add_tag_suffix is required to be set."
18
23
  end
24
+ @output_key ||= @key
19
25
  end
20
26
 
21
27
  def start
@@ -37,25 +43,34 @@ module Fluent
37
43
 
38
44
  def filter_record(tag, time, record)
39
45
  begin
40
- # @4000000052f88ea32489532c
41
- # 0123456789012345678901234
42
- # 0 1 2
43
- # |-------------||------|
44
- if record[key][0,2] == '@4' then
45
- ts = record[key][2,15].hex
46
- tf = record[key][17,8].hex
47
- t = Time.at(ts-10,tf/1000.0)
48
- record_time = t.strftime("%Y-%m-%d %X.%9N")
49
- else
50
- record_time = nil
51
- end
52
-
53
- record[parsed_time_tag] = record_time
54
-
55
- rescue ArgumentError => error
56
- $log.warn("out_tai64n_parser: #{error.message}")
46
+ record[output_key] = replace_tai64n(record[key])
47
+ rescue => error
48
+ log.warn("out_tai64n_parser: #{error.class} #{error.message} #{error.backtrace.first}")
57
49
  end
58
50
  super(tag, time, record)
59
51
  end
52
+
53
+ def replace_tai64n(str)
54
+ tai64n, rest = str[0,25], str[25..-1]
55
+ parsed = parse_tai64n(tai64n)
56
+ if parsed
57
+ "#{parsed}#{rest}"
58
+ else
59
+ log.info("out_tai64n_parser: record['#{key}']='#{str}' does not start with valid tai64n")
60
+ str
61
+ end
62
+ end
63
+
64
+ def parse_tai64n(tai64n)
65
+ # @4000000052f88ea32489532c
66
+ # 0123456789012345678901234
67
+ # 0 1 2
68
+ # |-------------||------|
69
+ return nil unless tai64n[0,2] == '@4'
70
+ ts = tai64n[2,15].hex
71
+ tf = tai64n[17,8].hex
72
+ t = Time.at(ts-10,tf/1000.0)
73
+ t.strftime("%Y-%m-%d %X.%9N")
74
+ end
60
75
  end
61
76
  end
data/test/plugin/test_out_tai64n_parser.rb CHANGED
@@ -4,6 +4,7 @@ require 'test_helper'
4
4
  class Tai64nParserOutputTest < Test::Unit::TestCase
5
5
 
6
6
  TAI64N_TIME = "@4000000052f88ea32489532c"
7
+ PARSED_TIME = "2014-02-10 17:32:25.612979500"
7
8
  BAD_TAI64N_TIME = "4000000052f88ea32489"
8
9
 
9
10
  def setup
@@ -19,9 +20,11 @@ class Tai64nParserOutputTest < Test::Unit::TestCase
19
20
  def test_configure
20
21
  d = create_driver(%[
21
22
  key test
23
+ output_key parsed_time
22
24
  add_tag_prefix parsed.
23
25
  ])
24
26
  assert_equal 'test', d.instance.key
27
+ assert_equal 'parsed_time', d.instance.output_key
25
28
  assert_equal 'parsed.', d.instance.add_tag_prefix
26
29
 
27
30
  #Default Key
@@ -29,12 +32,14 @@ class Tai64nParserOutputTest < Test::Unit::TestCase
29
32
  add_tag_prefix parsed.
30
33
  ])
31
34
  assert_equal 'tai64n', d.instance.key
35
+ assert_equal 'tai64n', d.instance.output_key
32
36
  assert_equal 'parsed.', d.instance.add_tag_prefix
33
37
  end
34
38
 
35
39
  def test_filter_record
36
40
  d = create_driver(%[
37
41
  key tai64n
42
+ output_key parsed_time
38
43
  add_tag_prefix parsed.
39
44
  ])
40
45
  tag = 'test'
@@ -48,22 +53,24 @@ class Tai64nParserOutputTest < Test::Unit::TestCase
48
53
  def test_filter_record_bad_parameters
49
54
  d = create_driver(%[
50
55
  key tai64n
56
+ output_key parsed_time
51
57
  add_tag_prefix parsed.
52
58
  ])
53
59
  tag = 'test'
54
60
  record = {'tai64n' => BAD_TAI64N_TIME}
55
61
 
56
62
  d.instance.filter_record('test', Time.now, record)
57
- assert_equal record['parsed_time'], nil
63
+ assert_equal record['parsed_time'], BAD_TAI64N_TIME
58
64
 
59
65
  record = {'tai64n' => "this is not a date"}
60
66
  d.instance.filter_record('test', Time.now, record)
61
- assert_equal record['parsed_time'], nil
67
+ assert_equal record['parsed_time'], "this is not a date"
62
68
  end
63
69
 
64
70
  def test_emit
65
71
  d = create_driver(%[
66
72
  key tai64n
73
+ output_key parsed_time
67
74
  add_tag_prefix parsed.
68
75
  ])
69
76
 
@@ -78,6 +85,7 @@ class Tai64nParserOutputTest < Test::Unit::TestCase
78
85
  def test_emit_multi
79
86
  d = create_driver(%[
80
87
  key tai64n
88
+ output_key parsed_time
81
89
  add_tag_prefix parsed.
82
90
  ])
83
91
 
@@ -98,6 +106,7 @@ class Tai64nParserOutputTest < Test::Unit::TestCase
98
106
  def test_emit_with_invalid_tai64n
99
107
  d = create_driver(%[
100
108
  key tai64n
109
+ output_key parsed_time
101
110
  add_tag_prefix parsed.
102
111
  ])
103
112
  wrong_time = 'wrong time'
@@ -109,4 +118,19 @@ class Tai64nParserOutputTest < Test::Unit::TestCase
109
118
  assert_equal wrong_time, emits[0][2]['tai64n']
110
119
  end
111
120
 
121
+ def test_emit_with_replace
122
+ d = create_driver(%[
123
+ key message
124
+ parsed_time_tag message
125
+ add_tag_prefix parsed.
126
+ ])
127
+
128
+ d.run { d.emit('message' => "#{TAI64N_TIME}foobar") }
129
+ emits = d.emits
130
+
131
+ assert_equal 1, emits.count
132
+ assert_equal 'parsed.test', emits[0][0]
133
+ assert_equal "#{PARSED_TIME}foobar", emits[0][2]['message']
134
+ end
135
+
112
136
  end
metadata CHANGED
@@ -1,14 +1,15 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-tai64n_parser
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.1
4
+ version: 0.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Akira Maeda
8
+ - Naotoshi Seo
8
9
  autorequire:
9
10
  bindir: bin
10
11
  cert_chain: []
11
- date: 2014-02-12 00:00:00.000000000 Z
12
+ date: 2014-02-19 00:00:00.000000000 Z
12
13
  dependencies:
13
14
  - !ruby/object:Gem::Dependency
14
15
  name: rake
@@ -69,6 +70,7 @@ dependencies:
69
70
  description: Fluentd plugin to parse the tai64n format log.
70
71
  email:
71
72
  - glidenote+github@gmail.com
73
+ - sonots@gmail.com
72
74
  executables: []
73
75
  extensions: []
74
76
  extra_rdoc_files: []
@@ -83,7 +85,7 @@ files:
83
85
  - lib/fluent/plugin/out_tai64n_parser.rb
84
86
  - test/plugin/test_out_tai64n_parser.rb
85
87
  - test/test_helper.rb
86
- homepage: ''
88
+ homepage: https://github.com/glidenote/fluent-plugin-tai64n_parser
87
89
  licenses: []
88
90
  metadata: {}
89
91
  post_install_message: