fluent-plugin-tai64n_parser 0.0.1 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +52 -2
  3. data/fluent-plugin-tai64n_parser.gemspec +4 -4
  4. data/lib/fluent/plugin/out_tai64n_parser.rb +33 -18
  5. data/test/plugin/test_out_tai64n_parser.rb +26 -2
  6. metadata +5 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: d71dccb155099860504f8f0c7ed61a60afb462e6
4
- data.tar.gz: 8ebf64e76a9e2c03b48946d1acfb87843cf182bd
3
+ metadata.gz: 1f47b272961f4be0457ff8ef982c0f29fa396f74
4
+ data.tar.gz: 0863699cd861ef18b4b999681cfeb7383c100841
5
5
  SHA512:
6
- metadata.gz: b829d4ffb506122ab704986834efc918d5a55224dcf2fe11b46bd3de6c6c0c52d0b4fac25df7dfb0ff888556627756c52a7a21c3866c4ac9b83af8940510eff7
7
- data.tar.gz: 220286072d8ea0f22ed55b61fd83add6a15c25cc591b0cb0c079643e54e72ced3d9a6d1495226cfe39685495c10a119b87b253f2c6032b61a996aadc67fb4b9f
6
+ metadata.gz: 0225747aa31a264174f99c84e63372e37227a28a3a2863f09b2d34030497b3ca32baa703170b6487838a9eb688fbcf2aa2c793d1ba25c82c1e016e3469598def
7
+ data.tar.gz: d7506a97b2d672c990e9ea9701e831e9b20684199d286220eda0de7b8ae78c0e1e90f72d86720ffce5054bfcf7af115143feb8d87e40b0447197f6e5abe39a3c
data/README.md CHANGED
@@ -14,14 +14,14 @@ gem install fluent-plugin-tai64n_parser
14
14
 
15
15
  ## Configuration
16
16
 
17
- Exampe:
17
+ ### Basic Example:
18
18
 
19
19
  ```
20
20
  <match test.**>
21
21
  type tai64n_parser
22
22
 
23
23
  key tai64n
24
- parsed_time_tag parsed_time
24
+ output_key parsed_time
25
25
  add_tag_prefix parsed.
26
26
  </match>
27
27
  ```
@@ -43,6 +43,56 @@ then output becomes as below (indented):
43
43
  }
44
44
  ```
45
45
 
46
+ ### Parse qmail log example:
47
+
48
+ ```
49
+ <match raw.qmail.sent>
50
+ type parser
51
+ remove_prefix raw
52
+ format /^(?<tai64n>[^ ]+) (?<message>(((?:new|end) msg (?<key>[0-9]+)|info msg (?<key>[0-9]+): bytes (?:\d+) from <(?<address>[^>]*)> |starting delivery (?<delivery_id>[0-9]+): msg (?<key>[0-9]+) to (?:local|remote) (?<address>.+)|delivery (?<delivery_id>[0-9]+))?.*))$/
53
+ key_name message
54
+ suppress_parse_error_log true
55
+ </match>
56
+
57
+ <match qmail.sent>
58
+ type tai64n_parser
59
+
60
+ key tai64n
61
+ output_key parsed_time
62
+ add_tag_prefix parsed.
63
+ </match>
64
+
65
+ <match parsed.qmail.sent>
66
+ type file
67
+ path /var/log/td-agent/qmail_tai64n_parsed.log
68
+ </match>
69
+ ```
70
+
71
+ Assume following input is coming:
72
+
73
+ ```
74
+ @4000000052fafd8d3298434c new msg 3890
75
+ @4000000052fafd8d32984b1c info msg 3890: bytes 372 from <root@**********.pb> qp 31835 uid 0
76
+ @4000000052fafd8d373b5dbc starting delivery 9: msg 3890 to remote glidenote@********.co.jp
77
+ @4000000052fafd8d373b6974 status: local 0/120 remote 1/60
78
+ @4000000052fafd8d38754cec delivery 9: success: ***.***.***.***_accepted_message./Remote_host_said:_250_ok_1392180611_qp_10394/
79
+ @4000000052fafd8d387554bc status: local 0/120 remote 0/60
80
+ @4000000052fafd8d387554bc end msg 3890
81
+ ```
82
+
83
+ then output becomes as below:
84
+
85
+ ```
86
+ 2014-02-12T13:50:11+09:00 parsed.qmail.sent {"tai64n":"@4000000052fafd8d3298434c","message":"new msg 3890","key":"3890","parsed_time":"2014-02-12 13:50:11.848839500"}
87
+ 2014-02-12T13:50:11+09:00 parsed.qmail.sent {"tai64n":"@4000000052fafd8d32984b1c","message":"info msg 3890: bytes 372 from <root@**********.pb> qp 31835 uid 0","key":"3890","address":"root@**********.pb","parsed_time":"2014-02-12 13:50:11.848841500"}
88
+ 2014-02-12T13:50:11+09:00 parsed.qmail.sent {"tai64n":"@4000000052fafd8d373b5dbc","message":"starting delivery 9: msg 3890 to remote glidenote@**********.co.jp","key":"3890","address":"glidenote@**********.co.jp","delivery_id":"9","parsed_time":"2014-02-12 13:50:11.926637500"}
89
+ 2014-02-12T13:50:11+09:00 parsed.qmail.sent {"tai64n":"@4000000052fafd8d373b6974","message":"status: local 0/120 remote 1/60","parsed_time":"2014-02-12 13:50:11.926640500"}
90
+ 2014-02-12T13:50:11+09:00 parsed.qmail.sent {"tai64n":"@4000000052fafd8d38754cec","message":"delivery 9: success: ***.***.***.***_accepted_message./Remote_host_said:_250_ok_1392180611_qp_10394/","
91
+ delivery_id":"9","parsed_time":"2014-02-12 13:50:11.947211500"}
92
+ 2014-02-12T13:50:11+09:00 parsed.qmail.sent {"tai64n":"@4000000052fafd8d387554bc","message":"status: local 0/120 remote 0/60","parsed_time":"2014-02-12 13:50:11.947213500"}
93
+ 2014-02-12T13:50:11+09:00 parsed.qmail.sent {"tai64n":"@4000000052fafd8d387554bc","message":"end msg 3890","key":"3890","parsed_time":"2014-02-12 13:50:11.947213500"}
94
+ ```
95
+
46
96
  ## Contributing
47
97
 
48
98
  1. Fork it
data/fluent-plugin-tai64n_parser.gemspec CHANGED
@@ -1,9 +1,9 @@
1
1
  Gem::Specification.new do |gem|
2
2
  gem.name = 'fluent-plugin-tai64n_parser'
3
- gem.version = '0.0.1'
4
- gem.authors = ['Akira Maeda']
5
- gem.email = ['glidenote+github@gmail.com']
6
- gem.homepage = ''
3
+ gem.version = '0.1.0'
4
+ gem.authors = ['Akira Maeda','Naotoshi Seo']
5
+ gem.email = ['glidenote+github@gmail.com','sonots@gmail.com']
6
+ gem.homepage = 'https://github.com/glidenote/fluent-plugin-tai64n_parser'
7
7
  gem.description = %q{Fluentd plugin to parse the tai64n format log.}
8
8
  gem.summary = %q{Fluentd plugin to parse the tai64n format log.}
9
9
 
data/lib/fluent/plugin/out_tai64n_parser.rb CHANGED
@@ -3,8 +3,13 @@ module Fluent
3
3
  include Fluent::HandleTagNameMixin
4
4
  Fluent::Plugin.register_output('tai64n_parser', self)
5
5
 
6
+ # Define `log` method for v0.10.42 or earlier
7
+ unless method_defined?(:log)
8
+ define_method("log") { $log }
9
+ end
10
+
6
11
  config_param :key, :string, :default => 'tai64n'
7
- config_param :parsed_time_tag, :string, :default => 'parsed_time'
12
+ config_param :output_key, :string, :default => nil
8
13
 
9
14
  def configure(conf)
10
15
  super
@@ -16,6 +21,7 @@ module Fluent
16
21
  )
17
22
  raise ConfigError, "out_tai64n_parser: At least one of remove_tag_prefix/remove_tag_suffix/add_tag_prefix/add_tag_suffix is required to be set."
18
23
  end
24
+ @output_key ||= @key
19
25
  end
20
26
 
21
27
  def start
@@ -37,25 +43,34 @@ module Fluent
37
43
 
38
44
  def filter_record(tag, time, record)
39
45
  begin
40
- # @4000000052f88ea32489532c
41
- # 0123456789012345678901234
42
- # 0 1 2
43
- # |-------------||------|
44
- if record[key][0,2] == '@4' then
45
- ts = record[key][2,15].hex
46
- tf = record[key][17,8].hex
47
- t = Time.at(ts-10,tf/1000.0)
48
- record_time = t.strftime("%Y-%m-%d %X.%9N")
49
- else
50
- record_time = nil
51
- end
52
-
53
- record[parsed_time_tag] = record_time
54
-
55
- rescue ArgumentError => error
56
- $log.warn("out_tai64n_parser: #{error.message}")
46
+ record[output_key] = replace_tai64n(record[key])
47
+ rescue => error
48
+ log.warn("out_tai64n_parser: #{error.class} #{error.message} #{error.backtrace.first}")
57
49
  end
58
50
  super(tag, time, record)
59
51
  end
52
+
53
+ def replace_tai64n(str)
54
+ tai64n, rest = str[0,25], str[25..-1]
55
+ parsed = parse_tai64n(tai64n)
56
+ if parsed
57
+ "#{parsed}#{rest}"
58
+ else
59
+ log.info("out_tai64n_parser: record['#{key}']='#{str}' does not start with valid tai64n")
60
+ str
61
+ end
62
+ end
63
+
64
+ def parse_tai64n(tai64n)
65
+ # @4000000052f88ea32489532c
66
+ # 0123456789012345678901234
67
+ # 0 1 2
68
+ # |-------------||------|
69
+ return nil unless tai64n[0,2] == '@4'
70
+ ts = tai64n[2,15].hex
71
+ tf = tai64n[17,8].hex
72
+ t = Time.at(ts-10,tf/1000.0)
73
+ t.strftime("%Y-%m-%d %X.%9N")
74
+ end
60
75
  end
61
76
  end
data/test/plugin/test_out_tai64n_parser.rb CHANGED
@@ -4,6 +4,7 @@ require 'test_helper'
4
4
  class Tai64nParserOutputTest < Test::Unit::TestCase
5
5
 
6
6
  TAI64N_TIME = "@4000000052f88ea32489532c"
7
+ PARSED_TIME = "2014-02-10 17:32:25.612979500"
7
8
  BAD_TAI64N_TIME = "4000000052f88ea32489"
8
9
 
9
10
  def setup
@@ -19,9 +20,11 @@ class Tai64nParserOutputTest < Test::Unit::TestCase
19
20
  def test_configure
20
21
  d = create_driver(%[
21
22
  key test
23
+ output_key parsed_time
22
24
  add_tag_prefix parsed.
23
25
  ])
24
26
  assert_equal 'test', d.instance.key
27
+ assert_equal 'parsed_time', d.instance.output_key
25
28
  assert_equal 'parsed.', d.instance.add_tag_prefix
26
29
 
27
30
  #Default Key
@@ -29,12 +32,14 @@ class Tai64nParserOutputTest < Test::Unit::TestCase
29
32
  add_tag_prefix parsed.
30
33
  ])
31
34
  assert_equal 'tai64n', d.instance.key
35
+ assert_equal 'tai64n', d.instance.output_key
32
36
  assert_equal 'parsed.', d.instance.add_tag_prefix
33
37
  end
34
38
 
35
39
  def test_filter_record
36
40
  d = create_driver(%[
37
41
  key tai64n
42
+ output_key parsed_time
38
43
  add_tag_prefix parsed.
39
44
  ])
40
45
  tag = 'test'
@@ -48,22 +53,24 @@ class Tai64nParserOutputTest < Test::Unit::TestCase
48
53
  def test_filter_record_bad_parameters
49
54
  d = create_driver(%[
50
55
  key tai64n
56
+ output_key parsed_time
51
57
  add_tag_prefix parsed.
52
58
  ])
53
59
  tag = 'test'
54
60
  record = {'tai64n' => BAD_TAI64N_TIME}
55
61
 
56
62
  d.instance.filter_record('test', Time.now, record)
57
- assert_equal record['parsed_time'], nil
63
+ assert_equal record['parsed_time'], BAD_TAI64N_TIME
58
64
 
59
65
  record = {'tai64n' => "this is not a date"}
60
66
  d.instance.filter_record('test', Time.now, record)
61
- assert_equal record['parsed_time'], nil
67
+ assert_equal record['parsed_time'], "this is not a date"
62
68
  end
63
69
 
64
70
  def test_emit
65
71
  d = create_driver(%[
66
72
  key tai64n
73
+ output_key parsed_time
67
74
  add_tag_prefix parsed.
68
75
  ])
69
76
 
@@ -78,6 +85,7 @@ class Tai64nParserOutputTest < Test::Unit::TestCase
78
85
  def test_emit_multi
79
86
  d = create_driver(%[
80
87
  key tai64n
88
+ output_key parsed_time
81
89
  add_tag_prefix parsed.
82
90
  ])
83
91
 
@@ -98,6 +106,7 @@ class Tai64nParserOutputTest < Test::Unit::TestCase
98
106
  def test_emit_with_invalid_tai64n
99
107
  d = create_driver(%[
100
108
  key tai64n
109
+ output_key parsed_time
101
110
  add_tag_prefix parsed.
102
111
  ])
103
112
  wrong_time = 'wrong time'
@@ -109,4 +118,19 @@ class Tai64nParserOutputTest < Test::Unit::TestCase
109
118
  assert_equal wrong_time, emits[0][2]['tai64n']
110
119
  end
111
120
 
121
+ def test_emit_with_replace
122
+ d = create_driver(%[
123
+ key message
124
+ parsed_time_tag message
125
+ add_tag_prefix parsed.
126
+ ])
127
+
128
+ d.run { d.emit('message' => "#{TAI64N_TIME}foobar") }
129
+ emits = d.emits
130
+
131
+ assert_equal 1, emits.count
132
+ assert_equal 'parsed.test', emits[0][0]
133
+ assert_equal "#{PARSED_TIME}foobar", emits[0][2]['message']
134
+ end
135
+
112
136
  end
metadata CHANGED
@@ -1,14 +1,15 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-tai64n_parser
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.1
4
+ version: 0.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Akira Maeda
8
+ - Naotoshi Seo
8
9
  autorequire:
9
10
  bindir: bin
10
11
  cert_chain: []
11
- date: 2014-02-12 00:00:00.000000000 Z
12
+ date: 2014-02-19 00:00:00.000000000 Z
12
13
  dependencies:
13
14
  - !ruby/object:Gem::Dependency
14
15
  name: rake
@@ -69,6 +70,7 @@ dependencies:
69
70
  description: Fluentd plugin to parse the tai64n format log.
70
71
  email:
71
72
  - glidenote+github@gmail.com
73
+ - sonots@gmail.com
72
74
  executables: []
73
75
  extensions: []
74
76
  extra_rdoc_files: []
@@ -83,7 +85,7 @@ files:
83
85
  - lib/fluent/plugin/out_tai64n_parser.rb
84
86
  - test/plugin/test_out_tai64n_parser.rb
85
87
  - test/test_helper.rb
86
- homepage: ''
88
+ homepage: https://github.com/glidenote/fluent-plugin-tai64n_parser
87
89
  licenses: []
88
90
  metadata: {}
89
91
  post_install_message: