fluent-plugin-parser 0.1.2 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +13 -0
- data/fluent-plugin-parser.gemspec +1 -1
- data/lib/fluent/plugin/fixed_parser.rb +17 -4
- data/test/plugin/test_out_parser.rb +72 -0
- metadata +2 -2
data/README.md
CHANGED
@@ -52,6 +52,19 @@ Format 'json' is also supported:
|
|
52
52
|
key_name sales
|
53
53
|
</match>
|
54
54
|
|
55
|
+
Format 'ltsv'(Labeled-TSV (Tab separated values)) is also supported:
|
56
|
+
|
57
|
+
<match raw.sales.*>
|
58
|
+
type parser
|
59
|
+
tag sales
|
60
|
+
format ltsv
|
61
|
+
key_name sales
|
62
|
+
</match>
|
63
|
+
|
64
|
+
'LTSV' is format like below, unlinke json, easy to write with simple formatter (ex: LogFormat of apache):
|
65
|
+
|
66
|
+
KEY1:VALUE1 [TAB] KEY2:VALUE2 [TAB] ...
|
67
|
+
|
55
68
|
### DeparserOutput
|
56
69
|
|
57
70
|
To build CSV from field 'store','item','num', as field 'csv', without raw data:
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# -*- encoding: utf-8 -*-
|
2
2
|
Gem::Specification.new do |gem|
|
3
3
|
gem.name = "fluent-plugin-parser"
|
4
|
-
gem.version = "0.
|
4
|
+
gem.version = "0.2.0"
|
5
5
|
gem.authors = ["TAGOMORI Satoshi"]
|
6
6
|
gem.email = ["tagomoris@gmail.com"]
|
7
7
|
gem.description = %q{fluentd plugin to parse single field, or to combine log structure into single field}
|
@@ -47,15 +47,13 @@ class FluentExt::TextParser
|
|
47
47
|
end
|
48
48
|
end
|
49
49
|
|
50
|
-
class
|
50
|
+
class GenericParser
|
51
51
|
include Fluent::Configurable
|
52
52
|
|
53
53
|
config_param :time_key, :string, :default => 'time'
|
54
54
|
config_param :time_format, :string, :default => nil
|
55
55
|
|
56
|
-
def
|
57
|
-
record = Yajl.load(text)
|
58
|
-
|
56
|
+
def parse_time(record)
|
59
57
|
time = nil
|
60
58
|
|
61
59
|
if value = record.delete(@time_key)
|
@@ -67,15 +65,30 @@ class FluentExt::TextParser
|
|
67
65
|
end
|
68
66
|
|
69
67
|
return time, record
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
class JSONParser < GenericParser
|
72
|
+
def call(text)
|
73
|
+
record = Yajl.load(text)
|
74
|
+
return parse_time(record)
|
70
75
|
rescue Yajl::ParseError
|
71
76
|
return nil, nil
|
72
77
|
end
|
73
78
|
end
|
74
79
|
|
80
|
+
class LabeledTSVParser < GenericParser
|
81
|
+
def call(text)
|
82
|
+
record = Hash[text.split("\t").map{|p| p.split(":", 2)}]
|
83
|
+
parse_time(record)
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
75
87
|
TEMPLATES = {
|
76
88
|
'apache' => RegexpParser.new(/^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/, {'time_format'=>"%d/%b/%Y:%H:%M:%S %z"}),
|
77
89
|
'syslog' => RegexpParser.new(/^(?<time>[^ ]*\s*[^ ]* [^ ]*) (?<host>[^ ]*) (?<ident>[a-zA-Z0-9_\/\.\-]*)(?:\[(?<pid>[0-9]+)\])?[^\:]*\: *(?<message>.*)$/, {'time_format'=>"%b %d %H:%M:%S"}),
|
78
90
|
'json' => JSONParser.new,
|
91
|
+
'ltsv' => LabeledTSVParser.new,
|
79
92
|
}
|
80
93
|
|
81
94
|
def self.register_template(name, regexp_or_proc, time_format=nil)
|
@@ -51,6 +51,22 @@ class ParserOutputTest < Test::Unit::TestCase
|
|
51
51
|
key_name foo
|
52
52
|
]
|
53
53
|
}
|
54
|
+
assert_nothing_raised {
|
55
|
+
d = create_driver %[
|
56
|
+
remove_prefix foo.baz
|
57
|
+
add_prefix foo.bar
|
58
|
+
format json
|
59
|
+
key_name foo
|
60
|
+
]
|
61
|
+
}
|
62
|
+
assert_nothing_raised {
|
63
|
+
d = create_driver %[
|
64
|
+
remove_prefix foo.baz
|
65
|
+
add_prefix foo.bar
|
66
|
+
format ltsv
|
67
|
+
key_name foo
|
68
|
+
]
|
69
|
+
}
|
54
70
|
d = create_driver %[
|
55
71
|
tag foo.bar
|
56
72
|
key_name foo
|
@@ -189,4 +205,60 @@ class ParserOutputTest < Test::Unit::TestCase
|
|
189
205
|
assert_equal 'x', second[2]['xxx']
|
190
206
|
assert_equal 'y', second[2]['yyy']
|
191
207
|
end
|
208
|
+
|
209
|
+
CONFIG_LTSV = %[
|
210
|
+
remove_prefix foo.baz
|
211
|
+
add_prefix foo.bar
|
212
|
+
format ltsv
|
213
|
+
key_name data
|
214
|
+
]
|
215
|
+
def test_emit_ltsv
|
216
|
+
d = create_driver(CONFIG_LTSV, 'foo.baz.test')
|
217
|
+
time = Time.parse("2012-04-02 18:20:59").to_i
|
218
|
+
d.run do
|
219
|
+
d.emit({'data' => "xxx:first\tyyy:second", 'xxx' => 'x', 'yyy' => 'y'}, time)
|
220
|
+
d.emit({'data' => "xxx:first\tyyy:second2", 'xxx' => 'x', 'yyy' => 'y'}, time)
|
221
|
+
end
|
222
|
+
emits = d.emits
|
223
|
+
assert_equal 2, emits.length
|
224
|
+
|
225
|
+
first = emits[0]
|
226
|
+
assert_equal 'foo.bar.test', first[0]
|
227
|
+
assert_equal time, first[1]
|
228
|
+
assert_nil first[2]['data']
|
229
|
+
assert_equal 'first', first[2]['xxx']
|
230
|
+
assert_equal 'second', first[2]['yyy']
|
231
|
+
|
232
|
+
second = emits[1]
|
233
|
+
assert_equal 'foo.bar.test', second[0]
|
234
|
+
assert_equal time, second[1]
|
235
|
+
assert_nil first[2]['data']
|
236
|
+
assert_equal 'first', second[2]['xxx']
|
237
|
+
assert_equal 'second2', second[2]['yyy']
|
238
|
+
|
239
|
+
d = create_driver(CONFIG_LTSV + %[
|
240
|
+
reserve_data yes
|
241
|
+
], 'foo.baz.test')
|
242
|
+
time = Time.parse("2012-04-02 18:20:59").to_i
|
243
|
+
d.run do
|
244
|
+
d.emit({'data' => "xxx:first\tyyy:second", 'xxx' => 'x', 'yyy' => 'y'}, time)
|
245
|
+
d.emit({'data' => "xxx:first\tyyy:second2", 'xxx' => 'x', 'yyy' => 'y'}, time)
|
246
|
+
end
|
247
|
+
emits = d.emits
|
248
|
+
assert_equal 2, emits.length
|
249
|
+
|
250
|
+
first = emits[0]
|
251
|
+
assert_equal 'foo.bar.test', first[0]
|
252
|
+
assert_equal time, first[1]
|
253
|
+
assert_equal "xxx:first\tyyy:second", first[2]['data']
|
254
|
+
assert_equal 'first', first[2]['xxx']
|
255
|
+
assert_equal 'second', first[2]['yyy']
|
256
|
+
|
257
|
+
second = emits[1]
|
258
|
+
assert_equal 'foo.bar.test', second[0]
|
259
|
+
assert_equal time, second[1]
|
260
|
+
assert_equal "xxx:first\tyyy:second", first[2]['data']
|
261
|
+
assert_equal 'first', second[2]['xxx']
|
262
|
+
assert_equal 'second2', second[2]['yyy']
|
263
|
+
end
|
192
264
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-parser
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2013-02-06 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rake
|