fluent-plugin-parser 0.2.3 → 0.2.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/README.md +13 -1
- data/fluent-plugin-parser.gemspec +1 -1
- data/lib/fluent/plugin/fixed_parser.rb +9 -9
- data/lib/fluent/plugin/out_parser.rb +11 -3
- data/test/plugin/test_out_parser.rb +35 -6
- metadata +12 -17
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 2e0393ad371a16008916ab9e6bf614d56be0a6a7
|
4
|
+
data.tar.gz: 98efdde5b556caa2452769a66582beba39bce9de
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 3cb147341062d8f32d9e3eeaee6971bbf1d539994148b9eb63260c2aafc77c0a684ceb042f1a01c6c8abea3ecd1f18bf366108f92b7d2e796d98c5728df4dcc0
|
7
|
+
data.tar.gz: d4c5308d819343ca895f62949c38d6e2fd35bf4b63ae78c42aa1aba4da9513ac293e14242e57abc4722b14475c8bb4d93f8b4a302ebc8d046db9e4e98d820d62
|
data/README.md
CHANGED
@@ -67,7 +67,7 @@ Format 'ltsv'(Labeled-TSV (Tab separated values)) is also supported:
|
|
67
67
|
|
68
68
|
About LTSV, see: http://ltsv.org/
|
69
69
|
|
70
|
-
If you want to suppress 'pattern not match' log, specify '
|
70
|
+
If you want to suppress 'pattern not match' log, specify 'suppress\_parse\_error\_log true' to configuration.
|
71
71
|
default value is false.
|
72
72
|
|
73
73
|
<match in.hogelog>
|
@@ -78,6 +78,18 @@ default value is false.
|
|
78
78
|
suppress_parse_error_log true
|
79
79
|
</match>
|
80
80
|
|
81
|
+
To store parsed values with specified key name prefix, use `inject_key_prefix` option:
|
82
|
+
|
83
|
+
<match raw.sales.*>
|
84
|
+
type parser
|
85
|
+
tag sales
|
86
|
+
format json
|
87
|
+
key_name sales
|
88
|
+
reserve_data yes
|
89
|
+
inject_key_prefix sales.
|
90
|
+
</match>
|
91
|
+
# input string of 'sales': {"user":1,"num":2}
|
92
|
+
# output data: {"sales":"{\"user\":1,\"num\":2}","sales.user":1, "sales.num":2}
|
81
93
|
|
82
94
|
### DeparserOutput
|
83
95
|
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# -*- encoding: utf-8 -*-
|
2
2
|
Gem::Specification.new do |gem|
|
3
3
|
gem.name = "fluent-plugin-parser"
|
4
|
-
gem.version = "0.2.
|
4
|
+
gem.version = "0.2.4"
|
5
5
|
gem.authors = ["TAGOMORI Satoshi"]
|
6
6
|
gem.email = ["tagomoris@gmail.com"]
|
7
7
|
gem.description = %q{fluentd plugin to parse single field, or to combine log structure into single field}
|
@@ -141,25 +141,25 @@ class FluentExt::TextParser
|
|
141
141
|
|
142
142
|
user = m['user']
|
143
143
|
user = (user == '-') ? nil : user
|
144
|
-
|
144
|
+
|
145
145
|
time = m['time']
|
146
146
|
time = Time.strptime(time, "%d/%b/%Y:%H:%M:%S %z").to_i
|
147
|
-
|
147
|
+
|
148
148
|
method = m['method']
|
149
149
|
path = m['path']
|
150
|
-
|
151
|
-
code = m['code'].to_i
|
150
|
+
|
151
|
+
code = m['code'].to_i
|
152
152
|
code = nil if code == 0
|
153
153
|
|
154
154
|
size = m['size']
|
155
155
|
size = (size == '-') ? nil : size.to_i
|
156
|
-
|
156
|
+
|
157
157
|
referer = m['referer']
|
158
158
|
referer = (referer == '-') ? nil : referer
|
159
|
-
|
159
|
+
|
160
160
|
agent = m['agent']
|
161
161
|
agent = (agent == '-') ? nil : agent
|
162
|
-
|
162
|
+
|
163
163
|
record = {
|
164
164
|
"host" => host,
|
165
165
|
"user" => user,
|
@@ -169,7 +169,7 @@ class FluentExt::TextParser
|
|
169
169
|
"size" => size,
|
170
170
|
"referer" => referer,
|
171
171
|
"agent" => agent,
|
172
|
-
}
|
172
|
+
}
|
173
173
|
|
174
174
|
return time, record
|
175
175
|
end
|
@@ -187,7 +187,7 @@ class FluentExt::TextParser
|
|
187
187
|
}
|
188
188
|
|
189
189
|
def self.register_template(name, regexp_or_proc, time_format=nil)
|
190
|
-
|
190
|
+
|
191
191
|
factory = if regexp_or_proc.is_a?(Regexp)
|
192
192
|
regexp = regexp_or_proc
|
193
193
|
Proc.new { RegexpParser.new(regexp, {'time_format'=>time_format}) }
|
@@ -8,6 +8,7 @@ class Fluent::ParserOutput < Fluent::Output
|
|
8
8
|
config_param :add_prefix, :string, :default => nil
|
9
9
|
config_param :key_name, :string
|
10
10
|
config_param :reserve_data, :bool, :default => false
|
11
|
+
config_param :inject_key_prefix, :string, :default => nil
|
11
12
|
config_param :replace_invalid_sequence, :bool, :default => false
|
12
13
|
|
13
14
|
def initialize
|
@@ -43,8 +44,8 @@ class Fluent::ParserOutput < Fluent::Output
|
|
43
44
|
if @remove_prefix and
|
44
45
|
( (tag.start_with?(@removed_prefix_string) and tag.length > @removed_length) or tag == @remove_prefix)
|
45
46
|
tag = tag[@removed_length..-1]
|
46
|
-
end
|
47
|
-
if @add_prefix
|
47
|
+
end
|
48
|
+
if @add_prefix
|
48
49
|
tag = if tag and tag.length > 0
|
49
50
|
@added_prefix_string + tag
|
50
51
|
else
|
@@ -63,6 +64,9 @@ class Fluent::ParserOutput < Fluent::Output
|
|
63
64
|
end
|
64
65
|
t ||= time
|
65
66
|
r = if values
|
67
|
+
if @inject_key_prefix
|
68
|
+
values = Hash[values.map{|k,v| [ @inject_key_prefix + k, v ]}]
|
69
|
+
end
|
66
70
|
record.merge(values)
|
67
71
|
else
|
68
72
|
record
|
@@ -73,7 +77,11 @@ class Fluent::ParserOutput < Fluent::Output
|
|
73
77
|
es.each {|time,record|
|
74
78
|
value = record[@key_name]
|
75
79
|
t,values = if value
|
76
|
-
parse(value)
|
80
|
+
parsed = parse(value)
|
81
|
+
if @inject_key_prefix
|
82
|
+
parsed = Hash[parsed.map{|k,v| [ @inject_key_prefix + k, v ]}]
|
83
|
+
end
|
84
|
+
parsed
|
77
85
|
else
|
78
86
|
[nil, nil]
|
79
87
|
end
|
@@ -4,7 +4,7 @@ class ParserOutputTest < Test::Unit::TestCase
|
|
4
4
|
def setup
|
5
5
|
Fluent::Test.setup
|
6
6
|
end
|
7
|
-
|
7
|
+
|
8
8
|
CONFIG = %[
|
9
9
|
remove_prefix test
|
10
10
|
add_prefix parsed
|
@@ -292,7 +292,7 @@ class ParserOutputTest < Test::Unit::TestCase
|
|
292
292
|
key_name data
|
293
293
|
keys key1,key2,key3
|
294
294
|
]
|
295
|
-
def
|
295
|
+
def test_emit_tsv
|
296
296
|
d = create_driver(CONFIG_TSV, 'foo.baz.test')
|
297
297
|
time = Time.parse("2012-04-02 18:20:59").to_i
|
298
298
|
d.run do
|
@@ -317,7 +317,7 @@ class ParserOutputTest < Test::Unit::TestCase
|
|
317
317
|
key_name data
|
318
318
|
keys key1,key2,key3
|
319
319
|
]
|
320
|
-
def
|
320
|
+
def test_emit_csv
|
321
321
|
d = create_driver(CONFIG_CSV, 'foo.baz.test')
|
322
322
|
time = Time.parse("2012-04-02 18:20:59").to_i
|
323
323
|
d.run do
|
@@ -335,6 +335,35 @@ class ParserOutputTest < Test::Unit::TestCase
|
|
335
335
|
assert_equal 'value"ThreeYes!', first[2]['key3']
|
336
336
|
end
|
337
337
|
|
338
|
+
CONFIG_KEY_PREFIX = %[
|
339
|
+
remove_prefix foo.baz
|
340
|
+
add_prefix foo.bar
|
341
|
+
format json
|
342
|
+
key_name data
|
343
|
+
reserve_data yes
|
344
|
+
inject_key_prefix data.
|
345
|
+
]
|
346
|
+
def test_inject_key_prefix
|
347
|
+
d = create_driver(CONFIG_KEY_PREFIX, 'foo.baz.test')
|
348
|
+
time = Time.parse("2012-04-02 18:20:59").to_i
|
349
|
+
d.run do
|
350
|
+
d.emit({'data' => '{"xxx":"first","yyy":"second"}', 'xxx' => 'x', 'yyy' => 'y'}, time)
|
351
|
+
end
|
352
|
+
emits = d.emits
|
353
|
+
assert_equal 1, emits.length
|
354
|
+
|
355
|
+
first = emits[0]
|
356
|
+
assert_equal 'foo.bar.test', first[0]
|
357
|
+
assert_equal time, first[1]
|
358
|
+
|
359
|
+
assert_equal '{"xxx":"first","yyy":"second"}', first[2]['data']
|
360
|
+
assert_equal 'x', first[2]['xxx']
|
361
|
+
assert_equal 'y', first[2]['yyy']
|
362
|
+
assert_equal 'first', first[2]['data.xxx']
|
363
|
+
assert_equal 'second', first[2]['data.yyy']
|
364
|
+
assert_equal 5, first[2].keys.size
|
365
|
+
end
|
366
|
+
|
338
367
|
#TODO: apache2
|
339
368
|
# REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
|
340
369
|
|
@@ -393,18 +422,18 @@ class ParserOutputTest < Test::Unit::TestCase
|
|
393
422
|
assert_equal '?'.force_encoding('US-ASCII'), emits[0][2]['message']
|
394
423
|
end
|
395
424
|
|
396
|
-
# suppress_parse_error_log test
|
425
|
+
# suppress_parse_error_log test
|
397
426
|
CONFIG_DISABELED_SUPPRESS_PARSE_ERROR_LOG = %[
|
398
427
|
tag hogelog
|
399
428
|
format /^col1=(?<col1>.+) col2=(?<col2>.+)$/
|
400
429
|
key_name message
|
401
|
-
suppress_parse_error_log false
|
430
|
+
suppress_parse_error_log false
|
402
431
|
]
|
403
432
|
CONFIG_ENABELED_SUPPRESS_PARSE_ERROR_LOG = %[
|
404
433
|
tag hogelog
|
405
434
|
format /^col1=(?<col1>.+) col2=(?<col2>.+)$/
|
406
435
|
key_name message
|
407
|
-
suppress_parse_error_log true
|
436
|
+
suppress_parse_error_log true
|
408
437
|
]
|
409
438
|
CONFIG_DEFAULT_SUPPRESS_PARSE_ERROR_LOG = %[
|
410
439
|
tag hogelog
|
metadata
CHANGED
@@ -1,46 +1,41 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-parser
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
5
|
-
prerelease:
|
4
|
+
version: 0.2.4
|
6
5
|
platform: ruby
|
7
6
|
authors:
|
8
7
|
- TAGOMORI Satoshi
|
9
8
|
autorequire:
|
10
9
|
bindir: bin
|
11
10
|
cert_chain: []
|
12
|
-
date: 2013-
|
11
|
+
date: 2013-08-01 00:00:00.000000000 Z
|
13
12
|
dependencies:
|
14
13
|
- !ruby/object:Gem::Dependency
|
15
14
|
name: rake
|
16
15
|
requirement: !ruby/object:Gem::Requirement
|
17
|
-
none: false
|
18
16
|
requirements:
|
19
|
-
- -
|
17
|
+
- - '>='
|
20
18
|
- !ruby/object:Gem::Version
|
21
19
|
version: '0'
|
22
20
|
type: :development
|
23
21
|
prerelease: false
|
24
22
|
version_requirements: !ruby/object:Gem::Requirement
|
25
|
-
none: false
|
26
23
|
requirements:
|
27
|
-
- -
|
24
|
+
- - '>='
|
28
25
|
- !ruby/object:Gem::Version
|
29
26
|
version: '0'
|
30
27
|
- !ruby/object:Gem::Dependency
|
31
28
|
name: fluentd
|
32
29
|
requirement: !ruby/object:Gem::Requirement
|
33
|
-
none: false
|
34
30
|
requirements:
|
35
|
-
- -
|
31
|
+
- - '>='
|
36
32
|
- !ruby/object:Gem::Version
|
37
33
|
version: '0'
|
38
34
|
type: :runtime
|
39
35
|
prerelease: false
|
40
36
|
version_requirements: !ruby/object:Gem::Requirement
|
41
|
-
none: false
|
42
37
|
requirements:
|
43
|
-
- -
|
38
|
+
- - '>='
|
44
39
|
- !ruby/object:Gem::Version
|
45
40
|
version: '0'
|
46
41
|
description: fluentd plugin to parse single field, or to combine log structure into
|
@@ -65,29 +60,29 @@ files:
|
|
65
60
|
- test/plugin/test_out_parser.rb
|
66
61
|
homepage: https://github.com/tagomoris/fluent-plugin-parser
|
67
62
|
licenses: []
|
63
|
+
metadata: {}
|
68
64
|
post_install_message:
|
69
65
|
rdoc_options: []
|
70
66
|
require_paths:
|
71
67
|
- lib
|
72
68
|
required_ruby_version: !ruby/object:Gem::Requirement
|
73
|
-
none: false
|
74
69
|
requirements:
|
75
|
-
- -
|
70
|
+
- - '>='
|
76
71
|
- !ruby/object:Gem::Version
|
77
72
|
version: '0'
|
78
73
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
79
|
-
none: false
|
80
74
|
requirements:
|
81
|
-
- -
|
75
|
+
- - '>='
|
82
76
|
- !ruby/object:Gem::Version
|
83
77
|
version: '0'
|
84
78
|
requirements: []
|
85
79
|
rubyforge_project:
|
86
|
-
rubygems_version:
|
80
|
+
rubygems_version: 2.0.2
|
87
81
|
signing_key:
|
88
|
-
specification_version:
|
82
|
+
specification_version: 4
|
89
83
|
summary: plugin to parse/combine fluentd log messages
|
90
84
|
test_files:
|
91
85
|
- test/helper.rb
|
92
86
|
- test/plugin/test_deparser.rb
|
93
87
|
- test/plugin/test_out_parser.rb
|
88
|
+
has_rdoc:
|