fluent-plugin-parser 0.3.0 → 0.3.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +17 -2
- data/fluent-plugin-parser.gemspec +1 -1
- data/lib/fluent/plugin/fixed_parser.rb +33 -36
- data/test/plugin/test_out_parser.rb +66 -0
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 981df9c360ea929797303eedebec95b3f884a574
|
4
|
+
data.tar.gz: ea2f3528e33cc43af7627b81ddc9b6be9ae8137a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 61c26d1823b7e64f01495e6aae818d84e1e8cbc06d3545342f80d235474fed197e8965dbdfd33e2b746c8e54e24c35516bea7e295c4c75a793094d5f8ddffdfc
|
7
|
+
data.tar.gz: 82fa801ee3ecd797a43628f5d0c1019898b815d7197339efc11625374570687a6364a66920c97f56fc0e4d3ac58d18c90a72926690cdfe33bb50a76f902fb9ed
|
data/README.md
CHANGED
@@ -107,6 +107,19 @@ Other options (ex: `reserve_data`, `inject_key_prefix`) are available with `hash
|
|
107
107
|
|
108
108
|
# output data: {"sales":"{\"user\":1,\"num\":2}", "parsed":{"sales.user":1, "sales.num":2}}
|
109
109
|
|
110
|
+
Not to parse times (reserve that field like 'time' in record), specify `time_parse no`:
|
111
|
+
|
112
|
+
<match raw.sales.*>
|
113
|
+
type parser
|
114
|
+
tag sales
|
115
|
+
format json
|
116
|
+
key_name sales
|
117
|
+
hash_value_field parsed
|
118
|
+
time_parse no
|
119
|
+
</match>
|
120
|
+
# input string of 'sales': {"user":1,"num":2,"time":"2013-10-31 12:48:33"}
|
121
|
+
# output data: {"parsed":{"user":1, "num":2,"time":"2013-10-31 12:48:33"}}
|
122
|
+
|
110
123
|
### DeparserOutput
|
111
124
|
|
112
125
|
To build CSV from field 'store','item','num', as field 'csv', without raw data:
|
@@ -137,5 +150,7 @@ To build same CSV, as additional field 'csv', with reserved raw fields:
|
|
137
150
|
|
138
151
|
## Copyright
|
139
152
|
|
140
|
-
|
141
|
-
|
153
|
+
* Copyright
|
154
|
+
* Copyright (c) 2012- TAGOMORI Satoshi (tagomoris)
|
155
|
+
* License
|
156
|
+
* Apache License, Version 2.0
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# -*- encoding: utf-8 -*-
|
2
2
|
Gem::Specification.new do |gem|
|
3
3
|
gem.name = "fluent-plugin-parser"
|
4
|
-
gem.version = "0.3.
|
4
|
+
gem.version = "0.3.1"
|
5
5
|
gem.authors = ["TAGOMORI Satoshi"]
|
6
6
|
gem.email = ["tagomoris@gmail.com"]
|
7
7
|
gem.description = %q{fluentd plugin to parse single field, or to combine log structure into single field}
|
@@ -4,10 +4,40 @@
|
|
4
4
|
module FluentExt; end
|
5
5
|
|
6
6
|
class FluentExt::TextParser
|
7
|
-
class
|
7
|
+
class GenericParser
|
8
8
|
include Fluent::Configurable
|
9
9
|
|
10
|
+
config_param :time_key, :string, :default => 'time'
|
10
11
|
config_param :time_format, :string, :default => nil
|
12
|
+
config_param :time_parse, :bool, :default => true
|
13
|
+
|
14
|
+
def parse_time(record)
|
15
|
+
time = nil
|
16
|
+
|
17
|
+
unless @time_parse
|
18
|
+
return time, record
|
19
|
+
end
|
20
|
+
|
21
|
+
if value = record.delete(@time_key)
|
22
|
+
begin
|
23
|
+
time = if @time_format
|
24
|
+
Time.strptime(value, @time_format).to_i
|
25
|
+
else
|
26
|
+
Time.parse(value).to_i
|
27
|
+
end
|
28
|
+
rescue TypeError, ArgumentError => e
|
29
|
+
$log.warn "Failed to parse time", :key => @time_key, :value => value
|
30
|
+
record[@time_key] = value
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
return time, record
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
class RegexpParser < GenericParser
|
39
|
+
include Fluent::Configurable
|
40
|
+
|
11
41
|
config_param :suppress_parse_error_log, :bool, :default => false
|
12
42
|
|
13
43
|
def initialize(regexp, conf={})
|
@@ -28,44 +58,11 @@ class FluentExt::TextParser
|
|
28
58
|
return nil, nil
|
29
59
|
end
|
30
60
|
|
31
|
-
time = nil
|
32
61
|
record = {}
|
33
|
-
|
34
62
|
m.names.each {|name|
|
35
|
-
|
36
|
-
case name
|
37
|
-
when "time"
|
38
|
-
if @time_format
|
39
|
-
time = Time.strptime(value, @time_format).to_i
|
40
|
-
else
|
41
|
-
time = Time.parse(value).to_i
|
42
|
-
end
|
43
|
-
else
|
44
|
-
record[name] = value
|
45
|
-
end
|
46
|
-
end
|
63
|
+
record[name] = m[name] if m[name]
|
47
64
|
}
|
48
|
-
|
49
|
-
return time, record
|
50
|
-
end
|
51
|
-
end
|
52
|
-
|
53
|
-
class GenericParser
|
54
|
-
include Fluent::Configurable
|
55
|
-
|
56
|
-
config_param :time_key, :string, :default => 'time'
|
57
|
-
config_param :time_format, :string, :default => nil
|
58
|
-
|
59
|
-
def parse_time(record)
|
60
|
-
time = nil
|
61
|
-
if value = record.delete(@time_key)
|
62
|
-
time = if @time_format
|
63
|
-
Time.strptime(value, @time_format).to_i
|
64
|
-
else
|
65
|
-
Time.parse(value).to_i
|
66
|
-
end
|
67
|
-
end
|
68
|
-
return time, record
|
65
|
+
parse_time(record)
|
69
66
|
end
|
70
67
|
end
|
71
68
|
|
@@ -442,6 +442,72 @@ class ParserOutputTest < Test::Unit::TestCase
|
|
442
442
|
assert_equal({"data.xxx"=>"first","data.yyy"=>"second"}, record['parsed'])
|
443
443
|
end
|
444
444
|
|
445
|
+
CONFIG_DONT_PARSE_TIME = %[
|
446
|
+
remove_prefix test
|
447
|
+
key_name data
|
448
|
+
format json
|
449
|
+
time_parse no
|
450
|
+
]
|
451
|
+
def test_time_should_be_reserved
|
452
|
+
t = Time.now.to_i
|
453
|
+
d = create_driver(CONFIG_DONT_PARSE_TIME, 'test.in')
|
454
|
+
|
455
|
+
assert_equal false, d.instance.instance_eval{ @parser }.instance_eval{ @parser }.time_parse
|
456
|
+
|
457
|
+
d.run do
|
458
|
+
d.emit({'data' => '{"time":1383190430, "f1":"v1"}'}, t)
|
459
|
+
d.emit({'data' => '{"time":"1383190430", "f1":"v1"}'}, t)
|
460
|
+
d.emit({'data' => '{"time":"2013-10-31 12:34:03 +0900", "f1":"v1"}'}, t)
|
461
|
+
end
|
462
|
+
emits = d.emits
|
463
|
+
assert_equal 3, emits.length
|
464
|
+
|
465
|
+
assert_equal 'in', emits[0][0]
|
466
|
+
assert_equal 'v1', emits[0][2]['f1']
|
467
|
+
assert_equal 1383190430, emits[0][2]['time']
|
468
|
+
assert_equal t, emits[0][1]
|
469
|
+
|
470
|
+
assert_equal 'in', emits[1][0]
|
471
|
+
assert_equal 'v1', emits[1][2]['f1']
|
472
|
+
assert_equal "1383190430", emits[1][2]['time']
|
473
|
+
assert_equal t, emits[1][1]
|
474
|
+
|
475
|
+
assert_equal 'in', emits[2][0]
|
476
|
+
assert_equal 'v1', emits[2][2]['f1']
|
477
|
+
assert_equal '2013-10-31 12:34:03 +0900', emits[2][2]['time']
|
478
|
+
assert_equal t, emits[2][1]
|
479
|
+
end
|
480
|
+
|
481
|
+
CONFIG_INVALID_TIME_VALUE = %[
|
482
|
+
remove_prefix test
|
483
|
+
key_name data
|
484
|
+
format json
|
485
|
+
] # 'time' is implicit @time_key
|
486
|
+
def test_invalid_time_data
|
487
|
+
# should not raise errors
|
488
|
+
t = Time.now.to_i
|
489
|
+
d = create_driver(CONFIG_INVALID_TIME_VALUE, 'test.in')
|
490
|
+
assert_nothing_raised {
|
491
|
+
d.run do
|
492
|
+
d.emit({'data' => '{"time":[], "f1":"v1"}'}, t)
|
493
|
+
d.emit({'data' => '{"time":"thisisnottime", "f1":"v1"}'}, t)
|
494
|
+
end
|
495
|
+
}
|
496
|
+
emits = d.emits
|
497
|
+
assert_equal 2, emits.length
|
498
|
+
|
499
|
+
assert_equal 'in', emits[0][0]
|
500
|
+
assert_equal t, emits[0][1]
|
501
|
+
assert_equal 'v1', emits[0][2]['f1']
|
502
|
+
assert_equal [], emits[0][2]['time']
|
503
|
+
|
504
|
+
assert_equal 'in', emits[1][0]
|
505
|
+
assert_equal t, emits[1][1]
|
506
|
+
assert_equal 'v1', emits[1][2]['f1']
|
507
|
+
assert_equal 'thisisnottime', emits[1][2]['time']
|
508
|
+
end
|
509
|
+
|
510
|
+
|
445
511
|
#TODO: apache2
|
446
512
|
# REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
|
447
513
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-parser
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- TAGOMORI Satoshi
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2013-10-
|
11
|
+
date: 2013-10-31 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rake
|