fluent-plugin-parser 0.3.3 → 0.3.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/fluent-plugin-parser.gemspec +1 -1
- data/lib/fluent/plugin/fixed_parser.rb +33 -12
- data/lib/fluent/plugin/out_parser.rb +3 -1
- data/test/plugin/test_out_parser.rb +56 -33
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 11c8d0bf5065c2ea089887da434f380c6477797c
|
4
|
+
data.tar.gz: 05dd0a148aed3f8020eb839fd5042298464296ed
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4bffef34b921fe845581381aa759fdeea68bf40ea7b95031f7e8909bd5516828f9b31964b550048bd1507e2dd337394e3c2f51ca38afd814c37c17c610353d40
|
7
|
+
data.tar.gz: 3471c464f75070f30e4ce2bd97e6bb5a10662e779d7cce00b4d0f41f2efdf6d5ba60c77f36a73ad46a81aa48f17ea10ff0e3f6527f9e1f82b7872ebb47fb1fbf
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# -*- encoding: utf-8 -*-
|
2
2
|
Gem::Specification.new do |gem|
|
3
3
|
gem.name = "fluent-plugin-parser"
|
4
|
-
gem.version = "0.3.
|
4
|
+
gem.version = "0.3.4"
|
5
5
|
gem.authors = ["TAGOMORI Satoshi"]
|
6
6
|
gem.email = ["tagomoris@gmail.com"]
|
7
7
|
gem.description = %q{fluentd plugin to parse single field, or to combine log structure into single field}
|
@@ -11,10 +11,18 @@ class FluentExt::TextParser
|
|
11
11
|
config_param :time_format, :string, :default => nil
|
12
12
|
config_param :time_parse, :bool, :default => true
|
13
13
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
14
|
+
attr_accessor :log
|
15
|
+
|
16
|
+
def initialize
|
17
|
+
super
|
18
|
+
|
19
|
+
@cache1_key = nil
|
20
|
+
@cache1_time = nil
|
21
|
+
@cache2_key = nil
|
22
|
+
@cache2_time = nil
|
23
|
+
|
24
|
+
@log = nil
|
25
|
+
end
|
18
26
|
|
19
27
|
def parse_time(record)
|
20
28
|
time = nil
|
@@ -40,7 +48,7 @@ class FluentExt::TextParser
|
|
40
48
|
@cache2_key = value
|
41
49
|
@cache2_time = time
|
42
50
|
rescue TypeError, ArgumentError => e
|
43
|
-
|
51
|
+
@log.warn "Failed to parse time", :key => @time_key, :value => value
|
44
52
|
record[@time_key] = value
|
45
53
|
end
|
46
54
|
end
|
@@ -67,7 +75,7 @@ class FluentExt::TextParser
|
|
67
75
|
m = @regexp.match(text)
|
68
76
|
unless m
|
69
77
|
unless @suppress_parse_error_log
|
70
|
-
|
78
|
+
@log.warn "pattern not match: #{text}"
|
71
79
|
end
|
72
80
|
|
73
81
|
return nil, nil
|
@@ -87,7 +95,7 @@ class FluentExt::TextParser
|
|
87
95
|
return parse_time(record)
|
88
96
|
rescue Yajl::ParseError
|
89
97
|
unless @suppress_parse_error_log
|
90
|
-
|
98
|
+
@log.warn "pattern not match(json): #{text.inspect}: #{$!}"
|
91
99
|
end
|
92
100
|
|
93
101
|
return nil, nil
|
@@ -133,16 +141,23 @@ class FluentExt::TextParser
|
|
133
141
|
end
|
134
142
|
end
|
135
143
|
|
136
|
-
class ApacheParser
|
144
|
+
class ApacheParser < GenericParser
|
137
145
|
include Fluent::Configurable
|
138
146
|
|
139
147
|
REGEXP = /^(?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<code>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)")?$/
|
140
148
|
|
149
|
+
def initialize
|
150
|
+
super
|
151
|
+
|
152
|
+
@time_key = "time"
|
153
|
+
@time_format = "%d/%b/%Y:%H:%M:%S %z"
|
154
|
+
end
|
155
|
+
|
141
156
|
def call(text)
|
142
157
|
m = REGEXP.match(text)
|
143
158
|
unless m
|
144
159
|
unless @suppress_parse_error_log
|
145
|
-
|
160
|
+
@log.warn "pattern not match: #{text.inspect}"
|
146
161
|
end
|
147
162
|
|
148
163
|
return nil, nil
|
@@ -155,7 +170,6 @@ class FluentExt::TextParser
|
|
155
170
|
user = (user == '-') ? nil : user
|
156
171
|
|
157
172
|
time = m['time']
|
158
|
-
time = Time.strptime(time, "%d/%b/%Y:%H:%M:%S %z").to_i
|
159
173
|
|
160
174
|
method = m['method']
|
161
175
|
path = m['path']
|
@@ -173,6 +187,7 @@ class FluentExt::TextParser
|
|
173
187
|
agent = (agent == '-') ? nil : agent
|
174
188
|
|
175
189
|
record = {
|
190
|
+
"time" => time,
|
176
191
|
"host" => host,
|
177
192
|
"user" => user,
|
178
193
|
"method" => method,
|
@@ -183,7 +198,7 @@ class FluentExt::TextParser
|
|
183
198
|
"agent" => agent,
|
184
199
|
}
|
185
200
|
|
186
|
-
|
201
|
+
parse_time(record)
|
187
202
|
end
|
188
203
|
end
|
189
204
|
|
@@ -209,7 +224,11 @@ class FluentExt::TextParser
|
|
209
224
|
TEMPLATE_FACTORIES[name] = factory
|
210
225
|
end
|
211
226
|
|
212
|
-
|
227
|
+
attr_accessor :log
|
228
|
+
attr_reader :parser
|
229
|
+
|
230
|
+
def initialize(logger)
|
231
|
+
@log = logger
|
213
232
|
@parser = nil
|
214
233
|
end
|
215
234
|
|
@@ -246,6 +265,8 @@ class FluentExt::TextParser
|
|
246
265
|
|
247
266
|
end
|
248
267
|
|
268
|
+
@parser.log = @log
|
269
|
+
|
249
270
|
if @parser.respond_to?(:configure)
|
250
271
|
@parser.configure(conf)
|
251
272
|
end
|
@@ -12,6 +12,8 @@ class Fluent::ParserOutput < Fluent::Output
|
|
12
12
|
config_param :replace_invalid_sequence, :bool, :default => false
|
13
13
|
config_param :hash_value_field, :string, :default => nil
|
14
14
|
|
15
|
+
attr_reader :parser
|
16
|
+
|
15
17
|
def initialize
|
16
18
|
super
|
17
19
|
require 'time'
|
@@ -39,7 +41,7 @@ class Fluent::ParserOutput < Fluent::Output
|
|
39
41
|
@added_prefix_string = @add_prefix + '.'
|
40
42
|
end
|
41
43
|
|
42
|
-
@parser = FluentExt::TextParser.new
|
44
|
+
@parser = FluentExt::TextParser.new(log())
|
43
45
|
@parser.configure(conf)
|
44
46
|
end
|
45
47
|
|
@@ -617,51 +617,74 @@ class ParserOutputTest < Test::Unit::TestCase
|
|
617
617
|
end
|
618
618
|
end
|
619
619
|
|
620
|
+
def swap_logger(instance)
|
621
|
+
raise "use with block" unless block_given?
|
622
|
+
parser_logger = instance.parser.log
|
623
|
+
dummy = DummyLogger.new
|
624
|
+
instance.parser.log = dummy
|
625
|
+
instance.parser.parser.log = dummy
|
626
|
+
|
627
|
+
restore = if instance.respond_to?("log=".to_sym)
|
628
|
+
saved_logger = instance.log
|
629
|
+
instance.log = dummy
|
630
|
+
lambda{ instance.log = saved_logger; instance.parser.log = instance.parser.parser.log = parser_logger }
|
631
|
+
else
|
632
|
+
saved_logger = $log
|
633
|
+
$log = dummy
|
634
|
+
lambda{ $log = saved_logger; instance.parser.log = instance.parser.parser.log = parser_logger }
|
635
|
+
end
|
636
|
+
|
637
|
+
yield
|
638
|
+
|
639
|
+
restore.call
|
640
|
+
end
|
641
|
+
|
620
642
|
def test_suppress_parse_error_log
|
621
643
|
# default(disabled) 'suppress_parse_error_log' is not specify
|
622
644
|
d = create_driver(CONFIG_DEFAULT_SUPPRESS_PARSE_ERROR_LOG, 'test.in')
|
623
645
|
|
624
|
-
|
625
|
-
|
626
|
-
|
627
|
-
|
628
|
-
|
629
|
-
|
630
|
-
|
631
|
-
|
632
|
-
|
633
|
-
|
634
|
-
|
635
|
-
|
636
|
-
|
637
|
-
}
|
646
|
+
swap_logger(d.instance) do
|
647
|
+
assert_raise(DummyLoggerWarnedException) {
|
648
|
+
d.run do
|
649
|
+
d.emit({'message' => INVALID_MESSAGE}, Time.now.to_i)
|
650
|
+
end
|
651
|
+
}
|
652
|
+
|
653
|
+
assert_nothing_raised {
|
654
|
+
d.run do
|
655
|
+
d.emit({'message' => VALID_MESSAGE}, Time.now.to_i)
|
656
|
+
end
|
657
|
+
}
|
658
|
+
end
|
638
659
|
|
639
660
|
# disabled 'suppress_parse_error_log'
|
640
661
|
d = create_driver(CONFIG_DISABELED_SUPPRESS_PARSE_ERROR_LOG, 'test.in')
|
641
662
|
|
642
|
-
|
643
|
-
|
644
|
-
d.
|
645
|
-
|
646
|
-
|
647
|
-
|
648
|
-
|
649
|
-
|
650
|
-
d.
|
651
|
-
|
652
|
-
|
663
|
+
swap_logger(d.instance) do
|
664
|
+
assert_raise(DummyLoggerWarnedException) {
|
665
|
+
d.run do
|
666
|
+
d.emit({'message' => INVALID_MESSAGE}, Time.now.to_i)
|
667
|
+
end
|
668
|
+
}
|
669
|
+
|
670
|
+
assert_nothing_raised {
|
671
|
+
d.run do
|
672
|
+
d.emit({'message' => VALID_MESSAGE}, Time.now.to_i)
|
673
|
+
end
|
674
|
+
}
|
675
|
+
end
|
653
676
|
|
654
677
|
# enabled 'suppress_parse_error_log'
|
655
678
|
d = create_driver(CONFIG_ENABELED_SUPPRESS_PARSE_ERROR_LOG, 'test.in')
|
656
679
|
|
657
|
-
|
658
|
-
|
659
|
-
d.
|
660
|
-
|
661
|
-
|
662
|
-
|
663
|
-
|
664
|
-
|
680
|
+
swap_logger(d.instance) do
|
681
|
+
assert_nothing_raised {
|
682
|
+
d.run do
|
683
|
+
d.emit({'message' => INVALID_MESSAGE}, Time.now.to_i)
|
684
|
+
d.emit({'message' => VALID_MESSAGE}, Time.now.to_i)
|
685
|
+
end
|
686
|
+
}
|
687
|
+
end
|
665
688
|
end
|
666
689
|
|
667
690
|
end
|