fluent-plugin-detect-exceptions 0.0.1 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +6 -6
- data/fluent-plugin-detect-exceptions.gemspec +1 -1
- data/lib/fluent/plugin/exception_detector.rb +33 -17
- data/lib/fluent/plugin/out_detect_exceptions.rb +6 -4
- data/test/plugin/bench_exception_detector.rb +6 -6
- data/test/plugin/test_exception_detector.rb +56 -2
- data/test/plugin/test_out_detect_exceptions.rb +6 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1a26a5e2085d4bc44296565c82c0baacc5c1e33f
|
4
|
+
data.tar.gz: b0268e7263ca8bb5d5bfb3cb8913c04acb3b5893
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c170490ab76e47973fa288d49280d0c8e7cc0232f99fea45b9a144f7b1dc1c6514ca54d9fb57bfcff7f83ef7a809ea50f0e1c4f93ec64a9690e9fa9e5b376d6a
|
7
|
+
data.tar.gz: ebd8383c06d79737a213c7756573db8b61044aaed9c0ea0262bdf6be5062cfa4740eaa192ca8e956fa69f2e391767b91d2c163c868696aa828a1eccffe8832b0
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
fluent-plugin-detect-exceptions (0.0.
|
4
|
+
fluent-plugin-detect-exceptions (0.0.3)
|
5
5
|
fluentd (~> 0.10, <= 0.13)
|
6
6
|
|
7
7
|
GEM
|
@@ -22,9 +22,9 @@ GEM
|
|
22
22
|
http_parser.rb (0.6.0)
|
23
23
|
json (2.0.2)
|
24
24
|
msgpack (0.5.12)
|
25
|
-
parser (2.3.1
|
25
|
+
parser (2.3.3.1)
|
26
26
|
ast (~> 2.2)
|
27
|
-
power_assert (0.
|
27
|
+
power_assert (0.4.1)
|
28
28
|
powerpack (0.1.1)
|
29
29
|
rainbow (2.1.0)
|
30
30
|
rake (10.5.0)
|
@@ -37,14 +37,14 @@ GEM
|
|
37
37
|
ruby-progressbar (1.8.1)
|
38
38
|
sigdump (0.2.4)
|
39
39
|
string-scrub (0.0.5)
|
40
|
-
test-unit (3.2.
|
40
|
+
test-unit (3.2.3)
|
41
41
|
power_assert
|
42
42
|
thread_safe (0.3.5)
|
43
43
|
tzinfo (1.2.2)
|
44
44
|
thread_safe (~> 0.1)
|
45
|
-
tzinfo-data (1.2016.
|
45
|
+
tzinfo-data (1.2016.10)
|
46
46
|
tzinfo (>= 1.0.0)
|
47
|
-
unicode-display_width (1.1.
|
47
|
+
unicode-display_width (1.1.2)
|
48
48
|
yajl-ruby (1.3.0)
|
49
49
|
|
50
50
|
PLATFORMS
|
@@ -11,7 +11,7 @@ eos
|
|
11
11
|
gem.homepage = \
|
12
12
|
'https://github.com/GoogleCloudPlatform/fluent-plugin-detect-exceptions'
|
13
13
|
gem.license = 'Apache-2.0'
|
14
|
-
gem.version = '0.0.
|
14
|
+
gem.version = '0.0.3'
|
15
15
|
gem.authors = ['Thomas Schickinger']
|
16
16
|
gem.email = ['schickin@google.com']
|
17
17
|
gem.required_ruby_version = Gem::Requirement.new('>= 2.0')
|
@@ -66,8 +66,9 @@ module Fluent
|
|
66
66
|
].freeze
|
67
67
|
|
68
68
|
PHP_RULES = [
|
69
|
-
rule(:start_state, /
|
70
|
-
|
69
|
+
rule(:start_state, /
|
70
|
+
(?:PHP\ (?:Notice|Parse\ error|Fatal\ error|Warning):)|
|
71
|
+
(?:exception\ '[^']+'\ with\ message\ ')/x, :php_stack_begin),
|
71
72
|
rule(:php_stack_begin, /^Stack trace:/, :php_stack_frames),
|
72
73
|
rule(:php_stack_frames, /^#\d/, :php_stack_frames),
|
73
74
|
rule(:php_stack_frames, /^\s+thrown in /, :start_state)
|
@@ -190,34 +191,36 @@ module Fluent
|
|
190
191
|
# message_field may contain the empty string. In this case, the
|
191
192
|
# TraceAccumulator 'learns' the field name from the first record by checking
|
192
193
|
# for some pre-defined common field names of text logs.
|
193
|
-
|
194
|
+
# The named parameters max_lines and max_bytes limit the maximum amount
|
195
|
+
# of data to be buffered. The default value 0 indicates 'no limit'.
|
196
|
+
def initialize(message_field, languages, max_lines: 0, max_bytes: 0,
|
197
|
+
&emit_callback)
|
194
198
|
@exception_detector = Fluent::ExceptionDetector.new(*languages)
|
199
|
+
@max_lines = max_lines
|
200
|
+
@max_bytes = max_bytes
|
195
201
|
@message_field = message_field
|
196
202
|
@messages = []
|
197
203
|
@buffer_start_time = Time.now
|
204
|
+
@buffer_size = 0
|
198
205
|
@first_record = nil
|
199
206
|
@first_timestamp = nil
|
200
207
|
@emit = emit_callback
|
201
208
|
end
|
202
209
|
|
203
210
|
def push(time_sec, record)
|
204
|
-
|
205
|
-
ExceptionDetectorConfig::DEFAULT_FIELDS.each do |f|
|
206
|
-
if record.key?(f)
|
207
|
-
@message_field = f
|
208
|
-
break
|
209
|
-
end
|
210
|
-
end
|
211
|
-
end
|
212
|
-
message = @message_field.nil? ? record : record[@message_field]
|
211
|
+
message = extract_message(record)
|
213
212
|
if message.nil?
|
214
213
|
@exception_detector.reset
|
215
214
|
detection_status = :no_trace
|
216
215
|
else
|
216
|
+
force_flush if @max_bytes > 0 &&
|
217
|
+
@buffer_size + message.length > @max_bytes
|
217
218
|
detection_status = @exception_detector.update(message)
|
218
219
|
end
|
219
220
|
|
220
221
|
update_buffer(detection_status, time_sec, record, message)
|
222
|
+
|
223
|
+
force_flush if @max_lines > 0 && @messages.length == @max_lines
|
221
224
|
end
|
222
225
|
|
223
226
|
def flush
|
@@ -239,6 +242,7 @@ module Fluent
|
|
239
242
|
@messages = []
|
240
243
|
@first_record = nil
|
241
244
|
@first_timestamp = nil
|
245
|
+
@buffer_size = 0
|
242
246
|
end
|
243
247
|
|
244
248
|
def force_flush
|
@@ -246,12 +250,20 @@ module Fluent
|
|
246
250
|
@exception_detector.reset
|
247
251
|
end
|
248
252
|
|
249
|
-
def length
|
250
|
-
@messages.length
|
251
|
-
end
|
252
|
-
|
253
253
|
private
|
254
254
|
|
255
|
+
def extract_message(record)
|
256
|
+
if !@message_field.nil? && @message_field.empty?
|
257
|
+
ExceptionDetectorConfig::DEFAULT_FIELDS.each do |f|
|
258
|
+
if record.key?(f)
|
259
|
+
@message_field = f
|
260
|
+
break
|
261
|
+
end
|
262
|
+
end
|
263
|
+
end
|
264
|
+
@message_field.nil? ? record : record[@message_field]
|
265
|
+
end
|
266
|
+
|
255
267
|
def update_buffer(detection_status, time_sec, record, message)
|
256
268
|
trigger_emit = detection_status == :no_trace ||
|
257
269
|
detection_status == :end_trace
|
@@ -259,6 +271,7 @@ module Fluent
|
|
259
271
|
@emit.call(time_sec, record)
|
260
272
|
return
|
261
273
|
end
|
274
|
+
|
262
275
|
case detection_status
|
263
276
|
when :inside_trace
|
264
277
|
add(time_sec, record, message)
|
@@ -281,7 +294,10 @@ module Fluent
|
|
281
294
|
@first_timestamp = time_sec
|
282
295
|
@buffer_start_time = Time.now
|
283
296
|
end
|
284
|
-
|
297
|
+
unless message.nil?
|
298
|
+
@messages << message
|
299
|
+
@buffer_size += message.length
|
300
|
+
end
|
285
301
|
end
|
286
302
|
end
|
287
303
|
end
|
@@ -30,8 +30,10 @@ module Fluent
|
|
30
30
|
config_param :multiline_flush_interval, :time, default: nil
|
31
31
|
desc 'Programming languages for which to detect exceptions. Default: all.'
|
32
32
|
config_param :languages, :array, value_type: :string, default: []
|
33
|
-
desc 'Maximum number of lines
|
33
|
+
desc 'Maximum number of lines to flush (0 means no limit). Default: 1000.'
|
34
34
|
config_param :max_lines, :integer, default: 1000
|
35
|
+
desc 'Maximum number of bytes to flush (0 means no limit). Default: 0.'
|
36
|
+
config_param :max_bytes, :integer, default: 0
|
35
37
|
|
36
38
|
Fluent::Plugin.register_output('detect_exceptions', self)
|
37
39
|
|
@@ -85,14 +87,14 @@ module Fluent
|
|
85
87
|
unless @accumulators.key?(tag)
|
86
88
|
out_tag = tag.sub(/^#{Regexp.escape(remove_tag_prefix)}\./, '')
|
87
89
|
@accumulators[tag] =
|
88
|
-
Fluent::TraceAccumulator.new(message, @languages
|
90
|
+
Fluent::TraceAccumulator.new(message, @languages,
|
91
|
+
max_lines: max_lines,
|
92
|
+
max_bytes: max_bytes) do |t, r|
|
89
93
|
router.emit(out_tag, t, r)
|
90
94
|
end
|
91
95
|
end
|
92
96
|
|
93
97
|
@accumulators[tag].push(time_sec, record)
|
94
|
-
|
95
|
-
@accumulators[tag].force_flush if @accumulators[tag].length > max_lines
|
96
98
|
end
|
97
99
|
end
|
98
100
|
|
@@ -48,22 +48,22 @@ random_text = (1..(size / line_length)).collect do
|
|
48
48
|
end
|
49
49
|
|
50
50
|
exceptions = {
|
51
|
-
|
52
|
-
|
51
|
+
java: (JAVA_EXC * (size / JAVA_EXC.length)).lines,
|
52
|
+
python: (PYTHON_EXC * (size / PYTHON_EXC.length)).lines
|
53
53
|
}
|
54
54
|
|
55
55
|
puts "Start benchmark. Input size #{size_in_m}M."
|
56
56
|
Benchmark.bm do |x|
|
57
57
|
languages = Fluent::ExceptionDetectorConfig::RULES_BY_LANG.keys
|
58
58
|
languages.each do |lang|
|
59
|
-
buffer = Fluent::
|
59
|
+
buffer = Fluent::TraceAccumulator.new(nil, lang) {}
|
60
60
|
x.report("#{lang}_detector_random_text") do
|
61
61
|
random_text.each { |l| buffer.push(0, l) }
|
62
62
|
end
|
63
63
|
end
|
64
|
-
|
65
|
-
buffer = Fluent::
|
66
|
-
exc_languages = detector_lang ==
|
64
|
+
[:java, :python, :all].each do |detector_lang|
|
65
|
+
buffer = Fluent::TraceAccumulator.new(nil, detector_lang) {}
|
66
|
+
exc_languages = detector_lang == :all ? exceptions.keys : [detector_lang]
|
67
67
|
exc_languages.each do |exc_lang|
|
68
68
|
x.report("#{detector_lang}_detector_#{exc_lang}_stacks") do
|
69
69
|
exceptions[exc_lang].each { |l| buffer.push(0, l) }
|
@@ -107,6 +107,14 @@ Exception: ('spam', 'eggs')
|
|
107
107
|
END
|
108
108
|
|
109
109
|
PHP_EXC = <<END.freeze
|
110
|
+
exception 'Exception' with message 'Custom exception' in /home/joe/work/test-php/test.php:5
|
111
|
+
Stack trace:
|
112
|
+
#0 /home/joe/work/test-php/test.php(9): func1()
|
113
|
+
#1 /home/joe/work/test-php/test.php(13): func2()
|
114
|
+
#2 {main}
|
115
|
+
END
|
116
|
+
|
117
|
+
PHP_ON_GAE_EXC = <<END.freeze
|
110
118
|
PHP Fatal error: Uncaught exception 'Exception' with message 'message' in /base/data/home/apps/s~crash-example-php/1.388306779641080894/errors.php:60
|
111
119
|
Stack trace:
|
112
120
|
#0 [internal function]: ErrorEntryGenerator::{closure}()
|
@@ -227,7 +235,8 @@ END
|
|
227
235
|
end
|
228
236
|
|
229
237
|
def test_php
|
230
|
-
check_exception(PHP_EXC,
|
238
|
+
check_exception(PHP_EXC, false)
|
239
|
+
check_exception(PHP_ON_GAE_EXC, true)
|
231
240
|
end
|
232
241
|
|
233
242
|
def test_go
|
@@ -243,7 +252,8 @@ END
|
|
243
252
|
check_exception(PYTHON_EXC, true)
|
244
253
|
check_exception(COMPLEX_JAVA_EXC, false)
|
245
254
|
check_exception(NODE_JS_EXC, false)
|
246
|
-
check_exception(PHP_EXC,
|
255
|
+
check_exception(PHP_EXC, false)
|
256
|
+
check_exception(PHP_ON_GAE_EXC, true)
|
247
257
|
check_exception(CLIENT_JS_EXC, false)
|
248
258
|
check_exception(GO_EXC, false)
|
249
259
|
check_exception(CSHARP_EXC, false)
|
@@ -350,4 +360,48 @@ END
|
|
350
360
|
assert_equal(expected_json(s.actual_field, s.output), out, s.desc)
|
351
361
|
end
|
352
362
|
end
|
363
|
+
|
364
|
+
def test_max_lines_limit
|
365
|
+
# Limit is equal to the first part of the exception and forces it to be
|
366
|
+
# flushed before the rest of the exception is processed.
|
367
|
+
max_lines = JAVA_EXC_PART1.lines.length
|
368
|
+
out = []
|
369
|
+
buffer = Fluent::TraceAccumulator.new(nil,
|
370
|
+
[:all],
|
371
|
+
max_lines: max_lines) do |_, m|
|
372
|
+
out << m
|
373
|
+
end
|
374
|
+
feed_lines(buffer, JAVA_EXC)
|
375
|
+
assert_equal([JAVA_EXC_PART1] + JAVA_EXC_PART2.lines, out)
|
376
|
+
end
|
377
|
+
|
378
|
+
def test_high_max_bytes_limit
|
379
|
+
# Limit is just too small to add one more line to the buffered first part of
|
380
|
+
# the exception.
|
381
|
+
max_bytes = JAVA_EXC_PART1.length + JAVA_EXC_PART2.lines[0].length - 1
|
382
|
+
out = []
|
383
|
+
buffer = Fluent::TraceAccumulator.new(nil,
|
384
|
+
[:all],
|
385
|
+
max_bytes: max_bytes) do |_, m|
|
386
|
+
out << m
|
387
|
+
end
|
388
|
+
feed_lines(buffer, JAVA_EXC)
|
389
|
+
# Check that the trace is flushed after the first part.
|
390
|
+
assert_equal([JAVA_EXC_PART1] + JAVA_EXC_PART2.lines, out)
|
391
|
+
end
|
392
|
+
|
393
|
+
def test_low_max_bytes_limit
|
394
|
+
# Limit is exceeded by the character that follows the buffered first part of
|
395
|
+
# the exception.
|
396
|
+
max_bytes = JAVA_EXC_PART1.length
|
397
|
+
out = []
|
398
|
+
buffer = Fluent::TraceAccumulator.new(nil,
|
399
|
+
[:all],
|
400
|
+
max_bytes: max_bytes) do |_, m|
|
401
|
+
out << m
|
402
|
+
end
|
403
|
+
feed_lines(buffer, JAVA_EXC)
|
404
|
+
# Check that the trace is flushed after the first part.
|
405
|
+
assert_equal([JAVA_EXC_PART1] + JAVA_EXC_PART2.lines, out)
|
406
|
+
end
|
353
407
|
end
|
@@ -166,8 +166,13 @@ END
|
|
166
166
|
d.run do
|
167
167
|
feed_lines(d, t, PYTHON_EXC, JAVA_EXC)
|
168
168
|
end
|
169
|
+
# Expected: the first two lines of the exception are buffered and combined.
|
170
|
+
# Then the max_lines setting kicks in and the rest of the Python exception
|
171
|
+
# is logged line-by-line (since it's not an exception stack in itself).
|
172
|
+
# Finally, the Java exception is logged in its entirety, since it only
|
173
|
+
# has two lines.
|
169
174
|
expected =
|
170
|
-
[PYTHON_EXC.lines[0..
|
175
|
+
[PYTHON_EXC.lines[0..1].join] + PYTHON_EXC.lines[2..-1] + [JAVA_EXC]
|
171
176
|
assert_equal(make_logs(t, *expected), d.events)
|
172
177
|
end
|
173
178
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-detect-exceptions
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Thomas Schickinger
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-12-09 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|