gitlab-fluent-plugin-detect-exceptions 0.0.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CONTRIBUTING +24 -0
- data/Gemfile +3 -0
- data/Gemfile.lock +67 -0
- data/LICENSE +201 -0
- data/README.rdoc +130 -0
- data/Rakefile +48 -0
- data/gitlab-fluent-plugin-detect-exceptions.gemspec +29 -0
- data/lib/fluent/plugin/exception_detector.rb +428 -0
- data/lib/fluent/plugin/out_detect_exceptions.rb +143 -0
- data/test/helper.rb +46 -0
- data/test/plugin/bench_exception_detector.rb +73 -0
- data/test/plugin/test_exception_detector.rb +1115 -0
- data/test/plugin/test_out_detect_exceptions.rb +350 -0
- metadata +134 -0
@@ -0,0 +1,428 @@
|
|
1
|
+
# Copyright 2016 Google Inc. All rights reserved.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
#
|
15
|
+
module Fluent
|
16
|
+
Struct.new('Rule', :from_states, :pattern, :to_state)
|
17
|
+
|
18
|
+
# Configuration of the state machine that detects exceptions.
|
19
|
+
module ExceptionDetectorConfig # rubocop:disable Metrics/ModuleLength
|
20
|
+
# Rule for a state transition: if pattern matches go to the given state.
|
21
|
+
class RuleTarget
|
22
|
+
attr_accessor :pattern, :to_state
|
23
|
+
|
24
|
+
def initialize(p, s)
|
25
|
+
@pattern = p
|
26
|
+
@to_state = s
|
27
|
+
end
|
28
|
+
|
29
|
+
def ==(other)
|
30
|
+
other.class == self.class && other.state == state
|
31
|
+
end
|
32
|
+
|
33
|
+
alias eql? ==
|
34
|
+
|
35
|
+
def hash
|
36
|
+
state.hash
|
37
|
+
end
|
38
|
+
|
39
|
+
def state
|
40
|
+
[@pattern, @to_state]
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def self.rule(from_state_or_states, pattern, to_state)
|
45
|
+
from_state_or_states = [from_state_or_states] unless
|
46
|
+
from_state_or_states.is_a?(Array)
|
47
|
+
Struct::Rule.new(from_state_or_states, pattern, to_state)
|
48
|
+
end
|
49
|
+
|
50
|
+
def self.supported
|
51
|
+
RULES_BY_LANG.keys
|
52
|
+
end
|
53
|
+
|
54
|
+
JAVA_RULES = [
|
55
|
+
rule([:start_state, :java_start_exception],
|
56
|
+
/(?:Exception|Error|Throwable|V8 errors stack trace)[:\r\n]/,
|
57
|
+
:java_after_exception),
|
58
|
+
rule(:java_after_exception, /^[\t ]*nested exception is:[\t ]*/,
|
59
|
+
:java_start_exception),
|
60
|
+
rule(:java_after_exception, /^[\r\n]*$/, :java_after_exception),
|
61
|
+
rule([:java_after_exception, :java], /^[\t ]+(?:eval )?at /, :java),
|
62
|
+
|
63
|
+
rule([:java_after_exception, :java],
|
64
|
+
# C# nested exception.
|
65
|
+
/^[\t ]+--- End of inner exception stack trace ---$/,
|
66
|
+
:java),
|
67
|
+
|
68
|
+
rule([:java_after_exception, :java],
|
69
|
+
# C# exception from async code.
|
70
|
+
/^--- End of stack trace from previous (?x:
|
71
|
+
)location where exception was thrown ---$/,
|
72
|
+
:java),
|
73
|
+
|
74
|
+
rule([:java_after_exception, :java], /^[\t ]*(?:Caused by|Suppressed):/,
|
75
|
+
:java_after_exception),
|
76
|
+
rule([:java_after_exception, :java],
|
77
|
+
/^[\t ]*... \d+ (?:more|common frames omitted)/, :java)
|
78
|
+
].freeze
|
79
|
+
|
80
|
+
PYTHON_RULES = [
|
81
|
+
rule(:start_state, /^Traceback \(most recent call last\):$/, :python),
|
82
|
+
rule(:python, /^[\t ]+File /, :python_code),
|
83
|
+
rule(:python_code, /[^\t ]/, :python),
|
84
|
+
rule(:python, /^(?:[^\s.():]+\.)*[^\s.():]+:/, :start_state)
|
85
|
+
].freeze
|
86
|
+
|
87
|
+
PHP_RULES = [
|
88
|
+
rule(:start_state, /
|
89
|
+
(?:PHP\ (?:Notice|Parse\ error|Fatal\ error|Warning):)|
|
90
|
+
(?:exception\ '[^']+'\ with\ message\ ')/x, :php_stack_begin),
|
91
|
+
rule(:php_stack_begin, /^Stack trace:/, :php_stack_frames),
|
92
|
+
rule(:php_stack_frames, /^#\d/, :php_stack_frames),
|
93
|
+
rule(:php_stack_frames, /^\s+thrown in /, :start_state)
|
94
|
+
].freeze
|
95
|
+
|
96
|
+
GO_RULES = [
|
97
|
+
rule(:start_state, /\bpanic: /, :go_after_panic),
|
98
|
+
rule(:start_state, /http: panic serving/, :go_goroutine),
|
99
|
+
rule(:go_after_panic, /^$/, :go_goroutine),
|
100
|
+
rule([:go_after_panic, :go_after_signal, :go_frame_1],
|
101
|
+
/^$/, :go_goroutine),
|
102
|
+
rule(:go_after_panic, /^\[signal /, :go_after_signal),
|
103
|
+
rule(:go_goroutine, /^goroutine \d+ \[[^\]]+\]:$/, :go_frame_1),
|
104
|
+
rule(:go_frame_1, /^(?:[^\s.:]+\.)*[^\s.():]+\(|^created by /,
|
105
|
+
:go_frame_2),
|
106
|
+
rule(:go_frame_2, /^\s/, :go_frame_1)
|
107
|
+
].freeze
|
108
|
+
|
109
|
+
RUBY_ERROR_RULES = [
|
110
|
+
rule(:start_state, /Error \(.*\):$/, :ruby_before_rails_trace),
|
111
|
+
rule(:ruby_before_rails_trace, /^ $/, :ruby),
|
112
|
+
rule(:ruby_before_rails_trace, /^[\t ]+.*?\.rb:\d+:in `/, :ruby),
|
113
|
+
rule(:ruby, /^[\t ]+.*?\.rb:\d+:in `/, :ruby)
|
114
|
+
].freeze
|
115
|
+
|
116
|
+
RUBY_SEGFAULT_RULES = [
|
117
|
+
rule(:start_state,
|
118
|
+
/:\d+:\s\[BUG\] Segmentation fault/, :ruby_description),
|
119
|
+
rule(:ruby_description, /^ruby\n*/, :ruby_description_end),
|
120
|
+
rule(:ruby_description_end, /\n$/, :ruby_control_frame_begin),
|
121
|
+
rule(:ruby_control_frame_begin,
|
122
|
+
/^-- Control frame information --/, :ruby_control_frames),
|
123
|
+
rule(:ruby_control_frames, /^c:/, :ruby_control_frames),
|
124
|
+
rule(:ruby_control_frames, /^\n$/, :ruby_level_backtrace_frame_start),
|
125
|
+
rule(:ruby_level_backtrace_frame_start,
|
126
|
+
/-- Ruby level backtrace information --/,
|
127
|
+
:ruby_level_backtrace_frames),
|
128
|
+
rule(:ruby_level_backtrace_frames, /:\d+:in /,
|
129
|
+
:ruby_level_backtrace_frames),
|
130
|
+
rule(:ruby_level_backtrace_frames, /^\n$/, :ruby_level_backtrace_end),
|
131
|
+
rule(:ruby_level_backtrace_end,
|
132
|
+
/^-- Machine register context --/, :ruby_machine_registers),
|
133
|
+
rule(:ruby_machine_registers, /: /, :ruby_machine_registers),
|
134
|
+
rule(:ruby_machine_registers, /^\n$/, :ruby_machine_registers_end),
|
135
|
+
rule(:ruby_machine_registers_end,
|
136
|
+
/^-- C level backtrace information --/,
|
137
|
+
:ruby_c_level_backtrace_frames),
|
138
|
+
rule(:ruby_c_level_backtrace_frames, /\[.*\]/,
|
139
|
+
:ruby_c_level_backtrace_frames),
|
140
|
+
rule(:ruby_c_level_backtrace_frames, / .*:\d+$/,
|
141
|
+
:ruby_c_level_backtrace_frames),
|
142
|
+
rule(:ruby_c_level_backtrace_frames, /^\n$/,
|
143
|
+
:ruby_c_level_backtrace_end),
|
144
|
+
rule(:ruby_c_level_backtrace_end,
|
145
|
+
/^-- Other runtime information/, :ruby_other_runtime_info),
|
146
|
+
rule(:ruby_other_runtime_info, /^\n$/, :ruby_other_runtime_info),
|
147
|
+
rule(:ruby_other_runtime_info, /^* Loaded script:/, :ruby_loaded_script),
|
148
|
+
rule(:ruby_loaded_script, /^\n$/, :ruby_loaded_features),
|
149
|
+
rule(:ruby_loaded_features, /^* Loaded features:/, :ruby_loaded_features),
|
150
|
+
rule(:ruby_loaded_features, /^\n$/, :ruby_loaded_features_frames),
|
151
|
+
rule(:ruby_loaded_features_frames,
|
152
|
+
/\d/, :ruby_loaded_features_frames),
|
153
|
+
rule(:ruby_loaded_features_frames,
|
154
|
+
/^\n$/, :ruby_process_memory_map),
|
155
|
+
rule(:ruby_process_memory_map,
|
156
|
+
/^* Process memory map:/, :ruby_process_memory_map),
|
157
|
+
rule(:ruby_process_memory_map,
|
158
|
+
/^\n$/, :ruby_process_memory_map_frames),
|
159
|
+
rule(:ruby_process_memory_map_frames,
|
160
|
+
/\-/, :ruby_process_memory_map_frames),
|
161
|
+
rule(:ruby_process_memory_map_frames, /^\n$/, :start_state)
|
162
|
+
].freeze
|
163
|
+
|
164
|
+
DART_RULES = [
|
165
|
+
rule(:start_state, /^Unhandled exception:$/, :dart_exc),
|
166
|
+
rule(:dart_exc, /^Instance of/, :dart_stack),
|
167
|
+
rule(:dart_exc, /^Exception/, :dart_stack),
|
168
|
+
rule(:dart_exc, /^Bad state/, :dart_stack),
|
169
|
+
rule(:dart_exc, /^IntegerDivisionByZeroException/, :dart_stack),
|
170
|
+
rule(:dart_exc, /^Invalid argument/, :dart_stack),
|
171
|
+
rule(:dart_exc, /^RangeError/, :dart_stack),
|
172
|
+
rule(:dart_exc, /^Assertion failed/, :dart_stack),
|
173
|
+
rule(:dart_exc, /^Cannot instantiate/, :dart_stack),
|
174
|
+
rule(:dart_exc, /^Reading static variable/, :dart_stack),
|
175
|
+
rule(:dart_exc, /^UnimplementedError/, :dart_stack),
|
176
|
+
rule(:dart_exc, /^Unsupported operation/, :dart_stack),
|
177
|
+
rule(:dart_exc, /^Concurrent modification/, :dart_stack),
|
178
|
+
rule(:dart_exc, /^Out of Memory/, :dart_stack),
|
179
|
+
rule(:dart_exc, /^Stack Overflow/, :dart_stack),
|
180
|
+
rule(:dart_exc, /^'.+?':.+?$/, :dart_type_err_1),
|
181
|
+
rule(:dart_type_err_1, /^#\d+\s+.+?\(.+?\)$/, :dart_stack),
|
182
|
+
rule(:dart_type_err_1, /^.+?$/, :dart_type_err_2),
|
183
|
+
rule(:dart_type_err_2, /^.*?\^.*?$/, :dart_type_err_3),
|
184
|
+
rule(:dart_type_err_3, /^$/, :dart_type_err_4),
|
185
|
+
rule(:dart_type_err_4, /^$/, :dart_stack),
|
186
|
+
rule(:dart_exc, /^FormatException/, :dart_format_err_1),
|
187
|
+
rule(:dart_format_err_1, /^#\d+\s+.+?\(.+?\)$/, :dart_stack),
|
188
|
+
rule(:dart_format_err_1, /^./, :dart_format_err_2),
|
189
|
+
rule(:dart_format_err_2, /^.*?\^/, :dart_format_err_3),
|
190
|
+
rule(:dart_format_err_3, /^$/, :dart_stack),
|
191
|
+
rule(:dart_exc, /^NoSuchMethodError:/, :dart_method_err_1),
|
192
|
+
rule(:dart_method_err_1, /^Receiver:/, :dart_method_err_2),
|
193
|
+
rule(:dart_method_err_2, /^Tried calling:/, :dart_method_err_3),
|
194
|
+
rule(:dart_method_err_3, /^Found:/, :dart_stack),
|
195
|
+
rule(:dart_method_err_3, /^#\d+\s+.+?\(.+?\)$/, :dart_stack),
|
196
|
+
rule(:dart_stack, /^#\d+\s+.+?\(.+?\)$/, :dart_stack),
|
197
|
+
rule(:dart_stack, /^<asynchronous suspension>$/, :dart_stack)
|
198
|
+
].freeze
|
199
|
+
|
200
|
+
RUBY_RULES = (
|
201
|
+
RUBY_ERROR_RULES + RUBY_SEGFAULT_RULES
|
202
|
+
).freeze
|
203
|
+
|
204
|
+
ALL_RULES = (
|
205
|
+
JAVA_RULES + PYTHON_RULES + PHP_RULES + GO_RULES + RUBY_RULES + DART_RULES
|
206
|
+
).freeze
|
207
|
+
|
208
|
+
RULES_BY_LANG = {
|
209
|
+
java: JAVA_RULES,
|
210
|
+
javascript: JAVA_RULES,
|
211
|
+
js: JAVA_RULES,
|
212
|
+
csharp: JAVA_RULES,
|
213
|
+
py: PYTHON_RULES,
|
214
|
+
python: PYTHON_RULES,
|
215
|
+
php: PHP_RULES,
|
216
|
+
go: GO_RULES,
|
217
|
+
rb: RUBY_RULES,
|
218
|
+
ruby: RUBY_RULES,
|
219
|
+
dart: DART_RULES,
|
220
|
+
all: ALL_RULES
|
221
|
+
}.freeze
|
222
|
+
|
223
|
+
DEFAULT_FIELDS = %w(message log).freeze
|
224
|
+
end
|
225
|
+
|
226
|
+
# State machine that consumes individual log lines and detects
|
227
|
+
# multi-line stack traces.
|
228
|
+
class ExceptionDetector
|
229
|
+
def initialize(*languages)
|
230
|
+
@state = :start_state
|
231
|
+
@rules = Hash.new { |h, k| h[k] = [] }
|
232
|
+
|
233
|
+
languages = [:all] if languages.empty?
|
234
|
+
|
235
|
+
languages.each do |lang|
|
236
|
+
rule_config =
|
237
|
+
ExceptionDetectorConfig::RULES_BY_LANG.fetch(lang.downcase) do |_k|
|
238
|
+
raise ArgumentError, "Unknown language: #{lang}"
|
239
|
+
end
|
240
|
+
|
241
|
+
rule_config.each do |r|
|
242
|
+
target = ExceptionDetectorConfig::RuleTarget.new(r[:pattern],
|
243
|
+
r[:to_state])
|
244
|
+
r[:from_states].each do |from_state|
|
245
|
+
@rules[from_state] << target
|
246
|
+
end
|
247
|
+
end
|
248
|
+
end
|
249
|
+
|
250
|
+
@rules.each_value(&:uniq!)
|
251
|
+
end
|
252
|
+
|
253
|
+
# Updates the state machine and returns the trace detection status:
|
254
|
+
# - no_trace: 'line' does not belong to an exception trace,
|
255
|
+
# - start_trace: 'line' starts a detected exception trace,
|
256
|
+
# - inside: 'line' is part of a detected exception trace,
|
257
|
+
# - end: the detected exception trace ends after 'line'.
|
258
|
+
def update(line)
|
259
|
+
trace_seen_before = transition(line)
|
260
|
+
# If the state machine fell back to the start state because there is no
|
261
|
+
# defined transition for 'line', trigger another state transition because
|
262
|
+
# 'line' may contain the beginning of another exception.
|
263
|
+
transition(line) unless trace_seen_before
|
264
|
+
new_state = @state
|
265
|
+
trace_seen_after = new_state != :start_state
|
266
|
+
|
267
|
+
case [trace_seen_before, trace_seen_after]
|
268
|
+
when [true, true]
|
269
|
+
:inside_trace
|
270
|
+
when [true, false]
|
271
|
+
:end_trace
|
272
|
+
when [false, true]
|
273
|
+
:start_trace
|
274
|
+
else
|
275
|
+
:no_trace
|
276
|
+
end
|
277
|
+
end
|
278
|
+
|
279
|
+
def reset
|
280
|
+
@state = :start_state
|
281
|
+
end
|
282
|
+
|
283
|
+
private
|
284
|
+
|
285
|
+
# Executes a transition of the state machine for the given line.
|
286
|
+
# Returns false if the line does not match any transition rule and the
|
287
|
+
# state machine was reset to the initial state.
|
288
|
+
def transition(line)
|
289
|
+
@rules[@state].each do |r|
|
290
|
+
next unless line =~ r.pattern
|
291
|
+
@state = r.to_state
|
292
|
+
return true
|
293
|
+
end
|
294
|
+
@state = :start_state
|
295
|
+
false
|
296
|
+
end
|
297
|
+
end
|
298
|
+
|
299
|
+
# Buffers and groups log records if they contain exception stack traces.
|
300
|
+
class TraceAccumulator
|
301
|
+
attr_reader :buffer_start_time
|
302
|
+
|
303
|
+
# If message_field is nil, the instance is set up to accumulate
|
304
|
+
# records that are plain strings (i.e. the whole record is concatenated).
|
305
|
+
# Otherwise, the instance accepts records that are dictionaries (usually
|
306
|
+
# originating from structured JSON logs) and accumulates just the
|
307
|
+
# content of the given message field.
|
308
|
+
# message_field may contain the empty string. In this case, the
|
309
|
+
# TraceAccumulator 'learns' the field name from the first record by checking
|
310
|
+
# for some pre-defined common field names of text logs.
|
311
|
+
# The option parameter can be used to pass the following parameters:
|
312
|
+
# force_line_breaks adds line breaks when combining exception stacks
|
313
|
+
# max_lines and max_bytes limit the maximum amount
|
314
|
+
# of data to be buffered. The default value 0 indicates 'no limit'.
|
315
|
+
def initialize(message_field, languages, **options, &emit_callback)
|
316
|
+
@exception_detector = Fluent::ExceptionDetector.new(*languages)
|
317
|
+
@message_field = message_field
|
318
|
+
@force_line_breaks = options[:force_line_breaks] || false
|
319
|
+
@max_lines = options[:max_lines] || 0
|
320
|
+
@max_bytes = options[:max_bytes] || 0
|
321
|
+
@emit = emit_callback
|
322
|
+
@messages = []
|
323
|
+
@buffer_start_time = Time.now
|
324
|
+
@buffer_size = 0
|
325
|
+
@first_record = nil
|
326
|
+
@first_timestamp = nil
|
327
|
+
end
|
328
|
+
|
329
|
+
def push(time_sec, record)
|
330
|
+
message = extract_message(record)
|
331
|
+
if message.nil?
|
332
|
+
@exception_detector.reset
|
333
|
+
detection_status = :no_trace
|
334
|
+
else
|
335
|
+
force_flush if @max_bytes > 0 &&
|
336
|
+
@buffer_size + message.length > @max_bytes
|
337
|
+
detection_status = @exception_detector.update(message)
|
338
|
+
end
|
339
|
+
|
340
|
+
update_buffer(detection_status, time_sec, record, message)
|
341
|
+
|
342
|
+
force_flush if @max_lines > 0 && @messages.length == @max_lines
|
343
|
+
end
|
344
|
+
|
345
|
+
def flush
|
346
|
+
case @messages.length
|
347
|
+
when 0
|
348
|
+
return
|
349
|
+
when 1
|
350
|
+
@emit.call(@first_timestamp, @first_record)
|
351
|
+
else
|
352
|
+
combined_message = @messages.join
|
353
|
+
if @message_field.nil?
|
354
|
+
output_record = combined_message
|
355
|
+
else
|
356
|
+
output_record = @first_record
|
357
|
+
output_record[@message_field] = combined_message
|
358
|
+
end
|
359
|
+
@emit.call(@first_timestamp, output_record)
|
360
|
+
end
|
361
|
+
@messages = []
|
362
|
+
@first_record = nil
|
363
|
+
@first_timestamp = nil
|
364
|
+
@buffer_size = 0
|
365
|
+
end
|
366
|
+
|
367
|
+
def force_flush
|
368
|
+
flush
|
369
|
+
@exception_detector.reset
|
370
|
+
end
|
371
|
+
|
372
|
+
private
|
373
|
+
|
374
|
+
def extract_message(record)
|
375
|
+
if !@message_field.nil? && @message_field.empty?
|
376
|
+
ExceptionDetectorConfig::DEFAULT_FIELDS.each do |f|
|
377
|
+
if record.key?(f)
|
378
|
+
@message_field = f
|
379
|
+
break
|
380
|
+
end
|
381
|
+
end
|
382
|
+
end
|
383
|
+
@message_field.nil? ? record : record[@message_field]
|
384
|
+
end
|
385
|
+
|
386
|
+
def update_buffer(detection_status, time_sec, record, message)
|
387
|
+
trigger_emit = detection_status == :no_trace ||
|
388
|
+
detection_status == :end_trace
|
389
|
+
if @messages.empty? && trigger_emit
|
390
|
+
@emit.call(time_sec, record)
|
391
|
+
return
|
392
|
+
end
|
393
|
+
|
394
|
+
case detection_status
|
395
|
+
when :inside_trace
|
396
|
+
add(time_sec, record, message)
|
397
|
+
when :end_trace
|
398
|
+
add(time_sec, record, message)
|
399
|
+
flush
|
400
|
+
when :no_trace
|
401
|
+
flush
|
402
|
+
add(time_sec, record, message)
|
403
|
+
flush
|
404
|
+
when :start_trace
|
405
|
+
flush
|
406
|
+
add(time_sec, record, message)
|
407
|
+
end
|
408
|
+
end
|
409
|
+
|
410
|
+
def add(time_sec, record, message)
|
411
|
+
if @messages.empty?
|
412
|
+
@first_record = record unless @message_field.nil?
|
413
|
+
@first_timestamp = time_sec
|
414
|
+
@buffer_start_time = Time.now
|
415
|
+
end
|
416
|
+
unless message.nil?
|
417
|
+
message_with_line_break =
|
418
|
+
if @force_line_breaks && !@messages.empty? && !message.include?("\n")
|
419
|
+
"\n" + message
|
420
|
+
else
|
421
|
+
message
|
422
|
+
end
|
423
|
+
@messages << message_with_line_break
|
424
|
+
@buffer_size += message_with_line_break.length
|
425
|
+
end
|
426
|
+
end
|
427
|
+
end
|
428
|
+
end
|
@@ -0,0 +1,143 @@
|
|
1
|
+
#
|
2
|
+
# Copyright 2016 Google Inc. All rights reserved.
|
3
|
+
#
|
4
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
5
|
+
# you may not use this file except in compliance with the License.
|
6
|
+
# You may obtain a copy of the License at
|
7
|
+
#
|
8
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
9
|
+
#
|
10
|
+
# Unless required by applicable law or agreed to in writing, software
|
11
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
12
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
13
|
+
# See the License for the specific language governing permissions and
|
14
|
+
# limitations under the License.
|
15
|
+
|
16
|
+
require 'fluent/plugin/exception_detector'
|
17
|
+
require 'fluent/output'
|
18
|
+
|
19
|
+
module Fluent
|
20
|
+
# This output plugin consumes a log stream of JSON objects which contain
|
21
|
+
# single-line log messages. If a consecutive sequence of log messages form
|
22
|
+
# an exception stack trace, they forwarded as a single, combined JSON
|
23
|
+
# object. Otherwise, the input log data is forwarded as is.
|
24
|
+
class DetectExceptionsOutput < Output
|
25
|
+
desc 'The prefix to be removed from the input tag when outputting a record.'
|
26
|
+
config_param :remove_tag_prefix, :string
|
27
|
+
desc 'The field which contains the raw message text in the input JSON data.'
|
28
|
+
config_param :message, :string, default: ''
|
29
|
+
desc 'The interval of flushing the buffer for multiline format.'
|
30
|
+
config_param :multiline_flush_interval, :time, default: nil
|
31
|
+
desc 'Programming languages for which to detect exceptions. Default: all.'
|
32
|
+
config_param :languages, :array, value_type: :string, default: []
|
33
|
+
desc 'Force live breaks when combining exception stacks. Default: false.'
|
34
|
+
config_param :force_line_breaks, :bool, default: false
|
35
|
+
desc 'Maximum number of lines to flush (0 means no limit). Default: 1000.'
|
36
|
+
config_param :max_lines, :integer, default: 1000
|
37
|
+
desc 'Maximum number of bytes to flush (0 means no limit). Default: 0.'
|
38
|
+
config_param :max_bytes, :integer, default: 0
|
39
|
+
desc 'Separate log streams by this field in the input JSON data.'
|
40
|
+
config_param :stream, :string, default: ''
|
41
|
+
|
42
|
+
Fluent::Plugin.register_output('detect_exceptions', self)
|
43
|
+
|
44
|
+
def configure(conf)
|
45
|
+
super
|
46
|
+
|
47
|
+
if multiline_flush_interval
|
48
|
+
@check_flush_interval = [multiline_flush_interval * 0.1, 1].max
|
49
|
+
end
|
50
|
+
|
51
|
+
@languages = languages.map(&:to_sym)
|
52
|
+
|
53
|
+
# Maps log stream tags to a corresponding TraceAccumulator.
|
54
|
+
@accumulators = {}
|
55
|
+
end
|
56
|
+
|
57
|
+
def start
|
58
|
+
super
|
59
|
+
|
60
|
+
if multiline_flush_interval
|
61
|
+
@flush_buffer_mutex = Mutex.new
|
62
|
+
@stop_check = false
|
63
|
+
@thread = Thread.new(&method(:check_flush_loop))
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
def before_shutdown
|
68
|
+
flush_buffers
|
69
|
+
super if defined?(super)
|
70
|
+
end
|
71
|
+
|
72
|
+
def shutdown
|
73
|
+
# Before shutdown is not available in older fluentd versions.
|
74
|
+
# Hence, we make sure that we flush the buffers here as well.
|
75
|
+
flush_buffers
|
76
|
+
@thread.join if @multiline_flush_interval
|
77
|
+
super
|
78
|
+
end
|
79
|
+
|
80
|
+
def emit(tag, es, chain)
|
81
|
+
es.each do |time_sec, record|
|
82
|
+
process_record(tag, time_sec, record)
|
83
|
+
end
|
84
|
+
chain.next
|
85
|
+
end
|
86
|
+
|
87
|
+
private
|
88
|
+
|
89
|
+
def process_record(tag, time_sec, record)
|
90
|
+
synchronize do
|
91
|
+
log_id = [tag]
|
92
|
+
log_id.push(record.fetch(@stream, '')) unless @stream.empty?
|
93
|
+
unless @accumulators.key?(log_id)
|
94
|
+
out_tag = tag.sub(/^#{Regexp.escape(@remove_tag_prefix)}\./, '')
|
95
|
+
@accumulators[log_id] =
|
96
|
+
Fluent::TraceAccumulator.new(
|
97
|
+
@message,
|
98
|
+
@languages,
|
99
|
+
force_line_breaks: @force_line_breaks,
|
100
|
+
max_lines: @max_lines,
|
101
|
+
max_bytes: @max_bytes
|
102
|
+
) do |t, r|
|
103
|
+
router.emit(out_tag, t, r)
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
@accumulators[log_id].push(time_sec, record)
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
def flush_buffers
|
112
|
+
synchronize do
|
113
|
+
@stop_check = true
|
114
|
+
@accumulators.each_value(&:force_flush)
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
def check_flush_loop
|
119
|
+
@flush_buffer_mutex.synchronize do
|
120
|
+
loop do
|
121
|
+
@flush_buffer_mutex.sleep(@check_flush_interval)
|
122
|
+
now = Time.now
|
123
|
+
break if @stop_check
|
124
|
+
@accumulators.each_value do |acc|
|
125
|
+
acc.force_flush if now - acc.buffer_start_time >
|
126
|
+
@multiline_flush_interval
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
130
|
+
rescue
|
131
|
+
log.error 'error in check_flush_loop', error: $ERROR_INFO.to_s
|
132
|
+
log.error_backtrace
|
133
|
+
end
|
134
|
+
|
135
|
+
def synchronize(&block)
|
136
|
+
if @multiline_flush_interval
|
137
|
+
@flush_buffer_mutex.synchronize(&block)
|
138
|
+
else
|
139
|
+
yield
|
140
|
+
end
|
141
|
+
end
|
142
|
+
end
|
143
|
+
end
|
data/test/helper.rb
ADDED
@@ -0,0 +1,46 @@
|
|
1
|
+
# Copyright 2016 Google Inc. All rights reserved.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
require 'rubygems'
|
16
|
+
require 'bundler'
|
17
|
+
|
18
|
+
begin
|
19
|
+
Bundler.setup(:default, :development)
|
20
|
+
rescue Bundler::BundlerError => e
|
21
|
+
$stderr.puts e.message
|
22
|
+
$stderr.puts 'Run `bundle install` to install missing gems'
|
23
|
+
exit e.status_code
|
24
|
+
end
|
25
|
+
|
26
|
+
require 'test/unit'
|
27
|
+
|
28
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
29
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
30
|
+
require 'fluent/test'
|
31
|
+
|
32
|
+
unless ENV.key?('VERBOSE')
|
33
|
+
nulllogger = Object.new
|
34
|
+
nulllogger.instance_eval do |_|
|
35
|
+
def respond_to_missing?
|
36
|
+
true
|
37
|
+
end
|
38
|
+
|
39
|
+
def method_missing(_method, *_args) # rubocop:disable Style/MethodMissing
|
40
|
+
end
|
41
|
+
end
|
42
|
+
# global $log variable is used by fluentd
|
43
|
+
$log = nulllogger # rubocop:disable Style/GlobalVars
|
44
|
+
end
|
45
|
+
|
46
|
+
require 'fluent/plugin/out_detect_exceptions'
|
@@ -0,0 +1,73 @@
|
|
1
|
+
# Copyright 2016 Google Inc. All rights reserved.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
require 'benchmark'
|
16
|
+
|
17
|
+
require 'fluent/plugin/exception_detector'
|
18
|
+
|
19
|
+
size_in_m = 25
|
20
|
+
line_length = 50
|
21
|
+
|
22
|
+
size = size_in_m << 20
|
23
|
+
|
24
|
+
JAVA_EXC = <<END.freeze
|
25
|
+
Jul 09, 2015 3:23:29 PM com.google.devtools.search.cloud.feeder.MakeLog: RuntimeException: Run from this message!
|
26
|
+
at com.my.app.Object.do$a1(MakeLog.java:50)
|
27
|
+
at java.lang.Thing.call(Thing.java:10)
|
28
|
+
at com.my.app.Object.help(MakeLog.java:40)
|
29
|
+
at sun.javax.API.method(API.java:100)
|
30
|
+
at com.jetty.Framework.main(MakeLog.java:30)
|
31
|
+
END
|
32
|
+
|
33
|
+
PYTHON_EXC = <<END.freeze
|
34
|
+
Traceback (most recent call last):
|
35
|
+
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.5.2/webapp2.py", line 1535, in __call__
|
36
|
+
rv = self.handle_exception(request, response, e)
|
37
|
+
File "/base/data/home/apps/s~nearfieldspy/1.378705245900539993/nearfieldspy.py", line 17, in start
|
38
|
+
return get()
|
39
|
+
File "/base/data/home/apps/s~nearfieldspy/1.378705245900539993/nearfieldspy.py", line 5, in get
|
40
|
+
raise Exception('spam', 'eggs')
|
41
|
+
Exception: ('spam', 'eggs')
|
42
|
+
END
|
43
|
+
|
44
|
+
chars = [('a'..'z'), ('A'..'Z')].map(&:to_a).flatten
|
45
|
+
|
46
|
+
random_text = (1..(size / line_length)).collect do
|
47
|
+
(0...line_length).map { chars[rand(chars.length)] }.join
|
48
|
+
end
|
49
|
+
|
50
|
+
exceptions = {
|
51
|
+
java: (JAVA_EXC * (size / JAVA_EXC.length)).lines,
|
52
|
+
python: (PYTHON_EXC * (size / PYTHON_EXC.length)).lines
|
53
|
+
}
|
54
|
+
|
55
|
+
puts "Start benchmark. Input size #{size_in_m}M."
|
56
|
+
Benchmark.bm do |x|
|
57
|
+
languages = Fluent::ExceptionDetectorConfig::RULES_BY_LANG.keys
|
58
|
+
languages.each do |lang|
|
59
|
+
buffer = Fluent::TraceAccumulator.new(nil, lang) {}
|
60
|
+
x.report("#{lang}_detector_random_text") do
|
61
|
+
random_text.each { |l| buffer.push(0, l) }
|
62
|
+
end
|
63
|
+
end
|
64
|
+
[:java, :python, :all].each do |detector_lang|
|
65
|
+
buffer = Fluent::TraceAccumulator.new(nil, detector_lang) {}
|
66
|
+
exc_languages = detector_lang == :all ? exceptions.keys : [detector_lang]
|
67
|
+
exc_languages.each do |exc_lang|
|
68
|
+
x.report("#{detector_lang}_detector_#{exc_lang}_stacks") do
|
69
|
+
exceptions[exc_lang].each { |l| buffer.push(0, l) }
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|