gitlab-fluent-plugin-detect-exceptions 0.0.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,428 @@
1
+ # Copyright 2016 Google Inc. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+ #
15
+ module Fluent
16
+ Struct.new('Rule', :from_states, :pattern, :to_state)
17
+
18
+ # Configuration of the state machine that detects exceptions.
19
+ module ExceptionDetectorConfig # rubocop:disable Metrics/ModuleLength
20
+ # Rule for a state transition: if pattern matches go to the given state.
21
+ class RuleTarget
22
+ attr_accessor :pattern, :to_state
23
+
24
+ def initialize(p, s)
25
+ @pattern = p
26
+ @to_state = s
27
+ end
28
+
29
+ def ==(other)
30
+ other.class == self.class && other.state == state
31
+ end
32
+
33
+ alias eql? ==
34
+
35
+ def hash
36
+ state.hash
37
+ end
38
+
39
+ def state
40
+ [@pattern, @to_state]
41
+ end
42
+ end
43
+
44
+ def self.rule(from_state_or_states, pattern, to_state)
45
+ from_state_or_states = [from_state_or_states] unless
46
+ from_state_or_states.is_a?(Array)
47
+ Struct::Rule.new(from_state_or_states, pattern, to_state)
48
+ end
49
+
50
+ def self.supported
51
+ RULES_BY_LANG.keys
52
+ end
53
+
54
+ JAVA_RULES = [
55
+ rule([:start_state, :java_start_exception],
56
+ /(?:Exception|Error|Throwable|V8 errors stack trace)[:\r\n]/,
57
+ :java_after_exception),
58
+ rule(:java_after_exception, /^[\t ]*nested exception is:[\t ]*/,
59
+ :java_start_exception),
60
+ rule(:java_after_exception, /^[\r\n]*$/, :java_after_exception),
61
+ rule([:java_after_exception, :java], /^[\t ]+(?:eval )?at /, :java),
62
+
63
+ rule([:java_after_exception, :java],
64
+ # C# nested exception.
65
+ /^[\t ]+--- End of inner exception stack trace ---$/,
66
+ :java),
67
+
68
+ rule([:java_after_exception, :java],
69
+ # C# exception from async code.
70
+ /^--- End of stack trace from previous (?x:
71
+ )location where exception was thrown ---$/,
72
+ :java),
73
+
74
+ rule([:java_after_exception, :java], /^[\t ]*(?:Caused by|Suppressed):/,
75
+ :java_after_exception),
76
+ rule([:java_after_exception, :java],
77
+ /^[\t ]*... \d+ (?:more|common frames omitted)/, :java)
78
+ ].freeze
79
+
80
+ PYTHON_RULES = [
81
+ rule(:start_state, /^Traceback \(most recent call last\):$/, :python),
82
+ rule(:python, /^[\t ]+File /, :python_code),
83
+ rule(:python_code, /[^\t ]/, :python),
84
+ rule(:python, /^(?:[^\s.():]+\.)*[^\s.():]+:/, :start_state)
85
+ ].freeze
86
+
87
+ PHP_RULES = [
88
+ rule(:start_state, /
89
+ (?:PHP\ (?:Notice|Parse\ error|Fatal\ error|Warning):)|
90
+ (?:exception\ '[^']+'\ with\ message\ ')/x, :php_stack_begin),
91
+ rule(:php_stack_begin, /^Stack trace:/, :php_stack_frames),
92
+ rule(:php_stack_frames, /^#\d/, :php_stack_frames),
93
+ rule(:php_stack_frames, /^\s+thrown in /, :start_state)
94
+ ].freeze
95
+
96
+ GO_RULES = [
97
+ rule(:start_state, /\bpanic: /, :go_after_panic),
98
+ rule(:start_state, /http: panic serving/, :go_goroutine),
99
+ rule(:go_after_panic, /^$/, :go_goroutine),
100
+ rule([:go_after_panic, :go_after_signal, :go_frame_1],
101
+ /^$/, :go_goroutine),
102
+ rule(:go_after_panic, /^\[signal /, :go_after_signal),
103
+ rule(:go_goroutine, /^goroutine \d+ \[[^\]]+\]:$/, :go_frame_1),
104
+ rule(:go_frame_1, /^(?:[^\s.:]+\.)*[^\s.():]+\(|^created by /,
105
+ :go_frame_2),
106
+ rule(:go_frame_2, /^\s/, :go_frame_1)
107
+ ].freeze
108
+
109
+ RUBY_ERROR_RULES = [
110
+ rule(:start_state, /Error \(.*\):$/, :ruby_before_rails_trace),
111
+ rule(:ruby_before_rails_trace, /^ $/, :ruby),
112
+ rule(:ruby_before_rails_trace, /^[\t ]+.*?\.rb:\d+:in `/, :ruby),
113
+ rule(:ruby, /^[\t ]+.*?\.rb:\d+:in `/, :ruby)
114
+ ].freeze
115
+
116
+ RUBY_SEGFAULT_RULES = [
117
+ rule(:start_state,
118
+ /:\d+:\s\[BUG\] Segmentation fault/, :ruby_description),
119
+ rule(:ruby_description, /^ruby\n*/, :ruby_description_end),
120
+ rule(:ruby_description_end, /\n$/, :ruby_control_frame_begin),
121
+ rule(:ruby_control_frame_begin,
122
+ /^-- Control frame information --/, :ruby_control_frames),
123
+ rule(:ruby_control_frames, /^c:/, :ruby_control_frames),
124
+ rule(:ruby_control_frames, /^\n$/, :ruby_level_backtrace_frame_start),
125
+ rule(:ruby_level_backtrace_frame_start,
126
+ /-- Ruby level backtrace information --/,
127
+ :ruby_level_backtrace_frames),
128
+ rule(:ruby_level_backtrace_frames, /:\d+:in /,
129
+ :ruby_level_backtrace_frames),
130
+ rule(:ruby_level_backtrace_frames, /^\n$/, :ruby_level_backtrace_end),
131
+ rule(:ruby_level_backtrace_end,
132
+ /^-- Machine register context --/, :ruby_machine_registers),
133
+ rule(:ruby_machine_registers, /: /, :ruby_machine_registers),
134
+ rule(:ruby_machine_registers, /^\n$/, :ruby_machine_registers_end),
135
+ rule(:ruby_machine_registers_end,
136
+ /^-- C level backtrace information --/,
137
+ :ruby_c_level_backtrace_frames),
138
+ rule(:ruby_c_level_backtrace_frames, /\[.*\]/,
139
+ :ruby_c_level_backtrace_frames),
140
+ rule(:ruby_c_level_backtrace_frames, / .*:\d+$/,
141
+ :ruby_c_level_backtrace_frames),
142
+ rule(:ruby_c_level_backtrace_frames, /^\n$/,
143
+ :ruby_c_level_backtrace_end),
144
+ rule(:ruby_c_level_backtrace_end,
145
+ /^-- Other runtime information/, :ruby_other_runtime_info),
146
+ rule(:ruby_other_runtime_info, /^\n$/, :ruby_other_runtime_info),
147
+ rule(:ruby_other_runtime_info, /^* Loaded script:/, :ruby_loaded_script),
148
+ rule(:ruby_loaded_script, /^\n$/, :ruby_loaded_features),
149
+ rule(:ruby_loaded_features, /^* Loaded features:/, :ruby_loaded_features),
150
+ rule(:ruby_loaded_features, /^\n$/, :ruby_loaded_features_frames),
151
+ rule(:ruby_loaded_features_frames,
152
+ /\d/, :ruby_loaded_features_frames),
153
+ rule(:ruby_loaded_features_frames,
154
+ /^\n$/, :ruby_process_memory_map),
155
+ rule(:ruby_process_memory_map,
156
+ /^* Process memory map:/, :ruby_process_memory_map),
157
+ rule(:ruby_process_memory_map,
158
+ /^\n$/, :ruby_process_memory_map_frames),
159
+ rule(:ruby_process_memory_map_frames,
160
+ /\-/, :ruby_process_memory_map_frames),
161
+ rule(:ruby_process_memory_map_frames, /^\n$/, :start_state)
162
+ ].freeze
163
+
164
+ DART_RULES = [
165
+ rule(:start_state, /^Unhandled exception:$/, :dart_exc),
166
+ rule(:dart_exc, /^Instance of/, :dart_stack),
167
+ rule(:dart_exc, /^Exception/, :dart_stack),
168
+ rule(:dart_exc, /^Bad state/, :dart_stack),
169
+ rule(:dart_exc, /^IntegerDivisionByZeroException/, :dart_stack),
170
+ rule(:dart_exc, /^Invalid argument/, :dart_stack),
171
+ rule(:dart_exc, /^RangeError/, :dart_stack),
172
+ rule(:dart_exc, /^Assertion failed/, :dart_stack),
173
+ rule(:dart_exc, /^Cannot instantiate/, :dart_stack),
174
+ rule(:dart_exc, /^Reading static variable/, :dart_stack),
175
+ rule(:dart_exc, /^UnimplementedError/, :dart_stack),
176
+ rule(:dart_exc, /^Unsupported operation/, :dart_stack),
177
+ rule(:dart_exc, /^Concurrent modification/, :dart_stack),
178
+ rule(:dart_exc, /^Out of Memory/, :dart_stack),
179
+ rule(:dart_exc, /^Stack Overflow/, :dart_stack),
180
+ rule(:dart_exc, /^'.+?':.+?$/, :dart_type_err_1),
181
+ rule(:dart_type_err_1, /^#\d+\s+.+?\(.+?\)$/, :dart_stack),
182
+ rule(:dart_type_err_1, /^.+?$/, :dart_type_err_2),
183
+ rule(:dart_type_err_2, /^.*?\^.*?$/, :dart_type_err_3),
184
+ rule(:dart_type_err_3, /^$/, :dart_type_err_4),
185
+ rule(:dart_type_err_4, /^$/, :dart_stack),
186
+ rule(:dart_exc, /^FormatException/, :dart_format_err_1),
187
+ rule(:dart_format_err_1, /^#\d+\s+.+?\(.+?\)$/, :dart_stack),
188
+ rule(:dart_format_err_1, /^./, :dart_format_err_2),
189
+ rule(:dart_format_err_2, /^.*?\^/, :dart_format_err_3),
190
+ rule(:dart_format_err_3, /^$/, :dart_stack),
191
+ rule(:dart_exc, /^NoSuchMethodError:/, :dart_method_err_1),
192
+ rule(:dart_method_err_1, /^Receiver:/, :dart_method_err_2),
193
+ rule(:dart_method_err_2, /^Tried calling:/, :dart_method_err_3),
194
+ rule(:dart_method_err_3, /^Found:/, :dart_stack),
195
+ rule(:dart_method_err_3, /^#\d+\s+.+?\(.+?\)$/, :dart_stack),
196
+ rule(:dart_stack, /^#\d+\s+.+?\(.+?\)$/, :dart_stack),
197
+ rule(:dart_stack, /^<asynchronous suspension>$/, :dart_stack)
198
+ ].freeze
199
+
200
+ RUBY_RULES = (
201
+ RUBY_ERROR_RULES + RUBY_SEGFAULT_RULES
202
+ ).freeze
203
+
204
+ ALL_RULES = (
205
+ JAVA_RULES + PYTHON_RULES + PHP_RULES + GO_RULES + RUBY_RULES + DART_RULES
206
+ ).freeze
207
+
208
+ RULES_BY_LANG = {
209
+ java: JAVA_RULES,
210
+ javascript: JAVA_RULES,
211
+ js: JAVA_RULES,
212
+ csharp: JAVA_RULES,
213
+ py: PYTHON_RULES,
214
+ python: PYTHON_RULES,
215
+ php: PHP_RULES,
216
+ go: GO_RULES,
217
+ rb: RUBY_RULES,
218
+ ruby: RUBY_RULES,
219
+ dart: DART_RULES,
220
+ all: ALL_RULES
221
+ }.freeze
222
+
223
+ DEFAULT_FIELDS = %w(message log).freeze
224
+ end
225
+
226
+ # State machine that consumes individual log lines and detects
227
+ # multi-line stack traces.
228
+ class ExceptionDetector
229
+ def initialize(*languages)
230
+ @state = :start_state
231
+ @rules = Hash.new { |h, k| h[k] = [] }
232
+
233
+ languages = [:all] if languages.empty?
234
+
235
+ languages.each do |lang|
236
+ rule_config =
237
+ ExceptionDetectorConfig::RULES_BY_LANG.fetch(lang.downcase) do |_k|
238
+ raise ArgumentError, "Unknown language: #{lang}"
239
+ end
240
+
241
+ rule_config.each do |r|
242
+ target = ExceptionDetectorConfig::RuleTarget.new(r[:pattern],
243
+ r[:to_state])
244
+ r[:from_states].each do |from_state|
245
+ @rules[from_state] << target
246
+ end
247
+ end
248
+ end
249
+
250
+ @rules.each_value(&:uniq!)
251
+ end
252
+
253
+ # Updates the state machine and returns the trace detection status:
254
+ # - no_trace: 'line' does not belong to an exception trace,
255
+ # - start_trace: 'line' starts a detected exception trace,
256
+ # - inside: 'line' is part of a detected exception trace,
257
+ # - end: the detected exception trace ends after 'line'.
258
+ def update(line)
259
+ trace_seen_before = transition(line)
260
+ # If the state machine fell back to the start state because there is no
261
+ # defined transition for 'line', trigger another state transition because
262
+ # 'line' may contain the beginning of another exception.
263
+ transition(line) unless trace_seen_before
264
+ new_state = @state
265
+ trace_seen_after = new_state != :start_state
266
+
267
+ case [trace_seen_before, trace_seen_after]
268
+ when [true, true]
269
+ :inside_trace
270
+ when [true, false]
271
+ :end_trace
272
+ when [false, true]
273
+ :start_trace
274
+ else
275
+ :no_trace
276
+ end
277
+ end
278
+
279
+ def reset
280
+ @state = :start_state
281
+ end
282
+
283
+ private
284
+
285
+ # Executes a transition of the state machine for the given line.
286
+ # Returns false if the line does not match any transition rule and the
287
+ # state machine was reset to the initial state.
288
+ def transition(line)
289
+ @rules[@state].each do |r|
290
+ next unless line =~ r.pattern
291
+ @state = r.to_state
292
+ return true
293
+ end
294
+ @state = :start_state
295
+ false
296
+ end
297
+ end
298
+
299
+ # Buffers and groups log records if they contain exception stack traces.
300
+ class TraceAccumulator
301
+ attr_reader :buffer_start_time
302
+
303
+ # If message_field is nil, the instance is set up to accumulate
304
+ # records that are plain strings (i.e. the whole record is concatenated).
305
+ # Otherwise, the instance accepts records that are dictionaries (usually
306
+ # originating from structured JSON logs) and accumulates just the
307
+ # content of the given message field.
308
+ # message_field may contain the empty string. In this case, the
309
+ # TraceAccumulator 'learns' the field name from the first record by checking
310
+ # for some pre-defined common field names of text logs.
311
+ # The option parameter can be used to pass the following parameters:
312
+ # force_line_breaks adds line breaks when combining exception stacks
313
+ # max_lines and max_bytes limit the maximum amount
314
+ # of data to be buffered. The default value 0 indicates 'no limit'.
315
+ def initialize(message_field, languages, **options, &emit_callback)
316
+ @exception_detector = Fluent::ExceptionDetector.new(*languages)
317
+ @message_field = message_field
318
+ @force_line_breaks = options[:force_line_breaks] || false
319
+ @max_lines = options[:max_lines] || 0
320
+ @max_bytes = options[:max_bytes] || 0
321
+ @emit = emit_callback
322
+ @messages = []
323
+ @buffer_start_time = Time.now
324
+ @buffer_size = 0
325
+ @first_record = nil
326
+ @first_timestamp = nil
327
+ end
328
+
329
+ def push(time_sec, record)
330
+ message = extract_message(record)
331
+ if message.nil?
332
+ @exception_detector.reset
333
+ detection_status = :no_trace
334
+ else
335
+ force_flush if @max_bytes > 0 &&
336
+ @buffer_size + message.length > @max_bytes
337
+ detection_status = @exception_detector.update(message)
338
+ end
339
+
340
+ update_buffer(detection_status, time_sec, record, message)
341
+
342
+ force_flush if @max_lines > 0 && @messages.length == @max_lines
343
+ end
344
+
345
+ def flush
346
+ case @messages.length
347
+ when 0
348
+ return
349
+ when 1
350
+ @emit.call(@first_timestamp, @first_record)
351
+ else
352
+ combined_message = @messages.join
353
+ if @message_field.nil?
354
+ output_record = combined_message
355
+ else
356
+ output_record = @first_record
357
+ output_record[@message_field] = combined_message
358
+ end
359
+ @emit.call(@first_timestamp, output_record)
360
+ end
361
+ @messages = []
362
+ @first_record = nil
363
+ @first_timestamp = nil
364
+ @buffer_size = 0
365
+ end
366
+
367
+ def force_flush
368
+ flush
369
+ @exception_detector.reset
370
+ end
371
+
372
+ private
373
+
374
+ def extract_message(record)
375
+ if !@message_field.nil? && @message_field.empty?
376
+ ExceptionDetectorConfig::DEFAULT_FIELDS.each do |f|
377
+ if record.key?(f)
378
+ @message_field = f
379
+ break
380
+ end
381
+ end
382
+ end
383
+ @message_field.nil? ? record : record[@message_field]
384
+ end
385
+
386
+ def update_buffer(detection_status, time_sec, record, message)
387
+ trigger_emit = detection_status == :no_trace ||
388
+ detection_status == :end_trace
389
+ if @messages.empty? && trigger_emit
390
+ @emit.call(time_sec, record)
391
+ return
392
+ end
393
+
394
+ case detection_status
395
+ when :inside_trace
396
+ add(time_sec, record, message)
397
+ when :end_trace
398
+ add(time_sec, record, message)
399
+ flush
400
+ when :no_trace
401
+ flush
402
+ add(time_sec, record, message)
403
+ flush
404
+ when :start_trace
405
+ flush
406
+ add(time_sec, record, message)
407
+ end
408
+ end
409
+
410
+ def add(time_sec, record, message)
411
+ if @messages.empty?
412
+ @first_record = record unless @message_field.nil?
413
+ @first_timestamp = time_sec
414
+ @buffer_start_time = Time.now
415
+ end
416
+ unless message.nil?
417
+ message_with_line_break =
418
+ if @force_line_breaks && !@messages.empty? && !message.include?("\n")
419
+ "\n" + message
420
+ else
421
+ message
422
+ end
423
+ @messages << message_with_line_break
424
+ @buffer_size += message_with_line_break.length
425
+ end
426
+ end
427
+ end
428
+ end
@@ -0,0 +1,143 @@
1
+ #
2
+ # Copyright 2016 Google Inc. All rights reserved.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License");
5
+ # you may not use this file except in compliance with the License.
6
+ # You may obtain a copy of the License at
7
+ #
8
+ # http://www.apache.org/licenses/LICENSE-2.0
9
+ #
10
+ # Unless required by applicable law or agreed to in writing, software
11
+ # distributed under the License is distributed on an "AS IS" BASIS,
12
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+ # See the License for the specific language governing permissions and
14
+ # limitations under the License.
15
+
16
+ require 'fluent/plugin/exception_detector'
17
+ require 'fluent/output'
18
+
19
+ module Fluent
20
+ # This output plugin consumes a log stream of JSON objects which contain
21
+ # single-line log messages. If a consecutive sequence of log messages form
22
+ # an exception stack trace, they forwarded as a single, combined JSON
23
+ # object. Otherwise, the input log data is forwarded as is.
24
+ class DetectExceptionsOutput < Output
25
+ desc 'The prefix to be removed from the input tag when outputting a record.'
26
+ config_param :remove_tag_prefix, :string
27
+ desc 'The field which contains the raw message text in the input JSON data.'
28
+ config_param :message, :string, default: ''
29
+ desc 'The interval of flushing the buffer for multiline format.'
30
+ config_param :multiline_flush_interval, :time, default: nil
31
+ desc 'Programming languages for which to detect exceptions. Default: all.'
32
+ config_param :languages, :array, value_type: :string, default: []
33
+ desc 'Force live breaks when combining exception stacks. Default: false.'
34
+ config_param :force_line_breaks, :bool, default: false
35
+ desc 'Maximum number of lines to flush (0 means no limit). Default: 1000.'
36
+ config_param :max_lines, :integer, default: 1000
37
+ desc 'Maximum number of bytes to flush (0 means no limit). Default: 0.'
38
+ config_param :max_bytes, :integer, default: 0
39
+ desc 'Separate log streams by this field in the input JSON data.'
40
+ config_param :stream, :string, default: ''
41
+
42
+ Fluent::Plugin.register_output('detect_exceptions', self)
43
+
44
+ def configure(conf)
45
+ super
46
+
47
+ if multiline_flush_interval
48
+ @check_flush_interval = [multiline_flush_interval * 0.1, 1].max
49
+ end
50
+
51
+ @languages = languages.map(&:to_sym)
52
+
53
+ # Maps log stream tags to a corresponding TraceAccumulator.
54
+ @accumulators = {}
55
+ end
56
+
57
+ def start
58
+ super
59
+
60
+ if multiline_flush_interval
61
+ @flush_buffer_mutex = Mutex.new
62
+ @stop_check = false
63
+ @thread = Thread.new(&method(:check_flush_loop))
64
+ end
65
+ end
66
+
67
+ def before_shutdown
68
+ flush_buffers
69
+ super if defined?(super)
70
+ end
71
+
72
+ def shutdown
73
+ # Before shutdown is not available in older fluentd versions.
74
+ # Hence, we make sure that we flush the buffers here as well.
75
+ flush_buffers
76
+ @thread.join if @multiline_flush_interval
77
+ super
78
+ end
79
+
80
+ def emit(tag, es, chain)
81
+ es.each do |time_sec, record|
82
+ process_record(tag, time_sec, record)
83
+ end
84
+ chain.next
85
+ end
86
+
87
+ private
88
+
89
+ def process_record(tag, time_sec, record)
90
+ synchronize do
91
+ log_id = [tag]
92
+ log_id.push(record.fetch(@stream, '')) unless @stream.empty?
93
+ unless @accumulators.key?(log_id)
94
+ out_tag = tag.sub(/^#{Regexp.escape(@remove_tag_prefix)}\./, '')
95
+ @accumulators[log_id] =
96
+ Fluent::TraceAccumulator.new(
97
+ @message,
98
+ @languages,
99
+ force_line_breaks: @force_line_breaks,
100
+ max_lines: @max_lines,
101
+ max_bytes: @max_bytes
102
+ ) do |t, r|
103
+ router.emit(out_tag, t, r)
104
+ end
105
+ end
106
+
107
+ @accumulators[log_id].push(time_sec, record)
108
+ end
109
+ end
110
+
111
+ def flush_buffers
112
+ synchronize do
113
+ @stop_check = true
114
+ @accumulators.each_value(&:force_flush)
115
+ end
116
+ end
117
+
118
+ def check_flush_loop
119
+ @flush_buffer_mutex.synchronize do
120
+ loop do
121
+ @flush_buffer_mutex.sleep(@check_flush_interval)
122
+ now = Time.now
123
+ break if @stop_check
124
+ @accumulators.each_value do |acc|
125
+ acc.force_flush if now - acc.buffer_start_time >
126
+ @multiline_flush_interval
127
+ end
128
+ end
129
+ end
130
+ rescue
131
+ log.error 'error in check_flush_loop', error: $ERROR_INFO.to_s
132
+ log.error_backtrace
133
+ end
134
+
135
+ def synchronize(&block)
136
+ if @multiline_flush_interval
137
+ @flush_buffer_mutex.synchronize(&block)
138
+ else
139
+ yield
140
+ end
141
+ end
142
+ end
143
+ end
data/test/helper.rb ADDED
@@ -0,0 +1,46 @@
1
+ # Copyright 2016 Google Inc. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ require 'rubygems'
16
+ require 'bundler'
17
+
18
+ begin
19
+ Bundler.setup(:default, :development)
20
+ rescue Bundler::BundlerError => e
21
+ $stderr.puts e.message
22
+ $stderr.puts 'Run `bundle install` to install missing gems'
23
+ exit e.status_code
24
+ end
25
+
26
+ require 'test/unit'
27
+
28
+ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
29
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
30
+ require 'fluent/test'
31
+
32
+ unless ENV.key?('VERBOSE')
33
+ nulllogger = Object.new
34
+ nulllogger.instance_eval do |_|
35
+ def respond_to_missing?
36
+ true
37
+ end
38
+
39
+ def method_missing(_method, *_args) # rubocop:disable Style/MethodMissing
40
+ end
41
+ end
42
+ # global $log variable is used by fluentd
43
+ $log = nulllogger # rubocop:disable Style/GlobalVars
44
+ end
45
+
46
+ require 'fluent/plugin/out_detect_exceptions'
@@ -0,0 +1,73 @@
1
+ # Copyright 2016 Google Inc. All rights reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ require 'benchmark'
16
+
17
+ require 'fluent/plugin/exception_detector'
18
+
19
+ size_in_m = 25
20
+ line_length = 50
21
+
22
+ size = size_in_m << 20
23
+
24
+ JAVA_EXC = <<END.freeze
25
+ Jul 09, 2015 3:23:29 PM com.google.devtools.search.cloud.feeder.MakeLog: RuntimeException: Run from this message!
26
+ at com.my.app.Object.do$a1(MakeLog.java:50)
27
+ at java.lang.Thing.call(Thing.java:10)
28
+ at com.my.app.Object.help(MakeLog.java:40)
29
+ at sun.javax.API.method(API.java:100)
30
+ at com.jetty.Framework.main(MakeLog.java:30)
31
+ END
32
+
33
+ PYTHON_EXC = <<END.freeze
34
+ Traceback (most recent call last):
35
+ File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.5.2/webapp2.py", line 1535, in __call__
36
+ rv = self.handle_exception(request, response, e)
37
+ File "/base/data/home/apps/s~nearfieldspy/1.378705245900539993/nearfieldspy.py", line 17, in start
38
+ return get()
39
+ File "/base/data/home/apps/s~nearfieldspy/1.378705245900539993/nearfieldspy.py", line 5, in get
40
+ raise Exception('spam', 'eggs')
41
+ Exception: ('spam', 'eggs')
42
+ END
43
+
44
+ chars = [('a'..'z'), ('A'..'Z')].map(&:to_a).flatten
45
+
46
+ random_text = (1..(size / line_length)).collect do
47
+ (0...line_length).map { chars[rand(chars.length)] }.join
48
+ end
49
+
50
+ exceptions = {
51
+ java: (JAVA_EXC * (size / JAVA_EXC.length)).lines,
52
+ python: (PYTHON_EXC * (size / PYTHON_EXC.length)).lines
53
+ }
54
+
55
+ puts "Start benchmark. Input size #{size_in_m}M."
56
+ Benchmark.bm do |x|
57
+ languages = Fluent::ExceptionDetectorConfig::RULES_BY_LANG.keys
58
+ languages.each do |lang|
59
+ buffer = Fluent::TraceAccumulator.new(nil, lang) {}
60
+ x.report("#{lang}_detector_random_text") do
61
+ random_text.each { |l| buffer.push(0, l) }
62
+ end
63
+ end
64
+ [:java, :python, :all].each do |detector_lang|
65
+ buffer = Fluent::TraceAccumulator.new(nil, detector_lang) {}
66
+ exc_languages = detector_lang == :all ? exceptions.keys : [detector_lang]
67
+ exc_languages.each do |exc_lang|
68
+ x.report("#{detector_lang}_detector_#{exc_lang}_stacks") do
69
+ exceptions[exc_lang].each { |l| buffer.push(0, l) }
70
+ end
71
+ end
72
+ end
73
+ end