fluent-plugin-group-exceptions 0.0.14
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/CONTRIBUTING +24 -0
- data/Gemfile +3 -0
- data/Gemfile.lock +86 -0
- data/LICENSE +201 -0
- data/README.rdoc +124 -0
- data/Rakefile +48 -0
- data/fluent-plugin-group-exceptions.gemspec +29 -0
- data/fluent-plugin-multi-exceptions-0.0.13.gem +0 -0
- data/fluent-plugin-multi-exceptions-0.0.14.gem +0 -0
- data/lib/fluent/plugin/exception_detector.rb +368 -0
- data/lib/fluent/plugin/out_detect_exceptions.rb +141 -0
- data/test/helper.rb +46 -0
- data/test/plugin/bench_exception_detector.rb +73 -0
- data/test/plugin/test_exception_detector.rb +849 -0
- data/test/plugin/test_out_detect_exceptions.rb +307 -0
- metadata +136 -0
data/Rakefile
ADDED
@@ -0,0 +1,48 @@
|
|
1
|
+
#!/usr/bin/env rake
|
2
|
+
|
3
|
+
require 'bundler'
|
4
|
+
Bundler::GemHelper.install_tasks
|
5
|
+
|
6
|
+
require 'rake/testtask'
|
7
|
+
require 'rubocop/rake_task'
|
8
|
+
|
9
|
+
desc 'Run Rubocop to check for style violations'
|
10
|
+
RuboCop::RakeTask.new
|
11
|
+
|
12
|
+
desc 'Run benchmark tests'
|
13
|
+
Rake::TestTask.new(:bench) do |test|
|
14
|
+
test.libs << 'lib' << 'test'
|
15
|
+
test.test_files = FileList['test/plugin/bench*.rb']
|
16
|
+
test.verbose = true
|
17
|
+
end
|
18
|
+
|
19
|
+
desc 'Run unit tests'
|
20
|
+
Rake::TestTask.new(:test) do |test|
|
21
|
+
test.libs << 'lib' << 'test'
|
22
|
+
test.test_files = FileList['test/plugin/test*.rb']
|
23
|
+
test.verbose = true
|
24
|
+
end
|
25
|
+
|
26
|
+
# Building the gem will use the local file mode, so ensure it's world-readable.
|
27
|
+
# https://github.com/GoogleCloudPlatform/fluent-plugin-detect-exceptions/issues/32
|
28
|
+
desc 'Fix file permissions'
|
29
|
+
task :fix_perms do
|
30
|
+
files = [
|
31
|
+
'lib/fluent/plugin/*.rb'
|
32
|
+
].flat_map do |file|
|
33
|
+
file.include?('*') ? Dir.glob(file) : [file]
|
34
|
+
end
|
35
|
+
|
36
|
+
files.each do |file|
|
37
|
+
mode = File.stat(file).mode & 0o777
|
38
|
+
next unless mode & 0o444 != 0o444
|
39
|
+
puts "Changing mode of #{file} from #{mode.to_s(8)} to "\
|
40
|
+
"#{(mode | 0o444).to_s(8)}"
|
41
|
+
chmod mode | 0o444, file
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
desc 'Run unit tests and RuboCop to check for style violations'
|
46
|
+
task all: [:test, :rubocop, :fix_perms]
|
47
|
+
|
48
|
+
task default: :all
|
@@ -0,0 +1,29 @@
|
|
1
|
+
Gem::Specification.new do |gem|
|
2
|
+
gem.name = 'fluent-plugin-group-exceptions'
|
3
|
+
gem.description = <<-eos
|
4
|
+
Fluentd output plugin which detects exception stack traces in a stream of
|
5
|
+
JSON log messages and combines all single-line messages that belong to the
|
6
|
+
same stack trace into one multi-line message.
|
7
|
+
This is Not an official Google Ruby gem. Added Multiworker to true
|
8
|
+
eos
|
9
|
+
gem.summary = \
|
10
|
+
'fluentd output plugin for combining stack traces as multi-line JSON logs'
|
11
|
+
gem.homepage = \
|
12
|
+
'https://github.com/GoogleCloudPlatform/fluent-plugin-group-exceptions'
|
13
|
+
gem.license = 'Apache-2.0'
|
14
|
+
gem.version = '0.0.14'
|
15
|
+
gem.authors = ['Deloitte Agents']
|
16
|
+
gem.email = ['gatolgaj@gmail.com']
|
17
|
+
gem.required_ruby_version = Gem::Requirement.new('>= 2.0')
|
18
|
+
|
19
|
+
gem.files = Dir['**/*'].keep_if { |file| File.file?(file) }
|
20
|
+
gem.test_files = gem.files.grep(/^(test)/)
|
21
|
+
gem.require_paths = ['lib']
|
22
|
+
|
23
|
+
gem.add_runtime_dependency 'fluentd', '>= 0.10'
|
24
|
+
|
25
|
+
gem.add_development_dependency 'rake', '~> 10.3'
|
26
|
+
gem.add_development_dependency 'rubocop', '= 0.42.0'
|
27
|
+
gem.add_development_dependency 'test-unit', '~> 3.0'
|
28
|
+
gem.add_development_dependency 'flexmock', '~> 2.0'
|
29
|
+
end
|
Binary file
|
Binary file
|
@@ -0,0 +1,368 @@
|
|
1
|
+
# Copyright 2016 Google Inc. All rights reserved.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
#
|
15
|
+
module Fluent
|
16
|
+
Struct.new('Rule', :from_states, :pattern, :to_state)
|
17
|
+
|
18
|
+
# Configuration of the state machine that detects exceptions.
|
19
|
+
module ExceptionDetectorConfig
|
20
|
+
# Rule for a state transition: if pattern matches go to the given state.
|
21
|
+
class RuleTarget
|
22
|
+
attr_accessor :pattern, :to_state
|
23
|
+
|
24
|
+
def initialize(p, s)
|
25
|
+
@pattern = p
|
26
|
+
@to_state = s
|
27
|
+
end
|
28
|
+
|
29
|
+
def ==(other)
|
30
|
+
other.class == self.class && other.state == state
|
31
|
+
end
|
32
|
+
|
33
|
+
alias eql? ==
|
34
|
+
|
35
|
+
def hash
|
36
|
+
state.hash
|
37
|
+
end
|
38
|
+
|
39
|
+
def state
|
40
|
+
[@pattern, @to_state]
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def self.rule(from_state_or_states, pattern, to_state)
|
45
|
+
from_state_or_states = [from_state_or_states] unless
|
46
|
+
from_state_or_states.is_a?(Array)
|
47
|
+
Struct::Rule.new(from_state_or_states, pattern, to_state)
|
48
|
+
end
|
49
|
+
|
50
|
+
def self.supported
|
51
|
+
RULES_BY_LANG.keys
|
52
|
+
end
|
53
|
+
|
54
|
+
JAVA_RULES = [
|
55
|
+
rule([:start_state, :java_start_exception],
|
56
|
+
/(?:Exception|Error|Throwable|V8 errors stack trace)[:\r\n]/,
|
57
|
+
:java_after_exception),
|
58
|
+
rule(:java_after_exception, /^[\t ]*nested exception is:[\t ]*/,
|
59
|
+
:java_start_exception),
|
60
|
+
rule(:java_after_exception, /^[\r\n]*$/, :java_after_exception),
|
61
|
+
rule([:java_after_exception, :java], /^[\t ]+(?:eval )?at /, :java),
|
62
|
+
|
63
|
+
rule([:java_after_exception, :java],
|
64
|
+
# C# nested exception.
|
65
|
+
/^[\t ]+--- End of inner exception stack trace ---$/,
|
66
|
+
:java),
|
67
|
+
|
68
|
+
rule([:java_after_exception, :java],
|
69
|
+
# C# exception from async code.
|
70
|
+
/^--- End of stack trace from previous (?x:
|
71
|
+
)location where exception was thrown ---$/,
|
72
|
+
:java),
|
73
|
+
|
74
|
+
rule([:java_after_exception, :java], /^[\t ]*(?:Caused by|Suppressed):/,
|
75
|
+
:java_after_exception),
|
76
|
+
rule([:java_after_exception, :java],
|
77
|
+
/^[\t ]*... \d+ (?:more|common frames omitted)/, :java)
|
78
|
+
].freeze
|
79
|
+
|
80
|
+
PYTHON_RULES = [
|
81
|
+
rule(:start_state, /^Traceback \(most recent call last\):$/, :python),
|
82
|
+
rule(:python, /^[\t ]+File /, :python_code),
|
83
|
+
rule(:python_code, /[^\t ]/, :python),
|
84
|
+
rule(:python, /^(?:[^\s.():]+\.)*[^\s.():]+:/, :start_state)
|
85
|
+
].freeze
|
86
|
+
|
87
|
+
PHP_RULES = [
|
88
|
+
rule(:start_state, /
|
89
|
+
(?:PHP\ (?:Notice|Parse\ error|Fatal\ error|Warning):)|
|
90
|
+
(?:exception\ '[^']+'\ with\ message\ ')/x, :php_stack_begin),
|
91
|
+
rule(:php_stack_begin, /^Stack trace:/, :php_stack_frames),
|
92
|
+
rule(:php_stack_frames, /^#\d/, :php_stack_frames),
|
93
|
+
rule(:php_stack_frames, /^\s+thrown in /, :start_state)
|
94
|
+
].freeze
|
95
|
+
|
96
|
+
GO_RULES = [
|
97
|
+
rule(:start_state, /\bpanic: /, :go_after_panic),
|
98
|
+
rule(:start_state, /http: panic serving/, :go_goroutine),
|
99
|
+
rule(:go_after_panic, /^$/, :go_goroutine),
|
100
|
+
rule([:go_after_panic, :go_after_signal, :go_frame_1],
|
101
|
+
/^$/, :go_goroutine),
|
102
|
+
rule(:go_after_panic, /^\[signal /, :go_after_signal),
|
103
|
+
rule(:go_goroutine, /^goroutine \d+ \[[^\]]+\]:$/, :go_frame_1),
|
104
|
+
rule(:go_frame_1, /^(?:[^\s.:]+\.)*[^\s.():]+\(|^created by /,
|
105
|
+
:go_frame_2),
|
106
|
+
rule(:go_frame_2, /^\s/, :go_frame_1)
|
107
|
+
].freeze
|
108
|
+
|
109
|
+
RUBY_RULES = [
|
110
|
+
rule(:start_state, /Error \(.*\):$/, :ruby_before_rails_trace),
|
111
|
+
rule(:ruby_before_rails_trace, /^ $/, :ruby),
|
112
|
+
rule(:ruby_before_rails_trace, /^[\t ]+.*?\.rb:\d+:in `/, :ruby),
|
113
|
+
rule(:ruby, /^[\t ]+.*?\.rb:\d+:in `/, :ruby)
|
114
|
+
].freeze
|
115
|
+
|
116
|
+
DART_RULES = [
|
117
|
+
rule(:start_state, /^Unhandled exception:$/, :dart_exc),
|
118
|
+
rule(:dart_exc, /^Instance of/, :dart_stack),
|
119
|
+
rule(:dart_exc, /^Exception/, :dart_stack),
|
120
|
+
rule(:dart_exc, /^Bad state/, :dart_stack),
|
121
|
+
rule(:dart_exc, /^IntegerDivisionByZeroException/, :dart_stack),
|
122
|
+
rule(:dart_exc, /^Invalid argument/, :dart_stack),
|
123
|
+
rule(:dart_exc, /^RangeError/, :dart_stack),
|
124
|
+
rule(:dart_exc, /^Assertion failed/, :dart_stack),
|
125
|
+
rule(:dart_exc, /^Cannot instantiate/, :dart_stack),
|
126
|
+
rule(:dart_exc, /^Reading static variable/, :dart_stack),
|
127
|
+
rule(:dart_exc, /^UnimplementedError/, :dart_stack),
|
128
|
+
rule(:dart_exc, /^Unsupported operation/, :dart_stack),
|
129
|
+
rule(:dart_exc, /^Concurrent modification/, :dart_stack),
|
130
|
+
rule(:dart_exc, /^Out of Memory/, :dart_stack),
|
131
|
+
rule(:dart_exc, /^Stack Overflow/, :dart_stack),
|
132
|
+
rule(:dart_exc, /^'.+?':.+?$/, :dart_type_err_1),
|
133
|
+
rule(:dart_type_err_1, /^#\d+\s+.+?\(.+?\)$/, :dart_stack),
|
134
|
+
rule(:dart_type_err_1, /^.+?$/, :dart_type_err_2),
|
135
|
+
rule(:dart_type_err_2, /^.*?\^.*?$/, :dart_type_err_3),
|
136
|
+
rule(:dart_type_err_3, /^$/, :dart_type_err_4),
|
137
|
+
rule(:dart_type_err_4, /^$/, :dart_stack),
|
138
|
+
rule(:dart_exc, /^FormatException/, :dart_format_err_1),
|
139
|
+
rule(:dart_format_err_1, /^#\d+\s+.+?\(.+?\)$/, :dart_stack),
|
140
|
+
rule(:dart_format_err_1, /^./, :dart_format_err_2),
|
141
|
+
rule(:dart_format_err_2, /^.*?\^/, :dart_format_err_3),
|
142
|
+
rule(:dart_format_err_3, /^$/, :dart_stack),
|
143
|
+
rule(:dart_exc, /^NoSuchMethodError:/, :dart_method_err_1),
|
144
|
+
rule(:dart_method_err_1, /^Receiver:/, :dart_method_err_2),
|
145
|
+
rule(:dart_method_err_2, /^Tried calling:/, :dart_method_err_3),
|
146
|
+
rule(:dart_method_err_3, /^Found:/, :dart_stack),
|
147
|
+
rule(:dart_method_err_3, /^#\d+\s+.+?\(.+?\)$/, :dart_stack),
|
148
|
+
rule(:dart_stack, /^#\d+\s+.+?\(.+?\)$/, :dart_stack),
|
149
|
+
rule(:dart_stack, /^<asynchronous suspension>$/, :dart_stack)
|
150
|
+
].freeze
|
151
|
+
|
152
|
+
ALL_RULES = (
|
153
|
+
JAVA_RULES + PYTHON_RULES + PHP_RULES + GO_RULES + RUBY_RULES + DART_RULES
|
154
|
+
).freeze
|
155
|
+
|
156
|
+
RULES_BY_LANG = {
|
157
|
+
java: JAVA_RULES,
|
158
|
+
javascript: JAVA_RULES,
|
159
|
+
js: JAVA_RULES,
|
160
|
+
csharp: JAVA_RULES,
|
161
|
+
py: PYTHON_RULES,
|
162
|
+
python: PYTHON_RULES,
|
163
|
+
php: PHP_RULES,
|
164
|
+
go: GO_RULES,
|
165
|
+
rb: RUBY_RULES,
|
166
|
+
ruby: RUBY_RULES,
|
167
|
+
dart: DART_RULES,
|
168
|
+
all: ALL_RULES
|
169
|
+
}.freeze
|
170
|
+
|
171
|
+
DEFAULT_FIELDS = %w(message log).freeze
|
172
|
+
end
|
173
|
+
|
174
|
+
# State machine that consumes individual log lines and detects
|
175
|
+
# multi-line stack traces.
|
176
|
+
class ExceptionDetector
|
177
|
+
def initialize(*languages)
|
178
|
+
@state = :start_state
|
179
|
+
@rules = Hash.new { |h, k| h[k] = [] }
|
180
|
+
|
181
|
+
languages = [:all] if languages.empty?
|
182
|
+
|
183
|
+
languages.each do |lang|
|
184
|
+
rule_config =
|
185
|
+
ExceptionDetectorConfig::RULES_BY_LANG.fetch(lang.downcase) do |_k|
|
186
|
+
raise ArgumentError, "Unknown language: #{lang}"
|
187
|
+
end
|
188
|
+
|
189
|
+
rule_config.each do |r|
|
190
|
+
target = ExceptionDetectorConfig::RuleTarget.new(r[:pattern],
|
191
|
+
r[:to_state])
|
192
|
+
r[:from_states].each do |from_state|
|
193
|
+
@rules[from_state] << target
|
194
|
+
end
|
195
|
+
end
|
196
|
+
end
|
197
|
+
|
198
|
+
@rules.each_value(&:uniq!)
|
199
|
+
end
|
200
|
+
|
201
|
+
# Updates the state machine and returns the trace detection status:
|
202
|
+
# - no_trace: 'line' does not belong to an exception trace,
|
203
|
+
# - start_trace: 'line' starts a detected exception trace,
|
204
|
+
# - inside: 'line' is part of a detected exception trace,
|
205
|
+
# - end: the detected exception trace ends after 'line'.
|
206
|
+
def update(line)
|
207
|
+
trace_seen_before = transition(line)
|
208
|
+
# If the state machine fell back to the start state because there is no
|
209
|
+
# defined transition for 'line', trigger another state transition because
|
210
|
+
# 'line' may contain the beginning of another exception.
|
211
|
+
transition(line) unless trace_seen_before
|
212
|
+
new_state = @state
|
213
|
+
trace_seen_after = new_state != :start_state
|
214
|
+
|
215
|
+
case [trace_seen_before, trace_seen_after]
|
216
|
+
when [true, true]
|
217
|
+
:inside_trace
|
218
|
+
when [true, false]
|
219
|
+
:end_trace
|
220
|
+
when [false, true]
|
221
|
+
:start_trace
|
222
|
+
else
|
223
|
+
:no_trace
|
224
|
+
end
|
225
|
+
end
|
226
|
+
|
227
|
+
def reset
|
228
|
+
@state = :start_state
|
229
|
+
end
|
230
|
+
|
231
|
+
private
|
232
|
+
|
233
|
+
# Executes a transition of the state machine for the given line.
|
234
|
+
# Returns false if the line does not match any transition rule and the
|
235
|
+
# state machine was reset to the initial state.
|
236
|
+
def transition(line)
|
237
|
+
@rules[@state].each do |r|
|
238
|
+
next unless line =~ r.pattern
|
239
|
+
@state = r.to_state
|
240
|
+
return true
|
241
|
+
end
|
242
|
+
@state = :start_state
|
243
|
+
false
|
244
|
+
end
|
245
|
+
end
|
246
|
+
|
247
|
+
# Buffers and groups log records if they contain exception stack traces.
|
248
|
+
class TraceAccumulator
|
249
|
+
attr_reader :buffer_start_time
|
250
|
+
|
251
|
+
# If message_field is nil, the instance is set up to accumulate
|
252
|
+
# records that are plain strings (i.e. the whole record is concatenated).
|
253
|
+
# Otherwise, the instance accepts records that are dictionaries (usually
|
254
|
+
# originating from structured JSON logs) and accumulates just the
|
255
|
+
# content of the given message field.
|
256
|
+
# message_field may contain the empty string. In this case, the
|
257
|
+
# TraceAccumulator 'learns' the field name from the first record by checking
|
258
|
+
# for some pre-defined common field names of text logs.
|
259
|
+
# The named parameters max_lines and max_bytes limit the maximum amount
|
260
|
+
# of data to be buffered. The default value 0 indicates 'no limit'.
|
261
|
+
def initialize(message_field, languages, max_lines: 0, max_bytes: 0,
|
262
|
+
&emit_callback)
|
263
|
+
@exception_detector = Fluent::ExceptionDetector.new(*languages)
|
264
|
+
@max_lines = max_lines
|
265
|
+
@max_bytes = max_bytes
|
266
|
+
@message_field = message_field
|
267
|
+
@messages = []
|
268
|
+
@buffer_start_time = Time.now
|
269
|
+
@buffer_size = 0
|
270
|
+
@first_record = nil
|
271
|
+
@first_timestamp = nil
|
272
|
+
@emit = emit_callback
|
273
|
+
end
|
274
|
+
|
275
|
+
def push(time_sec, record)
|
276
|
+
message = extract_message(record)
|
277
|
+
if message.nil?
|
278
|
+
@exception_detector.reset
|
279
|
+
detection_status = :no_trace
|
280
|
+
else
|
281
|
+
force_flush if @max_bytes > 0 &&
|
282
|
+
@buffer_size + message.length > @max_bytes
|
283
|
+
detection_status = @exception_detector.update(message)
|
284
|
+
end
|
285
|
+
|
286
|
+
update_buffer(detection_status, time_sec, record, message)
|
287
|
+
|
288
|
+
force_flush if @max_lines > 0 && @messages.length == @max_lines
|
289
|
+
end
|
290
|
+
|
291
|
+
def flush
|
292
|
+
case @messages.length
|
293
|
+
when 0
|
294
|
+
return
|
295
|
+
when 1
|
296
|
+
@emit.call(@first_timestamp, @first_record)
|
297
|
+
else
|
298
|
+
combined_message = @messages.join
|
299
|
+
if @message_field.nil?
|
300
|
+
output_record = combined_message
|
301
|
+
else
|
302
|
+
output_record = @first_record
|
303
|
+
output_record[@message_field] = combined_message
|
304
|
+
end
|
305
|
+
@emit.call(@first_timestamp, output_record)
|
306
|
+
end
|
307
|
+
@messages = []
|
308
|
+
@first_record = nil
|
309
|
+
@first_timestamp = nil
|
310
|
+
@buffer_size = 0
|
311
|
+
end
|
312
|
+
|
313
|
+
def force_flush
|
314
|
+
flush
|
315
|
+
@exception_detector.reset
|
316
|
+
end
|
317
|
+
|
318
|
+
private
|
319
|
+
|
320
|
+
def extract_message(record)
|
321
|
+
if !@message_field.nil? && @message_field.empty?
|
322
|
+
ExceptionDetectorConfig::DEFAULT_FIELDS.each do |f|
|
323
|
+
if record.key?(f)
|
324
|
+
@message_field = f
|
325
|
+
break
|
326
|
+
end
|
327
|
+
end
|
328
|
+
end
|
329
|
+
@message_field.nil? ? record : record[@message_field]
|
330
|
+
end
|
331
|
+
|
332
|
+
def update_buffer(detection_status, time_sec, record, message)
|
333
|
+
trigger_emit = detection_status == :no_trace ||
|
334
|
+
detection_status == :end_trace
|
335
|
+
if @messages.empty? && trigger_emit
|
336
|
+
@emit.call(time_sec, record)
|
337
|
+
return
|
338
|
+
end
|
339
|
+
|
340
|
+
case detection_status
|
341
|
+
when :inside_trace
|
342
|
+
add(time_sec, record, message)
|
343
|
+
when :end_trace
|
344
|
+
add(time_sec, record, message)
|
345
|
+
flush
|
346
|
+
when :no_trace
|
347
|
+
flush
|
348
|
+
add(time_sec, record, message)
|
349
|
+
flush
|
350
|
+
when :start_trace
|
351
|
+
flush
|
352
|
+
add(time_sec, record, message)
|
353
|
+
end
|
354
|
+
end
|
355
|
+
|
356
|
+
def add(time_sec, record, message)
|
357
|
+
if @messages.empty?
|
358
|
+
@first_record = record unless @message_field.nil?
|
359
|
+
@first_timestamp = time_sec
|
360
|
+
@buffer_start_time = Time.now
|
361
|
+
end
|
362
|
+
unless message.nil?
|
363
|
+
@messages << message
|
364
|
+
@buffer_size += message.length
|
365
|
+
end
|
366
|
+
end
|
367
|
+
end
|
368
|
+
end
|