fluent-plugin-group-exceptions 0.0.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CONTRIBUTING +24 -0
- data/Gemfile +3 -0
- data/Gemfile.lock +86 -0
- data/LICENSE +201 -0
- data/README.rdoc +124 -0
- data/Rakefile +48 -0
- data/fluent-plugin-group-exceptions.gemspec +29 -0
- data/fluent-plugin-multi-exceptions-0.0.13.gem +0 -0
- data/fluent-plugin-multi-exceptions-0.0.14.gem +0 -0
- data/lib/fluent/plugin/exception_detector.rb +368 -0
- data/lib/fluent/plugin/out_detect_exceptions.rb +141 -0
- data/test/helper.rb +46 -0
- data/test/plugin/bench_exception_detector.rb +73 -0
- data/test/plugin/test_exception_detector.rb +849 -0
- data/test/plugin/test_out_detect_exceptions.rb +307 -0
- metadata +136 -0
data/Rakefile
ADDED
@@ -0,0 +1,48 @@
|
|
1
|
+
#!/usr/bin/env rake
|
2
|
+
|
3
|
+
require 'bundler'
|
4
|
+
Bundler::GemHelper.install_tasks
|
5
|
+
|
6
|
+
require 'rake/testtask'
|
7
|
+
require 'rubocop/rake_task'
|
8
|
+
|
9
|
+
desc 'Run Rubocop to check for style violations'
|
10
|
+
RuboCop::RakeTask.new
|
11
|
+
|
12
|
+
desc 'Run benchmark tests'
|
13
|
+
Rake::TestTask.new(:bench) do |test|
|
14
|
+
test.libs << 'lib' << 'test'
|
15
|
+
test.test_files = FileList['test/plugin/bench*.rb']
|
16
|
+
test.verbose = true
|
17
|
+
end
|
18
|
+
|
19
|
+
desc 'Run unit tests'
|
20
|
+
Rake::TestTask.new(:test) do |test|
|
21
|
+
test.libs << 'lib' << 'test'
|
22
|
+
test.test_files = FileList['test/plugin/test*.rb']
|
23
|
+
test.verbose = true
|
24
|
+
end
|
25
|
+
|
26
|
+
# Building the gem will use the local file mode, so ensure it's world-readable.
|
27
|
+
# https://github.com/GoogleCloudPlatform/fluent-plugin-detect-exceptions/issues/32
|
28
|
+
desc 'Fix file permissions'
|
29
|
+
task :fix_perms do
|
30
|
+
files = [
|
31
|
+
'lib/fluent/plugin/*.rb'
|
32
|
+
].flat_map do |file|
|
33
|
+
file.include?('*') ? Dir.glob(file) : [file]
|
34
|
+
end
|
35
|
+
|
36
|
+
files.each do |file|
|
37
|
+
mode = File.stat(file).mode & 0o777
|
38
|
+
next unless mode & 0o444 != 0o444
|
39
|
+
puts "Changing mode of #{file} from #{mode.to_s(8)} to "\
|
40
|
+
"#{(mode | 0o444).to_s(8)}"
|
41
|
+
chmod mode | 0o444, file
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
desc 'Run unit tests and RuboCop to check for style violations'
|
46
|
+
task all: [:test, :rubocop, :fix_perms]
|
47
|
+
|
48
|
+
task default: :all
|
@@ -0,0 +1,29 @@
|
|
1
|
+
Gem::Specification.new do |gem|
|
2
|
+
gem.name = 'fluent-plugin-group-exceptions'
|
3
|
+
gem.description = <<-eos
|
4
|
+
Fluentd output plugin which detects exception stack traces in a stream of
|
5
|
+
JSON log messages and combines all single-line messages that belong to the
|
6
|
+
same stack trace into one multi-line message.
|
7
|
+
This is Not an official Google Ruby gem. Added Multiworker to true
|
8
|
+
eos
|
9
|
+
gem.summary = \
|
10
|
+
'fluentd output plugin for combining stack traces as multi-line JSON logs'
|
11
|
+
gem.homepage = \
|
12
|
+
'https://github.com/GoogleCloudPlatform/fluent-plugin-group-exceptions'
|
13
|
+
gem.license = 'Apache-2.0'
|
14
|
+
gem.version = '0.0.14'
|
15
|
+
gem.authors = ['Deloitte Agents']
|
16
|
+
gem.email = ['gatolgaj@gmail.com']
|
17
|
+
gem.required_ruby_version = Gem::Requirement.new('>= 2.0')
|
18
|
+
|
19
|
+
gem.files = Dir['**/*'].keep_if { |file| File.file?(file) }
|
20
|
+
gem.test_files = gem.files.grep(/^(test)/)
|
21
|
+
gem.require_paths = ['lib']
|
22
|
+
|
23
|
+
gem.add_runtime_dependency 'fluentd', '>= 0.10'
|
24
|
+
|
25
|
+
gem.add_development_dependency 'rake', '~> 10.3'
|
26
|
+
gem.add_development_dependency 'rubocop', '= 0.42.0'
|
27
|
+
gem.add_development_dependency 'test-unit', '~> 3.0'
|
28
|
+
gem.add_development_dependency 'flexmock', '~> 2.0'
|
29
|
+
end
|
Binary file
|
Binary file
|
@@ -0,0 +1,368 @@
|
|
1
|
+
# Copyright 2016 Google Inc. All rights reserved.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
#
|
15
|
+
module Fluent
|
16
|
+
Struct.new('Rule', :from_states, :pattern, :to_state)
|
17
|
+
|
18
|
+
# Configuration of the state machine that detects exceptions.
|
19
|
+
module ExceptionDetectorConfig
|
20
|
+
# Rule for a state transition: if pattern matches go to the given state.
|
21
|
+
class RuleTarget
|
22
|
+
attr_accessor :pattern, :to_state
|
23
|
+
|
24
|
+
def initialize(p, s)
|
25
|
+
@pattern = p
|
26
|
+
@to_state = s
|
27
|
+
end
|
28
|
+
|
29
|
+
def ==(other)
|
30
|
+
other.class == self.class && other.state == state
|
31
|
+
end
|
32
|
+
|
33
|
+
alias eql? ==
|
34
|
+
|
35
|
+
def hash
|
36
|
+
state.hash
|
37
|
+
end
|
38
|
+
|
39
|
+
def state
|
40
|
+
[@pattern, @to_state]
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def self.rule(from_state_or_states, pattern, to_state)
|
45
|
+
from_state_or_states = [from_state_or_states] unless
|
46
|
+
from_state_or_states.is_a?(Array)
|
47
|
+
Struct::Rule.new(from_state_or_states, pattern, to_state)
|
48
|
+
end
|
49
|
+
|
50
|
+
def self.supported
|
51
|
+
RULES_BY_LANG.keys
|
52
|
+
end
|
53
|
+
|
54
|
+
JAVA_RULES = [
|
55
|
+
rule([:start_state, :java_start_exception],
|
56
|
+
/(?:Exception|Error|Throwable|V8 errors stack trace)[:\r\n]/,
|
57
|
+
:java_after_exception),
|
58
|
+
rule(:java_after_exception, /^[\t ]*nested exception is:[\t ]*/,
|
59
|
+
:java_start_exception),
|
60
|
+
rule(:java_after_exception, /^[\r\n]*$/, :java_after_exception),
|
61
|
+
rule([:java_after_exception, :java], /^[\t ]+(?:eval )?at /, :java),
|
62
|
+
|
63
|
+
rule([:java_after_exception, :java],
|
64
|
+
# C# nested exception.
|
65
|
+
/^[\t ]+--- End of inner exception stack trace ---$/,
|
66
|
+
:java),
|
67
|
+
|
68
|
+
rule([:java_after_exception, :java],
|
69
|
+
# C# exception from async code.
|
70
|
+
/^--- End of stack trace from previous (?x:
|
71
|
+
)location where exception was thrown ---$/,
|
72
|
+
:java),
|
73
|
+
|
74
|
+
rule([:java_after_exception, :java], /^[\t ]*(?:Caused by|Suppressed):/,
|
75
|
+
:java_after_exception),
|
76
|
+
rule([:java_after_exception, :java],
|
77
|
+
/^[\t ]*... \d+ (?:more|common frames omitted)/, :java)
|
78
|
+
].freeze
|
79
|
+
|
80
|
+
PYTHON_RULES = [
|
81
|
+
rule(:start_state, /^Traceback \(most recent call last\):$/, :python),
|
82
|
+
rule(:python, /^[\t ]+File /, :python_code),
|
83
|
+
rule(:python_code, /[^\t ]/, :python),
|
84
|
+
rule(:python, /^(?:[^\s.():]+\.)*[^\s.():]+:/, :start_state)
|
85
|
+
].freeze
|
86
|
+
|
87
|
+
PHP_RULES = [
|
88
|
+
rule(:start_state, /
|
89
|
+
(?:PHP\ (?:Notice|Parse\ error|Fatal\ error|Warning):)|
|
90
|
+
(?:exception\ '[^']+'\ with\ message\ ')/x, :php_stack_begin),
|
91
|
+
rule(:php_stack_begin, /^Stack trace:/, :php_stack_frames),
|
92
|
+
rule(:php_stack_frames, /^#\d/, :php_stack_frames),
|
93
|
+
rule(:php_stack_frames, /^\s+thrown in /, :start_state)
|
94
|
+
].freeze
|
95
|
+
|
96
|
+
GO_RULES = [
|
97
|
+
rule(:start_state, /\bpanic: /, :go_after_panic),
|
98
|
+
rule(:start_state, /http: panic serving/, :go_goroutine),
|
99
|
+
rule(:go_after_panic, /^$/, :go_goroutine),
|
100
|
+
rule([:go_after_panic, :go_after_signal, :go_frame_1],
|
101
|
+
/^$/, :go_goroutine),
|
102
|
+
rule(:go_after_panic, /^\[signal /, :go_after_signal),
|
103
|
+
rule(:go_goroutine, /^goroutine \d+ \[[^\]]+\]:$/, :go_frame_1),
|
104
|
+
rule(:go_frame_1, /^(?:[^\s.:]+\.)*[^\s.():]+\(|^created by /,
|
105
|
+
:go_frame_2),
|
106
|
+
rule(:go_frame_2, /^\s/, :go_frame_1)
|
107
|
+
].freeze
|
108
|
+
|
109
|
+
RUBY_RULES = [
|
110
|
+
rule(:start_state, /Error \(.*\):$/, :ruby_before_rails_trace),
|
111
|
+
rule(:ruby_before_rails_trace, /^ $/, :ruby),
|
112
|
+
rule(:ruby_before_rails_trace, /^[\t ]+.*?\.rb:\d+:in `/, :ruby),
|
113
|
+
rule(:ruby, /^[\t ]+.*?\.rb:\d+:in `/, :ruby)
|
114
|
+
].freeze
|
115
|
+
|
116
|
+
DART_RULES = [
|
117
|
+
rule(:start_state, /^Unhandled exception:$/, :dart_exc),
|
118
|
+
rule(:dart_exc, /^Instance of/, :dart_stack),
|
119
|
+
rule(:dart_exc, /^Exception/, :dart_stack),
|
120
|
+
rule(:dart_exc, /^Bad state/, :dart_stack),
|
121
|
+
rule(:dart_exc, /^IntegerDivisionByZeroException/, :dart_stack),
|
122
|
+
rule(:dart_exc, /^Invalid argument/, :dart_stack),
|
123
|
+
rule(:dart_exc, /^RangeError/, :dart_stack),
|
124
|
+
rule(:dart_exc, /^Assertion failed/, :dart_stack),
|
125
|
+
rule(:dart_exc, /^Cannot instantiate/, :dart_stack),
|
126
|
+
rule(:dart_exc, /^Reading static variable/, :dart_stack),
|
127
|
+
rule(:dart_exc, /^UnimplementedError/, :dart_stack),
|
128
|
+
rule(:dart_exc, /^Unsupported operation/, :dart_stack),
|
129
|
+
rule(:dart_exc, /^Concurrent modification/, :dart_stack),
|
130
|
+
rule(:dart_exc, /^Out of Memory/, :dart_stack),
|
131
|
+
rule(:dart_exc, /^Stack Overflow/, :dart_stack),
|
132
|
+
rule(:dart_exc, /^'.+?':.+?$/, :dart_type_err_1),
|
133
|
+
rule(:dart_type_err_1, /^#\d+\s+.+?\(.+?\)$/, :dart_stack),
|
134
|
+
rule(:dart_type_err_1, /^.+?$/, :dart_type_err_2),
|
135
|
+
rule(:dart_type_err_2, /^.*?\^.*?$/, :dart_type_err_3),
|
136
|
+
rule(:dart_type_err_3, /^$/, :dart_type_err_4),
|
137
|
+
rule(:dart_type_err_4, /^$/, :dart_stack),
|
138
|
+
rule(:dart_exc, /^FormatException/, :dart_format_err_1),
|
139
|
+
rule(:dart_format_err_1, /^#\d+\s+.+?\(.+?\)$/, :dart_stack),
|
140
|
+
rule(:dart_format_err_1, /^./, :dart_format_err_2),
|
141
|
+
rule(:dart_format_err_2, /^.*?\^/, :dart_format_err_3),
|
142
|
+
rule(:dart_format_err_3, /^$/, :dart_stack),
|
143
|
+
rule(:dart_exc, /^NoSuchMethodError:/, :dart_method_err_1),
|
144
|
+
rule(:dart_method_err_1, /^Receiver:/, :dart_method_err_2),
|
145
|
+
rule(:dart_method_err_2, /^Tried calling:/, :dart_method_err_3),
|
146
|
+
rule(:dart_method_err_3, /^Found:/, :dart_stack),
|
147
|
+
rule(:dart_method_err_3, /^#\d+\s+.+?\(.+?\)$/, :dart_stack),
|
148
|
+
rule(:dart_stack, /^#\d+\s+.+?\(.+?\)$/, :dart_stack),
|
149
|
+
rule(:dart_stack, /^<asynchronous suspension>$/, :dart_stack)
|
150
|
+
].freeze
|
151
|
+
|
152
|
+
ALL_RULES = (
|
153
|
+
JAVA_RULES + PYTHON_RULES + PHP_RULES + GO_RULES + RUBY_RULES + DART_RULES
|
154
|
+
).freeze
|
155
|
+
|
156
|
+
RULES_BY_LANG = {
|
157
|
+
java: JAVA_RULES,
|
158
|
+
javascript: JAVA_RULES,
|
159
|
+
js: JAVA_RULES,
|
160
|
+
csharp: JAVA_RULES,
|
161
|
+
py: PYTHON_RULES,
|
162
|
+
python: PYTHON_RULES,
|
163
|
+
php: PHP_RULES,
|
164
|
+
go: GO_RULES,
|
165
|
+
rb: RUBY_RULES,
|
166
|
+
ruby: RUBY_RULES,
|
167
|
+
dart: DART_RULES,
|
168
|
+
all: ALL_RULES
|
169
|
+
}.freeze
|
170
|
+
|
171
|
+
DEFAULT_FIELDS = %w(message log).freeze
|
172
|
+
end
|
173
|
+
|
174
|
+
# State machine that consumes individual log lines and detects
|
175
|
+
# multi-line stack traces.
|
176
|
+
class ExceptionDetector
|
177
|
+
def initialize(*languages)
|
178
|
+
@state = :start_state
|
179
|
+
@rules = Hash.new { |h, k| h[k] = [] }
|
180
|
+
|
181
|
+
languages = [:all] if languages.empty?
|
182
|
+
|
183
|
+
languages.each do |lang|
|
184
|
+
rule_config =
|
185
|
+
ExceptionDetectorConfig::RULES_BY_LANG.fetch(lang.downcase) do |_k|
|
186
|
+
raise ArgumentError, "Unknown language: #{lang}"
|
187
|
+
end
|
188
|
+
|
189
|
+
rule_config.each do |r|
|
190
|
+
target = ExceptionDetectorConfig::RuleTarget.new(r[:pattern],
|
191
|
+
r[:to_state])
|
192
|
+
r[:from_states].each do |from_state|
|
193
|
+
@rules[from_state] << target
|
194
|
+
end
|
195
|
+
end
|
196
|
+
end
|
197
|
+
|
198
|
+
@rules.each_value(&:uniq!)
|
199
|
+
end
|
200
|
+
|
201
|
+
# Updates the state machine and returns the trace detection status:
|
202
|
+
# - no_trace: 'line' does not belong to an exception trace,
|
203
|
+
# - start_trace: 'line' starts a detected exception trace,
|
204
|
+
# - inside: 'line' is part of a detected exception trace,
|
205
|
+
# - end: the detected exception trace ends after 'line'.
|
206
|
+
def update(line)
|
207
|
+
trace_seen_before = transition(line)
|
208
|
+
# If the state machine fell back to the start state because there is no
|
209
|
+
# defined transition for 'line', trigger another state transition because
|
210
|
+
# 'line' may contain the beginning of another exception.
|
211
|
+
transition(line) unless trace_seen_before
|
212
|
+
new_state = @state
|
213
|
+
trace_seen_after = new_state != :start_state
|
214
|
+
|
215
|
+
case [trace_seen_before, trace_seen_after]
|
216
|
+
when [true, true]
|
217
|
+
:inside_trace
|
218
|
+
when [true, false]
|
219
|
+
:end_trace
|
220
|
+
when [false, true]
|
221
|
+
:start_trace
|
222
|
+
else
|
223
|
+
:no_trace
|
224
|
+
end
|
225
|
+
end
|
226
|
+
|
227
|
+
def reset
|
228
|
+
@state = :start_state
|
229
|
+
end
|
230
|
+
|
231
|
+
private
|
232
|
+
|
233
|
+
# Executes a transition of the state machine for the given line.
|
234
|
+
# Returns false if the line does not match any transition rule and the
|
235
|
+
# state machine was reset to the initial state.
|
236
|
+
def transition(line)
|
237
|
+
@rules[@state].each do |r|
|
238
|
+
next unless line =~ r.pattern
|
239
|
+
@state = r.to_state
|
240
|
+
return true
|
241
|
+
end
|
242
|
+
@state = :start_state
|
243
|
+
false
|
244
|
+
end
|
245
|
+
end
|
246
|
+
|
247
|
+
# Buffers and groups log records if they contain exception stack traces.
|
248
|
+
class TraceAccumulator
|
249
|
+
attr_reader :buffer_start_time
|
250
|
+
|
251
|
+
# If message_field is nil, the instance is set up to accumulate
|
252
|
+
# records that are plain strings (i.e. the whole record is concatenated).
|
253
|
+
# Otherwise, the instance accepts records that are dictionaries (usually
|
254
|
+
# originating from structured JSON logs) and accumulates just the
|
255
|
+
# content of the given message field.
|
256
|
+
# message_field may contain the empty string. In this case, the
|
257
|
+
# TraceAccumulator 'learns' the field name from the first record by checking
|
258
|
+
# for some pre-defined common field names of text logs.
|
259
|
+
# The named parameters max_lines and max_bytes limit the maximum amount
|
260
|
+
# of data to be buffered. The default value 0 indicates 'no limit'.
|
261
|
+
def initialize(message_field, languages, max_lines: 0, max_bytes: 0,
|
262
|
+
&emit_callback)
|
263
|
+
@exception_detector = Fluent::ExceptionDetector.new(*languages)
|
264
|
+
@max_lines = max_lines
|
265
|
+
@max_bytes = max_bytes
|
266
|
+
@message_field = message_field
|
267
|
+
@messages = []
|
268
|
+
@buffer_start_time = Time.now
|
269
|
+
@buffer_size = 0
|
270
|
+
@first_record = nil
|
271
|
+
@first_timestamp = nil
|
272
|
+
@emit = emit_callback
|
273
|
+
end
|
274
|
+
|
275
|
+
def push(time_sec, record)
|
276
|
+
message = extract_message(record)
|
277
|
+
if message.nil?
|
278
|
+
@exception_detector.reset
|
279
|
+
detection_status = :no_trace
|
280
|
+
else
|
281
|
+
force_flush if @max_bytes > 0 &&
|
282
|
+
@buffer_size + message.length > @max_bytes
|
283
|
+
detection_status = @exception_detector.update(message)
|
284
|
+
end
|
285
|
+
|
286
|
+
update_buffer(detection_status, time_sec, record, message)
|
287
|
+
|
288
|
+
force_flush if @max_lines > 0 && @messages.length == @max_lines
|
289
|
+
end
|
290
|
+
|
291
|
+
def flush
|
292
|
+
case @messages.length
|
293
|
+
when 0
|
294
|
+
return
|
295
|
+
when 1
|
296
|
+
@emit.call(@first_timestamp, @first_record)
|
297
|
+
else
|
298
|
+
combined_message = @messages.join
|
299
|
+
if @message_field.nil?
|
300
|
+
output_record = combined_message
|
301
|
+
else
|
302
|
+
output_record = @first_record
|
303
|
+
output_record[@message_field] = combined_message
|
304
|
+
end
|
305
|
+
@emit.call(@first_timestamp, output_record)
|
306
|
+
end
|
307
|
+
@messages = []
|
308
|
+
@first_record = nil
|
309
|
+
@first_timestamp = nil
|
310
|
+
@buffer_size = 0
|
311
|
+
end
|
312
|
+
|
313
|
+
def force_flush
|
314
|
+
flush
|
315
|
+
@exception_detector.reset
|
316
|
+
end
|
317
|
+
|
318
|
+
private
|
319
|
+
|
320
|
+
def extract_message(record)
|
321
|
+
if !@message_field.nil? && @message_field.empty?
|
322
|
+
ExceptionDetectorConfig::DEFAULT_FIELDS.each do |f|
|
323
|
+
if record.key?(f)
|
324
|
+
@message_field = f
|
325
|
+
break
|
326
|
+
end
|
327
|
+
end
|
328
|
+
end
|
329
|
+
@message_field.nil? ? record : record[@message_field]
|
330
|
+
end
|
331
|
+
|
332
|
+
def update_buffer(detection_status, time_sec, record, message)
|
333
|
+
trigger_emit = detection_status == :no_trace ||
|
334
|
+
detection_status == :end_trace
|
335
|
+
if @messages.empty? && trigger_emit
|
336
|
+
@emit.call(time_sec, record)
|
337
|
+
return
|
338
|
+
end
|
339
|
+
|
340
|
+
case detection_status
|
341
|
+
when :inside_trace
|
342
|
+
add(time_sec, record, message)
|
343
|
+
when :end_trace
|
344
|
+
add(time_sec, record, message)
|
345
|
+
flush
|
346
|
+
when :no_trace
|
347
|
+
flush
|
348
|
+
add(time_sec, record, message)
|
349
|
+
flush
|
350
|
+
when :start_trace
|
351
|
+
flush
|
352
|
+
add(time_sec, record, message)
|
353
|
+
end
|
354
|
+
end
|
355
|
+
|
356
|
+
def add(time_sec, record, message)
|
357
|
+
if @messages.empty?
|
358
|
+
@first_record = record unless @message_field.nil?
|
359
|
+
@first_timestamp = time_sec
|
360
|
+
@buffer_start_time = Time.now
|
361
|
+
end
|
362
|
+
unless message.nil?
|
363
|
+
@messages << message
|
364
|
+
@buffer_size += message.length
|
365
|
+
end
|
366
|
+
end
|
367
|
+
end
|
368
|
+
end
|