fluent-plugin-detect-exceptions 0.0.14 → 0.0.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile.lock +74 -0
- data/Rakefile +2 -1
- data/fluent-plugin-detect-exceptions.gemspec +6 -6
- data/lib/fluent/plugin/exception_detector.rb +45 -45
- data/lib/fluent/plugin/out_detect_exceptions.rb +10 -11
- data/test/helper.rb +5 -5
- data/test/plugin/bench_exception_detector.rb +19 -19
- data/test/plugin/test_exception_detector.rb +510 -508
- data/test/plugin/test_out_detect_exceptions.rb +51 -49
- metadata +21 -20
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c1ac7bd2ba9a6630a4d3362da4892df13fb6e65f595e23e871fb76dbfffb444a
|
4
|
+
data.tar.gz: a339c63ea937fbf91770a77b86cf91f3cba806e8bbb0b2f2531e9a2f26485630
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4c9608e5ab97d683e9c0ea187afce6c9495196fb84a13aa05eeb1cefb74ce62ec50aa6c1bb7879be1516ce633368cfc9e8fcbcd2776b7d8b360472c1a884adf2
|
7
|
+
data.tar.gz: 0abf0b0c543582b141da5c24ac8c1a10e6c2db4b4b4dd0925499bb5158713883398180dfda2cd3cba26abe40d9e60008b5eed546022b66a925dde2a50f6efa96
|
data/Gemfile.lock
ADDED
@@ -0,0 +1,74 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
fluent-plugin-detect-exceptions (0.0.15)
|
5
|
+
fluentd (>= 0.10)
|
6
|
+
|
7
|
+
GEM
|
8
|
+
remote: https://rubygems.org/
|
9
|
+
specs:
|
10
|
+
ast (2.4.2)
|
11
|
+
cool.io (1.5.4)
|
12
|
+
dig_rb (1.0.1)
|
13
|
+
flexmock (2.3.6)
|
14
|
+
fluentd (1.6.3)
|
15
|
+
cool.io (>= 1.4.5, < 2.0.0)
|
16
|
+
dig_rb (~> 1.0.0)
|
17
|
+
http_parser.rb (>= 0.5.1, < 0.7.0)
|
18
|
+
msgpack (>= 0.7.0, < 2.0.0)
|
19
|
+
serverengine (>= 2.0.4, < 3.0.0)
|
20
|
+
sigdump (~> 0.2.2)
|
21
|
+
strptime (>= 0.2.2, < 1.0.0)
|
22
|
+
tzinfo (~> 1.0)
|
23
|
+
tzinfo-data (~> 1.0)
|
24
|
+
yajl-ruby (~> 1.0)
|
25
|
+
http_parser.rb (0.6.0)
|
26
|
+
json (2.6.3)
|
27
|
+
msgpack (1.3.1)
|
28
|
+
parallel (1.22.1)
|
29
|
+
parser (3.2.1.1)
|
30
|
+
ast (~> 2.4.1)
|
31
|
+
power_assert (1.1.4)
|
32
|
+
rainbow (3.1.1)
|
33
|
+
rake (10.5.0)
|
34
|
+
regexp_parser (2.7.0)
|
35
|
+
rexml (3.2.5)
|
36
|
+
rubocop (1.48.1)
|
37
|
+
json (~> 2.3)
|
38
|
+
parallel (~> 1.10)
|
39
|
+
parser (>= 3.2.0.0)
|
40
|
+
rainbow (>= 2.2.2, < 4.0)
|
41
|
+
regexp_parser (>= 1.8, < 3.0)
|
42
|
+
rexml (>= 3.2.5, < 4.0)
|
43
|
+
rubocop-ast (>= 1.26.0, < 2.0)
|
44
|
+
ruby-progressbar (~> 1.7)
|
45
|
+
unicode-display_width (>= 2.4.0, < 3.0)
|
46
|
+
rubocop-ast (1.27.0)
|
47
|
+
parser (>= 3.2.1.0)
|
48
|
+
ruby-progressbar (1.13.0)
|
49
|
+
serverengine (2.1.1)
|
50
|
+
sigdump (~> 0.2.2)
|
51
|
+
sigdump (0.2.4)
|
52
|
+
strptime (0.2.3)
|
53
|
+
test-unit (3.3.3)
|
54
|
+
power_assert
|
55
|
+
thread_safe (0.3.6)
|
56
|
+
tzinfo (1.2.5)
|
57
|
+
thread_safe (~> 0.1)
|
58
|
+
tzinfo-data (1.2019.2)
|
59
|
+
tzinfo (>= 1.0.0)
|
60
|
+
unicode-display_width (2.4.2)
|
61
|
+
yajl-ruby (1.4.1)
|
62
|
+
|
63
|
+
PLATFORMS
|
64
|
+
ruby
|
65
|
+
|
66
|
+
DEPENDENCIES
|
67
|
+
flexmock (~> 2.0)
|
68
|
+
fluent-plugin-detect-exceptions!
|
69
|
+
rake (~> 10.3)
|
70
|
+
rubocop (= 1.48.1)
|
71
|
+
test-unit (~> 3.0)
|
72
|
+
|
73
|
+
BUNDLED WITH
|
74
|
+
2.4.1
|
data/Rakefile
CHANGED
@@ -36,6 +36,7 @@ task :fix_perms do
|
|
36
36
|
files.each do |file|
|
37
37
|
mode = File.stat(file).mode & 0o777
|
38
38
|
next unless mode & 0o444 != 0o444
|
39
|
+
|
39
40
|
puts "Changing mode of #{file} from #{mode.to_s(8)} to "\
|
40
41
|
"#{(mode | 0o444).to_s(8)}"
|
41
42
|
chmod mode | 0o444, file
|
@@ -43,6 +44,6 @@ task :fix_perms do
|
|
43
44
|
end
|
44
45
|
|
45
46
|
desc 'Run unit tests and RuboCop to check for style violations'
|
46
|
-
task all: [
|
47
|
+
task all: %i[rubocop test fix_perms]
|
47
48
|
|
48
49
|
task default: :all
|
@@ -1,20 +1,20 @@
|
|
1
1
|
Gem::Specification.new do |gem|
|
2
2
|
gem.name = 'fluent-plugin-detect-exceptions'
|
3
|
-
gem.description = <<-
|
3
|
+
gem.description = <<-DESCRIPTION
|
4
4
|
Fluentd output plugin which detects exception stack traces in a stream of
|
5
5
|
JSON log messages and combines all single-line messages that belong to the
|
6
6
|
same stack trace into one multi-line message.
|
7
7
|
This is an official Google Ruby gem.
|
8
|
-
|
8
|
+
DESCRIPTION
|
9
9
|
gem.summary = \
|
10
10
|
'fluentd output plugin for combining stack traces as multi-line JSON logs'
|
11
11
|
gem.homepage = \
|
12
12
|
'https://github.com/GoogleCloudPlatform/fluent-plugin-detect-exceptions'
|
13
13
|
gem.license = 'Apache-2.0'
|
14
|
-
gem.version = '0.0.
|
14
|
+
gem.version = '0.0.15'
|
15
15
|
gem.authors = ['Stackdriver Agents']
|
16
16
|
gem.email = ['stackdriver-agents@google.com']
|
17
|
-
gem.required_ruby_version = Gem::Requirement.new('>= 2.
|
17
|
+
gem.required_ruby_version = Gem::Requirement.new('>= 2.6')
|
18
18
|
|
19
19
|
gem.files = Dir['**/*'].keep_if { |file| File.file?(file) }
|
20
20
|
gem.test_files = gem.files.grep(/^(test)/)
|
@@ -22,8 +22,8 @@ eos
|
|
22
22
|
|
23
23
|
gem.add_runtime_dependency 'fluentd', '>= 0.10'
|
24
24
|
|
25
|
+
gem.add_development_dependency 'flexmock', '~> 2.0'
|
25
26
|
gem.add_development_dependency 'rake', '~> 10.3'
|
26
|
-
gem.add_development_dependency 'rubocop', '=
|
27
|
+
gem.add_development_dependency 'rubocop', '= 1.48.1'
|
27
28
|
gem.add_development_dependency 'test-unit', '~> 3.0'
|
28
|
-
gem.add_development_dependency 'flexmock', '~> 2.0'
|
29
29
|
end
|
@@ -21,9 +21,9 @@ module Fluent
|
|
21
21
|
class RuleTarget
|
22
22
|
attr_accessor :pattern, :to_state
|
23
23
|
|
24
|
-
def initialize(
|
25
|
-
@pattern =
|
26
|
-
@to_state =
|
24
|
+
def initialize(pattern, state)
|
25
|
+
@pattern = pattern
|
26
|
+
@to_state = state
|
27
27
|
end
|
28
28
|
|
29
29
|
def ==(other)
|
@@ -52,28 +52,28 @@ module Fluent
|
|
52
52
|
end
|
53
53
|
|
54
54
|
JAVA_RULES = [
|
55
|
-
rule([
|
55
|
+
rule(%i[start_state java_start_exception],
|
56
56
|
/(?:Exception|Error|Throwable|V8 errors stack trace)[:\r\n]/,
|
57
57
|
:java_after_exception),
|
58
58
|
rule(:java_after_exception, /^[\t ]*nested exception is:[\t ]*/,
|
59
59
|
:java_start_exception),
|
60
60
|
rule(:java_after_exception, /^[\r\n]*$/, :java_after_exception),
|
61
|
-
rule([
|
61
|
+
rule(%i[java_after_exception java], /^[\t ]+(?:eval )?at /, :java),
|
62
62
|
|
63
|
-
rule([
|
63
|
+
rule(%i[java_after_exception java],
|
64
64
|
# C# nested exception.
|
65
65
|
/^[\t ]+--- End of inner exception stack trace ---$/,
|
66
66
|
:java),
|
67
67
|
|
68
|
-
rule([
|
68
|
+
rule(%i[java_after_exception java],
|
69
69
|
# C# exception from async code.
|
70
70
|
/^--- End of stack trace from previous (?x:
|
71
71
|
)location where exception was thrown ---$/,
|
72
72
|
:java),
|
73
73
|
|
74
|
-
rule([
|
74
|
+
rule(%i[java_after_exception java], /^[\t ]*(?:Caused by|Suppressed):/,
|
75
75
|
:java_after_exception),
|
76
|
-
rule([
|
76
|
+
rule(%i[java_after_exception java],
|
77
77
|
/^[\t ]*... \d+ (?:more|common frames omitted)/, :java)
|
78
78
|
].freeze
|
79
79
|
|
@@ -97,13 +97,13 @@ module Fluent
|
|
97
97
|
rule(:start_state, /\bpanic: /, :go_after_panic),
|
98
98
|
rule(:start_state, /http: panic serving/, :go_goroutine),
|
99
99
|
rule(:go_after_panic, /^$/, :go_goroutine),
|
100
|
-
rule([
|
100
|
+
rule(%i[go_after_panic go_after_signal go_frame_line1],
|
101
101
|
/^$/, :go_goroutine),
|
102
102
|
rule(:go_after_panic, /^\[signal /, :go_after_signal),
|
103
|
-
rule(:go_goroutine, /^goroutine \d+ \[[^\]]+\]:$/, :
|
104
|
-
rule(:
|
105
|
-
:
|
106
|
-
rule(:
|
103
|
+
rule(:go_goroutine, /^goroutine \d+ \[[^\]]+\]:$/, :go_frame_line1),
|
104
|
+
rule(:go_frame_line1, /^(?:[^\s.:]+\.)*[^\s.():]+\(|^created by /,
|
105
|
+
:go_frame_line2),
|
106
|
+
rule(:go_frame_line2, /^\s/, :go_frame_line1)
|
107
107
|
].freeze
|
108
108
|
|
109
109
|
RUBY_RULES = [
|
@@ -129,22 +129,22 @@ module Fluent
|
|
129
129
|
rule(:dart_exc, /^Concurrent modification/, :dart_stack),
|
130
130
|
rule(:dart_exc, /^Out of Memory/, :dart_stack),
|
131
131
|
rule(:dart_exc, /^Stack Overflow/, :dart_stack),
|
132
|
-
rule(:dart_exc, /^'.+?':.+?$/, :
|
133
|
-
rule(:
|
134
|
-
rule(:
|
135
|
-
rule(:
|
136
|
-
rule(:
|
137
|
-
rule(:
|
138
|
-
rule(:dart_exc, /^FormatException/, :
|
139
|
-
rule(:
|
140
|
-
rule(:
|
141
|
-
rule(:
|
142
|
-
rule(:
|
143
|
-
rule(:dart_exc, /^NoSuchMethodError:/, :
|
144
|
-
rule(:
|
145
|
-
rule(:
|
146
|
-
rule(:
|
147
|
-
rule(:
|
132
|
+
rule(:dart_exc, /^'.+?':.+?$/, :dart_type_err_line1),
|
133
|
+
rule(:dart_type_err_line1, /^#\d+\s+.+?\(.+?\)$/, :dart_stack),
|
134
|
+
rule(:dart_type_err_line1, /^.+?$/, :dart_type_err_line2),
|
135
|
+
rule(:dart_type_err_line2, /^.*?\^.*?$/, :dart_type_err_line3),
|
136
|
+
rule(:dart_type_err_line3, /^$/, :dart_type_err_line4),
|
137
|
+
rule(:dart_type_err_line4, /^$/, :dart_stack),
|
138
|
+
rule(:dart_exc, /^FormatException/, :dart_format_err_line1),
|
139
|
+
rule(:dart_format_err_line1, /^#\d+\s+.+?\(.+?\)$/, :dart_stack),
|
140
|
+
rule(:dart_format_err_line1, /^./, :dart_format_err_line2),
|
141
|
+
rule(:dart_format_err_line2, /^.*?\^/, :dart_format_err_line3),
|
142
|
+
rule(:dart_format_err_line3, /^$/, :dart_stack),
|
143
|
+
rule(:dart_exc, /^NoSuchMethodError:/, :dart_method_err_line1),
|
144
|
+
rule(:dart_method_err_line1, /^Receiver:/, :dart_method_err_line2),
|
145
|
+
rule(:dart_method_err_line2, /^Tried calling:/, :dart_method_err_line3),
|
146
|
+
rule(:dart_method_err_line3, /^Found:/, :dart_stack),
|
147
|
+
rule(:dart_method_err_line3, /^#\d+\s+.+?\(.+?\)$/, :dart_stack),
|
148
148
|
rule(:dart_stack, /^#\d+\s+.+?\(.+?\)$/, :dart_stack),
|
149
149
|
rule(:dart_stack, /^<asynchronous suspension>$/, :dart_stack)
|
150
150
|
].freeze
|
@@ -168,7 +168,7 @@ module Fluent
|
|
168
168
|
all: ALL_RULES
|
169
169
|
}.freeze
|
170
170
|
|
171
|
-
DEFAULT_FIELDS = %w
|
171
|
+
DEFAULT_FIELDS = %w[message log].freeze
|
172
172
|
end
|
173
173
|
|
174
174
|
# State machine that consumes individual log lines and detects
|
@@ -236,6 +236,7 @@ module Fluent
|
|
236
236
|
def transition(line)
|
237
237
|
@rules[@state].each do |r|
|
238
238
|
next unless line =~ r.pattern
|
239
|
+
|
239
240
|
@state = r.to_state
|
240
241
|
return true
|
241
242
|
end
|
@@ -280,14 +281,14 @@ module Fluent
|
|
280
281
|
@exception_detector.reset
|
281
282
|
detection_status = :no_trace
|
282
283
|
else
|
283
|
-
force_flush if @max_bytes
|
284
|
+
force_flush if @max_bytes.positive? &&
|
284
285
|
@buffer_size + message.length > @max_bytes
|
285
286
|
detection_status = @exception_detector.update(message)
|
286
287
|
end
|
287
288
|
|
288
289
|
update_buffer(detection_status, time_sec, record, message)
|
289
290
|
|
290
|
-
force_flush if @max_lines
|
291
|
+
force_flush if @max_lines.positive? && @messages.length == @max_lines
|
291
292
|
end
|
292
293
|
|
293
294
|
def flush
|
@@ -332,8 +333,7 @@ module Fluent
|
|
332
333
|
end
|
333
334
|
|
334
335
|
def update_buffer(detection_status, time_sec, record, message)
|
335
|
-
trigger_emit =
|
336
|
-
detection_status == :end_trace
|
336
|
+
trigger_emit = %i[no_trace end_trace].include?(detection_status)
|
337
337
|
if @messages.empty? && trigger_emit
|
338
338
|
@emit.call(time_sec, record)
|
339
339
|
return
|
@@ -361,16 +361,16 @@ module Fluent
|
|
361
361
|
@first_timestamp = time_sec
|
362
362
|
@buffer_start_time = Time.now
|
363
363
|
end
|
364
|
-
|
365
|
-
|
366
|
-
|
367
|
-
|
368
|
-
|
369
|
-
|
370
|
-
|
371
|
-
|
372
|
-
|
373
|
-
|
364
|
+
return if message.nil?
|
365
|
+
|
366
|
+
message_with_line_break =
|
367
|
+
if @force_line_breaks && !@messages.empty? && !message.include?("\n")
|
368
|
+
"\n#{message}"
|
369
|
+
else
|
370
|
+
message
|
371
|
+
end
|
372
|
+
@messages << message_with_line_break
|
373
|
+
@buffer_size += message_with_line_break.length
|
374
374
|
end
|
375
375
|
end
|
376
376
|
end
|
@@ -44,9 +44,7 @@ module Fluent
|
|
44
44
|
def configure(conf)
|
45
45
|
super
|
46
46
|
|
47
|
-
if multiline_flush_interval
|
48
|
-
@check_flush_interval = [multiline_flush_interval * 0.1, 1].max
|
49
|
-
end
|
47
|
+
@check_flush_interval = [multiline_flush_interval * 0.1, 1].max if multiline_flush_interval
|
50
48
|
|
51
49
|
@languages = languages.map(&:to_sym)
|
52
50
|
|
@@ -57,11 +55,11 @@ module Fluent
|
|
57
55
|
def start
|
58
56
|
super
|
59
57
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
58
|
+
return unless multiline_flush_interval
|
59
|
+
|
60
|
+
@flush_buffer_mutex = Mutex.new
|
61
|
+
@stop_check = false
|
62
|
+
@thread = Thread.new(&method(:check_flush_loop))
|
65
63
|
end
|
66
64
|
|
67
65
|
def before_shutdown
|
@@ -77,8 +75,8 @@ module Fluent
|
|
77
75
|
super
|
78
76
|
end
|
79
77
|
|
80
|
-
def emit(tag,
|
81
|
-
|
78
|
+
def emit(tag, entries, chain)
|
79
|
+
entries.each do |time_sec, record|
|
82
80
|
process_record(tag, time_sec, record)
|
83
81
|
end
|
84
82
|
chain.next
|
@@ -121,13 +119,14 @@ module Fluent
|
|
121
119
|
@flush_buffer_mutex.sleep(@check_flush_interval)
|
122
120
|
now = Time.now
|
123
121
|
break if @stop_check
|
122
|
+
|
124
123
|
@accumulators.each_value do |acc|
|
125
124
|
acc.force_flush if now - acc.buffer_start_time >
|
126
125
|
@multiline_flush_interval
|
127
126
|
end
|
128
127
|
end
|
129
128
|
end
|
130
|
-
rescue
|
129
|
+
rescue StandardError
|
131
130
|
log.error 'error in check_flush_loop', error: $ERROR_INFO.to_s
|
132
131
|
log.error_backtrace
|
133
132
|
end
|
data/test/helper.rb
CHANGED
@@ -14,29 +14,29 @@
|
|
14
14
|
|
15
15
|
require 'rubygems'
|
16
16
|
require 'bundler'
|
17
|
-
|
18
17
|
begin
|
19
18
|
Bundler.setup(:default, :development)
|
20
19
|
rescue Bundler::BundlerError => e
|
20
|
+
# rubocop:disable Style/StderrPuts
|
21
21
|
$stderr.puts e.message
|
22
22
|
$stderr.puts 'Run `bundle install` to install missing gems'
|
23
|
+
# rubocop:enable Style/StderrPuts
|
23
24
|
exit e.status_code
|
24
25
|
end
|
25
|
-
|
26
26
|
require 'test/unit'
|
27
27
|
|
28
28
|
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
29
29
|
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
30
30
|
require 'fluent/test'
|
31
|
-
|
32
31
|
unless ENV.key?('VERBOSE')
|
33
32
|
nulllogger = Object.new
|
34
33
|
nulllogger.instance_eval do |_|
|
35
|
-
def respond_to_missing?
|
34
|
+
def respond_to_missing?(_method, _include_private = false)
|
36
35
|
true
|
37
36
|
end
|
38
37
|
|
39
|
-
def method_missing(_method, *_args)
|
38
|
+
def method_missing(_method, *_args)
|
39
|
+
# pass
|
40
40
|
end
|
41
41
|
end
|
42
42
|
# global $log variable is used by fluentd
|
@@ -21,25 +21,25 @@ line_length = 50
|
|
21
21
|
|
22
22
|
size = size_in_m << 20
|
23
23
|
|
24
|
-
JAVA_EXC =
|
25
|
-
Jul 09, 2015 3:23:29 PM com.google.devtools.search.cloud.feeder.MakeLog: RuntimeException: Run from this message!
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
24
|
+
JAVA_EXC = <<~END_JAVA.freeze
|
25
|
+
Jul 09, 2015 3:23:29 PM com.google.devtools.search.cloud.feeder.MakeLog: RuntimeException: Run from this message!
|
26
|
+
at com.my.app.Object.do$a1(MakeLog.java:50)
|
27
|
+
at java.lang.Thing.call(Thing.java:10)
|
28
|
+
at com.my.app.Object.help(MakeLog.java:40)
|
29
|
+
at sun.javax.API.method(API.java:100)
|
30
|
+
at com.jetty.Framework.main(MakeLog.java:30)
|
31
|
+
END_JAVA
|
32
32
|
|
33
|
-
PYTHON_EXC =
|
34
|
-
Traceback (most recent call last):
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
Exception: ('spam', 'eggs')
|
42
|
-
|
33
|
+
PYTHON_EXC = <<~END_PYTHON.freeze
|
34
|
+
Traceback (most recent call last):
|
35
|
+
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.5.2/webapp2.py", line 1535, in __call__
|
36
|
+
rv = self.handle_exception(request, response, e)
|
37
|
+
File "/base/data/home/apps/s~nearfieldspy/1.378705245900539993/nearfieldspy.py", line 17, in start
|
38
|
+
return get()
|
39
|
+
File "/base/data/home/apps/s~nearfieldspy/1.378705245900539993/nearfieldspy.py", line 5, in get
|
40
|
+
raise Exception('spam', 'eggs')
|
41
|
+
Exception: ('spam', 'eggs')
|
42
|
+
END_PYTHON
|
43
43
|
|
44
44
|
chars = [('a'..'z'), ('A'..'Z')].map(&:to_a).flatten
|
45
45
|
|
@@ -61,7 +61,7 @@ Benchmark.bm do |x|
|
|
61
61
|
random_text.each { |l| buffer.push(0, l) }
|
62
62
|
end
|
63
63
|
end
|
64
|
-
[
|
64
|
+
%i[java python all].each do |detector_lang|
|
65
65
|
buffer = Fluent::TraceAccumulator.new(nil, detector_lang) {}
|
66
66
|
exc_languages = detector_lang == :all ? exceptions.keys : [detector_lang]
|
67
67
|
exc_languages.each do |exc_lang|
|