fluent-plugin-detect-exceptions 0.0.13 → 0.0.15
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +33 -25
- data/README.rdoc +16 -6
- data/Rakefile +2 -1
- data/fluent-plugin-detect-exceptions.gemspec +6 -6
- data/lib/fluent/plugin/exception_detector.rb +53 -45
- data/lib/fluent/plugin/out_detect_exceptions.rb +21 -16
- data/test/helper.rb +5 -5
- data/test/plugin/bench_exception_detector.rb +19 -19
- data/test/plugin/test_exception_detector.rb +510 -508
- data/test/plugin/test_out_detect_exceptions.rb +120 -55
- metadata +20 -20
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c1ac7bd2ba9a6630a4d3362da4892df13fb6e65f595e23e871fb76dbfffb444a
|
4
|
+
data.tar.gz: a339c63ea937fbf91770a77b86cf91f3cba806e8bbb0b2f2531e9a2f26485630
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4c9608e5ab97d683e9c0ea187afce6c9495196fb84a13aa05eeb1cefb74ce62ec50aa6c1bb7879be1516ce633368cfc9e8fcbcd2776b7d8b360472c1a884adf2
|
7
|
+
data.tar.gz: 0abf0b0c543582b141da5c24ac8c1a10e6c2db4b4b4dd0925499bb5158713883398180dfda2cd3cba26abe40d9e60008b5eed546022b66a925dde2a50f6efa96
|
data/Gemfile.lock
CHANGED
@@ -1,55 +1,63 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
fluent-plugin-detect-exceptions (0.0.
|
4
|
+
fluent-plugin-detect-exceptions (0.0.15)
|
5
5
|
fluentd (>= 0.10)
|
6
6
|
|
7
7
|
GEM
|
8
8
|
remote: https://rubygems.org/
|
9
9
|
specs:
|
10
|
-
ast (2.4.
|
11
|
-
concurrent-ruby (1.1.5)
|
10
|
+
ast (2.4.2)
|
12
11
|
cool.io (1.5.4)
|
13
12
|
dig_rb (1.0.1)
|
14
13
|
flexmock (2.3.6)
|
15
|
-
fluentd (1.
|
14
|
+
fluentd (1.6.3)
|
16
15
|
cool.io (>= 1.4.5, < 2.0.0)
|
17
16
|
dig_rb (~> 1.0.0)
|
18
17
|
http_parser.rb (>= 0.5.1, < 0.7.0)
|
19
|
-
msgpack (>=
|
18
|
+
msgpack (>= 0.7.0, < 2.0.0)
|
20
19
|
serverengine (>= 2.0.4, < 3.0.0)
|
21
20
|
sigdump (~> 0.2.2)
|
22
21
|
strptime (>= 0.2.2, < 1.0.0)
|
23
|
-
tzinfo (~>
|
22
|
+
tzinfo (~> 1.0)
|
24
23
|
tzinfo-data (~> 1.0)
|
25
24
|
yajl-ruby (~> 1.0)
|
26
25
|
http_parser.rb (0.6.0)
|
26
|
+
json (2.6.3)
|
27
27
|
msgpack (1.3.1)
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
rainbow (
|
33
|
-
rake
|
28
|
+
parallel (1.22.1)
|
29
|
+
parser (3.2.1.1)
|
30
|
+
ast (~> 2.4.1)
|
31
|
+
power_assert (1.1.4)
|
32
|
+
rainbow (3.1.1)
|
34
33
|
rake (10.5.0)
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
34
|
+
regexp_parser (2.7.0)
|
35
|
+
rexml (3.2.5)
|
36
|
+
rubocop (1.48.1)
|
37
|
+
json (~> 2.3)
|
38
|
+
parallel (~> 1.10)
|
39
|
+
parser (>= 3.2.0.0)
|
40
|
+
rainbow (>= 2.2.2, < 4.0)
|
41
|
+
regexp_parser (>= 1.8, < 3.0)
|
42
|
+
rexml (>= 3.2.5, < 4.0)
|
43
|
+
rubocop-ast (>= 1.26.0, < 2.0)
|
39
44
|
ruby-progressbar (~> 1.7)
|
40
|
-
unicode-display_width (
|
41
|
-
|
45
|
+
unicode-display_width (>= 2.4.0, < 3.0)
|
46
|
+
rubocop-ast (1.27.0)
|
47
|
+
parser (>= 3.2.1.0)
|
48
|
+
ruby-progressbar (1.13.0)
|
42
49
|
serverengine (2.1.1)
|
43
50
|
sigdump (~> 0.2.2)
|
44
51
|
sigdump (0.2.4)
|
45
52
|
strptime (0.2.3)
|
46
|
-
test-unit (3.3.
|
53
|
+
test-unit (3.3.3)
|
47
54
|
power_assert
|
48
|
-
|
49
|
-
|
50
|
-
|
55
|
+
thread_safe (0.3.6)
|
56
|
+
tzinfo (1.2.5)
|
57
|
+
thread_safe (~> 0.1)
|
58
|
+
tzinfo-data (1.2019.2)
|
51
59
|
tzinfo (>= 1.0.0)
|
52
|
-
unicode-display_width (
|
60
|
+
unicode-display_width (2.4.2)
|
53
61
|
yajl-ruby (1.4.1)
|
54
62
|
|
55
63
|
PLATFORMS
|
@@ -59,8 +67,8 @@ DEPENDENCIES
|
|
59
67
|
flexmock (~> 2.0)
|
60
68
|
fluent-plugin-detect-exceptions!
|
61
69
|
rake (~> 10.3)
|
62
|
-
rubocop (=
|
70
|
+
rubocop (= 1.48.1)
|
63
71
|
test-unit (~> 3.0)
|
64
72
|
|
65
73
|
BUNDLED WITH
|
66
|
-
2.
|
74
|
+
2.4.1
|
data/README.rdoc
CHANGED
@@ -42,6 +42,16 @@ will also install and configure the gem.
|
|
42
42
|
|
43
43
|
The plugin supports the following parameters:
|
44
44
|
|
45
|
+
=== Required
|
46
|
+
|
47
|
+
[remove_tag_prefix] The prefix to remove from the input tag when outputting
|
48
|
+
a record. A prefix has to be a complete tag part.
|
49
|
+
Example: If remove_tag_prefix is set to 'foo', the input
|
50
|
+
tag foo.bar.baz is transformed to bar.baz and the input tag
|
51
|
+
'foofoo.bar' is not modified.
|
52
|
+
|
53
|
+
=== Optional
|
54
|
+
|
45
55
|
[message] Name of the field in the JSON record that contains the
|
46
56
|
single-line log messages that shall be scanned for exceptions.
|
47
57
|
If this is set to '', the plugin will try 'message' and 'log',
|
@@ -49,12 +59,6 @@ The plugin supports the following parameters:
|
|
49
59
|
This parameter is only applicable to structured (JSON) log streams.
|
50
60
|
Default: ''.
|
51
61
|
|
52
|
-
[remove_tag_prefix] The prefix to remove from the input tag when outputting
|
53
|
-
a record. A prefix has to be a complete tag part.
|
54
|
-
Example: If remove_tag_prefix is set to 'foo', the input
|
55
|
-
tag foo.bar.baz is transformed to bar.baz and the input tag
|
56
|
-
'foofoo.bar' is not modified. Default: empty string.
|
57
|
-
|
58
62
|
[languages] A list of language for which exception stack traces shall be
|
59
63
|
detected. The values in the list can be separated by commas or
|
60
64
|
written as JSON list.
|
@@ -66,6 +70,12 @@ The plugin supports the following parameters:
|
|
66
70
|
forwarded. If not set, incomplete exceptions stacks
|
67
71
|
are not flushed.
|
68
72
|
|
73
|
+
[force_line_breaks] Force line breaks between each lines when comibining exception stacks.
|
74
|
+
This is useful if your exception is formatted
|
75
|
+
as a single line. i.e., logs retrieved from the docker's
|
76
|
+
logging driver don't have any line break.
|
77
|
+
Default: false.
|
78
|
+
|
69
79
|
[max_lines] Maximum number of lines in a detected exception stack trace.
|
70
80
|
If this maximum number is exceeded, the exception stack trace
|
71
81
|
that has been detected so far will be output as a single
|
data/Rakefile
CHANGED
@@ -36,6 +36,7 @@ task :fix_perms do
|
|
36
36
|
files.each do |file|
|
37
37
|
mode = File.stat(file).mode & 0o777
|
38
38
|
next unless mode & 0o444 != 0o444
|
39
|
+
|
39
40
|
puts "Changing mode of #{file} from #{mode.to_s(8)} to "\
|
40
41
|
"#{(mode | 0o444).to_s(8)}"
|
41
42
|
chmod mode | 0o444, file
|
@@ -43,6 +44,6 @@ task :fix_perms do
|
|
43
44
|
end
|
44
45
|
|
45
46
|
desc 'Run unit tests and RuboCop to check for style violations'
|
46
|
-
task all: [
|
47
|
+
task all: %i[rubocop test fix_perms]
|
47
48
|
|
48
49
|
task default: :all
|
@@ -1,20 +1,20 @@
|
|
1
1
|
Gem::Specification.new do |gem|
|
2
2
|
gem.name = 'fluent-plugin-detect-exceptions'
|
3
|
-
gem.description = <<-
|
3
|
+
gem.description = <<-DESCRIPTION
|
4
4
|
Fluentd output plugin which detects exception stack traces in a stream of
|
5
5
|
JSON log messages and combines all single-line messages that belong to the
|
6
6
|
same stack trace into one multi-line message.
|
7
7
|
This is an official Google Ruby gem.
|
8
|
-
|
8
|
+
DESCRIPTION
|
9
9
|
gem.summary = \
|
10
10
|
'fluentd output plugin for combining stack traces as multi-line JSON logs'
|
11
11
|
gem.homepage = \
|
12
12
|
'https://github.com/GoogleCloudPlatform/fluent-plugin-detect-exceptions'
|
13
13
|
gem.license = 'Apache-2.0'
|
14
|
-
gem.version = '0.0.
|
14
|
+
gem.version = '0.0.15'
|
15
15
|
gem.authors = ['Stackdriver Agents']
|
16
16
|
gem.email = ['stackdriver-agents@google.com']
|
17
|
-
gem.required_ruby_version = Gem::Requirement.new('>= 2.
|
17
|
+
gem.required_ruby_version = Gem::Requirement.new('>= 2.6')
|
18
18
|
|
19
19
|
gem.files = Dir['**/*'].keep_if { |file| File.file?(file) }
|
20
20
|
gem.test_files = gem.files.grep(/^(test)/)
|
@@ -22,8 +22,8 @@ eos
|
|
22
22
|
|
23
23
|
gem.add_runtime_dependency 'fluentd', '>= 0.10'
|
24
24
|
|
25
|
+
gem.add_development_dependency 'flexmock', '~> 2.0'
|
25
26
|
gem.add_development_dependency 'rake', '~> 10.3'
|
26
|
-
gem.add_development_dependency 'rubocop', '=
|
27
|
+
gem.add_development_dependency 'rubocop', '= 1.48.1'
|
27
28
|
gem.add_development_dependency 'test-unit', '~> 3.0'
|
28
|
-
gem.add_development_dependency 'flexmock', '~> 2.0'
|
29
29
|
end
|
@@ -21,9 +21,9 @@ module Fluent
|
|
21
21
|
class RuleTarget
|
22
22
|
attr_accessor :pattern, :to_state
|
23
23
|
|
24
|
-
def initialize(
|
25
|
-
@pattern =
|
26
|
-
@to_state =
|
24
|
+
def initialize(pattern, state)
|
25
|
+
@pattern = pattern
|
26
|
+
@to_state = state
|
27
27
|
end
|
28
28
|
|
29
29
|
def ==(other)
|
@@ -52,28 +52,28 @@ module Fluent
|
|
52
52
|
end
|
53
53
|
|
54
54
|
JAVA_RULES = [
|
55
|
-
rule([
|
55
|
+
rule(%i[start_state java_start_exception],
|
56
56
|
/(?:Exception|Error|Throwable|V8 errors stack trace)[:\r\n]/,
|
57
57
|
:java_after_exception),
|
58
58
|
rule(:java_after_exception, /^[\t ]*nested exception is:[\t ]*/,
|
59
59
|
:java_start_exception),
|
60
60
|
rule(:java_after_exception, /^[\r\n]*$/, :java_after_exception),
|
61
|
-
rule([
|
61
|
+
rule(%i[java_after_exception java], /^[\t ]+(?:eval )?at /, :java),
|
62
62
|
|
63
|
-
rule([
|
63
|
+
rule(%i[java_after_exception java],
|
64
64
|
# C# nested exception.
|
65
65
|
/^[\t ]+--- End of inner exception stack trace ---$/,
|
66
66
|
:java),
|
67
67
|
|
68
|
-
rule([
|
68
|
+
rule(%i[java_after_exception java],
|
69
69
|
# C# exception from async code.
|
70
70
|
/^--- End of stack trace from previous (?x:
|
71
71
|
)location where exception was thrown ---$/,
|
72
72
|
:java),
|
73
73
|
|
74
|
-
rule([
|
74
|
+
rule(%i[java_after_exception java], /^[\t ]*(?:Caused by|Suppressed):/,
|
75
75
|
:java_after_exception),
|
76
|
-
rule([
|
76
|
+
rule(%i[java_after_exception java],
|
77
77
|
/^[\t ]*... \d+ (?:more|common frames omitted)/, :java)
|
78
78
|
].freeze
|
79
79
|
|
@@ -97,13 +97,13 @@ module Fluent
|
|
97
97
|
rule(:start_state, /\bpanic: /, :go_after_panic),
|
98
98
|
rule(:start_state, /http: panic serving/, :go_goroutine),
|
99
99
|
rule(:go_after_panic, /^$/, :go_goroutine),
|
100
|
-
rule([
|
100
|
+
rule(%i[go_after_panic go_after_signal go_frame_line1],
|
101
101
|
/^$/, :go_goroutine),
|
102
102
|
rule(:go_after_panic, /^\[signal /, :go_after_signal),
|
103
|
-
rule(:go_goroutine, /^goroutine \d+ \[[^\]]+\]:$/, :
|
104
|
-
rule(:
|
105
|
-
:
|
106
|
-
rule(:
|
103
|
+
rule(:go_goroutine, /^goroutine \d+ \[[^\]]+\]:$/, :go_frame_line1),
|
104
|
+
rule(:go_frame_line1, /^(?:[^\s.:]+\.)*[^\s.():]+\(|^created by /,
|
105
|
+
:go_frame_line2),
|
106
|
+
rule(:go_frame_line2, /^\s/, :go_frame_line1)
|
107
107
|
].freeze
|
108
108
|
|
109
109
|
RUBY_RULES = [
|
@@ -129,22 +129,22 @@ module Fluent
|
|
129
129
|
rule(:dart_exc, /^Concurrent modification/, :dart_stack),
|
130
130
|
rule(:dart_exc, /^Out of Memory/, :dart_stack),
|
131
131
|
rule(:dart_exc, /^Stack Overflow/, :dart_stack),
|
132
|
-
rule(:dart_exc, /^'.+?':.+?$/, :
|
133
|
-
rule(:
|
134
|
-
rule(:
|
135
|
-
rule(:
|
136
|
-
rule(:
|
137
|
-
rule(:
|
138
|
-
rule(:dart_exc, /^FormatException/, :
|
139
|
-
rule(:
|
140
|
-
rule(:
|
141
|
-
rule(:
|
142
|
-
rule(:
|
143
|
-
rule(:dart_exc, /^NoSuchMethodError:/, :
|
144
|
-
rule(:
|
145
|
-
rule(:
|
146
|
-
rule(:
|
147
|
-
rule(:
|
132
|
+
rule(:dart_exc, /^'.+?':.+?$/, :dart_type_err_line1),
|
133
|
+
rule(:dart_type_err_line1, /^#\d+\s+.+?\(.+?\)$/, :dart_stack),
|
134
|
+
rule(:dart_type_err_line1, /^.+?$/, :dart_type_err_line2),
|
135
|
+
rule(:dart_type_err_line2, /^.*?\^.*?$/, :dart_type_err_line3),
|
136
|
+
rule(:dart_type_err_line3, /^$/, :dart_type_err_line4),
|
137
|
+
rule(:dart_type_err_line4, /^$/, :dart_stack),
|
138
|
+
rule(:dart_exc, /^FormatException/, :dart_format_err_line1),
|
139
|
+
rule(:dart_format_err_line1, /^#\d+\s+.+?\(.+?\)$/, :dart_stack),
|
140
|
+
rule(:dart_format_err_line1, /^./, :dart_format_err_line2),
|
141
|
+
rule(:dart_format_err_line2, /^.*?\^/, :dart_format_err_line3),
|
142
|
+
rule(:dart_format_err_line3, /^$/, :dart_stack),
|
143
|
+
rule(:dart_exc, /^NoSuchMethodError:/, :dart_method_err_line1),
|
144
|
+
rule(:dart_method_err_line1, /^Receiver:/, :dart_method_err_line2),
|
145
|
+
rule(:dart_method_err_line2, /^Tried calling:/, :dart_method_err_line3),
|
146
|
+
rule(:dart_method_err_line3, /^Found:/, :dart_stack),
|
147
|
+
rule(:dart_method_err_line3, /^#\d+\s+.+?\(.+?\)$/, :dart_stack),
|
148
148
|
rule(:dart_stack, /^#\d+\s+.+?\(.+?\)$/, :dart_stack),
|
149
149
|
rule(:dart_stack, /^<asynchronous suspension>$/, :dart_stack)
|
150
150
|
].freeze
|
@@ -168,7 +168,7 @@ module Fluent
|
|
168
168
|
all: ALL_RULES
|
169
169
|
}.freeze
|
170
170
|
|
171
|
-
DEFAULT_FIELDS = %w
|
171
|
+
DEFAULT_FIELDS = %w[message log].freeze
|
172
172
|
end
|
173
173
|
|
174
174
|
# State machine that consumes individual log lines and detects
|
@@ -236,6 +236,7 @@ module Fluent
|
|
236
236
|
def transition(line)
|
237
237
|
@rules[@state].each do |r|
|
238
238
|
next unless line =~ r.pattern
|
239
|
+
|
239
240
|
@state = r.to_state
|
240
241
|
return true
|
241
242
|
end
|
@@ -256,20 +257,22 @@ module Fluent
|
|
256
257
|
# message_field may contain the empty string. In this case, the
|
257
258
|
# TraceAccumulator 'learns' the field name from the first record by checking
|
258
259
|
# for some pre-defined common field names of text logs.
|
259
|
-
# The
|
260
|
+
# The option parameter can be used to pass the following parameters:
|
261
|
+
# force_line_breaks adds line breaks when combining exception stacks
|
262
|
+
# max_lines and max_bytes limit the maximum amount
|
260
263
|
# of data to be buffered. The default value 0 indicates 'no limit'.
|
261
|
-
def initialize(message_field, languages,
|
262
|
-
&emit_callback)
|
264
|
+
def initialize(message_field, languages, **options, &emit_callback)
|
263
265
|
@exception_detector = Fluent::ExceptionDetector.new(*languages)
|
264
|
-
@max_lines = max_lines
|
265
|
-
@max_bytes = max_bytes
|
266
266
|
@message_field = message_field
|
267
|
+
@force_line_breaks = options[:force_line_breaks] || false
|
268
|
+
@max_lines = options[:max_lines] || 0
|
269
|
+
@max_bytes = options[:max_bytes] || 0
|
270
|
+
@emit = emit_callback
|
267
271
|
@messages = []
|
268
272
|
@buffer_start_time = Time.now
|
269
273
|
@buffer_size = 0
|
270
274
|
@first_record = nil
|
271
275
|
@first_timestamp = nil
|
272
|
-
@emit = emit_callback
|
273
276
|
end
|
274
277
|
|
275
278
|
def push(time_sec, record)
|
@@ -278,14 +281,14 @@ module Fluent
|
|
278
281
|
@exception_detector.reset
|
279
282
|
detection_status = :no_trace
|
280
283
|
else
|
281
|
-
force_flush if @max_bytes
|
284
|
+
force_flush if @max_bytes.positive? &&
|
282
285
|
@buffer_size + message.length > @max_bytes
|
283
286
|
detection_status = @exception_detector.update(message)
|
284
287
|
end
|
285
288
|
|
286
289
|
update_buffer(detection_status, time_sec, record, message)
|
287
290
|
|
288
|
-
force_flush if @max_lines
|
291
|
+
force_flush if @max_lines.positive? && @messages.length == @max_lines
|
289
292
|
end
|
290
293
|
|
291
294
|
def flush
|
@@ -330,8 +333,7 @@ module Fluent
|
|
330
333
|
end
|
331
334
|
|
332
335
|
def update_buffer(detection_status, time_sec, record, message)
|
333
|
-
trigger_emit =
|
334
|
-
detection_status == :end_trace
|
336
|
+
trigger_emit = %i[no_trace end_trace].include?(detection_status)
|
335
337
|
if @messages.empty? && trigger_emit
|
336
338
|
@emit.call(time_sec, record)
|
337
339
|
return
|
@@ -359,10 +361,16 @@ module Fluent
|
|
359
361
|
@first_timestamp = time_sec
|
360
362
|
@buffer_start_time = Time.now
|
361
363
|
end
|
362
|
-
|
363
|
-
|
364
|
-
|
365
|
-
|
364
|
+
return if message.nil?
|
365
|
+
|
366
|
+
message_with_line_break =
|
367
|
+
if @force_line_breaks && !@messages.empty? && !message.include?("\n")
|
368
|
+
"\n#{message}"
|
369
|
+
else
|
370
|
+
message
|
371
|
+
end
|
372
|
+
@messages << message_with_line_break
|
373
|
+
@buffer_size += message_with_line_break.length
|
366
374
|
end
|
367
375
|
end
|
368
376
|
end
|
@@ -22,14 +22,16 @@ module Fluent
|
|
22
22
|
# an exception stack trace, they forwarded as a single, combined JSON
|
23
23
|
# object. Otherwise, the input log data is forwarded as is.
|
24
24
|
class DetectExceptionsOutput < Output
|
25
|
+
desc 'The prefix to be removed from the input tag when outputting a record.'
|
26
|
+
config_param :remove_tag_prefix, :string
|
25
27
|
desc 'The field which contains the raw message text in the input JSON data.'
|
26
28
|
config_param :message, :string, default: ''
|
27
|
-
desc 'The prefix to be removed from the input tag when outputting a record.'
|
28
|
-
config_param :remove_tag_prefix, :string, default: ''
|
29
29
|
desc 'The interval of flushing the buffer for multiline format.'
|
30
30
|
config_param :multiline_flush_interval, :time, default: nil
|
31
31
|
desc 'Programming languages for which to detect exceptions. Default: all.'
|
32
32
|
config_param :languages, :array, value_type: :string, default: []
|
33
|
+
desc 'Force live breaks when combining exception stacks. Default: false.'
|
34
|
+
config_param :force_line_breaks, :bool, default: false
|
33
35
|
desc 'Maximum number of lines to flush (0 means no limit). Default: 1000.'
|
34
36
|
config_param :max_lines, :integer, default: 1000
|
35
37
|
desc 'Maximum number of bytes to flush (0 means no limit). Default: 0.'
|
@@ -42,9 +44,7 @@ module Fluent
|
|
42
44
|
def configure(conf)
|
43
45
|
super
|
44
46
|
|
45
|
-
if multiline_flush_interval
|
46
|
-
@check_flush_interval = [multiline_flush_interval * 0.1, 1].max
|
47
|
-
end
|
47
|
+
@check_flush_interval = [multiline_flush_interval * 0.1, 1].max if multiline_flush_interval
|
48
48
|
|
49
49
|
@languages = languages.map(&:to_sym)
|
50
50
|
|
@@ -55,11 +55,11 @@ module Fluent
|
|
55
55
|
def start
|
56
56
|
super
|
57
57
|
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
58
|
+
return unless multiline_flush_interval
|
59
|
+
|
60
|
+
@flush_buffer_mutex = Mutex.new
|
61
|
+
@stop_check = false
|
62
|
+
@thread = Thread.new(&method(:check_flush_loop))
|
63
63
|
end
|
64
64
|
|
65
65
|
def before_shutdown
|
@@ -75,8 +75,8 @@ module Fluent
|
|
75
75
|
super
|
76
76
|
end
|
77
77
|
|
78
|
-
def emit(tag,
|
79
|
-
|
78
|
+
def emit(tag, entries, chain)
|
79
|
+
entries.each do |time_sec, record|
|
80
80
|
process_record(tag, time_sec, record)
|
81
81
|
end
|
82
82
|
chain.next
|
@@ -91,9 +91,13 @@ module Fluent
|
|
91
91
|
unless @accumulators.key?(log_id)
|
92
92
|
out_tag = tag.sub(/^#{Regexp.escape(@remove_tag_prefix)}\./, '')
|
93
93
|
@accumulators[log_id] =
|
94
|
-
Fluent::TraceAccumulator.new(
|
95
|
-
|
96
|
-
|
94
|
+
Fluent::TraceAccumulator.new(
|
95
|
+
@message,
|
96
|
+
@languages,
|
97
|
+
force_line_breaks: @force_line_breaks,
|
98
|
+
max_lines: @max_lines,
|
99
|
+
max_bytes: @max_bytes
|
100
|
+
) do |t, r|
|
97
101
|
router.emit(out_tag, t, r)
|
98
102
|
end
|
99
103
|
end
|
@@ -115,13 +119,14 @@ module Fluent
|
|
115
119
|
@flush_buffer_mutex.sleep(@check_flush_interval)
|
116
120
|
now = Time.now
|
117
121
|
break if @stop_check
|
122
|
+
|
118
123
|
@accumulators.each_value do |acc|
|
119
124
|
acc.force_flush if now - acc.buffer_start_time >
|
120
125
|
@multiline_flush_interval
|
121
126
|
end
|
122
127
|
end
|
123
128
|
end
|
124
|
-
rescue
|
129
|
+
rescue StandardError
|
125
130
|
log.error 'error in check_flush_loop', error: $ERROR_INFO.to_s
|
126
131
|
log.error_backtrace
|
127
132
|
end
|
data/test/helper.rb
CHANGED
@@ -14,29 +14,29 @@
|
|
14
14
|
|
15
15
|
require 'rubygems'
|
16
16
|
require 'bundler'
|
17
|
-
|
18
17
|
begin
|
19
18
|
Bundler.setup(:default, :development)
|
20
19
|
rescue Bundler::BundlerError => e
|
20
|
+
# rubocop:disable Style/StderrPuts
|
21
21
|
$stderr.puts e.message
|
22
22
|
$stderr.puts 'Run `bundle install` to install missing gems'
|
23
|
+
# rubocop:enable Style/StderrPuts
|
23
24
|
exit e.status_code
|
24
25
|
end
|
25
|
-
|
26
26
|
require 'test/unit'
|
27
27
|
|
28
28
|
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
29
29
|
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
30
30
|
require 'fluent/test'
|
31
|
-
|
32
31
|
unless ENV.key?('VERBOSE')
|
33
32
|
nulllogger = Object.new
|
34
33
|
nulllogger.instance_eval do |_|
|
35
|
-
def respond_to_missing?
|
34
|
+
def respond_to_missing?(_method, _include_private = false)
|
36
35
|
true
|
37
36
|
end
|
38
37
|
|
39
|
-
def method_missing(_method, *_args)
|
38
|
+
def method_missing(_method, *_args)
|
39
|
+
# pass
|
40
40
|
end
|
41
41
|
end
|
42
42
|
# global $log variable is used by fluentd
|
@@ -21,25 +21,25 @@ line_length = 50
|
|
21
21
|
|
22
22
|
size = size_in_m << 20
|
23
23
|
|
24
|
-
JAVA_EXC =
|
25
|
-
Jul 09, 2015 3:23:29 PM com.google.devtools.search.cloud.feeder.MakeLog: RuntimeException: Run from this message!
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
24
|
+
JAVA_EXC = <<~END_JAVA.freeze
|
25
|
+
Jul 09, 2015 3:23:29 PM com.google.devtools.search.cloud.feeder.MakeLog: RuntimeException: Run from this message!
|
26
|
+
at com.my.app.Object.do$a1(MakeLog.java:50)
|
27
|
+
at java.lang.Thing.call(Thing.java:10)
|
28
|
+
at com.my.app.Object.help(MakeLog.java:40)
|
29
|
+
at sun.javax.API.method(API.java:100)
|
30
|
+
at com.jetty.Framework.main(MakeLog.java:30)
|
31
|
+
END_JAVA
|
32
32
|
|
33
|
-
PYTHON_EXC =
|
34
|
-
Traceback (most recent call last):
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
Exception: ('spam', 'eggs')
|
42
|
-
|
33
|
+
PYTHON_EXC = <<~END_PYTHON.freeze
|
34
|
+
Traceback (most recent call last):
|
35
|
+
File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.5.2/webapp2.py", line 1535, in __call__
|
36
|
+
rv = self.handle_exception(request, response, e)
|
37
|
+
File "/base/data/home/apps/s~nearfieldspy/1.378705245900539993/nearfieldspy.py", line 17, in start
|
38
|
+
return get()
|
39
|
+
File "/base/data/home/apps/s~nearfieldspy/1.378705245900539993/nearfieldspy.py", line 5, in get
|
40
|
+
raise Exception('spam', 'eggs')
|
41
|
+
Exception: ('spam', 'eggs')
|
42
|
+
END_PYTHON
|
43
43
|
|
44
44
|
chars = [('a'..'z'), ('A'..'Z')].map(&:to_a).flatten
|
45
45
|
|
@@ -61,7 +61,7 @@ Benchmark.bm do |x|
|
|
61
61
|
random_text.each { |l| buffer.push(0, l) }
|
62
62
|
end
|
63
63
|
end
|
64
|
-
[
|
64
|
+
%i[java python all].each do |detector_lang|
|
65
65
|
buffer = Fluent::TraceAccumulator.new(nil, detector_lang) {}
|
66
66
|
exc_languages = detector_lang == :all ? exceptions.keys : [detector_lang]
|
67
67
|
exc_languages.each do |exc_lang|
|