fluent-plugin-detect-exceptions 0.0.1 → 0.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 3fc8ac42bca6d07ddc0c8131203df18a0e5eb2bf
4
- data.tar.gz: ac0298c30ec7cc47eca96ccf29dcd13b47b56bcc
3
+ metadata.gz: 1a26a5e2085d4bc44296565c82c0baacc5c1e33f
4
+ data.tar.gz: b0268e7263ca8bb5d5bfb3cb8913c04acb3b5893
5
5
  SHA512:
6
- metadata.gz: 57aa45c609ccecc24362272b55f99a58d5fa27b7c3eb85b73f52643755585c0a700917914b4c0e8d75b21f986df27b8fddf457a8604a0a0c4f0e1090cd8da204
7
- data.tar.gz: 7d20fbf7734e321285738856b022663764440090dc4cfaef21e38b6850863af9b1b9cfa28a9821ae21a2a7c1121f0b6f4d8320e611dfcea6424553d0db3ee357
6
+ metadata.gz: c170490ab76e47973fa288d49280d0c8e7cc0232f99fea45b9a144f7b1dc1c6514ca54d9fb57bfcff7f83ef7a809ea50f0e1c4f93ec64a9690e9fa9e5b376d6a
7
+ data.tar.gz: ebd8383c06d79737a213c7756573db8b61044aaed9c0ea0262bdf6be5062cfa4740eaa192ca8e956fa69f2e391767b91d2c163c868696aa828a1eccffe8832b0
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- fluent-plugin-detect-exceptions (0.0.1)
4
+ fluent-plugin-detect-exceptions (0.0.3)
5
5
  fluentd (~> 0.10, <= 0.13)
6
6
 
7
7
  GEM
@@ -22,9 +22,9 @@ GEM
22
22
  http_parser.rb (0.6.0)
23
23
  json (2.0.2)
24
24
  msgpack (0.5.12)
25
- parser (2.3.1.4)
25
+ parser (2.3.3.1)
26
26
  ast (~> 2.2)
27
- power_assert (0.3.1)
27
+ power_assert (0.4.1)
28
28
  powerpack (0.1.1)
29
29
  rainbow (2.1.0)
30
30
  rake (10.5.0)
@@ -37,14 +37,14 @@ GEM
37
37
  ruby-progressbar (1.8.1)
38
38
  sigdump (0.2.4)
39
39
  string-scrub (0.0.5)
40
- test-unit (3.2.2)
40
+ test-unit (3.2.3)
41
41
  power_assert
42
42
  thread_safe (0.3.5)
43
43
  tzinfo (1.2.2)
44
44
  thread_safe (~> 0.1)
45
- tzinfo-data (1.2016.9)
45
+ tzinfo-data (1.2016.10)
46
46
  tzinfo (>= 1.0.0)
47
- unicode-display_width (1.1.1)
47
+ unicode-display_width (1.1.2)
48
48
  yajl-ruby (1.3.0)
49
49
 
50
50
  PLATFORMS
@@ -11,7 +11,7 @@ eos
11
11
  gem.homepage = \
12
12
  'https://github.com/GoogleCloudPlatform/fluent-plugin-detect-exceptions'
13
13
  gem.license = 'Apache-2.0'
14
- gem.version = '0.0.1'
14
+ gem.version = '0.0.3'
15
15
  gem.authors = ['Thomas Schickinger']
16
16
  gem.email = ['schickin@google.com']
17
17
  gem.required_ruby_version = Gem::Requirement.new('>= 2.0')
@@ -66,8 +66,9 @@ module Fluent
66
66
  ].freeze
67
67
 
68
68
  PHP_RULES = [
69
- rule(:start_state, /PHP (?:Notice|Parse error|Fatal error|Warning):/,
70
- :php_stack_begin),
69
+ rule(:start_state, /
70
+ (?:PHP\ (?:Notice|Parse\ error|Fatal\ error|Warning):)|
71
+ (?:exception\ '[^']+'\ with\ message\ ')/x, :php_stack_begin),
71
72
  rule(:php_stack_begin, /^Stack trace:/, :php_stack_frames),
72
73
  rule(:php_stack_frames, /^#\d/, :php_stack_frames),
73
74
  rule(:php_stack_frames, /^\s+thrown in /, :start_state)
@@ -190,34 +191,36 @@ module Fluent
190
191
  # message_field may contain the empty string. In this case, the
191
192
  # TraceAccumulator 'learns' the field name from the first record by checking
192
193
  # for some pre-defined common field names of text logs.
193
- def initialize(message_field, languages, &emit_callback)
194
+ # The named parameters max_lines and max_bytes limit the maximum amount
195
+ # of data to be buffered. The default value 0 indicates 'no limit'.
196
+ def initialize(message_field, languages, max_lines: 0, max_bytes: 0,
197
+ &emit_callback)
194
198
  @exception_detector = Fluent::ExceptionDetector.new(*languages)
199
+ @max_lines = max_lines
200
+ @max_bytes = max_bytes
195
201
  @message_field = message_field
196
202
  @messages = []
197
203
  @buffer_start_time = Time.now
204
+ @buffer_size = 0
198
205
  @first_record = nil
199
206
  @first_timestamp = nil
200
207
  @emit = emit_callback
201
208
  end
202
209
 
203
210
  def push(time_sec, record)
204
- if !@message_field.nil? && @message_field.empty?
205
- ExceptionDetectorConfig::DEFAULT_FIELDS.each do |f|
206
- if record.key?(f)
207
- @message_field = f
208
- break
209
- end
210
- end
211
- end
212
- message = @message_field.nil? ? record : record[@message_field]
211
+ message = extract_message(record)
213
212
  if message.nil?
214
213
  @exception_detector.reset
215
214
  detection_status = :no_trace
216
215
  else
216
+ force_flush if @max_bytes > 0 &&
217
+ @buffer_size + message.length > @max_bytes
217
218
  detection_status = @exception_detector.update(message)
218
219
  end
219
220
 
220
221
  update_buffer(detection_status, time_sec, record, message)
222
+
223
+ force_flush if @max_lines > 0 && @messages.length == @max_lines
221
224
  end
222
225
 
223
226
  def flush
@@ -239,6 +242,7 @@ module Fluent
239
242
  @messages = []
240
243
  @first_record = nil
241
244
  @first_timestamp = nil
245
+ @buffer_size = 0
242
246
  end
243
247
 
244
248
  def force_flush
@@ -246,12 +250,20 @@ module Fluent
246
250
  @exception_detector.reset
247
251
  end
248
252
 
249
- def length
250
- @messages.length
251
- end
252
-
253
253
  private
254
254
 
255
+ def extract_message(record)
256
+ if !@message_field.nil? && @message_field.empty?
257
+ ExceptionDetectorConfig::DEFAULT_FIELDS.each do |f|
258
+ if record.key?(f)
259
+ @message_field = f
260
+ break
261
+ end
262
+ end
263
+ end
264
+ @message_field.nil? ? record : record[@message_field]
265
+ end
266
+
255
267
  def update_buffer(detection_status, time_sec, record, message)
256
268
  trigger_emit = detection_status == :no_trace ||
257
269
  detection_status == :end_trace
@@ -259,6 +271,7 @@ module Fluent
259
271
  @emit.call(time_sec, record)
260
272
  return
261
273
  end
274
+
262
275
  case detection_status
263
276
  when :inside_trace
264
277
  add(time_sec, record, message)
@@ -281,7 +294,10 @@ module Fluent
281
294
  @first_timestamp = time_sec
282
295
  @buffer_start_time = Time.now
283
296
  end
284
- @messages << message unless message.nil?
297
+ unless message.nil?
298
+ @messages << message
299
+ @buffer_size += message.length
300
+ end
285
301
  end
286
302
  end
287
303
  end
@@ -30,8 +30,10 @@ module Fluent
30
30
  config_param :multiline_flush_interval, :time, default: nil
31
31
  desc 'Programming languages for which to detect exceptions. Default: all.'
32
32
  config_param :languages, :array, value_type: :string, default: []
33
- desc 'Maximum number of lines in a detected stack trace. Default: 1000.'
33
+ desc 'Maximum number of lines to flush (0 means no limit). Default: 1000.'
34
34
  config_param :max_lines, :integer, default: 1000
35
+ desc 'Maximum number of bytes to flush (0 means no limit). Default: 0.'
36
+ config_param :max_bytes, :integer, default: 0
35
37
 
36
38
  Fluent::Plugin.register_output('detect_exceptions', self)
37
39
 
@@ -85,14 +87,14 @@ module Fluent
85
87
  unless @accumulators.key?(tag)
86
88
  out_tag = tag.sub(/^#{Regexp.escape(remove_tag_prefix)}\./, '')
87
89
  @accumulators[tag] =
88
- Fluent::TraceAccumulator.new(message, @languages) do |t, r|
90
+ Fluent::TraceAccumulator.new(message, @languages,
91
+ max_lines: max_lines,
92
+ max_bytes: max_bytes) do |t, r|
89
93
  router.emit(out_tag, t, r)
90
94
  end
91
95
  end
92
96
 
93
97
  @accumulators[tag].push(time_sec, record)
94
-
95
- @accumulators[tag].force_flush if @accumulators[tag].length > max_lines
96
98
  end
97
99
  end
98
100
 
@@ -48,22 +48,22 @@ random_text = (1..(size / line_length)).collect do
48
48
  end
49
49
 
50
50
  exceptions = {
51
- 'java' => (JAVA_EXC * (size / JAVA_EXC.length)).lines,
52
- 'python' => (PYTHON_EXC * (size / PYTHON_EXC.length)).lines
51
+ java: (JAVA_EXC * (size / JAVA_EXC.length)).lines,
52
+ python: (PYTHON_EXC * (size / PYTHON_EXC.length)).lines
53
53
  }
54
54
 
55
55
  puts "Start benchmark. Input size #{size_in_m}M."
56
56
  Benchmark.bm do |x|
57
57
  languages = Fluent::ExceptionDetectorConfig::RULES_BY_LANG.keys
58
58
  languages.each do |lang|
59
- buffer = Fluent::ExceptionBuffer.new(nil, lang) {}
59
+ buffer = Fluent::TraceAccumulator.new(nil, lang) {}
60
60
  x.report("#{lang}_detector_random_text") do
61
61
  random_text.each { |l| buffer.push(0, l) }
62
62
  end
63
63
  end
64
- %w(java python all).each do |detector_lang|
65
- buffer = Fluent::ExceptionBuffer.new(nil, lang) {}
66
- exc_languages = detector_lang == 'all' ? exceptions.keys : [detector_lang]
64
+ [:java, :python, :all].each do |detector_lang|
65
+ buffer = Fluent::TraceAccumulator.new(nil, detector_lang) {}
66
+ exc_languages = detector_lang == :all ? exceptions.keys : [detector_lang]
67
67
  exc_languages.each do |exc_lang|
68
68
  x.report("#{detector_lang}_detector_#{exc_lang}_stacks") do
69
69
  exceptions[exc_lang].each { |l| buffer.push(0, l) }
@@ -107,6 +107,14 @@ Exception: ('spam', 'eggs')
107
107
  END
108
108
 
109
109
  PHP_EXC = <<END.freeze
110
+ exception 'Exception' with message 'Custom exception' in /home/joe/work/test-php/test.php:5
111
+ Stack trace:
112
+ #0 /home/joe/work/test-php/test.php(9): func1()
113
+ #1 /home/joe/work/test-php/test.php(13): func2()
114
+ #2 {main}
115
+ END
116
+
117
+ PHP_ON_GAE_EXC = <<END.freeze
110
118
  PHP Fatal error: Uncaught exception 'Exception' with message 'message' in /base/data/home/apps/s~crash-example-php/1.388306779641080894/errors.php:60
111
119
  Stack trace:
112
120
  #0 [internal function]: ErrorEntryGenerator::{closure}()
@@ -227,7 +235,8 @@ END
227
235
  end
228
236
 
229
237
  def test_php
230
- check_exception(PHP_EXC, true)
238
+ check_exception(PHP_EXC, false)
239
+ check_exception(PHP_ON_GAE_EXC, true)
231
240
  end
232
241
 
233
242
  def test_go
@@ -243,7 +252,8 @@ END
243
252
  check_exception(PYTHON_EXC, true)
244
253
  check_exception(COMPLEX_JAVA_EXC, false)
245
254
  check_exception(NODE_JS_EXC, false)
246
- check_exception(PHP_EXC, true)
255
+ check_exception(PHP_EXC, false)
256
+ check_exception(PHP_ON_GAE_EXC, true)
247
257
  check_exception(CLIENT_JS_EXC, false)
248
258
  check_exception(GO_EXC, false)
249
259
  check_exception(CSHARP_EXC, false)
@@ -350,4 +360,48 @@ END
350
360
  assert_equal(expected_json(s.actual_field, s.output), out, s.desc)
351
361
  end
352
362
  end
363
+
364
+ def test_max_lines_limit
365
+ # Limit is equal to the first part of the exception and forces it to be
366
+ # flushed before the rest of the exception is processed.
367
+ max_lines = JAVA_EXC_PART1.lines.length
368
+ out = []
369
+ buffer = Fluent::TraceAccumulator.new(nil,
370
+ [:all],
371
+ max_lines: max_lines) do |_, m|
372
+ out << m
373
+ end
374
+ feed_lines(buffer, JAVA_EXC)
375
+ assert_equal([JAVA_EXC_PART1] + JAVA_EXC_PART2.lines, out)
376
+ end
377
+
378
+ def test_high_max_bytes_limit
379
+ # Limit is just too small to add one more line to the buffered first part of
380
+ # the exception.
381
+ max_bytes = JAVA_EXC_PART1.length + JAVA_EXC_PART2.lines[0].length - 1
382
+ out = []
383
+ buffer = Fluent::TraceAccumulator.new(nil,
384
+ [:all],
385
+ max_bytes: max_bytes) do |_, m|
386
+ out << m
387
+ end
388
+ feed_lines(buffer, JAVA_EXC)
389
+ # Check that the trace is flushed after the first part.
390
+ assert_equal([JAVA_EXC_PART1] + JAVA_EXC_PART2.lines, out)
391
+ end
392
+
393
+ def test_low_max_bytes_limit
394
+ # Limit is exceeded by the character that follows the buffered first part of
395
+ # the exception.
396
+ max_bytes = JAVA_EXC_PART1.length
397
+ out = []
398
+ buffer = Fluent::TraceAccumulator.new(nil,
399
+ [:all],
400
+ max_bytes: max_bytes) do |_, m|
401
+ out << m
402
+ end
403
+ feed_lines(buffer, JAVA_EXC)
404
+ # Check that the trace is flushed after the first part.
405
+ assert_equal([JAVA_EXC_PART1] + JAVA_EXC_PART2.lines, out)
406
+ end
353
407
  end
@@ -166,8 +166,13 @@ END
166
166
  d.run do
167
167
  feed_lines(d, t, PYTHON_EXC, JAVA_EXC)
168
168
  end
169
+ # Expected: the first two lines of the exception are buffered and combined.
170
+ # Then the max_lines setting kicks in and the rest of the Python exception
171
+ # is logged line-by-line (since it's not an exception stack in itself).
172
+ # Finally, the Java exception is logged in its entirety, since it only
173
+ # has two lines.
169
174
  expected =
170
- [PYTHON_EXC.lines[0..2].join] + PYTHON_EXC.lines[3..-1] + [JAVA_EXC]
175
+ [PYTHON_EXC.lines[0..1].join] + PYTHON_EXC.lines[2..-1] + [JAVA_EXC]
171
176
  assert_equal(make_logs(t, *expected), d.events)
172
177
  end
173
178
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-detect-exceptions
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.1
4
+ version: 0.0.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Thomas Schickinger
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-11-17 00:00:00.000000000 Z
11
+ date: 2016-12-09 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd