fluent-plugin-detect-exceptions 0.0.1 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3fc8ac42bca6d07ddc0c8131203df18a0e5eb2bf
-  data.tar.gz: ac0298c30ec7cc47eca96ccf29dcd13b47b56bcc
+  metadata.gz: 1a26a5e2085d4bc44296565c82c0baacc5c1e33f
+  data.tar.gz: b0268e7263ca8bb5d5bfb3cb8913c04acb3b5893
 SHA512:
-  metadata.gz: 57aa45c609ccecc24362272b55f99a58d5fa27b7c3eb85b73f52643755585c0a700917914b4c0e8d75b21f986df27b8fddf457a8604a0a0c4f0e1090cd8da204
-  data.tar.gz: 7d20fbf7734e321285738856b022663764440090dc4cfaef21e38b6850863af9b1b9cfa28a9821ae21a2a7c1121f0b6f4d8320e611dfcea6424553d0db3ee357
+  metadata.gz: c170490ab76e47973fa288d49280d0c8e7cc0232f99fea45b9a144f7b1dc1c6514ca54d9fb57bfcff7f83ef7a809ea50f0e1c4f93ec64a9690e9fa9e5b376d6a
+  data.tar.gz: ebd8383c06d79737a213c7756573db8b61044aaed9c0ea0262bdf6be5062cfa4740eaa192ca8e956fa69f2e391767b91d2c163c868696aa828a1eccffe8832b0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    fluent-plugin-detect-exceptions (0.0.1)
+    fluent-plugin-detect-exceptions (0.0.3)
       fluentd (~> 0.10, <= 0.13)
 
 GEM
@@ -22,9 +22,9 @@ GEM
     http_parser.rb (0.6.0)
     json (2.0.2)
     msgpack (0.5.12)
-    parser (2.3.1.4)
+    parser (2.3.3.1)
       ast (~> 2.2)
-    power_assert (0.3.1)
+    power_assert (0.4.1)
     powerpack (0.1.1)
     rainbow (2.1.0)
     rake (10.5.0)
@@ -37,14 +37,14 @@ GEM
     ruby-progressbar (1.8.1)
     sigdump (0.2.4)
     string-scrub (0.0.5)
-    test-unit (3.2.2)
+    test-unit (3.2.3)
       power_assert
     thread_safe (0.3.5)
     tzinfo (1.2.2)
       thread_safe (~> 0.1)
-    tzinfo-data (1.2016.9)
+    tzinfo-data (1.2016.10)
       tzinfo (>= 1.0.0)
-    unicode-display_width (1.1.1)
+    unicode-display_width (1.1.2)
     yajl-ruby (1.3.0)
 
 PLATFORMS

data/fluent-plugin-detect-exceptions.gemspec

@@ -11,7 +11,7 @@ eos
   gem.homepage      = \
     'https://github.com/GoogleCloudPlatform/fluent-plugin-detect-exceptions'
   gem.license       = 'Apache-2.0'
-  gem.version       = '0.0.1'
+  gem.version       = '0.0.3'
   gem.authors       = ['Thomas Schickinger']
   gem.email         = ['schickin@google.com']
   gem.required_ruby_version = Gem::Requirement.new('>= 2.0')

data/lib/fluent/plugin/exception_detector.rb

@@ -66,8 +66,9 @@ module Fluent
     ].freeze
 
     PHP_RULES = [
-      rule(:start_state, /PHP (?:Notice|Parse error|Fatal error|Warning):/,
-           :php_stack_begin),
+      rule(:start_state, /
+        (?:PHP\ (?:Notice|Parse\ error|Fatal\ error|Warning):)|
+        (?:exception\ '[^']+'\ with\ message\ ')/x, :php_stack_begin),
       rule(:php_stack_begin, /^Stack trace:/, :php_stack_frames),
       rule(:php_stack_frames, /^#\d/, :php_stack_frames),
       rule(:php_stack_frames, /^\s+thrown in /, :start_state)
@@ -190,34 +191,36 @@ module Fluent
     # message_field may contain the empty string. In this case, the
     # TraceAccumulator 'learns' the field name from the first record by checking
     # for some pre-defined common field names of text logs.
-    def initialize(message_field, languages, &emit_callback)
+    # The named parameters max_lines and max_bytes limit the maximum amount
+    # of data to be buffered. The default value 0 indicates 'no limit'.
+    def initialize(message_field, languages, max_lines: 0, max_bytes: 0,
+                   &emit_callback)
       @exception_detector = Fluent::ExceptionDetector.new(*languages)
+      @max_lines = max_lines
+      @max_bytes = max_bytes
       @message_field = message_field
       @messages = []
       @buffer_start_time = Time.now
+      @buffer_size = 0
       @first_record = nil
       @first_timestamp = nil
       @emit = emit_callback
     end
 
     def push(time_sec, record)
-      if !@message_field.nil? && @message_field.empty?
-        ExceptionDetectorConfig::DEFAULT_FIELDS.each do |f|
-          if record.key?(f)
-            @message_field = f
-            break
-          end
-        end
-      end
-      message = @message_field.nil? ? record : record[@message_field]
+      message = extract_message(record)
       if message.nil?
         @exception_detector.reset
         detection_status = :no_trace
       else
+        force_flush if @max_bytes > 0 &&
+                       @buffer_size + message.length > @max_bytes
         detection_status = @exception_detector.update(message)
       end
 
       update_buffer(detection_status, time_sec, record, message)
+
+      force_flush if @max_lines > 0 && @messages.length == @max_lines
     end
 
     def flush
@@ -239,6 +242,7 @@ module Fluent
       @messages = []
       @first_record = nil
       @first_timestamp = nil
+      @buffer_size = 0
     end
 
     def force_flush
@@ -246,12 +250,20 @@ module Fluent
       @exception_detector.reset
     end
 
-    def length
-      @messages.length
-    end
-
     private
 
+    def extract_message(record)
+      if !@message_field.nil? && @message_field.empty?
+        ExceptionDetectorConfig::DEFAULT_FIELDS.each do |f|
+          if record.key?(f)
+            @message_field = f
+            break
+          end
+        end
+      end
+      @message_field.nil? ? record : record[@message_field]
+    end
+
     def update_buffer(detection_status, time_sec, record, message)
       trigger_emit = detection_status == :no_trace ||
                      detection_status == :end_trace
@@ -259,6 +271,7 @@ module Fluent
         @emit.call(time_sec, record)
         return
       end
+
       case detection_status
       when :inside_trace
         add(time_sec, record, message)
@@ -281,7 +294,10 @@ module Fluent
         @first_timestamp = time_sec
         @buffer_start_time = Time.now
       end
-      @messages << message unless message.nil?
+      unless message.nil?
+        @messages << message
+        @buffer_size += message.length
+      end
     end
   end
 end

data/lib/fluent/plugin/out_detect_exceptions.rb

@@ -30,8 +30,10 @@ module Fluent
     config_param :multiline_flush_interval, :time, default: nil
     desc 'Programming languages for which to detect exceptions. Default: all.'
     config_param :languages, :array, value_type: :string, default: []
-    desc 'Maximum number of lines in a detected stack trace. Default: 1000.'
+    desc 'Maximum number of lines to flush (0 means no limit). Default: 1000.'
     config_param :max_lines, :integer, default: 1000
+    desc 'Maximum number of bytes to flush (0 means no limit). Default: 0.'
+    config_param :max_bytes, :integer, default: 0
 
     Fluent::Plugin.register_output('detect_exceptions', self)
 
@@ -85,14 +87,14 @@ module Fluent
         unless @accumulators.key?(tag)
           out_tag = tag.sub(/^#{Regexp.escape(remove_tag_prefix)}\./, '')
           @accumulators[tag] =
-            Fluent::TraceAccumulator.new(message, @languages) do |t, r|
+            Fluent::TraceAccumulator.new(message, @languages,
+                                         max_lines: max_lines,
+                                         max_bytes: max_bytes) do |t, r|
               router.emit(out_tag, t, r)
             end
         end
 
         @accumulators[tag].push(time_sec, record)
-
-        @accumulators[tag].force_flush if @accumulators[tag].length > max_lines
       end
     end
 

data/test/plugin/bench_exception_detector.rb

@@ -48,22 +48,22 @@ random_text = (1..(size / line_length)).collect do
 end
 
 exceptions = {
-  'java' => (JAVA_EXC * (size / JAVA_EXC.length)).lines,
-  'python' => (PYTHON_EXC * (size / PYTHON_EXC.length)).lines
+  java: (JAVA_EXC * (size / JAVA_EXC.length)).lines,
+  python: (PYTHON_EXC * (size / PYTHON_EXC.length)).lines
 }
 
 puts "Start benchmark. Input size #{size_in_m}M."
 Benchmark.bm do |x|
   languages = Fluent::ExceptionDetectorConfig::RULES_BY_LANG.keys
   languages.each do |lang|
-    buffer = Fluent::ExceptionBuffer.new(nil, lang) {}
+    buffer = Fluent::TraceAccumulator.new(nil, lang) {}
     x.report("#{lang}_detector_random_text") do
       random_text.each { |l| buffer.push(0, l) }
     end
   end
-  %w(java python all).each do |detector_lang|
-    buffer = Fluent::ExceptionBuffer.new(nil, lang) {}
-    exc_languages = detector_lang == 'all' ? exceptions.keys : [detector_lang]
+  [:java, :python, :all].each do |detector_lang|
+    buffer = Fluent::TraceAccumulator.new(nil, detector_lang) {}
+    exc_languages = detector_lang == :all ? exceptions.keys : [detector_lang]
     exc_languages.each do |exc_lang|
       x.report("#{detector_lang}_detector_#{exc_lang}_stacks") do
         exceptions[exc_lang].each { |l| buffer.push(0, l) }

data/test/plugin/test_exception_detector.rb

@@ -107,6 +107,14 @@ Exception: ('spam', 'eggs')
 END
 
   PHP_EXC = <<END.freeze
+exception 'Exception' with message 'Custom exception' in /home/joe/work/test-php/test.php:5
+Stack trace:
+#0 /home/joe/work/test-php/test.php(9): func1()
+#1 /home/joe/work/test-php/test.php(13): func2()
+#2 {main}
+END
+
+  PHP_ON_GAE_EXC = <<END.freeze
 PHP Fatal error:  Uncaught exception 'Exception' with message 'message' in /base/data/home/apps/s~crash-example-php/1.388306779641080894/errors.php:60
 Stack trace:
 #0 [internal function]: ErrorEntryGenerator::{closure}()
@@ -227,7 +235,8 @@ END
   end
 
   def test_php
-    check_exception(PHP_EXC, true)
+    check_exception(PHP_EXC, false)
+    check_exception(PHP_ON_GAE_EXC, true)
   end
 
   def test_go
@@ -243,7 +252,8 @@ END
     check_exception(PYTHON_EXC, true)
     check_exception(COMPLEX_JAVA_EXC, false)
     check_exception(NODE_JS_EXC, false)
-    check_exception(PHP_EXC, true)
+    check_exception(PHP_EXC, false)
+    check_exception(PHP_ON_GAE_EXC, true)
     check_exception(CLIENT_JS_EXC, false)
     check_exception(GO_EXC, false)
     check_exception(CSHARP_EXC, false)
@@ -350,4 +360,48 @@ END
       assert_equal(expected_json(s.actual_field, s.output), out, s.desc)
     end
   end
+
+  def test_max_lines_limit
+    # Limit is equal to the first part of the exception and forces it to be
+    # flushed before the rest of the exception is processed.
+    max_lines = JAVA_EXC_PART1.lines.length
+    out = []
+    buffer = Fluent::TraceAccumulator.new(nil,
+                                          [:all],
+                                          max_lines: max_lines) do |_, m|
+      out << m
+    end
+    feed_lines(buffer, JAVA_EXC)
+    assert_equal([JAVA_EXC_PART1] + JAVA_EXC_PART2.lines, out)
+  end
+
+  def test_high_max_bytes_limit
+    # Limit is just too small to add one more line to the buffered first part of
+    # the exception.
+    max_bytes = JAVA_EXC_PART1.length + JAVA_EXC_PART2.lines[0].length - 1
+    out = []
+    buffer = Fluent::TraceAccumulator.new(nil,
+                                          [:all],
+                                          max_bytes: max_bytes) do |_, m|
+      out << m
+    end
+    feed_lines(buffer, JAVA_EXC)
+    # Check that the trace is flushed after the first part.
+    assert_equal([JAVA_EXC_PART1] + JAVA_EXC_PART2.lines, out)
+  end
+
+  def test_low_max_bytes_limit
+    # Limit is exceeded by the character that follows the buffered first part of
+    # the exception.
+    max_bytes = JAVA_EXC_PART1.length
+    out = []
+    buffer = Fluent::TraceAccumulator.new(nil,
+                                          [:all],
+                                          max_bytes: max_bytes) do |_, m|
+      out << m
+    end
+    feed_lines(buffer, JAVA_EXC)
+    # Check that the trace is flushed after the first part.
+    assert_equal([JAVA_EXC_PART1] + JAVA_EXC_PART2.lines, out)
+  end
 end

data/test/plugin/test_out_detect_exceptions.rb

@@ -166,8 +166,13 @@ END
     d.run do
       feed_lines(d, t, PYTHON_EXC, JAVA_EXC)
     end
+    # Expected: the first two lines of the exception are buffered and combined.
+    # Then the max_lines setting kicks in and the rest of the Python exception
+    # is logged line-by-line (since it's not an exception stack in itself).
+    # Finally, the Java exception is logged in its entirety, since it only
+    # has two lines.
     expected =
-      [PYTHON_EXC.lines[0..2].join] + PYTHON_EXC.lines[3..-1] + [JAVA_EXC]
+      [PYTHON_EXC.lines[0..1].join] + PYTHON_EXC.lines[2..-1] + [JAVA_EXC]
     assert_equal(make_logs(t, *expected), d.events)
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-detect-exceptions
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.3
 platform: ruby
 authors:
 - Thomas Schickinger
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-17 00:00:00.000000000 Z
+date: 2016-12-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd