fluent-plugin-detect-exceptions-with-webflux-support 0.0.13

data/lib/fluent/plugin/exception_detector.rb

@@ -0,0 +1,382 @@
+# Copyright 2016 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module Fluent
+  Struct.new('Rule', :from_states, :pattern, :to_state)
+
+  # Configuration of the state machine that detects exceptions.
+  module ExceptionDetectorConfig
+    # Rule for a state transition: if pattern matches go to the given state.
+    class RuleTarget
+      attr_accessor :pattern, :to_state
+
+      def initialize(pattern, state)
+        @pattern = pattern
+        @to_state = state
+      end
+
+      def ==(other)
+        other.class == self.class && other.state == state
+      end
+
+      alias eql? ==
+
+      def hash
+        state.hash
+      end
+
+      def state
+        [@pattern, @to_state]
+      end
+    end
+
+    def self.rule(from_state_or_states, pattern, to_state)
+      from_state_or_states = [from_state_or_states] unless
+        from_state_or_states.is_a?(Array)
+      Struct::Rule.new(from_state_or_states, pattern, to_state)
+    end
+
+    def self.supported
+      RULES_BY_LANG.keys
+    end
+
+    JAVA_RULES = [
+      rule(%i[start_state java_start_exception],
+           /(?:Exception|Error|Throwable|V8 errors stack trace)[:\r\n]/,
+           :java_after_exception),
+      rule(:java_after_exception, /^[\t ]*nested exception is:[\t ]*/,
+           :java_start_exception),
+      rule(:java_after_exception, /^[\r\n]*$/, :java_after_exception),
+      rule(%i[java_after_exception java], /^[\t ]+(?:eval )?at /, :java),
+      rule([:java_after_exception, :java], /^[\t ]+(?:eval )?\*__checkpoint/,
+      :java),
+
+      rule(%i[java_after_exception java],
+           # C# nested exception.
+           /^[\t ]+--- End of inner exception stack trace ---$/,
+           :java),
+
+      rule(%i[java_after_exception java],
+           # C# exception from async code.
+           /^--- End of stack trace from previous (?x:
+           )location where exception was thrown ---$/,
+           :java),
+
+      rule(%i[java_after_exception java], /^[\t ]*(?:Caused by|Suppressed):/,
+           :java_after_exception),
+      rule([:java_after_exception, :java],
+           /^(?:Error has been observed at the following site\(s\)):/,
+           :java),
+      rule([:java_after_exception, :java], /^(?:Original Stack Trace):/, :java),
+      rule(%i[java_after_exception java],
+           /^[\t ]*... \d+ (?:more|common frames omitted)/, :java)
+    ].freeze
+
+    PYTHON_RULES = [
+      rule(:start_state, /^Traceback \(most recent call last\):$/, :python),
+      rule(:python, /^[\t ]+File /, :python_code),
+      rule(:python_code, /[^\t ]/, :python),
+      rule(:python, /^(?:[^\s.():]+\.)*[^\s.():]+:/, :start_state)
+    ].freeze
+
+    PHP_RULES = [
+      rule(:start_state, /
+        (?:PHP\ (?:Notice|Parse\ error|Fatal\ error|Warning):)|
+        (?:exception\ '[^']+'\ with\ message\ ')/x, :php_stack_begin),
+      rule(:php_stack_begin, /^Stack trace:/, :php_stack_frames),
+      rule(:php_stack_frames, /^#\d/, :php_stack_frames),
+      rule(:php_stack_frames, /^\s+thrown in /, :start_state)
+    ].freeze
+
+    GO_RULES = [
+      rule(:start_state, /\bpanic: /, :go_after_panic),
+      rule(:start_state, /http: panic serving/, :go_goroutine),
+      rule(:go_after_panic, /^$/, :go_goroutine),
+      rule(%i[go_after_panic go_after_signal go_frame_line1],
+           /^$/, :go_goroutine),
+      rule(:go_after_panic, /^\[signal /, :go_after_signal),
+      rule(:go_goroutine, /^goroutine \d+ \[[^\]]+\]:$/, :go_frame_line1),
+      rule(:go_frame_line1, /^(?:[^\s.:]+\.)*[^\s.():]+\(|^created by /,
+           :go_frame_line2),
+      rule(:go_frame_line2, /^\s/, :go_frame_line1)
+    ].freeze
+
+    RUBY_RULES = [
+      rule(:start_state, /Error \(.*\):$/, :ruby_before_rails_trace),
+      rule(:ruby_before_rails_trace, /^  $/, :ruby),
+      rule(:ruby_before_rails_trace, /^[\t ]+.*?\.rb:\d+:in `/, :ruby),
+      rule(:ruby, /^[\t ]+.*?\.rb:\d+:in `/, :ruby)
+    ].freeze
+
+    DART_RULES = [
+      rule(:start_state, /^Unhandled exception:$/, :dart_exc),
+      rule(:dart_exc, /^Instance of/, :dart_stack),
+      rule(:dart_exc, /^Exception/, :dart_stack),
+      rule(:dart_exc, /^Bad state/, :dart_stack),
+      rule(:dart_exc, /^IntegerDivisionByZeroException/, :dart_stack),
+      rule(:dart_exc, /^Invalid argument/, :dart_stack),
+      rule(:dart_exc, /^RangeError/, :dart_stack),
+      rule(:dart_exc, /^Assertion failed/, :dart_stack),
+      rule(:dart_exc, /^Cannot instantiate/, :dart_stack),
+      rule(:dart_exc, /^Reading static variable/, :dart_stack),
+      rule(:dart_exc, /^UnimplementedError/, :dart_stack),
+      rule(:dart_exc, /^Unsupported operation/, :dart_stack),
+      rule(:dart_exc, /^Concurrent modification/, :dart_stack),
+      rule(:dart_exc, /^Out of Memory/, :dart_stack),
+      rule(:dart_exc, /^Stack Overflow/, :dart_stack),
+      rule(:dart_exc, /^'.+?':.+?$/, :dart_type_err_line1),
+      rule(:dart_type_err_line1, /^#\d+\s+.+?\(.+?\)$/, :dart_stack),
+      rule(:dart_type_err_line1, /^.+?$/, :dart_type_err_line2),
+      rule(:dart_type_err_line2, /^.*?\^.*?$/, :dart_type_err_line3),
+      rule(:dart_type_err_line3, /^$/, :dart_type_err_line4),
+      rule(:dart_type_err_line4, /^$/, :dart_stack),
+      rule(:dart_exc, /^FormatException/, :dart_format_err_line1),
+      rule(:dart_format_err_line1, /^#\d+\s+.+?\(.+?\)$/, :dart_stack),
+      rule(:dart_format_err_line1, /^./, :dart_format_err_line2),
+      rule(:dart_format_err_line2, /^.*?\^/, :dart_format_err_line3),
+      rule(:dart_format_err_line3, /^$/, :dart_stack),
+      rule(:dart_exc, /^NoSuchMethodError:/, :dart_method_err_line1),
+      rule(:dart_method_err_line1, /^Receiver:/, :dart_method_err_line2),
+      rule(:dart_method_err_line2, /^Tried calling:/, :dart_method_err_line3),
+      rule(:dart_method_err_line3, /^Found:/, :dart_stack),
+      rule(:dart_method_err_line3, /^#\d+\s+.+?\(.+?\)$/, :dart_stack),
+      rule(:dart_stack, /^#\d+\s+.+?\(.+?\)$/, :dart_stack),
+      rule(:dart_stack, /^<asynchronous suspension>$/, :dart_stack)
+    ].freeze
+
+    ALL_RULES = (
+      JAVA_RULES + PYTHON_RULES + PHP_RULES + GO_RULES + RUBY_RULES + DART_RULES
+    ).freeze
+
+    RULES_BY_LANG = {
+      java: JAVA_RULES,
+      javascript: JAVA_RULES,
+      js: JAVA_RULES,
+      csharp: JAVA_RULES,
+      py: PYTHON_RULES,
+      python: PYTHON_RULES,
+      php: PHP_RULES,
+      go: GO_RULES,
+      rb: RUBY_RULES,
+      ruby: RUBY_RULES,
+      dart: DART_RULES,
+      all: ALL_RULES
+    }.freeze
+
+    DEFAULT_FIELDS = %w[message log].freeze
+  end
+
+  # State machine that consumes individual log lines and detects
+  # multi-line stack traces.
+  class ExceptionDetector
+    def initialize(*languages)
+      @state = :start_state
+      @rules = Hash.new { |h, k| h[k] = [] }
+
+      languages = [:all] if languages.empty?
+
+      languages.each do |lang|
+        rule_config =
+          ExceptionDetectorConfig::RULES_BY_LANG.fetch(lang.downcase) do |_k|
+            raise ArgumentError, "Unknown language: #{lang}"
+          end
+
+        rule_config.each do |r|
+          target = ExceptionDetectorConfig::RuleTarget.new(r[:pattern],
+                                                           r[:to_state])
+          r[:from_states].each do |from_state|
+            @rules[from_state] << target
+          end
+        end
+      end
+
+      @rules.each_value(&:uniq!)
+    end
+
+    # Updates the state machine and returns the trace detection status:
+    # - no_trace: 'line' does not belong to an exception trace,
+    # - start_trace: 'line' starts a detected exception trace,
+    # - inside: 'line' is part of a detected exception trace,
+    # - end: the detected exception trace ends after 'line'.
+    def update(line)
+      trace_seen_before = transition(line)
+      # If the state machine fell back to the start state because there is no
+      # defined transition for 'line', trigger another state transition because
+      # 'line' may contain the beginning of another exception.
+      transition(line) unless trace_seen_before
+      new_state = @state
+      trace_seen_after = new_state != :start_state
+
+      case [trace_seen_before, trace_seen_after]
+      when [true, true]
+        :inside_trace
+      when [true, false]
+        :end_trace
+      when [false, true]
+        :start_trace
+      else
+        :no_trace
+      end
+    end
+
+    def reset
+      @state = :start_state
+    end
+
+    private
+
+    # Executes a transition of the state machine for the given line.
+    # Returns false if the line does not match any transition rule and the
+    # state machine was reset to the initial state.
+    def transition(line)
+      @rules[@state].each do |r|
+        next unless line =~ r.pattern
+
+        @state = r.to_state
+        return true
+      end
+      @state = :start_state
+      false
+    end
+  end
+
+  # Buffers and groups log records if they contain exception stack traces.
+  class TraceAccumulator
+    attr_reader :buffer_start_time
+
+    # If message_field is nil, the instance is set up to accumulate
+    # records that are plain strings (i.e. the whole record is concatenated).
+    # Otherwise, the instance accepts records that are dictionaries (usually
+    # originating from structured JSON logs) and accumulates just the
+    # content of the given message field.
+    # message_field may contain the empty string. In this case, the
+    # TraceAccumulator 'learns' the field name from the first record by checking
+    # for some pre-defined common field names of text logs.
+    # The option parameter can be used to pass the following parameters:
+    # force_line_breaks adds line breaks when combining exception stacks
+    # max_lines and max_bytes limit the maximum amount
+    # of data to be buffered. The default value 0 indicates 'no limit'.
+    def initialize(message_field, languages, **options, &emit_callback)
+      @exception_detector = Fluent::ExceptionDetector.new(*languages)
+      @message_field = message_field
+      @force_line_breaks = options[:force_line_breaks] || false
+      @max_lines = options[:max_lines]           || 0
+      @max_bytes = options[:max_bytes]           || 0
+      @emit = emit_callback
+      @messages = []
+      @buffer_start_time = Time.now
+      @buffer_size = 0
+      @first_record = nil
+      @first_timestamp = nil
+    end
+
+    def push(time_sec, record)
+      message = extract_message(record)
+      if message.nil?
+        @exception_detector.reset
+        detection_status = :no_trace
+      else
+        force_flush if @max_bytes.positive? &&
+                       @buffer_size + message.length > @max_bytes
+        detection_status = @exception_detector.update(message)
+      end
+
+      update_buffer(detection_status, time_sec, record, message)
+
+      force_flush if @max_lines.positive? && @messages.length == @max_lines
+    end
+
+    def flush
+      case @messages.length
+      when 0
+        return
+      when 1
+        @emit.call(@first_timestamp, @first_record)
+      else
+        combined_message = @messages.join
+        if @message_field.nil?
+          output_record = combined_message
+        else
+          output_record = @first_record
+          output_record[@message_field] = combined_message
+        end
+        @emit.call(@first_timestamp, output_record)
+      end
+      @messages = []
+      @first_record = nil
+      @first_timestamp = nil
+      @buffer_size = 0
+    end
+
+    def force_flush
+      flush
+      @exception_detector.reset
+    end
+
+    private
+
+    def extract_message(record)
+      if !@message_field.nil? && @message_field.empty?
+        ExceptionDetectorConfig::DEFAULT_FIELDS.each do |f|
+          if record.key?(f)
+            @message_field = f
+            break
+          end
+        end
+      end
+      @message_field.nil? ? record : record[@message_field]
+    end
+
+    def update_buffer(detection_status, time_sec, record, message)
+      trigger_emit = %i[no_trace end_trace].include?(detection_status)
+      if @messages.empty? && trigger_emit
+        @emit.call(time_sec, record)
+        return
+      end
+
+      case detection_status
+      when :inside_trace
+        add(time_sec, record, message)
+      when :end_trace
+        add(time_sec, record, message)
+        flush
+      when :no_trace
+        flush
+        add(time_sec, record, message)
+        flush
+      when :start_trace
+        flush
+        add(time_sec, record, message)
+      end
+    end
+
+    def add(time_sec, record, message)
+      if @messages.empty?
+        @first_record = record unless @message_field.nil?
+        @first_timestamp = time_sec
+        @buffer_start_time = Time.now
+      end
+      return if message.nil?
+
+      message_with_line_break =
+        if @force_line_breaks && !@messages.empty? && !message.include?("\n")
+          "\n#{message}"
+        else
+          message
+        end
+      @messages << message_with_line_break
+      @buffer_size += message_with_line_break.length
+    end
+  end
+end

data/lib/fluent/plugin/out_detect_exceptions_with_webflux_support.rb

@@ -0,0 +1,142 @@
+#
+# Copyright 2016 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'fluent/plugin/exception_detector'
+require 'fluent/output'
+
+module Fluent
+  # This output plugin consumes a log stream of JSON objects which contain
+  # single-line log messages. If a consecutive sequence of log messages form
+  # an exception stack trace, they forwarded as a single, combined JSON
+  # object. Otherwise, the input log data is forwarded as is.
+  class DetectExceptionsOutput < Output
+    desc 'The prefix to be removed from the input tag when outputting a record.'
+    config_param :remove_tag_prefix, :string
+    desc 'The field which contains the raw message text in the input JSON data.'
+    config_param :message, :string, default: ''
+    desc 'The interval of flushing the buffer for multiline format.'
+    config_param :multiline_flush_interval, :time, default: nil
+    desc 'Programming languages for which to detect exceptions. Default: all.'
+    config_param :languages, :array, value_type: :string, default: []
+    desc 'Force live breaks when combining exception stacks. Default: false.'
+    config_param :force_line_breaks, :bool, default: false
+    desc 'Maximum number of lines to flush (0 means no limit). Default: 1000.'
+    config_param :max_lines, :integer, default: 1000
+    desc 'Maximum number of bytes to flush (0 means no limit). Default: 0.'
+    config_param :max_bytes, :integer, default: 0
+    desc 'Separate log streams by this field in the input JSON data.'
+    config_param :stream, :string, default: ''
+
+    Fluent::Plugin.register_output('detect_exceptions_with_webflux_support', self)
+
+    def configure(conf)
+      super
+
+      @check_flush_interval = [multiline_flush_interval * 0.1, 1].max if multiline_flush_interval
+
+      @languages = languages.map(&:to_sym)
+
+      # Maps log stream tags to a corresponding TraceAccumulator.
+      @accumulators = {}
+    end
+
+    def start
+      super
+
+      return unless multiline_flush_interval
+
+      @flush_buffer_mutex = Mutex.new
+      @stop_check = false
+      @thread = Thread.new(&method(:check_flush_loop))
+    end
+
+    def before_shutdown
+      flush_buffers
+      super if defined?(super)
+    end
+
+    def shutdown
+      # Before shutdown is not available in older fluentd versions.
+      # Hence, we make sure that we flush the buffers here as well.
+      flush_buffers
+      @thread.join if @multiline_flush_interval
+      super
+    end
+
+    def emit(tag, entries, chain)
+      entries.each do |time_sec, record|
+        process_record(tag, time_sec, record)
+      end
+      chain.next
+    end
+
+    private
+
+    def process_record(tag, time_sec, record)
+      synchronize do
+        log_id = [tag]
+        log_id.push(record.fetch(@stream, '')) unless @stream.empty?
+        unless @accumulators.key?(log_id)
+          out_tag = tag.sub(/^#{Regexp.escape(@remove_tag_prefix)}\./, '')
+          @accumulators[log_id] =
+            Fluent::TraceAccumulator.new(
+              @message,
+              @languages,
+              force_line_breaks: @force_line_breaks,
+              max_lines: @max_lines,
+              max_bytes: @max_bytes
+            ) do |t, r|
+              router.emit(out_tag, t, r)
+            end
+        end
+
+        @accumulators[log_id].push(time_sec, record)
+      end
+    end
+
+    def flush_buffers
+      synchronize do
+        @stop_check = true
+        @accumulators.each_value(&:force_flush)
+      end
+    end
+
+    def check_flush_loop
+      @flush_buffer_mutex.synchronize do
+        loop do
+          @flush_buffer_mutex.sleep(@check_flush_interval)
+          now = Time.now
+          break if @stop_check
+
+          @accumulators.each_value do |acc|
+            acc.force_flush if now - acc.buffer_start_time >
+                               @multiline_flush_interval
+          end
+        end
+      end
+    rescue StandardError
+      log.error 'error in check_flush_loop', error: $ERROR_INFO.to_s
+      log.error_backtrace
+    end
+
+    def synchronize(&block)
+      if @multiline_flush_interval
+        @flush_buffer_mutex.synchronize(&block)
+      else
+        yield
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,46 @@
+# Copyright 2016 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  # rubocop:disable Style/StderrPuts
+  $stderr.puts e.message
+  $stderr.puts 'Run `bundle install` to install missing gems'
+  # rubocop:enable Style/StderrPuts
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+unless ENV.key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval do |_|
+    def respond_to_missing?(_method, _include_private = false)
+      true
+    end
+
+    def method_missing(_method, *_args)
+      # pass
+    end
+  end
+  # global $log variable is used by fluentd
+  $log = nulllogger # rubocop:disable Style/GlobalVars
+end
+
+require 'fluent/plugin/out_detect_exceptions_with_webflux_support'

data/test/plugin/bench_exception_detector.rb

@@ -0,0 +1,73 @@
+# Copyright 2016 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'benchmark'
+
+require 'fluent/plugin/exception_detector'
+
+size_in_m = 25
+line_length = 50
+
+size = size_in_m << 20
+
+JAVA_EXC = <<~END_JAVA.freeze
+  Jul 09, 2015 3:23:29 PM com.google.devtools.search.cloud.feeder.MakeLog: RuntimeException: Run from this message!
+    at com.my.app.Object.do$a1(MakeLog.java:50)
+    at java.lang.Thing.call(Thing.java:10)
+    at com.my.app.Object.help(MakeLog.java:40)
+    at sun.javax.API.method(API.java:100)
+    at com.jetty.Framework.main(MakeLog.java:30)
+END_JAVA
+
+PYTHON_EXC = <<~END_PYTHON.freeze
+  Traceback (most recent call last):
+    File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.5.2/webapp2.py", line 1535, in __call__
+      rv = self.handle_exception(request, response, e)
+    File "/base/data/home/apps/s~nearfieldspy/1.378705245900539993/nearfieldspy.py", line 17, in start
+      return get()
+    File "/base/data/home/apps/s~nearfieldspy/1.378705245900539993/nearfieldspy.py", line 5, in get
+      raise Exception('spam', 'eggs')
+  Exception: ('spam', 'eggs')
+END_PYTHON
+
+chars = [('a'..'z'), ('A'..'Z')].map(&:to_a).flatten
+
+random_text = (1..(size / line_length)).collect do
+  (0...line_length).map { chars[rand(chars.length)] }.join
+end
+
+exceptions = {
+  java: (JAVA_EXC * (size / JAVA_EXC.length)).lines,
+  python: (PYTHON_EXC * (size / PYTHON_EXC.length)).lines
+}
+
+puts "Start benchmark. Input size #{size_in_m}M."
+Benchmark.bm do |x|
+  languages = Fluent::ExceptionDetectorConfig::RULES_BY_LANG.keys
+  languages.each do |lang|
+    buffer = Fluent::TraceAccumulator.new(nil, lang) {}
+    x.report("#{lang}_detector_random_text") do
+      random_text.each { |l| buffer.push(0, l) }
+    end
+  end
+  %i[java python all].each do |detector_lang|
+    buffer = Fluent::TraceAccumulator.new(nil, detector_lang) {}
+    exc_languages = detector_lang == :all ? exceptions.keys : [detector_lang]
+    exc_languages.each do |exc_lang|
+      x.report("#{detector_lang}_detector_#{exc_lang}_stacks") do
+        exceptions[exc_lang].each { |l| buffer.push(0, l) }
+      end
+    end
+  end
+end