fluent-plugin-detect-exceptions-with-error 0.0.1a

data/lib/fluent/plugin/exception_detector.rb

@@ -0,0 +1,302 @@
+# Copyright 2016 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module Fluent
+  Struct.new('Rule', :from_state, :pattern, :to_state)
+
+  # Configuration of the state machine that detects exceptions.
+  module ExceptionDetectorConfig
+    # Rule for a state transition: if pattern matches go to the given state.
+    class RuleTarget
+      attr_accessor :pattern, :to_state
+
+      def initialize(p, s)
+        @pattern = p
+        @to_state = s
+      end
+
+      def ==(other)
+        other.class == self.class && other.state == state
+      end
+
+      alias eql? ==
+
+      def hash
+        state.hash
+      end
+
+      def state
+        [@pattern, @to_state]
+      end
+    end
+
+    def self.rule(from_state, pattern, to_state)
+      Struct::Rule.new(from_state, pattern, to_state)
+    end
+
+    def self.supported
+      RULES_BY_LANG.keys
+    end
+
+    JAVA_RULES = [
+      rule(:start_state, /(?:ERROR)[:\r\n]/, :java),
+      rule(:java, /(?:Exception|Error|Throwable|V8 errors stack trace)[:\r\n]/, :java),
+      rule(:java, /^[\t ]+(?:eval )?at /, :java),
+      rule(:java, /^[\t ]*(?:Caused by|Suppressed):/, :java),
+      rule(:java, /^[\t ]*... \d+\ more/, :java)
+    ].freeze
+
+    PYTHON_RULES = [
+      rule(:start_state, /Traceback \(most recent call last\)/, :python),
+      rule(:python, /^[\t ]+File /, :python_code),
+      rule(:python_code, /[^\t ]/, :python),
+      rule(:python, /^(?:[^\s.():]+\.)*[^\s.():]+:/, :start_state)
+    ].freeze
+
+    PHP_RULES = [
+      rule(:start_state, /
+        (?:PHP\ (?:Notice|Parse\ error|Fatal\ error|Warning):)|
+        (?:exception\ '[^']+'\ with\ message\ ')/x, :php_stack_begin),
+      rule(:php_stack_begin, /^Stack trace:/, :php_stack_frames),
+      rule(:php_stack_frames, /^#\d/, :php_stack_frames),
+      rule(:php_stack_frames, /^\s+thrown in /, :start_state)
+    ].freeze
+
+    GO_RULES = [
+      rule(:start_state, /panic: /, :go_before_goroutine),
+      rule(:go_before_goroutine, /^$/, :go_goroutine),
+      rule(:go_goroutine, /^goroutine \d+ \[[^\]]+\]:$/, :go_frame_1),
+      rule(:go_frame_1, /(?:[^\s.():]+\.)*[^\s.():]\(/, :go_frame_2),
+      rule(:go_frame_1, /^$/, :go_before_goroutine),
+      rule(:go_frame_2, /^\s/, :go_frame_1)
+    ].freeze
+
+    RUBY_RULES = [
+      rule(:start_state, /Error \(.*\):$/, :ruby),
+      rule(:ruby, /^[\t ]+.*?\.rb:\d+:in `/, :ruby)
+    ].freeze
+
+    ALL_RULES = (
+      JAVA_RULES + PYTHON_RULES + PHP_RULES + GO_RULES + RUBY_RULES).freeze
+
+    RULES_BY_LANG = {
+      java: JAVA_RULES,
+      javascript: JAVA_RULES,
+      js: JAVA_RULES,
+      csharp: JAVA_RULES,
+      py: PYTHON_RULES,
+      python: PYTHON_RULES,
+      php: PHP_RULES,
+      go: GO_RULES,
+      rb: RUBY_RULES,
+      ruby: RUBY_RULES,
+      all: ALL_RULES
+    }.freeze
+
+    DEFAULT_FIELDS = %w(message log).freeze
+  end
+
+  # State machine that consumes individual log lines and detects
+  # multi-line stack traces.
+  class ExceptionDetector
+    def initialize(*languages)
+      @state = :start_state
+      @rules = Hash.new { |h, k| h[k] = [] }
+
+      languages = [:all] if languages.empty?
+
+      languages.each do |lang|
+        rule_config =
+          ExceptionDetectorConfig::RULES_BY_LANG.fetch(lang.downcase) do |_k|
+            raise ArgumentError, "Unknown language: #{lang}"
+          end
+
+        rule_config.each do |r|
+          target = ExceptionDetectorConfig::RuleTarget.new(r[:pattern],
+                                                           r[:to_state])
+          @rules[r[:from_state]] << target
+        end
+      end
+
+      @rules.each_value(&:uniq!)
+    end
+
+    # Updates the state machine and returns the trace detection status:
+    # - no_trace: 'line' does not belong to an exception trace,
+    # - start_trace: 'line' starts a detected exception trace,
+    # - inside: 'line' is part of a detected exception trace,
+    # - end: the detected exception trace ends after 'line'.
+    def update(line)
+      trace_seen_before = transition(line)
+      # If the state machine fell back to the start state because there is no
+      # defined transition for 'line', trigger another state transition because
+      # 'line' may contain the beginning of another exception.
+      transition(line) unless trace_seen_before
+      new_state = @state
+      trace_seen_after = new_state != :start_state
+
+      case [trace_seen_before, trace_seen_after]
+      when [true, true]
+        :inside_trace
+      when [true, false]
+        :end_trace
+      when [false, true]
+        :start_trace
+      else
+        :no_trace
+      end
+    end
+
+    def reset
+      @state = :start_state
+    end
+
+    private
+
+    # Executes a transition of the state machine for the given line.
+    # Returns false if the line does not match any transition rule and the
+    # state machine was reset to the initial state.
+    def transition(line)
+      @rules[@state].each do |r|
+        next unless line =~ r.pattern
+        @state = r.to_state
+        return true
+      end
+      @state = :start_state
+      false
+    end
+  end
+
+  # Buffers and groups log records if they contain exception stack traces.
+  class TraceAccumulator
+    attr_reader :buffer_start_time
+
+    # If message_field is nil, the instance is set up to accumulate
+    # records that are plain strings (i.e. the whole record is concatenated).
+    # Otherwise, the instance accepts records that are dictionaries (usually
+    # originating from structured JSON logs) and accumulates just the
+    # content of the given message field.
+    # message_field may contain the empty string. In this case, the
+    # TraceAccumulator 'learns' the field name from the first record by checking
+    # for some pre-defined common field names of text logs.
+    # The named parameters max_lines and max_bytes limit the maximum amount
+    # of data to be buffered. The default value 0 indicates 'no limit'.
+    def initialize(message_field, languages, max_lines: 0, max_bytes: 0,
+                   &emit_callback)
+      @exception_detector = Fluent::ExceptionDetector.new(*languages)
+      @max_lines = max_lines
+      @max_bytes = max_bytes
+      @message_field = message_field
+      @messages = []
+      @buffer_start_time = Time.now
+      @buffer_size = 0
+      @first_record = nil
+      @first_timestamp = nil
+      @emit = emit_callback
+    end
+
+    def push(time_sec, record)
+      message = extract_message(record)
+      if message.nil?
+        @exception_detector.reset
+        detection_status = :no_trace
+      else
+        force_flush if @max_bytes > 0 &&
+                       @buffer_size + message.length > @max_bytes
+        detection_status = @exception_detector.update(message)
+      end
+
+      update_buffer(detection_status, time_sec, record, message)
+
+      force_flush if @max_lines > 0 && @messages.length == @max_lines
+    end
+
+    def flush
+      case @messages.length
+      when 0
+        return
+      when 1
+        @emit.call(@first_timestamp, @first_record)
+      else
+        combined_message = @messages.join
+        if @message_field.nil?
+          output_record = combined_message
+        else
+          output_record = @first_record
+          output_record[@message_field] = combined_message
+        end
+        @emit.call(@first_timestamp, output_record)
+      end
+      @messages = []
+      @first_record = nil
+      @first_timestamp = nil
+      @buffer_size = 0
+    end
+
+    def force_flush
+      flush
+      @exception_detector.reset
+    end
+
+    private
+
+    def extract_message(record)
+      if !@message_field.nil? && @message_field.empty?
+        ExceptionDetectorConfig::DEFAULT_FIELDS.each do |f|
+          if record.key?(f)
+            @message_field = f
+            break
+          end
+        end
+      end
+      @message_field.nil? ? record : record[@message_field]
+    end
+
+    def update_buffer(detection_status, time_sec, record, message)
+      trigger_emit = detection_status == :no_trace ||
+                     detection_status == :end_trace
+      if @messages.empty? && trigger_emit
+        @emit.call(time_sec, record)
+        return
+      end
+
+      case detection_status
+      when :inside_trace
+        add(time_sec, record, message)
+      when :end_trace
+        add(time_sec, record, message)
+        flush
+      when :no_trace
+        flush
+        add(time_sec, record, message)
+        flush
+      when :start_trace
+        flush
+        add(time_sec, record, message)
+      end
+    end
+
+    def add(time_sec, record, message)
+      if @messages.empty?
+        @first_record = record unless @message_field.nil?
+        @first_timestamp = time_sec
+        @buffer_start_time = Time.now
+      end
+      unless message.nil?
+        @messages << message
+        @buffer_size += message.length
+      end
+    end
+  end
+end

data/lib/fluent/plugin/out_detect_exceptions.rb

@@ -0,0 +1,137 @@
+#
+# Copyright 2016 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'fluent/plugin/exception_detector'
+require 'fluent/output'
+
+module Fluent
+  # This output plugin consumes a log stream of JSON objects which contain
+  # single-line log messages. If a consecutive sequence of log messages form
+  # an exception stack trace, they forwarded as a single, combined JSON
+  # object. Otherwise, the input log data is forwarded as is.
+  class DetectExceptionsOutput < Output
+    desc 'The field which contains the raw message text in the input JSON data.'
+    config_param :message, :string, default: ''
+    desc 'The prefix to be removed from the input tag when outputting a record.'
+    config_param :remove_tag_prefix, :string, default: ''
+    desc 'The interval of flushing the buffer for multiline format.'
+    config_param :multiline_flush_interval, :time, default: nil
+    desc 'Programming languages for which to detect exceptions. Default: all.'
+    config_param :languages, :array, value_type: :string, default: []
+    desc 'Maximum number of lines to flush (0 means no limit). Default: 1000.'
+    config_param :max_lines, :integer, default: 1000
+    desc 'Maximum number of bytes to flush (0 means no limit). Default: 0.'
+    config_param :max_bytes, :integer, default: 0
+    desc 'Separate log streams by this field in the input JSON data.'
+    config_param :stream, :string, default: ''
+
+    Fluent::Plugin.register_output('detect_exceptions', self)
+
+    def configure(conf)
+      super
+
+      if multiline_flush_interval
+        @check_flush_interval = [multiline_flush_interval * 0.1, 1].max
+      end
+
+      @languages = languages.map(&:to_sym)
+
+      # Maps log stream tags to a corresponding TraceAccumulator.
+      @accumulators = {}
+    end
+
+    def start
+      super
+
+      if multiline_flush_interval
+        @flush_buffer_mutex = Mutex.new
+        @stop_check = false
+        @thread = Thread.new(&method(:check_flush_loop))
+      end
+    end
+
+    def before_shutdown
+      flush_buffers
+      super if defined?(super)
+    end
+
+    def shutdown
+      # Before shutdown is not available in older fluentd versions.
+      # Hence, we make sure that we flush the buffers here as well.
+      flush_buffers
+      @thread.join if @multiline_flush_interval
+      super
+    end
+
+    def emit(tag, es, chain)
+      es.each do |time_sec, record|
+        process_record(tag, time_sec, record)
+      end
+      chain.next
+    end
+
+    private
+
+    def process_record(tag, time_sec, record)
+      synchronize do
+        log_id = [tag]
+        log_id.push(record.fetch(@stream, '')) unless @stream.empty?
+        unless @accumulators.key?(log_id)
+          out_tag = tag.sub(/^#{Regexp.escape(@remove_tag_prefix)}\./, '')
+          @accumulators[log_id] =
+            Fluent::TraceAccumulator.new(@message, @languages,
+                                         max_lines: @max_lines,
+                                         max_bytes: @max_bytes) do |t, r|
+              router.emit(out_tag, t, r)
+            end
+        end
+
+        @accumulators[log_id].push(time_sec, record)
+      end
+    end
+
+    def flush_buffers
+      synchronize do
+        @stop_check = true
+        @accumulators.each_value(&:force_flush)
+      end
+    end
+
+    def check_flush_loop
+      @flush_buffer_mutex.synchronize do
+        loop do
+          @flush_buffer_mutex.sleep(@check_flush_interval)
+          now = Time.now
+          break if @stop_check
+          @accumulators.each_value do |acc|
+            acc.force_flush if now - acc.buffer_start_time >
+                               @multiline_flush_interval
+          end
+        end
+      end
+    rescue
+      log.error 'error in check_flush_loop', error: $ERROR_INFO.to_s
+      log.error_backtrace
+    end
+
+    def synchronize(&block)
+      if @multiline_flush_interval
+        @flush_buffer_mutex.synchronize(&block)
+      else
+        yield
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,46 @@
+# Copyright 2016 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'rubygems'
+require 'bundler'
+
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts 'Run `bundle install` to install missing gems'
+  exit e.status_code
+end
+
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+
+unless ENV.key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval do |_|
+    def respond_to_missing?
+      true
+    end
+
+    def method_missing(_method, *_args) # rubocop:disable Style/MethodMissing
+    end
+  end
+  # global $log variable is used by fluentd
+  $log = nulllogger # rubocop:disable Style/GlobalVars
+end
+
+require 'fluent/plugin/out_detect_exceptions'

data/test/plugin/bench_exception_detector.rb

@@ -0,0 +1,73 @@
+# Copyright 2016 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'benchmark'
+
+require 'fluent/plugin/exception_detector'
+
+size_in_m = 25
+line_length = 50
+
+size = size_in_m << 20
+
+JAVA_EXC = <<END.freeze
+Jul 09, 2015 3:23:29 PM com.google.devtools.search.cloud.feeder.MakeLog: RuntimeException: Run from this message!
+  at com.my.app.Object.do$a1(MakeLog.java:50)
+  at java.lang.Thing.call(Thing.java:10)
+  at com.my.app.Object.help(MakeLog.java:40)
+  at sun.javax.API.method(API.java:100)
+  at com.jetty.Framework.main(MakeLog.java:30)
+END
+
+PYTHON_EXC = <<END.freeze
+Traceback (most recent call last):
+  File "/base/data/home/runtimes/python27/python27_lib/versions/third_party/webapp2-2.5.2/webapp2.py", line 1535, in __call__
+    rv = self.handle_exception(request, response, e)
+  File "/base/data/home/apps/s~nearfieldspy/1.378705245900539993/nearfieldspy.py", line 17, in start
+    return get()
+  File "/base/data/home/apps/s~nearfieldspy/1.378705245900539993/nearfieldspy.py", line 5, in get
+    raise Exception('spam', 'eggs')
+Exception: ('spam', 'eggs')
+END
+
+chars = [('a'..'z'), ('A'..'Z')].map(&:to_a).flatten
+
+random_text = (1..(size / line_length)).collect do
+  (0...line_length).map { chars[rand(chars.length)] }.join
+end
+
+exceptions = {
+  java: (JAVA_EXC * (size / JAVA_EXC.length)).lines,
+  python: (PYTHON_EXC * (size / PYTHON_EXC.length)).lines
+}
+
+puts "Start benchmark. Input size #{size_in_m}M."
+Benchmark.bm do |x|
+  languages = Fluent::ExceptionDetectorConfig::RULES_BY_LANG.keys
+  languages.each do |lang|
+    buffer = Fluent::TraceAccumulator.new(nil, lang) {}
+    x.report("#{lang}_detector_random_text") do
+      random_text.each { |l| buffer.push(0, l) }
+    end
+  end
+  [:java, :python, :all].each do |detector_lang|
+    buffer = Fluent::TraceAccumulator.new(nil, detector_lang) {}
+    exc_languages = detector_lang == :all ? exceptions.keys : [detector_lang]
+    exc_languages.each do |exc_lang|
+      x.report("#{detector_lang}_detector_#{exc_lang}_stacks") do
+        exceptions[exc_lang].each { |l| buffer.push(0, l) }
+      end
+    end
+  end
+end