fluent-plugin-detect-exceptions 0.0.1

data/lib/fluent/plugin/exception_detector.rb

@@ -0,0 +1,287 @@
+# Copyright 2016 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module Fluent
+  Struct.new('Rule', :from_state, :pattern, :to_state)
+
+  # Configuration of the state machine that detects exceptions.
+  module ExceptionDetectorConfig
+    # Rule for a state transition: if pattern matches go to the given state.
+    class RuleTarget
+      attr_accessor :pattern, :to_state
+
+      def initialize(p, s)
+        @pattern = p
+        @to_state = s
+      end
+
+      def ==(other)
+        other.class == self.class && other.state == state
+      end
+
+      alias eql? ==
+
+      def hash
+        state.hash
+      end
+
+      def state
+        [@pattern, @to_state]
+      end
+    end
+
+    def self.rule(from_state, pattern, to_state)
+      Struct::Rule.new(from_state, pattern, to_state)
+    end
+
+    def self.supported
+      RULES_BY_LANG.keys
+    end
+
+    JAVA_RULES = [
+      rule(:start_state,
+           /(?:Exception|Error|Throwable|V8 errors stack trace)[:\r\n]/,
+           :java),
+      rule(:java, /^[\t ]+(?:eval )?at /, :java),
+      rule(:java, /^[\t ]*(?:Caused by|Suppressed):/, :java),
+      rule(:java, /^[\t ]*... \d+\ more/, :java)
+    ].freeze
+
+    PYTHON_RULES = [
+      rule(:start_state, /Traceback \(most recent call last\)/, :python),
+      rule(:python, /^[\t ]+File /, :python_code),
+      rule(:python_code, /[^\t ]/, :python),
+      rule(:python, /^(?:[^\s.():]+\.)*[^\s.():]+:/, :start_state)
+    ].freeze
+
+    PHP_RULES = [
+      rule(:start_state, /PHP (?:Notice|Parse error|Fatal error|Warning):/,
+           :php_stack_begin),
+      rule(:php_stack_begin, /^Stack trace:/, :php_stack_frames),
+      rule(:php_stack_frames, /^#\d/, :php_stack_frames),
+      rule(:php_stack_frames, /^\s+thrown in /, :start_state)
+    ].freeze
+
+    GO_RULES = [
+      rule(:start_state, /panic: /, :go_before_goroutine),
+      rule(:go_before_goroutine, /^$/, :go_goroutine),
+      rule(:go_goroutine, /^goroutine \d+ \[[^\]]+\]:$/, :go_frame_1),
+      rule(:go_frame_1, /(?:[^\s.():]+\.)*[^\s.():]\(/, :go_frame_2),
+      rule(:go_frame_1, /^$/, :go_before_goroutine),
+      rule(:go_frame_2, /^\s/, :go_frame_1)
+    ].freeze
+
+    RUBY_RULES = [
+      rule(:start_state, /Error \(.*\):$/, :ruby),
+      rule(:ruby, /^[\t ]+.*?\.rb:\d+:in `/, :ruby)
+    ].freeze
+
+    ALL_RULES = (
+      JAVA_RULES + PYTHON_RULES + PHP_RULES + GO_RULES + RUBY_RULES).freeze
+
+    RULES_BY_LANG = {
+      java: JAVA_RULES,
+      javascript: JAVA_RULES,
+      js: JAVA_RULES,
+      csharp: JAVA_RULES,
+      py: PYTHON_RULES,
+      python: PYTHON_RULES,
+      php: PHP_RULES,
+      go: GO_RULES,
+      rb: RUBY_RULES,
+      ruby: RUBY_RULES,
+      all: ALL_RULES
+    }.freeze
+
+    DEFAULT_FIELDS = %w(message log).freeze
+  end
+
+  # State machine that consumes individual log lines and detects
+  # multi-line stack traces.
+  class ExceptionDetector
+    def initialize(*languages)
+      @state = :start_state
+      @rules = Hash.new { |h, k| h[k] = [] }
+
+      languages = [:all] if languages.empty?
+
+      languages.each do |lang|
+        rule_config =
+          ExceptionDetectorConfig::RULES_BY_LANG.fetch(lang.downcase) do |_k|
+            raise ArgumentError, "Unknown language: #{lang}"
+          end
+
+        rule_config.each do |r|
+          target = ExceptionDetectorConfig::RuleTarget.new(r[:pattern],
+                                                           r[:to_state])
+          @rules[r[:from_state]] << target
+        end
+      end
+
+      @rules.each_value(&:uniq!)
+    end
+
+    # Updates the state machine and returns the trace detection status:
+    # - no_trace: 'line' does not belong to an exception trace,
+    # - start_trace: 'line' starts a detected exception trace,
+    # - inside: 'line' is part of a detected exception trace,
+    # - end: the detected exception trace ends after 'line'.
+    def update(line)
+      trace_seen_before = transition(line)
+      # If the state machine fell back to the start state because there is no
+      # defined transition for 'line', trigger another state transition because
+      # 'line' may contain the beginning of another exception.
+      transition(line) unless trace_seen_before
+      new_state = @state
+      trace_seen_after = new_state != :start_state
+
+      case [trace_seen_before, trace_seen_after]
+      when [true, true]
+        :inside_trace
+      when [true, false]
+        :end_trace
+      when [false, true]
+        :start_trace
+      else
+        :no_trace
+      end
+    end
+
+    def reset
+      @state = :start_state
+    end
+
+    private
+
+    # Executes a transition of the state machine for the given line.
+    # Returns false if the line does not match any transition rule and the
+    # state machine was reset to the initial state.
+    def transition(line)
+      @rules[@state].each do |r|
+        next unless line =~ r.pattern
+        @state = r.to_state
+        return true
+      end
+      @state = :start_state
+      false
+    end
+  end
+
+  # Buffers and groups log records if they contain exception stack traces.
+  class TraceAccumulator
+    attr_reader :buffer_start_time
+
+    # If message_field is nil, the instance is set up to accumulate
+    # records that are plain strings (i.e. the whole record is concatenated).
+    # Otherwise, the instance accepts records that are dictionaries (usually
+    # originating from structured JSON logs) and accumulates just the
+    # content of the given message field.
+    # message_field may contain the empty string. In this case, the
+    # TraceAccumulator 'learns' the field name from the first record by checking
+    # for some pre-defined common field names of text logs.
+    def initialize(message_field, languages, &emit_callback)
+      @exception_detector = Fluent::ExceptionDetector.new(*languages)
+      @message_field = message_field
+      @messages = []
+      @buffer_start_time = Time.now
+      @first_record = nil
+      @first_timestamp = nil
+      @emit = emit_callback
+    end
+
+    def push(time_sec, record)
+      if !@message_field.nil? && @message_field.empty?
+        ExceptionDetectorConfig::DEFAULT_FIELDS.each do |f|
+          if record.key?(f)
+            @message_field = f
+            break
+          end
+        end
+      end
+      message = @message_field.nil? ? record : record[@message_field]
+      if message.nil?
+        @exception_detector.reset
+        detection_status = :no_trace
+      else
+        detection_status = @exception_detector.update(message)
+      end
+
+      update_buffer(detection_status, time_sec, record, message)
+    end
+
+    def flush
+      case @messages.length
+      when 0
+        return
+      when 1
+        @emit.call(@first_timestamp, @first_record)
+      else
+        combined_message = @messages.join
+        if @message_field.nil?
+          output_record = combined_message
+        else
+          output_record = @first_record
+          output_record[@message_field] = combined_message
+        end
+        @emit.call(@first_timestamp, output_record)
+      end
+      @messages = []
+      @first_record = nil
+      @first_timestamp = nil
+    end
+
+    def force_flush
+      flush
+      @exception_detector.reset
+    end
+
+    def length
+      @messages.length
+    end
+
+    private
+
+    def update_buffer(detection_status, time_sec, record, message)
+      trigger_emit = detection_status == :no_trace ||
+                     detection_status == :end_trace
+      if @messages.empty? && trigger_emit
+        @emit.call(time_sec, record)
+        return
+      end
+      case detection_status
+      when :inside_trace
+        add(time_sec, record, message)
+      when :end_trace
+        add(time_sec, record, message)
+        flush
+      when :no_trace
+        flush
+        add(time_sec, record, message)
+        flush
+      when :start_trace
+        flush
+        add(time_sec, record, message)
+      end
+    end
+
+    def add(time_sec, record, message)
+      if @messages.empty?
+        @first_record = record unless @message_field.nil?
+        @first_timestamp = time_sec
+        @buffer_start_time = Time.now
+      end
+      @messages << message unless message.nil?
+    end
+  end
+end

data/lib/fluent/plugin/out_detect_exceptions.rb

@@ -0,0 +1,131 @@
+#
+# Copyright 2016 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'fluent/plugin/exception_detector'
+require 'fluent/output'
+
+module Fluent
+  # This output plugin consumes a log stream of JSON objects which contain
+  # single-line log messages. If a consecutive sequence of log messages form
+  # an exception stack trace, they forwarded as a single, combined JSON
+  # object. Otherwise, the input log data is forwarded as is.
+  class DetectExceptionsOutput < Output
+    desc 'The field which contains the raw message text in the input json data.'
+    config_param :message, :string, default: ''
+    desc 'The prefix to be removed from the input tag when outputting a record.'
+    config_param :remove_tag_prefix, :string, default: ''
+    desc 'The interval of flushing the buffer for multiline format.'
+    config_param :multiline_flush_interval, :time, default: nil
+    desc 'Programming languages for which to detect exceptions. Default: all.'
+    config_param :languages, :array, value_type: :string, default: []
+    desc 'Maximum number of lines in a detected stack trace. Default: 1000.'
+    config_param :max_lines, :integer, default: 1000
+
+    Fluent::Plugin.register_output('detect_exceptions', self)
+
+    def configure(conf)
+      super
+
+      if multiline_flush_interval
+        @check_flush_interval = [multiline_flush_interval * 0.1, 1].max
+      end
+
+      @languages = languages.map(&:to_sym)
+
+      # Maps log stream tags to a corresponding TraceAccumulator.
+      @accumulators = {}
+    end
+
+    def start
+      super
+
+      if multiline_flush_interval
+        @flush_buffer_mutex = Mutex.new
+        @stop_check = false
+        @thread = Thread.new(&method(:check_flush_loop))
+      end
+    end
+
+    def before_shutdown
+      flush_buffers
+      super if defined?(super)
+    end
+
+    def shutdown
+      # Before shutdown is not available in older fluentd versions.
+      # Hence, we make sure that we flush the buffers here as well.
+      flush_buffers
+      @thread.join if multiline_flush_interval
+      super
+    end
+
+    def emit(tag, es, chain)
+      es.each do |time_sec, record|
+        process_record(tag, time_sec, record)
+      end
+      chain.next
+    end
+
+    private
+
+    def process_record(tag, time_sec, record)
+      synchronize do
+        unless @accumulators.key?(tag)
+          out_tag = tag.sub(/^#{Regexp.escape(remove_tag_prefix)}\./, '')
+          @accumulators[tag] =
+            Fluent::TraceAccumulator.new(message, @languages) do |t, r|
+              router.emit(out_tag, t, r)
+            end
+        end
+
+        @accumulators[tag].push(time_sec, record)
+
+        @accumulators[tag].force_flush if @accumulators[tag].length > max_lines
+      end
+    end
+
+    def flush_buffers
+      synchronize do
+        @stop_check = true
+        @accumulators.each_value(&:force_flush)
+      end
+    end
+
+    def check_flush_loop
+      @flush_buffer_mutex.synchronize do
+        loop do
+          @flush_buffer_mutex.sleep(@check_flush_interval)
+          now = Time.now
+          break if @stop_check
+          @accumulators.each_value do |acc|
+            needs_flush = now - acc.buffer_start_time > multiline_flush_interval
+            acc.force_flush if needs_flush
+          end
+        end
+      end
+    rescue
+      log.error 'error in check_flush_loop', error: $ERROR_INFO.to_s
+      log.error_backtrace
+    end
+
+    def synchronize(&block)
+      if multiline_flush_interval
+        @flush_buffer_mutex.synchronize(&block)
+      else
+        yield
+      end
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,46 @@
+# Copyright 2016 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'rubygems'
+require 'bundler'
+
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts 'Run `bundle install` to install missing gems'
+  exit e.status_code
+end
+
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+
+unless ENV.key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval do |_|
+    def respond_to_missing?
+      true
+    end
+
+    def method_missing(_method, *_args) # rubocop:disable Style/MethodMissing
+    end
+  end
+  # global $log variable is used by fluentd
+  $log = nulllogger # rubocop:disable Style/GlobalVars
+end
+
+require 'fluent/plugin/out_detect_exceptions'