fluent-plugin-multi-exceptions 0.0.13

data/Rakefile

@@ -0,0 +1,48 @@
+#!/usr/bin/env rake
+
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+require 'rubocop/rake_task'
+
+desc 'Run Rubocop to check for style violations'
+RuboCop::RakeTask.new
+
+desc 'Run benchmark tests'
+Rake::TestTask.new(:bench) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/plugin/bench*.rb']
+  test.verbose = true
+end
+
+desc 'Run unit tests'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/plugin/test*.rb']
+  test.verbose = true
+end
+
+# Building the gem will use the local file mode, so ensure it's world-readable.
+# https://github.com/GoogleCloudPlatform/fluent-plugin-detect-exceptions/issues/32
+desc 'Fix file permissions'
+task :fix_perms do
+  files = [
+    'lib/fluent/plugin/*.rb'
+  ].flat_map do |file|
+    file.include?('*') ? Dir.glob(file) : [file]
+  end
+
+  files.each do |file|
+    mode = File.stat(file).mode & 0o777
+    next unless mode & 0o444 != 0o444
+    puts "Changing mode of #{file} from #{mode.to_s(8)} to "\
+         "#{(mode | 0o444).to_s(8)}"
+    chmod mode | 0o444, file
+  end
+end
+
+desc 'Run unit tests and RuboCop to check for style violations'
+task all: [:test, :rubocop, :fix_perms]
+
+task default: :all

data/fluent-plugin-multi-exceptions.gemspec

@@ -0,0 +1,29 @@
+Gem::Specification.new do |gem|
+  gem.name          = 'fluent-plugin-multi-exceptions'
+  gem.description   = <<-eos
+   Fluentd output plugin which detects exception stack traces in a stream of
+   JSON log messages and combines all single-line messages that belong to the
+   same stack trace into one multi-line message.
+   This is Not an official Google Ruby gem. Added Multiworker to true
+eos
+  gem.summary       = \
+    'fluentd output plugin for combining stack traces as multi-line JSON logs'
+  gem.homepage      = \
+    'https://github.com/GoogleCloudPlatform/fluent-plugin-detect-exceptions'
+  gem.license       = 'Apache-2.0'
+  gem.version       = '0.0.13'
+  gem.authors       = ['Deloitte Agents']
+  gem.email         = ['gatolgaj@gmail.com']
+  gem.required_ruby_version = Gem::Requirement.new('>= 2.0')
+
+  gem.files         = Dir['**/*'].keep_if { |file| File.file?(file) }
+  gem.test_files    = gem.files.grep(/^(test)/)
+  gem.require_paths = ['lib']
+
+  gem.add_runtime_dependency 'fluentd', '>= 0.10'
+
+  gem.add_development_dependency 'rake', '~> 10.3'
+  gem.add_development_dependency 'rubocop', '= 0.42.0'
+  gem.add_development_dependency 'test-unit', '~> 3.0'
+  gem.add_development_dependency 'flexmock', '~> 2.0'
+end

data/lib/fluent/plugin/exception_detector.rb

@@ -0,0 +1,368 @@
+# Copyright 2016 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module Fluent
+  Struct.new('Rule', :from_states, :pattern, :to_state)
+
+  # Configuration of the state machine that detects exceptions.
+  module ExceptionDetectorConfig
+    # Rule for a state transition: if pattern matches go to the given state.
+    class RuleTarget
+      attr_accessor :pattern, :to_state
+
+      def initialize(p, s)
+        @pattern = p
+        @to_state = s
+      end
+
+      def ==(other)
+        other.class == self.class && other.state == state
+      end
+
+      alias eql? ==
+
+      def hash
+        state.hash
+      end
+
+      def state
+        [@pattern, @to_state]
+      end
+    end
+
+    def self.rule(from_state_or_states, pattern, to_state)
+      from_state_or_states = [from_state_or_states] unless
+        from_state_or_states.is_a?(Array)
+      Struct::Rule.new(from_state_or_states, pattern, to_state)
+    end
+
+    def self.supported
+      RULES_BY_LANG.keys
+    end
+
+    JAVA_RULES = [
+      rule([:start_state, :java_start_exception],
+           /(?:Exception|Error|Throwable|V8 errors stack trace)[:\r\n]/,
+           :java_after_exception),
+      rule(:java_after_exception, /^[\t ]*nested exception is:[\t ]*/,
+           :java_start_exception),
+      rule(:java_after_exception, /^[\r\n]*$/, :java_after_exception),
+      rule([:java_after_exception, :java], /^[\t ]+(?:eval )?at /, :java),
+
+      rule([:java_after_exception, :java],
+           # C# nested exception.
+           /^[\t ]+--- End of inner exception stack trace ---$/,
+           :java),
+
+      rule([:java_after_exception, :java],
+           # C# exception from async code.
+           /^--- End of stack trace from previous (?x:
+           )location where exception was thrown ---$/,
+           :java),
+
+      rule([:java_after_exception, :java], /^[\t ]*(?:Caused by|Suppressed):/,
+           :java_after_exception),
+      rule([:java_after_exception, :java],
+           /^[\t ]*... \d+ (?:more|common frames omitted)/, :java)
+    ].freeze
+
+    PYTHON_RULES = [
+      rule(:start_state, /^Traceback \(most recent call last\):$/, :python),
+      rule(:python, /^[\t ]+File /, :python_code),
+      rule(:python_code, /[^\t ]/, :python),
+      rule(:python, /^(?:[^\s.():]+\.)*[^\s.():]+:/, :start_state)
+    ].freeze
+
+    PHP_RULES = [
+      rule(:start_state, /
+        (?:PHP\ (?:Notice|Parse\ error|Fatal\ error|Warning):)|
+        (?:exception\ '[^']+'\ with\ message\ ')/x, :php_stack_begin),
+      rule(:php_stack_begin, /^Stack trace:/, :php_stack_frames),
+      rule(:php_stack_frames, /^#\d/, :php_stack_frames),
+      rule(:php_stack_frames, /^\s+thrown in /, :start_state)
+    ].freeze
+
+    GO_RULES = [
+      rule(:start_state, /\bpanic: /, :go_after_panic),
+      rule(:start_state, /http: panic serving/, :go_goroutine),
+      rule(:go_after_panic, /^$/, :go_goroutine),
+      rule([:go_after_panic, :go_after_signal, :go_frame_1],
+           /^$/, :go_goroutine),
+      rule(:go_after_panic, /^\[signal /, :go_after_signal),
+      rule(:go_goroutine, /^goroutine \d+ \[[^\]]+\]:$/, :go_frame_1),
+      rule(:go_frame_1, /^(?:[^\s.:]+\.)*[^\s.():]+\(|^created by /,
+           :go_frame_2),
+      rule(:go_frame_2, /^\s/, :go_frame_1)
+    ].freeze
+
+    RUBY_RULES = [
+      rule(:start_state, /Error \(.*\):$/, :ruby_before_rails_trace),
+      rule(:ruby_before_rails_trace, /^  $/, :ruby),
+      rule(:ruby_before_rails_trace, /^[\t ]+.*?\.rb:\d+:in `/, :ruby),
+      rule(:ruby, /^[\t ]+.*?\.rb:\d+:in `/, :ruby)
+    ].freeze
+
+    DART_RULES = [
+      rule(:start_state, /^Unhandled exception:$/, :dart_exc),
+      rule(:dart_exc, /^Instance of/, :dart_stack),
+      rule(:dart_exc, /^Exception/, :dart_stack),
+      rule(:dart_exc, /^Bad state/, :dart_stack),
+      rule(:dart_exc, /^IntegerDivisionByZeroException/, :dart_stack),
+      rule(:dart_exc, /^Invalid argument/, :dart_stack),
+      rule(:dart_exc, /^RangeError/, :dart_stack),
+      rule(:dart_exc, /^Assertion failed/, :dart_stack),
+      rule(:dart_exc, /^Cannot instantiate/, :dart_stack),
+      rule(:dart_exc, /^Reading static variable/, :dart_stack),
+      rule(:dart_exc, /^UnimplementedError/, :dart_stack),
+      rule(:dart_exc, /^Unsupported operation/, :dart_stack),
+      rule(:dart_exc, /^Concurrent modification/, :dart_stack),
+      rule(:dart_exc, /^Out of Memory/, :dart_stack),
+      rule(:dart_exc, /^Stack Overflow/, :dart_stack),
+      rule(:dart_exc, /^'.+?':.+?$/, :dart_type_err_1),
+      rule(:dart_type_err_1, /^#\d+\s+.+?\(.+?\)$/, :dart_stack),
+      rule(:dart_type_err_1, /^.+?$/, :dart_type_err_2),
+      rule(:dart_type_err_2, /^.*?\^.*?$/, :dart_type_err_3),
+      rule(:dart_type_err_3, /^$/, :dart_type_err_4),
+      rule(:dart_type_err_4, /^$/, :dart_stack),
+      rule(:dart_exc, /^FormatException/, :dart_format_err_1),
+      rule(:dart_format_err_1, /^#\d+\s+.+?\(.+?\)$/, :dart_stack),
+      rule(:dart_format_err_1, /^./, :dart_format_err_2),
+      rule(:dart_format_err_2, /^.*?\^/, :dart_format_err_3),
+      rule(:dart_format_err_3, /^$/, :dart_stack),
+      rule(:dart_exc, /^NoSuchMethodError:/, :dart_method_err_1),
+      rule(:dart_method_err_1, /^Receiver:/, :dart_method_err_2),
+      rule(:dart_method_err_2, /^Tried calling:/, :dart_method_err_3),
+      rule(:dart_method_err_3, /^Found:/, :dart_stack),
+      rule(:dart_method_err_3, /^#\d+\s+.+?\(.+?\)$/, :dart_stack),
+      rule(:dart_stack, /^#\d+\s+.+?\(.+?\)$/, :dart_stack),
+      rule(:dart_stack, /^<asynchronous suspension>$/, :dart_stack)
+    ].freeze
+
+    ALL_RULES = (
+      JAVA_RULES + PYTHON_RULES + PHP_RULES + GO_RULES + RUBY_RULES + DART_RULES
+    ).freeze
+
+    RULES_BY_LANG = {
+      java: JAVA_RULES,
+      javascript: JAVA_RULES,
+      js: JAVA_RULES,
+      csharp: JAVA_RULES,
+      py: PYTHON_RULES,
+      python: PYTHON_RULES,
+      php: PHP_RULES,
+      go: GO_RULES,
+      rb: RUBY_RULES,
+      ruby: RUBY_RULES,
+      dart: DART_RULES,
+      all: ALL_RULES
+    }.freeze
+
+    DEFAULT_FIELDS = %w(message log).freeze
+  end
+
+  # State machine that consumes individual log lines and detects
+  # multi-line stack traces.
+  class ExceptionDetector
+    def initialize(*languages)
+      @state = :start_state
+      @rules = Hash.new { |h, k| h[k] = [] }
+
+      languages = [:all] if languages.empty?
+
+      languages.each do |lang|
+        rule_config =
+          ExceptionDetectorConfig::RULES_BY_LANG.fetch(lang.downcase) do |_k|
+            raise ArgumentError, "Unknown language: #{lang}"
+          end
+
+        rule_config.each do |r|
+          target = ExceptionDetectorConfig::RuleTarget.new(r[:pattern],
+                                                           r[:to_state])
+          r[:from_states].each do |from_state|
+            @rules[from_state] << target
+          end
+        end
+      end
+
+      @rules.each_value(&:uniq!)
+    end
+
+    # Updates the state machine and returns the trace detection status:
+    # - no_trace: 'line' does not belong to an exception trace,
+    # - start_trace: 'line' starts a detected exception trace,
+    # - inside: 'line' is part of a detected exception trace,
+    # - end: the detected exception trace ends after 'line'.
+    def update(line)
+      trace_seen_before = transition(line)
+      # If the state machine fell back to the start state because there is no
+      # defined transition for 'line', trigger another state transition because
+      # 'line' may contain the beginning of another exception.
+      transition(line) unless trace_seen_before
+      new_state = @state
+      trace_seen_after = new_state != :start_state
+
+      case [trace_seen_before, trace_seen_after]
+      when [true, true]
+        :inside_trace
+      when [true, false]
+        :end_trace
+      when [false, true]
+        :start_trace
+      else
+        :no_trace
+      end
+    end
+
+    def reset
+      @state = :start_state
+    end
+
+    private
+
+    # Executes a transition of the state machine for the given line.
+    # Returns false if the line does not match any transition rule and the
+    # state machine was reset to the initial state.
+    def transition(line)
+      @rules[@state].each do |r|
+        next unless line =~ r.pattern
+        @state = r.to_state
+        return true
+      end
+      @state = :start_state
+      false
+    end
+  end
+
+  # Buffers and groups log records if they contain exception stack traces.
+  class TraceAccumulator
+    attr_reader :buffer_start_time
+
+    # If message_field is nil, the instance is set up to accumulate
+    # records that are plain strings (i.e. the whole record is concatenated).
+    # Otherwise, the instance accepts records that are dictionaries (usually
+    # originating from structured JSON logs) and accumulates just the
+    # content of the given message field.
+    # message_field may contain the empty string. In this case, the
+    # TraceAccumulator 'learns' the field name from the first record by checking
+    # for some pre-defined common field names of text logs.
+    # The named parameters max_lines and max_bytes limit the maximum amount
+    # of data to be buffered. The default value 0 indicates 'no limit'.
+    def initialize(message_field, languages, max_lines: 0, max_bytes: 0,
+                   &emit_callback)
+      @exception_detector = Fluent::ExceptionDetector.new(*languages)
+      @max_lines = max_lines
+      @max_bytes = max_bytes
+      @message_field = message_field
+      @messages = []
+      @buffer_start_time = Time.now
+      @buffer_size = 0
+      @first_record = nil
+      @first_timestamp = nil
+      @emit = emit_callback
+    end
+
+    def push(time_sec, record)
+      message = extract_message(record)
+      if message.nil?
+        @exception_detector.reset
+        detection_status = :no_trace
+      else
+        force_flush if @max_bytes > 0 &&
+                       @buffer_size + message.length > @max_bytes
+        detection_status = @exception_detector.update(message)
+      end
+
+      update_buffer(detection_status, time_sec, record, message)
+
+      force_flush if @max_lines > 0 && @messages.length == @max_lines
+    end
+
+    def flush
+      case @messages.length
+      when 0
+        return
+      when 1
+        @emit.call(@first_timestamp, @first_record)
+      else
+        combined_message = @messages.join
+        if @message_field.nil?
+          output_record = combined_message
+        else
+          output_record = @first_record
+          output_record[@message_field] = combined_message
+        end
+        @emit.call(@first_timestamp, output_record)
+      end
+      @messages = []
+      @first_record = nil
+      @first_timestamp = nil
+      @buffer_size = 0
+    end
+
+    def force_flush
+      flush
+      @exception_detector.reset
+    end
+
+    private
+
+    def extract_message(record)
+      if !@message_field.nil? && @message_field.empty?
+        ExceptionDetectorConfig::DEFAULT_FIELDS.each do |f|
+          if record.key?(f)
+            @message_field = f
+            break
+          end
+        end
+      end
+      @message_field.nil? ? record : record[@message_field]
+    end
+
+    def update_buffer(detection_status, time_sec, record, message)
+      trigger_emit = detection_status == :no_trace ||
+                     detection_status == :end_trace
+      if @messages.empty? && trigger_emit
+        @emit.call(time_sec, record)
+        return
+      end
+
+      case detection_status
+      when :inside_trace
+        add(time_sec, record, message)
+      when :end_trace
+        add(time_sec, record, message)
+        flush
+      when :no_trace
+        flush
+        add(time_sec, record, message)
+        flush
+      when :start_trace
+        flush
+        add(time_sec, record, message)
+      end
+    end
+
+    def add(time_sec, record, message)
+      if @messages.empty?
+        @first_record = record unless @message_field.nil?
+        @first_timestamp = time_sec
+        @buffer_start_time = Time.now
+      end
+      unless message.nil?
+        @messages << message
+        @buffer_size += message.length
+      end
+    end
+  end
+end

data/lib/fluent/plugin/out_detect_exceptions.rb

@@ -0,0 +1,141 @@
+#
+# Copyright 2016 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'fluent/plugin/exception_detector'
+require 'fluent/output'
+
+module Fluent
+  # This output plugin consumes a log stream of JSON objects which contain
+  # single-line log messages. If a consecutive sequence of log messages form
+  # an exception stack trace, they forwarded as a single, combined JSON
+  # object. Otherwise, the input log data is forwarded as is.
+  class DetectExceptionsOutput < Output
+    desc 'The prefix to be removed from the input tag when outputting a record.'
+    config_param :remove_tag_prefix, :string
+    desc 'The field which contains the raw message text in the input JSON data.'
+    config_param :message, :string, default: ''
+    desc 'The interval of flushing the buffer for multiline format.'
+    config_param :multiline_flush_interval, :time, default: nil
+    desc 'Programming languages for which to detect exceptions. Default: all.'
+    config_param :languages, :array, value_type: :string, default: []
+    desc 'Maximum number of lines to flush (0 means no limit). Default: 1000.'
+    config_param :max_lines, :integer, default: 1000
+    desc 'Maximum number of bytes to flush (0 means no limit). Default: 0.'
+    config_param :max_bytes, :integer, default: 0
+    desc 'Separate log streams by this field in the input JSON data.'
+    config_param :stream, :string, default: ''
+
+    Fluent::Plugin.register_output('detect_exceptions', self)
+
+    def configure(conf)
+      super
+
+      if multiline_flush_interval
+        @check_flush_interval = [multiline_flush_interval * 0.1, 1].max
+      end
+
+      @languages = languages.map(&:to_sym)
+
+      # Maps log stream tags to a corresponding TraceAccumulator.
+      @accumulators = {}
+    end
+    
+    def multi_workers_ready?
+      true
+    end
+
+    def start
+      super
+
+      if multiline_flush_interval
+        @flush_buffer_mutex = Mutex.new
+        @stop_check = false
+        @thread = Thread.new(&method(:check_flush_loop))
+      end
+    end
+
+    def before_shutdown
+      flush_buffers
+      super if defined?(super)
+    end
+
+    def shutdown
+      # Before shutdown is not available in older fluentd versions.
+      # Hence, we make sure that we flush the buffers here as well.
+      flush_buffers
+      @thread.join if @multiline_flush_interval
+      super
+    end
+
+    def emit(tag, es, chain)
+      es.each do |time_sec, record|
+        process_record(tag, time_sec, record)
+      end
+      chain.next
+    end
+
+    private
+
+    def process_record(tag, time_sec, record)
+      synchronize do
+        log_id = [tag]
+        log_id.push(record.fetch(@stream, '')) unless @stream.empty?
+        unless @accumulators.key?(log_id)
+          out_tag = tag.sub(/^#{Regexp.escape(@remove_tag_prefix)}\./, '')
+          @accumulators[log_id] =
+            Fluent::TraceAccumulator.new(@message, @languages,
+                                         max_lines: @max_lines,
+                                         max_bytes: @max_bytes) do |t, r|
+              router.emit(out_tag, t, r)
+            end
+        end
+
+        @accumulators[log_id].push(time_sec, record)
+      end
+    end
+
+    def flush_buffers
+      synchronize do
+        @stop_check = true
+        @accumulators.each_value(&:force_flush)
+      end
+    end
+
+    def check_flush_loop
+      @flush_buffer_mutex.synchronize do
+        loop do
+          @flush_buffer_mutex.sleep(@check_flush_interval)
+          now = Time.now
+          break if @stop_check
+          @accumulators.each_value do |acc|
+            acc.force_flush if now - acc.buffer_start_time >
+                               @multiline_flush_interval
+          end
+        end
+      end
+    rescue
+      log.error 'error in check_flush_loop', error: $ERROR_INFO.to_s
+      log.error_backtrace
+    end
+
+    def synchronize(&block)
+      if @multiline_flush_interval
+        @flush_buffer_mutex.synchronize(&block)
+      else
+        yield
+      end
+    end
+  end
+end