fluent-plugin-detect-ft-memb-exceptions 0.0.3

data/fluent-plugin-detect-ft-memb-exceptions.gemspec

@@ -0,0 +1,28 @@
+Gem::Specification.new do |gem|
+  gem.name          = 'fluent-plugin-detect-ft-memb-exceptions'
+  gem.description   = <<-eos
+   Fluentd output plugin which detects ft membership specific exception stack traces in a stream of
+   JSON log messages and combines all single-line messages that belong to the
+   same stack trace into one multi-line message.
+   This is an adaption of an official Google Ruby gem.
+eos
+  gem.summary       = \
+    'fluentd output plugin for combining stack traces as multi-line JSON logs'
+  gem.homepage      = \
+    'https://github.com/Financial-Times/fluent-plugin-detect-exceptions'
+  gem.license       = 'Apache-2.0'
+  gem.version       = '0.0.3'
+  gem.authors       = ['Naomi stern']
+  gem.email         = ['sally.dixon@ft.com']
+  gem.required_ruby_version = Gem::Requirement.new('>= 2.0')
+
+  gem.files         = Dir['**/*'].keep_if { |file| File.file?(file) }
+  gem.test_files    = gem.files.grep(/^(test)/)
+  gem.require_paths = ['lib']
+
+  gem.add_runtime_dependency 'fluentd', '~> 0.10'
+
+  gem.add_development_dependency 'rake', '~> 10.3'
+  gem.add_development_dependency 'rubocop', '= 0.42.0'
+  gem.add_development_dependency 'test-unit', '~> 3.0'
+end

data/lib/fluent/plugin/exception_detector.rb

@@ -0,0 +1,328 @@
+# Copyright 2016 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+module Fluent
+  Struct.new('Rule', :from_state, :pattern, :to_state)
+
+  # Configuration of the state machine that detects exceptions.
+  module ExceptionDetectorConfig
+    # Rule for a state transition: if pattern matches go to the given state.
+    class RuleTarget
+      attr_accessor :pattern, :to_state
+
+      def initialize(p, s)
+        @pattern = p
+        @to_state = s
+      end
+
+      def ==(other)
+        other.class == self.class && other.state == state
+      end
+
+      alias eql? ==
+
+      def hash
+        state.hash
+      end
+
+      def state
+        [@pattern, @to_state]
+      end
+    end
+
+    def self.rule(from_state, pattern, to_state)
+      Struct::Rule.new(from_state, pattern, to_state)
+    end
+
+    def self.supported
+      RULES_BY_LANG.keys
+    end
+
+    JAVA_RULES = [
+      rule(:start_state, /(?:Exception|Error|Throwable|V8 errors stack trace)[:\r\n]/, :java),
+      rule(:start_state, /(?:ERROR|WARN)(\s+\[)/, :java_stack_begin),
+      rule(:java_stack_begin, /(?:Exception|Error|Throwable|V8 errors stack trace)[:\r\n]/, :java),
+      rule(:java, /^[\t ]+(?:eval )?at /, :java),
+      rule(:java, /^(?:eval )?! at/, :java),
+      rule(:java, /^[\t ]*(?:Caused by|Suppressed):/, :java),
+      rule(:java, /^[\t ]*... \d+\ (more|common frames omitted)/, :java)
+    ].freeze
+
+    KAFKA_RULES = [
+      rule(:start_state, /(?:ERROR|WARN)(\s+\[).*kafka/, :kafka_failure_info),
+      rule(:start_state, /^[0-9]{4}-[0-1][0-9]-[0-3][0-9]T[0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{3}Z (?:ERROR|WARN)/, :kafka_failure_info),
+      rule(:kafka_failure_info, /NoKafkaConnectionError/, :kafka_failure_info),
+      rule(:kafka_failure_info, /^(\s)*server: 'kafka/, :kafka_failure_info),
+      rule(:kafka_failure_info, /^(\s)*message:/, :kafka_failure_info),
+      rule(:kafka_failure_info, /^Message-Timestamp: /, :kafka_failure_info),
+      rule(:kafka_failure_info, /^Message-Type: /, :kafka_failure_info),
+      rule(:kafka_failure_info, /^Origin-System-Id: /, :kafka_failure_info),
+      rule(:kafka_failure_info, /^Origin-Host-Location: /, :kafka_failure_info),
+      rule(:kafka_failure_info, /^Content-Type: /, :kafka_failure_info),
+      rule(:kafka_failure_info, /^Origin-Host: /, :kafka_failure_info),
+      rule(:kafka_failure_info, /^{*.+}+*.partitionKey*.+topic/, :kafka_failure_info),
+      rule(:kafka_failure_info, /^Message-Id: [a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}/, :java),
+      rule(:kafka_failure_info, /^\[at /, :java),
+      rule(:kafka_failure_info, /(?:Exception|Error|Throwable|V8 errors stack trace)[:\r\n]/, :java),
+      rule(:java, /^(?:eval )?! at/, :java)      
+    ].freeze
+
+    PYTHON_RULES = [
+      rule(:start_state, /Traceback \(most recent call last\)/, :python),
+      rule(:python, /^[\t ]+File /, :python_code),
+      rule(:python_code, /[^\t ]/, :python),
+      rule(:python, /^(?:[^\s.():]+\.)*[^\s.():]+:/, :start_state)
+    ].freeze
+
+    ELIXIR_RULES = [
+      rule(:start_state, /^\d{2}:\d{2}:\d{2}\.\d{3} \[error\] /, :elixir_failure_info),
+      rule(:elixir_failure_info, /^[^\d]/, :elixir_failure_info)
+    ].freeze
+
+    PHP_RULES = [
+      rule(:start_state, /
+        (?:PHP\ (?:Notice|Parse\ error|Fatal\ error|Warning):)|
+        (?:exception\ '[^']+'\ with\ message\ ')/x, :php_stack_begin),
+      rule(:php_stack_begin, /^Stack trace:/, :php_stack_frames),
+      rule(:php_stack_frames, /^#\d/, :php_stack_frames),
+      rule(:php_stack_frames, /^\s+thrown in /, :start_state)
+    ].freeze
+
+    GO_RULES = [
+      rule(:start_state, /panic: /, :go_before_goroutine),
+      rule(:go_before_goroutine, /^$/, :go_goroutine),
+      rule(:go_goroutine, /^goroutine \d+ \[[^\]]+\]:$/, :go_frame_1),
+      rule(:go_frame_1, /(?:[^\s.():]+\.)*[^\s.():]\(/, :go_frame_2),
+      rule(:go_frame_1, /^$/, :go_before_goroutine),
+      rule(:go_frame_2, /^\s/, :go_frame_1)
+    ].freeze
+
+    RUBY_RULES = [
+      rule(:start_state, /Error \(.*\):$/, :ruby),
+      rule(:ruby, /^[\t ]+.*?\.rb:\d+:in `/, :ruby)
+    ].freeze
+
+    ALL_RULES = (
+      KAFKA_RULES + JAVA_RULES + PYTHON_RULES + PHP_RULES + GO_RULES + RUBY_RULES + ELIXIR_RULES).freeze
+
+    RULES_BY_LANG = {
+      java: JAVA_RULES + KAFKA_RULES,
+      javascript: JAVA_RULES + KAFKA_RULES,
+      js: JAVA_RULES + KAFKA_RULES,
+      csharp: JAVA_RULES,
+      py: PYTHON_RULES,
+      python: PYTHON_RULES,
+      php: PHP_RULES,
+      go: GO_RULES,
+      rb: RUBY_RULES,
+      ruby: RUBY_RULES,
+      elixir: ELIXIR_RULES,
+      all: ALL_RULES
+    }.freeze
+
+    DEFAULT_FIELDS = %w(message log).freeze
+  end
+
+  # State machine that consumes individual log lines and detects
+  # multi-line stack traces.
+  class ExceptionDetector
+    def initialize(*languages)
+      @state = :start_state
+      @rules = Hash.new { |h, k| h[k] = [] }
+
+      languages = [:all] if languages.empty?
+
+      languages.each do |lang|
+        rule_config =
+          ExceptionDetectorConfig::RULES_BY_LANG.fetch(lang.downcase) do |_k|
+            raise ArgumentError, "Unknown language: #{lang}"
+          end
+
+        rule_config.each do |r|
+          target = ExceptionDetectorConfig::RuleTarget.new(r[:pattern],
+                                                           r[:to_state])
+          @rules[r[:from_state]] << target
+        end
+      end
+
+      @rules.each_value(&:uniq!)
+    end
+
+    # Updates the state machine and returns the trace detection status:
+    # - no_trace: 'line' does not belong to an exception trace,
+    # - start_trace: 'line' starts a detected exception trace,
+    # - inside: 'line' is part of a detected exception trace,
+    # - end: the detected exception trace ends after 'line'.
+    def update(line)
+      trace_seen_before = transition(line)
+      # If the state machine fell back to the start state because there is no
+      # defined transition for 'line', trigger another state transition because
+      # 'line' may contain the beginning of another exception.
+      transition(line) unless trace_seen_before
+      new_state = @state
+      trace_seen_after = new_state != :start_state
+
+      case [trace_seen_before, trace_seen_after]
+      when [true, true]
+        :inside_trace
+      when [true, false]
+        :end_trace
+      when [false, true]
+        :start_trace
+      else
+        :no_trace
+      end
+    end
+
+    def reset
+      @state = :start_state
+    end
+
+    private
+
+    # Executes a transition of the state machine for the given line.
+    # Returns false if the line does not match any transition rule and the
+    # state machine was reset to the initial state.
+    def transition(line)
+      @rules[@state].each do |r|
+        next unless line =~ r.pattern
+        @state = r.to_state
+        return true
+      end
+      @state = :start_state
+      false
+    end
+  end
+
+  # Buffers and groups log records if they contain exception stack traces.
+  class TraceAccumulator
+    attr_reader :buffer_start_time
+
+    # If message_field is nil, the instance is set up to accumulate
+    # records that are plain strings (i.e. the whole record is concatenated).
+    # Otherwise, the instance accepts records that are dictionaries (usually
+    # originating from structured JSON logs) and accumulates just the
+    # content of the given message field.
+    # message_field may contain the empty string. In this case, the
+    # TraceAccumulator 'learns' the field name from the first record by checking
+    # for some pre-defined common field names of text logs.
+    # The named parameters max_lines and max_bytes limit the maximum amount
+    # of data to be buffered. The default value 0 indicates 'no limit'.
+    def initialize(message_field, languages, max_lines: 0, max_bytes: 0,
+                   &emit_callback)
+      @exception_detector = Fluent::ExceptionDetector.new(*languages)
+      @max_lines = max_lines
+      @max_bytes = max_bytes
+      @message_field = message_field
+      @messages = []
+      @buffer_start_time = Time.now
+      @buffer_size = 0
+      @first_record = nil
+      @first_timestamp = nil
+      @emit = emit_callback
+    end
+
+    def push(time_sec, record)
+      message = extract_message(record)
+      if message.nil?
+        @exception_detector.reset
+        detection_status = :no_trace
+      else
+        force_flush if @max_bytes > 0 &&
+                       @buffer_size + message.length > @max_bytes
+        detection_status = @exception_detector.update(message)
+      end
+      update_buffer(detection_status, time_sec, record, message)
+
+      force_flush if @max_lines > 0 && @messages.length == @max_lines
+    end
+
+    def flush
+      case @messages.length
+      when 0
+        return
+      when 1
+        @emit.call(@first_timestamp, @first_record)
+      else
+        combined_message = @messages.join
+        if @message_field.nil?
+          output_record = combined_message
+        else
+          output_record = @first_record
+          output_record[@message_field] = combined_message
+        end
+        @emit.call(@first_timestamp, output_record)
+      end
+      @messages = []
+      @first_record = nil
+      @first_timestamp = nil
+      @buffer_size = 0
+    end
+
+    def force_flush
+      flush
+      @exception_detector.reset
+    end
+
+    private
+
+    def extract_message(record)
+      if !@message_field.nil? && @message_field.empty?
+        ExceptionDetectorConfig::DEFAULT_FIELDS.each do |f|
+          if record.key?(f)
+            @message_field = f
+            break
+          end
+        end
+      end
+      @message_field.nil? ? record : record[@message_field]
+    end
+
+    def update_buffer(detection_status, time_sec, record, message)
+      trigger_emit = detection_status == :no_trace ||
+                     detection_status == :end_trace
+      if @messages.empty? && trigger_emit
+        @emit.call(time_sec, record)
+        return
+      end
+
+      case detection_status
+      when :inside_trace
+        add(time_sec, record, message)
+      when :end_trace
+        add(time_sec, record, message)
+        flush
+      when :no_trace
+        flush
+        add(time_sec, record, message)
+        flush
+      when :start_trace
+        flush
+        add(time_sec, record, message)
+      end
+    end
+
+    def add(time_sec, record, message)
+      if @messages.empty?
+        @first_record = record unless @message_field.nil?
+        @first_timestamp = time_sec
+        @buffer_start_time = Time.now
+      end
+      unless message.nil?
+        @messages << message
+        @buffer_size += message.length
+      end
+    end
+  end
+end

data/lib/fluent/plugin/out_detect_exceptions.rb

@@ -0,0 +1,136 @@
+#
+# Copyright 2016 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require 'fluent/plugin/exception_detector'
+require 'fluent/output'
+
+module Fluent
+  # This output plugin consumes a log stream of JSON objects which contain
+  # single-line log messages. If a consecutive sequence of log messages form
+  # an exception stack trace, they forwarded as a single, combined JSON
+  # object. Otherwise, the input log data is forwarded as is.
+  class DetectExceptionsOutput < Output
+    desc 'The field which contains the raw message text in the input JSON data.'
+    config_param :message, :string, default: ''
+    desc 'The prefix to be removed from the input tag when outputting a record.'
+    config_param :remove_tag_prefix, :string, default: ''
+    desc 'The interval of flushing the buffer for multiline format.'
+    config_param :multiline_flush_interval, :time, default: nil
+    desc 'Programming languages for which to detect exceptions. Default: all.'
+    config_param :languages, :array, value_type: :string, default: []
+    desc 'Maximum number of lines to flush (0 means no limit). Default: 1000.'
+    config_param :max_lines, :integer, default: 1000
+    desc 'Maximum number of bytes to flush (0 means no limit). Default: 0.'
+    config_param :max_bytes, :integer, default: 0
+    desc 'Separate log streams by this field in the input JSON data.'
+    config_param :stream, :string, default: ''
+
+    Fluent::Plugin.register_output('detect_exceptions', self)
+
+    def configure(conf)
+      super
+      if multiline_flush_interval
+        @check_flush_interval = [multiline_flush_interval * 0.1, 1].max
+      end
+
+      @languages = languages.map(&:to_sym)
+
+      # Maps log stream tags to a corresponding TraceAccumulator.
+      @accumulators = {}
+    end
+
+    def start
+      super
+      if multiline_flush_interval
+        @flush_buffer_mutex = Mutex.new
+        @stop_check = false
+        @thread = Thread.new(&method(:check_flush_loop))
+      end
+    end
+
+    def before_shutdown
+      flush_buffers
+      super if defined?(super)
+    end
+
+    def shutdown
+      # Before shutdown is not available in older fluentd versions.
+      # Hence, we make sure that we flush the buffers here as well.
+      flush_buffers
+      @thread.join if @multiline_flush_interval
+      super
+    end
+
+    def emit(tag, es, chain)
+      es.each do |time_sec, record|
+        process_record(tag, time_sec, record)
+      end
+      chain.next
+    end
+
+    private
+
+    def process_record(tag, time_sec, record)
+      synchronize do
+        log_id = [tag]
+        log_id.push(record.fetch(@stream, '')) unless @stream.empty?
+        unless @accumulators.key?(log_id)
+          out_tag = tag.sub(/^#{Regexp.escape(@remove_tag_prefix)}\./, '')
+          @accumulators[log_id] =
+            Fluent::TraceAccumulator.new(@message, @languages,
+                                         max_lines: @max_lines,
+                                         max_bytes: @max_bytes) do |t, r|
+              router.emit(out_tag, t, r)
+            end
+        end
+
+        @accumulators[log_id].push(time_sec, record)
+      end
+    end
+
+    def flush_buffers
+      synchronize do
+        @stop_check = true
+        @accumulators.each_value(&:force_flush)
+      end
+    end
+
+    def check_flush_loop
+      @flush_buffer_mutex.synchronize do
+        loop do
+          @flush_buffer_mutex.sleep(@check_flush_interval)
+          now = Time.now
+          break if @stop_check
+          log.debug 'Reached flush loop so stopping'
+          @accumulators.each_value do |acc|
+            acc.force_flush if now - acc.buffer_start_time >
+                               @multiline_flush_interval
+          end
+        end
+      end
+    rescue
+      log.error 'error in check_flush_loop', error: $ERROR_INFO.to_s
+      log.error_backtrace
+    end
+
+    def synchronize(&block)
+      if @multiline_flush_interval
+        @flush_buffer_mutex.synchronize(&block)
+      else
+        yield
+      end
+    end
+  end
+end