logstash-filter-grok 4.1.1 → 4.4.1

data/lib/logstash/filters/grok.rb

@@ -3,8 +3,8 @@
   require "logstash/namespace"
   require "logstash/environment"
   require "logstash/patterns/core"
+  require 'logstash/plugin_mixins/ecs_compatibility_support'
   require "grok-pure" # rubygem 'jls-grok'
-  require "set"
   require "timeout"
 
   # Parse arbitrary text and structure it.
@@ -140,10 +140,12 @@
   # `SYSLOGBASE` pattern which itself is defined by other patterns.
   #
   # Another option is to define patterns _inline_ in the filter using `pattern_definitions`.
-  # This is mostly for convenience and allows user to define a pattern which can be used just in that 
+  # This is mostly for convenience and allows user to define a pattern which can be used just in that
   # filter. This newly defined patterns in `pattern_definitions` will not be available outside of that particular `grok` filter.
   #
   class LogStash::Filters::Grok < LogStash::Filters::Base
+    include LogStash::PluginMixins::ECSCompatibilitySupport
+
     config_name "grok"
 
     # A hash of matches of field => value
@@ -168,7 +170,7 @@
     # necessarily need to define this yourself unless you are adding additional
     # patterns. You can point to multiple pattern directories using this setting.
     # Note that Grok will read all files in the directory matching the patterns_files_glob
-    # and assume it's a pattern file (including any tilde backup files). 
+    # and assume it's a pattern file (including any tilde backup files).
     # [source,ruby]
     #     patterns_dir => ["/opt/logstash/patterns", "/opt/logstash/extra_patterns"]
     #
@@ -204,6 +206,10 @@
     # If `true`, keep empty captures as event fields.
     config :keep_empty_captures, :validate => :boolean, :default => false
 
+    # Define the target field for placing the matched captures.
+    # If this setting is omitted, data gets stored at the root (top level) of the event.
+    config :target, :validate => :string
+
     # Append values to the `tags` field when there has been no
     # successful match
     config :tag_on_failure, :validate => :array, :default => ["_grokparsefailure"]
@@ -215,6 +221,16 @@
     # Set to 0 to disable timeouts
     config :timeout_millis, :validate => :number, :default => 30000
 
+    # When multiple patterns are provided to `match`,
+    # the timeout has historically applied to _each_ pattern, incurring overhead
+    # for each and every pattern that is attempted; when the grok filter is
+    # configured with `timeout_scope => 'event'`, the plugin instead enforces
+    # a single timeout across all attempted matches on the event, so it can
+    # achieve similar safeguard against runaway matchers with significantly
+    # less overhead.
+    # It's usually better to scope the timeout for the whole event.
+    config :timeout_scope, :validate => %w(pattern event), :default => "pattern"
+
     # Tag to apply if a grok regexp times out.
     config :tag_on_timeout, :validate => :string, :default => '_groktimeout'
 
@@ -236,22 +252,14 @@
     # will be parsed and `hello world` will overwrite the original message.
     config :overwrite, :validate => :array, :default => []
 
-    # Register default pattern paths
-    @@patterns_path ||= Set.new
-    @@patterns_path += [
-      LogStash::Patterns::Core.path,
-      LogStash::Environment.pattern_path("*")
-    ]
-
     def register
       # a cache of capture name handler methods.
       @handlers = {}
 
       @patternfiles = []
-
-      # Have @@patterns_path show first. Last-in pattern definitions win; this
-      # will let folks redefine built-in patterns at runtime.
-      @patternfiles += patterns_files_from_paths(@@patterns_path.to_a, "*")
+      # Have (default) patterns_path show first. Last-in pattern definitions wins
+      # this will let folks redefine built-in patterns at runtime
+      @patternfiles += patterns_files_from_paths(patterns_path, "*")
       @patternfiles += patterns_files_from_paths(@patterns_dir, @patterns_files_glob)
 
       @patterns = Hash.new { |h,k| h[k] = [] }
@@ -264,11 +272,11 @@
         patterns = [patterns] if patterns.is_a?(String)
         @metric_match_fields.gauge(field, patterns.length)
 
-        @logger.trace("Grok compile", :field => field, :patterns => patterns)
+        @logger.trace? && @logger.trace("Grok compile", :field => field, :patterns => patterns)
         patterns.each do |pattern|
-          @logger.debug? and @logger.debug("regexp: #{@type}/#{field}", :pattern => pattern)
+          @logger.debug? && @logger.debug("regexp: #{@type}/#{field}", :pattern => pattern)
           grok = Grok.new
-          grok.logger = @logger unless @logger.nil?
+          grok.logger = @logger
           add_patterns_from_files(@patternfiles, grok)
           add_patterns_from_inline_definition(@pattern_definitions, grok)
           grok.compile(pattern, @named_captures_only)
@@ -278,24 +286,23 @@
       @match_counter = metric.counter(:matches)
       @failure_counter = metric.counter(:failures)
 
-      # divide by float to allow fractionnal seconds, the Timeout class timeout value is in seconds but the underlying
-      # executor resolution is in microseconds so fractionnal second parameter down to microseconds is possible.
-      # see https://github.com/jruby/jruby/blob/9.2.7.0/core/src/main/java/org/jruby/ext/timeout/Timeout.java#L125
-      @timeout_seconds = @timeout_millis / 1000.0
+      @target = "[#{@target.strip}]" if @target && @target !~ /\[.*?\]/
+
+      @timeout = @timeout_millis > 0.0 ? RubyTimeout.new(@timeout_millis) : NoopTimeout::INSTANCE
+      @matcher = ( @timeout_scope.eql?('event') ? EventTimeoutMatcher : PatternTimeoutMatcher ).new(self)
     end # def register
 
     def filter(event)
       matched = false
 
-      @logger.debug? and @logger.debug("Running grok filter", :event => event)
+      @logger.debug? && @logger.debug("Running grok filter", :event => event.to_hash)
 
       @patterns.each do |field, groks|
         if match(groks, field, event)
           matched = true
           break if @break_on_match
         end
-        #break if done
-      end # @patterns.each
+      end
 
       if matched
         @match_counter.increment(1)
@@ -305,7 +312,7 @@
         @tag_on_failure.each {|tag| event.tag(tag)}
       end
 
-      @logger.debug? and @logger.debug("Event now: ", :event => event)
+      @logger.debug? && @logger.debug("Event now: ", :event => event.to_hash)
     rescue GrokTimeoutException => e
       @logger.warn(e.message)
       metric.increment(:timeouts)
@@ -317,6 +324,27 @@
 
     private
 
+    # The default pattern paths, depending on environment.
+    def patterns_path
+      patterns_path = []
+      case ecs_compatibility
+      when :disabled
+        patterns_path << LogStash::Patterns::Core.path # :legacy
+      when :v1
+        patterns_path << LogStash::Patterns::Core.path('ecs-v1')
+      when :v8
+        @logger.warn("ECS v8 support is a preview of the unreleased ECS v8, and uses the v1 patterns. When Version 8 of the Elastic Common Schema becomes available, this plugin will need to be updated")
+        patterns_path << LogStash::Patterns::Core.path('ecs-v1')
+      else
+        fail(NotImplementedError, "ECS #{ecs_compatibility} is not supported by this plugin.")
+      end
+      # allow plugin to be instantiated outside the LS environment (in tests)
+      if defined? LogStash::Environment.pattern_path
+        patterns_path << LogStash::Environment.pattern_path("*")
+      end
+      patterns_path
+    end
+
     def match(groks, field, event)
       input = event.get(field)
       if input.is_a?(Array)
@@ -329,55 +357,91 @@
         match_against_groks(groks, field, input, event)
       end
     rescue StandardError => e
-      @logger.warn("Grok regexp threw exception", :exception => e.message, :backtrace => e.backtrace, :class => e.class.name)
+      @logger.warn("Grok regexp threw exception", :message => e.message, :exception => e.class, :backtrace => e.backtrace)
       return false
     end
 
     def match_against_groks(groks, field, input, event)
-      input = input.to_s
-      matched = false
-      groks.each do |grok|
-        # Convert anything else to string (number, hash, etc)
-        matched = grok_till_timeout(grok, field, input)
-        if matched
-          grok.capture(matched) {|field, value| handle(field, value, event)}
-          break if @break_on_match
+      # Convert anything else to string (number, hash, etc)
+      context = GrokContext.new(field, input.to_s)
+      @matcher.match(context, groks, event, @break_on_match)
+    end
+
+    # Internal (base) helper to handle the global timeout switch.
+    # @private
+    class Matcher
+
+      def initialize(filter)
+        @filter = filter
+      end
+
+      def match(context, groks, event, break_on_match)
+        matched = false
+
+        groks.each do |grok|
+          context.set_grok(grok)
+
+          matched = execute(context, grok)
+          if matched
+            grok.capture(matched) { |field, value| @filter.handle(field, value, event) }
+            break if break_on_match
+          end
         end
+
+        matched
+      end
+
+      protected
+
+      def execute(context, grok)
+        grok.execute(context.input)
       end
-      
-      matched
+
     end
 
-    def grok_till_timeout(grok, field, value)
-      begin
-        @timeout_seconds > 0.0 ? Timeout.timeout(@timeout_seconds, TimeoutError) { grok.execute(value) } : grok.execute(value)
-      rescue TimeoutError
-        raise GrokTimeoutException.new(grok, field, value)
+    # @private
+    class EventTimeoutMatcher < Matcher
+      # @override
+      def match(context, groks, event, break_on_match)
+        @filter.with_timeout(context) { super }
+      end
+    end
+
+    # @private
+    class PatternTimeoutMatcher < Matcher
+      # @override
+      def execute(context, grok)
+        @filter.with_timeout(context) { super }
       end
     end
 
     def handle(field, value, event)
       return if (value.nil? || (value.is_a?(String) && value.empty?)) unless @keep_empty_captures
 
+      target_field = @target ? "#{@target}[#{field}]" : field
+
       if @overwrite.include?(field)
-        event.set(field, value)
+        event.set(target_field, value)
       else
-        v = event.get(field)
+        v = event.get(target_field)
         if v.nil?
-          event.set(field, value)
+          event.set(target_field, value)
         elsif v.is_a?(Array)
           # do not replace the code below with:
           #   event[field] << value
           # this assumes implementation specific feature of returning a mutable object
           # from a field ref which should not be assumed and will change in the future.
           v << value
-          event.set(field, v)
+          event.set(target_field, v)
         elsif v.is_a?(String)
           # Promote to array since we aren't overwriting.
-          event.set(field, [v, value])
+          event.set(target_field, [v, value])
+        else
+          @logger.debug("Not adding matched value - found existing (#{v.class})", :field => target_field, :value => value)
         end
       end
     end
+    public :handle
 
     def patterns_files_from_paths(paths, glob)
       patternfiles = []
@@ -438,4 +502,52 @@
         end
       end
     end
+
+    def with_timeout(context, &block)
+      @timeout.exec(&block)
+    rescue TimeoutError => error
+      handle_timeout(context, error)
+    end
+    public :with_timeout
+
+    def handle_timeout(context, error)
+      raise GrokTimeoutException.new(context.grok, context.field, context.input)
+    end
+
+    # @private
+    class GrokContext
+      attr_reader :grok, :field, :input
+
+      def initialize(field, input)
+        @field = field
+        @input = input
+      end
+
+      def set_grok(grok)
+        @grok = grok
+      end
+    end
+
+    # @private
+    class NoopTimeout
+      INSTANCE = new
+
+      def exec
+        yield
+      end
+    end
+
+    # @private
+    class RubyTimeout
+      def initialize(timeout_millis)
+        # divide by float to allow fractional seconds, the Timeout class timeout value is in seconds but the underlying
+        # executor resolution is in microseconds so fractional second parameter down to microseconds is possible.
+        # see https://github.com/jruby/jruby/blob/9.2.7.0/core/src/main/java/org/jruby/ext/timeout/Timeout.java#L125
+        @timeout_seconds = timeout_millis / 1000.0
+      end
+
+      def exec(&block)
+        Timeout.timeout(@timeout_seconds, TimeoutError, &block)
+      end
+    end
   end # class LogStash::Filters::Grok

data/logstash-filter-grok.gemspec

@@ -1,7 +1,6 @@
 Gem::Specification.new do |s|
-
   s.name            = 'logstash-filter-grok'
-  s.version         = '4.1.1'
+  s.version         = '4.4.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Parses unstructured event data into fields"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -22,10 +21,11 @@ Gem::Specification.new do |s|
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
   s.add_runtime_dependency "logstash-core", ">= 5.6.0"
+  s.add_runtime_dependency 'logstash-mixin-ecs_compatibility_support', '~> 1.0'
 
   s.add_runtime_dependency 'jls-grok', '~> 0.11.3'
   s.add_runtime_dependency 'stud', '~> 0.0.22'
-  s.add_runtime_dependency 'logstash-patterns-core'
+  s.add_runtime_dependency 'logstash-patterns-core', '>= 4.3.0', '< 5'
 
-  s.add_development_dependency 'logstash-devutils', '= 1.3.6'
+  s.add_development_dependency 'logstash-devutils'
 end

data/spec/filters/grok_performance_spec.rb

@@ -0,0 +1,144 @@
+# encoding: utf-8
+require_relative "../spec_helper"
+
+begin
+  require "rspec-benchmark"
+rescue LoadError # due testing against LS 5.x
+end
+RSpec.configure do |config|
+  config.include RSpec::Benchmark::Matchers if defined? RSpec::Benchmark::Matchers
+end
+
+require "logstash/filters/grok"
+
+describe LogStash::Filters::Grok do
+
+  subject do
+    described_class.new(config).tap { |filter| filter.register }
+  end
+
+  EVENT_COUNT = 300_000
+
+  describe "base-line performance", :performance => true do
+
+    EXPECTED_MIN_RATE = 30_000 # per second - based on Travis CI (docker) numbers
+
+    let(:config) do
+      { 'match' => { "message" => "%{SYSLOGLINE}" }, 'overwrite' => [ "message" ] }
+    end
+
+    it "matches at least #{EXPECTED_MIN_RATE} events/second" do
+      max_duration = EVENT_COUNT / EXPECTED_MIN_RATE
+      message = "Mar 16 00:01:25 evita postfix/smtpd[1713]: connect from camomile.cloud9.net[168.100.1.3]"
+      expect do
+        duration = measure do
+          EVENT_COUNT.times { subject.filter(LogStash::Event.new("message" => message)) }
+        end
+        puts "filters/grok parse rate: #{"%02.0f/sec" % (EVENT_COUNT / duration)}, elapsed: #{duration}s"
+      end.to perform_under(max_duration).warmup(1).sample(2).times
+    end
+
+  end
+
+  describe "timeout", :performance => true do
+
+    ACCEPTED_TIMEOUT_DEGRADATION = 100 # in % (compared to timeout-less run)
+    # TODO: with more real-world (pipeline) setup this usually gets bellow 10% on average
+
+    MATCH_PATTERNS = {
+      "message" => [
+        "foo0: %{NUMBER:bar}", "foo1: %{NUMBER:bar}", "foo2: %{NUMBER:bar}", "foo3: %{NUMBER:bar}", "foo4: %{NUMBER:bar}",
+        "foo5: %{NUMBER:bar}", "foo6: %{NUMBER:bar}", "foo7: %{NUMBER:bar}", "foo8: %{NUMBER:bar}", "foo9: %{NUMBER:bar}",
+        "%{SYSLOGLINE}"
+      ]
+    }
+
+    SAMPLE_MESSAGE = "Mar 16 00:01:25 evita postfix/smtpd[1713]: connect from aaaaaaaa.aaaaaa.net[111.111.11.1]".freeze
+
+    TIMEOUT_MILLIS = 5_000
+
+    let(:config_wout_timeout) do
+      {
+        'match' => MATCH_PATTERNS,
+        'timeout_scope' => "event",
+        'timeout_millis' => 0 # 0 - disabled timeout
+      }
+    end
+
+    let(:config_with_timeout) do
+      {
+        'match' => MATCH_PATTERNS,
+        'timeout_scope' => "event",
+        'timeout_millis' => TIMEOUT_MILLIS
+      }
+    end
+
+    SAMPLE_COUNT = 2
+
+    it "has less than #{ACCEPTED_TIMEOUT_DEGRADATION}% overhead" do
+      filter_wout_timeout = LogStash::Filters::Grok.new(config_wout_timeout).tap(&:register)
+      wout_timeout_duration = do_sample_filter(filter_wout_timeout) # warmup
+      puts "filters/grok(timeout => 0) warmed up in #{wout_timeout_duration}"
+      before_sample!
+      no_timeout_durations = Array.new(SAMPLE_COUNT).map do
+        do_sample_filter(filter_wout_timeout)
+      end
+      puts "filters/grok(timeout => 0) took #{no_timeout_durations}"
+
+      expected_duration = avg(no_timeout_durations)
+      expected_duration += (expected_duration / 100) * ACCEPTED_TIMEOUT_DEGRADATION
+      puts "expected_duration #{expected_duration}"
+
+      filter_with_timeout = LogStash::Filters::Grok.new(config_with_timeout).tap(&:register)
+      with_timeout_duration = do_sample_filter(filter_with_timeout) # warmup
+      puts "filters/grok(timeout_scope => event) warmed up in #{with_timeout_duration}"
+
+      try(3) do
+        before_sample!
+        durations = []
+        begin
+          expect do
+            do_sample_filter(filter_with_timeout).tap { |duration| durations << duration }
+          end.to perform_under(expected_duration).sample(SAMPLE_COUNT).times
+        ensure
+          puts "filters/grok(timeout_scope => event) took #{durations}"
+        end
+      end
+    end
+
+    @private
+
+    def do_sample_filter(filter)
+      sample_event = { "message" => SAMPLE_MESSAGE }
+      measure do
+        for _ in (1..EVENT_COUNT) do # EVENT_COUNT.times without the block cost
+          filter.filter(LogStash::Event.new(sample_event))
+        end
+      end
+    end
+
+  end
+
+  @private
+
+  def measure
+    start = Time.now
+    yield
+    Time.now - start
+  end
+
+  def avg(ary)
+    ary.inject(0) { |m, i| m + i } / ary.size.to_f
+  end
+
+  def before_sample!
+    2.times { JRuby.gc }
+    sleep TIMEOUT_MILLIS / 1000
+  end
+
+  def sleep(seconds)
+    puts "sleeping for #{seconds} seconds (redundant - potential timeout propagation)"
+    Kernel.sleep(seconds)
+  end
+
+end