logstash-filter-grok 3.1.2 → 3.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: caf1f9a43c9403b8c1dfaa59d356e1600b5f1881
-  data.tar.gz: 2404df9cf8936c1bb6f62867733e53838e87c1c9
+  metadata.gz: a34c685833867a04238f8544429c5938cd58a696
+  data.tar.gz: 096eb04af33607c064d210c9e007ffbfccb3a9fa
 SHA512:
-  metadata.gz: a655b0603e43911b7ab64b2f59936224f656cb5de4064a28a06c378bdc9cba77c1716b3fee14531721abfb2ff09c7d8d53747035d0bc7b269c8e88d7911d1515
-  data.tar.gz: 306b4b5a7345991750b63f49b497b3fe59773a045680b6e934791a5a9a76280d615c74b7825ca45fd533727fefe485e109f23a42f5af051ea731e0831b5bee97
+  metadata.gz: b7aacf53f2a102c96c65597710b529ef0acef10730358ae4c5d4fd32c9dd053e5f226a2696535921fccc75504f205c8580762287ead404d413359bac8e9758f1
+  data.tar.gz: 61bfb20dbb5452b957e08e82b5ad54d8b1f89d50dcdd2d0ba6bd37f7e06b2f630e2d92247ddd9e81de7fcefba720f12a3a32b5a21edb99eafb9d6a57331b7b54

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 3.2.0
+  - Add new timeout options to cancel grok execution if a threshold time is exceeded
+
 ## 3.1.2
   - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
 

data/lib/logstash/filters/grok/timeout_enforcer.rb

@@ -0,0 +1,77 @@
+class LogStash::Filters::Grok::TimeoutEnforcer
+  def initialize(logger, timeout_nanos)
+    @logger = logger
+    @running = true
+    @timeout_nanos = timeout_nanos
+
+    # Stores running matches with their start time, this is used to cancel long running matches
+    # Is a map of Thread => start_time
+    @timer_mutex = Mutex.new
+    @threads_to_start_time = {}        
+  end
+
+  def grok_till_timeout(event, grok, field, value)
+    begin
+      thread = Thread.current
+      start_thread_groking(thread)
+      yield
+    rescue ::LogStash::Filters::Grok::TimeoutException => e
+      # These fields aren't present at the time the exception was raised
+      # so we add them here.
+      # We could store this metadata in the @threads_to_start_time hash
+      # but that'd come at a perf cost and this works just as well.
+      e.grok = grok
+      e.field = field
+      e.value = value
+      raise e
+    ensure
+      stop_thread_groking(thread)
+    end
+  end
+
+  def start_thread_groking(thread)
+    @timer_mutex.synchronize do
+      @threads_to_start_time[thread] = java.lang.System.nanoTime()
+    end
+  end
+
+  def stop_thread_groking(thread)
+    @timer_mutex.synchronize do
+      @threads_to_start_time.delete(thread)
+    end
+  end
+
+  def cancel_timed_out!
+    @threads_to_start_time.each do |thread,start_time|
+      now = java.lang.System.nanoTime # save ourselves some nanotime calls
+      elapsed = java.lang.System.nanoTime - start_time
+      if elapsed > @timeout_nanos
+        elapsed_millis = elapsed / 1000
+        thread.raise(::LogStash::Filters::Grok::TimeoutException.new(elapsed_millis))
+      end
+    end
+  end
+
+  def start!
+    @timer_thread = Thread.new do
+      while @running
+        begin
+          cancel_timed_out!
+        rescue Exception => e
+          @logger.error("Error while attempting to check/cancel excessively long grok patterns",
+                        :message => e.message,
+                        :class => e.class.name,
+                        :backtrace => e.backtrace
+                       )
+        end                   
+        sleep 0.25
+      end
+    end
+  end
+
+  def stop!
+    @running = false
+    # Check for the thread mostly for a fast start/shutdown scenario
+    @timer_thread.join if @timer_thread
+  end
+end

data/lib/logstash/filters/grok/timeout_exception.rb

@@ -0,0 +1,23 @@
+class LogStash::Filters::Grok::TimeoutException < Exception
+  attr_accessor :elapsed_millis, :grok, :field, :value
+  
+  def initialize(elapsed_millis, grok=nil, field=nil, value=nil)
+    @elapsed_millis = elapsed_millis
+    @field = field
+    @value = value
+    @grok = grok
+  end
+
+  def message
+    "Timeout executing grok '#{@grok.pattern}' against field '#{field}' with value '#{trunc_value}'!"
+  end
+
+  def trunc_value
+    if value.size <= 255 # If no more than 255 chars
+      value
+    else
+      "Value too large to output (#{value.bytesize} bytes)! First 255 chars are: #{value[0..255]}"
+    end
+  end
+end
+

data/lib/logstash/filters/grok.rb

@@ -138,6 +138,8 @@
   # `SYSLOGBASE` pattern which itself is defined by other patterns.
   class LogStash::Filters::Grok < LogStash::Filters::Base
     config_name "grok"
+    require "logstash/filters/grok/timeout_enforcer"
+    require "logstash/filters/grok/timeout_exception"
 
     # A hash of matches of field => value
     #
@@ -159,9 +161,9 @@
     #
     # Logstash ships by default with a bunch of patterns, so you don't
     # necessarily need to define this yourself unless you are adding additional
-    # patterns. You can point to multiple pattern directories using this setting
+    # patterns. You can point to multiple pattern directories using this setting.
     # Note that Grok will read all files in the directory matching the patterns_files_glob
-    # and assume its a pattern file (including any tilde backup files)
+    # and assume it's a pattern file (including any tilde backup files)
     # [source,ruby]
     #     patterns_dir => ["/opt/logstash/patterns", "/opt/logstash/extra_patterns"]
     #
@@ -193,6 +195,15 @@
     # successful match
     config :tag_on_failure, :validate => :array, :default => ["_grokparsefailure"]
 
+    # Attempt to terminate regexps after this amount of time.
+    # This applies per pattern if multiple patterns are applied
+    # This will never timeout early, but may take a little longer to timeout.
+    # Actual timeout is approximate based on a 250ms quantization.
+    config :timeout_millis, :validate => :number, :default => 2000
+
+    # Tag to apply if a grok regexp times out.
+    config :tag_on_timeout, :validate => :string, :default => '_groktimeout'
+
     # The fields to overwrite.
     #
     # This allows you to overwrite a value in a field that already exists.
@@ -223,6 +234,9 @@
       super(params)
       # a cache of capture name handler methods.
       @handlers = {}
+
+      @timeout_enforcer = TimeoutEnforcer.new(@logger, @timeout_millis * 1000000)
+      @timeout_enforcer.start!
     end
 
     public
@@ -264,8 +278,11 @@
       done = false
 
       @logger.debug? and @logger.debug("Running grok filter", :event => event);
+
       @patterns.each do |field, groks|
-        if match(groks, field, event)
+        success = match(groks, field, event)
+        
+        if success
           matched = true
           break if @break_on_match
         end
@@ -277,10 +294,14 @@
         filter_matched(event)
       else
         metric.increment(:failures)
-        @tag_on_failure.each{|tag| event.tag(tag)}
+        @tag_on_failure.each {|tag| event.tag(tag)}
       end
 
       @logger.debug? and @logger.debug("Event now: ", :event => event)
+    rescue ::LogStash::Filters::Grok::TimeoutException => e      
+      @logger.warn(e.message)
+      metric.increment(:timeouts)
+      event.tag(@tag_on_timeout)
     end # def filter
 
     private
@@ -289,28 +310,32 @@
       if input.is_a?(Array)
         success = false
         input.each do |input|
-          success |= match_against_groks(groks, input, event)
+          success |= match_against_groks(groks, field, input, event)
         end
         return success
       else
-        return match_against_groks(groks, input, event)
+        match_against_groks(groks, field, input, event)
       end
     rescue StandardError => e
-      @logger.warn("Grok regexp threw exception", :exception => e.message)
+      @logger.warn("Grok regexp threw exception", :exception => e.message, :backtrace => e.backtrace, :class => e.class.name)
+      return false
     end
-
+    
     private
-    def match_against_groks(groks, input, event)
+    def match_against_groks(groks, field, input, event)
+      input = input.to_s
       matched = false
       groks.each do |grok|
         # Convert anything else to string (number, hash, etc)
-        matched = grok.match_and_capture(input.to_s) do |field, value|
-          matched = true
-          handle(field, value, event)
+
+        matched = @timeout_enforcer.grok_till_timeout(event, grok, field, input) { grok.execute(input) }
+        if matched
+          grok.capture(matched) {|field, value| handle(field, value, event)}
+          break if @break_on_match
         end
-        break if matched and @break_on_match
       end
-      return matched
+      
+      matched
     end
 
     private
@@ -364,4 +389,8 @@
       end
     end # def add_patterns_from_files
 
+    def close
+      @timeout_handler.stop!
+    end
+
   end # class LogStash::Filters::Grok

data/logstash-filter-grok.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-grok'
-  s.version         = '3.1.2'
+  s.version         = '3.2.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Parse arbitrary text and structure it."
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -22,7 +22,7 @@ Gem::Specification.new do |s|
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
 
-  s.add_runtime_dependency 'jls-grok', '~> 0.11.1'
+  s.add_runtime_dependency 'jls-grok', '~> 0.11.3'
   s.add_runtime_dependency 'logstash-patterns-core'
 
   s.add_development_dependency 'logstash-devutils'

data/spec/filters/grok_spec.rb

@@ -407,12 +407,30 @@ describe LogStash::Filters::Grok do
     end
   end
 
+  describe "timeout on failure" do
+    config <<-CONFIG
+      filter { 
+        grok {
+          match => { 
+            message => "(.*a){30}"
+          }
+          timeout_millis => 100
+        }
+      }
+    CONFIG
+
+    sample "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" do
+      expect(subject.get("tags")).to include("_groktimeout")
+      expect(subject.get("tags")).not_to include("_grokparsefailure")
+    end
+  end
+
   describe "tagging on failure" do
     config <<-CONFIG
       filter {
         grok {
           match => { "message" => "matchme %{NUMBER:fancy}" }
-          tag_on_failure => false
+          tag_on_failure => not_a_match
         }
       }
     CONFIG
@@ -422,7 +440,7 @@ describe LogStash::Filters::Grok do
     end
 
     sample "this will not be matched" do
-      insist { subject.get("tags") }.include?("false")
+      insist { subject.get("tags") }.include?("not_a_match")
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-grok
 version: !ruby/object:Gem::Version
-  version: 3.1.2
+  version: 3.2.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-07-14 00:00:00.000000000 Z
+date: 2016-08-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -35,7 +35,7 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.11.1
+        version: 0.11.3
   name: jls-grok
   prerelease: false
   type: :runtime
@@ -43,7 +43,7 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.11.1
+        version: 0.11.3
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -85,6 +85,8 @@ files:
 - NOTICE.TXT
 - README.md
 - lib/logstash/filters/grok.rb
+- lib/logstash/filters/grok/timeout_enforcer.rb
+- lib/logstash/filters/grok/timeout_exception.rb
 - logstash-filter-grok.gemspec
 - spec/filters/grok_spec.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html
@@ -109,7 +111,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.3
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: Parse arbitrary text and structure it.