logstash-filter-grok 4.0.4 → 4.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 16a3453f2bf94d8eb76f5cb9127750ef1e1f7b801d53265e1d862164645b6adf
-  data.tar.gz: d5f193a61bc62ab63ecab9b5f912fd98418b25ab045998f39769e9c50dcd7938
+  metadata.gz: 228ed753ac1ef592e06994285f15a05f8000ef24b70e12b13176f64d9ac9761e
+  data.tar.gz: 31c62f48ac6240644c0e4e0eebd55e80944b5f693769f2a204540b4cb05dfed2
 SHA512:
-  metadata.gz: b18bb87b598ff1310d0cdd7188f3ab2c07a4888dfb1846bc81d94501c13791e55e106e3d9b1b888bc48ccccfc99a5df40a84f52672d1282b4d1c76e92a4f14e3
-  data.tar.gz: 397b8f0c2acd590dfab7db3efc5f79579dc64ce39d0c9df99acc5752877db6cb1477abab689737a923eaf2d1a7adb1fc52305390fd1ed6408cda01158e6b3dc8
+  metadata.gz: cb7923ffbcc68987bcee124bc4c85155db5e99ecbd74baf9ef27daa1e400f7de0c7072cfcfe460e1697230b4a1adbedeb98d3f5943e3a73d9f0cce5f152a0b63
+  data.tar.gz: 5465de021ca4e73f95cd2ee3ca5d45bdf2cbc6e116ef16f436f89231911dd1332aeb26e37d3e16a409a8370c0072389d82ee5dcd69717c73fc2e4032cb1081d0

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 4.1.0
+ - Changed timeout handling using the Timeout class [#147](https://github.com/logstash-plugins/logstash-filter-grok/pull/147)
+
 ## 4.0.4
   - Added info and link to documentation for logstash-filter-dissect as another option for extracting unstructured event data into fields
   [#144](https://github.com/logstash-plugins/logstash-filter-grok/issues/144)

data/lib/logstash/filters/grok.rb

@@ -5,6 +5,7 @@
   require "logstash/patterns/core"
   require "grok-pure" # rubygem 'jls-grok'
   require "set"
+  require "timeout"
 
   # Parse arbitrary text and structure it.
   #
@@ -144,8 +145,6 @@
   #
   class LogStash::Filters::Grok < LogStash::Filters::Base
     config_name "grok"
-    require "logstash/filters/grok/timeout_enforcer"
-    require "logstash/filters/grok/timeout_exception"
 
     # A hash of matches of field => value
     #
@@ -237,8 +236,6 @@
     # will be parsed and `hello world` will overwrite the original message.
     config :overwrite, :validate => :array, :default => []
 
-    attr_reader :timeout_enforcer
-    
     # Register default pattern paths
     @@patterns_path ||= Set.new
     @@patterns_path += [
@@ -246,14 +243,10 @@
       LogStash::Environment.pattern_path("*")
     ]
 
-    public
     def register
       # a cache of capture name handler methods.
       @handlers = {}
 
-      @timeout_enforcer = TimeoutEnforcer.new(@logger, @timeout_millis * 1000000)
-      @timeout_enforcer.start! unless @timeout_millis == 0
-
       @patternfiles = []
 
       # Have @@patterns_path show first. Last-in pattern definitions win; this
@@ -284,9 +277,13 @@
       end # @match.each
       @match_counter = metric.counter(:matches)
       @failure_counter = metric.counter(:failures)
+
+      # divide by float to allow fractionnal seconds, the Timeout class timeout value is in seconds but the underlying
+      # executor resolution is in microseconds so fractionnal second parameter down to microseconds is possible.
+      # see https://github.com/jruby/jruby/blob/9.2.7.0/core/src/main/java/org/jruby/ext/timeout/Timeout.java#L125
+      @timeout_seconds = @timeout_millis / 1000.0
     end # def register
 
-    public
     def filter(event)
       matched = false
 
@@ -309,13 +306,17 @@
       end
 
       @logger.debug? and @logger.debug("Event now: ", :event => event)
-    rescue ::LogStash::Filters::Grok::TimeoutException => e
+    rescue GrokTimeoutException => e
       @logger.warn(e.message)
       metric.increment(:timeouts)
       event.tag(@tag_on_timeout)
     end # def filter
 
+    def close
+    end
+
     private
+
     def match(groks, field, event)
       input = event.get(field)
       if input.is_a?(Array)
@@ -331,15 +332,13 @@
       @logger.warn("Grok regexp threw exception", :exception => e.message, :backtrace => e.backtrace, :class => e.class.name)
       return false
     end
-    
-    private
+
     def match_against_groks(groks, field, input, event)
       input = input.to_s
       matched = false
       groks.each do |grok|
         # Convert anything else to string (number, hash, etc)
-
-        matched = @timeout_enforcer.grok_till_timeout(grok, field, input)
+        matched = grok_till_timeout(grok, field, input)
         if matched
           grok.capture(matched) {|field, value| handle(field, value, event)}
           break if @break_on_match
@@ -349,7 +348,14 @@
       matched
     end
 
-    private
+    def grok_till_timeout(grok, field, value)
+      begin
+        @timeout_seconds > 0.0 ? Timeout.timeout(@timeout_seconds, TimeoutError) { grok.execute(value) } : grok.execute(value)
+      rescue TimeoutError
+        raise GrokTimeoutException.new(grok, field, value)
+      end
+    end
+
     def handle(field, value, event)
       return if (value.nil? || (value.is_a?(String) && value.empty?)) unless @keep_empty_captures
 
@@ -373,7 +379,6 @@
       end
     end
 
-    private
     def patterns_files_from_paths(paths, glob)
       patternfiles = []
       @logger.debug("Grok patterns path", :paths => paths)
@@ -394,7 +399,6 @@
       patternfiles
     end # def patterns_files_from_paths
 
-    private
     def add_patterns_from_files(paths, grok)
       paths.each do |path|
         if !File.exists?(path)
@@ -404,7 +408,6 @@
       end
     end # def add_patterns_from_files
 
-    private
     def add_patterns_from_inline_definition(pattern_definitions, grok)
       pattern_definitions.each do |name, pattern|
         next if pattern.nil?
@@ -412,8 +415,27 @@
       end
     end
 
-    def close
-      @timeout_enforcer.stop!
-    end
+    class TimeoutError < RuntimeError; end
+
+    class GrokTimeoutException < Exception
+      attr_reader :grok, :field, :value
 
+      def initialize(grok, field, value)
+        @grok = grok
+        @field = field
+        @value = value
+      end
+
+      def message
+        "Timeout executing grok '#{@grok.pattern}' against field '#{field}' with value '#{trunc_value}'!"
+      end
+
+      def trunc_value
+        if value.size <= 255 # If no more than 255 chars
+          value
+        else
+          "Value too large to output (#{value.bytesize} bytes)! First 255 chars are: #{value[0..255]}"
+        end
+      end
+    end
   end # class LogStash::Filters::Grok

data/logstash-filter-grok.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-grok'
-  s.version         = '4.0.4'
+  s.version         = '4.1.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Parses unstructured event data into fields"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/filters/grok_spec.rb

@@ -850,26 +850,9 @@ describe LogStash::Filters::Grok do
       plugin.register
     end
 
-    it "should start the timeout enforcer" do
-      expect(plugin.timeout_enforcer.running).to be true
-    end
-
-    context "with the timeout enforcer disabled" do
-      let(:config) { super.merge("timeout_millis" => 0) }
-
-      it "should not start the timeout enforcer" do
-        expect(plugin.timeout_enforcer.running).to be false
-      end
-    end
-
     it "should close cleanly" do
       expect { plugin.do_close }.not_to raise_error
     end
-
-    it "should stop the timeout enforcer" do
-      plugin.do_close
-      expect(plugin.timeout_enforcer.running).to be false
-    end
   end
 
   describe "after grok when the event is JSON serialised the field values are unchanged" do
@@ -929,4 +912,37 @@ describe LogStash::Filters::Grok do
     end
   end
 
-end
+
+  describe "direct plugin testing" do
+    subject do
+      plugin = LogStash::Filters::Grok.new(options)
+      plugin.register
+      plugin
+    end
+
+    let(:data) { {"message" => message} }
+    let(:event) { LogStash::Event.new(data) }
+
+    context 'when timeouts are explicitly disabled' do
+      let(:options) do
+        {
+          "timeout_millis" => 0
+        }
+      end
+
+      context 'when given a pathological input' do
+        let(:message) { "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}
+        let(:options) { super().merge("match" => { "message" => "(.*a){30}" }) }
+
+        it 'blocks for at least 3 seconds' do
+          blocking_exception_class = Class.new(::Exception) # avoid RuntimeError
+          expect do
+            Timeout.timeout(3, blocking_exception_class) do
+              subject.filter(event)
+            end
+          end.to raise_exception(blocking_exception_class)
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-grok
 version: !ruby/object:Gem::Version
-  version: 4.0.4
+  version: 4.1.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-10-19 00:00:00.000000000 Z
+date: 2019-07-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -116,8 +116,6 @@ files:
 - README.md
 - docs/index.asciidoc
 - lib/logstash/filters/grok.rb
-- lib/logstash/filters/grok/timeout_enforcer.rb
-- lib/logstash/filters/grok/timeout_exception.rb
 - logstash-filter-grok.gemspec
 - spec/filters/grok_spec.rb
 homepage: http://www.elastic.co/guide/en/logstash/current/index.html

data/lib/logstash/filters/grok/timeout_enforcer.rb

@@ -1,72 +0,0 @@
-class LogStash::Filters::Grok::TimeoutEnforcer
-  def initialize(logger, timeout_nanos)
-    @logger = logger
-    @running = java.util.concurrent.atomic.AtomicBoolean.new(false)
-    @timeout_nanos = timeout_nanos
-
-    # Stores running matches with their start time, this is used to cancel long running matches
-    # Is a map of Thread => start_time
-    @threads_to_start_time = java.util.concurrent.ConcurrentHashMap.new
-  end
-
-  def running
-    @running.get()
-  end
-
-  def grok_till_timeout(grok, field, value)
-    begin
-      thread = java.lang.Thread.currentThread()
-      @threads_to_start_time.put(thread, java.lang.System.nanoTime)
-      grok.execute(value)
-    rescue InterruptedRegexpError, java.lang.InterruptedException => e
-      raise ::LogStash::Filters::Grok::TimeoutException.new(grok, field, value)
-    ensure
-      # If the regexp finished, but interrupt was called after, we'll want to
-      # clear the interrupted status anyway
-      @threads_to_start_time.remove(thread)
-      thread.interrupted
-    end
-  end
-
-  def start!
-    @running.set(true)
-    @timer_thread = Thread.new do
-      while @running.get()
-        begin
-          cancel_timed_out!
-        rescue Exception => e
-          @logger.error("Error while attempting to check/cancel excessively long grok patterns",
-                        :message => e.message,
-                        :class => e.class.name,
-                        :backtrace => e.backtrace
-                       )
-        end
-        sleep 0.25
-      end
-    end
-  end
-
-  def stop!
-    @running.set(false)
-    # Check for the thread mostly for a fast start/shutdown scenario
-    @timer_thread.join if @timer_thread
-  end
-
-  private
-
-  def cancel_timed_out!
-    now = java.lang.System.nanoTime # save ourselves some nanotime calls
-    @threads_to_start_time.keySet.each do |thread|
-      # Use compute to lock this value
-      @threads_to_start_time.computeIfPresent(thread) do |thread, start_time|
-        if start_time < now && now - start_time > @timeout_nanos
-          thread.interrupt
-          nil # Delete the key
-        else
-          start_time # preserve the key
-        end
-      end
-    end
-  end
-
-end

data/lib/logstash/filters/grok/timeout_exception.rb

@@ -1,21 +0,0 @@
-class LogStash::Filters::Grok::TimeoutException < Exception
-  attr_reader :grok, :field, :value
-  
-  def initialize(grok=nil, field=nil, value=nil)
-    @field = field
-    @value = value
-    @grok = grok
-  end
-
-  def message
-    "Timeout executing grok '#{@grok.pattern}' against field '#{field}' with value '#{trunc_value}'!"
-  end
-
-  def trunc_value
-    if value.size <= 255 # If no more than 255 chars
-      value
-    else
-      "Value too large to output (#{value.bytesize} bytes)! First 255 chars are: #{value[0..255]}"
-    end
-  end
-end