logstash-filter-grok 3.4.4 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8a7d6245f3e543f6bee1730eaa9055caeb392ae193fb9a06269829fe3e30e838
-  data.tar.gz: cc3a933bfa9dc1cf892743c209500eb9db910b40cb8c15583ec36d6f54672061
+  metadata.gz: d06febb2d4c48dd7c02d2d89ecbe2c379c345b980687d10e80542192f3b232d9
+  data.tar.gz: ddb45461cff71c3fee2c7cd9e84780e0c0c216b207c23cee42ed39d499be25bf
 SHA512:
-  metadata.gz: 04055244f69d3f18a8a7e774c1353dd215c4e2e8900d312d738deb796e00631e060bf1c6fdaccdad4743431bd49c770bc2da07bf377f25110595134bfae7f4ae
-  data.tar.gz: 645106767ebce79cb04fd509073cc929326b9f3bd9c3b3bf219c227978e46fd5dfd4178d680e99c4f83053b143bf0ef667916d6f8f8237d975aa48702324dafe
+  metadata.gz: 7db250b93474764efb1c769aab63fe9e8850fa9eac4e14ad4b5ec82bdb296d6fe6edc92e4604c7f70cc52199929b2ce31b523cd07f75abb7fd781da64f74b385
+  data.tar.gz: 55051f431618fa87a944984de8b62ec81daaed169e719fad74d84d323b034d776d76bf26d568f5b587cabd16008ff2154c6b245f640287df36cf26ed7b7a0f9c

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 4.0.0
+  - Major performance improvements due to reduced locking
+
+## 3.4.5
+  - version yanked due to breaking changes within .patch release cause logstash crashes in < 5.6
+
 ## 3.4.4
   - Update gemspec summary
 

data/lib/logstash/filters/grok.rb

@@ -287,19 +287,18 @@
           @patterns[field] << grok
         end
       end # @match.each
+      @match_counter = metric.counter(:matches)
+      @failure_counter = metric.counter(:failures)
     end # def register
 
     public
     def filter(event)
       matched = false
-      done = false
 
-      @logger.debug? and @logger.debug("Running grok filter", :event => event);
+      @logger.debug? and @logger.debug("Running grok filter", :event => event)
 
       @patterns.each do |field, groks|
-        success = match(groks, field, event)
-        
-        if success
+        if match(groks, field, event)
           matched = true
           break if @break_on_match
         end
@@ -307,10 +306,10 @@
       end # @patterns.each
 
       if matched
-        metric.increment(:matches)
+        @match_counter.increment(1)
         filter_matched(event)
       else
-        metric.increment(:failures)
+        @failure_counter.increment(1)
         @tag_on_failure.each {|tag| event.tag(tag)}
       end
 
@@ -345,7 +344,7 @@
       groks.each do |grok|
         # Convert anything else to string (number, hash, etc)
 
-        matched = @timeout_enforcer.grok_till_timeout(event, grok, field, input) { grok.execute(input) }
+        matched = @timeout_enforcer.grok_till_timeout(grok, field, input)
         if matched
           grok.capture(matched) {|field, value| handle(field, value, event)}
           break if @break_on_match

data/lib/logstash/filters/grok/timeout_enforcer.rb

@@ -1,5 +1,3 @@
-java_import java.util.concurrent.locks.ReentrantLock
-
 class LogStash::Filters::Grok::TimeoutEnforcer
   attr_reader :running
 
@@ -10,24 +8,30 @@ class LogStash::Filters::Grok::TimeoutEnforcer
 
     # Stores running matches with their start time, this is used to cancel long running matches
     # Is a map of Thread => start_time
-    @threads_to_start_time = {}
-    @state_lock = ReentrantLock.new
+    @threads_to_start_time = java.util.concurrent.ConcurrentHashMap.new
+    @cancel_mutex = Mutex.new
   end
 
-  def grok_till_timeout(event, grok, field, value)
+  def grok_till_timeout(grok, field, value)
     begin
       thread = java.lang.Thread.currentThread()
       start_thread_groking(thread)
-      yield
+      grok.execute(value)
     rescue InterruptedRegexpError => e
       raise ::LogStash::Filters::Grok::TimeoutException.new(grok, field, value)
     ensure
-      stop_thread_groking(thread)
-      # Clear any interrupts from any previous invocations that were not caught by Joni
-      # It may appear that this should go in #stop_thread_groking but that would actually
-      # break functionality! If this were moved there we would clear the interrupt
-      # immediately after setting it in #cancel_timed_out, hence this MUST be here
-      thread.interrupted
+      unless stop_thread_groking(thread)
+        @cancel_mutex.lock
+        begin
+          # Clear any interrupts from any previous invocations that were not caught by Joni
+          # It may appear that this should go in #stop_thread_groking but that would actually
+          # break functionality! If this were moved there we would clear the interrupt
+          # immediately after setting it in #cancel_timed_out, hence this MUST be here
+          java.lang.Thread.interrupted
+        ensure
+          @cancel_mutex.unlock
+        end
+      end
     end
   end
 
@@ -64,44 +68,38 @@ class LogStash::Filters::Grok::TimeoutEnforcer
 
   def start_thread_groking(thread)
     # Clear any interrupts from any previous invocations that were not caught by Joni
-    thread.interrupted
-    synchronize do
-      @threads_to_start_time[thread] = java.lang.System.nanoTime()
-    end
+    java.lang.Thread.interrupted
+    @threads_to_start_time.put(thread, java.lang.System.nanoTime)
   end
 
+  # Returns falsy in case there was no Grok execution in progress for the thread
   def stop_thread_groking(thread)
-    synchronize do
-      @threads_to_start_time.delete(thread)
-    end
+    @threads_to_start_time.remove(thread)
   end
 
   def cancel_timed_out!
-    synchronize do
-      @threads_to_start_time.each do |thread,start_time|
-        now = java.lang.System.nanoTime # save ourselves some nanotime calls
-        elapsed = java.lang.System.nanoTime - start_time
-        if elapsed > @timeout_nanos
-          elapsed_millis = elapsed / 1000
-          thread.interrupt()
-          # Ensure that we never attempt to cancel this thread twice in the event
-          # of weird races
-          stop_thread_groking(thread)
+    now = java.lang.System.nanoTime # save ourselves some nanotime calls
+    @threads_to_start_time.entry_set.each do |entry|
+      start_time = entry.get_value
+      if start_time < now && now - start_time > @timeout_nanos
+        thread  = entry.get_key
+        # Ensure that we never attempt to cancel this thread unless a Grok execution is in progress
+        # Theoretically there is a race condition here in case the entry's grok action changed
+        # between evaluating the above condition on the start_time and calling stop_thread_groking
+        # Practically this is impossible, since it would require a whole loop of writing to an
+        # output, pulling new input events and starting a new Grok execution in worker thread
+        # in between the above `if start_time < now && now - start_time > @timeout_nanos` and
+        # the call to `stop_thread_groking`.
+        if stop_thread_groking(thread)
+          @cancel_mutex.lock
+          begin
+            thread.interrupt()
+          ensure
+            @cancel_mutex.unlock
+          end
         end
       end
     end
   end
 
-  # We use this instead of a Mutex because JRuby mutexes are interruptible
-  # We actually don't want that behavior since we always clear the interrupt in
-  # grok_till_timeout
-  def synchronize
-    # The JRuby Mutex uses lockInterruptibly which is what we DO NOT want
-    @state_lock.lock()
-    yield
-  ensure
-    @state_lock.unlock()
-  end
-
-
 end

data/logstash-filter-grok.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-grok'
-  s.version         = '3.4.4'
+  s.version         = '4.0.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "Parses unstructured event data into fields"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -21,10 +21,11 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency "logstash-core", ">= 5.6.0"
 
   s.add_runtime_dependency 'jls-grok', '~> 0.11.3'
   s.add_runtime_dependency 'stud', '~> 0.0.22'
   s.add_runtime_dependency 'logstash-patterns-core'
 
-  s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'logstash-devutils', '= 1.3.6'
 end

data/spec/filters/grok_spec.rb

@@ -889,7 +889,7 @@ describe LogStash::Filters::Grok do
       insist { subject.to_json } =~ %r|"@version":"1"|
       insist { subject.to_json } =~ %r|"username"|i
       insist { subject.to_json } =~ %r|"testuser"|
-      insist { subject.to_json } =~ %r|"tags":\["ssh_failure"\]}|
+      insist { subject.to_json } =~ %r|"tags":\["ssh_failure"\]|
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-grok
 version: !ruby/object:Gem::Version
-  version: 3.4.4
+  version: 4.0.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-11-07 00:00:00.000000000 Z
+date: 2017-11-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -30,6 +30,20 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.6.0
+  name: logstash-core
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 5.6.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -75,17 +89,17 @@ dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.3.6
   name: logstash-devutils
   prerelease: false
   type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.3.6
 description: This gem is a Logstash plugin required to be installed on top of the
   Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
   gem is not a stand-alone program
@@ -128,7 +142,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key:
 specification_version: 4
 summary: Parses unstructured event data into fields