logstash-filter-throttle 4.0.0 → 4.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 29516e66e87030588ab5f56191b0ac346efbaaa7
-  data.tar.gz: 74a3f0a23b80d19244eca960b00bd5f80898b253
+  metadata.gz: fad6d0a8d7a465d2f90cfc4ccb065804890444fe
+  data.tar.gz: 201cd6b667a0d2034116ef37a154aa8f3bd32dd1
 SHA512:
-  metadata.gz: a992934db1063a273159ff44ab7f00bba78f51a8c144b508c5d84c287ca10f53c1a4fb77143dffee7c018737bea7e2865bb414f050ab64ee879cf1032434c218
-  data.tar.gz: 035919b7fd35b9274c1e099946f4f36299a671d5cbc76559ec5efebd5aa4e225fbc6c0079c61b9dae855ecb3ed3c58d2775a84f2a05cce059b2b6d8832b78921
+  metadata.gz: 089baf8ba028077104a6bcde68d87eee9298825cb7bc7414ac0291eff7aa8a9e80533be00330dc15efb66acacc80dd6a6bc182aa26f998cc1d7f7b3d2a79f248
+  data.tar.gz: 4fe4d68746fe2119a753876dfe7e927abd13b4c8ca5dd46f291d4c8593dbc3c99f79a768aee67d1ef3830a733d49b836996b6103117e7314060c12682a2564ad

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 4.0.1
+  - fix racing condition that causes logstash to crash under heavy load conditions
+
 ## 4.0.0
   - Full reimplementation of the plugin. The plugin is now thread-safe and properly tracks past events.
   - Updated tests and added runtime dependencies
@@ -6,15 +9,18 @@
   - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
 
 ## 3.0.1
-  - Republish all the gems under jruby.
+ - internal: Republish all the gems under jruby.
+
 ## 3.0.0
-  - Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
-# 2.0.4
-  - Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
-# 2.0.3
-  - New dependency requirements for logstash-core for the 5.0 release
+ - internal,deps: Update the plugin to the version 2.0 of the plugin api, this change is required for Logstash 5.0 compatibility. See https://github.com/elastic/logstash/issues/5141
+
+## 2.0.4
+ - internal,deps: Depend on logstash-core-plugin-api instead of logstash-core, removing the need to mass update plugins on major releases of logstash
+
+## 2.0.3
+ - internal,deps: New dependency requirements for logstash-core for the 5.0 release
+
 ## 2.0.0
- - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
+ - internal: Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully, 
    instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
- - Dependency on logstash-core update to 2.0
-
+ - internal,deps: Dependency on logstash-core update to 2.0

data/lib/logstash/filters/throttle.rb

@@ -221,27 +221,38 @@ class LogStash::Filters::Throttle < LogStash::Filters::Base
   # filters the event
   public
   def filter(event)
-    key = event.sprintf(@key)                # substitute field
-    period = event.sprintf(@period).to_i     # substitute period
-    period = 60 if period == 0               # fallback if unparsable
-    epoch = event.timestamp.to_i             # event epoch time
-
-    @key_cache.compute_if_absent(key) do     # initialise timeslot cache
-      ThreadSafe::TimeslotCache.new(epoch)   # and add to key cache
+    key = event.sprintf(@key)                 # substitute field
+    period = event.sprintf(@period).to_i      # substitute period
+    period = 60 if period == 0                # fallback if unparsable
+    epoch = event.timestamp.to_i              # event epoch time
+
+    while true
+      # initialise timeslot cache (if required)
+      @key_cache.compute_if_absent(key) { ThreadSafe::TimeslotCache.new(epoch) }
+      timeslot_cache = @key_cache[key]        # try to get timeslot cache
+      break unless timeslot_cache.nil?        # retry until succesful
+
+      @logger.warn? and @logger.warn(
+        "filters/#{self.class.name}: timeslot cache disappeared, increase max_counters to prevent this.")
     end
 
-    timeslot_cache = @key_cache[key]         # get timeslot cache
-    timeslot_cache.latest = epoch            # update to latest epoch
+    timeslot_cache.latest = epoch             # update to latest epoch
 
     # find target timeslot
     timeslot_key = epoch - (epoch - timeslot_cache.created) % period
 
-    # initialise timeslot and counter (if required)
-    timeslot_cache.compute_if_absent(timeslot_key) { Atomic.new(0) }
+    while true
+      # initialise timeslot and counter (if required)
+      timeslot_cache.compute_if_absent(timeslot_key) { Atomic.new(0) }
+      timeslot = timeslot_cache[timeslot_key] # try to get timeslot
+      break unless timeslot.nil?              # retry until succesful
 
-    timeslot = timeslot_cache[timeslot_key]  # get timeslot
-    timeslot.update { |v| v + 1 }            # increment counter
-    count = timeslot.value                   # get latest counter value
+      @logger.warn? and @logger.warn(
+        "filters/#{self.class.name}: timeslot disappeared, increase max_age to prevent this.")
+    end
+      
+    timeslot.update { |v| v + 1 }             # increment counter
+    count = timeslot.value                    # get latest counter value
 
     @logger.debug? and @logger.debug(
       "filters/#{self.class.name}: counter incremented",
@@ -267,13 +278,13 @@ class LogStash::Filters::Throttle < LogStash::Filters::Base
 
   public
   def flush(options = {})
-    max_latest = 0                           # get maximum epoch
+    max_latest = 0                            # get maximum epoch
     @key_cache.each_value { |tc| max_latest = tc.latest if tc.latest > max_latest }
 
     total_counters = 0
     @key_cache.each_pair do |key,timeslot_cache|
       if timeslot_cache.latest < max_latest - @max_age
-        @key_cache.delete(key)               # delete expired timeslot cache
+        @key_cache.delete(key)                # delete expired timeslot cache
       else
         total_counters += timeslot_cache.size # get total number of counters
       end

data/logstash-filter-throttle.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-throttle'
-  s.version         = '4.0.0'
+  s.version         = '4.0.1'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "The throttle filter is for throttling the number of events received."
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-throttle
 version: !ruby/object:Gem::Version
-  version: 4.0.0
+  version: 4.0.1
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-07-31 00:00:00.000000000 Z
+date: 2016-09-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement