logstash-filter-throttle 3.0.2 → 4.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 49222db6e126acebec2ed52567967ab4ae8a6d43
-  data.tar.gz: e28c425f6b7be002bb1f1441fba0745e7ad7bfb8
+  metadata.gz: 29516e66e87030588ab5f56191b0ac346efbaaa7
+  data.tar.gz: 74a3f0a23b80d19244eca960b00bd5f80898b253
 SHA512:
-  metadata.gz: f4604cb0222bced5174872b4a6b7c908b70a972b77d84128151722f4230f28dd78ddbeb444a8250e67393d72a7f173db00bfa83aa2182a117ea617c35d4a5478
-  data.tar.gz: d9c69b7b908c8fdfb5e4a2c9a2291eab3333c8cc3850af0d8d2125357e6e6e686f4a57ebb1840b6d1c7e106dc981247faf63d73b833fa5905032a7601a436e33
+  metadata.gz: a992934db1063a273159ff44ab7f00bba78f51a8c144b508c5d84c287ca10f53c1a4fb77143dffee7c018737bea7e2865bb414f050ab64ee879cf1032434c218
+  data.tar.gz: 035919b7fd35b9274c1e099946f4f36299a671d5cbc76559ec5efebd5aa4e225fbc6c0079c61b9dae855ecb3ed3c58d2775a84f2a05cce059b2b6d8832b78921

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 4.0.0
+  - Full reimplementation of the plugin. The plugin is now thread-safe and properly tracks past events.
+  - Updated tests and added runtime dependencies
+
 ## 3.0.2
   - Relax constraint on logstash-core-plugin-api to >= 1.60 <= 2.99
 

data/CONTRIBUTORS

@@ -6,6 +6,7 @@ Contributors:
 * Pier-Hugues Pellerin (ph)
 * Richard Pijnenburg (electrical)
 * Suyog Rao (suyograo)
+* Frank de Jong (frapex)
 
 Note: If you've sent us patches, bug reports, or otherwise contributed to
 Logstash, and you aren't on the list above and want to be, please let us know

data/lib/logstash/filters/throttle.rb

@@ -1,18 +1,22 @@
 require "logstash/filters/base"
 require "logstash/namespace"
+require "thread_safe"
+require "atomic"
 
-# The throttle filter is for throttling the number of events received. The filter
-# is configured with a lower bound, the `before_count`, and upper bound, the `after_count`,
-# and a period of time. All events passing through the filter will be counted based on 
-# a key. As long as the count is less than the `before_count` or greater than the 
-# `after_count`, the event will be "throttled" which means the filter will be considered 
-# successful and any tags or fields will be added.
+# The throttle filter is for throttling the number of events.  The filter is
+# configured with a lower bound, the "before_count", and upper bound, the "after_count",
+# and a period of time.  All events passing through the filter will be counted based on
+# their key and the event timestamp.  As long as the count is less than the "before_count"
+# or greater than the "after_count", the event will be "throttled" which means the filter
+# will be considered successful and any tags or fields will be added (or removed).
 #
-# For example, if you wanted to throttle events so you only receive an event after 2 
-# occurrences and you get no more than 3 in 10 minutes, you would use the 
-# configuration:
+# The plugin is thread-safe and properly tracks past events.
+#
+# For example, if you wanted to throttle events so you only receive an event after 2
+# occurrences and you get no more than 3 in 10 minutes, you would use the configuration:
 # [source,ruby]
 #     period => 600
+#     max_age => 1200
 #     before_count => 3
 #     after_count => 5
 #
@@ -35,10 +39,11 @@ require "logstash/namespace"
 #     event 6 - throttled (successful filter)
 #     ...
 # ==========================
-# Another example is if you wanted to throttle events so you only receive 1 event per 
-# hour, you would use the configuration:
+# Another example is if you wanted to throttle events so you only
+# receive 1 event per hour, you would use the configuration:
 # [source,ruby]
 #     period => 3600
+#     max_age => 7200
 #     before_count => -1
 #     after_count => 1
 #
@@ -56,8 +61,8 @@ require "logstash/namespace"
 #     event 4 - throttled (successful filter)
 #     ...
 # ==========================
-# A common use case would be to use the throttle filter to throttle events before 3 and 
-# after 5 while using multiple fields for the key and then use the drop filter to remove 
+# A common use case would be to use the throttle filter to throttle events before 3 and
+# after 5 while using multiple fields for the key and then use the drop filter to remove
 # throttled events. This configuration might appear as:
 # [source,ruby]
 #     filter {
@@ -65,6 +70,7 @@ require "logstash/namespace"
 #         before_count => 3
 #         after_count => 5
 #         period => 3600
+#         max_age => 7200
 #         key => "%{host}%{message}"
 #         add_tag => "throttled"
 #       }
@@ -73,8 +79,8 @@ require "logstash/namespace"
 #       }
 #     }
 #
-# Another case would be to store all events, but only email non-throttled 
-# events so the op's inbox isn't flooded with emails in the event of a system error. 
+# Another case would be to store all events, but only email non-throttled events
+# so the op's inbox isn't flooded with emails in the event of a system error.
 # This configuration might appear as:
 # [source,ruby]
 #     filter {
@@ -82,6 +88,7 @@ require "logstash/namespace"
 #         before_count => 3
 #         after_count => 5
 #         period => 3600
+#         max_age => 7200
 #         key => "%{message}"
 #         add_tag => "throttled"
 #       }
@@ -89,12 +96,12 @@ require "logstash/namespace"
 #     output {
 #       if "throttled" not in [tags] {
 #         email {
-#    	    from => "logstash@mycompany.com"
-#    	    subject => "Production System Alert"
-#    	    to => "ops@mycompany.com"
-#    	    via => "sendmail"
-#    	    body => "Alert on %{host} from path %{path}:\n\n%{message}"
-#    	    options => { "location" => "/usr/sbin/sendmail" }
+#           from => "logstash@mycompany.com"
+#           subject => "Production System Alert"
+#           to => "ops@mycompany.com"
+#           via => "sendmail"
+#           body => "Alert on %{host} from path %{path}:\n\n%{message}"
+#           options => { "location" => "/usr/sbin/sendmail" }
 #         }
 #       }
 #       elasticsearch_http {
@@ -103,161 +110,200 @@ require "logstash/namespace"
 #       }
 #     }
 #
-# The event counts are cleared after the configured period elapses since the 
-# first instance of the event. That is, all the counts don't reset at the same 
-# time but rather the throttle period is per unique key value.
+# When an event is received, the event key is stored in a key_cache.  The key references
+# a timeslot_cache.  The event is allocated to a timeslot (created dynamically) based on
+# the timestamp of the event.  The timeslot counter is incremented.  When the next event is
+# received (same key), within the same "period", it is allocated to the same timeslot.
+# The timeslot counter is incremented once again.
+#
+# The timeslot expires if the maximum age has been exceeded.  The age is calculated
+# based on the latest event timestamp and the max_age configuration option.
+#
+#         ---[::.. DESIGN ..::]---
 #
+# +- [key_cache] -+  +-- [timeslot_cache] --+
+# |               |  | @created: 1439839636 |
+#                    | @latest:  1439839836 |
+#    [a.b.c]  =>     +----------------------+
+#                    | [1439839636] => 1    |
+#                    | [1439839736] => 3    |
+#                    | [1439839836] => 2    |
+#                    +----------------------+
+#
+#                    +-- [timeslot_cache] --+
+#                    | @created: eeeeeeeeee |
+#                    | @latest:  llllllllll |
+#    [x.y.z]  =>     +----------------------+
+#                    | [0000000060] => x    |
+#                    | [0000000120] => y    |
+# |               |  | [..........] => N    |
+# +---------------+  +----------------------+
+#
+# Frank de Jong (@frapex)
 # Mike Pilone (@mikepilone)
-# 
-class LogStash::Filters::Throttle < LogStash::Filters::Base
+#
+
+class ThreadSafe::TimeslotCache < ThreadSafe::Cache
+  attr_reader :created
+
+  def initialize(epoch, options = nil, &block)
+    @created = epoch
+    @latest = Atomic.new(epoch)
+
+    super(options, &block)
+  end
 
+  def latest
+    @latest.value
+  end
+
+  def latest=(val)
+    # only update if greater than current
+    @latest.update { |v| v = (val > v) ? val : v }
+  end
+end
+
+class LogStash::Filters::Throttle < LogStash::Filters::Base
   # The name to use in configuration files.
   config_name "throttle"
 
+  # The memory control mechanism automatically ajusts the maximum age
+  # of a timeslot based on the maximum number of counters.
+  MC_MIN_PCT = 5    # Lower bound percentage.
+  MC_MAX_PCT = 100  # Upper bound percentage.
+  MC_INCR_PCT = 80  # Increase if total below percentage.
+  MC_STEP_PCT = 5   # Increase/decrease by this percentage at a time.
+
+  # Call the filter flush method at regular interval.  It is used by the memory
+  # control mechanism.  Set to false if you like your VM to go (B)OOM.
+  config :periodic_flush, :validate => :boolean, :default => true
 
-  # The key used to identify events. Events with the same key will be throttled
-  # as a group.  Field substitutions are allowed, so you can combine multiple
-  # fields.
+  # The key used to identify events.  Events with the same key are grouped together.
+  # Field substitutions are allowed, so you can combine multiple fields.
   config :key, :validate => :string, :required => true
-  
-  # Events less than this count will be throttled. Setting this value to -1, the 
-  # default, will cause no messages to be throttled based on the lower bound.
+
+  # Events less than this count will be throttled.  Setting this value to -1, the
+  # default, will cause no events to be throttled based on the lower bound.
   config :before_count, :validate => :number, :default => -1, :required => false
-  
-  # Events greater than this count will be throttled. Setting this value to -1, the 
-  # default, will cause no messages to be throttled based on the upper bound.
+
+  # Events greater than this count will be throttled.  Setting this value to -1, the
+  # default, will cause no events to be throttled based on the upper bound.
   config :after_count, :validate => :number, :default => -1, :required => false
-  
-  # The period in seconds after the first occurrence of an event until the count is 
-  # reset for the event. This period is tracked per unique key value.  Field
-  # substitutions are allowed in this value.  They will be evaluated when the _first_
-  # event for a given key is seen.  This allows you to specify that certain kinds
-  # of events throttle for a specific period.
-  config :period, :validate => :string, :default => "3600", :required => false
-  
-  # The maximum number of counters to store before the oldest counter is purged. Setting 
-  # this value to -1 will prevent an upper bound no constraint on the number of counters  
-  # and they will only be purged after expiration. This configuration value should only 
-  # be used as a memory control mechanism and can cause early counter expiration if the 
-  # value is reached. It is recommended to leave the default value and ensure that your 
-  # key is selected such that it limits the number of counters required (i.e. don't 
-  # use UUID as the key!)
+
+  # The period in seconds after the first occurrence of an event until a new timeslot
+  # is created.  This period is tracked per unique key and per timeslot.
+  # Field substitutions are allowed in this value.  This allows you to specify that
+  # certain kinds of events throttle for a specific period of time.
+  config :period, :validate => :string, :default => "60", :required => false
+
+  # The maximum age of a timeslot.  Higher values allow better tracking of an asynchronous
+  # flow of events, but require more memory.  As a rule of thumb you should set this value
+  # to at least twice the period.  Or set this value to period + maximum time offset
+  # between unordered events with the same key.  Values below the specified period give
+  # unexpected results if unordered events are processed simultaneously.
+  config :max_age, :validate => :number, :default => 3600, :required => false
+
+  # The maximum number of counters to store before decreasing the maximum age of a timeslot.
+  # Setting this value to -1 will prevent an upper bound with no constraint on the
+  # number of counters.  This configuration value should only be used as a memory
+  # control mechanism and can cause early counter expiration if the value is reached.
+  # It is recommended to leave the default value and ensure that your key is selected
+  # such that it limits the number of counters required (i.e. don't use UUID as the key).
   config :max_counters, :validate => :number, :default => 100000, :required => false
 
-  # Performs initialization of the filter.
+  # performs initialization of the filter
   public
   def register
-    @threadsafe = false
-  
-    @event_counters = Hash.new
-    @next_expiration = nil
+    @key_cache = ThreadSafe::Cache.new
+    @max_age_orig = @max_age
   end # def register
 
-  # Filters the event. The filter is successful if the event should be throttled.
+  # filters the event
   public
   def filter(event)
-      	  
-    # Return nothing unless there's an actual filter event
-    
-    	  
-    now = Time.now
-    key = event.sprintf(@key)
-    
-    # Purge counters if too large to prevent OOM.
-    if @max_counters != -1 && @event_counters.size > @max_counters then
-      purgeOldestEventCounter()
-    end
-    
-    # Expire existing counter if needed
-    if @next_expiration.nil? || now >= @next_expiration then
-    	expireEventCounters(now)
+    key = event.sprintf(@key)                # substitute field
+    period = event.sprintf(@period).to_i     # substitute period
+    period = 60 if period == 0               # fallback if unparsable
+    epoch = event.timestamp.to_i             # event epoch time
+
+    @key_cache.compute_if_absent(key) do     # initialise timeslot cache
+      ThreadSafe::TimeslotCache.new(epoch)   # and add to key cache
     end
-    
+
+    timeslot_cache = @key_cache[key]         # get timeslot cache
+    timeslot_cache.latest = epoch            # update to latest epoch
+
+    # find target timeslot
+    timeslot_key = epoch - (epoch - timeslot_cache.created) % period
+
+    # initialise timeslot and counter (if required)
+    timeslot_cache.compute_if_absent(timeslot_key) { Atomic.new(0) }
+
+    timeslot = timeslot_cache[timeslot_key]  # get timeslot
+    timeslot.update { |v| v + 1 }            # increment counter
+    count = timeslot.value                   # get latest counter value
+
     @logger.debug? and @logger.debug(
-      	  "filters/#{self.class.name}: next expiration", 
-      	  { "next_expiration" => @next_expiration })
-    
-    # Create new counter for this event if this is the first occurrence
-    counter = nil
-    if !@event_counters.include?(key) then
-      period = event.sprintf(@period).to_i
-      period = 3600 if period == 0
-      expiration = now + period
-      @event_counters[key] = { :count => 0, :expiration => expiration }
-      
-      @logger.debug? and @logger.debug("filters/#{self.class.name}: new event", 
-      	  { :key => key, :expiration => expiration })
-    end
-    
-    # Fetch the counter
-    counter = @event_counters[key]
-    
-    # Count this event
-    counter[:count] = counter[:count] + 1;
-    
-    @logger.debug? and @logger.debug("filters/#{self.class.name}: current count", 
-      	  { :key => key, :count => counter[:count] })
-    
-    # Throttle if count is < before count or > after count
-    if ((@before_count != -1 && counter[:count] < @before_count) || 
-       (@after_count != -1 && counter[:count] > @after_count)) then
+      "filters/#{self.class.name}: counter incremented",
+      { key: key, epoch: epoch, timeslot: timeslot_key, count: count }
+    )
+
+    # throttle event if counter value not in range
+    if ((@before_count != -1 && count < @before_count) ||
+        (@after_count != -1 && count > @after_count))
       @logger.debug? and @logger.debug(
-      	  "filters/#{self.class.name}: throttling event", { :key => key })
-      	
+        "filters/#{self.class.name}: throttling event",
+        { key: key, epoch: epoch }
+      )
+
       filter_matched(event)
     end
-        
+
+    # Delete expired timeslots older than the latest.  Do not use variable
+    # timeslot_cache.latest for this.  If used, it might delete the latest timeslot.
+    latest_timeslot = timeslot_cache.keys.max || 0
+    timeslot_cache.each_key { |key| timeslot_cache.delete(key) if key < (latest_timeslot - @max_age) }
   end # def filter
-  
-  # Expires any counts where the period has elapsed. Sets the next expiration time 
-  # for when this method should be called again.
-  private
-  def expireEventCounters(now) 
-    
-    @next_expiration = nil
-    
-    @event_counters.delete_if do |key, counter|
-      expiration = counter[:expiration]
-      expired = expiration <= now
-    
-      if expired then
-      	@logger.debug? and @logger.debug(
-      	  "filters/#{self.class.name}: deleting expired counter", 
-      	  { :key => key })
-      	  
-      elsif @next_expiration.nil? || (expiration < @next_expiration)
-      	@next_expiration = expiration
+
+  public
+  def flush(options = {})
+    max_latest = 0                           # get maximum epoch
+    @key_cache.each_value { |tc| max_latest = tc.latest if tc.latest > max_latest }
+
+    total_counters = 0
+    @key_cache.each_pair do |key,timeslot_cache|
+      if timeslot_cache.latest < max_latest - @max_age
+        @key_cache.delete(key)               # delete expired timeslot cache
+      else
+        total_counters += timeslot_cache.size # get total number of counters
       end
-      
-      expired
     end
-  
-  end # def expireEventCounters
-  
-  # Purges the oldest event counter. This operation is for memory control only 
-  # and can cause early period expiration and thrashing if invoked.
-  private
-  def purgeOldestEventCounter()
-    
-    # Return unless we have something to purge
-    return unless @event_counters.size > 0
-    
-    oldestCounter = nil
-    oldestKey = nil
-    
-    @event_counters.each do |key, counter|
-      if oldestCounter.nil? || counter[:expiration] < oldestCounter[:expiration] then
-        oldestKey = key;
-        oldestCounter = counter;
+
+    @logger.debug? and @logger.debug(
+      "filters/#{self.class.name}: statistics",
+      { total_counters: total_counters, max_age: @max_age }
+    )
+
+    # memory control mechanism
+    if @max_counters != -1
+      over_limit = total_counters - @max_counters
+
+      # decrease max age of timeslot cache by x percent
+      if (over_limit > 0) && (@max_age > @max_age_orig * MC_MIN_PCT / 100)
+        @max_age -= @max_age_orig * MC_STEP_PCT / 100
+        @logger.warn? and @logger.warn(
+          "filters/#{self.class.name}: Decreased timeslot max_age to #{@max_age} because " +
+          "max_counters exceeded by #{over_limit}. Use a better key to prevent too many unique event counters.")
+
+      # increase max age of timeslot cache by x percent
+      elsif (@max_age < @max_age_orig * MC_MAX_PCT / 100) && (total_counters < (@max_counters * MC_INCR_PCT / 100))
+        @max_age += @max_age_orig * MC_STEP_PCT / 100
+        @logger.warn? and @logger.warn(
+          "filters/#{self.class.name}: Increased timeslot max_age to #{@max_age} because max_counters no longer exceeded.")
       end
     end
-    
-    @logger.warn? and @logger.warn(
-      "filters/#{self.class.name}: Purging oldest counter because max_counters " +
-      "exceeded. Use a better key to prevent too many unique event counters.", 
-      { :key => oldestKey, :expiration => oldestCounter[:expiration] })
-      	  
-    @event_counters.delete(oldestKey)
-    
-  end
+
+    return
+  end # def flush
+
 end # class LogStash::Filters::Throttle

data/logstash-filter-throttle.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-throttle'
-  s.version         = '3.0.2'
+  s.version         = '4.0.0'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "The throttle filter is for throttling the number of events received."
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -21,7 +21,8 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+  s.add_runtime_dependency "thread_safe"
+  s.add_runtime_dependency "atomic"
 
   s.add_development_dependency 'logstash-devutils'
 end
-

data/spec/filters/throttle_spec.rb

@@ -163,14 +163,13 @@ describe LogStash::Filters::Throttle do
       }
     end
   end
-  
-  describe "max_counter exceeded" do
+
+  describe "correct timeslot assigned/calculated, after_count exceeded" do
     config <<-CONFIG
       filter {
         throttle {
           period => 60
           after_count => 1
-          max_counters => 2
           key => "%{message}"
           add_tag => [ "throttled" ]
         }
@@ -178,19 +177,67 @@ describe LogStash::Filters::Throttle do
     CONFIG
 
     events = [{
-      "message" => "foo"
+      "@timestamp" => "2016-07-09T00:05:00.000Z",
+      "message"    => "server1"
     }, {
-      "message" => "bar"
+      "@timestamp" => "2016-07-09T00:05:59.000Z",
+      "message"    => "server1"
     }, {
-      "message" => "poo"
+      "@timestamp" => "2016-07-09T00:10:33.000Z",
+      "message"    => "server1"
     }, {
-      "message" => "foo"
+      "@timestamp" => "2016-07-09T00:10:34.000Z",
+      "message"    => "server1"
+    }, {
+      "@timestamp" => "2016-07-09T00:00:00.000Z",
+      "message"    => "server1"
+    }, {
+      "@timestamp" => "2016-07-09T00:00:45.000Z",
+      "message"    => "server1"
     }]
 
     sample events do
-      insist { subject[3].get("tags") } == nil
+      insist { subject[0].get("tags") } == nil
+      insist { subject[1].get("tags") } == [ "throttled" ]
+      insist { subject[2].get("tags") } == nil
+      insist { subject[3].get("tags") } == [ "throttled" ]
+      insist { subject[4].get("tags") } == nil
+      insist { subject[5].get("tags") } == [ "throttled" ]
     end
   end
 
-end # LogStash::Filters::Throttle
+  describe "asynchronous input, after_count exceeded" do
+    config <<-CONFIG
+      filter {
+        throttle {
+          period => 60
+          after_count => 1
+          key => "%{message}"
+          add_tag => [ "throttled" ]
+        }
+      }
+    CONFIG
+
+    events = [{
+      "@timestamp" => "2016-07-09T00:01:00.000Z",
+      "message"    => "server1"
+    }, {
+      "@timestamp" => "2016-07-09T00:00:30.000Z",
+      "message"    => "server1"
+    }, {
+      "@timestamp" => "2016-07-09T00:01:59.000Z",
+      "message"    => "server1"
+    }, {
+      "@timestamp" => "2016-07-09T00:00:59.000Z",
+      "message"    => "server1"
+    }]
+
+    sample events do
+      insist { subject[0].get("tags") } == nil
+      insist { subject[1].get("tags") } == nil
+      insist { subject[2].get("tags") } == [ "throttled" ]
+      insist { subject[3].get("tags") } == [ "throttled" ]
+    end
+  end
 
+end # LogStash::Filters::Throttle

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-throttle
 version: !ruby/object:Gem::Version
-  version: 3.0.2
+  version: 4.0.0
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-07-14 00:00:00.000000000 Z
+date: 2016-07-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -30,6 +30,34 @@ dependencies:
     - - "<="
       - !ruby/object:Gem::Version
         version: '2.99'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: thread_safe
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: atomic
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -81,7 +109,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.6.3
+rubygems_version: 2.4.8
 signing_key:
 specification_version: 4
 summary: The throttle filter is for throttling the number of events received.