logstash-filter-throttle 4.0.1 → 4.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fad6d0a8d7a465d2f90cfc4ccb065804890444fe
-  data.tar.gz: 201cd6b667a0d2034116ef37a154aa8f3bd32dd1
+  metadata.gz: 0d603f018f22197b49bacbe89a1e45b672bb9fad
+  data.tar.gz: 5d8dd804f76052b84a82035976ab7b2ae6a016a5
 SHA512:
-  metadata.gz: 089baf8ba028077104a6bcde68d87eee9298825cb7bc7414ac0291eff7aa8a9e80533be00330dc15efb66acacc80dd6a6bc182aa26f998cc1d7f7b3d2a79f248
-  data.tar.gz: 4fe4d68746fe2119a753876dfe7e927abd13b4c8ca5dd46f291d4c8593dbc3c99f79a768aee67d1ef3830a733d49b836996b6103117e7314060c12682a2564ad
+  metadata.gz: ecffd47a2d3c3c13aa4a846e99a3bbce3dca68ac99f79635d97678f090fd9a0c420b64c81dad6d5d6fa582f8f3f30d07b656c677988c1dbd561e674102d390f4
+  data.tar.gz: 9dd54ea19e284b7eda5506b6f2228c577c5d80f79162d782b7fd42639e90ff99cf367d33c54ffc8ceb2e0763f66b9a94082a3d9020886064b77a280a1a6735ec

data/Gemfile

@@ -1,4 +1,11 @@
 source 'https://rubygems.org'
 
-# Specify your gem's dependencies in logstash-mass_effect.gemspec
 gemspec
+
+logstash_path = ENV["LOGSTASH_PATH"] || "../../logstash"
+use_logstash_source = ENV["LOGSTASH_SOURCE"] && ENV["LOGSTASH_SOURCE"].to_s == "1"
+
+if Dir.exist?(logstash_path) && use_logstash_source
+  gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
+  gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
+end

data/docs/index.asciidoc

@@ -0,0 +1,252 @@
+:plugin: throttle
+:type: filter
+
+///////////////////////////////////////////
+START - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+:version: %VERSION%
+:release_date: %RELEASE_DATE%
+:changelog_url: %CHANGELOG_URL%
+:include_path: ../../../../logstash/docs/include
+///////////////////////////////////////////
+END - GENERATED VARIABLES, DO NOT EDIT!
+///////////////////////////////////////////
+
+[id="plugins-{type}-{plugin}"]
+
+=== Throttle filter plugin
+
+include::{include_path}/plugin_header.asciidoc[]
+
+==== Description
+
+The throttle filter is for throttling the number of events.  The filter is
+configured with a lower bound, the "before_count", and upper bound, the "after_count",
+and a period of time.  All events passing through the filter will be counted based on
+their key and the event timestamp.  As long as the count is less than the "before_count"
+or greater than the "after_count", the event will be "throttled" which means the filter
+will be considered successful and any tags or fields will be added (or removed).
+
+The plugin is thread-safe and properly tracks past events.
+
+For example, if you wanted to throttle events so you only receive an event after 2
+occurrences and you get no more than 3 in 10 minutes, you would use the configuration:
+[source,ruby]
+    period => 600
+    max_age => 1200
+    before_count => 3
+    after_count => 5
+
+Which would result in:
+==========================
+    event 1 - throttled (successful filter, period start)
+    event 2 - throttled (successful filter)
+    event 3 - not throttled
+    event 4 - not throttled
+    event 5 - not throttled
+    event 6 - throttled (successful filter)
+    event 7 - throttled (successful filter)
+    event x - throttled (successful filter)
+    period end
+    event 1 - throttled (successful filter, period start)
+    event 2 - throttled (successful filter)
+    event 3 - not throttled
+    event 4 - not throttled
+    event 5 - not throttled
+    event 6 - throttled (successful filter)
+    ...
+==========================
+Another example is if you wanted to throttle events so you only
+receive 1 event per hour, you would use the configuration:
+[source,ruby]
+    period => 3600
+    max_age => 7200
+    before_count => -1
+    after_count => 1
+
+Which would result in:
+==========================
+    event 1 - not throttled (period start)
+    event 2 - throttled (successful filter)
+    event 3 - throttled (successful filter)
+    event 4 - throttled (successful filter)
+    event x - throttled (successful filter)
+    period end
+    event 1 - not throttled (period start)
+    event 2 - throttled (successful filter)
+    event 3 - throttled (successful filter)
+    event 4 - throttled (successful filter)
+    ...
+==========================
+A common use case would be to use the throttle filter to throttle events before 3 and
+after 5 while using multiple fields for the key and then use the drop filter to remove
+throttled events. This configuration might appear as:
+[source,ruby]
+    filter {
+      throttle {
+        before_count => 3
+        after_count => 5
+        period => 3600
+        max_age => 7200
+        key => "%{host}%{message}"
+        add_tag => "throttled"
+      }
+      if "throttled" in [tags] {
+        drop { }
+      }
+    }
+
+Another case would be to store all events, but only email non-throttled events
+so the op's inbox isn't flooded with emails in the event of a system error.
+This configuration might appear as:
+[source,ruby]
+    filter {
+      throttle {
+        before_count => 3
+        after_count => 5
+        period => 3600
+        max_age => 7200
+        key => "%{message}"
+        add_tag => "throttled"
+      }
+    }
+    output {
+      if "throttled" not in [tags] {
+        email {
+          from => "logstash@mycompany.com"
+          subject => "Production System Alert"
+          to => "ops@mycompany.com"
+          via => "sendmail"
+          body => "Alert on %{host} from path %{path}:\n\n%{message}"
+          options => { "location" => "/usr/sbin/sendmail" }
+        }
+      }
+      elasticsearch_http {
+        host => "localhost"
+        port => "19200"
+      }
+    }
+
+When an event is received, the event key is stored in a key_cache.  The key references
+a timeslot_cache.  The event is allocated to a timeslot (created dynamically) based on
+the timestamp of the event.  The timeslot counter is incremented.  When the next event is
+received (same key), within the same "period", it is allocated to the same timeslot.
+The timeslot counter is incremented once again.
+
+The timeslot expires if the maximum age has been exceeded.  The age is calculated
+based on the latest event timestamp and the max_age configuration option.
+
+        ---[::.. DESIGN ..::]---
+
++- [key_cache] -+  +-- [timeslot_cache] --+
+|               |  | @created: 1439839636 |
+                   | @latest:  1439839836 |
+   [a.b.c]  =>     +----------------------+
+                   | [1439839636] => 1    |
+                   | [1439839736] => 3    |
+                   | [1439839836] => 2    |
+                   +----------------------+
+
+                   +-- [timeslot_cache] --+
+                   | @created: eeeeeeeeee |
+                   | @latest:  llllllllll |
+   [x.y.z]  =>     +----------------------+
+                   | [0000000060] => x    |
+                   | [0000000120] => y    |
+|               |  | [..........] => N    |
++---------------+  +----------------------+
+
+Frank de Jong (@frapex)
+Mike Pilone (@mikepilone)
+
+only update if greater than current
+
+[id="plugins-{type}s-{plugin}-options"]
+==== Throttle Filter Configuration Options
+
+This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
+
+[cols="<,<,<",options="header",]
+|=======================================================================
+|Setting |Input type|Required
+| <<plugins-{type}s-{plugin}-after_count>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-before_count>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-key>> |<<string,string>>|Yes
+| <<plugins-{type}s-{plugin}-max_age>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-max_counters>> |<<number,number>>|No
+| <<plugins-{type}s-{plugin}-period>> |<<string,string>>|No
+|=======================================================================
+
+Also see <<plugins-{type}s-{plugin}-common-options>> for a list of options supported by all
+filter plugins.
+
+&nbsp;
+
+[id="plugins-{type}s-{plugin}-after_count"]
+===== `after_count` 
+
+  * Value type is <<number,number>>
+  * Default value is `-1`
+
+Events greater than this count will be throttled.  Setting this value to -1, the
+default, will cause no events to be throttled based on the upper bound.
+
+[id="plugins-{type}s-{plugin}-before_count"]
+===== `before_count` 
+
+  * Value type is <<number,number>>
+  * Default value is `-1`
+
+Events less than this count will be throttled.  Setting this value to -1, the
+default, will cause no events to be throttled based on the lower bound.
+
+[id="plugins-{type}s-{plugin}-key"]
+===== `key` 
+
+  * This is a required setting.
+  * Value type is <<string,string>>
+  * There is no default value for this setting.
+
+The key used to identify events.  Events with the same key are grouped together.
+Field substitutions are allowed, so you can combine multiple fields.
+
+[id="plugins-{type}s-{plugin}-max_age"]
+===== `max_age` 
+
+  * Value type is <<number,number>>
+  * Default value is `3600`
+
+The maximum age of a timeslot.  Higher values allow better tracking of an asynchronous
+flow of events, but require more memory.  As a rule of thumb you should set this value
+to at least twice the period.  Or set this value to period + maximum time offset
+between unordered events with the same key.  Values below the specified period give
+unexpected results if unordered events are processed simultaneously.
+
+[id="plugins-{type}s-{plugin}-max_counters"]
+===== `max_counters` 
+
+  * Value type is <<number,number>>
+  * Default value is `100000`
+
+The maximum number of counters to store before decreasing the maximum age of a timeslot.
+Setting this value to -1 will prevent an upper bound with no constraint on the
+number of counters.  This configuration value should only be used as a memory
+control mechanism and can cause early counter expiration if the value is reached.
+It is recommended to leave the default value and ensure that your key is selected
+such that it limits the number of counters required (i.e. don't use UUID as the key).
+
+[id="plugins-{type}s-{plugin}-period"]
+===== `period` 
+
+  * Value type is <<string,string>>
+  * Default value is `"60"`
+
+The period in seconds after the first occurrence of an event until a new timeslot
+is created.  This period is tracked per unique key and per timeslot.
+Field substitutions are allowed in this value.  This allows you to specify that
+certain kinds of events throttle for a specific period of time.
+
+
+
+[id="plugins-{type}s-{plugin}-common-options"]
+include::{include_path}/{type}.asciidoc[]

data/logstash-filter-throttle.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name            = 'logstash-filter-throttle'
-  s.version         = '4.0.1'
+  s.version         = '4.0.2'
   s.licenses        = ['Apache License (2.0)']
   s.summary         = "The throttle filter is for throttling the number of events received."
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -11,7 +11,7 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   # Files
-  s.files = Dir['lib/**/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE','NOTICE.TXT']
+  s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
 
   # Tests
   s.test_files = s.files.grep(%r{^(test|spec|features)/})

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-throttle
 version: !ruby/object:Gem::Version
-  version: 4.0.1
+  version: 4.0.2
 platform: ruby
 authors:
 - Elastic
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-09-17 00:00:00.000000000 Z
+date: 2017-06-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -84,6 +84,7 @@ files:
 - LICENSE
 - NOTICE.TXT
 - README.md
+- docs/index.asciidoc
 - lib/logstash/filters/throttle.rb
 - logstash-filter-throttle.gemspec
 - spec/filters/throttle_spec.rb