logstash-filter-throttle 3.0.2 → 4.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/CONTRIBUTORS +1 -0
- data/lib/logstash/filters/throttle.rb +196 -150
- data/logstash-filter-throttle.gemspec +3 -2
- data/spec/filters/throttle_spec.rb +56 -9
- metadata +31 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 29516e66e87030588ab5f56191b0ac346efbaaa7
|
4
|
+
data.tar.gz: 74a3f0a23b80d19244eca960b00bd5f80898b253
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a992934db1063a273159ff44ab7f00bba78f51a8c144b508c5d84c287ca10f53c1a4fb77143dffee7c018737bea7e2865bb414f050ab64ee879cf1032434c218
|
7
|
+
data.tar.gz: 035919b7fd35b9274c1e099946f4f36299a671d5cbc76559ec5efebd5aa4e225fbc6c0079c61b9dae855ecb3ed3c58d2775a84f2a05cce059b2b6d8832b78921
|
data/CHANGELOG.md
CHANGED
data/CONTRIBUTORS
CHANGED
@@ -6,6 +6,7 @@ Contributors:
|
|
6
6
|
* Pier-Hugues Pellerin (ph)
|
7
7
|
* Richard Pijnenburg (electrical)
|
8
8
|
* Suyog Rao (suyograo)
|
9
|
+
* Frank de Jong (frapex)
|
9
10
|
|
10
11
|
Note: If you've sent us patches, bug reports, or otherwise contributed to
|
11
12
|
Logstash, and you aren't on the list above and want to be, please let us know
|
@@ -1,18 +1,22 @@
|
|
1
1
|
require "logstash/filters/base"
|
2
2
|
require "logstash/namespace"
|
3
|
+
require "thread_safe"
|
4
|
+
require "atomic"
|
3
5
|
|
4
|
-
# The throttle filter is for throttling the number of events
|
5
|
-
#
|
6
|
-
# and a period of time.
|
7
|
-
#
|
8
|
-
#
|
9
|
-
# successful and any tags or fields will be added.
|
6
|
+
# The throttle filter is for throttling the number of events. The filter is
|
7
|
+
# configured with a lower bound, the "before_count", and upper bound, the "after_count",
|
8
|
+
# and a period of time. All events passing through the filter will be counted based on
|
9
|
+
# their key and the event timestamp. As long as the count is less than the "before_count"
|
10
|
+
# or greater than the "after_count", the event will be "throttled" which means the filter
|
11
|
+
# will be considered successful and any tags or fields will be added (or removed).
|
10
12
|
#
|
11
|
-
#
|
12
|
-
#
|
13
|
-
#
|
13
|
+
# The plugin is thread-safe and properly tracks past events.
|
14
|
+
#
|
15
|
+
# For example, if you wanted to throttle events so you only receive an event after 2
|
16
|
+
# occurrences and you get no more than 3 in 10 minutes, you would use the configuration:
|
14
17
|
# [source,ruby]
|
15
18
|
# period => 600
|
19
|
+
# max_age => 1200
|
16
20
|
# before_count => 3
|
17
21
|
# after_count => 5
|
18
22
|
#
|
@@ -35,10 +39,11 @@ require "logstash/namespace"
|
|
35
39
|
# event 6 - throttled (successful filter)
|
36
40
|
# ...
|
37
41
|
# ==========================
|
38
|
-
# Another example is if you wanted to throttle events so you only
|
39
|
-
# hour, you would use the configuration:
|
42
|
+
# Another example is if you wanted to throttle events so you only
|
43
|
+
# receive 1 event per hour, you would use the configuration:
|
40
44
|
# [source,ruby]
|
41
45
|
# period => 3600
|
46
|
+
# max_age => 7200
|
42
47
|
# before_count => -1
|
43
48
|
# after_count => 1
|
44
49
|
#
|
@@ -56,8 +61,8 @@ require "logstash/namespace"
|
|
56
61
|
# event 4 - throttled (successful filter)
|
57
62
|
# ...
|
58
63
|
# ==========================
|
59
|
-
# A common use case would be to use the throttle filter to throttle events before 3 and
|
60
|
-
# after 5 while using multiple fields for the key and then use the drop filter to remove
|
64
|
+
# A common use case would be to use the throttle filter to throttle events before 3 and
|
65
|
+
# after 5 while using multiple fields for the key and then use the drop filter to remove
|
61
66
|
# throttled events. This configuration might appear as:
|
62
67
|
# [source,ruby]
|
63
68
|
# filter {
|
@@ -65,6 +70,7 @@ require "logstash/namespace"
|
|
65
70
|
# before_count => 3
|
66
71
|
# after_count => 5
|
67
72
|
# period => 3600
|
73
|
+
# max_age => 7200
|
68
74
|
# key => "%{host}%{message}"
|
69
75
|
# add_tag => "throttled"
|
70
76
|
# }
|
@@ -73,8 +79,8 @@ require "logstash/namespace"
|
|
73
79
|
# }
|
74
80
|
# }
|
75
81
|
#
|
76
|
-
# Another case would be to store all events, but only email non-throttled
|
77
|
-
#
|
82
|
+
# Another case would be to store all events, but only email non-throttled events
|
83
|
+
# so the op's inbox isn't flooded with emails in the event of a system error.
|
78
84
|
# This configuration might appear as:
|
79
85
|
# [source,ruby]
|
80
86
|
# filter {
|
@@ -82,6 +88,7 @@ require "logstash/namespace"
|
|
82
88
|
# before_count => 3
|
83
89
|
# after_count => 5
|
84
90
|
# period => 3600
|
91
|
+
# max_age => 7200
|
85
92
|
# key => "%{message}"
|
86
93
|
# add_tag => "throttled"
|
87
94
|
# }
|
@@ -89,12 +96,12 @@ require "logstash/namespace"
|
|
89
96
|
# output {
|
90
97
|
# if "throttled" not in [tags] {
|
91
98
|
# email {
|
92
|
-
#
|
93
|
-
#
|
94
|
-
#
|
95
|
-
#
|
96
|
-
#
|
97
|
-
#
|
99
|
+
# from => "logstash@mycompany.com"
|
100
|
+
# subject => "Production System Alert"
|
101
|
+
# to => "ops@mycompany.com"
|
102
|
+
# via => "sendmail"
|
103
|
+
# body => "Alert on %{host} from path %{path}:\n\n%{message}"
|
104
|
+
# options => { "location" => "/usr/sbin/sendmail" }
|
98
105
|
# }
|
99
106
|
# }
|
100
107
|
# elasticsearch_http {
|
@@ -103,161 +110,200 @@ require "logstash/namespace"
|
|
103
110
|
# }
|
104
111
|
# }
|
105
112
|
#
|
106
|
-
#
|
107
|
-
#
|
108
|
-
#
|
113
|
+
# When an event is received, the event key is stored in a key_cache. The key references
|
114
|
+
# a timeslot_cache. The event is allocated to a timeslot (created dynamically) based on
|
115
|
+
# the timestamp of the event. The timeslot counter is incremented. When the next event is
|
116
|
+
# received (same key), within the same "period", it is allocated to the same timeslot.
|
117
|
+
# The timeslot counter is incremented once again.
|
118
|
+
#
|
119
|
+
# The timeslot expires if the maximum age has been exceeded. The age is calculated
|
120
|
+
# based on the latest event timestamp and the max_age configuration option.
|
121
|
+
#
|
122
|
+
# ---[::.. DESIGN ..::]---
|
109
123
|
#
|
124
|
+
# +- [key_cache] -+ +-- [timeslot_cache] --+
|
125
|
+
# | | | @created: 1439839636 |
|
126
|
+
# | @latest: 1439839836 |
|
127
|
+
# [a.b.c] => +----------------------+
|
128
|
+
# | [1439839636] => 1 |
|
129
|
+
# | [1439839736] => 3 |
|
130
|
+
# | [1439839836] => 2 |
|
131
|
+
# +----------------------+
|
132
|
+
#
|
133
|
+
# +-- [timeslot_cache] --+
|
134
|
+
# | @created: eeeeeeeeee |
|
135
|
+
# | @latest: llllllllll |
|
136
|
+
# [x.y.z] => +----------------------+
|
137
|
+
# | [0000000060] => x |
|
138
|
+
# | [0000000120] => y |
|
139
|
+
# | | | [..........] => N |
|
140
|
+
# +---------------+ +----------------------+
|
141
|
+
#
|
142
|
+
# Frank de Jong (@frapex)
|
110
143
|
# Mike Pilone (@mikepilone)
|
111
|
-
#
|
112
|
-
|
144
|
+
#
|
145
|
+
|
146
|
+
class ThreadSafe::TimeslotCache < ThreadSafe::Cache
|
147
|
+
attr_reader :created
|
148
|
+
|
149
|
+
def initialize(epoch, options = nil, &block)
|
150
|
+
@created = epoch
|
151
|
+
@latest = Atomic.new(epoch)
|
152
|
+
|
153
|
+
super(options, &block)
|
154
|
+
end
|
113
155
|
|
156
|
+
def latest
|
157
|
+
@latest.value
|
158
|
+
end
|
159
|
+
|
160
|
+
def latest=(val)
|
161
|
+
# only update if greater than current
|
162
|
+
@latest.update { |v| v = (val > v) ? val : v }
|
163
|
+
end
|
164
|
+
end
|
165
|
+
|
166
|
+
class LogStash::Filters::Throttle < LogStash::Filters::Base
|
114
167
|
# The name to use in configuration files.
|
115
168
|
config_name "throttle"
|
116
169
|
|
170
|
+
# The memory control mechanism automatically ajusts the maximum age
|
171
|
+
# of a timeslot based on the maximum number of counters.
|
172
|
+
MC_MIN_PCT = 5 # Lower bound percentage.
|
173
|
+
MC_MAX_PCT = 100 # Upper bound percentage.
|
174
|
+
MC_INCR_PCT = 80 # Increase if total below percentage.
|
175
|
+
MC_STEP_PCT = 5 # Increase/decrease by this percentage at a time.
|
176
|
+
|
177
|
+
# Call the filter flush method at regular interval. It is used by the memory
|
178
|
+
# control mechanism. Set to false if you like your VM to go (B)OOM.
|
179
|
+
config :periodic_flush, :validate => :boolean, :default => true
|
117
180
|
|
118
|
-
# The key used to identify events.
|
119
|
-
#
|
120
|
-
# fields.
|
181
|
+
# The key used to identify events. Events with the same key are grouped together.
|
182
|
+
# Field substitutions are allowed, so you can combine multiple fields.
|
121
183
|
config :key, :validate => :string, :required => true
|
122
|
-
|
123
|
-
# Events less than this count will be throttled.
|
124
|
-
# default, will cause no
|
184
|
+
|
185
|
+
# Events less than this count will be throttled. Setting this value to -1, the
|
186
|
+
# default, will cause no events to be throttled based on the lower bound.
|
125
187
|
config :before_count, :validate => :number, :default => -1, :required => false
|
126
|
-
|
127
|
-
# Events greater than this count will be throttled.
|
128
|
-
# default, will cause no
|
188
|
+
|
189
|
+
# Events greater than this count will be throttled. Setting this value to -1, the
|
190
|
+
# default, will cause no events to be throttled based on the upper bound.
|
129
191
|
config :after_count, :validate => :number, :default => -1, :required => false
|
130
|
-
|
131
|
-
# The period in seconds after the first occurrence of an event until
|
132
|
-
#
|
133
|
-
# substitutions are allowed in this value.
|
134
|
-
#
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
#
|
139
|
-
#
|
140
|
-
#
|
141
|
-
#
|
142
|
-
|
143
|
-
|
144
|
-
#
|
192
|
+
|
193
|
+
# The period in seconds after the first occurrence of an event until a new timeslot
|
194
|
+
# is created. This period is tracked per unique key and per timeslot.
|
195
|
+
# Field substitutions are allowed in this value. This allows you to specify that
|
196
|
+
# certain kinds of events throttle for a specific period of time.
|
197
|
+
config :period, :validate => :string, :default => "60", :required => false
|
198
|
+
|
199
|
+
# The maximum age of a timeslot. Higher values allow better tracking of an asynchronous
|
200
|
+
# flow of events, but require more memory. As a rule of thumb you should set this value
|
201
|
+
# to at least twice the period. Or set this value to period + maximum time offset
|
202
|
+
# between unordered events with the same key. Values below the specified period give
|
203
|
+
# unexpected results if unordered events are processed simultaneously.
|
204
|
+
config :max_age, :validate => :number, :default => 3600, :required => false
|
205
|
+
|
206
|
+
# The maximum number of counters to store before decreasing the maximum age of a timeslot.
|
207
|
+
# Setting this value to -1 will prevent an upper bound with no constraint on the
|
208
|
+
# number of counters. This configuration value should only be used as a memory
|
209
|
+
# control mechanism and can cause early counter expiration if the value is reached.
|
210
|
+
# It is recommended to leave the default value and ensure that your key is selected
|
211
|
+
# such that it limits the number of counters required (i.e. don't use UUID as the key).
|
145
212
|
config :max_counters, :validate => :number, :default => 100000, :required => false
|
146
213
|
|
147
|
-
#
|
214
|
+
# performs initialization of the filter
|
148
215
|
public
|
149
216
|
def register
|
150
|
-
@
|
151
|
-
|
152
|
-
@event_counters = Hash.new
|
153
|
-
@next_expiration = nil
|
217
|
+
@key_cache = ThreadSafe::Cache.new
|
218
|
+
@max_age_orig = @max_age
|
154
219
|
end # def register
|
155
220
|
|
156
|
-
#
|
221
|
+
# filters the event
|
157
222
|
public
|
158
223
|
def filter(event)
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
# Purge counters if too large to prevent OOM.
|
167
|
-
if @max_counters != -1 && @event_counters.size > @max_counters then
|
168
|
-
purgeOldestEventCounter()
|
169
|
-
end
|
170
|
-
|
171
|
-
# Expire existing counter if needed
|
172
|
-
if @next_expiration.nil? || now >= @next_expiration then
|
173
|
-
expireEventCounters(now)
|
224
|
+
key = event.sprintf(@key) # substitute field
|
225
|
+
period = event.sprintf(@period).to_i # substitute period
|
226
|
+
period = 60 if period == 0 # fallback if unparsable
|
227
|
+
epoch = event.timestamp.to_i # event epoch time
|
228
|
+
|
229
|
+
@key_cache.compute_if_absent(key) do # initialise timeslot cache
|
230
|
+
ThreadSafe::TimeslotCache.new(epoch) # and add to key cache
|
174
231
|
end
|
175
|
-
|
232
|
+
|
233
|
+
timeslot_cache = @key_cache[key] # get timeslot cache
|
234
|
+
timeslot_cache.latest = epoch # update to latest epoch
|
235
|
+
|
236
|
+
# find target timeslot
|
237
|
+
timeslot_key = epoch - (epoch - timeslot_cache.created) % period
|
238
|
+
|
239
|
+
# initialise timeslot and counter (if required)
|
240
|
+
timeslot_cache.compute_if_absent(timeslot_key) { Atomic.new(0) }
|
241
|
+
|
242
|
+
timeslot = timeslot_cache[timeslot_key] # get timeslot
|
243
|
+
timeslot.update { |v| v + 1 } # increment counter
|
244
|
+
count = timeslot.value # get latest counter value
|
245
|
+
|
176
246
|
@logger.debug? and @logger.debug(
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
counter
|
182
|
-
if
|
183
|
-
|
184
|
-
period = 3600 if period == 0
|
185
|
-
expiration = now + period
|
186
|
-
@event_counters[key] = { :count => 0, :expiration => expiration }
|
187
|
-
|
188
|
-
@logger.debug? and @logger.debug("filters/#{self.class.name}: new event",
|
189
|
-
{ :key => key, :expiration => expiration })
|
190
|
-
end
|
191
|
-
|
192
|
-
# Fetch the counter
|
193
|
-
counter = @event_counters[key]
|
194
|
-
|
195
|
-
# Count this event
|
196
|
-
counter[:count] = counter[:count] + 1;
|
197
|
-
|
198
|
-
@logger.debug? and @logger.debug("filters/#{self.class.name}: current count",
|
199
|
-
{ :key => key, :count => counter[:count] })
|
200
|
-
|
201
|
-
# Throttle if count is < before count or > after count
|
202
|
-
if ((@before_count != -1 && counter[:count] < @before_count) ||
|
203
|
-
(@after_count != -1 && counter[:count] > @after_count)) then
|
247
|
+
"filters/#{self.class.name}: counter incremented",
|
248
|
+
{ key: key, epoch: epoch, timeslot: timeslot_key, count: count }
|
249
|
+
)
|
250
|
+
|
251
|
+
# throttle event if counter value not in range
|
252
|
+
if ((@before_count != -1 && count < @before_count) ||
|
253
|
+
(@after_count != -1 && count > @after_count))
|
204
254
|
@logger.debug? and @logger.debug(
|
205
|
-
|
206
|
-
|
255
|
+
"filters/#{self.class.name}: throttling event",
|
256
|
+
{ key: key, epoch: epoch }
|
257
|
+
)
|
258
|
+
|
207
259
|
filter_matched(event)
|
208
260
|
end
|
209
|
-
|
261
|
+
|
262
|
+
# Delete expired timeslots older than the latest. Do not use variable
|
263
|
+
# timeslot_cache.latest for this. If used, it might delete the latest timeslot.
|
264
|
+
latest_timeslot = timeslot_cache.keys.max || 0
|
265
|
+
timeslot_cache.each_key { |key| timeslot_cache.delete(key) if key < (latest_timeslot - @max_age) }
|
210
266
|
end # def filter
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
if expired then
|
224
|
-
@logger.debug? and @logger.debug(
|
225
|
-
"filters/#{self.class.name}: deleting expired counter",
|
226
|
-
{ :key => key })
|
227
|
-
|
228
|
-
elsif @next_expiration.nil? || (expiration < @next_expiration)
|
229
|
-
@next_expiration = expiration
|
267
|
+
|
268
|
+
public
|
269
|
+
def flush(options = {})
|
270
|
+
max_latest = 0 # get maximum epoch
|
271
|
+
@key_cache.each_value { |tc| max_latest = tc.latest if tc.latest > max_latest }
|
272
|
+
|
273
|
+
total_counters = 0
|
274
|
+
@key_cache.each_pair do |key,timeslot_cache|
|
275
|
+
if timeslot_cache.latest < max_latest - @max_age
|
276
|
+
@key_cache.delete(key) # delete expired timeslot cache
|
277
|
+
else
|
278
|
+
total_counters += timeslot_cache.size # get total number of counters
|
230
279
|
end
|
231
|
-
|
232
|
-
expired
|
233
280
|
end
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
281
|
+
|
282
|
+
@logger.debug? and @logger.debug(
|
283
|
+
"filters/#{self.class.name}: statistics",
|
284
|
+
{ total_counters: total_counters, max_age: @max_age }
|
285
|
+
)
|
286
|
+
|
287
|
+
# memory control mechanism
|
288
|
+
if @max_counters != -1
|
289
|
+
over_limit = total_counters - @max_counters
|
290
|
+
|
291
|
+
# decrease max age of timeslot cache by x percent
|
292
|
+
if (over_limit > 0) && (@max_age > @max_age_orig * MC_MIN_PCT / 100)
|
293
|
+
@max_age -= @max_age_orig * MC_STEP_PCT / 100
|
294
|
+
@logger.warn? and @logger.warn(
|
295
|
+
"filters/#{self.class.name}: Decreased timeslot max_age to #{@max_age} because " +
|
296
|
+
"max_counters exceeded by #{over_limit}. Use a better key to prevent too many unique event counters.")
|
297
|
+
|
298
|
+
# increase max age of timeslot cache by x percent
|
299
|
+
elsif (@max_age < @max_age_orig * MC_MAX_PCT / 100) && (total_counters < (@max_counters * MC_INCR_PCT / 100))
|
300
|
+
@max_age += @max_age_orig * MC_STEP_PCT / 100
|
301
|
+
@logger.warn? and @logger.warn(
|
302
|
+
"filters/#{self.class.name}: Increased timeslot max_age to #{@max_age} because max_counters no longer exceeded.")
|
252
303
|
end
|
253
304
|
end
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
{ :key => oldestKey, :expiration => oldestCounter[:expiration] })
|
259
|
-
|
260
|
-
@event_counters.delete(oldestKey)
|
261
|
-
|
262
|
-
end
|
305
|
+
|
306
|
+
return
|
307
|
+
end # def flush
|
308
|
+
|
263
309
|
end # class LogStash::Filters::Throttle
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-filter-throttle'
|
4
|
-
s.version = '
|
4
|
+
s.version = '4.0.0'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "The throttle filter is for throttling the number of events received."
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
@@ -21,7 +21,8 @@ Gem::Specification.new do |s|
|
|
21
21
|
|
22
22
|
# Gem dependencies
|
23
23
|
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
24
|
+
s.add_runtime_dependency "thread_safe"
|
25
|
+
s.add_runtime_dependency "atomic"
|
24
26
|
|
25
27
|
s.add_development_dependency 'logstash-devutils'
|
26
28
|
end
|
27
|
-
|
@@ -163,14 +163,13 @@ describe LogStash::Filters::Throttle do
|
|
163
163
|
}
|
164
164
|
end
|
165
165
|
end
|
166
|
-
|
167
|
-
describe "
|
166
|
+
|
167
|
+
describe "correct timeslot assigned/calculated, after_count exceeded" do
|
168
168
|
config <<-CONFIG
|
169
169
|
filter {
|
170
170
|
throttle {
|
171
171
|
period => 60
|
172
172
|
after_count => 1
|
173
|
-
max_counters => 2
|
174
173
|
key => "%{message}"
|
175
174
|
add_tag => [ "throttled" ]
|
176
175
|
}
|
@@ -178,19 +177,67 @@ describe LogStash::Filters::Throttle do
|
|
178
177
|
CONFIG
|
179
178
|
|
180
179
|
events = [{
|
181
|
-
"
|
180
|
+
"@timestamp" => "2016-07-09T00:05:00.000Z",
|
181
|
+
"message" => "server1"
|
182
182
|
}, {
|
183
|
-
"
|
183
|
+
"@timestamp" => "2016-07-09T00:05:59.000Z",
|
184
|
+
"message" => "server1"
|
184
185
|
}, {
|
185
|
-
"
|
186
|
+
"@timestamp" => "2016-07-09T00:10:33.000Z",
|
187
|
+
"message" => "server1"
|
186
188
|
}, {
|
187
|
-
"
|
189
|
+
"@timestamp" => "2016-07-09T00:10:34.000Z",
|
190
|
+
"message" => "server1"
|
191
|
+
}, {
|
192
|
+
"@timestamp" => "2016-07-09T00:00:00.000Z",
|
193
|
+
"message" => "server1"
|
194
|
+
}, {
|
195
|
+
"@timestamp" => "2016-07-09T00:00:45.000Z",
|
196
|
+
"message" => "server1"
|
188
197
|
}]
|
189
198
|
|
190
199
|
sample events do
|
191
|
-
insist { subject[
|
200
|
+
insist { subject[0].get("tags") } == nil
|
201
|
+
insist { subject[1].get("tags") } == [ "throttled" ]
|
202
|
+
insist { subject[2].get("tags") } == nil
|
203
|
+
insist { subject[3].get("tags") } == [ "throttled" ]
|
204
|
+
insist { subject[4].get("tags") } == nil
|
205
|
+
insist { subject[5].get("tags") } == [ "throttled" ]
|
192
206
|
end
|
193
207
|
end
|
194
208
|
|
195
|
-
|
209
|
+
describe "asynchronous input, after_count exceeded" do
|
210
|
+
config <<-CONFIG
|
211
|
+
filter {
|
212
|
+
throttle {
|
213
|
+
period => 60
|
214
|
+
after_count => 1
|
215
|
+
key => "%{message}"
|
216
|
+
add_tag => [ "throttled" ]
|
217
|
+
}
|
218
|
+
}
|
219
|
+
CONFIG
|
220
|
+
|
221
|
+
events = [{
|
222
|
+
"@timestamp" => "2016-07-09T00:01:00.000Z",
|
223
|
+
"message" => "server1"
|
224
|
+
}, {
|
225
|
+
"@timestamp" => "2016-07-09T00:00:30.000Z",
|
226
|
+
"message" => "server1"
|
227
|
+
}, {
|
228
|
+
"@timestamp" => "2016-07-09T00:01:59.000Z",
|
229
|
+
"message" => "server1"
|
230
|
+
}, {
|
231
|
+
"@timestamp" => "2016-07-09T00:00:59.000Z",
|
232
|
+
"message" => "server1"
|
233
|
+
}]
|
234
|
+
|
235
|
+
sample events do
|
236
|
+
insist { subject[0].get("tags") } == nil
|
237
|
+
insist { subject[1].get("tags") } == nil
|
238
|
+
insist { subject[2].get("tags") } == [ "throttled" ]
|
239
|
+
insist { subject[3].get("tags") } == [ "throttled" ]
|
240
|
+
end
|
241
|
+
end
|
196
242
|
|
243
|
+
end # LogStash::Filters::Throttle
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-filter-throttle
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 4.0.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-07-
|
11
|
+
date: 2016-07-31 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -30,6 +30,34 @@ dependencies:
|
|
30
30
|
- - "<="
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: '2.99'
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
requirement: !ruby/object:Gem::Requirement
|
35
|
+
requirements:
|
36
|
+
- - ">="
|
37
|
+
- !ruby/object:Gem::Version
|
38
|
+
version: '0'
|
39
|
+
name: thread_safe
|
40
|
+
prerelease: false
|
41
|
+
type: :runtime
|
42
|
+
version_requirements: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - ">="
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: '0'
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
requirements:
|
50
|
+
- - ">="
|
51
|
+
- !ruby/object:Gem::Version
|
52
|
+
version: '0'
|
53
|
+
name: atomic
|
54
|
+
prerelease: false
|
55
|
+
type: :runtime
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
requirements:
|
58
|
+
- - ">="
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: '0'
|
33
61
|
- !ruby/object:Gem::Dependency
|
34
62
|
requirement: !ruby/object:Gem::Requirement
|
35
63
|
requirements:
|
@@ -81,7 +109,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
81
109
|
version: '0'
|
82
110
|
requirements: []
|
83
111
|
rubyforge_project:
|
84
|
-
rubygems_version: 2.
|
112
|
+
rubygems_version: 2.4.8
|
85
113
|
signing_key:
|
86
114
|
specification_version: 4
|
87
115
|
summary: The throttle filter is for throttling the number of events received.
|