logstash-filter-aggregate 2.0.2 → 2.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +7 -1
- data/README.md +2 -0
- data/lib/logstash/filters/aggregate.rb +190 -192
- data/logstash-filter-aggregate.gemspec +2 -2
- data/spec/filters/aggregate_spec.rb +176 -168
- metadata +20 -20
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 6500c98a915954331abf2a5e589c58497d735a4c
|
|
4
|
+
data.tar.gz: a2936e86e9e33cb497a9d86b22356d1a9c5299ac
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 38d0789ad10b94338d1ece65b84e4d7dbf60ab320da55ef1921426509ffd4f4516c2457ccad2723a5c98b19fc5e2490e3676dba80496970d03dbec23312a6d3e
|
|
7
|
+
data.tar.gz: b19dbfc2e2af9202dd6b992fbbf925fd8c8280ef404e8c054e2291f2ffac80b4550c91f7c8502f9c158d0ef6774a6d515dd294f0c7426215795b586a09e1b721
|
data/CHANGELOG.md
CHANGED
|
@@ -1,4 +1,10 @@
|
|
|
1
|
-
|
|
1
|
+
# v 2.0.3
|
|
2
|
+
- fix issue #10 : numeric task_id is now well processed
|
|
3
|
+
|
|
4
|
+
# v 2.0.2
|
|
5
|
+
- fix issue #5 : when code call raises an exception, the error is logged and the event is tagged '_aggregateexception'. It avoids logstash crash.
|
|
6
|
+
|
|
7
|
+
# v 2.0.0
|
|
2
8
|
- Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
|
|
3
9
|
instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
|
|
4
10
|
- Dependency on logstash-core update to 2.0
|
data/README.md
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# Logstash Filter Aggregate Documentation
|
|
2
2
|
|
|
3
|
+
[](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-aggregate-unit/)
|
|
4
|
+
|
|
3
5
|
The aim of this filter is to aggregate information available among several events (typically log lines) belonging to a same task, and finally push aggregated information into final task event.
|
|
4
6
|
|
|
5
7
|
## Example #1
|
|
@@ -4,102 +4,102 @@ require "logstash/filters/base"
|
|
|
4
4
|
require "logstash/namespace"
|
|
5
5
|
require "thread"
|
|
6
6
|
|
|
7
|
-
#
|
|
7
|
+
#
|
|
8
8
|
# The aim of this filter is to aggregate information available among several events (typically log lines) belonging to a same task,
|
|
9
9
|
# and finally push aggregated information into final task event.
|
|
10
10
|
#
|
|
11
11
|
# ==== Example #1
|
|
12
|
-
#
|
|
12
|
+
#
|
|
13
13
|
# * with these given logs :
|
|
14
14
|
# [source,ruby]
|
|
15
15
|
# ----------------------------------
|
|
16
|
-
#
|
|
17
|
-
#
|
|
18
|
-
#
|
|
19
|
-
#
|
|
16
|
+
# INFO - 12345 - TASK_START - start
|
|
17
|
+
# INFO - 12345 - SQL - sqlQuery1 - 12
|
|
18
|
+
# INFO - 12345 - SQL - sqlQuery2 - 34
|
|
19
|
+
# INFO - 12345 - TASK_END - end
|
|
20
20
|
# ----------------------------------
|
|
21
|
-
#
|
|
21
|
+
#
|
|
22
22
|
# * you can aggregate "sql duration" for the whole task with this configuration :
|
|
23
23
|
# [source,ruby]
|
|
24
24
|
# ----------------------------------
|
|
25
|
-
#
|
|
26
|
-
#
|
|
27
|
-
#
|
|
28
|
-
#
|
|
29
|
-
#
|
|
30
|
-
#
|
|
31
|
-
#
|
|
32
|
-
#
|
|
33
|
-
#
|
|
34
|
-
#
|
|
35
|
-
#
|
|
36
|
-
#
|
|
37
|
-
#
|
|
38
|
-
#
|
|
39
|
-
#
|
|
40
|
-
#
|
|
41
|
-
#
|
|
42
|
-
#
|
|
43
|
-
#
|
|
44
|
-
#
|
|
45
|
-
#
|
|
46
|
-
#
|
|
47
|
-
#
|
|
48
|
-
#
|
|
49
|
-
#
|
|
50
|
-
#
|
|
51
|
-
#
|
|
52
|
-
#
|
|
53
|
-
#
|
|
54
|
-
#
|
|
55
|
-
#
|
|
25
|
+
# filter {
|
|
26
|
+
# grok {
|
|
27
|
+
# match => [ "message", "%{LOGLEVEL:loglevel} - %{NOTSPACE:taskid} - %{NOTSPACE:logger} - %{WORD:label}( - %{INT:duration:int})?" ]
|
|
28
|
+
# }
|
|
29
|
+
#
|
|
30
|
+
# if [logger] == "TASK_START" {
|
|
31
|
+
# aggregate {
|
|
32
|
+
# task_id => "%{taskid}"
|
|
33
|
+
# code => "map['sql_duration'] = 0"
|
|
34
|
+
# map_action => "create"
|
|
35
|
+
# }
|
|
36
|
+
# }
|
|
37
|
+
#
|
|
38
|
+
# if [logger] == "SQL" {
|
|
39
|
+
# aggregate {
|
|
40
|
+
# task_id => "%{taskid}"
|
|
41
|
+
# code => "map['sql_duration'] += event['duration']"
|
|
42
|
+
# map_action => "update"
|
|
43
|
+
# }
|
|
44
|
+
# }
|
|
45
|
+
#
|
|
46
|
+
# if [logger] == "TASK_END" {
|
|
47
|
+
# aggregate {
|
|
48
|
+
# task_id => "%{taskid}"
|
|
49
|
+
# code => "event['sql_duration'] = map['sql_duration']"
|
|
50
|
+
# map_action => "update"
|
|
51
|
+
# end_of_task => true
|
|
52
|
+
# timeout => 120
|
|
53
|
+
# }
|
|
54
|
+
# }
|
|
55
|
+
# }
|
|
56
56
|
# ----------------------------------
|
|
57
57
|
#
|
|
58
58
|
# * the final event then looks like :
|
|
59
59
|
# [source,ruby]
|
|
60
60
|
# ----------------------------------
|
|
61
61
|
# {
|
|
62
|
-
#
|
|
63
|
-
#
|
|
62
|
+
# "message" => "INFO - 12345 - TASK_END - end message",
|
|
63
|
+
# "sql_duration" => 46
|
|
64
64
|
# }
|
|
65
65
|
# ----------------------------------
|
|
66
|
-
#
|
|
66
|
+
#
|
|
67
67
|
# the field `sql_duration` is added and contains the sum of all sql queries durations.
|
|
68
|
-
#
|
|
68
|
+
#
|
|
69
69
|
# ==== Example #2
|
|
70
70
|
#
|
|
71
71
|
# * If you have the same logs than example #1, but without a start log :
|
|
72
72
|
# [source,ruby]
|
|
73
73
|
# ----------------------------------
|
|
74
|
-
#
|
|
75
|
-
#
|
|
76
|
-
#
|
|
74
|
+
# INFO - 12345 - SQL - sqlQuery1 - 12
|
|
75
|
+
# INFO - 12345 - SQL - sqlQuery2 - 34
|
|
76
|
+
# INFO - 12345 - TASK_END - end
|
|
77
77
|
# ----------------------------------
|
|
78
|
-
#
|
|
78
|
+
#
|
|
79
79
|
# * you can also aggregate "sql duration" with a slightly different configuration :
|
|
80
80
|
# [source,ruby]
|
|
81
81
|
# ----------------------------------
|
|
82
|
-
#
|
|
83
|
-
#
|
|
84
|
-
#
|
|
85
|
-
#
|
|
82
|
+
# filter {
|
|
83
|
+
# grok {
|
|
84
|
+
# match => [ "message", "%{LOGLEVEL:loglevel} - %{NOTSPACE:taskid} - %{NOTSPACE:logger} - %{WORD:label}( - %{INT:duration:int})?" ]
|
|
85
|
+
# }
|
|
86
86
|
#
|
|
87
|
-
#
|
|
88
|
-
#
|
|
89
|
-
#
|
|
90
|
-
#
|
|
91
|
-
#
|
|
92
|
-
#
|
|
87
|
+
# if [logger] == "SQL" {
|
|
88
|
+
# aggregate {
|
|
89
|
+
# task_id => "%{taskid}"
|
|
90
|
+
# code => "map['sql_duration'] ||= 0 ; map['sql_duration'] += event['duration']"
|
|
91
|
+
# }
|
|
92
|
+
# }
|
|
93
93
|
#
|
|
94
|
-
#
|
|
95
|
-
#
|
|
96
|
-
#
|
|
97
|
-
#
|
|
98
|
-
#
|
|
99
|
-
#
|
|
100
|
-
#
|
|
101
|
-
#
|
|
102
|
-
#
|
|
94
|
+
# if [logger] == "TASK_END" {
|
|
95
|
+
# aggregate {
|
|
96
|
+
# task_id => "%{taskid}"
|
|
97
|
+
# code => "event['sql_duration'] = map['sql_duration']"
|
|
98
|
+
# end_of_task => true
|
|
99
|
+
# timeout => 120
|
|
100
|
+
# }
|
|
101
|
+
# }
|
|
102
|
+
# }
|
|
103
103
|
# ----------------------------------
|
|
104
104
|
#
|
|
105
105
|
# * the final event is exactly the same than example #1
|
|
@@ -119,159 +119,157 @@ require "thread"
|
|
|
119
119
|
#
|
|
120
120
|
class LogStash::Filters::Aggregate < LogStash::Filters::Base
|
|
121
121
|
|
|
122
|
-
|
|
122
|
+
config_name "aggregate"
|
|
123
123
|
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
124
|
+
# The expression defining task ID to correlate logs.
|
|
125
|
+
#
|
|
126
|
+
# This value must uniquely identify the task in the system.
|
|
127
|
+
#
|
|
128
|
+
# Example value : "%{application}%{my_task_id}"
|
|
129
|
+
config :task_id, :validate => :string, :required => true
|
|
130
130
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
131
|
+
# The code to execute to update map, using current event.
|
|
132
|
+
#
|
|
133
|
+
# Or on the contrary, the code to execute to update event, using current map.
|
|
134
|
+
#
|
|
135
|
+
# You will have a 'map' variable and an 'event' variable available (that is the event itself).
|
|
136
|
+
#
|
|
137
|
+
# Example value : "map['sql_duration'] += event['duration']"
|
|
138
|
+
config :code, :validate => :string, :required => true
|
|
139
139
|
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
140
|
+
# Tell the filter what to do with aggregate map.
|
|
141
|
+
#
|
|
142
|
+
# `create`: create the map, and execute the code only if map wasn't created before
|
|
143
|
+
#
|
|
144
|
+
# `update`: doesn't create the map, and execute the code only if map was created before
|
|
145
|
+
#
|
|
146
|
+
# `create_or_update`: create the map if it wasn't created before, execute the code in all cases
|
|
147
|
+
config :map_action, :validate => :string, :default => "create_or_update"
|
|
148
148
|
|
|
149
|
-
|
|
150
|
-
|
|
149
|
+
# Tell the filter that task is ended, and therefore, to delete map after code execution.
|
|
150
|
+
config :end_of_task, :validate => :boolean, :default => false
|
|
151
151
|
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
152
|
+
# The amount of seconds after a task "end event" can be considered lost.
|
|
153
|
+
#
|
|
154
|
+
# The task "map" is evicted.
|
|
155
|
+
#
|
|
156
|
+
# Default value (`0`) means no timeout so no auto eviction.
|
|
157
|
+
config :timeout, :validate => :number, :required => false, :default => 0
|
|
158
158
|
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
159
|
+
|
|
160
|
+
# Default timeout (in seconds) when not defined in plugin configuration
|
|
161
|
+
DEFAULT_TIMEOUT = 1800
|
|
162
162
|
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
163
|
+
# This is the state of the filter.
|
|
164
|
+
# For each entry, key is "task_id" and value is a map freely updatable by 'code' config
|
|
165
|
+
@@aggregate_maps = {}
|
|
166
166
|
|
|
167
|
-
|
|
168
|
-
|
|
167
|
+
# Mutex used to synchronize access to 'aggregate_maps'
|
|
168
|
+
@@mutex = Mutex.new
|
|
169
169
|
|
|
170
|
-
|
|
171
|
-
|
|
170
|
+
# Aggregate instance which will evict all zombie Aggregate elements (older than timeout)
|
|
171
|
+
@@eviction_instance = nil
|
|
172
172
|
|
|
173
|
-
|
|
174
|
-
|
|
173
|
+
# last time where eviction was launched
|
|
174
|
+
@@last_eviction_timestamp = nil
|
|
175
175
|
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
176
|
+
# Initialize plugin
|
|
177
|
+
public
|
|
178
|
+
def register
|
|
179
|
+
# process lambda expression to call in each filter call
|
|
180
|
+
eval("@codeblock = lambda { |event, map| #{@code} }", binding, "(aggregate filter code)")
|
|
181
181
|
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
182
|
+
# define eviction_instance
|
|
183
|
+
@@mutex.synchronize do
|
|
184
|
+
if (@timeout > 0 && (@@eviction_instance.nil? || @timeout < @@eviction_instance.timeout))
|
|
185
|
+
@@eviction_instance = self
|
|
186
|
+
@logger.info("Aggregate, timeout: #{@timeout} seconds")
|
|
187
|
+
end
|
|
188
|
+
end
|
|
189
|
+
end
|
|
190
190
|
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
# return nothing unless there's an actual filter event
|
|
196
|
-
|
|
191
|
+
|
|
192
|
+
# This method is invoked each time an event matches the filter
|
|
193
|
+
public
|
|
194
|
+
def filter(event)
|
|
197
195
|
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
196
|
+
# define task id
|
|
197
|
+
task_id = event.sprintf(@task_id)
|
|
198
|
+
return if task_id.nil? || task_id == @task_id
|
|
201
199
|
|
|
202
|
-
|
|
200
|
+
noError = false
|
|
203
201
|
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
202
|
+
# protect aggregate_maps against concurrent access, using a mutex
|
|
203
|
+
@@mutex.synchronize do
|
|
204
|
+
|
|
205
|
+
# retrieve the current aggregate map
|
|
206
|
+
aggregate_maps_element = @@aggregate_maps[task_id]
|
|
207
|
+
if (aggregate_maps_element.nil?)
|
|
208
|
+
return if @map_action == "update"
|
|
209
|
+
aggregate_maps_element = LogStash::Filters::Aggregate::Element.new(Time.now);
|
|
210
|
+
@@aggregate_maps[task_id] = aggregate_maps_element
|
|
211
|
+
else
|
|
212
|
+
return if @map_action == "create"
|
|
213
|
+
end
|
|
214
|
+
map = aggregate_maps_element.map
|
|
217
215
|
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
216
|
+
# execute the code to read/update map and event
|
|
217
|
+
begin
|
|
218
|
+
@codeblock.call(event, map)
|
|
219
|
+
noError = true
|
|
220
|
+
rescue => exception
|
|
221
|
+
@logger.error("Aggregate exception occurred. Error: #{exception} ; Code: #{@code} ; Map: #{map} ; EventData: #{event.instance_variable_get('@data')}")
|
|
222
|
+
event.tag("_aggregateexception")
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
# delete the map if task is ended
|
|
226
|
+
@@aggregate_maps.delete(task_id) if @end_of_task
|
|
227
|
+
end
|
|
230
228
|
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
229
|
+
# match the filter, only if no error occurred
|
|
230
|
+
filter_matched(event) if noError
|
|
231
|
+
end
|
|
234
232
|
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
233
|
+
# Necessary to indicate logstash to periodically call 'flush' method
|
|
234
|
+
def periodic_flush
|
|
235
|
+
true
|
|
236
|
+
end
|
|
239
237
|
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
238
|
+
# This method is invoked by LogStash every 5 seconds.
|
|
239
|
+
def flush(options = {})
|
|
240
|
+
# Protection against no timeout defined by logstash conf : define a default eviction instance with timeout = DEFAULT_TIMEOUT seconds
|
|
241
|
+
if (@@eviction_instance.nil?)
|
|
242
|
+
@@eviction_instance = self
|
|
243
|
+
@timeout = DEFAULT_TIMEOUT
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
# Launch eviction only every interval of (@timeout / 2) seconds
|
|
247
|
+
if (@@eviction_instance == self && (@@last_eviction_timestamp.nil? || Time.now > @@last_eviction_timestamp + @timeout / 2))
|
|
248
|
+
remove_expired_elements()
|
|
249
|
+
@@last_eviction_timestamp = Time.now
|
|
250
|
+
end
|
|
251
|
+
|
|
252
|
+
return nil
|
|
253
|
+
end
|
|
256
254
|
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
255
|
+
|
|
256
|
+
# Remove the expired Aggregate elements from "aggregate_maps" if they are older than timeout
|
|
257
|
+
def remove_expired_elements()
|
|
258
|
+
min_timestamp = Time.now - @timeout
|
|
259
|
+
@@mutex.synchronize do
|
|
260
|
+
@@aggregate_maps.delete_if { |key, element| element.creation_timestamp < min_timestamp }
|
|
261
|
+
end
|
|
262
|
+
end
|
|
265
263
|
|
|
266
264
|
end # class LogStash::Filters::Aggregate
|
|
267
265
|
|
|
268
266
|
# Element of "aggregate_maps"
|
|
269
267
|
class LogStash::Filters::Aggregate::Element
|
|
270
268
|
|
|
271
|
-
|
|
269
|
+
attr_accessor :creation_timestamp, :map
|
|
272
270
|
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
271
|
+
def initialize(creation_timestamp)
|
|
272
|
+
@creation_timestamp = creation_timestamp
|
|
273
|
+
@map = {}
|
|
274
|
+
end
|
|
277
275
|
end
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Gem::Specification.new do |s|
|
|
2
2
|
s.name = 'logstash-filter-aggregate'
|
|
3
|
-
s.version = '2.0.
|
|
3
|
+
s.version = '2.0.3'
|
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
|
5
5
|
s.summary = "The aim of this filter is to aggregate information available among several events (typically log lines) belonging to a same task, and finally push aggregated information into final task event."
|
|
6
6
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
|
@@ -19,6 +19,6 @@ Gem::Specification.new do |s|
|
|
|
19
19
|
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
|
|
20
20
|
|
|
21
21
|
# Gem dependencies
|
|
22
|
-
s.add_runtime_dependency "logstash-core", ">= 2.0.0
|
|
22
|
+
s.add_runtime_dependency "logstash-core", ">= 2.0.0", "< 3.0.0"
|
|
23
23
|
s.add_development_dependency 'logstash-devutils', '~> 0'
|
|
24
24
|
end
|
|
@@ -5,173 +5,181 @@ require_relative "aggregate_spec_helper"
|
|
|
5
5
|
|
|
6
6
|
describe LogStash::Filters::Aggregate do
|
|
7
7
|
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
8
|
+
before(:each) do
|
|
9
|
+
set_eviction_instance(nil)
|
|
10
|
+
aggregate_maps.clear()
|
|
11
|
+
@start_filter = setup_filter({ "map_action" => "create", "code" => "map['sql_duration'] = 0" })
|
|
12
|
+
@update_filter = setup_filter({ "map_action" => "update", "code" => "map['sql_duration'] += event['duration']" })
|
|
13
|
+
@end_filter = setup_filter({ "map_action" => "update", "code" => "event.to_hash.merge!(map)", "end_of_task" => true, "timeout" => 5 })
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
context "Start event" do
|
|
17
|
+
describe "and receiving an event without task_id" do
|
|
18
|
+
it "does not record it" do
|
|
19
|
+
@start_filter.filter(event())
|
|
20
|
+
expect(aggregate_maps).to be_empty
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
describe "and receiving an event with task_id" do
|
|
24
|
+
it "records it" do
|
|
25
|
+
event = start_event("taskid" => "id123")
|
|
26
|
+
@start_filter.filter(event)
|
|
27
|
+
|
|
28
|
+
expect(aggregate_maps.size).to eq(1)
|
|
29
|
+
expect(aggregate_maps["id123"]).not_to be_nil
|
|
30
|
+
expect(aggregate_maps["id123"].creation_timestamp).to be >= event["@timestamp"]
|
|
31
|
+
expect(aggregate_maps["id123"].map["sql_duration"]).to eq(0)
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
describe "and receiving two 'start events' for the same task_id" do
|
|
36
|
+
it "keeps the first one and does nothing with the second one" do
|
|
37
|
+
|
|
38
|
+
first_start_event = start_event("taskid" => "id124")
|
|
39
|
+
@start_filter.filter(first_start_event)
|
|
40
|
+
|
|
41
|
+
first_update_event = update_event("taskid" => "id124", "duration" => 2)
|
|
42
|
+
@update_filter.filter(first_update_event)
|
|
43
|
+
|
|
44
|
+
sleep(1)
|
|
45
|
+
second_start_event = start_event("taskid" => "id124")
|
|
46
|
+
@start_filter.filter(second_start_event)
|
|
47
|
+
|
|
48
|
+
expect(aggregate_maps.size).to eq(1)
|
|
49
|
+
expect(aggregate_maps["id124"].creation_timestamp).to be < second_start_event["@timestamp"]
|
|
50
|
+
expect(aggregate_maps["id124"].map["sql_duration"]).to eq(first_update_event["duration"])
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
context "End event" do
|
|
56
|
+
describe "receiving an event without a previous 'start event'" do
|
|
57
|
+
describe "but without a previous 'start event'" do
|
|
58
|
+
it "does nothing with the event" do
|
|
59
|
+
end_event = end_event("taskid" => "id124")
|
|
60
|
+
@end_filter.filter(end_event)
|
|
61
|
+
|
|
62
|
+
expect(aggregate_maps).to be_empty
|
|
63
|
+
expect(end_event["sql_duration"]).to be_nil
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
context "Start/end events interaction" do
|
|
70
|
+
describe "receiving a 'start event'" do
|
|
71
|
+
before(:each) do
|
|
72
|
+
@task_id_value = "id_123"
|
|
73
|
+
@start_event = start_event({"taskid" => @task_id_value})
|
|
74
|
+
@start_filter.filter(@start_event)
|
|
75
|
+
expect(aggregate_maps.size).to eq(1)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
describe "and receiving an end event" do
|
|
79
|
+
describe "and without an id" do
|
|
80
|
+
it "does nothing" do
|
|
81
|
+
end_event = end_event()
|
|
82
|
+
@end_filter.filter(end_event)
|
|
83
|
+
expect(aggregate_maps.size).to eq(1)
|
|
84
|
+
expect(end_event["sql_duration"]).to be_nil
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
describe "and an id different from the one of the 'start event'" do
|
|
89
|
+
it "does nothing" do
|
|
90
|
+
different_id_value = @task_id_value + "_different"
|
|
91
|
+
@end_filter.filter(end_event("taskid" => different_id_value))
|
|
92
|
+
|
|
93
|
+
expect(aggregate_maps.size).to eq(1)
|
|
94
|
+
expect(aggregate_maps[@task_id_value]).not_to be_nil
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
describe "and the same id of the 'start event'" do
|
|
99
|
+
it "add 'sql_duration' field to the end event and deletes the aggregate map associated to taskid" do
|
|
100
|
+
expect(aggregate_maps.size).to eq(1)
|
|
101
|
+
|
|
102
|
+
@update_filter.filter(update_event("taskid" => @task_id_value, "duration" => 2))
|
|
103
|
+
|
|
104
|
+
end_event = end_event("taskid" => @task_id_value)
|
|
105
|
+
@end_filter.filter(end_event)
|
|
106
|
+
|
|
107
|
+
expect(aggregate_maps).to be_empty
|
|
108
|
+
expect(end_event["sql_duration"]).to eq(2)
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
context "Event with integer task id" do
|
|
117
|
+
it "works as well as with a string task id" do
|
|
118
|
+
start_event = start_event("taskid" => 124)
|
|
119
|
+
@start_filter.filter(start_event)
|
|
120
|
+
expect(aggregate_maps.size).to eq(1)
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
context "Event which causes an exception when code call" do
|
|
125
|
+
it "intercepts exception, logs the error and tags the event with '_aggregateexception'" do
|
|
126
|
+
@start_filter = setup_filter({ "code" => "fail 'Test'" })
|
|
127
|
+
start_event = start_event("taskid" => "id124")
|
|
128
|
+
@start_filter.filter(start_event)
|
|
129
|
+
|
|
130
|
+
expect(start_event["tags"]).to eq(["_aggregateexception"])
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
context "flush call" do
|
|
135
|
+
before(:each) do
|
|
136
|
+
@end_filter.timeout = 1
|
|
137
|
+
expect(@end_filter.timeout).to eq(1)
|
|
138
|
+
@task_id_value = "id_123"
|
|
139
|
+
@start_event = start_event({"taskid" => @task_id_value})
|
|
140
|
+
@start_filter.filter(@start_event)
|
|
141
|
+
expect(aggregate_maps.size).to eq(1)
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
describe "no timeout defined in none filter" do
|
|
145
|
+
it "defines a default timeout on a default filter" do
|
|
146
|
+
set_eviction_instance(nil)
|
|
147
|
+
expect(eviction_instance).to be_nil
|
|
148
|
+
@end_filter.flush()
|
|
149
|
+
expect(eviction_instance).to eq(@end_filter)
|
|
150
|
+
expect(@end_filter.timeout).to eq(LogStash::Filters::Aggregate::DEFAULT_TIMEOUT)
|
|
151
|
+
end
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
describe "timeout is defined on another filter" do
|
|
155
|
+
it "eviction_instance is not updated" do
|
|
156
|
+
expect(eviction_instance).not_to be_nil
|
|
157
|
+
@start_filter.flush()
|
|
158
|
+
expect(eviction_instance).not_to eq(@start_filter)
|
|
159
|
+
expect(eviction_instance).to eq(@end_filter)
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
describe "no timeout defined on the filter" do
|
|
164
|
+
it "event is not removed" do
|
|
165
|
+
sleep(2)
|
|
166
|
+
@start_filter.flush()
|
|
167
|
+
expect(aggregate_maps.size).to eq(1)
|
|
168
|
+
end
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
describe "timeout defined on the filter" do
|
|
172
|
+
it "event is not removed if not expired" do
|
|
173
|
+
@end_filter.flush()
|
|
174
|
+
expect(aggregate_maps.size).to eq(1)
|
|
175
|
+
end
|
|
176
|
+
it "event is removed if expired" do
|
|
177
|
+
sleep(2)
|
|
178
|
+
@end_filter.flush()
|
|
179
|
+
expect(aggregate_maps).to be_empty
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
end
|
|
176
184
|
|
|
177
185
|
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: logstash-filter-aggregate
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 2.0.
|
|
4
|
+
version: 2.0.3
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Elastic
|
|
@@ -9,58 +9,58 @@ authors:
|
|
|
9
9
|
autorequire:
|
|
10
10
|
bindir: bin
|
|
11
11
|
cert_chain: []
|
|
12
|
-
date: 2015-
|
|
12
|
+
date: 2015-11-03 00:00:00.000000000 Z
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
|
-
|
|
15
|
+
name: logstash-core
|
|
16
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
16
17
|
requirements:
|
|
17
18
|
- - '>='
|
|
18
19
|
- !ruby/object:Gem::Version
|
|
19
|
-
version: 2.0.0
|
|
20
|
+
version: 2.0.0
|
|
20
21
|
- - <
|
|
21
22
|
- !ruby/object:Gem::Version
|
|
22
23
|
version: 3.0.0
|
|
23
|
-
|
|
24
|
-
prerelease: false
|
|
25
|
-
type: :runtime
|
|
26
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
24
|
+
requirement: !ruby/object:Gem::Requirement
|
|
27
25
|
requirements:
|
|
28
26
|
- - '>='
|
|
29
27
|
- !ruby/object:Gem::Version
|
|
30
|
-
version: 2.0.0
|
|
28
|
+
version: 2.0.0
|
|
31
29
|
- - <
|
|
32
30
|
- !ruby/object:Gem::Version
|
|
33
31
|
version: 3.0.0
|
|
32
|
+
prerelease: false
|
|
33
|
+
type: :runtime
|
|
34
34
|
- !ruby/object:Gem::Dependency
|
|
35
|
-
|
|
35
|
+
name: logstash-devutils
|
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
36
37
|
requirements:
|
|
37
38
|
- - ~>
|
|
38
39
|
- !ruby/object:Gem::Version
|
|
39
40
|
version: '0'
|
|
40
|
-
|
|
41
|
-
prerelease: false
|
|
42
|
-
type: :development
|
|
43
|
-
version_requirements: !ruby/object:Gem::Requirement
|
|
41
|
+
requirement: !ruby/object:Gem::Requirement
|
|
44
42
|
requirements:
|
|
45
43
|
- - ~>
|
|
46
44
|
- !ruby/object:Gem::Version
|
|
47
45
|
version: '0'
|
|
46
|
+
prerelease: false
|
|
47
|
+
type: :development
|
|
48
48
|
description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
|
|
49
49
|
email: info@elastic.co
|
|
50
50
|
executables: []
|
|
51
51
|
extensions: []
|
|
52
52
|
extra_rdoc_files: []
|
|
53
53
|
files:
|
|
54
|
+
- lib/logstash/filters/aggregate.rb
|
|
55
|
+
- spec/filters/aggregate_spec.rb
|
|
56
|
+
- spec/filters/aggregate_spec_helper.rb
|
|
57
|
+
- logstash-filter-aggregate.gemspec
|
|
54
58
|
- BUILD.md
|
|
55
59
|
- CHANGELOG.md
|
|
60
|
+
- README.md
|
|
56
61
|
- CONTRIBUTORS
|
|
57
62
|
- Gemfile
|
|
58
63
|
- LICENSE
|
|
59
|
-
- README.md
|
|
60
|
-
- lib/logstash/filters/aggregate.rb
|
|
61
|
-
- logstash-filter-aggregate.gemspec
|
|
62
|
-
- spec/filters/aggregate_spec.rb
|
|
63
|
-
- spec/filters/aggregate_spec_helper.rb
|
|
64
64
|
homepage: https://github.com/logstash-plugins/logstash-filter-aggregate
|
|
65
65
|
licenses:
|
|
66
66
|
- Apache License (2.0)
|
|
@@ -83,7 +83,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
83
83
|
version: '0'
|
|
84
84
|
requirements: []
|
|
85
85
|
rubyforge_project:
|
|
86
|
-
rubygems_version: 2.
|
|
86
|
+
rubygems_version: 2.1.9
|
|
87
87
|
signing_key:
|
|
88
88
|
specification_version: 4
|
|
89
89
|
summary: The aim of this filter is to aggregate information available among several events (typically log lines) belonging to a same task, and finally push aggregated information into final task event.
|