logstash-filter-aggregate 0.1.4 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +3 -0
- data/lib/logstash/filters/aggregate.rb +190 -190
- data/logstash-filter-aggregate.gemspec +1 -1
- data/spec/filters/aggregate_spec.rb +176 -168
- metadata +3 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: a1149eb1e70a66aea89d164eb8046e14e5b51d3e
|
|
4
|
+
data.tar.gz: bdbb0e9df1f525de4548b1ab78b3c132d28b87b1
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 307eb7baf8d3be19b0fd3e7d28243c79b73a1b2a4b938be66365833156a2e676747ea867c4d79e4c9720bca6af35cecfde41a69206366c369133cb79a5722e66
|
|
7
|
+
data.tar.gz: 7f0a13541acf445ff46c74d8ed214901eb7d1d7de9d67be18f26612d4f9c88088aa18eb4d056063a7284067fdf775417c02e62a84673b4bacfe567f5700ba632
|
data/CHANGELOG.md
CHANGED
|
@@ -4,102 +4,102 @@ require "logstash/filters/base"
|
|
|
4
4
|
require "logstash/namespace"
|
|
5
5
|
require "thread"
|
|
6
6
|
|
|
7
|
-
#
|
|
7
|
+
#
|
|
8
8
|
# The aim of this filter is to aggregate information available among several events (typically log lines) belonging to a same task,
|
|
9
9
|
# and finally push aggregated information into final task event.
|
|
10
10
|
#
|
|
11
11
|
# ==== Example #1
|
|
12
|
-
#
|
|
12
|
+
#
|
|
13
13
|
# * with these given logs :
|
|
14
14
|
# [source,ruby]
|
|
15
15
|
# ----------------------------------
|
|
16
|
-
#
|
|
17
|
-
#
|
|
18
|
-
#
|
|
19
|
-
#
|
|
16
|
+
# INFO - 12345 - TASK_START - start
|
|
17
|
+
# INFO - 12345 - SQL - sqlQuery1 - 12
|
|
18
|
+
# INFO - 12345 - SQL - sqlQuery2 - 34
|
|
19
|
+
# INFO - 12345 - TASK_END - end
|
|
20
20
|
# ----------------------------------
|
|
21
|
-
#
|
|
21
|
+
#
|
|
22
22
|
# * you can aggregate "sql duration" for the whole task with this configuration :
|
|
23
23
|
# [source,ruby]
|
|
24
24
|
# ----------------------------------
|
|
25
|
-
#
|
|
26
|
-
#
|
|
27
|
-
#
|
|
28
|
-
#
|
|
29
|
-
#
|
|
30
|
-
#
|
|
31
|
-
#
|
|
32
|
-
#
|
|
33
|
-
#
|
|
34
|
-
#
|
|
35
|
-
#
|
|
36
|
-
#
|
|
37
|
-
#
|
|
38
|
-
#
|
|
39
|
-
#
|
|
40
|
-
#
|
|
41
|
-
#
|
|
42
|
-
#
|
|
43
|
-
#
|
|
44
|
-
#
|
|
45
|
-
#
|
|
46
|
-
#
|
|
47
|
-
#
|
|
48
|
-
#
|
|
49
|
-
#
|
|
50
|
-
#
|
|
51
|
-
#
|
|
52
|
-
#
|
|
53
|
-
#
|
|
54
|
-
#
|
|
55
|
-
#
|
|
25
|
+
# filter {
|
|
26
|
+
# grok {
|
|
27
|
+
# match => [ "message", "%{LOGLEVEL:loglevel} - %{NOTSPACE:taskid} - %{NOTSPACE:logger} - %{WORD:label}( - %{INT:duration:int})?" ]
|
|
28
|
+
# }
|
|
29
|
+
#
|
|
30
|
+
# if [logger] == "TASK_START" {
|
|
31
|
+
# aggregate {
|
|
32
|
+
# task_id => "%{taskid}"
|
|
33
|
+
# code => "map['sql_duration'] = 0"
|
|
34
|
+
# map_action => "create"
|
|
35
|
+
# }
|
|
36
|
+
# }
|
|
37
|
+
#
|
|
38
|
+
# if [logger] == "SQL" {
|
|
39
|
+
# aggregate {
|
|
40
|
+
# task_id => "%{taskid}"
|
|
41
|
+
# code => "map['sql_duration'] += event['duration']"
|
|
42
|
+
# map_action => "update"
|
|
43
|
+
# }
|
|
44
|
+
# }
|
|
45
|
+
#
|
|
46
|
+
# if [logger] == "TASK_END" {
|
|
47
|
+
# aggregate {
|
|
48
|
+
# task_id => "%{taskid}"
|
|
49
|
+
# code => "event['sql_duration'] = map['sql_duration']"
|
|
50
|
+
# map_action => "update"
|
|
51
|
+
# end_of_task => true
|
|
52
|
+
# timeout => 120
|
|
53
|
+
# }
|
|
54
|
+
# }
|
|
55
|
+
# }
|
|
56
56
|
# ----------------------------------
|
|
57
57
|
#
|
|
58
58
|
# * the final event then looks like :
|
|
59
59
|
# [source,ruby]
|
|
60
60
|
# ----------------------------------
|
|
61
61
|
# {
|
|
62
|
-
#
|
|
63
|
-
#
|
|
62
|
+
# "message" => "INFO - 12345 - TASK_END - end message",
|
|
63
|
+
# "sql_duration" => 46
|
|
64
64
|
# }
|
|
65
65
|
# ----------------------------------
|
|
66
|
-
#
|
|
66
|
+
#
|
|
67
67
|
# the field `sql_duration` is added and contains the sum of all sql queries durations.
|
|
68
|
-
#
|
|
68
|
+
#
|
|
69
69
|
# ==== Example #2
|
|
70
70
|
#
|
|
71
71
|
# * If you have the same logs than example #1, but without a start log :
|
|
72
72
|
# [source,ruby]
|
|
73
73
|
# ----------------------------------
|
|
74
|
-
#
|
|
75
|
-
#
|
|
76
|
-
#
|
|
74
|
+
# INFO - 12345 - SQL - sqlQuery1 - 12
|
|
75
|
+
# INFO - 12345 - SQL - sqlQuery2 - 34
|
|
76
|
+
# INFO - 12345 - TASK_END - end
|
|
77
77
|
# ----------------------------------
|
|
78
|
-
#
|
|
78
|
+
#
|
|
79
79
|
# * you can also aggregate "sql duration" with a slightly different configuration :
|
|
80
80
|
# [source,ruby]
|
|
81
81
|
# ----------------------------------
|
|
82
|
-
#
|
|
83
|
-
#
|
|
84
|
-
#
|
|
85
|
-
#
|
|
82
|
+
# filter {
|
|
83
|
+
# grok {
|
|
84
|
+
# match => [ "message", "%{LOGLEVEL:loglevel} - %{NOTSPACE:taskid} - %{NOTSPACE:logger} - %{WORD:label}( - %{INT:duration:int})?" ]
|
|
85
|
+
# }
|
|
86
86
|
#
|
|
87
|
-
#
|
|
88
|
-
#
|
|
89
|
-
#
|
|
90
|
-
#
|
|
91
|
-
#
|
|
92
|
-
#
|
|
87
|
+
# if [logger] == "SQL" {
|
|
88
|
+
# aggregate {
|
|
89
|
+
# task_id => "%{taskid}"
|
|
90
|
+
# code => "map['sql_duration'] ||= 0 ; map['sql_duration'] += event['duration']"
|
|
91
|
+
# }
|
|
92
|
+
# }
|
|
93
93
|
#
|
|
94
|
-
#
|
|
95
|
-
#
|
|
96
|
-
#
|
|
97
|
-
#
|
|
98
|
-
#
|
|
99
|
-
#
|
|
100
|
-
#
|
|
101
|
-
#
|
|
102
|
-
#
|
|
94
|
+
# if [logger] == "TASK_END" {
|
|
95
|
+
# aggregate {
|
|
96
|
+
# task_id => "%{taskid}"
|
|
97
|
+
# code => "event['sql_duration'] = map['sql_duration']"
|
|
98
|
+
# end_of_task => true
|
|
99
|
+
# timeout => 120
|
|
100
|
+
# }
|
|
101
|
+
# }
|
|
102
|
+
# }
|
|
103
103
|
# ----------------------------------
|
|
104
104
|
#
|
|
105
105
|
# * the final event is exactly the same than example #1
|
|
@@ -119,159 +119,159 @@ require "thread"
|
|
|
119
119
|
#
|
|
120
120
|
class LogStash::Filters::Aggregate < LogStash::Filters::Base
|
|
121
121
|
|
|
122
|
-
|
|
122
|
+
config_name "aggregate"
|
|
123
123
|
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
124
|
+
# The expression defining task ID to correlate logs.
|
|
125
|
+
#
|
|
126
|
+
# This value must uniquely identify the task in the system.
|
|
127
|
+
#
|
|
128
|
+
# Example value : "%{application}%{my_task_id}"
|
|
129
|
+
config :task_id, :validate => :string, :required => true
|
|
130
130
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
131
|
+
# The code to execute to update map, using current event.
|
|
132
|
+
#
|
|
133
|
+
# Or on the contrary, the code to execute to update event, using current map.
|
|
134
|
+
#
|
|
135
|
+
# You will have a 'map' variable and an 'event' variable available (that is the event itself).
|
|
136
|
+
#
|
|
137
|
+
# Example value : "map['sql_duration'] += event['duration']"
|
|
138
|
+
config :code, :validate => :string, :required => true
|
|
139
139
|
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
140
|
+
# Tell the filter what to do with aggregate map.
|
|
141
|
+
#
|
|
142
|
+
# `create`: create the map, and execute the code only if map wasn't created before
|
|
143
|
+
#
|
|
144
|
+
# `update`: doesn't create the map, and execute the code only if map was created before
|
|
145
|
+
#
|
|
146
|
+
# `create_or_update`: create the map if it wasn't created before, execute the code in all cases
|
|
147
|
+
config :map_action, :validate => :string, :default => "create_or_update"
|
|
148
148
|
|
|
149
|
-
|
|
150
|
-
|
|
149
|
+
# Tell the filter that task is ended, and therefore, to delete map after code execution.
|
|
150
|
+
config :end_of_task, :validate => :boolean, :default => false
|
|
151
151
|
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
152
|
+
# The amount of seconds after a task "end event" can be considered lost.
|
|
153
|
+
#
|
|
154
|
+
# The task "map" is evicted.
|
|
155
|
+
#
|
|
156
|
+
# Default value (`0`) means no timeout so no auto eviction.
|
|
157
|
+
config :timeout, :validate => :number, :required => false, :default => 0
|
|
158
158
|
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
159
|
+
|
|
160
|
+
# Default timeout (in seconds) when not defined in plugin configuration
|
|
161
|
+
DEFAULT_TIMEOUT = 1800
|
|
162
162
|
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
163
|
+
# This is the state of the filter.
|
|
164
|
+
# For each entry, key is "task_id" and value is a map freely updatable by 'code' config
|
|
165
|
+
@@aggregate_maps = {}
|
|
166
166
|
|
|
167
|
-
|
|
168
|
-
|
|
167
|
+
# Mutex used to synchronize access to 'aggregate_maps'
|
|
168
|
+
@@mutex = Mutex.new
|
|
169
169
|
|
|
170
|
-
|
|
171
|
-
|
|
170
|
+
# Aggregate instance which will evict all zombie Aggregate elements (older than timeout)
|
|
171
|
+
@@eviction_instance = nil
|
|
172
172
|
|
|
173
|
-
|
|
174
|
-
|
|
173
|
+
# last time where eviction was launched
|
|
174
|
+
@@last_eviction_timestamp = nil
|
|
175
175
|
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
176
|
+
# Initialize plugin
|
|
177
|
+
public
|
|
178
|
+
def register
|
|
179
|
+
# process lambda expression to call in each filter call
|
|
180
|
+
eval("@codeblock = lambda { |event, map| #{@code} }", binding, "(aggregate filter code)")
|
|
181
181
|
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
182
|
+
# define eviction_instance
|
|
183
|
+
@@mutex.synchronize do
|
|
184
|
+
if (@timeout > 0 && (@@eviction_instance.nil? || @timeout < @@eviction_instance.timeout))
|
|
185
|
+
@@eviction_instance = self
|
|
186
|
+
@logger.info("Aggregate, timeout: #{@timeout} seconds")
|
|
187
|
+
end
|
|
188
|
+
end
|
|
189
|
+
end
|
|
190
190
|
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
191
|
+
|
|
192
|
+
# This method is invoked each time an event matches the filter
|
|
193
|
+
public
|
|
194
|
+
def filter(event)
|
|
195
195
|
# return nothing unless there's an actual filter event
|
|
196
196
|
return unless filter?(event)
|
|
197
197
|
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
198
|
+
# define task id
|
|
199
|
+
task_id = event.sprintf(@task_id)
|
|
200
|
+
return if task_id.nil? || task_id == @task_id
|
|
201
201
|
|
|
202
|
-
|
|
202
|
+
noError = false
|
|
203
203
|
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
204
|
+
# protect aggregate_maps against concurrent access, using a mutex
|
|
205
|
+
@@mutex.synchronize do
|
|
206
|
+
|
|
207
|
+
# retrieve the current aggregate map
|
|
208
|
+
aggregate_maps_element = @@aggregate_maps[task_id]
|
|
209
|
+
if (aggregate_maps_element.nil?)
|
|
210
|
+
return if @map_action == "update"
|
|
211
|
+
aggregate_maps_element = LogStash::Filters::Aggregate::Element.new(Time.now);
|
|
212
|
+
@@aggregate_maps[task_id] = aggregate_maps_element
|
|
213
|
+
else
|
|
214
|
+
return if @map_action == "create"
|
|
215
|
+
end
|
|
216
|
+
map = aggregate_maps_element.map
|
|
217
217
|
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
218
|
+
# execute the code to read/update map and event
|
|
219
|
+
begin
|
|
220
|
+
@codeblock.call(event, map)
|
|
221
|
+
noError = true
|
|
222
|
+
rescue => exception
|
|
223
|
+
@logger.error("Aggregate exception occurred. Error: #{exception} ; Code: #{@code} ; Map: #{map} ; EventData: #{event.instance_variable_get('@data')}")
|
|
224
|
+
event.tag("_aggregateexception")
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
# delete the map if task is ended
|
|
228
|
+
@@aggregate_maps.delete(task_id) if @end_of_task
|
|
229
|
+
end
|
|
230
230
|
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
231
|
+
# match the filter, only if no error occurred
|
|
232
|
+
filter_matched(event) if noError
|
|
233
|
+
end
|
|
234
234
|
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
235
|
+
# Necessary to indicate logstash to periodically call 'flush' method
|
|
236
|
+
def periodic_flush
|
|
237
|
+
true
|
|
238
|
+
end
|
|
239
239
|
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
240
|
+
# This method is invoked by LogStash every 5 seconds.
|
|
241
|
+
def flush(options = {})
|
|
242
|
+
# Protection against no timeout defined by logstash conf : define a default eviction instance with timeout = DEFAULT_TIMEOUT seconds
|
|
243
|
+
if (@@eviction_instance.nil?)
|
|
244
|
+
@@eviction_instance = self
|
|
245
|
+
@timeout = DEFAULT_TIMEOUT
|
|
246
|
+
end
|
|
247
|
+
|
|
248
|
+
# Launch eviction only every interval of (@timeout / 2) seconds
|
|
249
|
+
if (@@eviction_instance == self && (@@last_eviction_timestamp.nil? || Time.now > @@last_eviction_timestamp + @timeout / 2))
|
|
250
|
+
remove_expired_elements()
|
|
251
|
+
@@last_eviction_timestamp = Time.now
|
|
252
|
+
end
|
|
253
|
+
|
|
254
|
+
return nil
|
|
255
|
+
end
|
|
256
256
|
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
257
|
+
|
|
258
|
+
# Remove the expired Aggregate elements from "aggregate_maps" if they are older than timeout
|
|
259
|
+
def remove_expired_elements()
|
|
260
|
+
min_timestamp = Time.now - @timeout
|
|
261
|
+
@@mutex.synchronize do
|
|
262
|
+
@@aggregate_maps.delete_if { |key, element| element.creation_timestamp < min_timestamp }
|
|
263
|
+
end
|
|
264
|
+
end
|
|
265
265
|
|
|
266
266
|
end # class LogStash::Filters::Aggregate
|
|
267
267
|
|
|
268
268
|
# Element of "aggregate_maps"
|
|
269
269
|
class LogStash::Filters::Aggregate::Element
|
|
270
270
|
|
|
271
|
-
|
|
271
|
+
attr_accessor :creation_timestamp, :map
|
|
272
272
|
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
273
|
+
def initialize(creation_timestamp)
|
|
274
|
+
@creation_timestamp = creation_timestamp
|
|
275
|
+
@map = {}
|
|
276
|
+
end
|
|
277
277
|
end
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Gem::Specification.new do |s|
|
|
2
2
|
s.name = 'logstash-filter-aggregate'
|
|
3
|
-
s.version = '0.1.
|
|
3
|
+
s.version = '0.1.5'
|
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
|
5
5
|
s.summary = "The aim of this filter is to aggregate information available among several events (typically log lines) belonging to a same task, and finally push aggregated information into final task event."
|
|
6
6
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
|
@@ -5,173 +5,181 @@ require_relative "aggregate_spec_helper"
|
|
|
5
5
|
|
|
6
6
|
describe LogStash::Filters::Aggregate do
|
|
7
7
|
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
8
|
+
before(:each) do
|
|
9
|
+
set_eviction_instance(nil)
|
|
10
|
+
aggregate_maps.clear()
|
|
11
|
+
@start_filter = setup_filter({ "map_action" => "create", "code" => "map['sql_duration'] = 0" })
|
|
12
|
+
@update_filter = setup_filter({ "map_action" => "update", "code" => "map['sql_duration'] += event['duration']" })
|
|
13
|
+
@end_filter = setup_filter({ "map_action" => "update", "code" => "event.to_hash.merge!(map)", "end_of_task" => true, "timeout" => 5 })
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
context "Start event" do
|
|
17
|
+
describe "and receiving an event without task_id" do
|
|
18
|
+
it "does not record it" do
|
|
19
|
+
@start_filter.filter(event())
|
|
20
|
+
expect(aggregate_maps).to be_empty
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
describe "and receiving an event with task_id" do
|
|
24
|
+
it "records it" do
|
|
25
|
+
event = start_event("taskid" => "id123")
|
|
26
|
+
@start_filter.filter(event)
|
|
27
|
+
|
|
28
|
+
expect(aggregate_maps.size).to eq(1)
|
|
29
|
+
expect(aggregate_maps["id123"]).not_to be_nil
|
|
30
|
+
expect(aggregate_maps["id123"].creation_timestamp).to be >= event["@timestamp"]
|
|
31
|
+
expect(aggregate_maps["id123"].map["sql_duration"]).to eq(0)
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
describe "and receiving two 'start events' for the same task_id" do
|
|
36
|
+
it "keeps the first one and does nothing with the second one" do
|
|
37
|
+
|
|
38
|
+
first_start_event = start_event("taskid" => "id124")
|
|
39
|
+
@start_filter.filter(first_start_event)
|
|
40
|
+
|
|
41
|
+
first_update_event = update_event("taskid" => "id124", "duration" => 2)
|
|
42
|
+
@update_filter.filter(first_update_event)
|
|
43
|
+
|
|
44
|
+
sleep(1)
|
|
45
|
+
second_start_event = start_event("taskid" => "id124")
|
|
46
|
+
@start_filter.filter(second_start_event)
|
|
47
|
+
|
|
48
|
+
expect(aggregate_maps.size).to eq(1)
|
|
49
|
+
expect(aggregate_maps["id124"].creation_timestamp).to be < second_start_event["@timestamp"]
|
|
50
|
+
expect(aggregate_maps["id124"].map["sql_duration"]).to eq(first_update_event["duration"])
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
context "End event" do
|
|
56
|
+
describe "receiving an event without a previous 'start event'" do
|
|
57
|
+
describe "but without a previous 'start event'" do
|
|
58
|
+
it "does nothing with the event" do
|
|
59
|
+
end_event = end_event("taskid" => "id124")
|
|
60
|
+
@end_filter.filter(end_event)
|
|
61
|
+
|
|
62
|
+
expect(aggregate_maps).to be_empty
|
|
63
|
+
expect(end_event["sql_duration"]).to be_nil
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
end
|
|
68
|
+
|
|
69
|
+
context "Start/end events interaction" do
|
|
70
|
+
describe "receiving a 'start event'" do
|
|
71
|
+
before(:each) do
|
|
72
|
+
@task_id_value = "id_123"
|
|
73
|
+
@start_event = start_event({"taskid" => @task_id_value})
|
|
74
|
+
@start_filter.filter(@start_event)
|
|
75
|
+
expect(aggregate_maps.size).to eq(1)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
describe "and receiving an end event" do
|
|
79
|
+
describe "and without an id" do
|
|
80
|
+
it "does nothing" do
|
|
81
|
+
end_event = end_event()
|
|
82
|
+
@end_filter.filter(end_event)
|
|
83
|
+
expect(aggregate_maps.size).to eq(1)
|
|
84
|
+
expect(end_event["sql_duration"]).to be_nil
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
describe "and an id different from the one of the 'start event'" do
|
|
89
|
+
it "does nothing" do
|
|
90
|
+
different_id_value = @task_id_value + "_different"
|
|
91
|
+
@end_filter.filter(end_event("taskid" => different_id_value))
|
|
92
|
+
|
|
93
|
+
expect(aggregate_maps.size).to eq(1)
|
|
94
|
+
expect(aggregate_maps[@task_id_value]).not_to be_nil
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
describe "and the same id of the 'start event'" do
|
|
99
|
+
it "add 'sql_duration' field to the end event and deletes the aggregate map associated to taskid" do
|
|
100
|
+
expect(aggregate_maps.size).to eq(1)
|
|
101
|
+
|
|
102
|
+
@update_filter.filter(update_event("taskid" => @task_id_value, "duration" => 2))
|
|
103
|
+
|
|
104
|
+
end_event = end_event("taskid" => @task_id_value)
|
|
105
|
+
@end_filter.filter(end_event)
|
|
106
|
+
|
|
107
|
+
expect(aggregate_maps).to be_empty
|
|
108
|
+
expect(end_event["sql_duration"]).to eq(2)
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
context "Event with integer task id" do
|
|
117
|
+
it "works as well as with a string task id" do
|
|
118
|
+
start_event = start_event("taskid" => 124)
|
|
119
|
+
@start_filter.filter(start_event)
|
|
120
|
+
expect(aggregate_maps.size).to eq(1)
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
context "Event which causes an exception when code call" do
|
|
125
|
+
it "intercepts exception, logs the error and tags the event with '_aggregateexception'" do
|
|
126
|
+
@start_filter = setup_filter({ "code" => "fail 'Test'" })
|
|
127
|
+
start_event = start_event("taskid" => "id124")
|
|
128
|
+
@start_filter.filter(start_event)
|
|
129
|
+
|
|
130
|
+
expect(start_event["tags"]).to eq(["_aggregateexception"])
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
context "flush call" do
|
|
135
|
+
before(:each) do
|
|
136
|
+
@end_filter.timeout = 1
|
|
137
|
+
expect(@end_filter.timeout).to eq(1)
|
|
138
|
+
@task_id_value = "id_123"
|
|
139
|
+
@start_event = start_event({"taskid" => @task_id_value})
|
|
140
|
+
@start_filter.filter(@start_event)
|
|
141
|
+
expect(aggregate_maps.size).to eq(1)
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
describe "no timeout defined in none filter" do
|
|
145
|
+
it "defines a default timeout on a default filter" do
|
|
146
|
+
set_eviction_instance(nil)
|
|
147
|
+
expect(eviction_instance).to be_nil
|
|
148
|
+
@end_filter.flush()
|
|
149
|
+
expect(eviction_instance).to eq(@end_filter)
|
|
150
|
+
expect(@end_filter.timeout).to eq(LogStash::Filters::Aggregate::DEFAULT_TIMEOUT)
|
|
151
|
+
end
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
describe "timeout is defined on another filter" do
|
|
155
|
+
it "eviction_instance is not updated" do
|
|
156
|
+
expect(eviction_instance).not_to be_nil
|
|
157
|
+
@start_filter.flush()
|
|
158
|
+
expect(eviction_instance).not_to eq(@start_filter)
|
|
159
|
+
expect(eviction_instance).to eq(@end_filter)
|
|
160
|
+
end
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
describe "no timeout defined on the filter" do
|
|
164
|
+
it "event is not removed" do
|
|
165
|
+
sleep(2)
|
|
166
|
+
@start_filter.flush()
|
|
167
|
+
expect(aggregate_maps.size).to eq(1)
|
|
168
|
+
end
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
describe "timeout defined on the filter" do
|
|
172
|
+
it "event is not removed if not expired" do
|
|
173
|
+
@end_filter.flush()
|
|
174
|
+
expect(aggregate_maps.size).to eq(1)
|
|
175
|
+
end
|
|
176
|
+
it "event is removed if expired" do
|
|
177
|
+
sleep(2)
|
|
178
|
+
@end_filter.flush()
|
|
179
|
+
expect(aggregate_maps).to be_empty
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
end
|
|
176
184
|
|
|
177
185
|
end
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: logstash-filter-aggregate
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.5
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Elastic
|
|
@@ -9,7 +9,7 @@ authors:
|
|
|
9
9
|
autorequire:
|
|
10
10
|
bindir: bin
|
|
11
11
|
cert_chain: []
|
|
12
|
-
date: 2015-
|
|
12
|
+
date: 2015-11-04 00:00:00.000000000 Z
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: logstash-core
|
|
@@ -83,7 +83,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
83
83
|
version: '0'
|
|
84
84
|
requirements: []
|
|
85
85
|
rubyforge_project:
|
|
86
|
-
rubygems_version: 2.4.
|
|
86
|
+
rubygems_version: 2.4.6
|
|
87
87
|
signing_key:
|
|
88
88
|
specification_version: 4
|
|
89
89
|
summary: The aim of this filter is to aggregate information available among several events (typically log lines) belonging to a same task, and finally push aggregated information into final task event.
|