logstash-filter-aggregate 2.0.2 → 2.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 54c0279cf7984071cc3f9a16b46c6880381ce1c8
4
- data.tar.gz: 4388fcf687de2dbdc244567faf9b377d1078c639
3
+ metadata.gz: 6500c98a915954331abf2a5e589c58497d735a4c
4
+ data.tar.gz: a2936e86e9e33cb497a9d86b22356d1a9c5299ac
5
5
  SHA512:
6
- metadata.gz: 7c1fea00dea5fce67dc57e5f24d402d116f966c4412e714b9ac385a2189a0b3284b3674c5d941fd4120a969e90dd562f2859884e26444be5885e85cf6725c651
7
- data.tar.gz: 1769add45737af42642d49cc762a93499df95e6c766c4e82ec03ed7978fa7535c81daf10b91ce96943286238b72e6faad3c65596075e4cfbc9bdab8eadb97d2e
6
+ metadata.gz: 38d0789ad10b94338d1ece65b84e4d7dbf60ab320da55ef1921426509ffd4f4516c2457ccad2723a5c98b19fc5e2490e3676dba80496970d03dbec23312a6d3e
7
+ data.tar.gz: b19dbfc2e2af9202dd6b992fbbf925fd8c8280ef404e8c054e2291f2ffac80b4550c91f7c8502f9c158d0ef6774a6d515dd294f0c7426215795b586a09e1b721
@@ -1,4 +1,10 @@
1
- ## 2.0.0
1
+ # v 2.0.3
2
+ - fix issue #10 : numeric task_id is now well processed
3
+
4
+ # v 2.0.2
5
+ - fix issue #5 : when code call raises an exception, the error is logged and the event is tagged '_aggregateexception'. It avoids logstash crash.
6
+
7
+ # v 2.0.0
2
8
  - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
3
9
  instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
4
10
  - Dependency on logstash-core update to 2.0
data/README.md CHANGED
@@ -1,5 +1,7 @@
1
1
  # Logstash Filter Aggregate Documentation
2
2
 
3
+ [![Build Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-aggregate-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-aggregate-unit/)
4
+
3
5
  The aim of this filter is to aggregate information available among several events (typically log lines) belonging to a same task, and finally push aggregated information into final task event.
4
6
 
5
7
  ## Example #1
@@ -4,102 +4,102 @@ require "logstash/filters/base"
4
4
  require "logstash/namespace"
5
5
  require "thread"
6
6
 
7
- #
7
+ #
8
8
  # The aim of this filter is to aggregate information available among several events (typically log lines) belonging to a same task,
9
9
  # and finally push aggregated information into final task event.
10
10
  #
11
11
  # ==== Example #1
12
- #
12
+ #
13
13
  # * with these given logs :
14
14
  # [source,ruby]
15
15
  # ----------------------------------
16
- # INFO - 12345 - TASK_START - start
17
- # INFO - 12345 - SQL - sqlQuery1 - 12
18
- # INFO - 12345 - SQL - sqlQuery2 - 34
19
- # INFO - 12345 - TASK_END - end
16
+ # INFO - 12345 - TASK_START - start
17
+ # INFO - 12345 - SQL - sqlQuery1 - 12
18
+ # INFO - 12345 - SQL - sqlQuery2 - 34
19
+ # INFO - 12345 - TASK_END - end
20
20
  # ----------------------------------
21
- #
21
+ #
22
22
  # * you can aggregate "sql duration" for the whole task with this configuration :
23
23
  # [source,ruby]
24
24
  # ----------------------------------
25
- # filter {
26
- # grok {
27
- # match => [ "message", "%{LOGLEVEL:loglevel} - %{NOTSPACE:taskid} - %{NOTSPACE:logger} - %{WORD:label}( - %{INT:duration:int})?" ]
28
- # }
29
- #
30
- # if [logger] == "TASK_START" {
31
- # aggregate {
32
- # task_id => "%{taskid}"
33
- # code => "map['sql_duration'] = 0"
34
- # map_action => "create"
35
- # }
36
- # }
37
- #
38
- # if [logger] == "SQL" {
39
- # aggregate {
40
- # task_id => "%{taskid}"
41
- # code => "map['sql_duration'] += event['duration']"
42
- # map_action => "update"
43
- # }
44
- # }
45
- #
46
- # if [logger] == "TASK_END" {
47
- # aggregate {
48
- # task_id => "%{taskid}"
49
- # code => "event['sql_duration'] = map['sql_duration']"
50
- # map_action => "update"
51
- # end_of_task => true
52
- # timeout => 120
53
- # }
54
- # }
55
- # }
25
+ # filter {
26
+ # grok {
27
+ # match => [ "message", "%{LOGLEVEL:loglevel} - %{NOTSPACE:taskid} - %{NOTSPACE:logger} - %{WORD:label}( - %{INT:duration:int})?" ]
28
+ # }
29
+ #
30
+ # if [logger] == "TASK_START" {
31
+ # aggregate {
32
+ # task_id => "%{taskid}"
33
+ # code => "map['sql_duration'] = 0"
34
+ # map_action => "create"
35
+ # }
36
+ # }
37
+ #
38
+ # if [logger] == "SQL" {
39
+ # aggregate {
40
+ # task_id => "%{taskid}"
41
+ # code => "map['sql_duration'] += event['duration']"
42
+ # map_action => "update"
43
+ # }
44
+ # }
45
+ #
46
+ # if [logger] == "TASK_END" {
47
+ # aggregate {
48
+ # task_id => "%{taskid}"
49
+ # code => "event['sql_duration'] = map['sql_duration']"
50
+ # map_action => "update"
51
+ # end_of_task => true
52
+ # timeout => 120
53
+ # }
54
+ # }
55
+ # }
56
56
  # ----------------------------------
57
57
  #
58
58
  # * the final event then looks like :
59
59
  # [source,ruby]
60
60
  # ----------------------------------
61
61
  # {
62
- # "message" => "INFO - 12345 - TASK_END - end message",
63
- # "sql_duration" => 46
62
+ # "message" => "INFO - 12345 - TASK_END - end message",
63
+ # "sql_duration" => 46
64
64
  # }
65
65
  # ----------------------------------
66
- #
66
+ #
67
67
  # the field `sql_duration` is added and contains the sum of all sql queries durations.
68
- #
68
+ #
69
69
  # ==== Example #2
70
70
  #
71
71
  # * If you have the same logs than example #1, but without a start log :
72
72
  # [source,ruby]
73
73
  # ----------------------------------
74
- # INFO - 12345 - SQL - sqlQuery1 - 12
75
- # INFO - 12345 - SQL - sqlQuery2 - 34
76
- # INFO - 12345 - TASK_END - end
74
+ # INFO - 12345 - SQL - sqlQuery1 - 12
75
+ # INFO - 12345 - SQL - sqlQuery2 - 34
76
+ # INFO - 12345 - TASK_END - end
77
77
  # ----------------------------------
78
- #
78
+ #
79
79
  # * you can also aggregate "sql duration" with a slightly different configuration :
80
80
  # [source,ruby]
81
81
  # ----------------------------------
82
- # filter {
83
- # grok {
84
- # match => [ "message", "%{LOGLEVEL:loglevel} - %{NOTSPACE:taskid} - %{NOTSPACE:logger} - %{WORD:label}( - %{INT:duration:int})?" ]
85
- # }
82
+ # filter {
83
+ # grok {
84
+ # match => [ "message", "%{LOGLEVEL:loglevel} - %{NOTSPACE:taskid} - %{NOTSPACE:logger} - %{WORD:label}( - %{INT:duration:int})?" ]
85
+ # }
86
86
  #
87
- # if [logger] == "SQL" {
88
- # aggregate {
89
- # task_id => "%{taskid}"
90
- # code => "map['sql_duration'] ||= 0 ; map['sql_duration'] += event['duration']"
91
- # }
92
- # }
87
+ # if [logger] == "SQL" {
88
+ # aggregate {
89
+ # task_id => "%{taskid}"
90
+ # code => "map['sql_duration'] ||= 0 ; map['sql_duration'] += event['duration']"
91
+ # }
92
+ # }
93
93
  #
94
- # if [logger] == "TASK_END" {
95
- # aggregate {
96
- # task_id => "%{taskid}"
97
- # code => "event['sql_duration'] = map['sql_duration']"
98
- # end_of_task => true
99
- # timeout => 120
100
- # }
101
- # }
102
- # }
94
+ # if [logger] == "TASK_END" {
95
+ # aggregate {
96
+ # task_id => "%{taskid}"
97
+ # code => "event['sql_duration'] = map['sql_duration']"
98
+ # end_of_task => true
99
+ # timeout => 120
100
+ # }
101
+ # }
102
+ # }
103
103
  # ----------------------------------
104
104
  #
105
105
  # * the final event is exactly the same than example #1
@@ -119,159 +119,157 @@ require "thread"
119
119
  #
120
120
  class LogStash::Filters::Aggregate < LogStash::Filters::Base
121
121
 
122
- config_name "aggregate"
122
+ config_name "aggregate"
123
123
 
124
- # The expression defining task ID to correlate logs.
125
- #
126
- # This value must uniquely identify the task in the system.
127
- #
128
- # Example value : "%{application}%{my_task_id}"
129
- config :task_id, :validate => :string, :required => true
124
+ # The expression defining task ID to correlate logs.
125
+ #
126
+ # This value must uniquely identify the task in the system.
127
+ #
128
+ # Example value : "%{application}%{my_task_id}"
129
+ config :task_id, :validate => :string, :required => true
130
130
 
131
- # The code to execute to update map, using current event.
132
- #
133
- # Or on the contrary, the code to execute to update event, using current map.
134
- #
135
- # You will have a 'map' variable and an 'event' variable available (that is the event itself).
136
- #
137
- # Example value : "map['sql_duration'] += event['duration']"
138
- config :code, :validate => :string, :required => true
131
+ # The code to execute to update map, using current event.
132
+ #
133
+ # Or on the contrary, the code to execute to update event, using current map.
134
+ #
135
+ # You will have a 'map' variable and an 'event' variable available (that is the event itself).
136
+ #
137
+ # Example value : "map['sql_duration'] += event['duration']"
138
+ config :code, :validate => :string, :required => true
139
139
 
140
- # Tell the filter what to do with aggregate map.
141
- #
142
- # `create`: create the map, and execute the code only if map wasn't created before
143
- #
144
- # `update`: doesn't create the map, and execute the code only if map was created before
145
- #
146
- # `create_or_update`: create the map if it wasn't created before, execute the code in all cases
147
- config :map_action, :validate => :string, :default => "create_or_update"
140
+ # Tell the filter what to do with aggregate map.
141
+ #
142
+ # `create`: create the map, and execute the code only if map wasn't created before
143
+ #
144
+ # `update`: doesn't create the map, and execute the code only if map was created before
145
+ #
146
+ # `create_or_update`: create the map if it wasn't created before, execute the code in all cases
147
+ config :map_action, :validate => :string, :default => "create_or_update"
148
148
 
149
- # Tell the filter that task is ended, and therefore, to delete map after code execution.
150
- config :end_of_task, :validate => :boolean, :default => false
149
+ # Tell the filter that task is ended, and therefore, to delete map after code execution.
150
+ config :end_of_task, :validate => :boolean, :default => false
151
151
 
152
- # The amount of seconds after a task "end event" can be considered lost.
153
- #
154
- # The task "map" is evicted.
155
- #
156
- # Default value (`0`) means no timeout so no auto eviction.
157
- config :timeout, :validate => :number, :required => false, :default => 0
152
+ # The amount of seconds after a task "end event" can be considered lost.
153
+ #
154
+ # The task "map" is evicted.
155
+ #
156
+ # Default value (`0`) means no timeout so no auto eviction.
157
+ config :timeout, :validate => :number, :required => false, :default => 0
158
158
 
159
-
160
- # Default timeout (in seconds) when not defined in plugin configuration
161
- DEFAULT_TIMEOUT = 1800
159
+
160
+ # Default timeout (in seconds) when not defined in plugin configuration
161
+ DEFAULT_TIMEOUT = 1800
162
162
 
163
- # This is the state of the filter.
164
- # For each entry, key is "task_id" and value is a map freely updatable by 'code' config
165
- @@aggregate_maps = {}
163
+ # This is the state of the filter.
164
+ # For each entry, key is "task_id" and value is a map freely updatable by 'code' config
165
+ @@aggregate_maps = {}
166
166
 
167
- # Mutex used to synchronize access to 'aggregate_maps'
168
- @@mutex = Mutex.new
167
+ # Mutex used to synchronize access to 'aggregate_maps'
168
+ @@mutex = Mutex.new
169
169
 
170
- # Aggregate instance which will evict all zombie Aggregate elements (older than timeout)
171
- @@eviction_instance = nil
170
+ # Aggregate instance which will evict all zombie Aggregate elements (older than timeout)
171
+ @@eviction_instance = nil
172
172
 
173
- # last time where eviction was launched
174
- @@last_eviction_timestamp = nil
173
+ # last time where eviction was launched
174
+ @@last_eviction_timestamp = nil
175
175
 
176
- # Initialize plugin
177
- public
178
- def register
179
- # process lambda expression to call in each filter call
180
- eval("@codeblock = lambda { |event, map| #{@code} }", binding, "(aggregate filter code)")
176
+ # Initialize plugin
177
+ public
178
+ def register
179
+ # process lambda expression to call in each filter call
180
+ eval("@codeblock = lambda { |event, map| #{@code} }", binding, "(aggregate filter code)")
181
181
 
182
- # define eviction_instance
183
- @@mutex.synchronize do
184
- if (@timeout > 0 && (@@eviction_instance.nil? || @timeout < @@eviction_instance.timeout))
185
- @@eviction_instance = self
186
- @logger.info("Aggregate, timeout: #{@timeout} seconds")
187
- end
188
- end
189
- end
182
+ # define eviction_instance
183
+ @@mutex.synchronize do
184
+ if (@timeout > 0 && (@@eviction_instance.nil? || @timeout < @@eviction_instance.timeout))
185
+ @@eviction_instance = self
186
+ @logger.info("Aggregate, timeout: #{@timeout} seconds")
187
+ end
188
+ end
189
+ end
190
190
 
191
-
192
- # This method is invoked each time an event matches the filter
193
- public
194
- def filter(event)
195
- # return nothing unless there's an actual filter event
196
-
191
+
192
+ # This method is invoked each time an event matches the filter
193
+ public
194
+ def filter(event)
197
195
 
198
- # define task id
199
- task_id = event.sprintf(@task_id)
200
- return if task_id.nil? || task_id.empty? || task_id == @task_id
196
+ # define task id
197
+ task_id = event.sprintf(@task_id)
198
+ return if task_id.nil? || task_id == @task_id
201
199
 
202
- noError = false
200
+ noError = false
203
201
 
204
- # protect aggregate_maps against concurrent access, using a mutex
205
- @@mutex.synchronize do
206
-
207
- # retrieve the current aggregate map
208
- aggregate_maps_element = @@aggregate_maps[task_id]
209
- if (aggregate_maps_element.nil?)
210
- return if @map_action == "update"
211
- aggregate_maps_element = LogStash::Filters::Aggregate::Element.new(Time.now);
212
- @@aggregate_maps[task_id] = aggregate_maps_element
213
- else
214
- return if @map_action == "create"
215
- end
216
- map = aggregate_maps_element.map
202
+ # protect aggregate_maps against concurrent access, using a mutex
203
+ @@mutex.synchronize do
204
+
205
+ # retrieve the current aggregate map
206
+ aggregate_maps_element = @@aggregate_maps[task_id]
207
+ if (aggregate_maps_element.nil?)
208
+ return if @map_action == "update"
209
+ aggregate_maps_element = LogStash::Filters::Aggregate::Element.new(Time.now);
210
+ @@aggregate_maps[task_id] = aggregate_maps_element
211
+ else
212
+ return if @map_action == "create"
213
+ end
214
+ map = aggregate_maps_element.map
217
215
 
218
- # execute the code to read/update map and event
219
- begin
220
- @codeblock.call(event, map)
221
- noError = true
222
- rescue => exception
223
- @logger.error("Aggregate exception occurred. Error: #{exception} ; Code: #{@code} ; Map: #{map} ; EventData: #{event.instance_variable_get('@data')}")
224
- event.tag("_aggregateexception")
225
- end
226
-
227
- # delete the map if task is ended
228
- @@aggregate_maps.delete(task_id) if @end_of_task
229
- end
216
+ # execute the code to read/update map and event
217
+ begin
218
+ @codeblock.call(event, map)
219
+ noError = true
220
+ rescue => exception
221
+ @logger.error("Aggregate exception occurred. Error: #{exception} ; Code: #{@code} ; Map: #{map} ; EventData: #{event.instance_variable_get('@data')}")
222
+ event.tag("_aggregateexception")
223
+ end
224
+
225
+ # delete the map if task is ended
226
+ @@aggregate_maps.delete(task_id) if @end_of_task
227
+ end
230
228
 
231
- # match the filter, only if no error occurred
232
- filter_matched(event) if noError
233
- end
229
+ # match the filter, only if no error occurred
230
+ filter_matched(event) if noError
231
+ end
234
232
 
235
- # Necessary to indicate logstash to periodically call 'flush' method
236
- def periodic_flush
237
- true
238
- end
233
+ # Necessary to indicate logstash to periodically call 'flush' method
234
+ def periodic_flush
235
+ true
236
+ end
239
237
 
240
- # This method is invoked by LogStash every 5 seconds.
241
- def flush(options = {})
242
- # Protection against no timeout defined by logstash conf : define a default eviction instance with timeout = DEFAULT_TIMEOUT seconds
243
- if (@@eviction_instance.nil?)
244
- @@eviction_instance = self
245
- @timeout = DEFAULT_TIMEOUT
246
- end
247
-
248
- # Launch eviction only every interval of (@timeout / 2) seconds
249
- if (@@eviction_instance == self && (@@last_eviction_timestamp.nil? || Time.now > @@last_eviction_timestamp + @timeout / 2))
250
- remove_expired_elements()
251
- @@last_eviction_timestamp = Time.now
252
- end
253
-
254
- return nil
255
- end
238
+ # This method is invoked by LogStash every 5 seconds.
239
+ def flush(options = {})
240
+ # Protection against no timeout defined by logstash conf : define a default eviction instance with timeout = DEFAULT_TIMEOUT seconds
241
+ if (@@eviction_instance.nil?)
242
+ @@eviction_instance = self
243
+ @timeout = DEFAULT_TIMEOUT
244
+ end
245
+
246
+ # Launch eviction only every interval of (@timeout / 2) seconds
247
+ if (@@eviction_instance == self && (@@last_eviction_timestamp.nil? || Time.now > @@last_eviction_timestamp + @timeout / 2))
248
+ remove_expired_elements()
249
+ @@last_eviction_timestamp = Time.now
250
+ end
251
+
252
+ return nil
253
+ end
256
254
 
257
-
258
- # Remove the expired Aggregate elements from "aggregate_maps" if they are older than timeout
259
- def remove_expired_elements()
260
- min_timestamp = Time.now - @timeout
261
- @@mutex.synchronize do
262
- @@aggregate_maps.delete_if { |key, element| element.creation_timestamp < min_timestamp }
263
- end
264
- end
255
+
256
+ # Remove the expired Aggregate elements from "aggregate_maps" if they are older than timeout
257
+ def remove_expired_elements()
258
+ min_timestamp = Time.now - @timeout
259
+ @@mutex.synchronize do
260
+ @@aggregate_maps.delete_if { |key, element| element.creation_timestamp < min_timestamp }
261
+ end
262
+ end
265
263
 
266
264
  end # class LogStash::Filters::Aggregate
267
265
 
268
266
  # Element of "aggregate_maps"
269
267
  class LogStash::Filters::Aggregate::Element
270
268
 
271
- attr_accessor :creation_timestamp, :map
269
+ attr_accessor :creation_timestamp, :map
272
270
 
273
- def initialize(creation_timestamp)
274
- @creation_timestamp = creation_timestamp
275
- @map = {}
276
- end
271
+ def initialize(creation_timestamp)
272
+ @creation_timestamp = creation_timestamp
273
+ @map = {}
274
+ end
277
275
  end
@@ -1,6 +1,6 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-filter-aggregate'
3
- s.version = '2.0.2'
3
+ s.version = '2.0.3'
4
4
  s.licenses = ['Apache License (2.0)']
5
5
  s.summary = "The aim of this filter is to aggregate information available among several events (typically log lines) belonging to a same task, and finally push aggregated information into final task event."
6
6
  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -19,6 +19,6 @@ Gem::Specification.new do |s|
19
19
  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
20
20
 
21
21
  # Gem dependencies
22
- s.add_runtime_dependency "logstash-core", ">= 2.0.0.beta2", "< 3.0.0"
22
+ s.add_runtime_dependency "logstash-core", ">= 2.0.0", "< 3.0.0"
23
23
  s.add_development_dependency 'logstash-devutils', '~> 0'
24
24
  end
@@ -5,173 +5,181 @@ require_relative "aggregate_spec_helper"
5
5
 
6
6
  describe LogStash::Filters::Aggregate do
7
7
 
8
- before(:each) do
9
- set_eviction_instance(nil)
10
- aggregate_maps.clear()
11
- @start_filter = setup_filter({ "map_action" => "create", "code" => "map['sql_duration'] = 0" })
12
- @update_filter = setup_filter({ "map_action" => "update", "code" => "map['sql_duration'] += event['duration']" })
13
- @end_filter = setup_filter({ "map_action" => "update", "code" => "event.to_hash.merge!(map)", "end_of_task" => true, "timeout" => 5 })
14
- end
15
-
16
- context "Start event" do
17
- describe "and receiving an event without task_id" do
18
- it "does not record it" do
19
- @start_filter.filter(event())
20
- expect(aggregate_maps).to be_empty
21
- end
22
- end
23
- describe "and receiving an event with task_id" do
24
- it "records it" do
25
- event = start_event("taskid" => "id123")
26
- @start_filter.filter(event)
27
-
28
- expect(aggregate_maps.size).to eq(1)
29
- expect(aggregate_maps["id123"]).not_to be_nil
30
- expect(aggregate_maps["id123"].creation_timestamp).to be >= event["@timestamp"]
31
- expect(aggregate_maps["id123"].map["sql_duration"]).to eq(0)
32
- end
33
- end
34
-
35
- describe "and receiving two 'start events' for the same task_id" do
36
- it "keeps the first one and does nothing with the second one" do
37
-
38
- first_start_event = start_event("taskid" => "id124")
39
- @start_filter.filter(first_start_event)
40
-
41
- first_update_event = update_event("taskid" => "id124", "duration" => 2)
42
- @update_filter.filter(first_update_event)
43
-
44
- sleep(1)
45
- second_start_event = start_event("taskid" => "id124")
46
- @start_filter.filter(second_start_event)
47
-
48
- expect(aggregate_maps.size).to eq(1)
49
- expect(aggregate_maps["id124"].creation_timestamp).to be < second_start_event["@timestamp"]
50
- expect(aggregate_maps["id124"].map["sql_duration"]).to eq(first_update_event["duration"])
51
- end
52
- end
53
- end
54
-
55
- context "End event" do
56
- describe "receiving an event without a previous 'start event'" do
57
- describe "but without a previous 'start event'" do
58
- it "does nothing with the event" do
59
- end_event = end_event("taskid" => "id124")
60
- @end_filter.filter(end_event)
61
-
62
- expect(aggregate_maps).to be_empty
63
- expect(end_event["sql_duration"]).to be_nil
64
- end
65
- end
66
- end
67
- end
68
-
69
- context "Start/end events interaction" do
70
- describe "receiving a 'start event'" do
71
- before(:each) do
72
- @task_id_value = "id_123"
73
- @start_event = start_event({"taskid" => @task_id_value})
74
- @start_filter.filter(@start_event)
75
- expect(aggregate_maps.size).to eq(1)
76
- end
77
-
78
- describe "and receiving an end event" do
79
- describe "and without an id" do
80
- it "does nothing" do
81
- end_event = end_event()
82
- @end_filter.filter(end_event)
83
- expect(aggregate_maps.size).to eq(1)
84
- expect(end_event["sql_duration"]).to be_nil
85
- end
86
- end
87
-
88
- describe "and an id different from the one of the 'start event'" do
89
- it "does nothing" do
90
- different_id_value = @task_id_value + "_different"
91
- @end_filter.filter(end_event("taskid" => different_id_value))
92
-
93
- expect(aggregate_maps.size).to eq(1)
94
- expect(aggregate_maps[@task_id_value]).not_to be_nil
95
- end
96
- end
97
-
98
- describe "and the same id of the 'start event'" do
99
- it "add 'sql_duration' field to the end event and deletes the aggregate map associated to taskid" do
100
- expect(aggregate_maps.size).to eq(1)
101
-
102
- @update_filter.filter(update_event("taskid" => @task_id_value, "duration" => 2))
103
-
104
- end_event = end_event("taskid" => @task_id_value)
105
- @end_filter.filter(end_event)
106
-
107
- expect(aggregate_maps).to be_empty
108
- expect(end_event["sql_duration"]).to eq(2)
109
- end
110
-
111
- end
112
- end
113
- end
114
- end
115
-
116
- context "Event which causes an exception when code call" do
117
- it "intercepts exception, logs the error and tags the event with '_aggregateexception'" do
118
- @start_filter = setup_filter({ "code" => "fail 'Test'" })
119
- start_event = start_event("taskid" => "id124")
120
- @start_filter.filter(start_event)
121
-
122
- expect(start_event["tags"]).to eq(["_aggregateexception"])
123
- end
124
- end
125
-
126
- context "flush call" do
127
- before(:each) do
128
- @end_filter.timeout = 1
129
- expect(@end_filter.timeout).to eq(1)
130
- @task_id_value = "id_123"
131
- @start_event = start_event({"taskid" => @task_id_value})
132
- @start_filter.filter(@start_event)
133
- expect(aggregate_maps.size).to eq(1)
134
- end
135
-
136
- describe "no timeout defined in none filter" do
137
- it "defines a default timeout on a default filter" do
138
- set_eviction_instance(nil)
139
- expect(eviction_instance).to be_nil
140
- @end_filter.flush()
141
- expect(eviction_instance).to eq(@end_filter)
142
- expect(@end_filter.timeout).to eq(LogStash::Filters::Aggregate::DEFAULT_TIMEOUT)
143
- end
144
- end
145
-
146
- describe "timeout is defined on another filter" do
147
- it "eviction_instance is not updated" do
148
- expect(eviction_instance).not_to be_nil
149
- @start_filter.flush()
150
- expect(eviction_instance).not_to eq(@start_filter)
151
- expect(eviction_instance).to eq(@end_filter)
152
- end
153
- end
154
-
155
- describe "no timeout defined on the filter" do
156
- it "event is not removed" do
157
- sleep(2)
158
- @start_filter.flush()
159
- expect(aggregate_maps.size).to eq(1)
160
- end
161
- end
162
-
163
- describe "timeout defined on the filter" do
164
- it "event is not removed if not expired" do
165
- @end_filter.flush()
166
- expect(aggregate_maps.size).to eq(1)
167
- end
168
- it "event is removed if expired" do
169
- sleep(2)
170
- @end_filter.flush()
171
- expect(aggregate_maps).to be_empty
172
- end
173
- end
174
-
175
- end
8
+ before(:each) do
9
+ set_eviction_instance(nil)
10
+ aggregate_maps.clear()
11
+ @start_filter = setup_filter({ "map_action" => "create", "code" => "map['sql_duration'] = 0" })
12
+ @update_filter = setup_filter({ "map_action" => "update", "code" => "map['sql_duration'] += event['duration']" })
13
+ @end_filter = setup_filter({ "map_action" => "update", "code" => "event.to_hash.merge!(map)", "end_of_task" => true, "timeout" => 5 })
14
+ end
15
+
16
+ context "Start event" do
17
+ describe "and receiving an event without task_id" do
18
+ it "does not record it" do
19
+ @start_filter.filter(event())
20
+ expect(aggregate_maps).to be_empty
21
+ end
22
+ end
23
+ describe "and receiving an event with task_id" do
24
+ it "records it" do
25
+ event = start_event("taskid" => "id123")
26
+ @start_filter.filter(event)
27
+
28
+ expect(aggregate_maps.size).to eq(1)
29
+ expect(aggregate_maps["id123"]).not_to be_nil
30
+ expect(aggregate_maps["id123"].creation_timestamp).to be >= event["@timestamp"]
31
+ expect(aggregate_maps["id123"].map["sql_duration"]).to eq(0)
32
+ end
33
+ end
34
+
35
+ describe "and receiving two 'start events' for the same task_id" do
36
+ it "keeps the first one and does nothing with the second one" do
37
+
38
+ first_start_event = start_event("taskid" => "id124")
39
+ @start_filter.filter(first_start_event)
40
+
41
+ first_update_event = update_event("taskid" => "id124", "duration" => 2)
42
+ @update_filter.filter(first_update_event)
43
+
44
+ sleep(1)
45
+ second_start_event = start_event("taskid" => "id124")
46
+ @start_filter.filter(second_start_event)
47
+
48
+ expect(aggregate_maps.size).to eq(1)
49
+ expect(aggregate_maps["id124"].creation_timestamp).to be < second_start_event["@timestamp"]
50
+ expect(aggregate_maps["id124"].map["sql_duration"]).to eq(first_update_event["duration"])
51
+ end
52
+ end
53
+ end
54
+
55
+ context "End event" do
56
+ describe "receiving an event without a previous 'start event'" do
57
+ describe "but without a previous 'start event'" do
58
+ it "does nothing with the event" do
59
+ end_event = end_event("taskid" => "id124")
60
+ @end_filter.filter(end_event)
61
+
62
+ expect(aggregate_maps).to be_empty
63
+ expect(end_event["sql_duration"]).to be_nil
64
+ end
65
+ end
66
+ end
67
+ end
68
+
69
+ context "Start/end events interaction" do
70
+ describe "receiving a 'start event'" do
71
+ before(:each) do
72
+ @task_id_value = "id_123"
73
+ @start_event = start_event({"taskid" => @task_id_value})
74
+ @start_filter.filter(@start_event)
75
+ expect(aggregate_maps.size).to eq(1)
76
+ end
77
+
78
+ describe "and receiving an end event" do
79
+ describe "and without an id" do
80
+ it "does nothing" do
81
+ end_event = end_event()
82
+ @end_filter.filter(end_event)
83
+ expect(aggregate_maps.size).to eq(1)
84
+ expect(end_event["sql_duration"]).to be_nil
85
+ end
86
+ end
87
+
88
+ describe "and an id different from the one of the 'start event'" do
89
+ it "does nothing" do
90
+ different_id_value = @task_id_value + "_different"
91
+ @end_filter.filter(end_event("taskid" => different_id_value))
92
+
93
+ expect(aggregate_maps.size).to eq(1)
94
+ expect(aggregate_maps[@task_id_value]).not_to be_nil
95
+ end
96
+ end
97
+
98
+ describe "and the same id of the 'start event'" do
99
+ it "add 'sql_duration' field to the end event and deletes the aggregate map associated to taskid" do
100
+ expect(aggregate_maps.size).to eq(1)
101
+
102
+ @update_filter.filter(update_event("taskid" => @task_id_value, "duration" => 2))
103
+
104
+ end_event = end_event("taskid" => @task_id_value)
105
+ @end_filter.filter(end_event)
106
+
107
+ expect(aggregate_maps).to be_empty
108
+ expect(end_event["sql_duration"]).to eq(2)
109
+ end
110
+
111
+ end
112
+ end
113
+ end
114
+ end
115
+
116
+ context "Event with integer task id" do
117
+ it "works as well as with a string task id" do
118
+ start_event = start_event("taskid" => 124)
119
+ @start_filter.filter(start_event)
120
+ expect(aggregate_maps.size).to eq(1)
121
+ end
122
+ end
123
+
124
+ context "Event which causes an exception when code call" do
125
+ it "intercepts exception, logs the error and tags the event with '_aggregateexception'" do
126
+ @start_filter = setup_filter({ "code" => "fail 'Test'" })
127
+ start_event = start_event("taskid" => "id124")
128
+ @start_filter.filter(start_event)
129
+
130
+ expect(start_event["tags"]).to eq(["_aggregateexception"])
131
+ end
132
+ end
133
+
134
+ context "flush call" do
135
+ before(:each) do
136
+ @end_filter.timeout = 1
137
+ expect(@end_filter.timeout).to eq(1)
138
+ @task_id_value = "id_123"
139
+ @start_event = start_event({"taskid" => @task_id_value})
140
+ @start_filter.filter(@start_event)
141
+ expect(aggregate_maps.size).to eq(1)
142
+ end
143
+
144
+ describe "no timeout defined in none filter" do
145
+ it "defines a default timeout on a default filter" do
146
+ set_eviction_instance(nil)
147
+ expect(eviction_instance).to be_nil
148
+ @end_filter.flush()
149
+ expect(eviction_instance).to eq(@end_filter)
150
+ expect(@end_filter.timeout).to eq(LogStash::Filters::Aggregate::DEFAULT_TIMEOUT)
151
+ end
152
+ end
153
+
154
+ describe "timeout is defined on another filter" do
155
+ it "eviction_instance is not updated" do
156
+ expect(eviction_instance).not_to be_nil
157
+ @start_filter.flush()
158
+ expect(eviction_instance).not_to eq(@start_filter)
159
+ expect(eviction_instance).to eq(@end_filter)
160
+ end
161
+ end
162
+
163
+ describe "no timeout defined on the filter" do
164
+ it "event is not removed" do
165
+ sleep(2)
166
+ @start_filter.flush()
167
+ expect(aggregate_maps.size).to eq(1)
168
+ end
169
+ end
170
+
171
+ describe "timeout defined on the filter" do
172
+ it "event is not removed if not expired" do
173
+ @end_filter.flush()
174
+ expect(aggregate_maps.size).to eq(1)
175
+ end
176
+ it "event is removed if expired" do
177
+ sleep(2)
178
+ @end_filter.flush()
179
+ expect(aggregate_maps).to be_empty
180
+ end
181
+ end
182
+
183
+ end
176
184
 
177
185
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-filter-aggregate
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.2
4
+ version: 2.0.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
@@ -9,58 +9,58 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2015-10-14 00:00:00.000000000 Z
12
+ date: 2015-11-03 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
- requirement: !ruby/object:Gem::Requirement
15
+ name: logstash-core
16
+ version_requirements: !ruby/object:Gem::Requirement
16
17
  requirements:
17
18
  - - '>='
18
19
  - !ruby/object:Gem::Version
19
- version: 2.0.0.beta2
20
+ version: 2.0.0
20
21
  - - <
21
22
  - !ruby/object:Gem::Version
22
23
  version: 3.0.0
23
- name: logstash-core
24
- prerelease: false
25
- type: :runtime
26
- version_requirements: !ruby/object:Gem::Requirement
24
+ requirement: !ruby/object:Gem::Requirement
27
25
  requirements:
28
26
  - - '>='
29
27
  - !ruby/object:Gem::Version
30
- version: 2.0.0.beta2
28
+ version: 2.0.0
31
29
  - - <
32
30
  - !ruby/object:Gem::Version
33
31
  version: 3.0.0
32
+ prerelease: false
33
+ type: :runtime
34
34
  - !ruby/object:Gem::Dependency
35
- requirement: !ruby/object:Gem::Requirement
35
+ name: logstash-devutils
36
+ version_requirements: !ruby/object:Gem::Requirement
36
37
  requirements:
37
38
  - - ~>
38
39
  - !ruby/object:Gem::Version
39
40
  version: '0'
40
- name: logstash-devutils
41
- prerelease: false
42
- type: :development
43
- version_requirements: !ruby/object:Gem::Requirement
41
+ requirement: !ruby/object:Gem::Requirement
44
42
  requirements:
45
43
  - - ~>
46
44
  - !ruby/object:Gem::Version
47
45
  version: '0'
46
+ prerelease: false
47
+ type: :development
48
48
  description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
49
49
  email: info@elastic.co
50
50
  executables: []
51
51
  extensions: []
52
52
  extra_rdoc_files: []
53
53
  files:
54
+ - lib/logstash/filters/aggregate.rb
55
+ - spec/filters/aggregate_spec.rb
56
+ - spec/filters/aggregate_spec_helper.rb
57
+ - logstash-filter-aggregate.gemspec
54
58
  - BUILD.md
55
59
  - CHANGELOG.md
60
+ - README.md
56
61
  - CONTRIBUTORS
57
62
  - Gemfile
58
63
  - LICENSE
59
- - README.md
60
- - lib/logstash/filters/aggregate.rb
61
- - logstash-filter-aggregate.gemspec
62
- - spec/filters/aggregate_spec.rb
63
- - spec/filters/aggregate_spec_helper.rb
64
64
  homepage: https://github.com/logstash-plugins/logstash-filter-aggregate
65
65
  licenses:
66
66
  - Apache License (2.0)
@@ -83,7 +83,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
83
83
  version: '0'
84
84
  requirements: []
85
85
  rubyforge_project:
86
- rubygems_version: 2.4.8
86
+ rubygems_version: 2.1.9
87
87
  signing_key:
88
88
  specification_version: 4
89
89
  summary: The aim of this filter is to aggregate information available among several events (typically log lines) belonging to a same task, and finally push aggregated information into final task event.