logstash-filter-aggregate 2.0.5 → 2.1.0

Sign up to get free protection for your applications and to get access to all the features.
@@ -1,24 +1,24 @@
1
- Gem::Specification.new do |s|
2
- s.name = 'logstash-filter-aggregate'
3
- s.version = '2.0.5'
4
- s.licenses = ['Apache License (2.0)']
5
- s.summary = "The aim of this filter is to aggregate information available among several events (typically log lines) belonging to a same task, and finally push aggregated information into final task event."
6
- s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
7
- s.authors = ["Elastic", "Fabien Baligand"]
8
- s.email = 'info@elastic.co'
9
- s.homepage = "https://github.com/logstash-plugins/logstash-filter-aggregate"
10
- s.require_paths = ["lib"]
11
-
12
- # Files
13
- s.files = Dir['lib/**/*','spec/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE']
14
-
15
- # Tests
16
- s.test_files = s.files.grep(%r{^(test|spec|features)/})
17
-
18
- # Special flag to let us know this is actually a logstash plugin
19
- s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
20
-
21
- # Gem dependencies
22
- s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0"
23
- s.add_development_dependency 'logstash-devutils', '~> 0'
24
- end
1
+ Gem::Specification.new do |s|
2
+ s.name = 'logstash-filter-aggregate'
3
+ s.version = '2.1.0'
4
+ s.licenses = ['Apache License (2.0)']
5
+ s.summary = "The aim of this filter is to aggregate information available among several events (typically log lines) belonging to a same task, and finally push aggregated information into final task event."
6
+ s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
7
+ s.authors = ["Elastic", "Fabien Baligand"]
8
+ s.email = 'info@elastic.co'
9
+ s.homepage = "https://github.com/logstash-plugins/logstash-filter-aggregate"
10
+ s.require_paths = ["lib"]
11
+
12
+ # Files
13
+ s.files = Dir['lib/**/*','spec/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE']
14
+
15
+ # Tests
16
+ s.test_files = s.files.grep(%r{^(test|spec|features)/})
17
+
18
+ # Special flag to let us know this is actually a logstash plugin
19
+ s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" }
20
+
21
+ # Gem dependencies
22
+ s.add_runtime_dependency "logstash-core-plugin-api", "~> 1.0"
23
+ s.add_development_dependency 'logstash-devutils', '~> 0'
24
+ end
@@ -1,185 +1,210 @@
1
- # encoding: utf-8
2
- require "logstash/devutils/rspec/spec_helper"
3
- require "logstash/filters/aggregate"
4
- require_relative "aggregate_spec_helper"
5
-
6
- describe LogStash::Filters::Aggregate do
7
-
8
- before(:each) do
9
- set_eviction_instance(nil)
10
- aggregate_maps.clear()
11
- @start_filter = setup_filter({ "map_action" => "create", "code" => "map['sql_duration'] = 0" })
12
- @update_filter = setup_filter({ "map_action" => "update", "code" => "map['sql_duration'] += event['duration']" })
13
- @end_filter = setup_filter({ "map_action" => "update", "code" => "event.to_hash.merge!(map)", "end_of_task" => true, "timeout" => 5 })
14
- end
15
-
16
- context "Start event" do
17
- describe "and receiving an event without task_id" do
18
- it "does not record it" do
19
- @start_filter.filter(event())
20
- expect(aggregate_maps).to be_empty
21
- end
22
- end
23
- describe "and receiving an event with task_id" do
24
- it "records it" do
25
- event = start_event("taskid" => "id123")
26
- @start_filter.filter(event)
27
-
28
- expect(aggregate_maps.size).to eq(1)
29
- expect(aggregate_maps["id123"]).not_to be_nil
30
- expect(aggregate_maps["id123"].creation_timestamp).to be >= event["@timestamp"]
31
- expect(aggregate_maps["id123"].map["sql_duration"]).to eq(0)
32
- end
33
- end
34
-
35
- describe "and receiving two 'start events' for the same task_id" do
36
- it "keeps the first one and does nothing with the second one" do
37
-
38
- first_start_event = start_event("taskid" => "id124")
39
- @start_filter.filter(first_start_event)
40
-
41
- first_update_event = update_event("taskid" => "id124", "duration" => 2)
42
- @update_filter.filter(first_update_event)
43
-
44
- sleep(1)
45
- second_start_event = start_event("taskid" => "id124")
46
- @start_filter.filter(second_start_event)
47
-
48
- expect(aggregate_maps.size).to eq(1)
49
- expect(aggregate_maps["id124"].creation_timestamp).to be < second_start_event["@timestamp"]
50
- expect(aggregate_maps["id124"].map["sql_duration"]).to eq(first_update_event["duration"])
51
- end
52
- end
53
- end
54
-
55
- context "End event" do
56
- describe "receiving an event without a previous 'start event'" do
57
- describe "but without a previous 'start event'" do
58
- it "does nothing with the event" do
59
- end_event = end_event("taskid" => "id124")
60
- @end_filter.filter(end_event)
61
-
62
- expect(aggregate_maps).to be_empty
63
- expect(end_event["sql_duration"]).to be_nil
64
- end
65
- end
66
- end
67
- end
68
-
69
- context "Start/end events interaction" do
70
- describe "receiving a 'start event'" do
71
- before(:each) do
72
- @task_id_value = "id_123"
73
- @start_event = start_event({"taskid" => @task_id_value})
74
- @start_filter.filter(@start_event)
75
- expect(aggregate_maps.size).to eq(1)
76
- end
77
-
78
- describe "and receiving an end event" do
79
- describe "and without an id" do
80
- it "does nothing" do
81
- end_event = end_event()
82
- @end_filter.filter(end_event)
83
- expect(aggregate_maps.size).to eq(1)
84
- expect(end_event["sql_duration"]).to be_nil
85
- end
86
- end
87
-
88
- describe "and an id different from the one of the 'start event'" do
89
- it "does nothing" do
90
- different_id_value = @task_id_value + "_different"
91
- @end_filter.filter(end_event("taskid" => different_id_value))
92
-
93
- expect(aggregate_maps.size).to eq(1)
94
- expect(aggregate_maps[@task_id_value]).not_to be_nil
95
- end
96
- end
97
-
98
- describe "and the same id of the 'start event'" do
99
- it "add 'sql_duration' field to the end event and deletes the aggregate map associated to taskid" do
100
- expect(aggregate_maps.size).to eq(1)
101
-
102
- @update_filter.filter(update_event("taskid" => @task_id_value, "duration" => 2))
103
-
104
- end_event = end_event("taskid" => @task_id_value)
105
- @end_filter.filter(end_event)
106
-
107
- expect(aggregate_maps).to be_empty
108
- expect(end_event["sql_duration"]).to eq(2)
109
- end
110
-
111
- end
112
- end
113
- end
114
- end
115
-
116
- context "Event with integer task id" do
117
- it "works as well as with a string task id" do
118
- start_event = start_event("taskid" => 124)
119
- @start_filter.filter(start_event)
120
- expect(aggregate_maps.size).to eq(1)
121
- end
122
- end
123
-
124
- context "Event which causes an exception when code call" do
125
- it "intercepts exception, logs the error and tags the event with '_aggregateexception'" do
126
- @start_filter = setup_filter({ "code" => "fail 'Test'" })
127
- start_event = start_event("taskid" => "id124")
128
- @start_filter.filter(start_event)
129
-
130
- expect(start_event["tags"]).to eq(["_aggregateexception"])
131
- end
132
- end
133
-
134
- context "flush call" do
135
- before(:each) do
136
- @end_filter.timeout = 1
137
- expect(@end_filter.timeout).to eq(1)
138
- @task_id_value = "id_123"
139
- @start_event = start_event({"taskid" => @task_id_value})
140
- @start_filter.filter(@start_event)
141
- expect(aggregate_maps.size).to eq(1)
142
- end
143
-
144
- describe "no timeout defined in none filter" do
145
- it "defines a default timeout on a default filter" do
146
- set_eviction_instance(nil)
147
- expect(eviction_instance).to be_nil
148
- @end_filter.flush()
149
- expect(eviction_instance).to eq(@end_filter)
150
- expect(@end_filter.timeout).to eq(LogStash::Filters::Aggregate::DEFAULT_TIMEOUT)
151
- end
152
- end
153
-
154
- describe "timeout is defined on another filter" do
155
- it "eviction_instance is not updated" do
156
- expect(eviction_instance).not_to be_nil
157
- @start_filter.flush()
158
- expect(eviction_instance).not_to eq(@start_filter)
159
- expect(eviction_instance).to eq(@end_filter)
160
- end
161
- end
162
-
163
- describe "no timeout defined on the filter" do
164
- it "event is not removed" do
165
- sleep(2)
166
- @start_filter.flush()
167
- expect(aggregate_maps.size).to eq(1)
168
- end
169
- end
170
-
171
- describe "timeout defined on the filter" do
172
- it "event is not removed if not expired" do
173
- @end_filter.flush()
174
- expect(aggregate_maps.size).to eq(1)
175
- end
176
- it "event is removed if expired" do
177
- sleep(2)
178
- @end_filter.flush()
179
- expect(aggregate_maps).to be_empty
180
- end
181
- end
182
-
183
- end
184
-
185
- end
1
+ # encoding: utf-8
2
+ require "logstash/devutils/rspec/spec_helper"
3
+ require "logstash/filters/aggregate"
4
+ require_relative "aggregate_spec_helper"
5
+
6
+ describe LogStash::Filters::Aggregate do
7
+
8
+ before(:each) do
9
+ set_eviction_instance(nil)
10
+ aggregate_maps.clear()
11
+ @start_filter = setup_filter({ "map_action" => "create", "code" => "map['sql_duration'] = 0" })
12
+ @update_filter = setup_filter({ "map_action" => "update", "code" => "map['sql_duration'] += event['duration']" })
13
+ @end_filter = setup_filter({ "map_action" => "update", "code" => "event.to_hash.merge!(map)", "end_of_task" => true, "timeout" => 5 })
14
+ end
15
+
16
+ context "Start event" do
17
+ describe "and receiving an event without task_id" do
18
+ it "does not record it" do
19
+ @start_filter.filter(event())
20
+ expect(aggregate_maps).to be_empty
21
+ end
22
+ end
23
+ describe "and receiving an event with task_id" do
24
+ it "records it" do
25
+ event = start_event("taskid" => "id123")
26
+ @start_filter.filter(event)
27
+
28
+ expect(aggregate_maps.size).to eq(1)
29
+ expect(aggregate_maps["id123"]).not_to be_nil
30
+ expect(aggregate_maps["id123"].creation_timestamp).to be >= event["@timestamp"]
31
+ expect(aggregate_maps["id123"].map["sql_duration"]).to eq(0)
32
+ end
33
+ end
34
+
35
+ describe "and receiving two 'start events' for the same task_id" do
36
+ it "keeps the first one and does nothing with the second one" do
37
+
38
+ first_start_event = start_event("taskid" => "id124")
39
+ @start_filter.filter(first_start_event)
40
+
41
+ first_update_event = update_event("taskid" => "id124", "duration" => 2)
42
+ @update_filter.filter(first_update_event)
43
+
44
+ sleep(1)
45
+ second_start_event = start_event("taskid" => "id124")
46
+ @start_filter.filter(second_start_event)
47
+
48
+ expect(aggregate_maps.size).to eq(1)
49
+ expect(aggregate_maps["id124"].creation_timestamp).to be < second_start_event["@timestamp"]
50
+ expect(aggregate_maps["id124"].map["sql_duration"]).to eq(first_update_event["duration"])
51
+ end
52
+ end
53
+ end
54
+
55
+ context "End event" do
56
+ describe "receiving an event without a previous 'start event'" do
57
+ describe "but without a previous 'start event'" do
58
+ it "does nothing with the event" do
59
+ end_event = end_event("taskid" => "id124")
60
+ @end_filter.filter(end_event)
61
+
62
+ expect(aggregate_maps).to be_empty
63
+ expect(end_event["sql_duration"]).to be_nil
64
+ end
65
+ end
66
+ end
67
+ end
68
+
69
+ context "Start/end events interaction" do
70
+ describe "receiving a 'start event'" do
71
+ before(:each) do
72
+ @task_id_value = "id_123"
73
+ @start_event = start_event({"taskid" => @task_id_value})
74
+ @start_filter.filter(@start_event)
75
+ expect(aggregate_maps.size).to eq(1)
76
+ end
77
+
78
+ describe "and receiving an end event" do
79
+ describe "and without an id" do
80
+ it "does nothing" do
81
+ end_event = end_event()
82
+ @end_filter.filter(end_event)
83
+ expect(aggregate_maps.size).to eq(1)
84
+ expect(end_event["sql_duration"]).to be_nil
85
+ end
86
+ end
87
+
88
+ describe "and an id different from the one of the 'start event'" do
89
+ it "does nothing" do
90
+ different_id_value = @task_id_value + "_different"
91
+ @end_filter.filter(end_event("taskid" => different_id_value))
92
+
93
+ expect(aggregate_maps.size).to eq(1)
94
+ expect(aggregate_maps[@task_id_value]).not_to be_nil
95
+ end
96
+ end
97
+
98
+ describe "and the same id of the 'start event'" do
99
+ it "add 'sql_duration' field to the end event and deletes the aggregate map associated to taskid" do
100
+ expect(aggregate_maps.size).to eq(1)
101
+
102
+ @update_filter.filter(update_event("taskid" => @task_id_value, "duration" => 2))
103
+
104
+ end_event = end_event("taskid" => @task_id_value)
105
+ @end_filter.filter(end_event)
106
+
107
+ expect(aggregate_maps).to be_empty
108
+ expect(end_event["sql_duration"]).to eq(2)
109
+ end
110
+
111
+ end
112
+ end
113
+ end
114
+ end
115
+
116
+ context "Event with integer task id" do
117
+ it "works as well as with a string task id" do
118
+ start_event = start_event("taskid" => 124)
119
+ @start_filter.filter(start_event)
120
+ expect(aggregate_maps.size).to eq(1)
121
+ end
122
+ end
123
+
124
+ context "Event which causes an exception when code call" do
125
+ it "intercepts exception, logs the error and tags the event with '_aggregateexception'" do
126
+ @start_filter = setup_filter({ "code" => "fail 'Test'" })
127
+ start_event = start_event("taskid" => "id124")
128
+ @start_filter.filter(start_event)
129
+
130
+ expect(start_event["tags"]).to eq(["_aggregateexception"])
131
+ end
132
+ end
133
+
134
+ context "flush call" do
135
+ before(:each) do
136
+ @end_filter.timeout = 1
137
+ expect(@end_filter.timeout).to eq(1)
138
+ @task_id_value = "id_123"
139
+ @start_event = start_event({"taskid" => @task_id_value})
140
+ @start_filter.filter(@start_event)
141
+ expect(aggregate_maps.size).to eq(1)
142
+ end
143
+
144
+ describe "no timeout defined in none filter" do
145
+ it "defines a default timeout on a default filter" do
146
+ set_eviction_instance(nil)
147
+ expect(eviction_instance).to be_nil
148
+ @end_filter.flush()
149
+ expect(eviction_instance).to eq(@end_filter)
150
+ expect(@end_filter.timeout).to eq(LogStash::Filters::Aggregate::DEFAULT_TIMEOUT)
151
+ end
152
+ end
153
+
154
+ describe "timeout is defined on another filter" do
155
+ it "eviction_instance is not updated" do
156
+ expect(eviction_instance).not_to be_nil
157
+ @start_filter.flush()
158
+ expect(eviction_instance).not_to eq(@start_filter)
159
+ expect(eviction_instance).to eq(@end_filter)
160
+ end
161
+ end
162
+
163
+ describe "no timeout defined on the filter" do
164
+ it "event is not removed" do
165
+ sleep(2)
166
+ @start_filter.flush()
167
+ expect(aggregate_maps.size).to eq(1)
168
+ end
169
+ end
170
+
171
+ describe "timeout defined on the filter" do
172
+ it "event is not removed if not expired" do
173
+ @end_filter.flush()
174
+ expect(aggregate_maps.size).to eq(1)
175
+ end
176
+ it "event is removed if expired" do
177
+ sleep(2)
178
+ @end_filter.flush()
179
+ expect(aggregate_maps).to be_empty
180
+ end
181
+ end
182
+
183
+ end
184
+
185
+ context "aggregate_maps_path option is defined, " do
186
+ describe "close event append then register event append, " do
187
+ it "stores aggregate maps to configured file and then loads aggregate maps from file" do
188
+
189
+ store_file = "aggregate_maps"
190
+ expect(File.exist?(store_file)).to be false
191
+
192
+ store_filter = setup_filter({ "code" => "map['sql_duration'] = 0", "aggregate_maps_path" => store_file })
193
+ expect(aggregate_maps).to be_empty
194
+
195
+ start_event = start_event("taskid" => 124)
196
+ filter = store_filter.filter(start_event)
197
+ expect(aggregate_maps.size).to eq(1)
198
+
199
+ store_filter.close()
200
+ expect(File.exist?(store_file)).to be true
201
+ expect(aggregate_maps).to be_empty
202
+
203
+ store_filter = setup_filter({ "code" => "map['sql_duration'] = 0", "aggregate_maps_path" => store_file })
204
+ expect(File.exist?(store_file)).to be false
205
+ expect(aggregate_maps.size).to eq(1)
206
+
207
+ end
208
+ end
209
+ end
210
+ end
@@ -1,49 +1,49 @@
1
- # encoding: utf-8
2
- require "logstash/filters/aggregate"
3
-
4
- def event(data = {})
5
- data["message"] ||= "Log message"
6
- data["@timestamp"] ||= Time.now
7
- LogStash::Event.new(data)
8
- end
9
-
10
- def start_event(data = {})
11
- data["logger"] = "TASK_START"
12
- event(data)
13
- end
14
-
15
- def update_event(data = {})
16
- data["logger"] = "SQL"
17
- event(data)
18
- end
19
-
20
- def end_event(data = {})
21
- data["logger"] = "TASK_END"
22
- event(data)
23
- end
24
-
25
- def setup_filter(config = {})
26
- config["task_id"] ||= "%{taskid}"
27
- filter = LogStash::Filters::Aggregate.new(config)
28
- filter.register()
29
- return filter
30
- end
31
-
32
- def filter(event)
33
- @start_filter.filter(event)
34
- @update_filter.filter(event)
35
- @end_filter.filter(event)
36
- end
37
-
38
- def aggregate_maps()
39
- LogStash::Filters::Aggregate.class_variable_get(:@@aggregate_maps)
40
- end
41
-
42
- def eviction_instance()
43
- LogStash::Filters::Aggregate.class_variable_get(:@@eviction_instance)
44
- end
45
-
46
- def set_eviction_instance(new_value)
47
- LogStash::Filters::Aggregate.class_variable_set(:@@eviction_instance, new_value)
48
- end
49
-
1
+ # encoding: utf-8
2
+ require "logstash/filters/aggregate"
3
+
4
+ def event(data = {})
5
+ data["message"] ||= "Log message"
6
+ data["@timestamp"] ||= Time.now
7
+ LogStash::Event.new(data)
8
+ end
9
+
10
+ def start_event(data = {})
11
+ data["logger"] = "TASK_START"
12
+ event(data)
13
+ end
14
+
15
+ def update_event(data = {})
16
+ data["logger"] = "SQL"
17
+ event(data)
18
+ end
19
+
20
+ def end_event(data = {})
21
+ data["logger"] = "TASK_END"
22
+ event(data)
23
+ end
24
+
25
+ def setup_filter(config = {})
26
+ config["task_id"] ||= "%{taskid}"
27
+ filter = LogStash::Filters::Aggregate.new(config)
28
+ filter.register()
29
+ return filter
30
+ end
31
+
32
+ def filter(event)
33
+ @start_filter.filter(event)
34
+ @update_filter.filter(event)
35
+ @end_filter.filter(event)
36
+ end
37
+
38
+ def aggregate_maps()
39
+ LogStash::Filters::Aggregate.class_variable_get(:@@aggregate_maps)
40
+ end
41
+
42
+ def eviction_instance()
43
+ LogStash::Filters::Aggregate.class_variable_get(:@@eviction_instance)
44
+ end
45
+
46
+ def set_eviction_instance(new_value)
47
+ LogStash::Filters::Aggregate.class_variable_set(:@@eviction_instance, new_value)
48
+ end
49
+