logstash-filter-foreach 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/lib/logstash/filters/foreach.rb +17 -8
- data/logstash-filter-foreach.gemspec +1 -1
- data/spec/filters/foreach_spec.rb +128 -0
- metadata +27 -27
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: '060786427644bdb8d829fd5f6f44adca1b1f7a48a3b0cea726cc1b3a92c71f1d'
|
4
|
+
data.tar.gz: 78ebd5c7a394f5ffa1ebbe7a3bc84506f4f1d25cca0f1dca1b1b1eea3937bbb6
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 82d2d3c2b1f61085f19e082e2131ef42181608eb711c92e456814d5665def3ee01948f00ff167ce5cd8953c19e97b6e85dfa5810116009e7967f4a76d8c167b2
|
7
|
+
data.tar.gz: 7c3c7960e299d4b3ece6b22bc80520b7eee44cb949ff7ef3d04ea965da26e7fdf10f6a20ffffc53b4d9cf3a29452f3d7907afe1230bcfbb48a09c5af53990f65
|
@@ -73,7 +73,7 @@ class LogStash::Filters::Foreach < LogStash::Filters::Base
|
|
73
73
|
public
|
74
74
|
def filter(event)
|
75
75
|
|
76
|
-
@logger.debug("Foreach plugin:", :task_id => @task_id, :array_field => @array_field, :join_fields => @join_fields, :end => @end, :timeout => @timeout, :event => event.to_hash)
|
76
|
+
@logger.debug("Foreach plugin:", :task_id => @task_id, :array_field => @array_field, :join_fields => @join_fields, :end => @end, :timeout => @timeout, :event => event.to_hash, :metadata => event.get('@metadata'))
|
77
77
|
|
78
78
|
passthrough = false
|
79
79
|
|
@@ -127,7 +127,7 @@ class LogStash::Filters::Foreach < LogStash::Filters::Base
|
|
127
127
|
|
128
128
|
@logger.trace("Foreach plugin: else !array_field.is_a?(Array)");
|
129
129
|
|
130
|
-
@@event_data[task_id] = LogStash::Filters::Foreach::Element.new(configuration, Time.now(), event.clone, configuration.join_fields)
|
130
|
+
@@event_data[task_id] = LogStash::Filters::Foreach::Element.new(configuration, Time.now(), event.clone, event.get('@metadata').clone, configuration.join_fields)
|
131
131
|
event_data = @@event_data[task_id]
|
132
132
|
|
133
133
|
if array_field.length == 0
|
@@ -177,8 +177,9 @@ class LogStash::Filters::Foreach < LogStash::Filters::Base
|
|
177
177
|
|
178
178
|
if event_data.sub_events_count == 0
|
179
179
|
|
180
|
-
|
181
|
-
|
180
|
+
ret_event = event_data.event()
|
181
|
+
filter_matched(ret_event)
|
182
|
+
yield ret_event
|
182
183
|
@@event_data.delete(task_id)
|
183
184
|
|
184
185
|
else
|
@@ -197,8 +198,9 @@ class LogStash::Filters::Foreach < LogStash::Filters::Base
|
|
197
198
|
configuration.join_fields.each do |join_field|
|
198
199
|
event_data.initial_event.set(join_field, event_data.join_fields[join_field])
|
199
200
|
end
|
200
|
-
|
201
|
-
|
201
|
+
ret_event = event_data.event()
|
202
|
+
filter_matched(ret_event)
|
203
|
+
yield ret_event
|
202
204
|
@@event_data.delete(task_id)
|
203
205
|
end
|
204
206
|
|
@@ -273,13 +275,14 @@ end
|
|
273
275
|
|
274
276
|
class LogStash::Filters::Foreach::Element
|
275
277
|
|
276
|
-
attr_accessor :initial_event, :counter, :sub_events_count, :join_fields, :lastevent_timestamp, :configuration
|
278
|
+
attr_accessor :initial_event, :initial_metadata, :counter, :sub_events_count, :join_fields, :lastevent_timestamp, :configuration
|
277
279
|
|
278
|
-
def initialize(configuration, creation_timestamp, event, join_fields)
|
280
|
+
def initialize(configuration, creation_timestamp, event, metadata, join_fields)
|
279
281
|
# @creation_timestamp = creation_timestamp
|
280
282
|
@configuration = configuration
|
281
283
|
@lastevent_timestamp = creation_timestamp
|
282
284
|
@initial_event = event
|
285
|
+
@initial_metadata = metadata
|
283
286
|
@counter = 0
|
284
287
|
@sub_events_count = event.get(configuration.array_field).length
|
285
288
|
@join_fields = {}
|
@@ -287,4 +290,10 @@ class LogStash::Filters::Foreach::Element
|
|
287
290
|
@join_fields[join_field] = []
|
288
291
|
end
|
289
292
|
end
|
293
|
+
|
294
|
+
def event()
|
295
|
+
e = @initial_event.clone
|
296
|
+
e.set('@metadata', @initial_metadata)
|
297
|
+
return e
|
298
|
+
end
|
290
299
|
end
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-filter-foreach'
|
3
|
-
s.version = '0.
|
3
|
+
s.version = '0.2.0'
|
4
4
|
s.licenses = ['Apache-2.0']
|
5
5
|
s.summary = 'Process filters for every array item'
|
6
6
|
s.description = 'Plugin splits event for every item in array, then you could process other filters for every item and then join event back'
|
@@ -153,6 +153,72 @@ describe LogStash::Filters::Foreach do
|
|
153
153
|
end
|
154
154
|
end
|
155
155
|
|
156
|
+
describe "should split and join correctly with nested field" do
|
157
|
+
let(:config) do
|
158
|
+
<<-CONFIG
|
159
|
+
filter {
|
160
|
+
foreach {
|
161
|
+
task_id => "%{task_id}"
|
162
|
+
array_field => "[array][nested]"
|
163
|
+
join_fields => ["join"]
|
164
|
+
}
|
165
|
+
|
166
|
+
mutate {
|
167
|
+
add_field => { "join" => "%{[array][nested]}_changed" }
|
168
|
+
}
|
169
|
+
|
170
|
+
foreach {
|
171
|
+
task_id => "%{task_id}"
|
172
|
+
end => true
|
173
|
+
}
|
174
|
+
}
|
175
|
+
CONFIG
|
176
|
+
end
|
177
|
+
|
178
|
+
sample("task_id" => 1, "array" => {"nested" => ["big", "bird", "sesame street"]}, "unchanged" => "unchanged_value") do
|
179
|
+
insist { subject.is_a?(LogStash::Event) } == true
|
180
|
+
insist { subject.get("[array][nested]").is_a?(Array) } == true
|
181
|
+
insist { subject.get("[array][nested]") } == ["big", "bird", "sesame street"]
|
182
|
+
insist { subject.get("join").is_a?(Array) } == true
|
183
|
+
insist { subject.get("join") } == ["big_changed", "bird_changed", "sesame street_changed"]
|
184
|
+
insist { subject.get("unchanged").is_a?(String) } == true
|
185
|
+
insist { subject.get("unchanged") } == "unchanged_value"
|
186
|
+
end
|
187
|
+
end
|
188
|
+
|
189
|
+
describe "should split and join correctly with array joins" do
|
190
|
+
let(:config) do
|
191
|
+
<<-CONFIG
|
192
|
+
filter {
|
193
|
+
foreach {
|
194
|
+
task_id => "%{task_id}"
|
195
|
+
array_field => "array"
|
196
|
+
join_fields => ["join"]
|
197
|
+
}
|
198
|
+
|
199
|
+
mutate {
|
200
|
+
add_field => { "join" => ["%{array}_changed", "%{array}_changed2"] }
|
201
|
+
}
|
202
|
+
|
203
|
+
foreach {
|
204
|
+
task_id => "%{task_id}"
|
205
|
+
end => true
|
206
|
+
}
|
207
|
+
}
|
208
|
+
CONFIG
|
209
|
+
end
|
210
|
+
|
211
|
+
sample("task_id" => 1, "array" => ["big", "bird", "sesame street"], "unchanged" => "unchanged_value") do
|
212
|
+
insist { subject.is_a?(LogStash::Event) } == true
|
213
|
+
insist { subject.get("array").is_a?(Array) } == true
|
214
|
+
insist { subject.get("array") } == ["big", "bird", "sesame street"]
|
215
|
+
insist { subject.get("join").is_a?(Array) } == true
|
216
|
+
insist { subject.get("join") } == ["big_changed", "big_changed2", "bird_changed", "bird_changed2", "sesame street_changed", "sesame street_changed2"]
|
217
|
+
insist { subject.get("unchanged").is_a?(String) } == true
|
218
|
+
insist { subject.get("unchanged") } == "unchanged_value"
|
219
|
+
end
|
220
|
+
end
|
221
|
+
|
156
222
|
describe "should passthrough event with incorrect task_id" do
|
157
223
|
let(:config) do
|
158
224
|
<<-CONFIG
|
@@ -364,5 +430,67 @@ describe LogStash::Filters::Foreach do
|
|
364
430
|
end
|
365
431
|
|
366
432
|
end
|
433
|
+
|
434
|
+
describe "should work with @metadata fields" do
|
435
|
+
let(:config) do
|
436
|
+
<<-CONFIG
|
437
|
+
filter {
|
438
|
+
foreach {
|
439
|
+
task_id => "%{task_id}"
|
440
|
+
array_field => "array"
|
441
|
+
join_fields => ["join"]
|
442
|
+
}
|
443
|
+
|
444
|
+
mutate {
|
445
|
+
add_field => { "join" => "%{array}_changed" }
|
446
|
+
}
|
447
|
+
|
448
|
+
foreach {
|
449
|
+
task_id => "%{task_id}"
|
450
|
+
end => true
|
451
|
+
}
|
452
|
+
}
|
453
|
+
CONFIG
|
454
|
+
end
|
455
|
+
|
456
|
+
sample("task_id" => 1, "array" => ["big", "bird", "sesame street"], "@metadata" => {"unchanged" => "unchanged_value"}) do
|
457
|
+
insist { subject.is_a?(LogStash::Event) } == true
|
458
|
+
insist { subject.get("array").is_a?(Array) } == true
|
459
|
+
insist { subject.get("array") } == ["big", "bird", "sesame street"]
|
460
|
+
insist { subject.get("join").is_a?(Array) } == true
|
461
|
+
insist { subject.get("join") } == ["big_changed", "bird_changed", "sesame street_changed"]
|
462
|
+
insist { subject.get("@metadata").is_a?(Object) } == true
|
463
|
+
insist { subject.get("[@metadata][unchanged]") } == "unchanged_value"
|
464
|
+
end
|
465
|
+
end
|
466
|
+
|
467
|
+
describe "should pass @metadata fields" do
|
468
|
+
let(:config) do
|
469
|
+
<<-CONFIG
|
470
|
+
filter {
|
471
|
+
foreach {
|
472
|
+
task_id => "%{task_id}"
|
473
|
+
array_field => "array"
|
474
|
+
join_fields => ["join"]
|
475
|
+
}
|
476
|
+
|
477
|
+
mutate {
|
478
|
+
add_field => { "join" => "%{array}_changed" }
|
479
|
+
}
|
480
|
+
|
481
|
+
foreach {
|
482
|
+
task_id => "%{task_id}"
|
483
|
+
end => true
|
484
|
+
}
|
485
|
+
}
|
486
|
+
CONFIG
|
487
|
+
end
|
488
|
+
|
489
|
+
sample("task_id" => 1, "@metadata" => {"unchanged" => "unchanged_value"}) do
|
490
|
+
insist { subject.is_a?(LogStash::Event) } == true
|
491
|
+
insist { subject.get("@metadata").is_a?(Object) } == true
|
492
|
+
insist { subject.get("[@metadata][unchanged]") } == "unchanged_value"
|
493
|
+
end
|
494
|
+
end
|
367
495
|
end
|
368
496
|
end
|
metadata
CHANGED
@@ -1,75 +1,75 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-filter-foreach
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- IIIEII
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-08-
|
11
|
+
date: 2017-08-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name: logstash-core-plugin-api
|
15
14
|
requirement: !ruby/object:Gem::Requirement
|
16
15
|
requirements:
|
17
|
-
- - ~>
|
16
|
+
- - "~>"
|
18
17
|
- !ruby/object:Gem::Version
|
19
18
|
version: '2.0'
|
20
|
-
|
19
|
+
name: logstash-core-plugin-api
|
21
20
|
prerelease: false
|
21
|
+
type: :runtime
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- - ~>
|
24
|
+
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '2.0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
|
-
name: logstash-filter-mutate
|
29
28
|
requirement: !ruby/object:Gem::Requirement
|
30
29
|
requirements:
|
31
|
-
- -
|
30
|
+
- - ">="
|
32
31
|
- !ruby/object:Gem::Version
|
33
32
|
version: '0'
|
34
|
-
|
33
|
+
name: logstash-filter-mutate
|
35
34
|
prerelease: false
|
35
|
+
type: :development
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- -
|
38
|
+
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
|
-
name: logstash-filter-drop
|
43
42
|
requirement: !ruby/object:Gem::Requirement
|
44
43
|
requirements:
|
45
|
-
- -
|
44
|
+
- - ">="
|
46
45
|
- !ruby/object:Gem::Version
|
47
46
|
version: '0'
|
48
|
-
|
47
|
+
name: logstash-filter-drop
|
49
48
|
prerelease: false
|
49
|
+
type: :development
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- -
|
52
|
+
- - ">="
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '0'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
|
-
name: logstash-devutils
|
57
56
|
requirement: !ruby/object:Gem::Requirement
|
58
57
|
requirements:
|
59
|
-
- - ~>
|
58
|
+
- - "~>"
|
60
59
|
- !ruby/object:Gem::Version
|
61
60
|
version: '1.3'
|
62
|
-
- -
|
61
|
+
- - ">="
|
63
62
|
- !ruby/object:Gem::Version
|
64
63
|
version: 1.3.1
|
65
|
-
|
64
|
+
name: logstash-devutils
|
66
65
|
prerelease: false
|
66
|
+
type: :development
|
67
67
|
version_requirements: !ruby/object:Gem::Requirement
|
68
68
|
requirements:
|
69
|
-
- - ~>
|
69
|
+
- - "~>"
|
70
70
|
- !ruby/object:Gem::Version
|
71
71
|
version: '1.3'
|
72
|
-
- -
|
72
|
+
- - ">="
|
73
73
|
- !ruby/object:Gem::Version
|
74
74
|
version: 1.3.1
|
75
75
|
description: Plugin splits event for every item in array, then you could process other
|
@@ -95,24 +95,24 @@ licenses:
|
|
95
95
|
metadata:
|
96
96
|
logstash_plugin: 'true'
|
97
97
|
logstash_group: filter
|
98
|
-
post_install_message:
|
98
|
+
post_install_message:
|
99
99
|
rdoc_options: []
|
100
100
|
require_paths:
|
101
101
|
- lib
|
102
102
|
required_ruby_version: !ruby/object:Gem::Requirement
|
103
103
|
requirements:
|
104
|
-
- -
|
104
|
+
- - ">="
|
105
105
|
- !ruby/object:Gem::Version
|
106
106
|
version: '0'
|
107
107
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
108
108
|
requirements:
|
109
|
-
- -
|
109
|
+
- - ">="
|
110
110
|
- !ruby/object:Gem::Version
|
111
111
|
version: '0'
|
112
112
|
requirements: []
|
113
|
-
rubyforge_project:
|
114
|
-
rubygems_version: 2.
|
115
|
-
signing_key:
|
113
|
+
rubyforge_project:
|
114
|
+
rubygems_version: 2.6.11
|
115
|
+
signing_key:
|
116
116
|
specification_version: 4
|
117
117
|
summary: Process filters for every array item
|
118
118
|
test_files:
|