logstash-filter-prune 3.0.3 → 3.0.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +6 -0
- data/LICENSE +1 -1
- data/docs/index.asciidoc +4 -3
- data/lib/logstash/filters/prune.rb +1 -1
- data/logstash-filter-prune.gemspec +1 -1
- data/spec/filters/prune_spec.rb +214 -373
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 28e7aa1f366437289ce58f0efe100c88d183e700cbd31c4410281e86297a652b
|
4
|
+
data.tar.gz: c1694004e92a1fdb785f5c097c0a5d32be6635fdf14d7babb839f82a2fd0e6cc
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c94b68626810f30c9064b5a0adb06d18cd7453c61bb0bb6ed1e3ba0d7c856311a52cc69825224b12fecc1422f0412b94d8dee47d7277b83ce81c0f7a7f3d6913
|
7
|
+
data.tar.gz: f06d4af667368912eed28d90f38a49c928036fe5e41ad8c1c2771b6f2f7f571b033cde42fc0a26a7bbc310ac11c379b1a45bf708ab8b77a9e52eb1f8e2e23c00
|
data/CHANGELOG.md
CHANGED
@@ -1,3 +1,9 @@
|
|
1
|
+
# 3.0.4
|
2
|
+
- Fixed regex to prevent Exception in sprintf call [#25](https://github.com/logstash-plugins/logstash-filter-prune/pull/25)
|
3
|
+
- Changed testing to docker [#27](https://github.com/logstash-plugins/logstash-filter-prune/pull/27)
|
4
|
+
- Added clarification in docs around whitelist_values
|
5
|
+
- Changed tests from insist to expect
|
6
|
+
|
1
7
|
## 3.0.3
|
2
8
|
- Update gemspec summary
|
3
9
|
|
data/LICENSE
CHANGED
data/docs/index.asciidoc
CHANGED
@@ -24,8 +24,8 @@ The prune filter is for removing fields from events based on
|
|
24
24
|
whitelists or blacklist of field names or their values (names and
|
25
25
|
values can also be regular expressions).
|
26
26
|
|
27
|
-
This can e.g. be useful if you have a
|
28
|
-
or
|
27
|
+
This can e.g. be useful if you have a {logstash-ref}/plugins-filters-json.html[json]
|
28
|
+
or {logstash-ref}/plugins-filters-kv.html[kv] filter that creates a number of fields
|
29
29
|
with names that you don't necessarily know the names of beforehand,
|
30
30
|
and you only want to keep a subset of them.
|
31
31
|
|
@@ -139,6 +139,7 @@ Include only fields only if their names match specified regexps, default to empt
|
|
139
139
|
|
140
140
|
Include specified fields only if their values match one of the supplied regular expressions.
|
141
141
|
In case field values are arrays, each array item is matched against the regular expressions and only matching array items will be included.
|
142
|
+
By default all fields that are not listed in this setting are kept unless pruned by other settings.
|
142
143
|
[source,ruby]
|
143
144
|
filter {
|
144
145
|
prune {
|
@@ -151,4 +152,4 @@ In case field values are arrays, each array item is matched against the regular
|
|
151
152
|
|
152
153
|
|
153
154
|
[id="plugins-{type}s-{plugin}-common-options"]
|
154
|
-
include::{include_path}/{type}.asciidoc[]
|
155
|
+
include::{include_path}/{type}.asciidoc[]
|
@@ -63,7 +63,7 @@ class LogStash::Filters::Prune < LogStash::Filters::Base
|
|
63
63
|
# blacklist_names => [ "method", "(referrer|status)", "${some}_field" ]
|
64
64
|
# }
|
65
65
|
# }
|
66
|
-
config :blacklist_names, :validate => :array, :default => [ "
|
66
|
+
config :blacklist_names, :validate => :array, :default => [ "%\\{[^}]+\\}" ]
|
67
67
|
|
68
68
|
# Include specified fields only if their values match one of the supplied regular expressions.
|
69
69
|
# In case field values are arrays, each array item is matched against the regular expressions and only matching array items will be included.
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-filter-prune'
|
4
|
-
s.version = '3.0.
|
4
|
+
s.version = '3.0.4'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "Prunes event data based on a list of fields to blacklist or whitelist"
|
7
7
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
data/spec/filters/prune_spec.rb
CHANGED
@@ -6,18 +6,11 @@ require "logstash/filters/prune"
|
|
6
6
|
#
|
7
7
|
# See the 'whitelist field values with interpolation' test for a commented
|
8
8
|
# explanation of my confusion.
|
9
|
-
describe LogStash::Filters::Prune
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
config <<-CONFIG
|
15
|
-
filter {
|
16
|
-
prune { }
|
17
|
-
}
|
18
|
-
CONFIG
|
19
|
-
|
20
|
-
sample(
|
9
|
+
describe LogStash::Filters::Prune do
|
10
|
+
subject { described_class.new(config) }
|
11
|
+
let(:config) { {} }
|
12
|
+
let(:event_data) do
|
13
|
+
{
|
21
14
|
"firstname" => "Borat",
|
22
15
|
"lastname" => "Sagdiyev",
|
23
16
|
"fullname" => "Borat Sagdiyev",
|
@@ -26,416 +19,264 @@ describe LogStash::Filters::Prune, :if => false do
|
|
26
19
|
"hobby" => "Cloud",
|
27
20
|
"status" => "200",
|
28
21
|
"Borat_saying" => "Cloud is not ready for enterprise if is not integrate with single server running Active Directory.",
|
29
|
-
|
30
|
-
) do
|
31
|
-
insist { subject.get("firstname") } == "Borat"
|
32
|
-
insist { subject.get("lastname") } == "Sagdiyev"
|
33
|
-
insist { subject.get("fullname") } == "Borat Sagdiyev"
|
34
|
-
insist { subject.get("country") } == "Kazakhstan"
|
35
|
-
insist { subject.get("location") } == "Somethere in Kazakhstan"
|
36
|
-
insist { subject.get("hobby") } == "Cloud"
|
37
|
-
insist { subject.get("status") } == "200"
|
38
|
-
insist { subject.get("Borat_saying") } == "Cloud is not ready for enterprise if is not integrate with single server running Active Directory."
|
39
|
-
insist { subject.get("%{hmm}") } == nil
|
40
|
-
end
|
22
|
+
}
|
41
23
|
end
|
42
24
|
|
43
|
-
|
25
|
+
let(:event) { LogStash::Event.new(event_data) }
|
44
26
|
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
}
|
50
|
-
}
|
51
|
-
CONFIG
|
27
|
+
before(:each) do
|
28
|
+
subject.register
|
29
|
+
subject.filter(event)
|
30
|
+
end
|
52
31
|
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
"
|
59
|
-
"
|
60
|
-
|
61
|
-
|
62
|
-
"%{hmm}" => "doh"
|
63
|
-
) do
|
64
|
-
insist { subject.get("firstname") } == "Borat"
|
65
|
-
insist { subject.get("lastname") } == nil
|
66
|
-
insist { subject.get("fullname") } == nil
|
67
|
-
insist { subject.get("country") } == nil
|
68
|
-
insist { subject.get("location") } == nil
|
69
|
-
insist { subject.get("hobby") } == "Cloud"
|
70
|
-
insist { subject.get("status") } == "200"
|
71
|
-
insist { subject.get("Borat_saying") } == nil
|
72
|
-
insist { subject.get("%{hmm}") } == nil
|
32
|
+
describe "default behaviour" do
|
33
|
+
it "retains all fields since whiteliste_names is empty" do
|
34
|
+
expect(event.to_hash.keys).to include(*event_data.keys)
|
35
|
+
end
|
36
|
+
describe "blacklist_names" do
|
37
|
+
let(:event_data) { super.merge("%{hmm}" => "doh") }
|
38
|
+
it "drops unresolved field references" do
|
39
|
+
expect(event.get("%{hmm}")).to be_nil
|
40
|
+
end
|
73
41
|
end
|
74
42
|
end
|
75
43
|
|
76
|
-
describe "
|
44
|
+
describe "whitelist_names" do
|
77
45
|
|
78
|
-
config
|
79
|
-
|
80
|
-
prune {
|
81
|
-
whitelist_names => [ "firstname", "(hobby|status)", "%{firstname}_saying" ]
|
82
|
-
interpolate => true
|
83
|
-
}
|
84
|
-
}
|
85
|
-
CONFIG
|
86
|
-
|
87
|
-
sample(
|
88
|
-
"firstname" => "Borat",
|
89
|
-
"lastname" => "Sagdiyev",
|
90
|
-
"fullname" => "Borat Sagdiyev",
|
91
|
-
"country" => "Kazakhstan",
|
92
|
-
"location" => "Somethere in Kazakhstan",
|
93
|
-
"hobby" => "Cloud",
|
94
|
-
"status" => "200",
|
95
|
-
"Borat_saying" => "Cloud is not ready for enterprise if is not integrate with single server running Active Directory.",
|
96
|
-
"%{hmm}" => "doh"
|
97
|
-
) do
|
98
|
-
insist { subject.get("firstname") } == "Borat"
|
99
|
-
insist { subject.get("lastname") } == nil
|
100
|
-
insist { subject.get("fullname") } == nil
|
101
|
-
insist { subject.get("country") } == nil
|
102
|
-
insist { subject.get("location") } == nil
|
103
|
-
insist { subject.get("hobby") } == "Cloud"
|
104
|
-
insist { subject.get("status") } == "200"
|
105
|
-
insist { subject.get("Borat_saying") } == "Cloud is not ready for enterprise if is not integrate with single server running Active Directory."
|
106
|
-
insist { subject.get("%{hmm}") } == nil
|
46
|
+
let(:config) do
|
47
|
+
{ "whitelist_names" => [ "firstname", "(hobby|status)", "%{firstname}_saying" ] }
|
107
48
|
end
|
108
|
-
end
|
109
49
|
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
blacklist_names => [ "firstname", "(hobby|status)", "%{firstname}_saying" ]
|
116
|
-
}
|
117
|
-
}
|
118
|
-
CONFIG
|
50
|
+
it "keeps fields in the list" do
|
51
|
+
expect(event.get("firstname")).to eq("Borat")
|
52
|
+
expect(event.get("hobby")).to eq("Cloud")
|
53
|
+
expect(event.get("status")).to eq("200")
|
54
|
+
end
|
119
55
|
|
120
|
-
|
121
|
-
"
|
122
|
-
"
|
123
|
-
"
|
124
|
-
"
|
125
|
-
"
|
126
|
-
"
|
127
|
-
"status" => "200",
|
128
|
-
"Borat_saying" => "Cloud is not ready for enterprise if is not integrate with single server running Active Directory.",
|
129
|
-
"%{hmm}" => "doh"
|
130
|
-
) do
|
131
|
-
insist { subject.get("firstname") } == nil
|
132
|
-
insist { subject.get("lastname") } == "Sagdiyev"
|
133
|
-
insist { subject.get("fullname") } == "Borat Sagdiyev"
|
134
|
-
insist { subject.get("country") } == "Kazakhstan"
|
135
|
-
insist { subject.get("location") } == "Somethere in Kazakhstan"
|
136
|
-
insist { subject.get("hobby") } == nil
|
137
|
-
insist { subject.get("status") } == nil
|
138
|
-
insist { subject.get("Borat_saying") } == "Cloud is not ready for enterprise if is not integrate with single server running Active Directory."
|
139
|
-
insist { subject.get("%{hmm}") } == "doh"
|
56
|
+
it "drops fields not described in the whitelist" do
|
57
|
+
expect(event.get("lastname")).to be_nil
|
58
|
+
expect(event.get("fullname")).to be_nil
|
59
|
+
expect(event.get("country")).to be_nil
|
60
|
+
expect(event.get("location")).to be_nil
|
61
|
+
expect(event.get("Borat_saying")).to be_nil
|
62
|
+
expect(event.get("%{hmm}")).to be_nil
|
140
63
|
end
|
141
|
-
end
|
142
64
|
|
143
|
-
|
65
|
+
context "with interpolation" do
|
144
66
|
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
interpolate => true
|
67
|
+
let(:config) do
|
68
|
+
{
|
69
|
+
"whitelist_names" => [ "firstname", "%{firstname}_saying" ],
|
70
|
+
"interpolate" => true
|
150
71
|
}
|
151
|
-
|
152
|
-
CONFIG
|
72
|
+
end
|
153
73
|
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
"country" => "Kazakhstan",
|
159
|
-
"location" => "Somethere in Kazakhstan",
|
160
|
-
"hobby" => "Cloud",
|
161
|
-
"status" => "200",
|
162
|
-
"Borat_saying" => "Cloud is not ready for enterprise if is not integrate with single server running Active Directory.",
|
163
|
-
"%{hmm}" => "doh"
|
164
|
-
) do
|
165
|
-
insist { subject.get("firstname") } == nil
|
166
|
-
insist { subject.get("lastname") } == "Sagdiyev"
|
167
|
-
insist { subject.get("fullname") } == "Borat Sagdiyev"
|
168
|
-
insist { subject.get("country") } == "Kazakhstan"
|
169
|
-
insist { subject.get("location") } == "Somethere in Kazakhstan"
|
170
|
-
insist { subject.get("hobby") } == nil
|
171
|
-
insist { subject.get("status") } == nil
|
172
|
-
insist { subject.get("Borat_saying") } == nil
|
173
|
-
insist { subject.get("%{hmm}") } == "doh"
|
74
|
+
it "retains fields that match after interpolation" do
|
75
|
+
expect(event.get("firstname")).to eq("Borat")
|
76
|
+
expect(event.get("Borat_saying")).to eq("Cloud is not ready for enterprise if is not integrate with single server running Active Directory.")
|
77
|
+
end
|
174
78
|
end
|
175
79
|
end
|
176
80
|
|
177
|
-
describe "
|
178
|
-
|
179
|
-
config <<-CONFIG
|
180
|
-
filter {
|
181
|
-
prune {
|
182
|
-
# This should only permit fields named 'firstname', 'fullname',
|
183
|
-
# 'location', 'status', etc.
|
184
|
-
whitelist_values => [ "firstname", "^Borat$",
|
185
|
-
"fullname", "%{firstname} Sagdiyev",
|
186
|
-
"location", "no no no",
|
187
|
-
"status", "^2",
|
188
|
-
"%{firstname}_saying", "%{hobby}.*Active" ]
|
189
|
-
}
|
190
|
-
}
|
191
|
-
CONFIG
|
81
|
+
describe "blacklist_names" do
|
192
82
|
|
193
|
-
|
194
|
-
"
|
195
|
-
"lastname" => "Sagdiyev",
|
196
|
-
"fullname" => "Borat Sagdiyev",
|
197
|
-
"country" => "Kazakhstan",
|
198
|
-
"location" => "Somethere in Kazakhstan",
|
199
|
-
"hobby" => "Cloud",
|
200
|
-
"status" => "200",
|
201
|
-
"Borat_saying" => "Cloud is not ready for enterprise if is not integrate with single server running Active Directory.",
|
202
|
-
"%{hmm}" => "doh"
|
203
|
-
) do
|
204
|
-
insist { subject.get("firstname") } == "Borat"
|
205
|
-
|
206
|
-
# TODO(sissel): According to the config above, this should be nil because
|
207
|
-
# it is not in the list of whitelisted fields, but we expect it to be
|
208
|
-
# "Sagdiyev" ? I am confused.
|
209
|
-
insist { subject.get("lastname") } == "Sagdiyev"
|
210
|
-
insist { subject.get("fullname") } == nil
|
211
|
-
insist { subject.get("country") } == "Kazakhstan"
|
212
|
-
insist { subject.get("location") } == nil
|
213
|
-
insist { subject.get("hobby") } == "Cloud"
|
214
|
-
insist { subject.get("status") } == "200"
|
215
|
-
insist { subject.get("Borat_saying") } == "Cloud is not ready for enterprise if is not integrate with single server running Active Directory."
|
216
|
-
|
217
|
-
# TODO(sissel): Contrary to the 'lastname' check, we expect %{hmm} field
|
218
|
-
# to be nil because it is not whitelisted, yes? Contradictory insists
|
219
|
-
# here. I don't know what the intended behavior is... Seems like
|
220
|
-
# whitelist means 'anything not here' but since this test is written
|
221
|
-
# confusingly, I dont' know how to move forward.
|
222
|
-
insist { subject.get("%{hmm}") } == nil
|
83
|
+
let(:config) do
|
84
|
+
{ "blacklist_names" => [ "firstname", "(hobby|status)", "%{firstname}_saying" ] }
|
223
85
|
end
|
224
|
-
end
|
225
86
|
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
whitelist_values => [ "firstname", "^Borat$",
|
232
|
-
"fullname", "%{firstname} Sagdiyev",
|
233
|
-
"location", "no no no",
|
234
|
-
"status", "^2",
|
235
|
-
"%{firstname}_saying", "%{hobby}.*Active" ]
|
236
|
-
interpolate => true
|
237
|
-
}
|
238
|
-
}
|
239
|
-
CONFIG
|
87
|
+
it "drops fields in the list" do
|
88
|
+
expect(event.get("firstname")).to eq(nil)
|
89
|
+
expect(event.get("hobby")).to eq(nil)
|
90
|
+
expect(event.get("status")).to eq(nil)
|
91
|
+
end
|
240
92
|
|
241
|
-
|
242
|
-
"
|
243
|
-
"
|
244
|
-
"
|
245
|
-
"
|
246
|
-
"
|
247
|
-
"hobby" => "Cloud",
|
248
|
-
"status" => "200",
|
249
|
-
"Borat_saying" => "Cloud is not ready for enterprise if is not integrate with single server running Active Directory.",
|
250
|
-
"%{hmm}" => "doh"
|
251
|
-
) do
|
252
|
-
insist { subject.get("firstname") } == "Borat"
|
253
|
-
insist { subject.get("lastname") } == "Sagdiyev"
|
254
|
-
insist { subject.get("fullname") } == "Borat Sagdiyev"
|
255
|
-
insist { subject.get("country") } == "Kazakhstan"
|
256
|
-
insist { subject.get("location") } == nil
|
257
|
-
insist { subject.get("hobby") } == "Cloud"
|
258
|
-
insist { subject.get("status") } == "200"
|
259
|
-
insist { subject.get("Borat_saying") } == "Cloud is not ready for enterprise if is not integrate with single server running Active Directory."
|
260
|
-
insist { subject.get("%{hmm}") } == nil
|
93
|
+
it "keeps the remaining fields" do
|
94
|
+
expect(event.get("lastname")).to eq("Sagdiyev")
|
95
|
+
expect(event.get("fullname")).to eq("Borat Sagdiyev")
|
96
|
+
expect(event.get("country")).to eq("Kazakhstan")
|
97
|
+
expect(event.get("location")).to eq("Somethere in Kazakhstan")
|
98
|
+
expect(event.get("Borat_saying")).to eq("Cloud is not ready for enterprise if is not integrate with single server running Active Directory.")
|
261
99
|
end
|
262
|
-
end
|
263
100
|
|
264
|
-
|
101
|
+
context "if there are non resolved field references" do
|
102
|
+
let(:event_data) { super.merge("%{hmm}" => "doh") }
|
103
|
+
it "also drops them" do
|
104
|
+
expect(event.get("%{hmm}")).to eq("doh")
|
105
|
+
end
|
106
|
+
end
|
107
|
+
context "with interpolation" do
|
265
108
|
|
266
|
-
|
267
|
-
filter {
|
268
|
-
prune {
|
269
|
-
blacklist_values => [ "firstname", "^Borat$",
|
270
|
-
"fullname", "%{firstname} Sagdiyev",
|
271
|
-
"location", "no no no",
|
272
|
-
"status", "^2",
|
273
|
-
"%{firstname}_saying", "%{hobby}.*Active" ]
|
274
|
-
}
|
275
|
-
}
|
276
|
-
CONFIG
|
109
|
+
let(:config) { super.merge("interpolate" => true) }
|
277
110
|
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
"fullname" => "Borat Sagdiyev",
|
282
|
-
"country" => "Kazakhstan",
|
283
|
-
"location" => "Somethere in Kazakhstan",
|
284
|
-
"hobby" => "Cloud",
|
285
|
-
"status" => "200",
|
286
|
-
"Borat_saying" => "Cloud is not ready for enterprise if is not integrate with single server running Active Directory.",
|
287
|
-
"%{hmm}" => "doh"
|
288
|
-
) do
|
289
|
-
insist { subject.get("firstname") } == nil
|
290
|
-
insist { subject.get("lastname") } == "Sagdiyev"
|
291
|
-
insist { subject.get("fullname") } == "Borat Sagdiyev"
|
292
|
-
insist { subject.get("country") } == "Kazakhstan"
|
293
|
-
insist { subject.get("location") } == "Somethere in Kazakhstan"
|
294
|
-
insist { subject.get("hobby") } == "Cloud"
|
295
|
-
insist { subject.get("status") } == nil
|
296
|
-
insist { subject.get("Borat_saying") } == "Cloud is not ready for enterprise if is not integrate with single server running Active Directory."
|
297
|
-
insist { subject.get("%{hmm}") } == nil
|
111
|
+
it "drops fields after interpolation" do
|
112
|
+
expect(event.get("Borat_saying")).to be_nil
|
113
|
+
end
|
298
114
|
end
|
299
115
|
end
|
300
|
-
|
301
|
-
|
302
|
-
|
303
|
-
|
304
|
-
|
305
|
-
|
306
|
-
|
307
|
-
|
308
|
-
|
309
|
-
|
310
|
-
|
311
|
-
|
116
|
+
describe "whitelist_values" do
|
117
|
+
|
118
|
+
let(:config) do
|
119
|
+
{
|
120
|
+
# This should only permit fields named 'firstname', 'fullname',
|
121
|
+
# 'location', 'status', etc.
|
122
|
+
"whitelist_values" => {
|
123
|
+
"firstname" => "^Borat$",
|
124
|
+
"fullname" => "%{firstname} Sagdiyev",
|
125
|
+
"location" => "no no no",
|
126
|
+
"status" => "^2",
|
127
|
+
"%{firstname}_saying" => "%{hobby}.*Active"
|
312
128
|
}
|
313
129
|
}
|
314
|
-
|
130
|
+
end
|
315
131
|
|
316
|
-
|
317
|
-
"firstname"
|
318
|
-
"
|
319
|
-
"fullname" => "Borat Sagdiyev",
|
320
|
-
"country" => "Kazakhstan",
|
321
|
-
"location" => "Somethere in Kazakhstan",
|
322
|
-
"hobby" => "Cloud",
|
323
|
-
"status" => "200",
|
324
|
-
"Borat_saying" => "Cloud is not ready for enterprise if is not integrate with single server running Active Directory.",
|
325
|
-
"%{hmm}" => "doh"
|
326
|
-
) do
|
327
|
-
insist { subject.get("firstname") } == nil
|
328
|
-
insist { subject.get("lastname") } == "Sagdiyev"
|
329
|
-
insist { subject.get("fullname") } == nil
|
330
|
-
insist { subject.get("country") } == "Kazakhstan"
|
331
|
-
insist { subject.get("location") } == "Somethere in Kazakhstan"
|
332
|
-
insist { subject.get("hobby") } == "Cloud"
|
333
|
-
insist { subject.get("status") } == nil
|
334
|
-
insist { subject.get("Borat_saying") } == nil
|
335
|
-
insist { subject.get("%{hmm}") } == nil
|
132
|
+
it "keeps fields in the whitelist if the value matches" do
|
133
|
+
expect(event.get("firstname")).to eq("Borat")
|
134
|
+
expect(event.get("status")).to eq("200")
|
336
135
|
end
|
337
|
-
end
|
338
136
|
|
339
|
-
|
137
|
+
it "drops fields in the whitelist if the value doesn't match" do
|
138
|
+
expect(event.get("fullname")).to be_nil
|
139
|
+
expect(event.get("location")).to be_nil
|
140
|
+
end
|
141
|
+
|
142
|
+
it "include all other fields" do
|
143
|
+
# whitelist_values will only filter configured fields
|
144
|
+
# all others are still governed by the whitelist_names setting
|
145
|
+
# which means they're all kept by default
|
146
|
+
expect(event.get("lastname")).to eq("Sagdiyev")
|
147
|
+
expect(event.get("country")).to eq("Kazakhstan")
|
148
|
+
expect(event.get("hobby")).to eq("Cloud")
|
149
|
+
expect(event.get("Borat_saying")).to eq("Cloud is not ready for enterprise if is not integrate with single server running Active Directory.")
|
150
|
+
end
|
340
151
|
|
341
|
-
|
342
|
-
|
343
|
-
|
344
|
-
|
345
|
-
|
346
|
-
|
152
|
+
context "with interpolation" do
|
153
|
+
|
154
|
+
let(:config) do
|
155
|
+
{
|
156
|
+
"whitelist_values" => {
|
157
|
+
"firstname" => "^Borat$",
|
158
|
+
"fullname" => "%{firstname} Sagdiyev",
|
159
|
+
"location" => "no no no",
|
160
|
+
"status" => "^2",
|
161
|
+
"%{firstname}_saying" => "%{hobby}.*Active"
|
162
|
+
},
|
163
|
+
"interpolate" => true
|
347
164
|
}
|
348
|
-
|
349
|
-
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
|
356
|
-
|
357
|
-
|
358
|
-
|
359
|
-
|
360
|
-
|
165
|
+
end
|
166
|
+
let(:event_data) { super.merge("%{hmm}" => "doh") }
|
167
|
+
it "keeps field values after interpolation" do
|
168
|
+
expect(event.get("fullname")).to eq("Borat Sagdiyev")
|
169
|
+
expect(event.get("Borat_saying")).to eq("Cloud is not ready for enterprise if is not integrate with single server running Active Directory.")
|
170
|
+
end
|
171
|
+
end
|
172
|
+
context "with array values" do
|
173
|
+
|
174
|
+
let(:config) do
|
175
|
+
{
|
176
|
+
"whitelist_values" => {
|
177
|
+
"status" => "^(1|2|3)",
|
178
|
+
"xxx" => "3",
|
179
|
+
"error" => "%{blah}"
|
180
|
+
}
|
181
|
+
}
|
182
|
+
end
|
183
|
+
|
184
|
+
let(:event_data) do
|
185
|
+
{
|
186
|
+
"blah" => "foo",
|
187
|
+
"xxx" => [ "1 2 3", "3 4 5" ],
|
188
|
+
"status" => [ "100", "200", "300", "400", "500" ],
|
189
|
+
"error" => [ "This is foolish" , "Need smthing smart too" ]
|
190
|
+
}
|
191
|
+
end
|
192
|
+
|
193
|
+
it "drops fields if no value matches" do
|
194
|
+
expect(event.get("error")).to eq(nil)
|
195
|
+
end
|
196
|
+
|
197
|
+
it "keeps only elements that match" do
|
198
|
+
expect(event.get("status")).to eq([ "100", "200", "300" ])
|
199
|
+
end
|
200
|
+
|
201
|
+
it "keeps values intact if they all match" do
|
202
|
+
expect(event.get("xxx")).to eq([ "1 2 3", "3 4 5" ])
|
203
|
+
end
|
204
|
+
context "with interpolation" do
|
205
|
+
let(:config) { super.merge("interpolate" => true) }
|
206
|
+
it "keeps values that match after interpolation" do
|
207
|
+
expect(event.get("error")).to eq([ "This is foolish" ])
|
208
|
+
end
|
209
|
+
end
|
361
210
|
end
|
362
211
|
end
|
363
212
|
|
364
|
-
describe "
|
213
|
+
describe "blacklist_values" do
|
365
214
|
|
366
|
-
config
|
367
|
-
|
368
|
-
|
369
|
-
|
370
|
-
|
371
|
-
|
215
|
+
let(:config) do
|
216
|
+
{
|
217
|
+
"blacklist_values" => {
|
218
|
+
"firstname" => "^Borat$",
|
219
|
+
"fullname" => "%{firstname} Sagdiyev",
|
220
|
+
"location" => "no no no",
|
221
|
+
"status" => "^2",
|
222
|
+
"%{firstname}_saying" => "%{hobby}.*Active"
|
372
223
|
}
|
373
224
|
}
|
374
|
-
CONFIG
|
375
|
-
|
376
|
-
sample(
|
377
|
-
"blah" => "foo",
|
378
|
-
"xxx" => [ "1 2 3", "3 4 5" ],
|
379
|
-
"status" => [ "100", "200", "300", "400", "500" ],
|
380
|
-
"error" => [ "This is foolish", "Need smthing smart too" ]
|
381
|
-
) do
|
382
|
-
insist { subject.get("blah") } == "foo"
|
383
|
-
insist { subject.get("error") } == [ "This is foolish", "Need smthing smart too" ]
|
384
|
-
insist { subject.get("xxx") } == nil
|
385
|
-
insist { subject.get("status") } == [ "400", "500" ]
|
386
225
|
end
|
387
|
-
end
|
388
226
|
|
389
|
-
|
227
|
+
it "drops fields that match the values" do
|
228
|
+
expect(event.get("firstname")).to eq(nil)
|
229
|
+
expect(event.get("status")).to eq(nil)
|
230
|
+
end
|
390
231
|
|
391
|
-
|
392
|
-
|
393
|
-
|
394
|
-
whitelist_values => [ "status", "^(1|2|3)",
|
395
|
-
"xxx", "3",
|
396
|
-
"error", "%{blah}" ]
|
397
|
-
interpolate => true
|
398
|
-
}
|
399
|
-
}
|
400
|
-
CONFIG
|
401
|
-
|
402
|
-
sample(
|
403
|
-
"blah" => "foo",
|
404
|
-
"xxx" => [ "1 2 3", "3 4 5" ],
|
405
|
-
"status" => [ "100", "200", "300", "400", "500" ],
|
406
|
-
"error" => [ "This is foolish" , "Need smthing smart too" ]
|
407
|
-
) do
|
408
|
-
insist { subject.get("blah") } == "foo"
|
409
|
-
insist { subject.get("error") } == [ "This is foolish" ]
|
410
|
-
insist { subject.get("xxx") } == [ "1 2 3", "3 4 5" ]
|
411
|
-
insist { subject.get("status") } == [ "100", "200", "300" ]
|
232
|
+
it "keeps fields that don't match the values" do
|
233
|
+
expect(event.get("fullname")).to eq("Borat Sagdiyev")
|
234
|
+
expect(event.get("location")).to eq("Somethere in Kazakhstan")
|
412
235
|
end
|
413
|
-
end
|
414
236
|
|
415
|
-
|
237
|
+
context "with interpolation" do
|
416
238
|
|
417
|
-
|
418
|
-
|
419
|
-
|
420
|
-
|
421
|
-
|
422
|
-
|
423
|
-
|
239
|
+
let(:config) { super.merge("interpolate" => true) }
|
240
|
+
|
241
|
+
it "drops fields that match after interpolation" do
|
242
|
+
expect(event.get("fullname")).to eq(nil)
|
243
|
+
expect(event.get("Borat_saying")).to eq(nil)
|
244
|
+
end
|
245
|
+
end
|
246
|
+
context "with array values" do
|
247
|
+
|
248
|
+
let(:config) do
|
249
|
+
{
|
250
|
+
"blacklist_values" => {
|
251
|
+
"status" => "^(1|2|3)",
|
252
|
+
"xxx" => "3",
|
253
|
+
"error" => "%{blah}"
|
254
|
+
}
|
424
255
|
}
|
425
|
-
|
426
|
-
|
427
|
-
|
428
|
-
|
429
|
-
|
430
|
-
|
431
|
-
|
432
|
-
|
433
|
-
|
434
|
-
|
435
|
-
|
436
|
-
|
437
|
-
|
256
|
+
end
|
257
|
+
let(:event_data) do
|
258
|
+
{
|
259
|
+
"blah" => "foo",
|
260
|
+
"xxx" => [ "1 2 3", "3 4 5" ],
|
261
|
+
"status" => [ "100", "200", "300", "400", "500" ],
|
262
|
+
"error" => [ "This is foolish", "Need smthing smart too" ]
|
263
|
+
}
|
264
|
+
end
|
265
|
+
it "drops fields if no elements match" do
|
266
|
+
expect(event.get("xxx")).to eq(nil)
|
267
|
+
end
|
268
|
+
|
269
|
+
it "keeps values that don't match" do
|
270
|
+
expect(event.get("error")).to eq([ "This is foolish", "Need smthing smart too" ])
|
271
|
+
expect(event.get("status")).to eq([ "400", "500" ])
|
272
|
+
end
|
273
|
+
|
274
|
+
context "with interpolation" do
|
275
|
+
let(:config) { super.merge("interpolate" => true) }
|
276
|
+
it "drops values that match after interpolation" do
|
277
|
+
expect(event.get("error")).to eq([ "Need smthing smart too" ])
|
278
|
+
end
|
279
|
+
end
|
438
280
|
end
|
439
281
|
end
|
440
|
-
|
441
282
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-filter-prune
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.0.
|
4
|
+
version: 3.0.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2019-09-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -84,7 +84,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
84
84
|
version: '0'
|
85
85
|
requirements: []
|
86
86
|
rubyforge_project:
|
87
|
-
rubygems_version: 2.6.
|
87
|
+
rubygems_version: 2.6.13
|
88
88
|
signing_key:
|
89
89
|
specification_version: 4
|
90
90
|
summary: Prunes event data based on a list of fields to blacklist or whitelist
|