logstash-filter-mutate 2.0.2 → 2.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -1
- data/README.md +3 -0
- data/lib/logstash/filters/mutate.rb +116 -110
- data/logstash-filter-mutate.gemspec +1 -1
- metadata +17 -17
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fd1c151098361f3b30635ab7d3091585514f2f6a
|
4
|
+
data.tar.gz: bc60754791477e9e23211cc2a2da52e5f26e95de
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 59c5e5b86234a514c5cf5ce735c05f788117fdbd7aff5476a0b2fb034022c7a7e35f68c76132f381d42a63fa8f5ccfd654899afcbb2b9b16af3569dd5a9ecdc5
|
7
|
+
data.tar.gz: 49b29f761184665f0f4da21a2bb3a692732d5c801f93cd67d63b72cdb91613194f77cf031102f33c99a748eb683c0b11a6e1bffad3ed2c3c579b6c3f81d1125f
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,8 @@
|
|
1
|
+
## 2.0.3
|
2
|
+
- Code cleanups and fix field assignments
|
3
|
+
|
1
4
|
## 2.0.0
|
2
|
-
- Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
|
5
|
+
- Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
|
3
6
|
instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
|
4
7
|
- Dependency on logstash-core update to 2.0
|
5
8
|
|
data/README.md
CHANGED
@@ -1,5 +1,8 @@
|
|
1
1
|
# Logstash Plugin
|
2
2
|
|
3
|
+
[](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-mutate-unit/)
|
5
|
+
|
3
6
|
This is a plugin for [Logstash](https://github.com/elastic/logstash).
|
4
7
|
|
5
8
|
It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
|
@@ -33,7 +33,7 @@ class LogStash::Filters::Mutate < LogStash::Filters::Base
|
|
33
33
|
# filters.
|
34
34
|
config :remove, :validate => :array, :deprecated => true
|
35
35
|
|
36
|
-
# Replace a field with a new value. The new value can include
|
36
|
+
# Replace a field with a new value. The new value can include `%{foo}` strings
|
37
37
|
# to help you build a new value from other parts of the event.
|
38
38
|
#
|
39
39
|
# Example:
|
@@ -176,24 +176,33 @@ class LogStash::Filters::Mutate < LogStash::Filters::Base
|
|
176
176
|
# }
|
177
177
|
config :merge, :validate => :hash
|
178
178
|
|
179
|
-
|
179
|
+
TRUE_REGEX = (/^(true|t|yes|y|1)$/i).freeze
|
180
|
+
FALSE_REGEX = (/^(false|f|no|n|0)$/i).freeze
|
181
|
+
CONVERT_PREFIX = "convert_".freeze
|
182
|
+
|
180
183
|
def register
|
181
184
|
valid_conversions = %w(string integer float boolean)
|
182
185
|
# TODO(sissel): Validate conversion requests if provided.
|
183
186
|
@convert.nil? or @convert.each do |field, type|
|
184
187
|
if !valid_conversions.include?(type)
|
185
|
-
raise LogStash::ConfigurationError, I18n.t(
|
186
|
-
|
187
|
-
:
|
188
|
+
raise LogStash::ConfigurationError, I18n.t(
|
189
|
+
"logstash.agent.configuration.invalid_plugin_register",
|
190
|
+
:plugin => "filter",
|
191
|
+
:type => "mutate",
|
192
|
+
:error => "Invalid conversion type '#{type}', expected one of '#{valid_conversions.join(',')}'"
|
193
|
+
)
|
188
194
|
end
|
189
|
-
end
|
195
|
+
end
|
190
196
|
|
191
197
|
@gsub_parsed = []
|
192
198
|
@gsub.nil? or @gsub.each_slice(3) do |field, needle, replacement|
|
193
199
|
if [field, needle, replacement].any? {|n| n.nil?}
|
194
|
-
raise LogStash::ConfigurationError, I18n.t(
|
195
|
-
|
196
|
-
:
|
200
|
+
raise LogStash::ConfigurationError, I18n.t(
|
201
|
+
"logstash.agent.configuration.invalid_plugin_register",
|
202
|
+
:plugin => "filter",
|
203
|
+
:type => "mutate",
|
204
|
+
:error => "Invalid gsub configuration #{[field, needle, replacement]}. gsub requires 3 non-nil elements per config entry"
|
205
|
+
)
|
197
206
|
end
|
198
207
|
|
199
208
|
@gsub_parsed << {
|
@@ -202,12 +211,9 @@ class LogStash::Filters::Mutate < LogStash::Filters::Base
|
|
202
211
|
:replacement => replacement
|
203
212
|
}
|
204
213
|
end
|
205
|
-
end
|
214
|
+
end
|
206
215
|
|
207
|
-
public
|
208
216
|
def filter(event)
|
209
|
-
|
210
|
-
|
211
217
|
rename(event) if @rename
|
212
218
|
update(event) if @update
|
213
219
|
replace(event) if @replace
|
@@ -222,17 +228,17 @@ class LogStash::Filters::Mutate < LogStash::Filters::Base
|
|
222
228
|
merge(event) if @merge
|
223
229
|
|
224
230
|
filter_matched(event)
|
225
|
-
end
|
231
|
+
end
|
226
232
|
|
227
233
|
private
|
234
|
+
|
228
235
|
def remove(event)
|
229
236
|
@remove.each do |field|
|
230
237
|
field = event.sprintf(field)
|
231
238
|
event.remove(field)
|
232
239
|
end
|
233
|
-
end
|
240
|
+
end
|
234
241
|
|
235
|
-
private
|
236
242
|
def rename(event)
|
237
243
|
@rename.each do |old, new|
|
238
244
|
old = event.sprintf(old)
|
@@ -240,98 +246,94 @@ class LogStash::Filters::Mutate < LogStash::Filters::Base
|
|
240
246
|
next unless event.include?(old)
|
241
247
|
event[new] = event.remove(old)
|
242
248
|
end
|
243
|
-
end
|
249
|
+
end
|
244
250
|
|
245
|
-
private
|
246
251
|
def update(event)
|
247
252
|
@update.each do |field, newvalue|
|
248
253
|
next unless event.include?(field)
|
249
254
|
event[field] = event.sprintf(newvalue)
|
250
255
|
end
|
251
|
-
end
|
256
|
+
end
|
252
257
|
|
253
|
-
private
|
254
258
|
def replace(event)
|
255
259
|
@replace.each do |field, newvalue|
|
256
260
|
event[field] = event.sprintf(newvalue)
|
257
261
|
end
|
258
|
-
end
|
262
|
+
end
|
259
263
|
|
260
264
|
def convert(event)
|
261
265
|
@convert.each do |field, type|
|
262
266
|
next unless event.include?(field)
|
263
267
|
original = event[field]
|
264
268
|
# calls convert_{string,integer,float,boolean} depending on type requested.
|
265
|
-
converter = method(
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
@logger.debug("I don't know how to type convert a hash, skipping",
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
269
|
+
converter = method(CONVERT_PREFIX + type)
|
270
|
+
|
271
|
+
case original
|
272
|
+
when Hash
|
273
|
+
@logger.debug? && @logger.debug("I don't know how to type convert a hash, skipping", :field => field, :value => original)
|
274
|
+
when Array
|
275
|
+
event[field] = original.map { |v| converter.call(v) }
|
276
|
+
when NilClass
|
277
|
+
# ignore
|
274
278
|
else
|
275
|
-
|
279
|
+
event[field] = converter.call(original)
|
276
280
|
end
|
277
|
-
event[field] = value
|
278
281
|
end
|
279
|
-
end
|
282
|
+
end
|
280
283
|
|
281
284
|
def convert_string(value)
|
282
285
|
# since this is a filter and all inputs should be already UTF-8
|
283
286
|
# we wont check valid_encoding? but just force UTF-8 for
|
284
287
|
# the Fixnum#to_s case which always result in US-ASCII
|
288
|
+
# also not that force_encoding checks current encoding against the
|
289
|
+
# target encoding and only change if necessary, so calling
|
290
|
+
# valid_encoding? is redundant
|
285
291
|
# see https://twitter.com/jordansissel/status/444613207143903232
|
286
|
-
|
287
|
-
end
|
292
|
+
value.to_s.force_encoding(Encoding::UTF_8)
|
293
|
+
end
|
288
294
|
|
289
295
|
def convert_integer(value)
|
290
|
-
|
291
|
-
end
|
296
|
+
value.to_i
|
297
|
+
end
|
292
298
|
|
293
299
|
def convert_float(value)
|
294
|
-
|
295
|
-
end
|
300
|
+
value.to_f
|
301
|
+
end
|
296
302
|
|
297
303
|
def convert_boolean(value)
|
298
|
-
return true if value =~
|
299
|
-
return false if value.empty? || value =~
|
304
|
+
return true if value =~ TRUE_REGEX
|
305
|
+
return false if value.empty? || value =~ FALSE_REGEX
|
300
306
|
@logger.warn("Failed to convert #{value} into boolean.")
|
301
|
-
|
302
|
-
end
|
307
|
+
value
|
308
|
+
end
|
303
309
|
|
304
|
-
private
|
305
310
|
def gsub(event)
|
306
311
|
@gsub_parsed.each do |config|
|
307
312
|
field = config[:field]
|
308
313
|
needle = config[:needle]
|
309
314
|
replacement = config[:replacement]
|
310
315
|
|
311
|
-
|
312
|
-
|
313
|
-
|
314
|
-
|
315
|
-
|
316
|
-
v
|
317
|
-
else
|
316
|
+
value = event[field]
|
317
|
+
case value
|
318
|
+
when Array
|
319
|
+
event[field] = value.map do |v|
|
320
|
+
if v.is_a?(String)
|
318
321
|
gsub_dynamic_fields(event, v, needle, replacement)
|
322
|
+
else
|
323
|
+
@logger.warn("gsub mutation is only applicable for Strings, skipping", :field => field, :value => v)
|
324
|
+
v
|
319
325
|
end
|
320
326
|
end
|
327
|
+
when String
|
328
|
+
event[field] = gsub_dynamic_fields(event, value, needle, replacement)
|
321
329
|
else
|
322
|
-
|
323
|
-
@logger.debug("gsub mutation is only applicable for Strings, " +
|
324
|
-
"skipping", :field => field, :value => event[field])
|
325
|
-
next
|
326
|
-
end
|
327
|
-
event[field] = gsub_dynamic_fields(event, event[field], needle, replacement)
|
330
|
+
@logger.debug? && @logger.debug("gsub mutation is only applicable for Strings, skipping", :field => field, :value => event[field])
|
328
331
|
end
|
329
|
-
end
|
330
|
-
end
|
332
|
+
end
|
333
|
+
end
|
331
334
|
|
332
|
-
private
|
333
335
|
def gsub_dynamic_fields(event, original, needle, replacement)
|
334
|
-
if needle.is_a?
|
336
|
+
if needle.is_a?(Regexp)
|
335
337
|
original.gsub(needle, event.sprintf(replacement))
|
336
338
|
else
|
337
339
|
# we need to replace any dynamic fields
|
@@ -339,7 +341,6 @@ class LogStash::Filters::Mutate < LogStash::Filters::Base
|
|
339
341
|
end
|
340
342
|
end
|
341
343
|
|
342
|
-
private
|
343
344
|
def uppercase(event)
|
344
345
|
@uppercase.each do |field|
|
345
346
|
original = event[field]
|
@@ -347,95 +348,100 @@ class LogStash::Filters::Mutate < LogStash::Filters::Base
|
|
347
348
|
# therefore we can't assume that we are modifying the actual value behind
|
348
349
|
# the key so read, modify and overwrite
|
349
350
|
event[field] = case original
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
|
354
|
-
|
355
|
-
end
|
356
|
-
when String
|
357
|
-
# nil means no change was made to the String
|
358
|
-
original.upcase! || original
|
359
|
-
else
|
360
|
-
@logger.debug("Can't uppercase something that isn't a string",
|
361
|
-
:field => field, :value => original)
|
362
|
-
original
|
351
|
+
when Array
|
352
|
+
# can't map upcase! as it replaces an already upcase value with nil
|
353
|
+
# ["ABCDEF"].map(&:upcase!) => [nil]
|
354
|
+
original.map do |elem|
|
355
|
+
(elem.is_a?(String) ? elem.upcase : elem)
|
363
356
|
end
|
357
|
+
when String
|
358
|
+
# nil means no change was made to the String
|
359
|
+
original.upcase! || original
|
360
|
+
else
|
361
|
+
@logger.debug? && @logger.debug("Can't uppercase something that isn't a string", :field => field, :value => original)
|
362
|
+
original
|
363
|
+
end
|
364
364
|
end
|
365
|
-
end
|
365
|
+
end
|
366
366
|
|
367
|
-
private
|
368
367
|
def lowercase(event)
|
369
368
|
#see comments for #uppercase
|
370
369
|
@lowercase.each do |field|
|
371
370
|
original = event[field]
|
372
371
|
event[field] = case original
|
373
|
-
|
374
|
-
|
375
|
-
|
376
|
-
end
|
377
|
-
when String
|
378
|
-
original.downcase! || original
|
379
|
-
else
|
380
|
-
@logger.debug("Can't lowercase something that isn't a string",
|
381
|
-
:field => field, :value => original)
|
382
|
-
original
|
372
|
+
when Array
|
373
|
+
original.map! do |elem|
|
374
|
+
(elem.is_a?(String) ? elem.downcase : elem)
|
383
375
|
end
|
376
|
+
when String
|
377
|
+
original.downcase! || original
|
378
|
+
else
|
379
|
+
@logger.debug? && @logger.debug("Can't lowercase something that isn't a string", :field => field, :value => original)
|
380
|
+
original
|
381
|
+
end
|
384
382
|
end
|
385
|
-
end
|
386
|
-
|
383
|
+
end
|
387
384
|
|
388
|
-
private
|
389
385
|
def split(event)
|
390
386
|
@split.each do |field, separator|
|
391
|
-
|
392
|
-
|
387
|
+
value = event[field]
|
388
|
+
if value.is_a?(String)
|
389
|
+
event[field] = value.split(separator)
|
393
390
|
else
|
394
|
-
@logger.debug("Can't split something that isn't a string",
|
395
|
-
:field => field, :value => event[field])
|
391
|
+
@logger.debug? && @logger.debug("Can't split something that isn't a string", :field => field, :value => event[field])
|
396
392
|
end
|
397
393
|
end
|
398
394
|
end
|
399
395
|
|
400
|
-
private
|
401
396
|
def join(event)
|
402
397
|
@join.each do |field, separator|
|
403
|
-
|
404
|
-
|
398
|
+
value = event[field]
|
399
|
+
if value.is_a?(Array)
|
400
|
+
event[field] = value.join(separator)
|
405
401
|
end
|
406
402
|
end
|
407
403
|
end
|
408
404
|
|
409
|
-
private
|
410
405
|
def strip(event)
|
411
406
|
@strip.each do |field|
|
412
|
-
|
413
|
-
|
414
|
-
|
415
|
-
event[field] =
|
407
|
+
value = event[field]
|
408
|
+
case value
|
409
|
+
when Array
|
410
|
+
event[field] = value.map{|s| s.strip }
|
411
|
+
when String
|
412
|
+
event[field] = value.strip
|
416
413
|
end
|
417
414
|
end
|
418
415
|
end
|
419
416
|
|
420
|
-
private
|
421
417
|
def merge(event)
|
422
418
|
@merge.each do |dest_field, added_fields|
|
423
419
|
# When multiple calls, added_field is an array
|
420
|
+
|
421
|
+
dest_field_value = event[dest_field]
|
422
|
+
|
424
423
|
Array(added_fields).each do |added_field|
|
425
|
-
|
424
|
+
added_field_value = event[added_field]
|
425
|
+
|
426
|
+
if dest_field_value.is_a?(Hash) ^ added_field_value.is_a?(Hash)
|
426
427
|
@logger.error("Not possible to merge an array and a hash: ", :dest_field => dest_field, :added_field => added_field )
|
427
428
|
next
|
428
429
|
end
|
429
430
|
|
430
|
-
|
431
|
-
|
432
|
-
event[dest_field].update
|
431
|
+
# No need to test the other
|
432
|
+
if dest_field_value.is_a?(Hash)
|
433
|
+
# do not use event[dest_field].update because the returned object from event[dest_field]
|
434
|
+
# can/will be a copy of the actual event data and directly updating it will not update
|
435
|
+
# the Event internal data. The updated value must be reassigned in the Event.
|
436
|
+
event[dest_field] = dest_field_value.update(added_field_value)
|
433
437
|
else
|
434
|
-
event[dest_field]
|
435
|
-
event
|
438
|
+
# do not use event[dest_field].concat because the returned object from event[dest_field]
|
439
|
+
# can/will be a copy of the actual event data and directly updating it will not update
|
440
|
+
# the Event internal data. The updated value must be reassigned in the Event.
|
441
|
+
event[dest_field] = Array(dest_field_value).concat(Array(added_field_value))
|
436
442
|
end
|
437
443
|
end
|
438
444
|
end
|
439
445
|
end
|
440
446
|
|
441
|
-
end
|
447
|
+
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-filter-mutate'
|
4
|
-
s.version = '2.0.
|
4
|
+
s.version = '2.0.3'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "The mutate filter allows you to perform general mutations on fields. You can rename, remove, replace, and modify fields in your events."
|
7
7
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
metadata
CHANGED
@@ -1,17 +1,18 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-filter-mutate
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.0.
|
4
|
+
version: 2.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Elastic
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-12-16 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
|
14
|
+
name: logstash-core
|
15
|
+
version_requirements: !ruby/object:Gem::Requirement
|
15
16
|
requirements:
|
16
17
|
- - '>='
|
17
18
|
- !ruby/object:Gem::Version
|
@@ -19,10 +20,7 @@ dependencies:
|
|
19
20
|
- - <
|
20
21
|
- !ruby/object:Gem::Version
|
21
22
|
version: 3.0.0
|
22
|
-
|
23
|
-
prerelease: false
|
24
|
-
type: :runtime
|
25
|
-
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirement: !ruby/object:Gem::Requirement
|
26
24
|
requirements:
|
27
25
|
- - '>='
|
28
26
|
- !ruby/object:Gem::Version
|
@@ -30,48 +28,50 @@ dependencies:
|
|
30
28
|
- - <
|
31
29
|
- !ruby/object:Gem::Version
|
32
30
|
version: 3.0.0
|
31
|
+
prerelease: false
|
32
|
+
type: :runtime
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
|
+
name: logstash-patterns-core
|
35
|
+
version_requirements: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - '>='
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: '0'
|
34
40
|
requirement: !ruby/object:Gem::Requirement
|
35
41
|
requirements:
|
36
42
|
- - '>='
|
37
43
|
- !ruby/object:Gem::Version
|
38
44
|
version: '0'
|
39
|
-
name: logstash-patterns-core
|
40
45
|
prerelease: false
|
41
46
|
type: :runtime
|
47
|
+
- !ruby/object:Gem::Dependency
|
48
|
+
name: logstash-filter-grok
|
42
49
|
version_requirements: !ruby/object:Gem::Requirement
|
43
50
|
requirements:
|
44
51
|
- - '>='
|
45
52
|
- !ruby/object:Gem::Version
|
46
53
|
version: '0'
|
47
|
-
- !ruby/object:Gem::Dependency
|
48
54
|
requirement: !ruby/object:Gem::Requirement
|
49
55
|
requirements:
|
50
56
|
- - '>='
|
51
57
|
- !ruby/object:Gem::Version
|
52
58
|
version: '0'
|
53
|
-
name: logstash-filter-grok
|
54
59
|
prerelease: false
|
55
60
|
type: :runtime
|
61
|
+
- !ruby/object:Gem::Dependency
|
62
|
+
name: logstash-devutils
|
56
63
|
version_requirements: !ruby/object:Gem::Requirement
|
57
64
|
requirements:
|
58
65
|
- - '>='
|
59
66
|
- !ruby/object:Gem::Version
|
60
67
|
version: '0'
|
61
|
-
- !ruby/object:Gem::Dependency
|
62
68
|
requirement: !ruby/object:Gem::Requirement
|
63
69
|
requirements:
|
64
70
|
- - '>='
|
65
71
|
- !ruby/object:Gem::Version
|
66
72
|
version: '0'
|
67
|
-
name: logstash-devutils
|
68
73
|
prerelease: false
|
69
74
|
type: :development
|
70
|
-
version_requirements: !ruby/object:Gem::Requirement
|
71
|
-
requirements:
|
72
|
-
- - '>='
|
73
|
-
- !ruby/object:Gem::Version
|
74
|
-
version: '0'
|
75
75
|
description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
|
76
76
|
email: info@elastic.co
|
77
77
|
executables: []
|