logstash-filter-mutate 2.0.2 → 2.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: a2373e3dacb12fba8e76b73ad1a25ae4936fa4ca
4
- data.tar.gz: 61b0f5308de9d387655f822ce13cd15536fa9b1f
3
+ metadata.gz: fd1c151098361f3b30635ab7d3091585514f2f6a
4
+ data.tar.gz: bc60754791477e9e23211cc2a2da52e5f26e95de
5
5
  SHA512:
6
- metadata.gz: 6590939b5d4885d99efa079852caca9c99571b4296c9f0eec30a0cf54e52f56031231897818ce7fd67e10a15b8643b71ad53ea6b5717160fe9a27750cb7ac672
7
- data.tar.gz: c629d3ee574f556fce2b33dfff71f77864811d77761401bb117aebdfc965aacc1366046a948b97f44121032d2af327c7013ddb3ebf67d10244ed28aa22e4dcf5
6
+ metadata.gz: 59c5e5b86234a514c5cf5ce735c05f788117fdbd7aff5476a0b2fb034022c7a7e35f68c76132f381d42a63fa8f5ccfd654899afcbb2b9b16af3569dd5a9ecdc5
7
+ data.tar.gz: 49b29f761184665f0f4da21a2bb3a692732d5c801f93cd67d63b72cdb91613194f77cf031102f33c99a748eb683c0b11a6e1bffad3ed2c3c579b6c3f81d1125f
data/CHANGELOG.md CHANGED
@@ -1,5 +1,8 @@
1
+ ## 2.0.3
2
+ - Code cleanups and fix field assignments
3
+
1
4
  ## 2.0.0
2
- - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
5
+ - Plugins were updated to follow the new shutdown semantic, this mainly allows Logstash to instruct input plugins to terminate gracefully,
3
6
  instead of using Thread.raise on the plugins' threads. Ref: https://github.com/elastic/logstash/pull/3895
4
7
  - Dependency on logstash-core update to 2.0
5
8
 
data/README.md CHANGED
@@ -1,5 +1,8 @@
1
1
  # Logstash Plugin
2
2
 
3
+ [![Build
4
+ Status](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-mutate-unit/badge/icon)](http://build-eu-00.elastic.co/view/LS%20Plugins/view/LS%20Filters/job/logstash-plugin-filter-mutate-unit/)
5
+
3
6
  This is a plugin for [Logstash](https://github.com/elastic/logstash).
4
7
 
5
8
  It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
@@ -33,7 +33,7 @@ class LogStash::Filters::Mutate < LogStash::Filters::Base
33
33
  # filters.
34
34
  config :remove, :validate => :array, :deprecated => true
35
35
 
36
- # Replace a field with a new value. The new value can include %{foo} strings
36
+ # Replace a field with a new value. The new value can include `%{foo}` strings
37
37
  # to help you build a new value from other parts of the event.
38
38
  #
39
39
  # Example:
@@ -176,24 +176,33 @@ class LogStash::Filters::Mutate < LogStash::Filters::Base
176
176
  # }
177
177
  config :merge, :validate => :hash
178
178
 
179
- public
179
+ TRUE_REGEX = (/^(true|t|yes|y|1)$/i).freeze
180
+ FALSE_REGEX = (/^(false|f|no|n|0)$/i).freeze
181
+ CONVERT_PREFIX = "convert_".freeze
182
+
180
183
  def register
181
184
  valid_conversions = %w(string integer float boolean)
182
185
  # TODO(sissel): Validate conversion requests if provided.
183
186
  @convert.nil? or @convert.each do |field, type|
184
187
  if !valid_conversions.include?(type)
185
- raise LogStash::ConfigurationError, I18n.t("logstash.agent.configuration.invalid_plugin_register",
186
- :plugin => "filter", :type => "mutate",
187
- :error => "Invalid conversion type '#{type}', expected one of '#{valid_conversions.join(',')}'")
188
+ raise LogStash::ConfigurationError, I18n.t(
189
+ "logstash.agent.configuration.invalid_plugin_register",
190
+ :plugin => "filter",
191
+ :type => "mutate",
192
+ :error => "Invalid conversion type '#{type}', expected one of '#{valid_conversions.join(',')}'"
193
+ )
188
194
  end
189
- end # @convert.each
195
+ end
190
196
 
191
197
  @gsub_parsed = []
192
198
  @gsub.nil? or @gsub.each_slice(3) do |field, needle, replacement|
193
199
  if [field, needle, replacement].any? {|n| n.nil?}
194
- raise LogStash::ConfigurationError, I18n.t("logstash.agent.configuration.invalid_plugin_register",
195
- :plugin => "filter", :type => "mutate",
196
- :error => "Invalid gsub configuration #{[field, needle, replacement]}. gsub requires 3 non-nil elements per config entry")
200
+ raise LogStash::ConfigurationError, I18n.t(
201
+ "logstash.agent.configuration.invalid_plugin_register",
202
+ :plugin => "filter",
203
+ :type => "mutate",
204
+ :error => "Invalid gsub configuration #{[field, needle, replacement]}. gsub requires 3 non-nil elements per config entry"
205
+ )
197
206
  end
198
207
 
199
208
  @gsub_parsed << {
@@ -202,12 +211,9 @@ class LogStash::Filters::Mutate < LogStash::Filters::Base
202
211
  :replacement => replacement
203
212
  }
204
213
  end
205
- end # def register
214
+ end
206
215
 
207
- public
208
216
  def filter(event)
209
-
210
-
211
217
  rename(event) if @rename
212
218
  update(event) if @update
213
219
  replace(event) if @replace
@@ -222,17 +228,17 @@ class LogStash::Filters::Mutate < LogStash::Filters::Base
222
228
  merge(event) if @merge
223
229
 
224
230
  filter_matched(event)
225
- end # def filter
231
+ end
226
232
 
227
233
  private
234
+
228
235
  def remove(event)
229
236
  @remove.each do |field|
230
237
  field = event.sprintf(field)
231
238
  event.remove(field)
232
239
  end
233
- end # def remove
240
+ end
234
241
 
235
- private
236
242
  def rename(event)
237
243
  @rename.each do |old, new|
238
244
  old = event.sprintf(old)
@@ -240,98 +246,94 @@ class LogStash::Filters::Mutate < LogStash::Filters::Base
240
246
  next unless event.include?(old)
241
247
  event[new] = event.remove(old)
242
248
  end
243
- end # def rename
249
+ end
244
250
 
245
- private
246
251
  def update(event)
247
252
  @update.each do |field, newvalue|
248
253
  next unless event.include?(field)
249
254
  event[field] = event.sprintf(newvalue)
250
255
  end
251
- end # def update
256
+ end
252
257
 
253
- private
254
258
  def replace(event)
255
259
  @replace.each do |field, newvalue|
256
260
  event[field] = event.sprintf(newvalue)
257
261
  end
258
- end # def replace
262
+ end
259
263
 
260
264
  def convert(event)
261
265
  @convert.each do |field, type|
262
266
  next unless event.include?(field)
263
267
  original = event[field]
264
268
  # calls convert_{string,integer,float,boolean} depending on type requested.
265
- converter = method("convert_" + type)
266
- if original.nil?
267
- next
268
- elsif original.is_a?(Hash)
269
- @logger.debug("I don't know how to type convert a hash, skipping",
270
- :field => field, :value => original)
271
- next
272
- elsif original.is_a?(Array)
273
- value = original.map { |v| converter.call(v) }
269
+ converter = method(CONVERT_PREFIX + type)
270
+
271
+ case original
272
+ when Hash
273
+ @logger.debug? && @logger.debug("I don't know how to type convert a hash, skipping", :field => field, :value => original)
274
+ when Array
275
+ event[field] = original.map { |v| converter.call(v) }
276
+ when NilClass
277
+ # ignore
274
278
  else
275
- value = converter.call(original)
279
+ event[field] = converter.call(original)
276
280
  end
277
- event[field] = value
278
281
  end
279
- end # def convert
282
+ end
280
283
 
281
284
  def convert_string(value)
282
285
  # since this is a filter and all inputs should be already UTF-8
283
286
  # we wont check valid_encoding? but just force UTF-8 for
284
287
  # the Fixnum#to_s case which always result in US-ASCII
288
+ # also not that force_encoding checks current encoding against the
289
+ # target encoding and only change if necessary, so calling
290
+ # valid_encoding? is redundant
285
291
  # see https://twitter.com/jordansissel/status/444613207143903232
286
- return value.to_s.force_encoding(Encoding::UTF_8)
287
- end # def convert_string
292
+ value.to_s.force_encoding(Encoding::UTF_8)
293
+ end
288
294
 
289
295
  def convert_integer(value)
290
- return value.to_i
291
- end # def convert_integer
296
+ value.to_i
297
+ end
292
298
 
293
299
  def convert_float(value)
294
- return value.to_f
295
- end # def convert_float
300
+ value.to_f
301
+ end
296
302
 
297
303
  def convert_boolean(value)
298
- return true if value =~ (/^(true|t|yes|y|1)$/i)
299
- return false if value.empty? || value =~ (/^(false|f|no|n|0)$/i)
304
+ return true if value =~ TRUE_REGEX
305
+ return false if value.empty? || value =~ FALSE_REGEX
300
306
  @logger.warn("Failed to convert #{value} into boolean.")
301
- return value
302
- end # def convert_boolean
307
+ value
308
+ end
303
309
 
304
- private
305
310
  def gsub(event)
306
311
  @gsub_parsed.each do |config|
307
312
  field = config[:field]
308
313
  needle = config[:needle]
309
314
  replacement = config[:replacement]
310
315
 
311
- if event[field].is_a?(Array)
312
- event[field] = event[field].map do |v|
313
- if not v.is_a?(String)
314
- @logger.warn("gsub mutation is only applicable for Strings, " +
315
- "skipping", :field => field, :value => v)
316
- v
317
- else
316
+ value = event[field]
317
+ case value
318
+ when Array
319
+ event[field] = value.map do |v|
320
+ if v.is_a?(String)
318
321
  gsub_dynamic_fields(event, v, needle, replacement)
322
+ else
323
+ @logger.warn("gsub mutation is only applicable for Strings, skipping", :field => field, :value => v)
324
+ v
319
325
  end
320
326
  end
327
+ when String
328
+ event[field] = gsub_dynamic_fields(event, value, needle, replacement)
321
329
  else
322
- if not event[field].is_a?(String)
323
- @logger.debug("gsub mutation is only applicable for Strings, " +
324
- "skipping", :field => field, :value => event[field])
325
- next
326
- end
327
- event[field] = gsub_dynamic_fields(event, event[field], needle, replacement)
330
+ @logger.debug? && @logger.debug("gsub mutation is only applicable for Strings, skipping", :field => field, :value => event[field])
328
331
  end
329
- end # @gsub_parsed.each
330
- end # def gsub
332
+ end
333
+ end
331
334
 
332
- private
333
335
  def gsub_dynamic_fields(event, original, needle, replacement)
334
- if needle.is_a? Regexp
336
+ if needle.is_a?(Regexp)
335
337
  original.gsub(needle, event.sprintf(replacement))
336
338
  else
337
339
  # we need to replace any dynamic fields
@@ -339,7 +341,6 @@ class LogStash::Filters::Mutate < LogStash::Filters::Base
339
341
  end
340
342
  end
341
343
 
342
- private
343
344
  def uppercase(event)
344
345
  @uppercase.each do |field|
345
346
  original = event[field]
@@ -347,95 +348,100 @@ class LogStash::Filters::Mutate < LogStash::Filters::Base
347
348
  # therefore we can't assume that we are modifying the actual value behind
348
349
  # the key so read, modify and overwrite
349
350
  event[field] = case original
350
- when Array
351
- # can't map upcase! as it replaces an already upcase value with nil
352
- # ["ABCDEF"].map(&:upcase!) => [nil]
353
- original.map do |elem|
354
- (elem.is_a?(String) ? elem.upcase : elem)
355
- end
356
- when String
357
- # nil means no change was made to the String
358
- original.upcase! || original
359
- else
360
- @logger.debug("Can't uppercase something that isn't a string",
361
- :field => field, :value => original)
362
- original
351
+ when Array
352
+ # can't map upcase! as it replaces an already upcase value with nil
353
+ # ["ABCDEF"].map(&:upcase!) => [nil]
354
+ original.map do |elem|
355
+ (elem.is_a?(String) ? elem.upcase : elem)
363
356
  end
357
+ when String
358
+ # nil means no change was made to the String
359
+ original.upcase! || original
360
+ else
361
+ @logger.debug? && @logger.debug("Can't uppercase something that isn't a string", :field => field, :value => original)
362
+ original
363
+ end
364
364
  end
365
- end # def uppercase
365
+ end
366
366
 
367
- private
368
367
  def lowercase(event)
369
368
  #see comments for #uppercase
370
369
  @lowercase.each do |field|
371
370
  original = event[field]
372
371
  event[field] = case original
373
- when Array
374
- original.map! do |elem|
375
- (elem.is_a?(String) ? elem.downcase : elem)
376
- end
377
- when String
378
- original.downcase! || original
379
- else
380
- @logger.debug("Can't lowercase something that isn't a string",
381
- :field => field, :value => original)
382
- original
372
+ when Array
373
+ original.map! do |elem|
374
+ (elem.is_a?(String) ? elem.downcase : elem)
383
375
  end
376
+ when String
377
+ original.downcase! || original
378
+ else
379
+ @logger.debug? && @logger.debug("Can't lowercase something that isn't a string", :field => field, :value => original)
380
+ original
381
+ end
384
382
  end
385
- end # def lowercase
386
-
383
+ end
387
384
 
388
- private
389
385
  def split(event)
390
386
  @split.each do |field, separator|
391
- if event[field].is_a?(String)
392
- event[field] = event[field].split(separator)
387
+ value = event[field]
388
+ if value.is_a?(String)
389
+ event[field] = value.split(separator)
393
390
  else
394
- @logger.debug("Can't split something that isn't a string",
395
- :field => field, :value => event[field])
391
+ @logger.debug? && @logger.debug("Can't split something that isn't a string", :field => field, :value => event[field])
396
392
  end
397
393
  end
398
394
  end
399
395
 
400
- private
401
396
  def join(event)
402
397
  @join.each do |field, separator|
403
- if event[field].is_a?(Array)
404
- event[field] = event[field].join(separator)
398
+ value = event[field]
399
+ if value.is_a?(Array)
400
+ event[field] = value.join(separator)
405
401
  end
406
402
  end
407
403
  end
408
404
 
409
- private
410
405
  def strip(event)
411
406
  @strip.each do |field|
412
- if event[field].is_a?(Array)
413
- event[field] = event[field].map{|s| s.strip }
414
- elsif event[field].is_a?(String)
415
- event[field] = event[field].strip
407
+ value = event[field]
408
+ case value
409
+ when Array
410
+ event[field] = value.map{|s| s.strip }
411
+ when String
412
+ event[field] = value.strip
416
413
  end
417
414
  end
418
415
  end
419
416
 
420
- private
421
417
  def merge(event)
422
418
  @merge.each do |dest_field, added_fields|
423
419
  # When multiple calls, added_field is an array
420
+
421
+ dest_field_value = event[dest_field]
422
+
424
423
  Array(added_fields).each do |added_field|
425
- if event[dest_field].is_a?(Hash) ^ event[added_field].is_a?(Hash)
424
+ added_field_value = event[added_field]
425
+
426
+ if dest_field_value.is_a?(Hash) ^ added_field_value.is_a?(Hash)
426
427
  @logger.error("Not possible to merge an array and a hash: ", :dest_field => dest_field, :added_field => added_field )
427
428
  next
428
429
  end
429
430
 
430
- if event[dest_field].is_a?(Hash)
431
- # No need to test the other
432
- event[dest_field].update(event[added_field])
431
+ # No need to test the other
432
+ if dest_field_value.is_a?(Hash)
433
+ # do not use event[dest_field].update because the returned object from event[dest_field]
434
+ # can/will be a copy of the actual event data and directly updating it will not update
435
+ # the Event internal data. The updated value must be reassigned in the Event.
436
+ event[dest_field] = dest_field_value.update(added_field_value)
433
437
  else
434
- event[dest_field] = Array(event[dest_field])
435
- event[dest_field].concat(Array(event[added_field]))
438
+ # do not use event[dest_field].concat because the returned object from event[dest_field]
439
+ # can/will be a copy of the actual event data and directly updating it will not update
440
+ # the Event internal data. The updated value must be reassigned in the Event.
441
+ event[dest_field] = Array(dest_field_value).concat(Array(added_field_value))
436
442
  end
437
443
  end
438
444
  end
439
445
  end
440
446
 
441
- end # class LogStash::Filters::Mutate
447
+ end
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-filter-mutate'
4
- s.version = '2.0.2'
4
+ s.version = '2.0.3'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = "The mutate filter allows you to perform general mutations on fields. You can rename, remove, replace, and modify fields in your events."
7
7
  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
metadata CHANGED
@@ -1,17 +1,18 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-filter-mutate
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.0.2
4
+ version: 2.0.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elastic
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-10-14 00:00:00.000000000 Z
11
+ date: 2015-12-16 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
- requirement: !ruby/object:Gem::Requirement
14
+ name: logstash-core
15
+ version_requirements: !ruby/object:Gem::Requirement
15
16
  requirements:
16
17
  - - '>='
17
18
  - !ruby/object:Gem::Version
@@ -19,10 +20,7 @@ dependencies:
19
20
  - - <
20
21
  - !ruby/object:Gem::Version
21
22
  version: 3.0.0
22
- name: logstash-core
23
- prerelease: false
24
- type: :runtime
25
- version_requirements: !ruby/object:Gem::Requirement
23
+ requirement: !ruby/object:Gem::Requirement
26
24
  requirements:
27
25
  - - '>='
28
26
  - !ruby/object:Gem::Version
@@ -30,48 +28,50 @@ dependencies:
30
28
  - - <
31
29
  - !ruby/object:Gem::Version
32
30
  version: 3.0.0
31
+ prerelease: false
32
+ type: :runtime
33
33
  - !ruby/object:Gem::Dependency
34
+ name: logstash-patterns-core
35
+ version_requirements: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - '>='
38
+ - !ruby/object:Gem::Version
39
+ version: '0'
34
40
  requirement: !ruby/object:Gem::Requirement
35
41
  requirements:
36
42
  - - '>='
37
43
  - !ruby/object:Gem::Version
38
44
  version: '0'
39
- name: logstash-patterns-core
40
45
  prerelease: false
41
46
  type: :runtime
47
+ - !ruby/object:Gem::Dependency
48
+ name: logstash-filter-grok
42
49
  version_requirements: !ruby/object:Gem::Requirement
43
50
  requirements:
44
51
  - - '>='
45
52
  - !ruby/object:Gem::Version
46
53
  version: '0'
47
- - !ruby/object:Gem::Dependency
48
54
  requirement: !ruby/object:Gem::Requirement
49
55
  requirements:
50
56
  - - '>='
51
57
  - !ruby/object:Gem::Version
52
58
  version: '0'
53
- name: logstash-filter-grok
54
59
  prerelease: false
55
60
  type: :runtime
61
+ - !ruby/object:Gem::Dependency
62
+ name: logstash-devutils
56
63
  version_requirements: !ruby/object:Gem::Requirement
57
64
  requirements:
58
65
  - - '>='
59
66
  - !ruby/object:Gem::Version
60
67
  version: '0'
61
- - !ruby/object:Gem::Dependency
62
68
  requirement: !ruby/object:Gem::Requirement
63
69
  requirements:
64
70
  - - '>='
65
71
  - !ruby/object:Gem::Version
66
72
  version: '0'
67
- name: logstash-devutils
68
73
  prerelease: false
69
74
  type: :development
70
- version_requirements: !ruby/object:Gem::Requirement
71
- requirements:
72
- - - '>='
73
- - !ruby/object:Gem::Version
74
- version: '0'
75
75
  description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
76
76
  email: info@elastic.co
77
77
  executables: []