fluent-plugin-mutate_filter 0.2.1 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +8 -2
- data/fluent-plugin-mutate_filter.gemspec +1 -1
- data/lib/fluent/plugin/filter_mutate.rb +65 -19
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 37b660658d4e6fd887a3955692e5bb9dc705af04
|
4
|
+
data.tar.gz: 960455819ab4a23dbe3672be563368a330f04699
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 8ef1cc5d483709fe9d4234094c5d57ab9a568467894928a6b8f8a3686019180dc06953246cea0391b561d0dcc78029128ed6460a5cacd6af979a0ad7cd0125fc
|
7
|
+
data.tar.gz: '0120295c8e6331d44e5aa3a98609401b9cb51cb8673c8ad29c4634e32c1f2824a40e0b072a0135efa5aba747dfdb1fb11dda765804a83de63b130fdaabf23a9e'
|
data/CHANGELOG.md
CHANGED
@@ -1,10 +1,16 @@
|
|
1
1
|
# Changelog
|
2
2
|
|
3
|
-
|
3
|
+
v0.3.0
|
4
|
+
-------------
|
5
|
+
* Resolve bug in error handling
|
6
|
+
* Add support for %e{} tags to replace with environment variables
|
7
|
+
* Add support for %e{hostname} tag to replace with hostname
|
8
|
+
|
9
|
+
v0.2.1
|
4
10
|
-------------
|
5
11
|
* Resolve bug in datetime converter
|
6
12
|
|
7
|
-
|
13
|
+
v0.2.0
|
8
14
|
-------------
|
9
15
|
* Fix MutateEvent initializer bug
|
10
16
|
* Add support for event_tag and event_time substitutions %{event_time}
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# coding: utf-8
|
2
2
|
Gem::Specification.new do |spec|
|
3
3
|
spec.name = "fluent-plugin-mutate_filter"
|
4
|
-
spec.version = "0.
|
4
|
+
spec.version = "0.3.0"
|
5
5
|
spec.authors = ["Jonathan Serafini"]
|
6
6
|
spec.email = ["jonathan@serafini.ca"]
|
7
7
|
spec.summary = %q{A mutate filter for Fluent which functions like Logstash.}
|
@@ -1,3 +1,4 @@
|
|
1
|
+
require 'socket'
|
1
2
|
require 'fluent/filter'
|
2
3
|
require 'fluent/plugin_mixin/mutate_event'
|
3
4
|
|
@@ -20,8 +21,8 @@ module Fluent
|
|
20
21
|
|
21
22
|
# Update an existing field with a new value.
|
22
23
|
# - If the field does not exist then no action will be taken.
|
23
|
-
# - If the new value contains a placeholder %{}, then the value will be
|
24
|
-
# expanded to the related event record field.
|
24
|
+
# - If the new value contains a placeholder %{}, then the value will be
|
25
|
+
# expanded to the related event record field.
|
25
26
|
# @example
|
26
27
|
# update {
|
27
28
|
# "message": "%{hostname}: new message"
|
@@ -59,7 +60,7 @@ module Fluent
|
|
59
60
|
|
60
61
|
# Convert a string field by applying a regular expression and replacement.
|
61
62
|
# - If the field is not a string, then no action will be taken.
|
62
|
-
#
|
63
|
+
#
|
63
64
|
# The configuration takes an array consisting of 3 elements per field/sub.
|
64
65
|
#
|
65
66
|
# @example
|
@@ -84,14 +85,14 @@ module Fluent
|
|
84
85
|
config_param :lowercase, :array, default: Array.new
|
85
86
|
|
86
87
|
# Strip whitespace from field.
|
87
|
-
# @example
|
88
|
+
# @example
|
88
89
|
# strip [
|
89
90
|
# "field1"
|
90
91
|
# ]
|
91
92
|
config_param :strip, :array, default: Array.new
|
92
93
|
|
93
94
|
# Split a field to an array using a separator character
|
94
|
-
# @example
|
95
|
+
# @example
|
95
96
|
# split {
|
96
97
|
# "field1": ","
|
97
98
|
# }
|
@@ -111,7 +112,7 @@ module Fluent
|
|
111
112
|
# }
|
112
113
|
config_param :merge, :hash, default: Hash.new
|
113
114
|
|
114
|
-
# List of all possible mutate actions, in the order that we will apply
|
115
|
+
# List of all possible mutate actions, in the order that we will apply
|
115
116
|
# them. As it stands, this is the order in which Logstash would apply them.
|
116
117
|
MUTATE_ACTIONS = %w(
|
117
118
|
rename
|
@@ -138,6 +139,9 @@ module Fluent
|
|
138
139
|
TRUE_REGEX = (/^(true|t|yes|y|1)$/i).freeze
|
139
140
|
FALSE_REGEX = (/^(false|f|no|n|0)$/i).freeze
|
140
141
|
|
142
|
+
# Placeholder regex
|
143
|
+
ENVIRONMENT_TAG_REGEXP = /%e\{[^}]+\}/
|
144
|
+
|
141
145
|
# Placeholder regex
|
142
146
|
TEMPLATE_TAG_REGEXP = /%\{[^}]+\}/
|
143
147
|
|
@@ -149,13 +153,13 @@ module Fluent
|
|
149
153
|
|
150
154
|
@convert.nil? or @convert.each do |field, type|
|
151
155
|
if !VALID_CONVERSIONS.include?(type)
|
152
|
-
raise ConfigError,
|
156
|
+
raise ConfigError,
|
153
157
|
"convert #{type} is not one of #{VALID_CONVERSIONS.join(',')}."
|
154
158
|
end
|
155
159
|
end
|
156
160
|
|
157
161
|
@gsub_parsed = []
|
158
|
-
@gsub.nil? or
|
162
|
+
@gsub.nil? or
|
159
163
|
@gsub.each_slice(3) do |field, needle, replacement|
|
160
164
|
if [field, needle, replacement].any? {|n| n.nil?}
|
161
165
|
raise ConfigError,
|
@@ -197,6 +201,15 @@ module Fluent
|
|
197
201
|
|
198
202
|
protected
|
199
203
|
|
204
|
+
# Expand replacable patterns on the event
|
205
|
+
# @since 0.3.0
|
206
|
+
# @return [String] the modified string
|
207
|
+
def expand_patterns(event, string)
|
208
|
+
string = expand_references(event, string)
|
209
|
+
string = expand_environment(event, string)
|
210
|
+
string
|
211
|
+
end
|
212
|
+
|
200
213
|
# Expand %{} strings to the related event fields.
|
201
214
|
# @since 0.1.0
|
202
215
|
# @return [String] the modified string
|
@@ -231,6 +244,39 @@ module Fluent
|
|
231
244
|
new_string
|
232
245
|
end
|
233
246
|
|
247
|
+
# Expand %e{} strings to the related environment variables.
|
248
|
+
# @since 0.3.0
|
249
|
+
# @return [String] the modified string
|
250
|
+
def expand_environment(event, string)
|
251
|
+
new_string = ''
|
252
|
+
|
253
|
+
position = 0
|
254
|
+
matches = string.scan(ENVIRONMENT_TAG_REGEXP).map{|m| $~}
|
255
|
+
|
256
|
+
matches.each do |match|
|
257
|
+
reference_tag = match[0][2..-2]
|
258
|
+
reference_value = case reference_tag
|
259
|
+
when "hostname" then Socket.gethostname
|
260
|
+
else ENV[reference_tag]
|
261
|
+
end
|
262
|
+
if reference_value.nil?
|
263
|
+
@log.error "failed to replace tag", field: reference_tag
|
264
|
+
reference_value = match.to_s
|
265
|
+
end
|
266
|
+
|
267
|
+
start = match.offset(0).first
|
268
|
+
new_string << string[position..(start-1)] if start > 0
|
269
|
+
new_string << reference_value
|
270
|
+
position = match.offset(0).last
|
271
|
+
end
|
272
|
+
|
273
|
+
if position < string.size
|
274
|
+
new_string << string[position..-1]
|
275
|
+
end
|
276
|
+
|
277
|
+
new_string
|
278
|
+
end
|
279
|
+
|
234
280
|
# Remove fields from the event hash
|
235
281
|
# @since 0.1.0
|
236
282
|
def remove(event)
|
@@ -254,7 +300,7 @@ module Fluent
|
|
254
300
|
# @since 0.1.0
|
255
301
|
def update(event)
|
256
302
|
@update.each do |field, newvalue|
|
257
|
-
newvalue =
|
303
|
+
newvalue = expand_patterns(event, newvalue)
|
258
304
|
next unless event.include?(field)
|
259
305
|
event.set(field, newvalue)
|
260
306
|
end
|
@@ -264,7 +310,7 @@ module Fluent
|
|
264
310
|
# @since 0.1.0
|
265
311
|
def replace(event)
|
266
312
|
@replace.each do |field, newvalue|
|
267
|
-
newvalue =
|
313
|
+
newvalue = expand_patterns(event, newvalue)
|
268
314
|
event.set(field, newvalue)
|
269
315
|
end
|
270
316
|
end
|
@@ -324,8 +370,8 @@ module Fluent
|
|
324
370
|
when String
|
325
371
|
original.upcase! || original
|
326
372
|
else
|
327
|
-
@log.error("can't uppercase field",
|
328
|
-
field: field,
|
373
|
+
@log.error("can't uppercase field",
|
374
|
+
field: field,
|
329
375
|
value: original)
|
330
376
|
original
|
331
377
|
end
|
@@ -346,8 +392,8 @@ module Fluent
|
|
346
392
|
when String
|
347
393
|
original.downcase! || original
|
348
394
|
else
|
349
|
-
@log.error("can't lowercase field",
|
350
|
-
field: field,
|
395
|
+
@log.error("can't lowercase field",
|
396
|
+
field: field,
|
351
397
|
value: original)
|
352
398
|
original
|
353
399
|
end
|
@@ -363,7 +409,7 @@ module Fluent
|
|
363
409
|
if value.is_a?(String)
|
364
410
|
event.set(field, value.split(separator))
|
365
411
|
else
|
366
|
-
@
|
412
|
+
@log.error("can't split field",
|
367
413
|
field: field,
|
368
414
|
value: value)
|
369
415
|
end
|
@@ -435,9 +481,9 @@ module Fluent
|
|
435
481
|
result = value.map do |v|
|
436
482
|
if v.is_a?(String)
|
437
483
|
gsub_dynamic_fields(event, v, needle, replacement)
|
438
|
-
else
|
439
|
-
@log.error('cannot gsub non Strings',
|
440
|
-
field: field,
|
484
|
+
else
|
485
|
+
@log.error('cannot gsub non Strings',
|
486
|
+
field: field,
|
441
487
|
value: v)
|
442
488
|
end
|
443
489
|
event.set(field, result)
|
@@ -452,7 +498,7 @@ module Fluent
|
|
452
498
|
end
|
453
499
|
|
454
500
|
def gsub_dynamic_fields(event, original, needle, replacement)
|
455
|
-
replacement =
|
501
|
+
replacement = expand_patterns(event, replacement)
|
456
502
|
if needle.is_a?(Regexp)
|
457
503
|
original.gsub(needle, replacement)
|
458
504
|
else
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-mutate_filter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Jonathan Serafini
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-11-16 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -94,7 +94,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
94
94
|
version: '0'
|
95
95
|
requirements: []
|
96
96
|
rubyforge_project:
|
97
|
-
rubygems_version: 2.2
|
97
|
+
rubygems_version: 2.5.2
|
98
98
|
signing_key:
|
99
99
|
specification_version: 4
|
100
100
|
summary: A mutate filter for Fluent which functions like Logstash.
|