fluent-plugin-rewrite-tag-filter 2.3.0 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +30 -0
- data/fluent-plugin-rewrite-tag-filter.gemspec +1 -1
- data/lib/fluent/plugin/out_rewrite_tag_filter.rb +34 -22
- data/test/plugin/test_out_rewrite_tag_filter.rb +71 -0
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a12e8f6567cd07462a236f0cda38e98bac14b921a2c7b762e9851cb04e5bf119
|
4
|
+
data.tar.gz: b198b1e6b8a48d386f7cc62521f358cdff745754edc8dda753ac176ced1e3e67
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cc556895e92c3f53ca3c41af6866072e48ac53b6b170cfe163f1563d04310c8b89cc802cabefd957dab95da29362ff76164621f0a83d806619d67df521e866b6
|
7
|
+
data.tar.gz: 1777b76d00afe128f20da230ca26d0c30d4f62e60f4800efec6b59450040e54de7b861ee7913ed82af8f91b9f3cf5e11344b5ec5b08e57af97d611a0060d0ddf
|
data/README.md
CHANGED
@@ -52,6 +52,7 @@ For more details, see [Plugin Management](https://docs.fluentd.org/deployment/pl
|
|
52
52
|
`/regexp/` is preferred because `/regexp/` style can support character classes such as `/[a-z]/`.
|
53
53
|
The pattern without slashes will cause errors if you use patterns start with character classes.
|
54
54
|
* **tag** (string) (required): New tag
|
55
|
+
* **label** (string) (optional): New label. If specified, label can be changed per-rule.
|
55
56
|
* **invert** (bool) (optional): If true, rewrite tag when unmatch pattern
|
56
57
|
* Default value: `false`
|
57
58
|
|
@@ -300,6 +301,35 @@ It's a sample to rewrite a tag with placeholder.
|
|
300
301
|
</match>
|
301
302
|
```
|
302
303
|
|
304
|
+
### Altering Labels
|
305
|
+
|
306
|
+
In addition to changing tags, you can also change event's route by setting
|
307
|
+
the label for the re-emitted event.
|
308
|
+
|
309
|
+
For example, given this configuration:
|
310
|
+
|
311
|
+
```
|
312
|
+
<match apache.access>
|
313
|
+
@type rewrite_tag_filter
|
314
|
+
<rule>
|
315
|
+
key domain
|
316
|
+
pattern ^www\.example\.com$
|
317
|
+
tag web.${tag}
|
318
|
+
</rule>
|
319
|
+
<rule>
|
320
|
+
key domain
|
321
|
+
pattern ^(.*)\.example\.com$
|
322
|
+
tag other.$1
|
323
|
+
label other
|
324
|
+
</rule>
|
325
|
+
</match>
|
326
|
+
```
|
327
|
+
|
328
|
+
message: `{"domain": "www.example.com"}` will get its tag changed to
|
329
|
+
`web.apache.access`, while message
|
330
|
+
`{"domain": "api.example.com"}` will get its tag changed to `other.api` and
|
331
|
+
be sent to label `other`
|
332
|
+
|
303
333
|
## Example
|
304
334
|
|
305
335
|
- Example1: how to analyze response_time, response_code and user_agent for each virtual domain websites.
|
@@ -2,7 +2,7 @@ $:.push File.expand_path("../lib", __FILE__)
|
|
2
2
|
|
3
3
|
Gem::Specification.new do |s|
|
4
4
|
s.name = "fluent-plugin-rewrite-tag-filter"
|
5
|
-
s.version = "2.
|
5
|
+
s.version = "2.4.0"
|
6
6
|
s.license = "Apache-2.0"
|
7
7
|
s.authors = ["Kentaro Yoshida"]
|
8
8
|
s.email = ["y.ken.studio@gmail.com"]
|
@@ -24,6 +24,8 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
|
|
24
24
|
config_param :pattern, :regexp
|
25
25
|
desc "New tag"
|
26
26
|
config_param :tag, :string
|
27
|
+
desc "New label. If specified, label can be changed per-rule."
|
28
|
+
config_param :label, :string, default: nil
|
27
29
|
desc "If true, rewrite tag when unmatch pattern"
|
28
30
|
config_param :invert, :bool, default: false
|
29
31
|
end
|
@@ -43,7 +45,7 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
|
|
43
45
|
end
|
44
46
|
|
45
47
|
invert = rule.invert ? MATCH_OPERATOR_EXCLUDE : ""
|
46
|
-
@rewriterules.push([record_accessor_create(rule.key), rule.pattern, invert, rule.tag])
|
48
|
+
@rewriterules.push([record_accessor_create(rule.key), rule.pattern, invert, rule.tag, rule.label])
|
47
49
|
rewriterule_names.push(rule.key + invert + rule.pattern.to_s)
|
48
50
|
log.info "adding rewrite_tag_filter rule: #{rule.key} #{@rewriterules.last}"
|
49
51
|
end
|
@@ -75,35 +77,45 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
|
|
75
77
|
true
|
76
78
|
end
|
77
79
|
|
80
|
+
def get_router(tgt_label)
|
81
|
+
label_router = if tgt_label.nil? || tgt_label.empty?
|
82
|
+
router
|
83
|
+
else
|
84
|
+
event_emitter_router(tgt_label)
|
85
|
+
end
|
86
|
+
#log.trace "Got router for #{tgt_label}: #{!label_router.nil?}"
|
87
|
+
return label_router
|
88
|
+
end
|
89
|
+
|
78
90
|
def process(tag, es)
|
79
91
|
placeholder = get_placeholder(tag)
|
80
|
-
if @batch_mode
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
end
|
88
|
-
new_event_streams[rewrited_tag].add(time, record)
|
92
|
+
new_event_streams = Hash.new {|hh, kk| hh[kk] = Hash.new {|h, k| h[k] = Fluent::MultiEventStream.new }}if @batch_mode
|
93
|
+
|
94
|
+
es.each do |time, record|
|
95
|
+
rewrited_tag, rewrited_label = rewrite_tag(tag, record, placeholder)
|
96
|
+
if (rewrited_tag.nil? || tag == rewrited_tag) && rewrited_label.nil?
|
97
|
+
log.trace("rewrite_tag_filter: tag has not been rewritten", record)
|
98
|
+
next
|
89
99
|
end
|
90
|
-
|
91
|
-
|
100
|
+
rewrited_tag = tag if rewrited_tag.nil?
|
101
|
+
if new_event_streams.nil?
|
102
|
+
get_router(rewrited_label).emit(rewrited_tag, time, record)
|
103
|
+
else
|
104
|
+
new_event_streams[rewrited_label][rewrited_tag].add(time, record)
|
92
105
|
end
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
106
|
+
end
|
107
|
+
if !new_event_streams.nil?
|
108
|
+
new_event_streams.each do |rewrited_label, label_event_streams |
|
109
|
+
labeled_router = get_router(rewrited_label)
|
110
|
+
label_event_streams.each do |rewrited_tag, new_es|
|
111
|
+
labeled_router.emit_stream(rewrited_tag, new_es)
|
99
112
|
end
|
100
|
-
router.emit(rewrited_tag, time, record)
|
101
113
|
end
|
102
114
|
end
|
103
115
|
end
|
104
116
|
|
105
117
|
def rewrite_tag(tag, record, placeholder)
|
106
|
-
@rewriterules.each do |record_accessor, regexp, match_operator, rewritetag|
|
118
|
+
@rewriterules.each do |record_accessor, regexp, match_operator, rewritetag, rewritelabel|
|
107
119
|
rewritevalue = record_accessor.call(record).to_s
|
108
120
|
next if rewritevalue.empty? && match_operator != MATCH_OPERATOR_EXCLUDE
|
109
121
|
last_match = regexp_last_match(regexp, rewritevalue)
|
@@ -119,9 +131,9 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
|
|
119
131
|
log.warn "rewrite_tag_filter: unknown placeholder found. :placeholder=>#{$1} :tag=>#{tag} :rewritetag=>#{rewritetag}" unless placeholder.include?($1)
|
120
132
|
placeholder[$1]
|
121
133
|
end
|
122
|
-
return rewritetag
|
134
|
+
return rewritetag, rewritelabel
|
123
135
|
end
|
124
|
-
return nil
|
136
|
+
return nil, nil
|
125
137
|
end
|
126
138
|
|
127
139
|
def regexp_last_match(regexp, rewritevalue)
|
@@ -346,6 +346,77 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
|
|
346
346
|
assert_equal "com.example", events[0][0]
|
347
347
|
end
|
348
348
|
|
349
|
+
test "get_router" do
|
350
|
+
conf = %[
|
351
|
+
<rule>
|
352
|
+
key key
|
353
|
+
pattern /^(odd|even)$/
|
354
|
+
tag $1
|
355
|
+
label new_label
|
356
|
+
</rule>
|
357
|
+
<rule>
|
358
|
+
key key
|
359
|
+
pattern /^(.*)$/
|
360
|
+
tag $1
|
361
|
+
</rule>
|
362
|
+
]
|
363
|
+
time = event_time
|
364
|
+
d = create_driver(conf)
|
365
|
+
assert_equal(d.instance.router, d.instance.get_router(nil))
|
366
|
+
assert_equal(d.instance.router, d.instance.get_router(""))
|
367
|
+
new_label_router = d.instance.get_router("new_label")
|
368
|
+
refute_equal(d.instance.router, new_label_router)
|
369
|
+
end
|
370
|
+
|
371
|
+
test "relabel" do
|
372
|
+
conf = %[
|
373
|
+
emit_mode record
|
374
|
+
<rule>
|
375
|
+
key key
|
376
|
+
pattern /^(odd)$/
|
377
|
+
tag $1
|
378
|
+
label odd_label
|
379
|
+
</rule>
|
380
|
+
<rule>
|
381
|
+
key key
|
382
|
+
pattern /^(even)$/
|
383
|
+
tag ${tag}
|
384
|
+
label even_label
|
385
|
+
</rule>
|
386
|
+
<rule>
|
387
|
+
key key
|
388
|
+
pattern /^(.*)$/
|
389
|
+
tag $1
|
390
|
+
</rule>
|
391
|
+
]
|
392
|
+
time = event_time
|
393
|
+
d = create_driver(conf)
|
394
|
+
# Router only called for default label
|
395
|
+
mock.proxy(d.instance.router).emit(anything, anything, anything).times(2)
|
396
|
+
mock.proxy(d.instance).get_router("odd_label").times(2)
|
397
|
+
mock.proxy(d.instance).get_router("even_label").times(2)
|
398
|
+
mock.proxy(d.instance).get_router(nil).times(2)
|
399
|
+
mock.proxy(d.instance.router).emit_stream(anything, anything).times(0)
|
400
|
+
d.run(default_tag: "input") do
|
401
|
+
d.feed([[time, { "key" => "odd", "message" => "message-1" }],
|
402
|
+
[time, { "key" => "even", "message" => "message-2" }],
|
403
|
+
[time, { "key" => "zero", "message" => "message-3" }],
|
404
|
+
[time, { "key" => "odd", "message" => "message-4" }],
|
405
|
+
[time, { "key" => "even", "message" => "message-5" }],
|
406
|
+
[time, { "key" => "zero", "message" => "message-6" }]])
|
407
|
+
end
|
408
|
+
events = d.events
|
409
|
+
expected_events = [
|
410
|
+
["odd", time, { "key" => "odd", "message" => "message-1" }],
|
411
|
+
["input", time, { "key" => "even", "message" => "message-2" }],
|
412
|
+
["zero", time, { "key" => "zero", "message" => "message-3" }],
|
413
|
+
["odd", time, { "key" => "odd", "message" => "message-4" }],
|
414
|
+
["input", time, { "key" => "even", "message" => "message-5" }],
|
415
|
+
["zero", time, { "key" => "zero", "message" => "message-6" }],
|
416
|
+
]
|
417
|
+
assert_equal(events, expected_events)
|
418
|
+
end
|
419
|
+
|
349
420
|
sub_test_case "emit_mode" do
|
350
421
|
test "record" do
|
351
422
|
conf = %[
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-rewrite-tag-filter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kentaro Yoshida
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-12-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: test-unit
|
@@ -124,7 +124,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
124
124
|
- !ruby/object:Gem::Version
|
125
125
|
version: '0'
|
126
126
|
requirements: []
|
127
|
-
rubygems_version: 3.1.
|
127
|
+
rubygems_version: 3.1.4
|
128
128
|
signing_key:
|
129
129
|
specification_version: 4
|
130
130
|
summary: Fluentd Output filter plugin. It has designed to rewrite tag like mod_rewrite.
|