fluent-plugin-rewrite-tag-filter 2.1.1 → 2.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +4 -2
- data/fluent-plugin-rewrite-tag-filter.gemspec +3 -2
- data/lib/fluent/plugin/out_rewrite_tag_filter.rb +25 -6
- data/test/helper.rb +4 -0
- data/test/plugin/test_out_rewrite_tag_filter.rb +68 -0
- metadata +17 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8253bb2b1859438ea797e1fd2046d0b18290a424b340de8a88524b9afd33388a
|
4
|
+
data.tar.gz: 33e91a5b0570810f2ad802885df4a7e87eb03c4816b5779d847c5004b25afb0a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f28d04ee233a9bfdd668f5dc114b7e9b7d5531d275d1c5318100d4a76171607ddf9e03041081b139ba2f7a55b6b73bdbf64c44cae67097df983ae80395d04e69
|
7
|
+
data.tar.gz: 4e6ee684e6da26e6d0e449cc0650dd482f9cbd5cd6623d540bb63b1b991e48f0ede38f9424f04ef8750e6d7f451afa62c89bda412e3bff66589b3101407414d3
|
data/README.md
CHANGED
@@ -36,12 +36,14 @@ For more details, see [Plugin Management](https://docs.fluentd.org/v0.14/article
|
|
36
36
|
## Configuration
|
37
37
|
|
38
38
|
* **rewriterule\<num\>** (string) (optional) \<attribute\> \<regex_pattern\> \<new_tag\>
|
39
|
-
* Obsoleted: Use
|
39
|
+
* Obsoleted: Use \<rule\> section
|
40
40
|
* **capitalize_regex_backreference** (bool) (optional): Capitalize letter for every matched regex backreference. (ex: maps -> Maps) for more details, see usage.
|
41
41
|
* Default value: no
|
42
42
|
* **remove_tag_prefix** (string) (optional): Remove tag prefix for tag placeholder. (see the section of "Tag placeholder")
|
43
43
|
* **hostname_command** (string) (optional): Override hostname command for placeholder. (see the section of "Tag placeholder")
|
44
|
-
* Default value: `hostname
|
44
|
+
* Default value: `hostname`
|
45
|
+
* **emit_mode** (enum) (required): Specify emit_mode to `batch` or `record`. `batch` will emit events per rewritten tag, and decrease IO. `record` will emit events per record.
|
46
|
+
* Default value: `batch`
|
45
47
|
|
46
48
|
### \<rule\> section (optional) (multiple)
|
47
49
|
|
@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = "fluent-plugin-rewrite-tag-filter"
|
6
|
-
s.version = "2.
|
6
|
+
s.version = "2.2.0"
|
7
7
|
s.license = "Apache-2.0"
|
8
8
|
s.authors = ["Kentaro Yoshida"]
|
9
9
|
s.email = ["y.ken.studio@gmail.com"]
|
@@ -16,7 +16,8 @@ Gem::Specification.new do |s|
|
|
16
16
|
s.require_paths = ["lib"]
|
17
17
|
|
18
18
|
s.add_development_dependency "test-unit", ">= 3.1.0"
|
19
|
+
s.add_development_dependency "test-unit-rr"
|
19
20
|
s.add_development_dependency "rake"
|
20
|
-
s.add_runtime_dependency "fluentd",
|
21
|
+
s.add_runtime_dependency "fluentd", ">= 0.14.2", "< 2"
|
21
22
|
s.add_runtime_dependency "fluent-config-regexp-type"
|
22
23
|
end
|
@@ -12,6 +12,8 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
|
|
12
12
|
config_param :remove_tag_prefix, :string, :default => nil
|
13
13
|
desc 'Override hostname command for placeholder.'
|
14
14
|
config_param :hostname_command, :string, :default => 'hostname'
|
15
|
+
desc "The emit mode. If `batch`, this plugin will emit events per rewritten tag."
|
16
|
+
config_param :emit_mode, :enum, list: [:record, :batch], default: :batch
|
15
17
|
|
16
18
|
config_section :rule, param_name: :rules, multi: true do
|
17
19
|
desc "The field name to which the regular expression is applied"
|
@@ -59,6 +61,8 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
|
|
59
61
|
unless @remove_tag_prefix.nil?
|
60
62
|
@remove_tag_prefix = /^#{Regexp.escape(@remove_tag_prefix)}\.?/
|
61
63
|
end
|
64
|
+
|
65
|
+
@batch_mode = @emit_mode == :batch
|
62
66
|
end
|
63
67
|
|
64
68
|
def multi_workers_ready?
|
@@ -67,13 +71,28 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
|
|
67
71
|
|
68
72
|
def process(tag, es)
|
69
73
|
placeholder = get_placeholder(tag)
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
74
|
+
if @batch_mode
|
75
|
+
new_event_streams = Hash.new {|h, k| h[k] = Fluent::MultiEventStream.new }
|
76
|
+
es.each do |time, record|
|
77
|
+
rewrited_tag = rewrite_tag(tag, record, placeholder)
|
78
|
+
if rewrited_tag.nil? || tag == rewrited_tag
|
79
|
+
log.trace("rewrite_tag_filter: tag has not been rewritten", record)
|
80
|
+
next
|
81
|
+
end
|
82
|
+
new_event_streams[rewrited_tag].add(time, record)
|
83
|
+
end
|
84
|
+
new_event_streams.each do |rewrited_tag, new_es|
|
85
|
+
router.emit_stream(rewrited_tag, new_es)
|
86
|
+
end
|
87
|
+
else
|
88
|
+
es.each do |time, record|
|
89
|
+
rewrited_tag = rewrite_tag(tag, record, placeholder)
|
90
|
+
if rewrited_tag.nil? || tag == rewrited_tag
|
91
|
+
log.trace("rewrite_tag_filter: tag has not been rewritten", record)
|
92
|
+
next
|
93
|
+
end
|
94
|
+
router.emit(rewrited_tag, time, record)
|
75
95
|
end
|
76
|
-
router.emit(rewrited_tag, time, record)
|
77
96
|
end
|
78
97
|
end
|
79
98
|
|
data/test/helper.rb
CHANGED
@@ -1,9 +1,13 @@
|
|
1
1
|
require 'bundler/setup'
|
2
2
|
require 'test/unit'
|
3
|
+
require 'test/unit/rr'
|
3
4
|
|
4
5
|
$LOAD_PATH.unshift(File.join(__dir__, '..', 'lib'))
|
5
6
|
$LOAD_PATH.unshift(__dir__)
|
6
7
|
require 'fluent/test'
|
7
8
|
require 'fluent/test/driver/output'
|
9
|
+
require 'fluent/test/helpers'
|
10
|
+
|
11
|
+
Test::Unit::TestCase.include(Fluent::Test::Helpers)
|
8
12
|
|
9
13
|
require 'fluent/plugin/out_rewrite_tag_filter'
|
@@ -306,5 +306,73 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
|
|
306
306
|
log.slice(/\[trace\]: (.+)$/, 1))
|
307
307
|
assert_equal "com.example", events[0][0]
|
308
308
|
end
|
309
|
+
|
310
|
+
sub_test_case "emit_mode" do
|
311
|
+
test "record" do
|
312
|
+
conf = %[
|
313
|
+
emit_mode record
|
314
|
+
<rule>
|
315
|
+
key key
|
316
|
+
pattern /^(odd|even)$/
|
317
|
+
tag $1
|
318
|
+
</rule>
|
319
|
+
]
|
320
|
+
time = event_time
|
321
|
+
d = create_driver(conf)
|
322
|
+
mock.proxy(d.instance.router).emit(anything, anything, anything).times(6)
|
323
|
+
mock.proxy(d.instance.router).emit_stream(anything, anything).times(0)
|
324
|
+
d.run(default_tag: "input") do
|
325
|
+
d.feed([[time, { "key" => "odd", "message" => "message-1" }],
|
326
|
+
[time, { "key" => "even", "message" => "message-2" }],
|
327
|
+
[time, { "key" => "odd", "message" => "message-3" }],
|
328
|
+
[time, { "key" => "even", "message" => "message-4" }],
|
329
|
+
[time, { "key" => "odd", "message" => "message-5" }],
|
330
|
+
[time, { "key" => "even", "message" => "message-6" }]])
|
331
|
+
end
|
332
|
+
events = d.events
|
333
|
+
expected_events = [
|
334
|
+
["odd", time, { "key" => "odd", "message" => "message-1" }],
|
335
|
+
["even", time, { "key" => "even", "message" => "message-2" }],
|
336
|
+
["odd", time, { "key" => "odd", "message" => "message-3" }],
|
337
|
+
["even", time, { "key" => "even", "message" => "message-4" }],
|
338
|
+
["odd", time, { "key" => "odd", "message" => "message-5" }],
|
339
|
+
["even", time, { "key" => "even", "message" => "message-6" }],
|
340
|
+
]
|
341
|
+
assert_equal(events, expected_events)
|
342
|
+
end
|
343
|
+
|
344
|
+
test "batch" do
|
345
|
+
conf = %[
|
346
|
+
emit_mode batch
|
347
|
+
<rule>
|
348
|
+
key key
|
349
|
+
pattern /^(odd|even)$/
|
350
|
+
tag $1
|
351
|
+
</rule>
|
352
|
+
]
|
353
|
+
time = event_time
|
354
|
+
d = create_driver(conf)
|
355
|
+
mock.proxy(d.instance.router).emit(anything, anything, anything).times(0)
|
356
|
+
mock.proxy(d.instance.router).emit_stream(anything, anything).times(2)
|
357
|
+
d.run(default_tag: "input") do
|
358
|
+
d.feed([[time, { "key" => "odd", "message" => "message-1" }],
|
359
|
+
[time, { "key" => "even", "message" => "message-2" }],
|
360
|
+
[time, { "key" => "odd", "message" => "message-3" }],
|
361
|
+
[time, { "key" => "even", "message" => "message-4" }],
|
362
|
+
[time, { "key" => "odd", "message" => "message-5" }],
|
363
|
+
[time, { "key" => "even", "message" => "message-6" }]])
|
364
|
+
end
|
365
|
+
events = d.events
|
366
|
+
expected_records = [
|
367
|
+
["odd", time, { "key" => "odd", "message" => "message-1" }],
|
368
|
+
["odd", time, { "key" => "odd", "message" => "message-3" }],
|
369
|
+
["odd", time, { "key" => "odd", "message" => "message-5" }],
|
370
|
+
["even", time, { "key" => "even", "message" => "message-2" }],
|
371
|
+
["even", time, { "key" => "even", "message" => "message-4" }],
|
372
|
+
["even", time, { "key" => "even", "message" => "message-6" }],
|
373
|
+
]
|
374
|
+
assert_equal(events, expected_records)
|
375
|
+
end
|
376
|
+
end
|
309
377
|
end
|
310
378
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-rewrite-tag-filter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kentaro Yoshida
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2019-03-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: test-unit
|
@@ -24,6 +24,20 @@ dependencies:
|
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: 3.1.0
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: test-unit-rr
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
27
41
|
- !ruby/object:Gem::Dependency
|
28
42
|
name: rake
|
29
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -110,8 +124,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
110
124
|
- !ruby/object:Gem::Version
|
111
125
|
version: '0'
|
112
126
|
requirements: []
|
113
|
-
|
114
|
-
rubygems_version: 2.7.6
|
127
|
+
rubygems_version: 3.0.1
|
115
128
|
signing_key:
|
116
129
|
specification_version: 4
|
117
130
|
summary: Fluentd Output filter plugin. It has designed to rewrite tag like mod_rewrite.
|