fluent-plugin-rewrite-tag-filter 2.1.1 → 2.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +4 -2
- data/fluent-plugin-rewrite-tag-filter.gemspec +3 -2
- data/lib/fluent/plugin/out_rewrite_tag_filter.rb +25 -6
- data/test/helper.rb +4 -0
- data/test/plugin/test_out_rewrite_tag_filter.rb +68 -0
- metadata +17 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8253bb2b1859438ea797e1fd2046d0b18290a424b340de8a88524b9afd33388a
|
4
|
+
data.tar.gz: 33e91a5b0570810f2ad802885df4a7e87eb03c4816b5779d847c5004b25afb0a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f28d04ee233a9bfdd668f5dc114b7e9b7d5531d275d1c5318100d4a76171607ddf9e03041081b139ba2f7a55b6b73bdbf64c44cae67097df983ae80395d04e69
|
7
|
+
data.tar.gz: 4e6ee684e6da26e6d0e449cc0650dd482f9cbd5cd6623d540bb63b1b991e48f0ede38f9424f04ef8750e6d7f451afa62c89bda412e3bff66589b3101407414d3
|
data/README.md
CHANGED
@@ -36,12 +36,14 @@ For more details, see [Plugin Management](https://docs.fluentd.org/v0.14/article
|
|
36
36
|
## Configuration
|
37
37
|
|
38
38
|
* **rewriterule\<num\>** (string) (optional) \<attribute\> \<regex_pattern\> \<new_tag\>
|
39
|
-
* Obsoleted: Use
|
39
|
+
* Obsoleted: Use \<rule\> section
|
40
40
|
* **capitalize_regex_backreference** (bool) (optional): Capitalize letter for every matched regex backreference. (ex: maps -> Maps) for more details, see usage.
|
41
41
|
* Default value: no
|
42
42
|
* **remove_tag_prefix** (string) (optional): Remove tag prefix for tag placeholder. (see the section of "Tag placeholder")
|
43
43
|
* **hostname_command** (string) (optional): Override hostname command for placeholder. (see the section of "Tag placeholder")
|
44
|
-
* Default value: `hostname
|
44
|
+
* Default value: `hostname`
|
45
|
+
* **emit_mode** (enum) (required): Specify emit_mode to `batch` or `record`. `batch` will emit events per rewritten tag, and decrease IO. `record` will emit events per record.
|
46
|
+
* Default value: `batch`
|
45
47
|
|
46
48
|
### \<rule\> section (optional) (multiple)
|
47
49
|
|
@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = "fluent-plugin-rewrite-tag-filter"
|
6
|
-
s.version = "2.
|
6
|
+
s.version = "2.2.0"
|
7
7
|
s.license = "Apache-2.0"
|
8
8
|
s.authors = ["Kentaro Yoshida"]
|
9
9
|
s.email = ["y.ken.studio@gmail.com"]
|
@@ -16,7 +16,8 @@ Gem::Specification.new do |s|
|
|
16
16
|
s.require_paths = ["lib"]
|
17
17
|
|
18
18
|
s.add_development_dependency "test-unit", ">= 3.1.0"
|
19
|
+
s.add_development_dependency "test-unit-rr"
|
19
20
|
s.add_development_dependency "rake"
|
20
|
-
s.add_runtime_dependency "fluentd",
|
21
|
+
s.add_runtime_dependency "fluentd", ">= 0.14.2", "< 2"
|
21
22
|
s.add_runtime_dependency "fluent-config-regexp-type"
|
22
23
|
end
|
@@ -12,6 +12,8 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
|
|
12
12
|
config_param :remove_tag_prefix, :string, :default => nil
|
13
13
|
desc 'Override hostname command for placeholder.'
|
14
14
|
config_param :hostname_command, :string, :default => 'hostname'
|
15
|
+
desc "The emit mode. If `batch`, this plugin will emit events per rewritten tag."
|
16
|
+
config_param :emit_mode, :enum, list: [:record, :batch], default: :batch
|
15
17
|
|
16
18
|
config_section :rule, param_name: :rules, multi: true do
|
17
19
|
desc "The field name to which the regular expression is applied"
|
@@ -59,6 +61,8 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
|
|
59
61
|
unless @remove_tag_prefix.nil?
|
60
62
|
@remove_tag_prefix = /^#{Regexp.escape(@remove_tag_prefix)}\.?/
|
61
63
|
end
|
64
|
+
|
65
|
+
@batch_mode = @emit_mode == :batch
|
62
66
|
end
|
63
67
|
|
64
68
|
def multi_workers_ready?
|
@@ -67,13 +71,28 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
|
|
67
71
|
|
68
72
|
def process(tag, es)
|
69
73
|
placeholder = get_placeholder(tag)
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
74
|
+
if @batch_mode
|
75
|
+
new_event_streams = Hash.new {|h, k| h[k] = Fluent::MultiEventStream.new }
|
76
|
+
es.each do |time, record|
|
77
|
+
rewrited_tag = rewrite_tag(tag, record, placeholder)
|
78
|
+
if rewrited_tag.nil? || tag == rewrited_tag
|
79
|
+
log.trace("rewrite_tag_filter: tag has not been rewritten", record)
|
80
|
+
next
|
81
|
+
end
|
82
|
+
new_event_streams[rewrited_tag].add(time, record)
|
83
|
+
end
|
84
|
+
new_event_streams.each do |rewrited_tag, new_es|
|
85
|
+
router.emit_stream(rewrited_tag, new_es)
|
86
|
+
end
|
87
|
+
else
|
88
|
+
es.each do |time, record|
|
89
|
+
rewrited_tag = rewrite_tag(tag, record, placeholder)
|
90
|
+
if rewrited_tag.nil? || tag == rewrited_tag
|
91
|
+
log.trace("rewrite_tag_filter: tag has not been rewritten", record)
|
92
|
+
next
|
93
|
+
end
|
94
|
+
router.emit(rewrited_tag, time, record)
|
75
95
|
end
|
76
|
-
router.emit(rewrited_tag, time, record)
|
77
96
|
end
|
78
97
|
end
|
79
98
|
|
data/test/helper.rb
CHANGED
@@ -1,9 +1,13 @@
|
|
1
1
|
require 'bundler/setup'
|
2
2
|
require 'test/unit'
|
3
|
+
require 'test/unit/rr'
|
3
4
|
|
4
5
|
$LOAD_PATH.unshift(File.join(__dir__, '..', 'lib'))
|
5
6
|
$LOAD_PATH.unshift(__dir__)
|
6
7
|
require 'fluent/test'
|
7
8
|
require 'fluent/test/driver/output'
|
9
|
+
require 'fluent/test/helpers'
|
10
|
+
|
11
|
+
Test::Unit::TestCase.include(Fluent::Test::Helpers)
|
8
12
|
|
9
13
|
require 'fluent/plugin/out_rewrite_tag_filter'
|
@@ -306,5 +306,73 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
|
|
306
306
|
log.slice(/\[trace\]: (.+)$/, 1))
|
307
307
|
assert_equal "com.example", events[0][0]
|
308
308
|
end
|
309
|
+
|
310
|
+
sub_test_case "emit_mode" do
|
311
|
+
test "record" do
|
312
|
+
conf = %[
|
313
|
+
emit_mode record
|
314
|
+
<rule>
|
315
|
+
key key
|
316
|
+
pattern /^(odd|even)$/
|
317
|
+
tag $1
|
318
|
+
</rule>
|
319
|
+
]
|
320
|
+
time = event_time
|
321
|
+
d = create_driver(conf)
|
322
|
+
mock.proxy(d.instance.router).emit(anything, anything, anything).times(6)
|
323
|
+
mock.proxy(d.instance.router).emit_stream(anything, anything).times(0)
|
324
|
+
d.run(default_tag: "input") do
|
325
|
+
d.feed([[time, { "key" => "odd", "message" => "message-1" }],
|
326
|
+
[time, { "key" => "even", "message" => "message-2" }],
|
327
|
+
[time, { "key" => "odd", "message" => "message-3" }],
|
328
|
+
[time, { "key" => "even", "message" => "message-4" }],
|
329
|
+
[time, { "key" => "odd", "message" => "message-5" }],
|
330
|
+
[time, { "key" => "even", "message" => "message-6" }]])
|
331
|
+
end
|
332
|
+
events = d.events
|
333
|
+
expected_events = [
|
334
|
+
["odd", time, { "key" => "odd", "message" => "message-1" }],
|
335
|
+
["even", time, { "key" => "even", "message" => "message-2" }],
|
336
|
+
["odd", time, { "key" => "odd", "message" => "message-3" }],
|
337
|
+
["even", time, { "key" => "even", "message" => "message-4" }],
|
338
|
+
["odd", time, { "key" => "odd", "message" => "message-5" }],
|
339
|
+
["even", time, { "key" => "even", "message" => "message-6" }],
|
340
|
+
]
|
341
|
+
assert_equal(events, expected_events)
|
342
|
+
end
|
343
|
+
|
344
|
+
test "batch" do
|
345
|
+
conf = %[
|
346
|
+
emit_mode batch
|
347
|
+
<rule>
|
348
|
+
key key
|
349
|
+
pattern /^(odd|even)$/
|
350
|
+
tag $1
|
351
|
+
</rule>
|
352
|
+
]
|
353
|
+
time = event_time
|
354
|
+
d = create_driver(conf)
|
355
|
+
mock.proxy(d.instance.router).emit(anything, anything, anything).times(0)
|
356
|
+
mock.proxy(d.instance.router).emit_stream(anything, anything).times(2)
|
357
|
+
d.run(default_tag: "input") do
|
358
|
+
d.feed([[time, { "key" => "odd", "message" => "message-1" }],
|
359
|
+
[time, { "key" => "even", "message" => "message-2" }],
|
360
|
+
[time, { "key" => "odd", "message" => "message-3" }],
|
361
|
+
[time, { "key" => "even", "message" => "message-4" }],
|
362
|
+
[time, { "key" => "odd", "message" => "message-5" }],
|
363
|
+
[time, { "key" => "even", "message" => "message-6" }]])
|
364
|
+
end
|
365
|
+
events = d.events
|
366
|
+
expected_records = [
|
367
|
+
["odd", time, { "key" => "odd", "message" => "message-1" }],
|
368
|
+
["odd", time, { "key" => "odd", "message" => "message-3" }],
|
369
|
+
["odd", time, { "key" => "odd", "message" => "message-5" }],
|
370
|
+
["even", time, { "key" => "even", "message" => "message-2" }],
|
371
|
+
["even", time, { "key" => "even", "message" => "message-4" }],
|
372
|
+
["even", time, { "key" => "even", "message" => "message-6" }],
|
373
|
+
]
|
374
|
+
assert_equal(events, expected_records)
|
375
|
+
end
|
376
|
+
end
|
309
377
|
end
|
310
378
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-rewrite-tag-filter
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
4
|
+
version: 2.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Kentaro Yoshida
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2019-03-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: test-unit
|
@@ -24,6 +24,20 @@ dependencies:
|
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: 3.1.0
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: test-unit-rr
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
27
41
|
- !ruby/object:Gem::Dependency
|
28
42
|
name: rake
|
29
43
|
requirement: !ruby/object:Gem::Requirement
|
@@ -110,8 +124,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
110
124
|
- !ruby/object:Gem::Version
|
111
125
|
version: '0'
|
112
126
|
requirements: []
|
113
|
-
|
114
|
-
rubygems_version: 2.7.6
|
127
|
+
rubygems_version: 3.0.1
|
115
128
|
signing_key:
|
116
129
|
specification_version: 4
|
117
130
|
summary: Fluentd Output filter plugin. It has designed to rewrite tag like mod_rewrite.
|