fluent-plugin-rewrite-tag-filter 2.1.1 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 9a976b12b3e1ab4b56116a3690d0c48949ff34788f1b165b39909c52d4e1983d
4
- data.tar.gz: 435ca4ea943d418f2803ee787422271850ededecbe20bd82efa518bb5ff12a7a
3
+ metadata.gz: 8253bb2b1859438ea797e1fd2046d0b18290a424b340de8a88524b9afd33388a
4
+ data.tar.gz: 33e91a5b0570810f2ad802885df4a7e87eb03c4816b5779d847c5004b25afb0a
5
5
  SHA512:
6
- metadata.gz: 49b4bf86556799b2ac17c37e214754af8ea80489d3817a7026e4aac725ac360605154538e885f4c5cc6843424a1d9575ea383a4fa99c6e3b045169347855ecd2
7
- data.tar.gz: 7f34dc3e1f1d5178051d3e2af5d433064bb8219638f4181854c6e712341fa0bdbe52f8baa527e9b66967c79686ea2eccf0641fcaadfce22379e47b9bb7747c59
6
+ metadata.gz: f28d04ee233a9bfdd668f5dc114b7e9b7d5531d275d1c5318100d4a76171607ddf9e03041081b139ba2f7a55b6b73bdbf64c44cae67097df983ae80395d04e69
7
+ data.tar.gz: 4e6ee684e6da26e6d0e449cc0650dd482f9cbd5cd6623d540bb63b1b991e48f0ede38f9424f04ef8750e6d7f451afa62c89bda412e3bff66589b3101407414d3
data/README.md CHANGED
@@ -36,12 +36,14 @@ For more details, see [Plugin Management](https://docs.fluentd.org/v0.14/article
36
36
  ## Configuration
37
37
 
38
38
  * **rewriterule\<num\>** (string) (optional) \<attribute\> \<regex_pattern\> \<new_tag\>
39
- * Obsoleted: Use <rule> section
39
+ * Obsoleted: Use \<rule\> section
40
40
  * **capitalize_regex_backreference** (bool) (optional): Capitalize letter for every matched regex backreference. (ex: maps -> Maps) for more details, see usage.
41
41
  * Default value: no
42
42
  * **remove_tag_prefix** (string) (optional): Remove tag prefix for tag placeholder. (see the section of "Tag placeholder")
43
43
  * **hostname_command** (string) (optional): Override hostname command for placeholder. (see the section of "Tag placeholder")
44
- * Default value: `hostname`.
44
+ * Default value: `hostname`
45
+ * **emit_mode** (enum) (required): Specify emit_mode to `batch` or `record`. `batch` will emit events per rewritten tag, and decrease IO. `record` will emit events per record.
46
+ * Default value: `batch`
45
47
 
46
48
  ### \<rule\> section (optional) (multiple)
47
49
 
@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
3
3
 
4
4
  Gem::Specification.new do |s|
5
5
  s.name = "fluent-plugin-rewrite-tag-filter"
6
- s.version = "2.1.1"
6
+ s.version = "2.2.0"
7
7
  s.license = "Apache-2.0"
8
8
  s.authors = ["Kentaro Yoshida"]
9
9
  s.email = ["y.ken.studio@gmail.com"]
@@ -16,7 +16,8 @@ Gem::Specification.new do |s|
16
16
  s.require_paths = ["lib"]
17
17
 
18
18
  s.add_development_dependency "test-unit", ">= 3.1.0"
19
+ s.add_development_dependency "test-unit-rr"
19
20
  s.add_development_dependency "rake"
20
- s.add_runtime_dependency "fluentd", [">= 0.14.2", "< 2"]
21
+ s.add_runtime_dependency "fluentd", ">= 0.14.2", "< 2"
21
22
  s.add_runtime_dependency "fluent-config-regexp-type"
22
23
  end
@@ -12,6 +12,8 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
12
12
  config_param :remove_tag_prefix, :string, :default => nil
13
13
  desc 'Override hostname command for placeholder.'
14
14
  config_param :hostname_command, :string, :default => 'hostname'
15
+ desc "The emit mode. If `batch`, this plugin will emit events per rewritten tag."
16
+ config_param :emit_mode, :enum, list: [:record, :batch], default: :batch
15
17
 
16
18
  config_section :rule, param_name: :rules, multi: true do
17
19
  desc "The field name to which the regular expression is applied"
@@ -59,6 +61,8 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
59
61
  unless @remove_tag_prefix.nil?
60
62
  @remove_tag_prefix = /^#{Regexp.escape(@remove_tag_prefix)}\.?/
61
63
  end
64
+
65
+ @batch_mode = @emit_mode == :batch
62
66
  end
63
67
 
64
68
  def multi_workers_ready?
@@ -67,13 +71,28 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
67
71
 
68
72
  def process(tag, es)
69
73
  placeholder = get_placeholder(tag)
70
- es.each do |time, record|
71
- rewrited_tag = rewrite_tag(tag, record, placeholder)
72
- if rewrited_tag.nil? || tag == rewrited_tag
73
- log.trace("rewrite_tag_filter: tag has not been rewritten", record)
74
- next
74
+ if @batch_mode
75
+ new_event_streams = Hash.new {|h, k| h[k] = Fluent::MultiEventStream.new }
76
+ es.each do |time, record|
77
+ rewrited_tag = rewrite_tag(tag, record, placeholder)
78
+ if rewrited_tag.nil? || tag == rewrited_tag
79
+ log.trace("rewrite_tag_filter: tag has not been rewritten", record)
80
+ next
81
+ end
82
+ new_event_streams[rewrited_tag].add(time, record)
83
+ end
84
+ new_event_streams.each do |rewrited_tag, new_es|
85
+ router.emit_stream(rewrited_tag, new_es)
86
+ end
87
+ else
88
+ es.each do |time, record|
89
+ rewrited_tag = rewrite_tag(tag, record, placeholder)
90
+ if rewrited_tag.nil? || tag == rewrited_tag
91
+ log.trace("rewrite_tag_filter: tag has not been rewritten", record)
92
+ next
93
+ end
94
+ router.emit(rewrited_tag, time, record)
75
95
  end
76
- router.emit(rewrited_tag, time, record)
77
96
  end
78
97
  end
79
98
 
@@ -1,9 +1,13 @@
1
1
  require 'bundler/setup'
2
2
  require 'test/unit'
3
+ require 'test/unit/rr'
3
4
 
4
5
  $LOAD_PATH.unshift(File.join(__dir__, '..', 'lib'))
5
6
  $LOAD_PATH.unshift(__dir__)
6
7
  require 'fluent/test'
7
8
  require 'fluent/test/driver/output'
9
+ require 'fluent/test/helpers'
10
+
11
+ Test::Unit::TestCase.include(Fluent::Test::Helpers)
8
12
 
9
13
  require 'fluent/plugin/out_rewrite_tag_filter'
@@ -306,5 +306,73 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
306
306
  log.slice(/\[trace\]: (.+)$/, 1))
307
307
  assert_equal "com.example", events[0][0]
308
308
  end
309
+
310
+ sub_test_case "emit_mode" do
311
+ test "record" do
312
+ conf = %[
313
+ emit_mode record
314
+ <rule>
315
+ key key
316
+ pattern /^(odd|even)$/
317
+ tag $1
318
+ </rule>
319
+ ]
320
+ time = event_time
321
+ d = create_driver(conf)
322
+ mock.proxy(d.instance.router).emit(anything, anything, anything).times(6)
323
+ mock.proxy(d.instance.router).emit_stream(anything, anything).times(0)
324
+ d.run(default_tag: "input") do
325
+ d.feed([[time, { "key" => "odd", "message" => "message-1" }],
326
+ [time, { "key" => "even", "message" => "message-2" }],
327
+ [time, { "key" => "odd", "message" => "message-3" }],
328
+ [time, { "key" => "even", "message" => "message-4" }],
329
+ [time, { "key" => "odd", "message" => "message-5" }],
330
+ [time, { "key" => "even", "message" => "message-6" }]])
331
+ end
332
+ events = d.events
333
+ expected_events = [
334
+ ["odd", time, { "key" => "odd", "message" => "message-1" }],
335
+ ["even", time, { "key" => "even", "message" => "message-2" }],
336
+ ["odd", time, { "key" => "odd", "message" => "message-3" }],
337
+ ["even", time, { "key" => "even", "message" => "message-4" }],
338
+ ["odd", time, { "key" => "odd", "message" => "message-5" }],
339
+ ["even", time, { "key" => "even", "message" => "message-6" }],
340
+ ]
341
+ assert_equal(events, expected_events)
342
+ end
343
+
344
+ test "batch" do
345
+ conf = %[
346
+ emit_mode batch
347
+ <rule>
348
+ key key
349
+ pattern /^(odd|even)$/
350
+ tag $1
351
+ </rule>
352
+ ]
353
+ time = event_time
354
+ d = create_driver(conf)
355
+ mock.proxy(d.instance.router).emit(anything, anything, anything).times(0)
356
+ mock.proxy(d.instance.router).emit_stream(anything, anything).times(2)
357
+ d.run(default_tag: "input") do
358
+ d.feed([[time, { "key" => "odd", "message" => "message-1" }],
359
+ [time, { "key" => "even", "message" => "message-2" }],
360
+ [time, { "key" => "odd", "message" => "message-3" }],
361
+ [time, { "key" => "even", "message" => "message-4" }],
362
+ [time, { "key" => "odd", "message" => "message-5" }],
363
+ [time, { "key" => "even", "message" => "message-6" }]])
364
+ end
365
+ events = d.events
366
+ expected_records = [
367
+ ["odd", time, { "key" => "odd", "message" => "message-1" }],
368
+ ["odd", time, { "key" => "odd", "message" => "message-3" }],
369
+ ["odd", time, { "key" => "odd", "message" => "message-5" }],
370
+ ["even", time, { "key" => "even", "message" => "message-2" }],
371
+ ["even", time, { "key" => "even", "message" => "message-4" }],
372
+ ["even", time, { "key" => "even", "message" => "message-6" }],
373
+ ]
374
+ assert_equal(events, expected_records)
375
+ end
376
+ end
309
377
  end
310
378
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-rewrite-tag-filter
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.1.1
4
+ version: 2.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Kentaro Yoshida
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-11-12 00:00:00.000000000 Z
11
+ date: 2019-03-12 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: test-unit
@@ -24,6 +24,20 @@ dependencies:
24
24
  - - ">="
25
25
  - !ruby/object:Gem::Version
26
26
  version: 3.1.0
27
+ - !ruby/object:Gem::Dependency
28
+ name: test-unit-rr
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: '0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
27
41
  - !ruby/object:Gem::Dependency
28
42
  name: rake
29
43
  requirement: !ruby/object:Gem::Requirement
@@ -110,8 +124,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
110
124
  - !ruby/object:Gem::Version
111
125
  version: '0'
112
126
  requirements: []
113
- rubyforge_project:
114
- rubygems_version: 2.7.6
127
+ rubygems_version: 3.0.1
115
128
  signing_key:
116
129
  specification_version: 4
117
130
  summary: Fluentd Output filter plugin. It has designed to rewrite tag like mod_rewrite.