fluent-plugin-rewrite-tag-filter 2.1.1 → 2.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9a976b12b3e1ab4b56116a3690d0c48949ff34788f1b165b39909c52d4e1983d
-  data.tar.gz: 435ca4ea943d418f2803ee787422271850ededecbe20bd82efa518bb5ff12a7a
+  metadata.gz: 8253bb2b1859438ea797e1fd2046d0b18290a424b340de8a88524b9afd33388a
+  data.tar.gz: 33e91a5b0570810f2ad802885df4a7e87eb03c4816b5779d847c5004b25afb0a
 SHA512:
-  metadata.gz: 49b4bf86556799b2ac17c37e214754af8ea80489d3817a7026e4aac725ac360605154538e885f4c5cc6843424a1d9575ea383a4fa99c6e3b045169347855ecd2
-  data.tar.gz: 7f34dc3e1f1d5178051d3e2af5d433064bb8219638f4181854c6e712341fa0bdbe52f8baa527e9b66967c79686ea2eccf0641fcaadfce22379e47b9bb7747c59
+  metadata.gz: f28d04ee233a9bfdd668f5dc114b7e9b7d5531d275d1c5318100d4a76171607ddf9e03041081b139ba2f7a55b6b73bdbf64c44cae67097df983ae80395d04e69
+  data.tar.gz: 4e6ee684e6da26e6d0e449cc0650dd482f9cbd5cd6623d540bb63b1b991e48f0ede38f9424f04ef8750e6d7f451afa62c89bda412e3bff66589b3101407414d3

data/README.md

@@ -36,12 +36,14 @@ For more details, see [Plugin Management](https://docs.fluentd.org/v0.14/article
 ## Configuration
 
 * **rewriterule\<num\>** (string) (optional) \<attribute\> \<regex_pattern\> \<new_tag\>
-  * Obsoleted: Use <rule> section
+  * Obsoleted: Use \<rule\> section
 * **capitalize_regex_backreference** (bool) (optional): Capitalize letter for every matched regex backreference. (ex: maps -> Maps) for more details, see usage.
   * Default value: no
 * **remove_tag_prefix** (string) (optional): Remove tag prefix for tag placeholder. (see the section of "Tag placeholder")
 * **hostname_command** (string) (optional): Override hostname command for placeholder. (see the section of "Tag placeholder")
-  * Default value: `hostname`.
+  * Default value: `hostname`
+* **emit_mode** (enum) (required): Specify emit_mode to `batch` or `record`. `batch` will emit events per rewritten tag, and decrease IO. `record` will emit events per record.
+  * Default value: `batch`
 
 ### \<rule\> section (optional) (multiple)
 

data/fluent-plugin-rewrite-tag-filter.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-rewrite-tag-filter"
-  s.version     = "2.1.1"
+  s.version     = "2.2.0"
   s.license     = "Apache-2.0"
   s.authors     = ["Kentaro Yoshida"]
   s.email       = ["y.ken.studio@gmail.com"]
@@ -16,7 +16,8 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   s.add_development_dependency "test-unit", ">= 3.1.0"
+  s.add_development_dependency "test-unit-rr"
   s.add_development_dependency "rake"
-  s.add_runtime_dependency "fluentd", [">= 0.14.2", "< 2"]
+  s.add_runtime_dependency "fluentd", ">= 0.14.2", "< 2"
   s.add_runtime_dependency "fluent-config-regexp-type"
 end

data/lib/fluent/plugin/out_rewrite_tag_filter.rb

@@ -12,6 +12,8 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
   config_param :remove_tag_prefix, :string, :default => nil
   desc 'Override hostname command for placeholder.'
   config_param :hostname_command, :string, :default => 'hostname'
+  desc "The emit mode. If `batch`, this plugin will emit events per rewritten tag."
+  config_param :emit_mode, :enum, list: [:record, :batch], default: :batch
 
   config_section :rule, param_name: :rules, multi: true do
     desc "The field name to which the regular expression is applied"
@@ -59,6 +61,8 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
     unless @remove_tag_prefix.nil?
       @remove_tag_prefix = /^#{Regexp.escape(@remove_tag_prefix)}\.?/
     end
+
+    @batch_mode = @emit_mode == :batch
   end
 
   def multi_workers_ready?
@@ -67,13 +71,28 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
 
   def process(tag, es)
     placeholder = get_placeholder(tag)
-    es.each do |time, record|
-      rewrited_tag = rewrite_tag(tag, record, placeholder)
-      if rewrited_tag.nil? || tag == rewrited_tag
-        log.trace("rewrite_tag_filter: tag has not been rewritten", record)
-        next
+    if @batch_mode
+      new_event_streams = Hash.new {|h, k| h[k] = Fluent::MultiEventStream.new }
+      es.each do |time, record|
+        rewrited_tag = rewrite_tag(tag, record, placeholder)
+        if rewrited_tag.nil? || tag == rewrited_tag
+          log.trace("rewrite_tag_filter: tag has not been rewritten", record)
+          next
+        end
+        new_event_streams[rewrited_tag].add(time, record)
+      end
+      new_event_streams.each do |rewrited_tag, new_es|
+        router.emit_stream(rewrited_tag, new_es)
+      end
+    else
+      es.each do |time, record|
+        rewrited_tag = rewrite_tag(tag, record, placeholder)
+        if rewrited_tag.nil? || tag == rewrited_tag
+          log.trace("rewrite_tag_filter: tag has not been rewritten", record)
+          next
+        end
+        router.emit(rewrited_tag, time, record)
       end
-      router.emit(rewrited_tag, time, record)
     end
   end
 

data/test/helper.rb

@@ -1,9 +1,13 @@
 require 'bundler/setup'
 require 'test/unit'
+require 'test/unit/rr'
 
 $LOAD_PATH.unshift(File.join(__dir__, '..', 'lib'))
 $LOAD_PATH.unshift(__dir__)
 require 'fluent/test'
 require 'fluent/test/driver/output'
+require 'fluent/test/helpers'
+
+Test::Unit::TestCase.include(Fluent::Test::Helpers)
 
 require 'fluent/plugin/out_rewrite_tag_filter'

data/test/plugin/test_out_rewrite_tag_filter.rb

@@ -306,5 +306,73 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
                    log.slice(/\[trace\]: (.+)$/, 1))
       assert_equal "com.example", events[0][0]
     end
+
+    sub_test_case "emit_mode" do
+      test "record" do
+        conf = %[
+          emit_mode record
+          <rule>
+            key key
+            pattern /^(odd|even)$/
+            tag $1
+          </rule>
+        ]
+        time = event_time
+        d = create_driver(conf)
+        mock.proxy(d.instance.router).emit(anything, anything, anything).times(6)
+        mock.proxy(d.instance.router).emit_stream(anything, anything).times(0)
+        d.run(default_tag: "input") do
+          d.feed([[time, { "key" => "odd", "message" => "message-1" }],
+                  [time, { "key" => "even", "message" => "message-2" }],
+                  [time, { "key" => "odd", "message" => "message-3" }],
+                  [time, { "key" => "even", "message" => "message-4" }],
+                  [time, { "key" => "odd", "message" => "message-5" }],
+                  [time, { "key" => "even", "message" => "message-6" }]])
+        end
+        events = d.events
+        expected_events = [
+          ["odd", time, { "key" => "odd", "message" => "message-1" }],
+          ["even", time, { "key" => "even", "message" => "message-2" }],
+          ["odd", time, { "key" => "odd", "message" => "message-3" }],
+          ["even", time, { "key" => "even", "message" => "message-4" }],
+          ["odd", time, { "key" => "odd", "message" => "message-5" }],
+          ["even", time, { "key" => "even", "message" => "message-6" }],
+        ]
+        assert_equal(events, expected_events)
+      end
+
+      test "batch" do
+        conf = %[
+          emit_mode batch
+          <rule>
+            key key
+            pattern /^(odd|even)$/
+            tag $1
+          </rule>
+        ]
+        time = event_time
+        d = create_driver(conf)
+        mock.proxy(d.instance.router).emit(anything, anything, anything).times(0)
+        mock.proxy(d.instance.router).emit_stream(anything, anything).times(2)
+        d.run(default_tag: "input") do
+          d.feed([[time, { "key" => "odd", "message" => "message-1" }],
+                  [time, { "key" => "even", "message" => "message-2" }],
+                  [time, { "key" => "odd", "message" => "message-3" }],
+                  [time, { "key" => "even", "message" => "message-4" }],
+                  [time, { "key" => "odd", "message" => "message-5" }],
+                  [time, { "key" => "even", "message" => "message-6" }]])
+        end
+        events = d.events
+        expected_records = [
+          ["odd", time, { "key" => "odd", "message" => "message-1" }],
+          ["odd", time, { "key" => "odd", "message" => "message-3" }],
+          ["odd", time, { "key" => "odd", "message" => "message-5" }],
+          ["even", time, { "key" => "even", "message" => "message-2" }],
+          ["even", time, { "key" => "even", "message" => "message-4" }],
+          ["even", time, { "key" => "even", "message" => "message-6" }],
+        ]
+        assert_equal(events, expected_records)
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-rewrite-tag-filter
 version: !ruby/object:Gem::Version
-  version: 2.1.1
+  version: 2.2.0
 platform: ruby
 authors:
 - Kentaro Yoshida
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-12 00:00:00.000000000 Z
+date: 2019-03-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: test-unit
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.1.0
+- !ruby/object:Gem::Dependency
+  name: test-unit-rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -110,8 +124,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.1
 signing_key: 
 specification_version: 4
 summary: Fluentd Output filter plugin. It has designed to rewrite tag like mod_rewrite.