fluent-plugin-rewrite-tag-filter 2.2.0 → 2.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8253bb2b1859438ea797e1fd2046d0b18290a424b340de8a88524b9afd33388a
-  data.tar.gz: 33e91a5b0570810f2ad802885df4a7e87eb03c4816b5779d847c5004b25afb0a
+  metadata.gz: a12e8f6567cd07462a236f0cda38e98bac14b921a2c7b762e9851cb04e5bf119
+  data.tar.gz: b198b1e6b8a48d386f7cc62521f358cdff745754edc8dda753ac176ced1e3e67
 SHA512:
-  metadata.gz: f28d04ee233a9bfdd668f5dc114b7e9b7d5531d275d1c5318100d4a76171607ddf9e03041081b139ba2f7a55b6b73bdbf64c44cae67097df983ae80395d04e69
-  data.tar.gz: 4e6ee684e6da26e6d0e449cc0650dd482f9cbd5cd6623d540bb63b1b991e48f0ede38f9424f04ef8750e6d7f451afa62c89bda412e3bff66589b3101407414d3
+  metadata.gz: cc556895e92c3f53ca3c41af6866072e48ac53b6b170cfe163f1563d04310c8b89cc802cabefd957dab95da29362ff76164621f0a83d806619d67df521e866b6
+  data.tar.gz: 1777b76d00afe128f20da230ca26d0c30d4f62e60f4800efec6b59450040e54de7b861ee7913ed82af8f91b9f3cf5e11344b5ec5b08e57af97d611a0060d0ddf

data/.travis.yml

@@ -1,10 +1,10 @@
 language: ruby
 
 rvm:
-  - 2.4.2
-  - 2.3.5
-  - 2.2
-  - 2.1
+  - 2.7.0
+  - 2.6.5
+  - 2.5.7
+  - 2.4.9
 
 # to avoid travis-ci issue since 2015-12-25
 before_install:

data/README.md

@@ -3,7 +3,7 @@
 ## Overview
 
 Rewrite Tag Filter for [Fluentd](http://fluentd.org). It is designed to rewrite tags like mod_rewrite.  
-Re-emit the record with rewrited tag when a value matches/unmatches with a regular expression.  
+Re-emit the record with rewritten tag when a value matches/unmatches with a regular expression.  
 Also you can change a tag from Apache log by domain, status code (ex. 500 error),  
 user-agent, request-uri, regex-backreference and so on with regular expression.
 
@@ -31,7 +31,7 @@ $ sudo td-agent-gem install fluent-plugin-rewrite-tag-filter -v 1.6.0
 $ sudo td-agent-gem install fluent-plugin-rewrite-tag-filter
 ```
 
-For more details, see [Plugin Management](https://docs.fluentd.org/v0.14/articles/plugin-management)
+For more details, see [Plugin Management](https://docs.fluentd.org/deployment/plugin-management)
 
 ## Configuration
 
@@ -52,6 +52,7 @@ For more details, see [Plugin Management](https://docs.fluentd.org/v0.14/article
   `/regexp/` is preferred because `/regexp/` style can support character classes such as `/[a-z]/`.
   The pattern without slashes will cause errors if you use patterns start with character classes.
 * **tag** (string) (required): New tag
+* **label** (string) (optional): New label. If specified, label can be changed per-rule.
 * **invert** (bool) (optional): If true, rewrite tag when unmatch pattern
   * Default value: `false`
 
@@ -200,7 +201,7 @@ When original tag is `kubernetes.var.log`, this will be converted to `default.ku
 
 ### Tag placeholder
 
-It is supported these placeholder for new_tag (rewrited tag).
+It is supported these placeholder for new_tag (rewritten tag).
 
 - `${tag}`
 - `__TAG__`
@@ -214,12 +215,16 @@ For example with `td.apache.access` tag, it will get `td` by `${tag_parts[0]}` a
 
 **Note** Currently, range expression ```${tag_parts[0..2]}``` is not supported.
 
-#### Placeholder Option
+#### Placeholder Options
 
 * `remove_tag_prefix`  
 
 This option adds removing tag prefix for `${tag}` or `__TAG__` in placeholder.
 
+* `remove_tag_regexp`  
+
+This option adds removing tag regexp for `${tag}` or `__TAG__` in placeholder.
+
 * `hostname_command` 
 
 By default, execute command as `hostname` to get full hostname.  
@@ -231,49 +236,100 @@ It comes short hostname with `hostname_command hostname -s` configuration specif
 It's a sample to rewrite a tag with placeholder.
 
 ```
-# It will get "rewrited.access.ExampleMail"
+# It will get "rewritten.access.ExampleMail"
 <match apache.access>
   @type rewrite_tag_filter
   remove_tag_prefix apache
   <rule>
     key     domain
     pattern ^(mail)\.(example)\.com$
-    tag     rewrited.${tag}.$2$1
+    tag     rewritten.${tag}.$2$1
+  </rule>
+</match>
+
+# It will get "rewritten.access.ExampleMail"
+<match apache.access>
+  @type rewrite_tag_filter
+  remove_tag_regexp /^apache\./
+  <rule>
+    key     domain
+    pattern ^(mail)\.(example)\.com$
+    tag     rewritten.${tag}.$2$1
+  </rule>
+</match>
+
+# It will get "http.access.log"
+<match input.{apache,nginx}.access.log>
+  @type rewrite_tag_filter
+  remove_tag_regexp /^input\.(apache|nginx)\./
+  <rule>
+    key     domain
+    pattern ^.+$
+    tag     http.${tag}
   </rule>
 </match>
 
-# It will get "rewrited.ExampleMail.app30-124.foo.com" when hostname is "app30-124.foo.com"
+# It will get "rewritten.ExampleMail.app30-124.foo.com" when hostname is "app30-124.foo.com"
 <match apache.access>
   @type rewrite_tag_filter
   <rule>
     key     domain
     pattern ^(mail)\.(example)\.com$
-    tag     rewrited.$2$1.${hostname}
+    tag     rewritten.$2$1.${hostname}
   </rule>
 </match>
 
-# It will get "rewrited.ExampleMail.app30-124" when hostname is "app30-124.foo.com"
+# It will get "rewritten.ExampleMail.app30-124" when hostname is "app30-124.foo.com"
 <match apache.access>
   @type rewrite_tag_filter
   hostname_command hostname -s
   <rule>
     key     domain
     pattern ^(mail)\.(example)\.com$
-    tag     rewrited.$2$1.${hostname}
+    tag     rewritten.$2$1.${hostname}
   </rule>
 </match>
 
-# It will get "rewrited.game.pool"
+# It will get "rewritten.game.pool"
 <match app.game.pool.activity>
   @type rewrite_tag_filter
   <rule>
     key     domain
     pattern ^.+$
-    tag     rewrited.${tag_parts[1]}.${tag_parts[2]}
+    tag     rewritten.${tag_parts[1]}.${tag_parts[2]}
   </rule>
 </match>
 ```
 
+### Altering Labels
+
+In addition to changing tags, you can also change event's route by setting
+ the label for the re-emitted event.
+
+For example, given this configuration:
+
+```
+<match apache.access>
+  @type rewrite_tag_filter
+  <rule>
+    key     domain
+    pattern ^www\.example\.com$
+    tag     web.${tag}
+  </rule>
+  <rule>
+    key     domain
+    pattern ^(.*)\.example\.com$
+    tag     other.$1
+    label   other
+  </rule>
+</match>
+```
+
+message: `{"domain": "www.example.com"}` will get its tag changed to 
+`web.apache.access`, while message 
+`{"domain": "api.example.com"}` will get its tag changed to `other.api` and
+ be sent to label `other`
+
 ## Example
 
 - Example1: how to analyze response_time, response_code and user_agent for each virtual domain websites.  

data/fluent-plugin-rewrite-tag-filter.gemspec

@@ -1,9 +1,8 @@
-# -*- encoding: utf-8 -*-
 $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-rewrite-tag-filter"
-  s.version     = "2.2.0"
+  s.version     = "2.4.0"
   s.license     = "Apache-2.0"
   s.authors     = ["Kentaro Yoshida"]
   s.email       = ["y.ken.studio@gmail.com"]

data/lib/fluent/plugin/out_rewrite_tag_filter.rb

@@ -10,6 +10,8 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
   config_param :capitalize_regex_backreference, :bool, :default => false
   desc 'Remove tag prefix for tag placeholder.'
   config_param :remove_tag_prefix, :string, :default => nil
+  desc "Remove tag regexp for tag placeholder."
+  config_param :remove_tag_regexp, :regexp, default: nil
   desc 'Override hostname command for placeholder.'
   config_param :hostname_command, :string, :default => 'hostname'
   desc "The emit mode. If `batch`, this plugin will emit events per rewritten tag."
@@ -22,6 +24,8 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
     config_param :pattern, :regexp
     desc "New tag"
     config_param :tag, :string
+    desc "New label. If specified, label can be changed per-rule."
+    config_param :label, :string, default: nil
     desc "If true, rewrite tag when unmatch pattern"
     config_param :invert, :bool, default: false
   end
@@ -41,7 +45,7 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
       end
 
       invert = rule.invert ? MATCH_OPERATOR_EXCLUDE : ""
-      @rewriterules.push([record_accessor_create(rule.key), rule.pattern, invert, rule.tag])
+      @rewriterules.push([record_accessor_create(rule.key), rule.pattern, invert, rule.tag, rule.label])
       rewriterule_names.push(rule.key + invert + rule.pattern.to_s)
       log.info "adding rewrite_tag_filter rule: #{rule.key} #{@rewriterules.last}"
     end
@@ -58,8 +62,12 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
       raise Fluent::ConfigError, "duplicated rewriterules found #{@rewriterules.inspect}"
     end
 
+    if @remove_tag_prefix && @remove_tag_regexp
+      raise Fluent::ConfigError, "remove_tag_prefix and remove_tag_regexp are exclusive"
+    end
+
     unless @remove_tag_prefix.nil?
-      @remove_tag_prefix = /^#{Regexp.escape(@remove_tag_prefix)}\.?/
+      @remove_tag_regexp = /^#{Regexp.escape(@remove_tag_prefix)}\.?/
     end
 
     @batch_mode = @emit_mode == :batch
@@ -69,35 +77,45 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
     true
   end
 
+  def get_router(tgt_label)
+    label_router = if tgt_label.nil? || tgt_label.empty?
+        router
+    else
+        event_emitter_router(tgt_label)
+    end
+    #log.trace "Got router for #{tgt_label}: #{!label_router.nil?}"
+    return label_router
+  end
+
   def process(tag, es)
     placeholder = get_placeholder(tag)
-    if @batch_mode
-      new_event_streams = Hash.new {|h, k| h[k] = Fluent::MultiEventStream.new }
-      es.each do |time, record|
-        rewrited_tag = rewrite_tag(tag, record, placeholder)
-        if rewrited_tag.nil? || tag == rewrited_tag
-          log.trace("rewrite_tag_filter: tag has not been rewritten", record)
-          next
-        end
-        new_event_streams[rewrited_tag].add(time, record)
+    new_event_streams =  Hash.new {|hh, kk| hh[kk] = Hash.new {|h, k| h[k] = Fluent::MultiEventStream.new }}if @batch_mode
+
+    es.each do |time, record|
+      rewrited_tag, rewrited_label  = rewrite_tag(tag, record, placeholder)
+      if (rewrited_tag.nil? || tag == rewrited_tag) && rewrited_label.nil?
+        log.trace("rewrite_tag_filter: tag has not been rewritten", record)
+        next
       end
-      new_event_streams.each do |rewrited_tag, new_es|
-        router.emit_stream(rewrited_tag, new_es)
+      rewrited_tag = tag if rewrited_tag.nil?
+      if new_event_streams.nil?
+        get_router(rewrited_label).emit(rewrited_tag, time, record)
+      else
+        new_event_streams[rewrited_label][rewrited_tag].add(time, record)
       end
-    else
-      es.each do |time, record|
-        rewrited_tag = rewrite_tag(tag, record, placeholder)
-        if rewrited_tag.nil? || tag == rewrited_tag
-          log.trace("rewrite_tag_filter: tag has not been rewritten", record)
-          next
+    end
+    if !new_event_streams.nil?
+      new_event_streams.each do |rewrited_label, label_event_streams |
+        labeled_router = get_router(rewrited_label)
+        label_event_streams.each do |rewrited_tag, new_es|
+            labeled_router.emit_stream(rewrited_tag, new_es)
         end
-        router.emit(rewrited_tag, time, record)
       end
     end
   end
 
   def rewrite_tag(tag, record, placeholder)
-    @rewriterules.each do |record_accessor, regexp, match_operator, rewritetag|
+    @rewriterules.each do |record_accessor, regexp, match_operator, rewritetag, rewritelabel|
       rewritevalue = record_accessor.call(record).to_s
       next if rewritevalue.empty? && match_operator != MATCH_OPERATOR_EXCLUDE
       last_match = regexp_last_match(regexp, rewritevalue)
@@ -113,9 +131,9 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
         log.warn "rewrite_tag_filter: unknown placeholder found. :placeholder=>#{$1} :tag=>#{tag} :rewritetag=>#{rewritetag}" unless placeholder.include?($1)
         placeholder[$1]
       end
-      return rewritetag
+      return rewritetag, rewritelabel
     end
-    return nil
+    return nil, nil
   end
 
   def regexp_last_match(regexp, rewritevalue)
@@ -135,7 +153,7 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
   end
 
   def get_placeholder(tag)
-    tag = tag.sub(@remove_tag_prefix, '') if @remove_tag_prefix
+    tag = tag.sub(@remove_tag_regexp, '') if @remove_tag_regexp
 
     result = {
       '__HOSTNAME__' => @hostname,

data/test/plugin/test_out_rewrite_tag_filter.rb

@@ -41,6 +41,21 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
       d = create_driver(conf)
       assert_equal(/.+/, d.instance.rules.first.pattern)
     end
+
+    test "remove_tag_prefix and remove_tag_regexp are exclusive" do
+      conf = %[
+        remove_tag_prefix prefix
+        remove_tag_regexp /^prefix\./
+        <rule>
+          key message
+          pattern .+
+          tag ${tag}
+        </rule>
+      ]
+      assert_raise(Fluent::ConfigError) do
+        create_driver(conf)
+      end
+    end
   end
 
   sub_test_case "section style" do
@@ -127,6 +142,30 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
       assert_equal 'access', events[0][0] # tag
     end
 
+    sub_test_case "remove_tag_regexp" do
+      test "plain" do
+        config = %[
+          remove_tag_regexp /^input\.(apache|nginx)\./
+          <rule>
+            key domain
+            pattern ^www\.google\.com$
+            tag rewritten.${tag}
+          </rule>
+        ]
+        d = create_driver(config)
+        d.run do
+          d.feed('input.apache.access', event_time, {'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
+          d.feed('input.nginx.access', event_time, {'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
+          d.feed('input.tomcat.access', event_time, {'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
+        end
+        events = d.events
+        assert_equal 3, events.length
+        assert_equal 'rewritten.access', events[0][0]
+        assert_equal 'rewritten.access', events[1][0]
+        assert_equal 'rewritten.input.tomcat.access', events[2][0]
+      end
+    end
+
     test "short hostname" do
       config = %[
         remove_tag_prefix input
@@ -307,6 +346,77 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
       assert_equal "com.example", events[0][0]
     end
 
+    test "get_router" do
+      conf = %[
+        <rule>
+          key key
+          pattern /^(odd|even)$/
+          tag $1
+          label new_label
+        </rule>
+        <rule>
+          key key
+          pattern /^(.*)$/
+          tag $1
+        </rule>
+      ]
+      time = event_time
+      d = create_driver(conf)
+      assert_equal(d.instance.router, d.instance.get_router(nil))
+      assert_equal(d.instance.router, d.instance.get_router(""))
+      new_label_router = d.instance.get_router("new_label")
+      refute_equal(d.instance.router, new_label_router)
+    end
+
+    test "relabel" do
+      conf = %[
+          emit_mode record
+          <rule>
+            key key
+            pattern /^(odd)$/
+            tag $1
+            label odd_label
+          </rule>
+          <rule>
+            key key
+            pattern /^(even)$/
+            tag ${tag}
+            label even_label
+          </rule>
+          <rule>
+            key key
+            pattern /^(.*)$/
+            tag $1
+          </rule>
+        ]
+      time = event_time
+      d = create_driver(conf)
+      # Router only called for default label
+      mock.proxy(d.instance.router).emit(anything, anything, anything).times(2)
+      mock.proxy(d.instance).get_router("odd_label").times(2)
+      mock.proxy(d.instance).get_router("even_label").times(2)
+      mock.proxy(d.instance).get_router(nil).times(2)
+      mock.proxy(d.instance.router).emit_stream(anything, anything).times(0)
+      d.run(default_tag: "input") do
+        d.feed([[time, { "key" => "odd", "message" => "message-1" }],
+                [time, { "key" => "even", "message" => "message-2" }],
+                [time, { "key" => "zero", "message" => "message-3" }],
+                [time, { "key" => "odd", "message" => "message-4" }],
+                [time, { "key" => "even", "message" => "message-5" }],
+                [time, { "key" => "zero", "message" => "message-6" }]])
+      end
+      events = d.events
+      expected_events = [
+        ["odd", time, { "key" => "odd", "message" => "message-1" }],
+        ["input", time, { "key" => "even", "message" => "message-2" }],
+        ["zero", time, { "key" => "zero", "message" => "message-3" }],
+        ["odd", time, { "key" => "odd", "message" => "message-4" }],
+        ["input", time, { "key" => "even", "message" => "message-5" }],
+        ["zero", time, { "key" => "zero", "message" => "message-6" }],
+      ]
+      assert_equal(events, expected_events)
+    end
+
     sub_test_case "emit_mode" do
       test "record" do
         conf = %[

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-rewrite-tag-filter
 version: !ruby/object:Gem::Version
-  version: 2.2.0
+  version: 2.4.0
 platform: ruby
 authors:
 - Kentaro Yoshida
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-12 00:00:00.000000000 Z
+date: 2020-12-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: test-unit
@@ -124,7 +124,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.1
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Fluentd Output filter plugin. It has designed to rewrite tag like mod_rewrite.