fluent-plugin-rewrite-tag-filter 2.0.0 → 2.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 2644562a37644d56fa8712f13d46aa0ff42cb9ec
-  data.tar.gz: 22ac2b253497f823babee508e41d695b9bdc8218
+SHA256:
+  metadata.gz: a12e8f6567cd07462a236f0cda38e98bac14b921a2c7b762e9851cb04e5bf119
+  data.tar.gz: b198b1e6b8a48d386f7cc62521f358cdff745754edc8dda753ac176ced1e3e67
 SHA512:
-  metadata.gz: eaa2ee232b41ba0b6f387adb69621e78a411919d8d074c3ce014d344b4211a1dad7d47b61a06698cbc4fa0791de55a87111bf91383c47cee3f56f71299ad53e1
-  data.tar.gz: ff33f9efd2106fc70e85023a139563764aa8414329066459bd9dd177133ee9ff8b9bff20119a50d8621c20573f3d7cdcfc3cb37c1d67a4f6210190b9523bb878
+  metadata.gz: cc556895e92c3f53ca3c41af6866072e48ac53b6b170cfe163f1563d04310c8b89cc802cabefd957dab95da29362ff76164621f0a83d806619d67df521e866b6
+  data.tar.gz: 1777b76d00afe128f20da230ca26d0c30d4f62e60f4800efec6b59450040e54de7b861ee7913ed82af8f91b9f3cf5e11344b5ec5b08e57af97d611a0060d0ddf

data/.travis.yml

@@ -1,10 +1,10 @@
 language: ruby
 
 rvm:
-  - 2.4.2
-  - 2.3.5
-  - 2.2
-  - 2.1
+  - 2.7.0
+  - 2.6.5
+  - 2.5.7
+  - 2.4.9
 
 # to avoid travis-ci issue since 2015-12-25
 before_install:

data/README.md

@@ -3,10 +3,12 @@
 ## Overview
 
 Rewrite Tag Filter for [Fluentd](http://fluentd.org). It is designed to rewrite tags like mod_rewrite.  
-Re-emit the record with rewrited tag when a value matches/unmatches with a regular expression.  
+Re-emit the record with rewritten tag when a value matches/unmatches with a regular expression.  
 Also you can change a tag from Apache log by domain, status code (ex. 500 error),  
 user-agent, request-uri, regex-backreference and so on with regular expression.
 
+This is an output plugin because fluentd's `filter` doesn't allow tag rewrite.
+
 ## Requirements
 
 | fluent-plugin-rewrite-tag-filter | Fluentd    | Ruby   |
@@ -16,34 +18,41 @@ user-agent, request-uri, regex-backreference and so on with regular expression.
 
 ## Installation
 
-Install with `gem`, `fluent-gem` or `td-agent-gem` command as:
+Install with `gem` or `td-agent-gem` command as:
 
 ```
 # for system installed fluentd
 $ gem install fluent-plugin-rewrite-tag-filter
 
-# for td-agent (Legacy)
-$ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-rewrite-tag-filter -v 1.5.6
-
 # for td-agent2 (with fluentd v0.12)
-$ sudo td-agent-gem install fluent-plugin-rewrite-tag-filter -v 1.5.6
+$ sudo td-agent-gem install fluent-plugin-rewrite-tag-filter -v 1.6.0
+
+# for td-agent3 (with fluentd v0.14)
+$ sudo td-agent-gem install fluent-plugin-rewrite-tag-filter
 ```
 
+For more details, see [Plugin Management](https://docs.fluentd.org/deployment/plugin-management)
+
 ## Configuration
 
 * **rewriterule\<num\>** (string) (optional) \<attribute\> \<regex_pattern\> \<new_tag\>
-  * Deprecated: Use <rule> section
+  * Obsoleted: Use \<rule\> section
 * **capitalize_regex_backreference** (bool) (optional): Capitalize letter for every matched regex backreference. (ex: maps -> Maps) for more details, see usage.
   * Default value: no
 * **remove_tag_prefix** (string) (optional): Remove tag prefix for tag placeholder. (see the section of "Tag placeholder")
 * **hostname_command** (string) (optional): Override hostname command for placeholder. (see the section of "Tag placeholder")
-  * Default value: `hostname`.
+  * Default value: `hostname`
+* **emit_mode** (enum) (required): Specify emit_mode to `batch` or `record`. `batch` will emit events per rewritten tag, and decrease IO. `record` will emit events per record.
+  * Default value: `batch`
 
 ### \<rule\> section (optional) (multiple)
 
 * **key** (string) (required): The field name to which the regular expression is applied
-* **pattern** (regexp) (required): The regular expression
+* **pattern** (regexp) (required): The regular expression.
+  `/regexp/` is preferred because `/regexp/` style can support character classes such as `/[a-z]/`.
+  The pattern without slashes will cause errors if you use patterns start with character classes.
 * **tag** (string) (required): New tag
+* **label** (string) (optional): New label. If specified, label can be changed per-rule.
 * **invert** (bool) (optional): If true, rewrite tag when unmatch pattern
   * Default value: `false`
 
@@ -70,39 +79,39 @@ It's a sample to exclude some static file log before split tag by domain.
   capitalize_regex_backreference yes
   <rule>
     key     path
-    pattern \.(gif|jpe?g|png|pdf|zip)$
+    pattern /\.(gif|jpe?g|png|pdf|zip)$/
     tag clear
   </rule>
   <rule>
     key     status
-    pattern ^200$
+    pattern /^200$/
     tag     clear
     invert  true
   </rule>
   <rule>
     key     domain
-    pattern ^.+\.com$
+    pattern /^.+\.com$/
     tag     clear
     invert  true
   </rule>
   <rule>
     key     domain
-    pattern ^maps\.example\.com$
+    pattern /^maps\.example\.com$/
     tag     site.ExampleMaps
   </rule>
   <rule>
     key     domain
-    pattern ^news\.example\.com$
+    pattern /^news\.example\.com$/
     tag     site.ExampleNews
   </rule>
   <rule>
     key     domain
-    pattern ^(mail)\.(example)\.com$
+    pattern /^(mail)\.(example)\.com$/
     tag     site.$2$1
   </rule>
   <rule>
     key     domain
-    pattern .+
+    pattern /.+/
     tag     site.unmatched
   </rule>
 </match>
@@ -157,7 +166,11 @@ Dot notation:
 ```
 <match kubernetes.**>
   @type rewrite_tag_filter
-  rewriterule1 $.kubernetes.namespace_name ^(.+)$ $1.${tag}
+  <rule>
+    key $.kubernetes.namespace_name
+    pattern ^(.+)$
+    tag $1.${tag}
+  </rule>
 </match>
 ```
 
@@ -166,7 +179,11 @@ Bracket notation:
 ```
 <match kubernetes.**>
   @type rewrite_tag_filter
-  rewriterule1 $['kubernetes']['namespace_name'] ^(.+)$ $1.${tag}
+  <rule>
+    key $['kubernetes']['namespace_name']
+    pattern ^(.+)$
+    tag $1.${tag}
+  </rule>
 </match>
 ```
 
@@ -184,7 +201,7 @@ When original tag is `kubernetes.var.log`, this will be converted to `default.ku
 
 ### Tag placeholder
 
-It is supported these placeholder for new_tag (rewrited tag).
+It is supported these placeholder for new_tag (rewritten tag).
 
 - `${tag}`
 - `__TAG__`
@@ -198,12 +215,16 @@ For example with `td.apache.access` tag, it will get `td` by `${tag_parts[0]}` a
 
 **Note** Currently, range expression ```${tag_parts[0..2]}``` is not supported.
 
-#### Placeholder Option
+#### Placeholder Options
 
 * `remove_tag_prefix`  
 
 This option adds removing tag prefix for `${tag}` or `__TAG__` in placeholder.
 
+* `remove_tag_regexp`  
+
+This option adds removing tag regexp for `${tag}` or `__TAG__` in placeholder.
+
 * `hostname_command` 
 
 By default, execute command as `hostname` to get full hostname.  
@@ -215,49 +236,100 @@ It comes short hostname with `hostname_command hostname -s` configuration specif
 It's a sample to rewrite a tag with placeholder.
 
 ```
-# It will get "rewrited.access.ExampleMail"
+# It will get "rewritten.access.ExampleMail"
 <match apache.access>
   @type rewrite_tag_filter
   remove_tag_prefix apache
   <rule>
     key     domain
     pattern ^(mail)\.(example)\.com$
-    tag     rewrited.${tag}.$2$1
+    tag     rewritten.${tag}.$2$1
+  </rule>
+</match>
+
+# It will get "rewritten.access.ExampleMail"
+<match apache.access>
+  @type rewrite_tag_filter
+  remove_tag_regexp /^apache\./
+  <rule>
+    key     domain
+    pattern ^(mail)\.(example)\.com$
+    tag     rewritten.${tag}.$2$1
+  </rule>
+</match>
+
+# It will get "http.access.log"
+<match input.{apache,nginx}.access.log>
+  @type rewrite_tag_filter
+  remove_tag_regexp /^input\.(apache|nginx)\./
+  <rule>
+    key     domain
+    pattern ^.+$
+    tag     http.${tag}
   </rule>
 </match>
 
-# It will get "rewrited.ExampleMail.app30-124.foo.com" when hostname is "app30-124.foo.com"
+# It will get "rewritten.ExampleMail.app30-124.foo.com" when hostname is "app30-124.foo.com"
 <match apache.access>
   @type rewrite_tag_filter
   <rule>
     key     domain
     pattern ^(mail)\.(example)\.com$
-    tag     rewrited.$2$1.${hostname}
+    tag     rewritten.$2$1.${hostname}
   </rule>
 </match>
 
-# It will get "rewrited.ExampleMail.app30-124" when hostname is "app30-124.foo.com"
+# It will get "rewritten.ExampleMail.app30-124" when hostname is "app30-124.foo.com"
 <match apache.access>
   @type rewrite_tag_filter
   hostname_command hostname -s
   <rule>
     key     domain
     pattern ^(mail)\.(example)\.com$
-    tag     rewrited.$2$1.${hostname}
+    tag     rewritten.$2$1.${hostname}
   </rule>
 </match>
 
-# It will get "rewrited.game.pool"
+# It will get "rewritten.game.pool"
 <match app.game.pool.activity>
   @type rewrite_tag_filter
   <rule>
     key     domain
     pattern ^.+$
-    tag     rewrited.${tag_parts[1]}.${tag_parts[2]}
+    tag     rewritten.${tag_parts[1]}.${tag_parts[2]}
   </rule>
 </match>
 ```
 
+### Altering Labels
+
+In addition to changing tags, you can also change event's route by setting
+ the label for the re-emitted event.
+
+For example, given this configuration:
+
+```
+<match apache.access>
+  @type rewrite_tag_filter
+  <rule>
+    key     domain
+    pattern ^www\.example\.com$
+    tag     web.${tag}
+  </rule>
+  <rule>
+    key     domain
+    pattern ^(.*)\.example\.com$
+    tag     other.$1
+    label   other
+  </rule>
+</match>
+```
+
+message: `{"domain": "www.example.com"}` will get its tag changed to 
+`web.apache.access`, while message 
+`{"domain": "api.example.com"}` will get its tag changed to `other.api` and
+ be sent to label `other`
+
 ## Example
 
 - Example1: how to analyze response_time, response_code and user_agent for each virtual domain websites.  

data/example.conf

@@ -2,15 +2,17 @@
 
 # appearing plugins:
 #  rewrite_tag_filter: http://rubygems.org/gems/fluent-plugin-rewrite-tag-filter
-#  forest: http://rubygems.org/gems/fluent-plugin-forest
 #  datacounter: http://rubygems.org/gems/fluent-plugin-datacounter
 #  growthforecast: http://rubygems.org/gems/fluent-plugin-growthforecast
 
 <source>
-  type tail
+  @type tail
   path /var/log/httpd/access_log
-  format /^(?<domain>[^ ]*) (?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<status>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<response_time>[^ ]*))?$/
-  time_format %d/%b/%Y:%H:%M:%S %z
+  <parse>
+    @type regexp
+    expression /^(?<domain>[^ ]*) (?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<status>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<response_time>[^ ]*))?$/
+    time_format %d/%b/%Y:%H:%M:%S %z
+  </parse>
   tag td.apache.access
   pos_file /var/log/td-agent/apache_access.pos
 </source>
@@ -18,139 +20,173 @@
 
 # Extract specified virtual domain
 <match td.apache.access>
-  type copy
+  @type copy
   <store>
-    type rewrite_tag_filter
+    @type rewrite_tag_filter
     capitalize_regex_backreference yes
-    rewriterule1 domain ^(maps|news|mail)\.google\.com$ site.Google$1
+    <rule>
+      key domain
+      pattern /^(maps|news|mail)\.google\.com$/
+      tag site.Google$1
+    </rule>
   </store>
   <store>
-    type rewrite_tag_filter
+    @type rewrite_tag_filter
     capitalize_regex_backreference yes
-    rewriterule1 domain ^(maps)\.google\.com$ sitepath.Google$1
+    <rule>
+      key domain
+      pattern /^(maps)\.google\.com$/
+      tag sitepath.Google$1
+    </rule>
   </store>
 </match>
 
 
 # Second level analyzing
 <match sitepath.GoogleMaps>
+  @type copy
   <store>
-    type rewrite_tag_filter
-    rewriterule1 path ^/labs site.GoogleMaps.Labs
-    rewriterule2 path ^/static/\d+ site.GoogleMaps.Static
-    rewriterule3 path ^/help/maps/(streetview|getmaps) site.GoogleMaps.Help
+    @type rewrite_tag_filter
+    <rule>
+      key path
+      pattern /^\/labs/
+      tag site.GoogleMaps.Labs
+    </rule>
+    <rule>
+      key path
+      pattern /^\/static/\d+/
+      tag site.GoogleMaps.Static
+    </rule>
+    <rule>
+      key path
+      pattern /^\/help\/maps\/(streetview|getmaps)/
+      tag site.GoogleMaps.Help
+    </rule>
   </store>
   <store>
-    type rewrite_tag_filter
-    rewriterule1 referer headlines\.yahoo\.co\.jp site.GoogleMaps.referer_YahooHeadlines
-    rewriterule2 referer news\.livedoor\.com site.GoogleMaps.referer_LivedoorNews
+    @type rewrite_tag_filter
+    <rule>
+      key referer
+      pattern /headlines\.yahoo\.co\.jp/
+      tag site.GoogleMaps.referer_YahooHeadlines
+    </rule>
+    <rule>
+      key referer
+      pattern /news\.livedoor\.com/
+      tag site.GoogleMaps.referer_LivedoorNews
+    </rule>
   </store>
   <store>
-    type rewrite_tag_filter
-    rewriterule1 agent Googlebot/ site.GoogleMaps.agent_Googlebot
-    rewriterule2 agent ^.* iPhone .+Googlebot-Mobile/.*$ site.GoogleMaps.agent_GooglebotSmartphone
-    rewriterule3 agent Googlebot-Mobile/ site.GoogleMaps.agent_GooglebotMobile
-    rewriterule4 agent bingbot site.GoogleMaps.agent_Bingbot
-    rewriterule5 agent Baiduspider site.GoogleMaps.agent_Baiduspider
+    @type rewrite_tag_filter
+    <rule>
+      key agent
+      pattern /Googlebot\//
+      tag site.GoogleMaps.agent_Googlebot
+    </rule>
+    <rule>
+      key agent
+      pattern /^.* iPhone .+Googlebot-Mobile\/.*$/
+      tag site.GoogleMaps.agent_GooglebotSmartphone
+    </rule>
+    <rule>
+      key agent
+      pattern /Googlebot-Mobile\//
+      tag site.GoogleMaps.agent_GooglebotMobile
+    </rule>
+    <rule>
+      key agent
+      pattern /bingbot/
+      tag site.GoogleMaps.agent_Bingbot
+    </rule>
+    <rule>
+      key agent
+      pattern /Baiduspider/
+      tag site.GoogleMaps.agent_Baiduspider
+    </rule>
   </store>
 </match>
 
 
 # Summarize
 <match site.**>
-  type copy
+  @type copy
   <store>
-    type forest
-    subtype datacounter
-    <template>
-      unit minute
-      count_key response_time
-      outcast_unmatched false
-      aggregate all
-      tag gf.responsetime.__TAG__
-      pattern1 0-100msec ^\d{1,5}$
-      pattern2 100-300msec ^[1-2]\d{5}$
-      pattern3 300-600msec ^[3-5]\d{5}$
-      pattern4 600msec-1sec ^[6-9]\d{5}$
-      pattern5 1-2sec ^1\d{6}$
-      pattern6 2-10sec ^[2-9]\d{6}$
-      pattern7 10sec_over ^\d{8,}$
-    </template>
+    @type datacounter
+    unit minute
+    count_key response_time
+    outcast_unmatched false
+    aggregate tag
+    tag_prefix gf.responsetime
+    output_per_tag yes
+    pattern1 0-100msec ^\d{1,5}$
+    pattern2 100-300msec ^[1-2]\d{5}$
+    pattern3 300-600msec ^[3-5]\d{5}$
+    pattern4 600msec-1sec ^[6-9]\d{5}$
+    pattern5 1-2sec ^1\d{6}$
+    pattern6 2-10sec ^[2-9]\d{6}$
+    pattern7 10sec_over ^\d{8,}$
   </store>
   <store>
-    type forest
-    subtype datacounter
-    <template>
-      unit minute
-      outcast_unmatched false
-      aggregate all
-      tag gf.responsecode.__TAG__
-      count_key status
-      pattern1 200 ^200$
-      pattern2 2xx ^2\d\d$
-      pattern3 301 ^301$
-      pattern4 302 ^302$
-      pattern5 3xx ^3\d\d$
-      pattern6 403 ^403$
-      pattern7 404 ^404$
-      pattern8 410 ^410$
-      pattern9 4xx ^4\d\d$
-      pattern10 500 ^5\d\d$
-    </template>
+    @type datacounter
+    unit minute
+    outcast_unmatched false
+    aggregate tag
+    tag_prefix gf.responsecode
+    output_per_tag yes
+    count_key status
+    pattern1 200 ^200$
+    pattern2 2xx ^2\d\d$
+    pattern3 301 ^301$
+    pattern4 302 ^302$
+    pattern5 3xx ^3\d\d$
+    pattern6 403 ^403$
+    pattern7 404 ^404$
+    pattern8 410 ^410$
+    pattern9 4xx ^4\d\d$
+    pattern10 500 ^5\d\d$
   </store>
   <store>
-    type forest
-    subtype datacounter
-    <template>
-      unit minute
-      count_key agent
-      outcast_unmatched false
-      aggregate all
-      tag gf.useragent.__TAG__
-      pattern1 api HttpRequest
-      pattern2 robot (spider|bot|crawler|\+http\:)
-      pattern3 smartphone (iPhone|iPod|Android|dream|CUPCAKE|blackberry|webOS|incognito|webmate|IEMobile)
-      pattern4 mobile (^KDDI|UP.Browser|DoCoMo|Vodafone|SoftBank|WILLCOM)
-      pattern5 pc .+
-    </template>
+    @type datacounter
+    unit minute
+    count_key agent
+    outcast_unmatched false
+    aggregate tag
+    tag_prefix gf.useragent
+    output_per_tag yes
+    pattern1 api HttpRequest
+    pattern2 robot (spider|bot|crawler|\+http\:)
+    pattern3 smartphone (iPhone|iPod|Android|dream|CUPCAKE|blackberry|webOS|incognito|webmate|IEMobile)
+    pattern4 mobile (^KDDI|UP.Browser|DoCoMo|Vodafone|SoftBank|WILLCOM)
+    pattern5 pc .+
   </store>
 </match>
 
 
 # Graph
 <match gf.responsetime.**>
-  type forest
-  subtype growthforecast
+  @type growthforecast
   remove_prefix gf.responsetime.site
-  <template>
-    gfapi_url http://localhost:5125/api/
-    service   __TAG__
-    section   response_time
-    name_keys 0-100msec_percentage,100-300msec_percentage,300-600msec_percentage,600msec-1sec_percentage,1-2sec_percentage,2-10sec_percentage,10sec_over_percentage
-  </template>
+  gfapi_url http://localhost:5125/api/
+  tag_for   service
+  section   response_time
+  name_keys 0-100msec_percentage,100-300msec_percentage,300-600msec_percentage,600msec-1sec_percentage,1-2sec_percentage,2-10sec_percentage,10sec_over_percentage
 </match>
 
 <match gf.responsecode.**>
-  type forest
-  subtype growthforecast
+  @type growthforecast
   remove_prefix gf.responsecode.site
-  <template>
-    gfapi_url http://localhost:5125/api/
-    service   __TAG__
-    section   response_code
-    name_keys 301_count,302_count,3xx_count,403_count,404_count,410_count,4xx_count,500_count
-  </template>
+  gfapi_url http://localhost:5125/api/
+  tag_for   service
+  section   response_code
+  name_keys 301_count,302_count,3xx_count,403_count,404_count,410_count,4xx_count,500_count
 </match>
 
 <match gf.useragent.**>
-  type forest
-  subtype growthforecast
+  @type growthforecast
   remove_prefix gf.useragent.site
-  <template>
-    gfapi_url http://localhost:5125/api/
-    service   __TAG__
-    section   useragent
-    name_keys pc_count,mobile_count,smartphone_count,robot_count,api_count
-  </template>
+  gfapi_url http://localhost:5125/api/
+  tag_for   service
+  section   useragent
+  name_keys pc_count,mobile_count,smartphone_count,robot_count,api_count
 </match>
 

data/example2.conf

@@ -7,10 +7,13 @@
 #  growthforecast: http://rubygems.org/gems/fluent-plugin-growthforecast
 
 <source>
-  type tail
+  @type tail
   path /var/log/httpd/access_log
-  format /^(?<domain>[^ ]*) (?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<status>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<response_time>[^ ]*))?$/
-  time_format %d/%b/%Y:%H:%M:%S %z
+  <parse>
+    @type regexp
+    expression /^(?<domain>[^ ]*) (?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<status>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<response_time>[^ ]*))?$/
+    time_format %d/%b/%Y:%H:%M:%S %z
+  </parse>
   tag td.apache.access
   pos_file /var/log/td-agent/apache_access.pos
 </source>
@@ -18,60 +21,83 @@
 
 # Extract specified virtual domain
 <match td.apache.access>
-  type rewrite_tag_filter
-  rewriterule1 domain ^maps\.google\.com$ filter.GoogleMap
+  @type rewrite_tag_filter
+  <rule>
+    key domain
+    pattern /^maps\.google\.com$/
+    tag filter.GoogleMap
+  </rule>
 </match>
 
 
 # Filtering
 <match filter.GoogleMap>
-  type rewrite_tag_filter
-  rewriterule1 path ^/(img|css|js|static|assets)/ clear
-  rewriterule2 status ^(?!200)$ clear
-  rewriterule3 method ^(?!GET)$ clear
-  rewriterule4 agent (spider|bot|crawler|\+http\:) clear
-  rewriterule5 path ^/(admin|api|backend) site.GoogleMap.backend
-  rewriterule6 path .+ site.GoogleMap.front
+  @type rewrite_tag_filter
+  <rule>
+    key path
+    pattern /^\/(img|css|js|static|assets)\//
+    tag clear
+  </rule>
+  <rule>
+    key status
+    pattern /^(?!200)$/
+    tag clear
+  </rule>
+  <rule>
+    key method
+    pattern /^(?!GET)$/
+    tag clear
+  </rule>
+  <rule>
+    key agent
+    pattern /(spider|bot|crawler|\+http\:)/
+    tag clean
+  </rule>
+  <rule>
+    key path
+    pattern /^\/(admin|api|backend)/
+    tag site.GoogleMap.backend
+  </rule>
+  <rule>
+    key path
+    pattern /.+/
+    tag site.GoogleMap.front
+  </rule>
 </match>
 
 
 # Summarize
 <match site.**>
-  type forest
-  subtype datacounter
-  <template>
-    unit minute
-    count_key response_time
-    outcast_unmatched false
-    aggregate all
-    tag gf.responsetime.__TAG__
-    pattern1 0-100msec ^\d{1,5}$
-    pattern2 100-300msec ^[1-2]\d{5}$
-    pattern3 300-600msec ^[3-5]\d{5}$
-    pattern4 600msec-1sec ^[6-9]\d{5}$
-    pattern5 1-2sec ^1\d{6}$
-    pattern6 2-10sec ^[2-9]\d{6}$
-    pattern7 10sec_over ^\d{8,}$
-  </template>
+  @type datacounter
+  unit minute
+  count_key response_time
+  outcast_unmatched false
+  aggregate tag
+  tag_prefix gf.responsetime
+  output_per_tag yes
+  pattern1 0-100msec ^\d{1,5}$
+  pattern2 100-300msec ^[1-2]\d{5}$
+  pattern3 300-600msec ^[3-5]\d{5}$
+  pattern4 600msec-1sec ^[6-9]\d{5}$
+  pattern5 1-2sec ^1\d{6}$
+  pattern6 2-10sec ^[2-9]\d{6}$
+  pattern7 10sec_over ^\d{8,}$
 </match>
 
 
 # Graph
 <match gf.responsetime.**>
-  type forest
-  subtype growthforecast
+  @type growthforecast
   remove_prefix gf.responsetime.site
-  <template>
-    gfapi_url http://localhost:5125/api/
-    service   __TAG__
-    section   response_time
-    name_keys 0-100msec_percentage,100-300msec_percentage,300-600msec_percentage,600msec-1sec_percentage,1-2sec_percentage,2-10sec_percentage,10sec_over_percentage
-  </template>
+  gfapi_url http://localhost:5125/api/
+  tag_for   service
+  section   response_time
+  name_keys 0-100msec_percentage,100-300msec_percentage,300-600msec_percentage,600msec-1sec_percentage,1-2sec_percentage,2-10sec_percentage,10sec_over_percentage
 </match>
 
 
 # Clear tag
 <match clear>
-  type null
+  @type null
 </match>
 

data/fluent-plugin-rewrite-tag-filter.gemspec

@@ -1,9 +1,8 @@
-# -*- encoding: utf-8 -*-
 $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-rewrite-tag-filter"
-  s.version     = "2.0.0"
+  s.version     = "2.4.0"
   s.license     = "Apache-2.0"
   s.authors     = ["Kentaro Yoshida"]
   s.email       = ["y.ken.studio@gmail.com"]
@@ -16,6 +15,8 @@ Gem::Specification.new do |s|
   s.require_paths = ["lib"]
 
   s.add_development_dependency "test-unit", ">= 3.1.0"
+  s.add_development_dependency "test-unit-rr"
   s.add_development_dependency "rake"
-  s.add_runtime_dependency "fluentd", "~> 0.14.2"
+  s.add_runtime_dependency "fluentd", ">= 0.14.2", "< 2"
+  s.add_runtime_dependency "fluent-config-regexp-type"
 end

data/lib/fluent/plugin/out_rewrite_tag_filter.rb

@@ -1,4 +1,5 @@
 require "fluent/plugin/output"
+require "fluent/config/regexp_type"
 
 class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
   Fluent::Plugin.register_output('rewrite_tag_filter', self)
@@ -9,18 +10,22 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
   config_param :capitalize_regex_backreference, :bool, :default => false
   desc 'Remove tag prefix for tag placeholder.'
   config_param :remove_tag_prefix, :string, :default => nil
+  desc "Remove tag regexp for tag placeholder."
+  config_param :remove_tag_regexp, :regexp, default: nil
   desc 'Override hostname command for placeholder.'
   config_param :hostname_command, :string, :default => 'hostname'
+  desc "The emit mode. If `batch`, this plugin will emit events per rewritten tag."
+  config_param :emit_mode, :enum, list: [:record, :batch], default: :batch
 
   config_section :rule, param_name: :rules, multi: true do
     desc "The field name to which the regular expression is applied"
     config_param :key, :string
     desc "The regular expression"
-    config_param :pattern do |value|
-      Regexp.compile(value)
-    end
+    config_param :pattern, :regexp
     desc "New tag"
     config_param :tag, :string
+    desc "New label. If specified, label can be changed per-rule."
+    config_param :label, :string, default: nil
     desc "If true, rewrite tag when unmatch pattern"
     config_param :invert, :bool, default: false
   end
@@ -40,7 +45,7 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
       end
 
       invert = rule.invert ? MATCH_OPERATOR_EXCLUDE : ""
-      @rewriterules.push([record_accessor_create(rule.key), rule.pattern, invert, rule.tag])
+      @rewriterules.push([record_accessor_create(rule.key), rule.pattern, invert, rule.tag, rule.label])
       rewriterule_names.push(rule.key + invert + rule.pattern.to_s)
       log.info "adding rewrite_tag_filter rule: #{rule.key} #{@rewriterules.last}"
     end
@@ -57,26 +62,60 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
       raise Fluent::ConfigError, "duplicated rewriterules found #{@rewriterules.inspect}"
     end
 
+    if @remove_tag_prefix && @remove_tag_regexp
+      raise Fluent::ConfigError, "remove_tag_prefix and remove_tag_regexp are exclusive"
+    end
+
     unless @remove_tag_prefix.nil?
-      @remove_tag_prefix = /^#{Regexp.escape(@remove_tag_prefix)}\.?/
+      @remove_tag_regexp = /^#{Regexp.escape(@remove_tag_prefix)}\.?/
     end
+
+    @batch_mode = @emit_mode == :batch
   end
 
   def multi_workers_ready?
     true
   end
 
+  def get_router(tgt_label)
+    label_router = if tgt_label.nil? || tgt_label.empty?
+        router
+    else
+        event_emitter_router(tgt_label)
+    end
+    #log.trace "Got router for #{tgt_label}: #{!label_router.nil?}"
+    return label_router
+  end
+
   def process(tag, es)
+    placeholder = get_placeholder(tag)
+    new_event_streams =  Hash.new {|hh, kk| hh[kk] = Hash.new {|h, k| h[k] = Fluent::MultiEventStream.new }}if @batch_mode
+
     es.each do |time, record|
-      rewrited_tag = rewrite_tag(tag, record)
-      next if rewrited_tag.nil? || tag == rewrited_tag
-      router.emit(rewrited_tag, time, record)
+      rewrited_tag, rewrited_label  = rewrite_tag(tag, record, placeholder)
+      if (rewrited_tag.nil? || tag == rewrited_tag) && rewrited_label.nil?
+        log.trace("rewrite_tag_filter: tag has not been rewritten", record)
+        next
+      end
+      rewrited_tag = tag if rewrited_tag.nil?
+      if new_event_streams.nil?
+        get_router(rewrited_label).emit(rewrited_tag, time, record)
+      else
+        new_event_streams[rewrited_label][rewrited_tag].add(time, record)
+      end
+    end
+    if !new_event_streams.nil?
+      new_event_streams.each do |rewrited_label, label_event_streams |
+        labeled_router = get_router(rewrited_label)
+        label_event_streams.each do |rewrited_tag, new_es|
+            labeled_router.emit_stream(rewrited_tag, new_es)
+        end
+      end
     end
   end
 
-  def rewrite_tag(tag, record)
-    placeholder = get_placeholder(tag)
-    @rewriterules.each do |record_accessor, regexp, match_operator, rewritetag|
+  def rewrite_tag(tag, record, placeholder)
+    @rewriterules.each do |record_accessor, regexp, match_operator, rewritetag, rewritelabel|
       rewritevalue = record_accessor.call(record).to_s
       next if rewritevalue.empty? && match_operator != MATCH_OPERATOR_EXCLUDE
       last_match = regexp_last_match(regexp, rewritevalue)
@@ -92,9 +131,9 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
         log.warn "rewrite_tag_filter: unknown placeholder found. :placeholder=>#{$1} :tag=>#{tag} :rewritetag=>#{rewritetag}" unless placeholder.include?($1)
         placeholder[$1]
       end
-      return rewritetag
+      return rewritetag, rewritelabel
     end
-    return nil
+    return nil, nil
   end
 
   def regexp_last_match(regexp, rewritevalue)
@@ -114,7 +153,7 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
   end
 
   def get_placeholder(tag)
-    tag = tag.sub(@remove_tag_prefix, '') if @remove_tag_prefix
+    tag = tag.sub(@remove_tag_regexp, '') if @remove_tag_regexp
 
     result = {
       '__HOSTNAME__' => @hostname,

data/test/helper.rb

@@ -1,9 +1,13 @@
 require 'bundler/setup'
 require 'test/unit'
+require 'test/unit/rr'
 
 $LOAD_PATH.unshift(File.join(__dir__, '..', 'lib'))
 $LOAD_PATH.unshift(__dir__)
 require 'fluent/test'
 require 'fluent/test/driver/output'
+require 'fluent/test/helpers'
+
+Test::Unit::TestCase.include(Fluent::Test::Helpers)
 
 require 'fluent/plugin/out_rewrite_tag_filter'

data/test/plugin/test_out_rewrite_tag_filter.rb

@@ -17,6 +17,45 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
         create_driver(conf)
       end
     end
+
+    test "pattern with character classes" do
+      conf = %[
+        <rule>
+          key $['email']['domain']
+          pattern /[sv]d[a-z]+\\d*$/
+          tag $2.$1
+        </rule>
+      ]
+      d = create_driver(conf)
+      assert_equal(/[sv]d[a-z]+\d*$/, d.instance.rules.first.pattern)
+    end
+
+    test "pattern w/o slashes" do
+      conf = %[
+        <rule>
+          key $['email']['domain']
+          pattern .+
+          tag $2.$1
+        </rule>
+      ]
+      d = create_driver(conf)
+      assert_equal(/.+/, d.instance.rules.first.pattern)
+    end
+
+    test "remove_tag_prefix and remove_tag_regexp are exclusive" do
+      conf = %[
+        remove_tag_prefix prefix
+        remove_tag_regexp /^prefix\./
+        <rule>
+          key message
+          pattern .+
+          tag ${tag}
+        </rule>
+      ]
+      assert_raise(Fluent::ConfigError) do
+        create_driver(conf)
+      end
+    end
   end
 
   sub_test_case "section style" do
@@ -103,6 +142,30 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
       assert_equal 'access', events[0][0] # tag
     end
 
+    sub_test_case "remove_tag_regexp" do
+      test "plain" do
+        config = %[
+          remove_tag_regexp /^input\.(apache|nginx)\./
+          <rule>
+            key domain
+            pattern ^www\.google\.com$
+            tag rewritten.${tag}
+          </rule>
+        ]
+        d = create_driver(config)
+        d.run do
+          d.feed('input.apache.access', event_time, {'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
+          d.feed('input.nginx.access', event_time, {'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
+          d.feed('input.tomcat.access', event_time, {'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
+        end
+        events = d.events
+        assert_equal 3, events.length
+        assert_equal 'rewritten.access', events[0][0]
+        assert_equal 'rewritten.access', events[1][0]
+        assert_equal 'rewritten.input.tomcat.access', events[2][0]
+      end
+    end
+
     test "short hostname" do
       config = %[
         remove_tag_prefix input
@@ -260,5 +323,166 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
       events = d.events
       assert_equal "com.example", events[0][0]
     end
+
+    test "tag has not been rewritten and log_level trace" do
+      conf = %[
+        @log_level trace
+        <rule>
+          key $['email']['domain']
+          pattern ^(example)\.(com)$
+          tag $2.$1
+        </rule>
+      ]
+      d = create_driver(conf)
+      d.run(default_tag: "input") do
+        d.feed({ "email" => { "localpart" => "john", "domain" => "example.com" }})
+        d.feed({ "email" => { "localpart" => "doe", "domain" => "example.jp" }})
+      end
+      events = d.events
+      assert_equal(1, events.size)
+      log = d.logs.grep(/\[trace\]/).first
+      assert_equal('rewrite_tag_filter: tag has not been rewritten email={"localpart"=>"doe", "domain"=>"example.jp"}',
+                   log.slice(/\[trace\]: (.+)$/, 1))
+      assert_equal "com.example", events[0][0]
+    end
+
+    test "get_router" do
+      conf = %[
+        <rule>
+          key key
+          pattern /^(odd|even)$/
+          tag $1
+          label new_label
+        </rule>
+        <rule>
+          key key
+          pattern /^(.*)$/
+          tag $1
+        </rule>
+      ]
+      time = event_time
+      d = create_driver(conf)
+      assert_equal(d.instance.router, d.instance.get_router(nil))
+      assert_equal(d.instance.router, d.instance.get_router(""))
+      new_label_router = d.instance.get_router("new_label")
+      refute_equal(d.instance.router, new_label_router)
+    end
+
+    test "relabel" do
+      conf = %[
+          emit_mode record
+          <rule>
+            key key
+            pattern /^(odd)$/
+            tag $1
+            label odd_label
+          </rule>
+          <rule>
+            key key
+            pattern /^(even)$/
+            tag ${tag}
+            label even_label
+          </rule>
+          <rule>
+            key key
+            pattern /^(.*)$/
+            tag $1
+          </rule>
+        ]
+      time = event_time
+      d = create_driver(conf)
+      # Router only called for default label
+      mock.proxy(d.instance.router).emit(anything, anything, anything).times(2)
+      mock.proxy(d.instance).get_router("odd_label").times(2)
+      mock.proxy(d.instance).get_router("even_label").times(2)
+      mock.proxy(d.instance).get_router(nil).times(2)
+      mock.proxy(d.instance.router).emit_stream(anything, anything).times(0)
+      d.run(default_tag: "input") do
+        d.feed([[time, { "key" => "odd", "message" => "message-1" }],
+                [time, { "key" => "even", "message" => "message-2" }],
+                [time, { "key" => "zero", "message" => "message-3" }],
+                [time, { "key" => "odd", "message" => "message-4" }],
+                [time, { "key" => "even", "message" => "message-5" }],
+                [time, { "key" => "zero", "message" => "message-6" }]])
+      end
+      events = d.events
+      expected_events = [
+        ["odd", time, { "key" => "odd", "message" => "message-1" }],
+        ["input", time, { "key" => "even", "message" => "message-2" }],
+        ["zero", time, { "key" => "zero", "message" => "message-3" }],
+        ["odd", time, { "key" => "odd", "message" => "message-4" }],
+        ["input", time, { "key" => "even", "message" => "message-5" }],
+        ["zero", time, { "key" => "zero", "message" => "message-6" }],
+      ]
+      assert_equal(events, expected_events)
+    end
+
+    sub_test_case "emit_mode" do
+      test "record" do
+        conf = %[
+          emit_mode record
+          <rule>
+            key key
+            pattern /^(odd|even)$/
+            tag $1
+          </rule>
+        ]
+        time = event_time
+        d = create_driver(conf)
+        mock.proxy(d.instance.router).emit(anything, anything, anything).times(6)
+        mock.proxy(d.instance.router).emit_stream(anything, anything).times(0)
+        d.run(default_tag: "input") do
+          d.feed([[time, { "key" => "odd", "message" => "message-1" }],
+                  [time, { "key" => "even", "message" => "message-2" }],
+                  [time, { "key" => "odd", "message" => "message-3" }],
+                  [time, { "key" => "even", "message" => "message-4" }],
+                  [time, { "key" => "odd", "message" => "message-5" }],
+                  [time, { "key" => "even", "message" => "message-6" }]])
+        end
+        events = d.events
+        expected_events = [
+          ["odd", time, { "key" => "odd", "message" => "message-1" }],
+          ["even", time, { "key" => "even", "message" => "message-2" }],
+          ["odd", time, { "key" => "odd", "message" => "message-3" }],
+          ["even", time, { "key" => "even", "message" => "message-4" }],
+          ["odd", time, { "key" => "odd", "message" => "message-5" }],
+          ["even", time, { "key" => "even", "message" => "message-6" }],
+        ]
+        assert_equal(events, expected_events)
+      end
+
+      test "batch" do
+        conf = %[
+          emit_mode batch
+          <rule>
+            key key
+            pattern /^(odd|even)$/
+            tag $1
+          </rule>
+        ]
+        time = event_time
+        d = create_driver(conf)
+        mock.proxy(d.instance.router).emit(anything, anything, anything).times(0)
+        mock.proxy(d.instance.router).emit_stream(anything, anything).times(2)
+        d.run(default_tag: "input") do
+          d.feed([[time, { "key" => "odd", "message" => "message-1" }],
+                  [time, { "key" => "even", "message" => "message-2" }],
+                  [time, { "key" => "odd", "message" => "message-3" }],
+                  [time, { "key" => "even", "message" => "message-4" }],
+                  [time, { "key" => "odd", "message" => "message-5" }],
+                  [time, { "key" => "even", "message" => "message-6" }]])
+        end
+        events = d.events
+        expected_records = [
+          ["odd", time, { "key" => "odd", "message" => "message-1" }],
+          ["odd", time, { "key" => "odd", "message" => "message-3" }],
+          ["odd", time, { "key" => "odd", "message" => "message-5" }],
+          ["even", time, { "key" => "even", "message" => "message-2" }],
+          ["even", time, { "key" => "even", "message" => "message-4" }],
+          ["even", time, { "key" => "even", "message" => "message-6" }],
+        ]
+        assert_equal(events, expected_records)
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-rewrite-tag-filter
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.4.0
 platform: ruby
 authors:
 - Kentaro Yoshida
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-11-06 00:00:00.000000000 Z
+date: 2020-12-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: test-unit
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 3.1.0
+- !ruby/object:Gem::Dependency
+  name: test-unit-rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -42,16 +56,36 @@ dependencies:
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.14.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.14.2
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: fluent-config-regexp-type
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
 - y.ken.studio@gmail.com
@@ -90,8 +124,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.13
+rubygems_version: 3.1.4
 signing_key: 
 specification_version: 4
 summary: Fluentd Output filter plugin. It has designed to rewrite tag like mod_rewrite.