fluent-plugin-rewrite-tag-filter 1.3.1 → 1.4.0

data/.travis.yml

@@ -0,0 +1,6 @@
+language: ruby
+
+rvm:
+  - 2.0.0
+  - 1.9.3
+

data/README.md

@@ -1,4 +1,4 @@
-# fluent-plugin-rewrite-tag-filter
+# fluent-plugin-rewrite-tag-filter [![Build Status](https://travis-ci.org/fluent/fluent-plugin-rewrite-tag-filter.png?branch=master)](https://travis-ci.org/fluent/fluent-plugin-rewrite-tag-filter)
 
 ## Overview
 
@@ -11,12 +11,14 @@ user-agent, request-uri, regex-backreference and so on with regular expression.
 
 ## Installation
 
+install with gem or fluent-gem command as:
+
 ```
-# native gem
-gem install fluent-plugin-rewrite-tag-filter
+# for fluentd
+$ gem install fluent-plugin-rewrite-tag-filter
 
-# td-agent gem
-/usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-rewrite-tag-filter
+# for td-agent
+$ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-rewrite-tag-filter
 ```
 
 ## Configuration
@@ -26,7 +28,8 @@ gem install fluent-plugin-rewrite-tag-filter
 ```
 rewriterule<num> <attribute> <regex_pattern> <new_tag>
 
-# Optional: Capitalize every matched regex backreference. (ex: $1, $2)
+# Optional: Capitalize letter for every matched regex backreference. (ex: maps -> Maps)
+# for more details, see usage.
 capitalize_regex_backreference <yes/no> (default no)
 
 # Optional: remove tag prefix for tag placeholder. (see the section of "Tag placeholder")
@@ -50,6 +53,7 @@ It's a sample to exclude some static file log before split tag by domain.
   pos_file /var/log/td-agent/apache_access.pos
 </source>
 
+# "capitalize_regex_backreference yes" affects converting every matched first letter of backreference to upper case. ex: maps -> Maps
 # At rewriterule2, redirect to tag named "clear" which unmatched for status code 200.
 # At rewriterule3, redirect to tag named "clear" which is not end with ".com"
 # At rewriterule6, "site.$2$1" to be "site.ExampleMail" by capitalize_regex_backreference option.
@@ -110,19 +114,30 @@ $ tailf /var/log/td-agent/td-agent.log
 
 ### Tag placeholder
 
-It is supporting there placeholder for new_tag(rewrited tag).
+It is supported these placeholder for new_tag (rewrited tag).
 
 - `${tag}`
 - `__TAG__`
+- `{$tag_parts[n]}`
+- `__TAG_PARTS[n]__`
+- `${hostname}`
+- `__HOSTNAME__`
+
+The placeholder of `{$tag_parts[n]}` and `__TAG_PARTS[n]__` acts accessing the index which split the tag with "." (dot).  
+For example with `td.apache.access` tag, it will get `td` by `${tag_parts[0]}` and `apache` by `${tag_parts[1]}`.
+
+**Note** Currently, range expression ```${tag_parts[0..2]}``` is not supported.
+
+#### Placeholder Option
+
+* `remove_tag_prefix`  
 
-It's available to use this placeholder with `remove_tag_prefix` option.  
 This option adds removing tag prefix for `${tag}` or `__TAG__` in placeholder.
 
-- `${hostname}`
-- `__HOSTNAME__`
+* `hostname_command` 
 
 By default, execute command as `hostname` to get full hostname.  
-Also, you can override hostname command using `hostname_command` option.  
+On your needs, it could override hostname command using `hostname_command` option.  
 It comes short hostname with `hostname_command hostname -s` configuration specified.
 
 #### Placeholder Usage
@@ -149,6 +164,12 @@ It's a sample to rewrite a tag with placeholder.
   rewriterule1  domain  ^(mail)\.(example)\.com$  rewrited.$2$1.${hostname}
   hostname_command hostname -s
 </match>
+
+# It will get "rewrited.game.pool"
+<match app.game.pool.activity>
+  type rewrite_tag_filter
+  rewriterule1  domain  ^.+$  rewrited.${tag_parts[1]}.${tag_parts[2]}
+</match>
 ```
 
 ## Example
@@ -185,6 +206,17 @@ http://y-ken.hatenablog.com/entry/fluent-plugin-rewrite-tag-filter-v1.2.1
 - 待望の正規表現の否定パターンに対応した fluent-plugin-rewrite-tag-filter v1.3.0 をリリースしました #fluentd  
 http://y-ken.hatenablog.com/entry/fluent-plugin-rewrite-tag-filter-v1.3.0
 
+- 不具合修正版 fluent-plugin-rewrite-tag-filter v1.3.1 をリリースしました #fluentd  
+http://y-ken.hatenablog.com/entry/fluent-plugin-rewrite-tag-filter-v1.3.1
+
+- PostgreSQLのログをfluentdで回収する設定 — still deeper  
+http://chopl.in/blog/2013/06/07/postgresql_csv_log_with_fluentd.html
+
+- S3とFluentdを用いた効率的なログ管理 | SmartNews開発者ブログ  
+http://developer.smartnews.be/blog/2013/09/02/an-effective-log-management-technique-which-uses-fluentd-and-s3/
+
+- fluentd(td-agent) の導入 : Raccoon Tech Blog [株式会社ラクーン 技術戦略部ブログ]  
+http://techblog.raccoon.ne.jp/archives/35031163.html
 
 ## TODO
 

data/fluent-plugin-rewrite-tag-filter.gemspec

@@ -3,7 +3,8 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-rewrite-tag-filter"
-  s.version     = "1.3.1"
+  s.version     = "1.4.0"
+  s.license     = "Apache 2.0"
   s.authors     = ["Kentaro Yoshida"]
   s.email       = ["y.ken.studio@gmail.com"]
   s.homepage    = "https://github.com/y-ken/fluent-plugin-rewrite-tag-filter"

data/lib/fluent/plugin/out_rewrite_tag_filter.rb

@@ -19,6 +19,11 @@ class Fluent::RewriteTagFilterOutput < Fluent::Output
       if regexp.nil? || rewritetag.nil?
         raise Fluent::ConfigError, "failed to parse rewriterules at #{key} #{conf[key]}"
       end
+
+      unless rewritetag.match(/\$\{tag_parts\[\d\.\.\.?\d\]\}/).nil? or rewritetag.match(/__TAG_PARTS\[\d\.\.\.?\d\]__/).nil?
+        raise Fluent::ConfigError, "${tag_parts[n]} and __TAG_PARTS[n]__ placeholder does not support range specify at #{key} #{conf[key]}"
+      end
+
       @rewriterules.push([rewritekey, /#{trim_regex_quote(regexp)}/, get_match_operator(regexp), rewritetag])
       rewriterule_names.push(rewritekey + regexp)
       $log.info "adding rewrite_tag_filter rule: #{key} #{@rewriterules.last}"
@@ -61,7 +66,10 @@ class Fluent::RewriteTagFilterOutput < Fluent::Output
         backreference_table = get_backreference_table($~.captures)
         rewritetag = rewritetag.gsub(/\$\d+/, backreference_table)
       end
-      rewritetag = rewritetag.gsub(/(\${[a-z]+}|__[A-Z]+__)/, placeholder)
+      rewritetag = rewritetag.gsub(/(\${[a-z_]+(\[[0-9]+\])?}|__[A-Z_]+__)/) do
+        $log.warn "rewrite_tag_filter: unknown placeholder found. :placeholder=>#{$1} :tag=>#{tag} :rewritetag=>#{rewritetag}" unless placeholder.include?($1)
+        placeholder[$1]
+      end
       return rewritetag
     end
     return nil
@@ -99,12 +107,20 @@ class Fluent::RewriteTagFilterOutput < Fluent::Output
 
   def get_placeholder(tag)
     tag = tag.sub(@remove_tag_prefix, '') if @remove_tag_prefix
-    return {
+
+    result = {
       '__HOSTNAME__' => @hostname,
       '${hostname}' => @hostname,
       '__TAG__' => tag,
       '${tag}' => tag,
     }
+
+    tag.split('.').each_with_index do |t, idx|
+      result.store("${tag_parts[#{idx}]}", t)
+      result.store("__TAG_PARTS[#{idx}]__", t)
+    end
+
+    return result
   end
 end
 

data/test/plugin/test_out_rewrite_tag_filter.rb

@@ -55,6 +55,14 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
     rewriterule20 domain ^news\.google\.com$ site.GoogleNews
   ]
 
+  # split by tag
+  CONFIG_SPLIT_BY_TAG = %[
+    rewriterule1 user_name ^Lynn Minmay$ vip.${tag_parts[1]}.remember_love
+    rewriterule2 user_name ^Harlock$ ${tag_parts[2]}.${tag_parts[0]}.${tag_parts[1]}
+    rewriterule3 world ^(alice|chaos)$ application.${tag_parts[0]}.$1_server
+    rewriterule4 world ^[a-z]+$ application.${tag_parts[1]}.future_server
+  ]
+
   def create_driver(conf=CONFIG,tag='test')
     Fluent::Test::OutputTestDriver.new(Fluent::RewriteTagFilterOutput, tag).configure(conf)
   end
@@ -69,6 +77,12 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
     assert_raise(Fluent::ConfigError) {
       d = create_driver('rewriterule1 foo foo')
     }
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver('rewriterule1 hoge hoge.${tag_parts[0..2]}.__TAG_PARTS[0..2]__')
+    }
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver('rewriterule1 fuga fuga.${tag_parts[1...2]}.__TAG_PARTS[1...2]__')
+    }
     d = create_driver %[
       rewriterule1 domain ^www.google.com$ site.Google
       rewriterule2 domain ^news.google.com$ site.GoogleNews
@@ -188,5 +202,30 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
     assert_equal 'site.GoogleNews', emits[1][0] # tag
   end
 
+  def test_emit8_split_by_tag
+    d1 = create_driver(CONFIG_SPLIT_BY_TAG, 'game.production.api')
+    d1.run do
+      d1.emit({'user_id' => '10000', 'world' => 'chaos', 'user_name' => 'gamagoori'})
+      d1.emit({'user_id' => '10001', 'world' => 'chaos', 'user_name' => 'sanageyama'})
+      d1.emit({'user_id' => '10002', 'world' => 'nehan', 'user_name' => 'inumuta'})
+      d1.emit({'user_id' => '77777', 'world' => 'space', 'user_name' => 'Lynn Minmay'})
+      d1.emit({'user_id' => '99999', 'world' => 'space', 'user_name' => 'Harlock'})
+    end
+    emits = d1.emits
+    p emits
+    assert_equal 5, emits.length
+    p emits[0]
+    assert_equal 'application.game.chaos_server', emits[0][0]
+    p emits[1]
+    assert_equal 'application.game.chaos_server', emits[1][0]
+    p emits[2]
+    assert_equal 'application.production.future_server', emits[2][0]
+    p emits[3]
+    assert_equal 'vip.production.remember_love', emits[3][0]
+    p emits[4]
+    assert_equal 'api.game.production', emits[4][0]
+  end
+
+
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-rewrite-tag-filter
 version: !ruby/object:Gem::Version
-  version: 1.3.1
+  version: 1.4.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-21 00:00:00.000000000 Z
+date: 2013-12-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -51,6 +51,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -62,7 +63,8 @@ files:
 - test/helper.rb
 - test/plugin/test_out_rewrite_tag_filter.rb
 homepage: https://github.com/y-ken/fluent-plugin-rewrite-tag-filter
-licenses: []
+licenses:
+- Apache 2.0
 post_install_message: 
 rdoc_options: []
 require_paths: