fluent-plugin-rewrite-tag-filter 0.0.8 → 0.0.9

data/README.rdoc

@@ -6,29 +6,30 @@
 
 It has designed to rewrite tags like mod_rewrite.
 Also you can change a tag from apache log by domain, status-code(ex. 500 error), 
-user-agent, and request-uri and so on with regular expression. 
+user-agent, request-uri, regex-backreference and so on with regular expression. 
 
 == Installation
 
-    # native fluentd user
+    # native gem
     gem install fluent-plugin-rewrite-tag-filter
     
-    # td-agent user
+    # td-agent gem
     /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-rewrite-tag-filter
 
 == Configuration
 
-Syntax
+=== Syntax
 
     rewruterule<num:1-200> <attribute> <regex_pattern> <new_tag>
     
-    # Option: Capitalize every matched regex backreference (ex: $1, $2).
+    # Option: Capitalize every matched regex backreference. (ex: $1, $2)
     # It has described on usage section below.
-    capitalize_regex_backreference <default:no>
+    capitalize_regex_backreference <yes/no(default)>
 
-Usage
+=== Usage
+
+It's a sample to exclude some static file log before split tag by domain.
 
-    # It's a sample to exclude some static file log before split tag by domain.
     <source>
       type tail
       path /var/log/httpd/access_log
@@ -38,15 +39,15 @@ Usage
       pos_file /var/log/td-agent/apache_access.pos
     </source>
 
+    # At rewriterule4, "site.$2$1" to be "site.ExampleMail" by capitalize_regex_backreference option.
     <match td.apache.access>
       type rewrite_tag_filter
       capitalize_regex_backreference yes
-      rewriterule1 path \.(gif|jpe?g|png|pdf|zip)$ clear
-      rewriterule2 domain ^maps\.example\.com$ site.ExampleMaps
-      rewriterule3 domain ^news\.example\.com$ site.ExampleNews
-      # "site.$2$1" to be "site.ExampleMail" by capitalize_regex_backreference option.
-      rewriterule4 domain ^(mail)\.(example)\.com$ site.$2$1
-      rewriterule5 domain .+ site.unmatched
+      rewriterule1 path   \.(gif|jpe?g|png|pdf|zip)$  clear
+      rewriterule2 domain ^maps\.example\.com$        site.ExampleMaps
+      rewriterule3 domain ^news\.example\.com$        site.ExampleNews
+      rewriterule4 domain ^(mail)\.(example)\.com$    site.$2$1
+      rewriterule5 domain .+                          site.unmatched
     </match>
 
     <match site.*>
@@ -63,7 +64,7 @@ Usage
       type null
     </match>
 
-Result
+=== Result
 
     $ mongo
     MongoDB shell version: 2.2.0
@@ -75,6 +76,18 @@ Result
     ExampleMail
     unmatched
 
+=== Debug
+
+Logging supported like below.
+
+    $ tailf /var/log/td-agent/td-agent.log
+    2012-09-16 18:10:51 +0900: adding match pattern="td.apache.access" type="rewrite_tag_filter"
+    2012-09-16 18:10:51 +0900: adding rewrite_tag_filter rule: [1, "path", /\.(gif|jpe?g|png|pdf|zip)$/, "clear"]
+    2012-09-16 18:10:51 +0900: adding rewrite_tag_filter rule: [2, "domain", /^maps\.example\.com$/, "site.ExampleMaps"]
+    2012-09-16 18:10:51 +0900: adding rewrite_tag_filter rule: [3, "domain", /^news\.example\.com$/, "site.ExampleNews"]
+    2012-09-16 18:10:51 +0900: adding rewrite_tag_filter rule: [4, "domain", /^(mail)\.(example)\.com$/, "site.$2$1"]
+    2012-09-16 18:10:51 +0900: adding rewrite_tag_filter rule: [5, "domain", /.+/, "site.unmatched"]
+
 == Example
 
 - Example1: how to analyze response_time, response_code and user_agent for each virtual domain websites.
@@ -83,11 +96,14 @@ Result
 - Example2: how to exclude specified patterns before analyze response_time for each virtual domain websites.
   - https://github.com/y-ken/fluent-plugin-rewrite-tag-filter/blob/master/example2.conf
 
-== Review
+== Topic
 
 - 自在にタグを書き換える fluent-plugin-rewrite-tag-filter でログ解析が捗るお話 #fluentd
   - http://d.hatena.ne.jp/yoshi-ken/20120701/1341137269
 
+- Fluentd & TreasureDataで こっそり始めるログ集計 Fluentd Meetup #2 @mikeda
+  - http://www.slideshare.net/baguzy/fluentd-meetup-2-14073930
+
 == TODO
 
 - patches welcome!

data/example.conf

@@ -21,36 +21,37 @@
   type copy
   <store>
     type rewrite_tag_filter
-    rewriterule1 domain ^maps\.google\.com$ site.GoogleMap
-    rewriterule2 domain ^news\.google\.com$ site.GoogleNews
-    rewriterule3 domain ^mail\.google\.com$ site.GoogleMail
+    capitalize_regex_backreference yes
+    rewriterule1 domain ^(maps|news|mail)\.google\.com$ site.Google$1
   </store>
   <store>
     type rewrite_tag_filter
-    rewriterule1 domain ^maps\.google\.com$ sitepath.GoogleMap
+    capitalize_regex_backreference yes
+    rewriterule1 domain ^(maps)\.google\.com$ sitepath.Google$1
   </store>
 </match>
 
 
 # Second level analyzing
-<match sitepath.GoogleMap>
+<match sitepath.GoogleMaps>
   <store>
     type rewrite_tag_filter
-    rewriterule1 path ^/labs site.GoogleMap.Labs
-    rewriterule2 path ^/static/\d+ site.GoogleMap.Static
-    rewriterule3 path ^/help/maps/(streetview|getmaps) site.GoogleMap.Help
+    rewriterule1 path ^/labs site.GoogleMaps.Labs
+    rewriterule2 path ^/static/\d+ site.GoogleMaps.Static
+    rewriterule3 path ^/help/maps/(streetview|getmaps) site.GoogleMaps.Help
   </store>
   <store>
     type rewrite_tag_filter
-    rewriterule1 referer headlines\.yahoo\.co\.jp site.GoogleMap.referer_YahooHeadlines
-    rewriterule2 referer news\.livedoor\.com site.GoogleMap.referer_LivedoorNews
+    rewriterule1 referer headlines\.yahoo\.co\.jp site.GoogleMaps.referer_YahooHeadlines
+    rewriterule2 referer news\.livedoor\.com site.GoogleMaps.referer_LivedoorNews
   </store>
   <store>
     type rewrite_tag_filter
-    rewriterule1 agent Googlebot/ site.GoogleMap.agent_Googlebot
-    rewriterule2 agent Googlebot-Mobile/ site.GoogleMap.agent_GooglebotMobile
-    rewriterule3 agent bingbot site.GoogleMap.agent_Bingbot
-    rewriterule4 agent Baiduspider site.GoogleMap.agent_Baiduspider
+    rewriterule1 agent Googlebot/ site.GoogleMaps.agent_Googlebot
+    rewriterule2 agent " iPhone .+Googlebot-Mobile/" site.GoogleMaps.agent_GooglebotSmartphoneMobile
+    rewriterule3 agent Googlebot-Mobile/ site.GoogleMaps.agent_GooglebotMobile
+    rewriterule4 agent bingbot site.GoogleMaps.agent_Bingbot
+    rewriterule5 agent Baiduspider site.GoogleMaps.agent_Baiduspider
   </store>
 </match>
 

data/example2.conf

@@ -29,10 +29,9 @@
   rewriterule1 path ^/(img|css|js|static|assets)/ clear
   rewriterule2 status ^(?!200)$ clear
   rewriterule3 method ^(?!GET)$ clear
-  rewriterule4 agent Mac\sOS\sX clear
-  rewriterule5 agent (spider|bot|crawler|\+http\:) clear
-  rewriterule6 path ^/(admin|api|backend) site.GoogleMap.backend
-  rewriterule7 path .+ site.GoogleMap.front
+  rewriterule4 agent (spider|bot|crawler|\+http\:) clear
+  rewriterule5 path ^/(admin|api|backend) site.GoogleMap.backend
+  rewriterule6 path .+ site.GoogleMap.front
 </match>
 
 

data/fluent-plugin-rewrite-tag-filter.gemspec

@@ -3,11 +3,11 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-rewrite-tag-filter"
-  s.version     = "0.0.8"
+  s.version     = "0.0.9"
   s.authors     = ["Kentaro Yoshida"]
   s.email       = ["y.ken.studio@gmail.com"]
   s.homepage    = "https://github.com/y-ken/fluent-plugin-rewrite-tag-filter"
-  s.summary     = %q{Fluentd Output filter plugin. It has designed to rewrite tag like mod_rewrite. Also you can change a tag from apache log by domain, status-code(ex. 500 error), user-agent, and request-uri and so on with regular expression.}
+  s.summary     = %q{Fluentd Output filter plugin. It has designed to rewrite tag like mod_rewrite. Also you can change a tag from apache log by domain, status-code(ex. 500 error), user-agent, request-uri, regex-backreference and so on with regular expression.}
   # s.description = %q{Fluentd Output filter plugin. It has designed to rewrite tag like mod_rewrite. Also you can change a tag from apache log by domain, status-code(ex. 500 error), user-agent, and request-uri and so on with regular expression.}
 
   s.files         = `git ls-files`.split("\n")

data/lib/fluent/plugin/out_rewrite_tag_filter.rb

@@ -21,13 +21,13 @@ class Fluent::RewriteTagFilterOutput < Fluent::Output
     end
     (1..PATTERN_MAX_NUM).each do |i|
       next unless conf["rewriterule#{i}"]
-      rewritekey,regexp,rewritetag = conf["rewriterule#{i}"].match(/([^ ]+)\s(.+)\s([^ ]+)/).captures
+      rewritekey,regexp,rewritetag = conf["rewriterule#{i}"].match(/^([^\s]+)\s+["\/]?(.+?)["\/]?\s+([^\s]+)$/).captures
       unless regexp != nil && rewritetag != nil
         raise Fluent::ConfigError, "missing values at rewriterule#{i} " + conf["rewriterule#{i}"].inspect
       end 
       @rewriterules.push([i, rewritekey, Regexp.new(regexp), rewritetag])
       rewriterule_names.push(rewritekey + regexp)
-      $log.info "register rewrite_tag_filter rule: #{@rewriterules.last}"
+      $log.info "adding rewrite_tag_filter rule: #{@rewriterules.last}"
     end
     rewriterule_index_list = conf.keys.select{|s| s =~ /^rewriterule\d$/}.map{|v| (/^rewriterule(\d)$/.match(v))[1].to_i}
     unless rewriterule_index_list.reduce(true){|v,i| v and @rewriterules[i - 1]}
@@ -44,14 +44,14 @@ class Fluent::RewriteTagFilterOutput < Fluent::Output
       @rewriterules.each do |index, rewritekey, regexp, rewritetag|
         rewritevalue = record[rewritekey]
         next if rewritevalue.nil?
-        if (regexp && regexp.match(rewritevalue))
-          rewrite = true
-          tag = rewritetag
-          if rewritetag.include?('$')
-            tag = rewritetag.gsub(/\$\d+/, map_regex_table($~.captures))
-          end
-          break
-        end
+        next unless (regexp && regexp.match(rewritevalue))
+        rewrite = true
+        tag = if rewritetag.include?('$')
+                rewritetag.gsub(/\$\d+/, map_regex_table($~.captures))
+              else
+                rewritetag
+              end
+        break
       end
       Fluent::Engine.emit(tag, time, record) if (rewrite)
     end

data/test/plugin/test_out_rewrite_tag_filter.rb

@@ -12,12 +12,14 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
     rewriterule4 agent (Googlebot|CustomBot)-([a-zA-Z]+) agent.$1-$2
   ]
 
+  # aggresive test
+  # indentation, comment, capitalize_regex_backreference, regex with space aside, slash/quote delimited
   CONFIG2 = %[
     capitalize_regex_backreference yes
-    rewriterule1 domain ^www\.google\.com$ site.Google
-    rewriterule2 domain ^(news)\.(google)\.com$ site.$2$1
-    rewriterule3 agent Mac\sOS\sX agent.MacOSX
-    rewriterule4 agent (Googlebot|CustomBot)-([a-zA-Z]+) agent.$1-$2
+    rewriterule1 domain "^www\.google\.com$"                site.Google # some comment
+    rewriterule2 domain /^(news)\.(google)\.com$/           site.$2$1
+    rewriterule3 agent  / Mac OS X /                        agent.MacOSX
+    rewriterule4 agent  /(Googlebot|CustomBot)-([a-zA-Z]+)/ agent.$1-$2
   ]
 
   def create_driver(conf=CONFIG,tag='test')

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-rewrite-tag-filter
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.9
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-09-15 00:00:00.000000000 Z
+date: 2012-09-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
-  requirement: &20127680 !ruby/object:Gem::Requirement
+  requirement: &10180420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *20127680
+  version_requirements: *10180420
 - !ruby/object:Gem::Dependency
   name: fluentd
-  requirement: &20126860 !ruby/object:Gem::Requirement
+  requirement: &10179520 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,7 +32,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *20126860
+  version_requirements: *10179520
 description: 
 email:
 - y.ken.studio@gmail.com
@@ -76,7 +76,7 @@ signing_key:
 specification_version: 3
 summary: Fluentd Output filter plugin. It has designed to rewrite tag like mod_rewrite.
   Also you can change a tag from apache log by domain, status-code(ex. 500 error),
-  user-agent, and request-uri and so on with regular expression.
+  user-agent, request-uri, regex-backreference and so on with regular expression.
 test_files:
 - test/helper.rb
 - test/plugin/test_out_rewrite_tag_filter.rb