fluent-plugin-rewrite-tag-filter 0.0.2 → 0.0.3

data/README.rdoc

@@ -18,7 +18,7 @@ Modify Tags with data matches any of specified regexp patterns in specified attr
 
 Format
 
-    rewruterule<num:1-200> <attribute> <regex_pattern> <rewrite_tag>
+    rewruterule<num:1-200> <attribute> <regex_pattern> <new_tag>
 
 Usage
 
@@ -34,15 +34,20 @@ Usage
       host 10.10.0.36
       database apache_access
       remove_tag_prefix site
+      tag_mapped
+      collection misc
       capped
       capped_size 100m
     </match>
 
-Example
+== Example
 
 - Example1: how to analyze response_time, response_code and user_agent for each virtual domain websites.
   - https://github.com/y-ken/fluent-plugin-rewrite-tag-filter/blob/master/example.conf
 
+- Example2: how to exclude specified patterns before analyze response_time for each virtual domain websites.
+  - https://github.com/y-ken/fluent-plugin-rewrite-tag-filter/blob/master/example2.conf
+
 == TODO
 
 - patches welcome!
@@ -51,3 +56,4 @@ Example
 
 Copyright:: Copyright (c) 2012- Kentaro Yoshida
 License::   Apache License, Version 2.0
+

data/example.conf

@@ -1,4 +1,10 @@
-# it is a sample how to analyze response_time, response_code and user_agent for each virtual domain websites.
+# It is a sample how to analyze response_time, response_code and user_agent for each virtual domain websites.
+
+# appearing plugins:
+#  rewrite_tag_filter: http://rubygems.org/gems/fluent-plugin-rewrite-tag-filter
+#  forest: http://rubygems.org/gems/fluent-plugin-forest
+#  datacounter: http://rubygems.org/gems/fluent-plugin-datacounter
+#  growthforecast: http://rubygems.org/gems/fluent-plugin-growthforecast
 
 <source>
   type tail
@@ -9,13 +15,47 @@
   pos_file /var/log/td-agent/apache_access.pos
 </source>
 
+
+# Extract specified virtual domain
 <match td.apache.access>
-  type rewrite_tag_filter
-  rewriterule1 domain ^maps\.google\.com$ site.GoogleMap
-  rewriterule2 domain ^news\.google\.com$ site.GoogleNews
-  rewriterule3 domain ^mail\.google\.com$ site.GoogleMail
+  type copy
+  <store>
+    type rewrite_tag_filter
+    rewriterule1 domain ^maps\.google\.com$ site.GoogleMap
+    rewriterule2 domain ^news\.google\.com$ site.GoogleNews
+    rewriterule3 domain ^mail\.google\.com$ site.GoogleMail
+  </store>
+  <store>
+    type rewrite_tag_filter
+    rewriterule1 domain ^maps\.google\.com$ sitepath.GoogleMap
+  </store>
 </match>
 
+
+# Second level analyzing
+<match sitepath.GoogleMap>
+  <store>
+    type rewrite_tag_filter
+    rewriterule1 path ^/labs site.GoogleMap.Labs
+    rewriterule2 path ^/static/\d+ site.GoogleMap.Static
+    rewriterule3 path ^/help/maps/(streetview|getmaps) site.GoogleMap.Help
+  </store>
+  <store>
+    type rewrite_tag_filter
+    rewriterule1 referer headlines\.yahoo\.co\.jp site.GoogleMap.referer_YahooHeadlines
+    rewriterule2 referer news\.livedoor\.com site.GoogleMap.referer_LivedoorNews
+  </store>
+  <store>
+    type rewrite_tag_filter
+    rewriterule1 agent Googlebot/ site.GoogleMap.agent_Googlebot
+    rewriterule2 agent Googlebot-Mobile/ site.GoogleMap.agent_GooglebotMobile
+    rewriterule3 agent bingbot site.GoogleMap.agent_Bingbot
+    rewriterule4 agent Baiduspider site.GoogleMap.agent_Baiduspider
+  </store>
+</match>
+
+
+# Summarize
 <match site.**>
   type copy
   <store>
@@ -75,27 +115,29 @@
   </store>
 </match>
 
-<match gf.responsecode.**>
+
+# Graph
+<match gf.responsetime.**>
   type forest
   subtype growthforecast
-  remove_prefix gf.responsecode.site
+  remove_prefix gf.responsetime.site
   <template>
-    gfapi_url http://10.10.0.55:5125/api/
+    gfapi_url http://localhost:5125/api/
     service   __TAG__
-    section   response_code
-    name_keys 301_count,302_count,3xx_count,403_count,404_count,410_count,4xx_count,500_count
+    section   response_time
+    name_keys 0-100msec_percentage,100-300msec_percentage,300-600msec_percentage,600msec-1sec_percentage,1-2sec_percentage,2-10sec_percentage,10sec_over_percentage
   </template>
 </match>
 
-<match gf.responsetime.**>
+<match gf.responsecode.**>
   type forest
   subtype growthforecast
-  remove_prefix gf.responsetime.site
+  remove_prefix gf.responsecode.site
   <template>
-    gfapi_url http://10.10.0.55:5125/api/
+    gfapi_url http://localhost:5125/api/
     service   __TAG__
-    section   response_time
-    name_keys 0-100msec_percentage,100-300msec_percentage,300-600msec_percentage,600msec-1sec_percentage,1-2sec_percentage,2-10sec_percentage,10sec_over_percentage
+    section   response_code
+    name_keys 301_count,302_count,3xx_count,403_count,404_count,410_count,4xx_count,500_count
   </template>
 </match>
 
@@ -104,7 +146,7 @@
   subtype growthforecast
   remove_prefix gf.useragent.site
   <template>
-    gfapi_url http://10.10.0.55:5125/api/
+    gfapi_url http://localhost:5125/api/
     service   __TAG__
     section   useragent
     name_keys pc_count,mobile_count,smartphone_count,robot_count,api_count

data/example2.conf

@@ -0,0 +1,79 @@
+# It is a sample how to exclude specified patterns before analyze response_time for each virtual domain websites.
+
+# appearing plugins:
+#  rewrite_tag_filter: http://rubygems.org/gems/fluent-plugin-rewrite-tag-filter
+#  forest: http://rubygems.org/gems/fluent-plugin-forest
+#  datacounter: http://rubygems.org/gems/fluent-plugin-datacounter
+#  growthforecast: http://rubygems.org/gems/fluent-plugin-growthforecast
+
+<source>
+  type tail
+  path /var/log/httpd/access_log
+  format /^(?<domain>[^ ]*) (?<host>[^ ]*) [^ ]* (?<user>[^ ]*) \[(?<time>[^\]]*)\] "(?<method>\S+)(?: +(?<path>[^ ]*) +\S*)?" (?<status>[^ ]*) (?<size>[^ ]*)(?: "(?<referer>[^\"]*)" "(?<agent>[^\"]*)" (?<response_time>[^ ]*))?$/
+  time_format %d/%b/%Y:%H:%M:%S %z
+  tag td.apache.access
+  pos_file /var/log/td-agent/apache_access.pos
+</source>
+
+
+# Extract specified virtual domain
+<match td.apache.access>
+  type rewrite_tag_filter
+  rewriterule1 domain ^maps\.google\.com$ filter.GoogleMap
+</match>
+
+
+# Filtering
+<match filter.GoogleMap>
+  <store>
+    type rewrite_tag_filter
+    rewriterule1 path ^/(img|css|js|static|assets)/ clear
+    rewriterule2 status ^(?!200)$ clear
+    rewriterule3 method ^(?!GET)$ clear
+    rewriterule4 agent (spider|bot|crawler|\+http\:) clear
+    rewriterule5 path ^/(admin|api|backend) site.GoogleMap.backend
+    rewriterule6 path .+ site.GoogleMap.front
+  </store>
+</match>
+
+
+# Summarize
+<match site.**>
+  type forest
+  subtype datacounter
+  <template>
+    unit minute
+    count_key response_time
+    outcast_unmatched false
+    aggregate all
+    tag gf.responsetime.__TAG__
+    pattern1 0-100msec ^\d{1,5}$
+    pattern2 100-300msec ^[1-2]\d{5}$
+    pattern3 300-600msec ^[3-5]\d{5}$
+    pattern4 600msec-1sec ^[6-9]\d{5}$
+    pattern5 1-2sec ^1\d{6}$
+    pattern6 2-10sec ^[2-9]\d{6}$
+    pattern7 10sec_over ^\d{8,}$
+  </template>
+</match>
+
+
+# Graph
+<match gf.responsetime.**>
+  type forest
+  subtype growthforecast
+  remove_prefix gf.responsetime.site
+  <template>
+    gfapi_url http://localhost:5125/api/
+    service   __TAG__
+    section   response_time
+    name_keys 0-100msec_percentage,100-300msec_percentage,300-600msec_percentage,600msec-1sec_percentage,1-2sec_percentage,2-10sec_percentage,10sec_over_percentage
+  </template>
+</match>
+
+
+# Clear tag
+<match clear>
+  type null
+</match>
+

data/fluent-plugin-rewrite-tag-filter.gemspec

@@ -3,12 +3,12 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-rewrite-tag-filter"
-  s.version     = "0.0.2"
+  s.version     = "0.0.3"
   s.authors     = ["Kentaro Yoshida"]
   s.email       = ["y.ken.studio@gmail.com"]
   s.homepage    = "https://github.com/y-ken/fluent-plugin-rewrite-tag-filter"
-  s.summary     = %q{Output filter plugin to rewrite tags that matches specified attribute}
-  s.description = %q{Modify Tags with data matches any of specified regexp patterns in specified attribute.}
+  s.summary     = %q{Fluentd Output filter plugin to rewrite tags that matches specified attribute.}
+  s.description = %q{Rewrite Tags with data matches any of specified regexp patterns in specified attribute.}
 
   s.rubyforge_project = "fluent-plugin-rewrite-tag-filter"
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-rewrite-tag-filter
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-06-30 00:00:00.000000000 Z
+date: 2012-07-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
-  requirement: &11789920 !ruby/object:Gem::Requirement
+  requirement: &17995380 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *11789920
+  version_requirements: *17995380
 - !ruby/object:Gem::Dependency
   name: fluentd
-  requirement: &11788560 !ruby/object:Gem::Requirement
+  requirement: &17994080 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,8 +32,8 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *11788560
-description: Modify Tags with data matches any of specified regexp patterns in specified
+  version_requirements: *17994080
+description: Rewrite Tags with data matches any of specified regexp patterns in specified
   attribute.
 email:
 - y.ken.studio@gmail.com
@@ -47,6 +47,7 @@ files:
 - README.rdoc
 - Rakefile
 - example.conf
+- example2.conf
 - fluent-plugin-rewrite-tag-filter.gemspec
 - lib/fluent/plugin/out_rewrite_tag_filter.rb
 - test/helper.rb
@@ -74,7 +75,7 @@ rubyforge_project: fluent-plugin-rewrite-tag-filter
 rubygems_version: 1.8.11
 signing_key: 
 specification_version: 3
-summary: Output filter plugin to rewrite tags that matches specified attribute
+summary: Fluentd Output filter plugin to rewrite tags that matches specified attribute.
 test_files:
 - test/helper.rb
 - test/plugin/test_out_rewrite_tag_filter.rb