fluent-plugin-rewrite-tag-filter 0.0.4 → 0.0.5

data/README.rdoc

@@ -4,7 +4,9 @@
 
 === RewriteTagFilterOutput
 
-Modify Tags with data matches any of specified regexp patterns in specified attribute.
+It has designed to rewrite tags like mod_rewrite.
+Also you can change a tag from apache log by domain, status-code(ex. 500 error), 
+user-agent, and request-uri and so on with regular expression. 
 
 == Installation
 
@@ -22,7 +24,7 @@ Format
 
 Usage
 
-    # It's a sample to exclude some patterns before split tag by domain.
+    # It's a sample to exclude some static file log before split tag by domain.
     <source>
       type tail
       path /var/log/httpd/access_log
@@ -38,6 +40,7 @@ Usage
       rewriterule2 domain ^maps\.google\.com$ site.GoogleMap
       rewriterule3 domain ^news\.google\.com$ site.GoogleNews
       rewriterule4 domain ^mail\.google\.com$ site.GoogleMail
+      rewriterule5 domain .+ site.unmatched
     </match>
 
     <match site.*>
@@ -46,7 +49,6 @@ Usage
       database apache_access
       remove_tag_prefix site
       tag_mapped
-      collection misc
       capped
       capped_size 100m
     </match>
@@ -65,6 +67,7 @@ Result
     GoogleMap
     GoogleNews
     GoogleMail
+    unmatched
 
 == Example
 
@@ -74,12 +77,17 @@ Result
 - Example2: how to exclude specified patterns before analyze response_time for each virtual domain websites.
   - https://github.com/y-ken/fluent-plugin-rewrite-tag-filter/blob/master/example2.conf
 
+== Review
+
+- 自在にタグを書き換える fluent-plugin-rewrite-tag-filter でログ解析が捗るお話 #fluentd
+  - http://d.hatena.ne.jp/yoshi-ken/20120701/1341137269
+
 == TODO
 
 - patches welcome!
 
 == Copyright
 
-Copyright:: Copyright (c) 2012- Kentaro Yoshida
+Copyright:: Copyright (c) 2012- Kentaro Yoshida (@yoshi_ken)
 License::   Apache License, Version 2.0
 

data/example2.conf

@@ -29,9 +29,10 @@
   rewriterule1 path ^/(img|css|js|static|assets)/ clear
   rewriterule2 status ^(?!200)$ clear
   rewriterule3 method ^(?!GET)$ clear
-  rewriterule4 agent (spider|bot|crawler|\+http\:) clear
-  rewriterule5 path ^/(admin|api|backend) site.GoogleMap.backend
-  rewriterule6 path .+ site.GoogleMap.front
+  rewriterule4 agent "Mac OS X" clear
+  rewriterule5 agent (spider|bot|crawler|\+http\:) clear
+  rewriterule6 path ^/(admin|api|backend) site.GoogleMap.backend
+  rewriterule7 path .+ site.GoogleMap.front
 </match>
 
 

data/fluent-plugin-rewrite-tag-filter.gemspec

@@ -3,8 +3,8 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-rewrite-tag-filter"
-  s.version     = "0.0.4"
-  s.authors     = ["Kentaro Yoshida"]
+  s.version     = "0.0.5"
+  s.authors     = ["Kentaro YOSHIDA"]
   s.email       = ["y.ken.studio@gmail.com"]
   s.homepage    = "https://github.com/y-ken/fluent-plugin-rewrite-tag-filter"
   s.summary     = %q{Fluentd Output filter plugin to rewrite tags that matches specified attribute.}

data/lib/fluent/plugin/out_rewrite_tag_filter.rb

@@ -20,7 +20,7 @@ class Fluent::RewriteTagFilterOutput < Fluent::Output
     end
     (1..PATTERN_MAX_NUM).each do |i|
       next unless conf["rewriterule#{i}"]
-      rewritekey,regexp,rewritetag = conf["rewriterule#{i}"].split(' ', 3)
+      rewritekey,regexp,rewritetag = conf["rewriterule#{i}"].split(/^(\S+) "?([\S\s]+?)"? (\S+)$/).reject(&:empty?)
       unless regexp != nil && rewritetag != nil
         raise Fluent::ConfigError, "missing values at " + "rewriterule#{i} " + conf["rewriterule#{i}"].inspect
       end 

data/test/plugin/test_out_rewrite_tag_filter.rb

@@ -8,6 +8,7 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
   CONFIG = %[
     rewriterule1 domain ^www\.google\.com$ site.Google
     rewriterule2 domain ^news\.google\.com$ site.GoogleNews
+    rewriterule3 agent "Mac OS X" agent.MacOSX
   ]
 
   def create_driver(conf=CONFIG,tag='test')
@@ -31,14 +32,16 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
     d1 = create_driver(CONFIG, 'input.access')
     time = Time.parse("2012-01-02 13:14:15").to_i
     d1.run do
-      d1.emit({'domain' => 'www.google.com', 'agent' => 'Googlebot', 'response_time' => 1000000})
-      d1.emit({'domain' => 'news.google.com', 'agent' => 'Googlebot-Mobile', 'response_time' => 900000})
+      d1.emit({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
+      d1.emit({'domain' => 'news.google.com', 'path' => '/', 'agent' => 'Googlebot-Mobile', 'response_time' => 900000})
+      d1.emit({'domain' => 'map.google.com', 'path' => '/', 'agent' => 'Macintosh; Intel Mac OS X 10_7_4', 'response_time' => 900000})
     end
     emits = d1.emits
-    assert_equal 2, emits.length
+    assert_equal 3, emits.length
     assert_equal 'site.Google', emits[0][0] # tag
     assert_equal 'site.GoogleNews', emits[1][0] # tag
     assert_equal 'news.google.com', emits[1][2]['domain']
+    assert_equal 'agent.MacOSX', emits[2][0] #tag
   end
 end
 

metadata

@@ -1,19 +1,19 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-rewrite-tag-filter
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
   prerelease: 
 platform: ruby
 authors:
-- Kentaro Yoshida
+- Kentaro YOSHIDA
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-07-04 00:00:00.000000000 Z
+date: 2012-07-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
-  requirement: &12250760 !ruby/object:Gem::Requirement
+  requirement: &21462300 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *12250760
+  version_requirements: *21462300
 - !ruby/object:Gem::Dependency
   name: fluentd
-  requirement: &12249400 !ruby/object:Gem::Requirement
+  requirement: &21461000 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,7 +32,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *12249400
+  version_requirements: *21461000
 description: Rewrite Tags with data matches any of specified regexp patterns in specified
   attribute.
 email: