fluent-plugin-rewrite-tag-filter 0.0.7 → 0.0.8

data/README.rdoc

@@ -14,13 +14,17 @@ user-agent, and request-uri and so on with regular expression.
     gem install fluent-plugin-rewrite-tag-filter
     
     # td-agent user
-    /usr/lib64/fluent/ruby/bin/gem install fluent-plugin-rewrite-tag-filter
+    /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-rewrite-tag-filter
 
 == Configuration
 
-Format
+Syntax
 
     rewruterule<num:1-200> <attribute> <regex_pattern> <new_tag>
+    
+    # Option: Capitalize every matched regex backreference (ex: $1, $2).
+    # It has described on usage section below.
+    capitalize_regex_backreference <default:no>
 
 Usage
 
@@ -36,10 +40,12 @@ Usage
 
     <match td.apache.access>
       type rewrite_tag_filter
+      capitalize_regex_backreference yes
       rewriterule1 path \.(gif|jpe?g|png|pdf|zip)$ clear
-      rewriterule2 domain ^maps\.google\.com$ site.GoogleMap
-      rewriterule3 domain ^news\.google\.com$ site.GoogleNews
-      rewriterule4 domain ^mail\.google\.com$ site.GoogleMail
+      rewriterule2 domain ^maps\.example\.com$ site.ExampleMaps
+      rewriterule3 domain ^news\.example\.com$ site.ExampleNews
+      # "site.$2$1" to be "site.ExampleMail" by capitalize_regex_backreference option.
+      rewriterule4 domain ^(mail)\.(example)\.com$ site.$2$1
       rewriterule5 domain .+ site.unmatched
     </match>
 
@@ -60,13 +66,13 @@ Usage
 Result
 
     $ mongo
-    MongoDB shell version: 2.0.6
+    MongoDB shell version: 2.2.0
     > use apache_access
     switched to db apache_access
     > show collections
-    GoogleMap
-    GoogleNews
-    GoogleMail
+    ExampleMaps
+    ExampleNews
+    ExampleMail
     unmatched
 
 == Example

data/example2.conf

@@ -29,7 +29,7 @@
   rewriterule1 path ^/(img|css|js|static|assets)/ clear
   rewriterule2 status ^(?!200)$ clear
   rewriterule3 method ^(?!GET)$ clear
-  rewriterule4 agent "Mac OS X" clear
+  rewriterule4 agent Mac\sOS\sX clear
   rewriterule5 agent (spider|bot|crawler|\+http\:) clear
   rewriterule6 path ^/(admin|api|backend) site.GoogleMap.backend
   rewriterule7 path .+ site.GoogleMap.front

data/fluent-plugin-rewrite-tag-filter.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-rewrite-tag-filter"
-  s.version     = "0.0.7"
+  s.version     = "0.0.8"
   s.authors     = ["Kentaro Yoshida"]
   s.email       = ["y.ken.studio@gmail.com"]
   s.homepage    = "https://github.com/y-ken/fluent-plugin-rewrite-tag-filter"

data/lib/fluent/plugin/out_rewrite_tag_filter.rb

@@ -7,6 +7,7 @@ class Fluent::RewriteTagFilterOutput < Fluent::Output
   (2..PATTERN_MAX_NUM).each do |i|
     config_param ('rewriterule' + i.to_s).to_sym, :string, :default => nil # NAME REGEXP
   end
+  config_param :capitalize_regex_backreference, :bool, :default => false
 
   def configure(conf)
     super
@@ -16,23 +17,24 @@ class Fluent::RewriteTagFilterOutput < Fluent::Output
 
     invalids = conf.keys.select{|k| k =~ /^rewriterule(\d+)$/}.select{|arg| arg =~ /^rewriterule(\d+)/ and not (1..PATTERN_MAX_NUM).include?($1.to_i)}
     if invalids.size > 0
-      $log.warn "invalid number rewriterules (valid rewriterule number:1-{PATTERN_MAX_NUM}):" + invalids.join(",")
+      $log.warn "invalid number rewriterules (valid rewriterule number:1-{PATTERN_MAX_NUM}): #{invalids.join(",")}"
     end
     (1..PATTERN_MAX_NUM).each do |i|
       next unless conf["rewriterule#{i}"]
-      rewritekey,regexp,rewritetag = conf["rewriterule#{i}"].split(/^(\S+) "?([\S\s]+?)"? (\S+)$/).reject(&:empty?)
+      rewritekey,regexp,rewritetag = conf["rewriterule#{i}"].match(/([^ ]+)\s(.+)\s([^ ]+)/).captures
       unless regexp != nil && rewritetag != nil
-        raise Fluent::ConfigError, "missing values at " + "rewriterule#{i} " + conf["rewriterule#{i}"].inspect
+        raise Fluent::ConfigError, "missing values at rewriterule#{i} " + conf["rewriterule#{i}"].inspect
       end 
       @rewriterules.push([i, rewritekey, Regexp.new(regexp), rewritetag])
       rewriterule_names.push(rewritekey + regexp)
+      $log.info "register rewrite_tag_filter rule: #{@rewriterules.last}"
     end
     rewriterule_index_list = conf.keys.select{|s| s =~ /^rewriterule\d$/}.map{|v| (/^rewriterule(\d)$/.match(v))[1].to_i}
     unless rewriterule_index_list.reduce(true){|v,i| v and @rewriterules[i - 1]}
-      raise Fluent::ConfigError, "jump of rewriterule index found" + @rewriterules.inspect
+      raise Fluent::ConfigError, "jump of rewriterule index found #{@rewriterules.inspect}"
     end
     unless @rewriterules.length == rewriterule_names.uniq.length
-      raise Fluent::ConfigError, "duplicated rewriterules found" + @rewriterules.inspect
+      raise Fluent::ConfigError, "duplicated rewriterules found #{@rewriterules.inspect}"
     end
   end
 
@@ -45,6 +47,9 @@ class Fluent::RewriteTagFilterOutput < Fluent::Output
         if (regexp && regexp.match(rewritevalue))
           rewrite = true
           tag = rewritetag
+          if rewritetag.include?('$')
+            tag = rewritetag.gsub(/\$\d+/, map_regex_table($~.captures))
+          end
           break
         end
       end
@@ -53,5 +58,15 @@ class Fluent::RewriteTagFilterOutput < Fluent::Output
 
     chain.next
   end
+
+  def map_regex_table(elements)
+    hash_table = Hash.new
+    index = 1
+    elements.each do |value|
+      hash_table["$#{index}"] = @capitalize_regex_backreference ? value.capitalize : value
+      index += 1
+    end
+    return hash_table
+  end
 end
 

data/test/plugin/test_out_rewrite_tag_filter.rb

@@ -4,11 +4,20 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
   end
-  
+
   CONFIG = %[
     rewriterule1 domain ^www\.google\.com$ site.Google
     rewriterule2 domain ^news\.google\.com$ site.GoogleNews
-    rewriterule3 agent "Mac OS X" agent.MacOSX
+    rewriterule3 agent Mac\sOS\sX agent.MacOSX
+    rewriterule4 agent (Googlebot|CustomBot)-([a-zA-Z]+) agent.$1-$2
+  ]
+
+  CONFIG2 = %[
+    capitalize_regex_backreference yes
+    rewriterule1 domain ^www\.google\.com$ site.Google
+    rewriterule2 domain ^(news)\.(google)\.com$ site.$2$1
+    rewriterule3 agent Mac\sOS\sX agent.MacOSX
+    rewriterule4 agent (Googlebot|CustomBot)-([a-zA-Z]+) agent.$1-$2
   ]
 
   def create_driver(conf=CONFIG,tag='test')
@@ -35,13 +44,41 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
       d1.emit({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
       d1.emit({'domain' => 'news.google.com', 'path' => '/', 'agent' => 'Googlebot-Mobile', 'response_time' => 900000})
       d1.emit({'domain' => 'map.google.com', 'path' => '/', 'agent' => 'Macintosh; Intel Mac OS X 10_7_4', 'response_time' => 900000})
+      d1.emit({'domain' => 'labs.google.com', 'path' => '/', 'agent' => 'Mozilla/5.0 Googlebot-FooBar/2.1', 'response_time' => 900000})
+    end
+    emits = d1.emits
+    assert_equal 4, emits.length
+    p emits[0]
+    assert_equal 'site.Google', emits[0][0] # tag
+    p emits[1]
+    assert_equal 'site.GoogleNews', emits[1][0] # tag
+    assert_equal 'news.google.com', emits[1][2]['domain']
+    p emits[2]
+    assert_equal 'agent.MacOSX', emits[2][0] #tag
+    p emits[3]
+    assert_equal 'agent.Googlebot-FooBar', emits[3][0] #tag
+  end
+
+  def test_emit2
+    d1 = create_driver(CONFIG2, 'input.access')
+    time = Time.parse("2012-01-02 13:14:15").to_i
+    d1.run do
+      d1.emit({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
+      d1.emit({'domain' => 'news.google.com', 'path' => '/', 'agent' => 'Googlebot-Mobile', 'response_time' => 900000})
+      d1.emit({'domain' => 'map.google.com', 'path' => '/', 'agent' => 'Macintosh; Intel Mac OS X 10_7_4', 'response_time' => 900000})
+      d1.emit({'domain' => 'labs.google.com', 'path' => '/', 'agent' => 'Mozilla/5.0 Googlebot-FooBar/2.1', 'response_time' => 900000})
     end
     emits = d1.emits
-    assert_equal 3, emits.length
+    assert_equal 4, emits.length
+    p emits[0]
     assert_equal 'site.Google', emits[0][0] # tag
+    p emits[1]
     assert_equal 'site.GoogleNews', emits[1][0] # tag
     assert_equal 'news.google.com', emits[1][2]['domain']
+    p emits[2]
     assert_equal 'agent.MacOSX', emits[2][0] #tag
+    p emits[3]
+    assert_equal 'agent.Googlebot-Foobar', emits[3][0] #tag
   end
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-rewrite-tag-filter
 version: !ruby/object:Gem::Version
-  version: 0.0.7
+  version: 0.0.8
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-07-28 00:00:00.000000000 Z
+date: 2012-09-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
-  requirement: &20535320 !ruby/object:Gem::Requirement
+  requirement: &20127680 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *20535320
+  version_requirements: *20127680
 - !ruby/object:Gem::Dependency
   name: fluentd
-  requirement: &20534460 !ruby/object:Gem::Requirement
+  requirement: &20126860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,7 +32,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *20534460
+  version_requirements: *20126860
 description: 
 email:
 - y.ken.studio@gmail.com