fluent-plugin-rewrite-tag-filter 1.4.0 → 1.4.1

data/.gitignore

@@ -2,3 +2,4 @@
 .bundle
 Gemfile.lock
 pkg/*
+vendor/*

data/.travis.yml

@@ -1,6 +1,7 @@
 language: ruby
 
 rvm:
+  - 2.1.0
   - 2.0.0
   - 1.9.3
 

data/README.md

@@ -2,16 +2,14 @@
 
 ## Overview
 
-### RewriteTagFilterOutput
-
-Fluentd Output filter plugin. It has designed to rewrite tag like mod_rewrite.  
-Re-emmit a record with rewrited tag when a value matches/unmatches with the regular expression.  
-Also you can change a tag from apache log by domain, status-code(ex. 500 error),  
+Rewrite Tag Filter for [Fluentd](http://fluentd.org). It is designed to rewrite tags like mod_rewrite.  
+Re-emit the record with rewrited tag when a value matches/unmatches with a regular expression.  
+Also you can change a tag from Apache log by domain, status code (ex. 500 error),  
 user-agent, request-uri, regex-backreference and so on with regular expression.
 
 ## Installation
 
-install with gem or fluent-gem command as:
+Install with gem or fluent-gem command as:
 
 ```
 # for fluentd

data/fluent-plugin-rewrite-tag-filter.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-rewrite-tag-filter"
-  s.version     = "1.4.0"
+  s.version     = "1.4.1"
   s.license     = "Apache 2.0"
   s.authors     = ["Kentaro Yoshida"]
   s.email       = ["y.ken.studio@gmail.com"]
@@ -17,4 +17,5 @@ Gem::Specification.new do |s|
 
   s.add_development_dependency "rake"
   s.add_runtime_dependency "fluentd"
+  s.add_runtime_dependency "string-scrub"
 end

data/lib/fluent/plugin/out_rewrite_tag_filter.rb

@@ -7,6 +7,11 @@ class Fluent::RewriteTagFilterOutput < Fluent::Output
 
   MATCH_OPERATOR_EXCLUDE = '!'
 
+  def initialize
+    super
+    require 'string/scrub'
+  end
+
   def configure(conf)
     super
 
@@ -57,13 +62,13 @@ class Fluent::RewriteTagFilterOutput < Fluent::Output
     @rewriterules.each do |rewritekey, regexp, match_operator, rewritetag|
       rewritevalue = record[rewritekey].to_s
       next if rewritevalue.empty? && match_operator != MATCH_OPERATOR_EXCLUDE
-      matched = regexp && regexp.match(rewritevalue)
+      last_match = regexp_last_match(regexp, rewritevalue)
       case match_operator
       when MATCH_OPERATOR_EXCLUDE
-        next if matched
+        next if last_match
       else
-        next if !matched
-        backreference_table = get_backreference_table($~.captures)
+        next if !last_match
+        backreference_table = get_backreference_table(last_match.captures)
         rewritetag = rewritetag.gsub(/\$\d+/, backreference_table)
       end
       rewritetag = rewritetag.gsub(/(\${[a-z_]+(\[[0-9]+\])?}|__[A-Z_]+__)/) do
@@ -75,6 +80,18 @@ class Fluent::RewriteTagFilterOutput < Fluent::Output
     return nil
   end
 
+  def regexp_last_match(regexp, rewritevalue)
+    begin
+      return if regexp.nil?
+      regexp.match(rewritevalue)
+      return $~
+    rescue ArgumentError => e
+      raise e unless e.message.index('invalid byte sequence in') == 0
+      regexp.match(rewritevalue.scrub('?'))
+      return $~
+    end
+  end
+
   def parse_rewriterule(rule)
     if rule.match(/^([^\s]+)\s+(.+?)\s+([^\s]+)$/)
       return $~.captures

data/test/plugin/test_out_rewrite_tag_filter.rb

@@ -63,6 +63,11 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
     rewriterule4 world ^[a-z]+$ application.${tag_parts[1]}.future_server
   ]
 
+  # test for invalid byte sequence in UTF-8 error
+  CONFIG_INVALID_BYTE = %[
+    rewriterule1 client_name (.+) app.$1
+  ]
+
   def create_driver(conf=CONFIG,tag='test')
     Fluent::Test::OutputTestDriver.new(Fluent::RewriteTagFilterOutput, tag).configure(conf)
   end
@@ -87,7 +92,6 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
       rewriterule1 domain ^www.google.com$ site.Google
       rewriterule2 domain ^news.google.com$ site.GoogleNews
     ]
-    puts d.instance.inspect
     assert_equal 'domain ^www.google.com$ site.Google', d.instance.config['rewriterule1']
     assert_equal 'domain ^news.google.com$ site.GoogleNews', d.instance.config['rewriterule2']
   end
@@ -104,16 +108,11 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
     end
     emits = d1.emits
     assert_equal 5, emits.length
-    p emits[0]
     assert_equal 'site.Google', emits[0][0] # tag
-    p emits[1]
     assert_equal 'site.GoogleNews', emits[1][0] # tag
     assert_equal 'news.google.com', emits[1][2]['domain']
-    p emits[2]
     assert_equal 'agent.MacOSX', emits[2][0] #tag
-    p emits[3]
     assert_equal 'agent.Googlebot-FooBar', emits[3][0] #tag
-    p emits[4]
     assert_equal 'site.input.access.tagtest', emits[4][0] #tag
   end
 
@@ -127,14 +126,10 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
     end
     emits = d1.emits
     assert_equal 4, emits.length
-    p emits[0]
     assert_equal 'site.Google', emits[0][0] # tag
-    p emits[1]
     assert_equal 'site.GoogleNews', emits[1][0] # tag
     assert_equal 'news.google.com', emits[1][2]['domain']
-    p emits[2]
     assert_equal 'agent.MacOSX', emits[2][0] #tag
-    p emits[3]
     assert_equal 'agent.Googlebot-Foobar', emits[3][0] #tag
   end
 
@@ -145,7 +140,6 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
     end
     emits = d1.emits
     assert_equal 1, emits.length
-    p emits[0]
     assert_equal 'access', emits[0][0] # tag
   end
 
@@ -156,7 +150,6 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
     end
     emits = d1.emits
     assert_equal 1, emits.length
-    p emits[0]
     assert_equal 'access', emits[0][0] # tag
   end
 
@@ -167,7 +160,6 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
     end
     emits = d1.emits
     assert_equal 1, emits.length
-    p emits[0]
     assert_equal `hostname -s`.chomp, emits[0][0] # tag
   end
 
@@ -180,11 +172,8 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
     end
     emits = d1.emits
     assert_equal 3, emits.length
-    p emits[0]
     assert_equal 'start_with_www', emits[0][0] # tag
-    p emits[1]
     assert_equal 'not_start_with_www', emits[1][0] # tag
-    p emits[2]
     assert_equal 'not_start_with_www', emits[2][0] # tag
   end
 
@@ -196,9 +185,7 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
     end
     emits = d1.emits
     assert_equal 2, emits.length
-    p emits[0]
     assert_equal 'site.Google', emits[0][0] # tag
-    p emits[1]
     assert_equal 'site.GoogleNews', emits[1][0] # tag
   end
 
@@ -212,20 +199,33 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
       d1.emit({'user_id' => '99999', 'world' => 'space', 'user_name' => 'Harlock'})
     end
     emits = d1.emits
-    p emits
     assert_equal 5, emits.length
-    p emits[0]
     assert_equal 'application.game.chaos_server', emits[0][0]
-    p emits[1]
     assert_equal 'application.game.chaos_server', emits[1][0]
-    p emits[2]
     assert_equal 'application.production.future_server', emits[2][0]
-    p emits[3]
     assert_equal 'vip.production.remember_love', emits[3][0]
-    p emits[4]
     assert_equal 'api.game.production', emits[4][0]
   end
 
+  def test_emit9_invalid_byte
+    invalid_utf8 = "\xff".force_encoding('UTF-8')
+    d1 = create_driver(CONFIG_INVALID_BYTE, 'input.activity')
+    d1.run do
+      d1.emit({'client_name' => invalid_utf8})
+    end
+    emits = d1.emits
+    assert_equal 1, emits.length
+    assert_equal "app.?", emits[0][0]
+    assert_equal invalid_utf8, emits[0][2]['client_name']
 
+    invalid_ascii = "\xff".force_encoding('US-ASCII')
+    d1 = create_driver(CONFIG_INVALID_BYTE, 'input.activity')
+    d1.run do
+      d1.emit({'client_name' => invalid_ascii})
+    end
+    emits = d1.emits
+    assert_equal 1, emits.length
+    assert_equal "app.?", emits[0][0]
+    assert_equal invalid_ascii, emits[0][2]['client_name']
+  end
 end
-

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-rewrite-tag-filter
 version: !ruby/object:Gem::Version
-  version: 1.4.0
+  version: 1.4.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-17 00:00:00.000000000 Z
+date: 2014-03-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -43,6 +43,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: string-scrub
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
 - y.ken.studio@gmail.com