fluent-plugin-rewrite-tag-filter 1.2.1 → 1.3.0

data/README.md

@@ -5,7 +5,7 @@
 ### RewriteTagFilterOutput
 
 Fluentd Output filter plugin. It has designed to rewrite tag like mod_rewrite.  
-Re-emmit a record with rewrited tag when a value matches with the regular expression.  
+Re-emmit a record with rewrited tag when a value matches/unmatches with the regular expression.  
 Also you can change a tag from apache log by domain, status-code(ex. 500 error),  
 user-agent, request-uri, regex-backreference and so on with regular expression.
 
@@ -24,15 +24,15 @@ gem install fluent-plugin-rewrite-tag-filter
 ### Syntax
 
 ```
-rewruterule<num:1-200> <attribute> <regex_pattern> <new_tag>
+rewriterule<num:1-200> <attribute> <regex_pattern> <new_tag>
 
 # Optional: Capitalize every matched regex backreference. (ex: $1, $2)
 capitalize_regex_backreference <yes/no> (default no)
 
-# Optional: remove tag prefix for tag placeholder.
+# Optional: remove tag prefix for tag placeholder. (see the section of "Tag placeholder")
 remove_tag_prefix <string>
 
-# Optional: set placeholder for hostname.
+# Optional: override hostname command for placeholder. (see the section of "Tag placeholder")
 hostname_command <string>
 ```
 
@@ -50,15 +50,19 @@ It's a sample to exclude some static file log before split tag by domain.
   pos_file /var/log/td-agent/apache_access.pos
 </source>
 
-# At rewriterule4, "site.$2$1" to be "site.ExampleMail" by capitalize_regex_backreference option.
+# At rewriterule2, redirect to tag named "clear" which unmatched for status code 200.
+# At rewriterule3, redirect to tag named "clear" which is not end with ".com"
+# At rewriterule6, "site.$2$1" to be "site.ExampleMail" by capitalize_regex_backreference option.
 <match td.apache.access>
   type rewrite_tag_filter
   capitalize_regex_backreference yes
   rewriterule1 path   \.(gif|jpe?g|png|pdf|zip)$  clear
-  rewriterule2 domain ^maps\.example\.com$        site.ExampleMaps
-  rewriterule3 domain ^news\.example\.com$        site.ExampleNews
-  rewriterule4 domain ^(mail)\.(example)\.com$    site.$2$1
-  rewriterule5 domain .+                          site.unmatched
+  rewriterule2 status !^200$                      clear
+  rewriterule3 domain !^.+\.com$                  clear
+  rewriterule4 domain ^maps\.example\.com$        site.ExampleMaps
+  rewriterule5 domain ^news\.example\.com$        site.ExampleNews
+  rewriterule6 domain ^(mail)\.(example)\.com$    site.$2$1
+  rewriterule7 domain .+                          site.unmatched
 </match>
 
 <match site.*>
@@ -106,14 +110,22 @@ $ tailf /var/log/td-agent/td-agent.log
 
 ### Tag placeholder
 
-Supporing these placeholder for new_tag(rewrited tag).  
+It is supporting there placeholder for new_tag(rewrited tag).
+
+- `${tag}`
+- `__TAG__`
+
 It's available to use this placeholder with `remove_tag_prefix` option.  
-This feature adds removing tag prefix for `${tag}` or `__TAG__` in placeholder.
+This option adds removing tag prefix for `${tag}` or `__TAG__` in placeholder.
 
 - `${hostname}`
 - `__HOSTNAME__`
-- `${tag}`
-- `__TAG__`
+
+By default, execute command as `hostname` to get full hostname.  
+Also, you can override hostname command using `hostname_command` option.  
+It comes short hostname with `hostname_command hostname -s` configuration specified.
+
+#### Placeholder Usage
 
 It's a sample to rewrite a tag with placeholder.
 

data/fluent-plugin-rewrite-tag-filter.gemspec

@@ -3,11 +3,11 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-rewrite-tag-filter"
-  s.version     = "1.2.1"
+  s.version     = "1.3.0"
   s.authors     = ["Kentaro Yoshida"]
   s.email       = ["y.ken.studio@gmail.com"]
   s.homepage    = "https://github.com/y-ken/fluent-plugin-rewrite-tag-filter"
-  s.summary     = %q{Fluentd Output filter plugin. It has designed to rewrite tag like mod_rewrite. Re-emmit a record with rewrited tag when a value matches with the regular expression. Also you can change a tag from apache log by domain, status-code(ex. 500 error), user-agent, request-uri, regex-backreference and so on with regular expression.}
+  s.summary     = %q{Fluentd Output filter plugin. It has designed to rewrite tag like mod_rewrite. Re-emmit a record with rewrited tag when a value matches/unmatches with the regular expression. Also you can change a tag from apache log by domain, status-code(ex. 500 error), user-agent, request-uri, regex-backreference and so on with regular expression.}
 
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")

data/lib/fluent/plugin/out_rewrite_tag_filter.rb

@@ -1,16 +1,12 @@
 class Fluent::RewriteTagFilterOutput < Fluent::Output
   Fluent::Plugin.register_output('rewrite_tag_filter', self)
 
-  PATTERN_MAX_NUM = 200
-
-  config_param :rewriterule1, :string # string: NAME REGEXP
-  (2..PATTERN_MAX_NUM).each do |i|
-    config_param ('rewriterule' + i.to_s).to_sym, :string, :default => nil # NAME REGEXP
-  end
   config_param :capitalize_regex_backreference, :bool, :default => false
   config_param :remove_tag_prefix, :string, :default => nil
   config_param :hostname_command, :string, :default => 'hostname'
 
+  MATCH_OPERATOR_EXCLUDE = '!'
+
   def configure(conf)
     super
 
@@ -18,24 +14,20 @@ class Fluent::RewriteTagFilterOutput < Fluent::Output
     rewriterule_names = []
     @hostname = `#{@hostname_command}`.chomp
 
-    invalids = conf.keys.select{|k| k =~ /^rewriterule(\d+)$/}.select{|arg| arg =~ /^rewriterule(\d+)/ and not (1..PATTERN_MAX_NUM).include?($1.to_i)}
-    if invalids.size > 0
-      $log.warn "invalid number rewriterules (valid rewriterule number:1-{PATTERN_MAX_NUM}): #{invalids.join(",")}"
-    end
-    (1..PATTERN_MAX_NUM).each do |i|
-      next unless conf["rewriterule#{i}"]
-      rewritekey,regexp,rewritetag = conf["rewriterule#{i}"].match(/^([^\s]+)\s+(.+?)\s+([^\s]+)$/).captures
+    conf.keys.select{|k| k =~ /^rewriterule(\d+)$/}.sort_by{|i| i.sub('rewriterule', '').to_i}.each do |key|
+      rewritekey,regexp,rewritetag = parse_rewriterule(conf[key])
       if regexp.nil? || rewritetag.nil?
-        raise Fluent::ConfigError, "missing values at rewriterule#{i} " + conf["rewriterule#{i}"].inspect
+        raise Fluent::ConfigError, "failed to parse rewriterules at #{key} #{conf[key]}"
       end
-      @rewriterules.push([i, rewritekey, Regexp.new(trim_regex_quote(regexp)), rewritetag])
+      @rewriterules.push([rewritekey, Regexp.new(trim_regex_quote(regexp)), get_match_operator(regexp), rewritetag])
       rewriterule_names.push(rewritekey + regexp)
-      $log.info "adding rewrite_tag_filter rule: #{@rewriterules.last}"
+      $log.info "adding rewrite_tag_filter rule: #{key} #{@rewriterules.last}"
     end
-    rewriterule_index_list = conf.keys.select{|s| s =~ /^rewriterule\d$/}.map{|v| (/^rewriterule(\d)$/.match(v))[1].to_i}
-    unless rewriterule_index_list.reduce(true){|v,i| v and @rewriterules[i - 1]}
-      raise Fluent::ConfigError, "jump of rewriterule index found #{@rewriterules.inspect}"
+
+    unless @rewriterules.length > 0
+      raise Fluent::ConfigError, "missing rewriterules"
     end
+
     unless @rewriterules.length == rewriterule_names.uniq.length
       raise Fluent::ConfigError, "duplicated rewriterules found #{@rewriterules.inspect}"
     end
@@ -46,34 +38,58 @@ class Fluent::RewriteTagFilterOutput < Fluent::Output
   end
 
   def emit(tag, es, chain)
-    placeholder = get_placeholder(tag)
     es.each do |time,record|
-      rewrite = false
-      @rewriterules.each do |index, rewritekey, regexp, rewritetag|
-        rewritevalue = record[rewritekey].to_s
-        next if rewritevalue.nil?
-        next unless (regexp && regexp.match(rewritevalue))
-        backreference_table = map_regex_table($~.captures)
-        rewrite = true
-        rewritetag.gsub!(/(\${[a-z]+}|__[A-Z]+__)/, placeholder)
-        tag = rewritetag.gsub(/\$\d+/, backreference_table)
-        break
-      end
-      Fluent::Engine.emit(tag, time, record) if (rewrite)
+      rewrited_tag = rewrite_tag(tag, record)
+      next if rewrited_tag.nil? || tag == rewrited_tag
+      Fluent::Engine.emit(rewrited_tag, time, record)
     end
 
     chain.next
   end
 
+  def rewrite_tag(tag, record)
+    placeholder = get_placeholder(tag)
+    @rewriterules.each do |rewritekey, regexp, match_operator, rewritetag|
+      rewritevalue = record[rewritekey].to_s
+      next if rewritevalue.empty? && match_operator != MATCH_OPERATOR_EXCLUDE
+      matched = regexp && regexp.match(rewritevalue)
+      case match_operator
+      when MATCH_OPERATOR_EXCLUDE
+        next if matched
+      else
+        next if !matched
+        backreference_table = get_backreference_table($~.captures)
+        rewritetag.gsub!(/\$\d+/, backreference_table)
+      end
+      rewritetag.gsub!(/(\${[a-z]+}|__[A-Z]+__)/, placeholder)
+      return rewritetag
+    end
+    return nil
+  end
+
+  def parse_rewriterule(rule)
+    if rule.match(/^([^\s]+)\s+(.+?)\s+([^\s]+)$/)
+      return $~.captures
+    end
+  end
+
   def trim_regex_quote(regexp)
     if regexp.start_with?('"') && regexp.end_with?('"')
       $log.info "rewrite_tag_filter: [DEPRECATED] Use ^....$ pattern for partial word match instead of double-quote-delimiter. #{regexp}"
-      return regexp[1..-2]
+      regexp = regexp[1..-2]
+    end
+    if regexp.start_with?('!')
+      regexp = regexp[1, regexp.length]
     end
     return regexp
   end
 
-  def map_regex_table(elements)
+  def get_match_operator(regexp)
+    return '!' if regexp.start_with?('!')
+    return ''
+  end
+
+  def get_backreference_table(elements)
     hash_table = Hash.new
     elements.each.with_index(1) do |value, index|
       hash_table["$#{index}"] = @capitalize_regex_backreference ? value.capitalize : value

data/test/plugin/test_out_rewrite_tag_filter.rb

@@ -16,7 +16,7 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
   # aggresive test
   # indentation, comment, capitalize_regex_backreference, regex with space aside.
   # [DEPLICATED] Use ^....$ pattern for partial word match instead of double-quote-delimiter.
-  CONFIG2 = %[
+  CONFIG_INDENT_SPACE_AND_CAPITALIZE_OPTION = %[
     capitalize_regex_backreference yes
     rewriterule1 domain ^www\.google\.com$                  site.Google # some comment
     rewriterule2 domain ^(news)\.(google)\.com$             site.$2$1
@@ -25,18 +25,30 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
   ]
 
   # remove_tag_prefix test
-  CONFIG3 = %[
+  CONFIG_REMOVE_TAG_PREFIX = %[
     rewriterule1 domain ^www\.google\.com$ ${tag}
     remove_tag_prefix input
   ]
 
   # hostname placeholder test
-  CONFIG4 = %[
+  CONFIG_SHORT_HOSTNAME = %[
     rewriterule1 domain ^www\.google\.com$ ${hostname}
     remove_tag_prefix input
     hostname_command hostname -s
   ]
 
+  # '!' character (exclamation mark) to specify a non-matching pattern
+  CONFIG_NON_MATCHING = %[
+    rewriterule1 domain !^www\..+$ not_start_with_www
+    rewriterule2 domain ^www\..+$ start_with_www
+  ]
+
+  # jump of index
+  CONFIG_JUMP_INDEX = %[
+    rewriterule10 domain ^www\.google\.com$ site.Google
+    rewriterule20 domain ^news\.google\.com$ site.GoogleNews
+  ]
+
   def create_driver(conf=CONFIG,tag='test')
     Fluent::Test::OutputTestDriver.new(Fluent::RewriteTagFilterOutput, tag).configure(conf)
   end
@@ -45,13 +57,19 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
     assert_raise(Fluent::ConfigError) {
       d = create_driver('')
     }
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver('rewriterule1 foo')
+    }
+    assert_raise(Fluent::ConfigError) {
+      d = create_driver('rewriterule1 foo foo')
+    }
     d = create_driver %[
       rewriterule1 domain ^www.google.com$ site.Google
       rewriterule2 domain ^news.google.com$ site.GoogleNews
     ]
-    d.instance.inspect
-    assert_equal 'domain ^www.google.com$ site.Google', d.instance.rewriterule1
-    assert_equal 'domain ^news.google.com$ site.GoogleNews', d.instance.rewriterule2
+    puts d.instance.inspect
+    assert_equal 'domain ^www.google.com$ site.Google', d.instance.config['rewriterule1']
+    assert_equal 'domain ^news.google.com$ site.GoogleNews', d.instance.config['rewriterule2']
   end
 
   def test_emit
@@ -62,6 +80,7 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
       d1.emit({'domain' => 'map.google.com', 'path' => '/', 'agent' => 'Macintosh; Intel Mac OS X 10_7_4', 'response_time' => 900000})
       d1.emit({'domain' => 'labs.google.com', 'path' => '/', 'agent' => 'Mozilla/5.0 Googlebot-FooBar/2.1', 'response_time' => 900000})
       d1.emit({'domain' => 'tagtest.google.com', 'path' => '/', 'agent' => 'Googlebot', 'response_time' => 900000})
+      d1.emit({'domain' => 'noop.example.com'}) # to be ignored
     end
     emits = d1.emits
     assert_equal 5, emits.length
@@ -78,8 +97,8 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
     assert_equal 'site.input.access.tagtest', emits[4][0] #tag
   end
 
-  def test_emit2
-    d1 = create_driver(CONFIG2, 'input.access')
+  def test_emit2_indent_and_capitalize_option
+    d1 = create_driver(CONFIG_INDENT_SPACE_AND_CAPITALIZE_OPTION, 'input.access')
     d1.run do
       d1.emit({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
       d1.emit({'domain' => 'news.google.com', 'path' => '/', 'agent' => 'Googlebot-Mobile', 'response_time' => 900000})
@@ -99,8 +118,8 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
     assert_equal 'agent.Googlebot-Foobar', emits[3][0] #tag
   end
 
-  def test_emit3
-    d1 = create_driver(CONFIG3, 'input.access')
+  def test_emit3_remove_tag_prefix
+    d1 = create_driver(CONFIG_REMOVE_TAG_PREFIX, 'input.access')
     d1.run do
       d1.emit({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
     end
@@ -110,8 +129,8 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
     assert_equal 'access', emits[0][0] # tag
   end
 
-  def test_emit4
-    d1 = create_driver(CONFIG4, 'input.access')
+  def test_emit4_short_hostname
+    d1 = create_driver(CONFIG_SHORT_HOSTNAME, 'input.access')
     d1.run do
       d1.emit({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
     end
@@ -120,5 +139,37 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
     p emits[0]
     assert_equal `hostname -s`.chomp, emits[0][0] # tag
   end
+
+  def test_emit5_non_matching
+    d1 = create_driver(CONFIG_NON_MATCHING, 'input.access')
+    d1.run do
+      d1.emit({'domain' => 'www.google.com'})
+      d1.emit({'path' => '/'})
+      d1.emit({'domain' => 'maps.google.com'})
+    end
+    emits = d1.emits
+    assert_equal 3, emits.length
+    p emits[0]
+    assert_equal 'start_with_www', emits[0][0] # tag
+    p emits[1]
+    assert_equal 'not_start_with_www', emits[1][0] # tag
+    p emits[2]
+    assert_equal 'not_start_with_www', emits[2][0] # tag
+  end
+
+  def test_emit6_jump_index
+    d1 = create_driver(CONFIG_JUMP_INDEX, 'input.access')
+    d1.run do
+      d1.emit({'domain' => 'www.google.com', 'path' => '/', 'agent' => 'Googlebot', 'response_time' => 1000000})
+      d1.emit({'domain' => 'news.google.com', 'path' => '/', 'agent' => 'Googlebot', 'response_time' => 900000})
+    end
+    emits = d1.emits
+    assert_equal 2, emits.length
+    p emits[0]
+    assert_equal 'site.Google', emits[0][0] # tag
+    p emits[1]
+    assert_equal 'site.GoogleNews', emits[1][0] # tag
+  end
+
 end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-rewrite-tag-filter
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.3.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-06-21 00:00:00.000000000 Z
+date: 2013-07-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -85,9 +85,10 @@ rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: Fluentd Output filter plugin. It has designed to rewrite tag like mod_rewrite.
-  Re-emmit a record with rewrited tag when a value matches with the regular expression.
-  Also you can change a tag from apache log by domain, status-code(ex. 500 error),
-  user-agent, request-uri, regex-backreference and so on with regular expression.
+  Re-emmit a record with rewrited tag when a value matches/unmatches with the regular
+  expression. Also you can change a tag from apache log by domain, status-code(ex.
+  500 error), user-agent, request-uri, regex-backreference and so on with regular
+  expression.
 test_files:
 - test/helper.rb
 - test/plugin/test_out_rewrite_tag_filter.rb