fluent-plugin-rewrite-tag-filter 2.0.0.rc1 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 47add5af0b04bb562a521772ee1a3fac2872a2f9
-  data.tar.gz: b9ec90c1ef4cca20a7fa70a95c105f74a1a0753a
+  metadata.gz: 2644562a37644d56fa8712f13d46aa0ff42cb9ec
+  data.tar.gz: 22ac2b253497f823babee508e41d695b9bdc8218
 SHA512:
-  metadata.gz: 84132912d4463769f217023558fc4d522a3bdd0e3aa66ae3a92f1aab63b99c8298a47699913dcdcea27fe56db79ef3d692860ed32b43b969783af10b926d7093
-  data.tar.gz: bc956b79041dc4966d46471425e9b879ae2df2f77f73ac212bd2b467e624ea565d8e259fb6283966018d32422b6bbd92caa504216c50600709f8ef2cff9ff03a
+  metadata.gz: eaa2ee232b41ba0b6f387adb69621e78a411919d8d074c3ce014d344b4211a1dad7d47b61a06698cbc4fa0791de55a87111bf91383c47cee3f56f71299ad53e1
+  data.tar.gz: ff33f9efd2106fc70e85023a139563764aa8414329066459bd9dd177133ee9ff8b9bff20119a50d8621c20573f3d7cdcfc3cb37c1d67a4f6210190b9523bb878

data/.travis.yml

@@ -1,6 +1,8 @@
 language: ruby
 
 rvm:
+  - 2.4.2
+  - 2.3.5
   - 2.2
   - 2.1
 

data/README.md

@@ -14,8 +14,6 @@ user-agent, request-uri, regex-backreference and so on with regular expression.
 | >= 2.0.0                         | >= v0.14.2 | >= 2.1 |
 | < 2.0.0                          | >= v0.12.0 | >= 1.9 |
 
-NOTE: fluent-plugin-rewrite-tag-filter v2.0.0 is now RC. We will release stable v2.0.0 soon.
-
 ## Installation
 
 Install with `gem`, `fluent-gem` or `td-agent-gem` command as:
@@ -33,24 +31,21 @@ $ sudo td-agent-gem install fluent-plugin-rewrite-tag-filter -v 1.5.6
 
 ## Configuration
 
-### Syntax
-
-```
-rewriterule<num> <attribute> <regex_pattern> <new_tag>
-
-# Optional: Capitalize letter for every matched regex backreference. (ex: maps -> Maps)
-# for more details, see usage.
-capitalize_regex_backreference <yes/no> (default no)
-
-# Optional: remove tag prefix for tag placeholder. (see the section of "Tag placeholder")
-remove_tag_prefix <string>
+* **rewriterule\<num\>** (string) (optional) \<attribute\> \<regex_pattern\> \<new_tag\>
+  * Deprecated: Use <rule> section
+* **capitalize_regex_backreference** (bool) (optional): Capitalize letter for every matched regex backreference. (ex: maps -> Maps) for more details, see usage.
+  * Default value: no
+* **remove_tag_prefix** (string) (optional): Remove tag prefix for tag placeholder. (see the section of "Tag placeholder")
+* **hostname_command** (string) (optional): Override hostname command for placeholder. (see the section of "Tag placeholder")
+  * Default value: `hostname`.
 
-# Optional: override hostname command for placeholder. (see the section of "Tag placeholder")
-hostname_command <string>
+### \<rule\> section (optional) (multiple)
 
-# Optional: Set log level for this plugin. (ex: trace, debug, info, warn, error, fatal)
-log_level        <string> (default info)
-```
+* **key** (string) (required): The field name to which the regular expression is applied
+* **pattern** (regexp) (required): The regular expression
+* **tag** (string) (required): New tag
+* **invert** (bool) (optional): If true, rewrite tag when unmatch pattern
+  * Default value: `false`
 
 ### Usage
 
@@ -67,19 +62,49 @@ It's a sample to exclude some static file log before split tag by domain.
 </source>
 
 # "capitalize_regex_backreference yes" affects converting every matched first letter of backreference to upper case. ex: maps -> Maps
-# At rewriterule2, redirect to tag named "clear" which unmatched for status code 200.
-# At rewriterule3, redirect to tag named "clear" which is not end with ".com"
-# At rewriterule6, "site.$2$1" to be "site.ExampleMail" by capitalize_regex_backreference option.
+# At 2nd <rule>, redirect to tag named "clear" which unmatched for status code 200.
+# At 3rd <rule>, redirect to tag named "clear" which is not end with ".com"
+# At 6th <rule>, "site.$2$1" to be "site.ExampleMail" by capitalize_regex_backreference option.
 <match td.apache.access>
   @type rewrite_tag_filter
   capitalize_regex_backreference yes
-  rewriterule1 path   \.(gif|jpe?g|png|pdf|zip)$  clear
-  rewriterule2 status !^200$                      clear
-  rewriterule3 domain !^.+\.com$                  clear
-  rewriterule4 domain ^maps\.example\.com$        site.ExampleMaps
-  rewriterule5 domain ^news\.example\.com$        site.ExampleNews
-  rewriterule6 domain ^(mail)\.(example)\.com$    site.$2$1
-  rewriterule7 domain .+                          site.unmatched
+  <rule>
+    key     path
+    pattern \.(gif|jpe?g|png|pdf|zip)$
+    tag clear
+  </rule>
+  <rule>
+    key     status
+    pattern ^200$
+    tag     clear
+    invert  true
+  </rule>
+  <rule>
+    key     domain
+    pattern ^.+\.com$
+    tag     clear
+    invert  true
+  </rule>
+  <rule>
+    key     domain
+    pattern ^maps\.example\.com$
+    tag     site.ExampleMaps
+  </rule>
+  <rule>
+    key     domain
+    pattern ^news\.example\.com$
+    tag     site.ExampleNews
+  </rule>
+  <rule>
+    key     domain
+    pattern ^(mail)\.(example)\.com$
+    tag     site.$2$1
+  </rule>
+  <rule>
+    key     domain
+    pattern .+
+    tag     site.unmatched
+  </rule>
 </match>
 
 <match site.*>
@@ -125,6 +150,38 @@ $ tailf /var/log/td-agent/td-agent.log
 2012-09-16 18:10:51 +0900: adding rewrite_tag_filter rule: [5, "domain", /.+/, "site.unmatched"]
 ```
 
+### Nested attributes
+
+Dot notation:
+
+```
+<match kubernetes.**>
+  @type rewrite_tag_filter
+  rewriterule1 $.kubernetes.namespace_name ^(.+)$ $1.${tag}
+</match>
+```
+
+Bracket notation:
+
+```
+<match kubernetes.**>
+  @type rewrite_tag_filter
+  rewriterule1 $['kubernetes']['namespace_name'] ^(.+)$ $1.${tag}
+</match>
+```
+
+These example configurations can process nested attributes like following:
+
+```
+{
+  "kubernetes": {
+    "namespace_name": "default"
+  }
+}
+```
+
+When original tag is `kubernetes.var.log`, this will be converted to `default.kubernetes.var.log`.
+
 ### Tag placeholder
 
 It is supported these placeholder for new_tag (rewrited tag).
@@ -161,27 +218,43 @@ It's a sample to rewrite a tag with placeholder.
 # It will get "rewrited.access.ExampleMail"
 <match apache.access>
   @type rewrite_tag_filter
-  rewriterule1  domain  ^(mail)\.(example)\.com$  rewrited.${tag}.$2$1
   remove_tag_prefix apache
+  <rule>
+    key     domain
+    pattern ^(mail)\.(example)\.com$
+    tag     rewrited.${tag}.$2$1
+  </rule>
 </match>
 
 # It will get "rewrited.ExampleMail.app30-124.foo.com" when hostname is "app30-124.foo.com"
 <match apache.access>
   @type rewrite_tag_filter
-  rewriterule1  domain  ^(mail)\.(example)\.com$  rewrited.$2$1.${hostname}
+  <rule>
+    key     domain
+    pattern ^(mail)\.(example)\.com$
+    tag     rewrited.$2$1.${hostname}
+  </rule>
 </match>
 
 # It will get "rewrited.ExampleMail.app30-124" when hostname is "app30-124.foo.com"
 <match apache.access>
   @type rewrite_tag_filter
-  rewriterule1  domain  ^(mail)\.(example)\.com$  rewrited.$2$1.${hostname}
   hostname_command hostname -s
+  <rule>
+    key     domain
+    pattern ^(mail)\.(example)\.com$
+    tag     rewrited.$2$1.${hostname}
+  </rule>
 </match>
 
 # It will get "rewrited.game.pool"
 <match app.game.pool.activity>
   @type rewrite_tag_filter
-  rewriterule1  domain  ^.+$  rewrited.${tag_parts[1]}.${tag_parts[2]}
+  <rule>
+    key     domain
+    pattern ^.+$
+    tag     rewrited.${tag_parts[1]}.${tag_parts[2]}
+  </rule>
 </match>
 ```
 

data/fluent-plugin-rewrite-tag-filter.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = "fluent-plugin-rewrite-tag-filter"
-  s.version     = "2.0.0.rc1"
+  s.version     = "2.0.0"
   s.license     = "Apache-2.0"
   s.authors     = ["Kentaro Yoshida"]
   s.email       = ["y.ken.studio@gmail.com"]

data/lib/fluent/plugin/out_rewrite_tag_filter.rb

@@ -3,7 +3,7 @@ require "fluent/plugin/output"
 class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
   Fluent::Plugin.register_output('rewrite_tag_filter', self)
 
-  helpers :event_emitter
+  helpers :event_emitter, :record_accessor
 
   desc 'Capitalize letter for every matched regex backreference.'
   config_param :capitalize_regex_backreference, :bool, :default => false
@@ -12,13 +12,21 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
   desc 'Override hostname command for placeholder.'
   config_param :hostname_command, :string, :default => 'hostname'
 
-  MATCH_OPERATOR_EXCLUDE = '!'
-
-  def initialize
-    super
-    require 'string/scrub' if RUBY_VERSION.to_f < 2.1
+  config_section :rule, param_name: :rules, multi: true do
+    desc "The field name to which the regular expression is applied"
+    config_param :key, :string
+    desc "The regular expression"
+    config_param :pattern do |value|
+      Regexp.compile(value)
+    end
+    desc "New tag"
+    config_param :tag, :string
+    desc "If true, rewrite tag when unmatch pattern"
+    config_param :invert, :bool, default: false
   end
 
+  MATCH_OPERATOR_EXCLUDE = '!'
+
   def configure(conf)
     super
 
@@ -26,19 +34,19 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
     rewriterule_names = []
     @hostname = `#{@hostname_command}`.chomp
 
-    conf.keys.select{|k| k =~ /^rewriterule(\d+)$/}.sort_by{|i| i.sub('rewriterule', '').to_i}.each do |key|
-      rewritekey,regexp,rewritetag = parse_rewriterule(conf[key])
-      if regexp.nil? || rewritetag.nil?
-        raise Fluent::ConfigError, "failed to parse rewriterules at #{key} #{conf[key]}"
+    @rules.each do |rule|
+      unless rule.tag.match(/\$\{tag_parts\[\d\.\.\.?\d\]\}/).nil? or rule.tag.match(/__TAG_PARTS\[\d\.\.\.?\d\]__/).nil?
+        raise Fluent::ConfigError, "${tag_parts[n]} and __TAG_PARTS[n]__ placeholder does not support range specify at #{rule}"
       end
 
-      unless rewritetag.match(/\$\{tag_parts\[\d\.\.\.?\d\]\}/).nil? or rewritetag.match(/__TAG_PARTS\[\d\.\.\.?\d\]__/).nil?
-        raise Fluent::ConfigError, "${tag_parts[n]} and __TAG_PARTS[n]__ placeholder does not support range specify at #{key} #{conf[key]}"
-      end
+      invert = rule.invert ? MATCH_OPERATOR_EXCLUDE : ""
+      @rewriterules.push([record_accessor_create(rule.key), rule.pattern, invert, rule.tag])
+      rewriterule_names.push(rule.key + invert + rule.pattern.to_s)
+      log.info "adding rewrite_tag_filter rule: #{rule.key} #{@rewriterules.last}"
+    end
 
-      @rewriterules.push([rewritekey, /#{trim_regex_quote(regexp)}/, get_match_operator(regexp), rewritetag])
-      rewriterule_names.push(rewritekey + regexp)
-      log.info "adding rewrite_tag_filter rule: #{key} #{@rewriterules.last}"
+    if conf.keys.any? {|k| k.start_with?("rewriterule") }
+      raise Fluent::ConfigError, "\"rewriterule<num>\" support has been dropped. Use <rule> section instead."
     end
 
     unless @rewriterules.length > 0
@@ -68,8 +76,8 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
 
   def rewrite_tag(tag, record)
     placeholder = get_placeholder(tag)
-    @rewriterules.each do |rewritekey, regexp, match_operator, rewritetag|
-      rewritevalue = record[rewritekey].to_s
+    @rewriterules.each do |record_accessor, regexp, match_operator, rewritetag|
+      rewritevalue = record_accessor.call(record).to_s
       next if rewritevalue.empty? && match_operator != MATCH_OPERATOR_EXCLUDE
       last_match = regexp_last_match(regexp, rewritevalue)
       case match_operator
@@ -90,37 +98,13 @@ class Fluent::Plugin::RewriteTagFilterOutput < Fluent::Plugin::Output
   end
 
   def regexp_last_match(regexp, rewritevalue)
-    begin
-      return if regexp.nil?
+    if rewritevalue.valid_encoding?
       regexp.match(rewritevalue)
-    rescue ArgumentError => e
-      raise e unless e.message.index('invalid byte sequence in') == 0
+    else
       regexp.match(rewritevalue.scrub('?'))
     end
   end
 
-  def parse_rewriterule(rule)
-    if m = rule.match(/^([^\s]+)\s+(.+?)\s+([^\s]+)$/)
-      return m.captures
-    end
-  end
-
-  def trim_regex_quote(regexp)
-    if regexp.start_with?('"') && regexp.end_with?('"')
-      log.info "rewrite_tag_filter: [DEPRECATED] Use ^....$ pattern for partial word match instead of double-quote-delimiter. #{regexp}"
-      regexp = regexp[1..-2]
-    end
-    if regexp.start_with?(MATCH_OPERATOR_EXCLUDE)
-      regexp = regexp[1, regexp.length]
-    end
-    return regexp
-  end
-
-  def get_match_operator(regexp)
-    return MATCH_OPERATOR_EXCLUDE if regexp.start_with?(MATCH_OPERATOR_EXCLUDE)
-    return ''
-  end
-
   def get_backreference_table(elements)
     hash_table = Hash.new
     elements.each.with_index(1) do |value, index|

data/test/helper.rb

@@ -1,29 +1,9 @@
-require 'rubygems'
-require 'bundler'
-begin
-  Bundler.setup(:default, :development)
-rescue Bundler::BundlerError => e
-  $stderr.puts e.message
-  $stderr.puts "Run `bundle install` to install missing gems"
-  exit e.status_code
-end
+require 'bundler/setup'
 require 'test/unit'
 
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(__dir__, '..', 'lib'))
+$LOAD_PATH.unshift(__dir__)
 require 'fluent/test'
 require 'fluent/test/driver/output'
-unless ENV.has_key?('VERBOSE')
-  nulllogger = Object.new
-  nulllogger.instance_eval {|obj|
-    def method_missing(method, *args)
-      # pass
-    end
-  }
-  $log = nulllogger
-end
 
 require 'fluent/plugin/out_rewrite_tag_filter'
-
-class Test::Unit::TestCase
-end

data/test/plugin/test_out_rewrite_tag_filter.rb

@@ -5,224 +5,260 @@ class RewriteTagFilterOutputTest < Test::Unit::TestCase
     Fluent::Test.setup
   end
 
-  CONFIG = %[
-    rewriterule1 domain ^www\.google\.com$ site.Google
-    rewriterule2 domain ^news\.google\.com$ site.GoogleNews
-    rewriterule3 agent .* Mac OS X .* agent.MacOSX
-    rewriterule4 agent (Googlebot|CustomBot)-([a-zA-Z]+) agent.$1-$2
-    rewriterule5 domain ^(tagtest)\.google\.com$ site.${tag}.$1
-  ]
-
-  # aggresive test
-  # indentation, comment, capitalize_regex_backreference, regex with space aside.
-  # [DEPLICATED] Use ^....$ pattern for partial word match instead of double-quote-delimiter.
-  CONFIG_INDENT_SPACE_AND_CAPITALIZE_OPTION = %[
-    capitalize_regex_backreference yes
-    rewriterule1 domain ^www\.google\.com$                  site.Google # some comment
-    rewriterule2 domain ^(news)\.(google)\.com$             site.$2$1
-    rewriterule3 agent  ^.* Mac OS X .*$                    agent.MacOSX
-    rewriterule4 agent  "(Googlebot|CustomBot)-([a-zA-Z]+)" agent.$1-$2
-  ]
-
-  # remove_tag_prefix test
-  CONFIG_REMOVE_TAG_PREFIX = %[
-    rewriterule1 domain ^www\.google\.com$ ${tag}
-    remove_tag_prefix input
-  ]
-
-  # remove_tag_prefix test2
-  CONFIG_REMOVE_TAG_PREFIX_WITH_DOT = %[
-    rewriterule1 domain ^www\.google\.com$ ${tag}
-    remove_tag_prefix input.
-  ]
-
-  # hostname placeholder test
-  CONFIG_SHORT_HOSTNAME = %[
-    rewriterule1 domain ^www\.google\.com$ ${hostname}
-    remove_tag_prefix input
-    hostname_command hostname -s
-  ]
-
-  # '!' character (exclamation mark) to specify a non-matching pattern
-  CONFIG_NON_MATCHING = %[
-    rewriterule1 domain !^www\..+$ not_start_with_www
-    rewriterule2 domain ^www\..+$ start_with_www
-  ]
-
-  # jump of index
-  CONFIG_JUMP_INDEX = %[
-    rewriterule10 domain ^www\.google\.com$ site.Google
-    rewriterule20 domain ^news\.google\.com$ site.GoogleNews
-  ]
-
-  # split by tag
-  CONFIG_SPLIT_BY_TAG = %[
-    rewriterule1 user_name ^Lynn Minmay$ vip.${tag_parts[1]}.remember_love
-    rewriterule2 user_name ^Harlock$ ${tag_parts[2]}.${tag_parts[0]}.${tag_parts[1]}
-    rewriterule3 world ^(alice|chaos)$ application.${tag_parts[0]}.$1_server
-    rewriterule4 world ^[a-z]+$ application.${tag_parts[1]}.future_server
-  ]
-
-  # test for invalid byte sequence in UTF-8 error
-  CONFIG_INVALID_BYTE = %[
-    rewriterule1 client_name (.+) app.$1
-  ]
-
-  def create_driver(conf = CONFIG)
+  def create_driver(conf)
     Fluent::Test::Driver::Output.new(Fluent::Plugin::RewriteTagFilterOutput).configure(conf)
   end
 
   sub_test_case "configure" do
     data("empty" => "",
-         "missing regexp" => "rewriterule1 foo",
-         "not regexp 1" => "rewriterule1 hoge hoge.${tag_parts[0..2]}.__TAG_PARTS[0..2]__",
-         "not regexp 2" => "rewriterule1 fuga fuga.${tag_parts[1...2]}.__TAG_PARTS[1...2]__")
+         "line style" => "rewriterule1 foo ^foo$ new_tag")
     test "invalid" do |conf|
       assert_raise(Fluent::ConfigError) do
         create_driver(conf)
       end
     end
-
-    test "valid" do
-      d = create_driver %[
-        rewriterule1 domain ^www.google.com$ site.Google
-        rewriterule2 domain ^news.google.com$ site.GoogleNews
-      ]
-      assert_equal 'domain ^www.google.com$ site.Google', d.instance.config['rewriterule1']
-      assert_equal 'domain ^news.google.com$ site.GoogleNews', d.instance.config['rewriterule2']
-    end
   end
 
-  def test_emit
-    d1 = create_driver(CONFIG)
-    d1.run(default_tag: "input.access") do
-      d1.feed({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
-      d1.feed({'domain' => 'news.google.com', 'path' => '/', 'agent' => 'Googlebot-Mobile', 'response_time' => 900000})
-      d1.feed({'domain' => 'map.google.com', 'path' => '/', 'agent' => 'Macintosh; Intel Mac OS X 10_7_4', 'response_time' => 900000})
-      d1.feed({'domain' => 'labs.google.com', 'path' => '/', 'agent' => 'Mozilla/5.0 Googlebot-FooBar/2.1', 'response_time' => 900000})
-      d1.feed({'domain' => 'tagtest.google.com', 'path' => '/', 'agent' => 'Googlebot', 'response_time' => 900000})
-      d1.feed({'domain' => 'noop.example.com'}) # to be ignored
+  sub_test_case "section style" do
+    test "simple" do
+      config = %[
+        <rule>
+          key domain
+          pattern ^www\.google\.com$
+          tag site.Google
+        </rule>
+        <rule>
+          key domain
+          pattern ^news\.google\.com$
+          tag site.GoogleNews
+        </rule>
+        <rule>
+          key agent
+          pattern .* Mac OS X .*
+          tag agent.MacOSX
+        </rule>
+        <rule>
+          key agent
+          pattern (Googlebot|CustomBot)-([a-zA-Z]+)
+          tag agent.$1-$2
+        </rule>
+        <rule>
+          key domain
+          pattern ^(tagtest)\.google\.com$
+          tag site.${tag}.$1
+        </rule>
+      ]
+      d = create_driver(config)
+      d.run(default_tag: "input.access") do
+        d.feed({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
+        d.feed({'domain' => 'news.google.com', 'path' => '/', 'agent' => 'Googlebot-Mobile', 'response_time' => 900000})
+        d.feed({'domain' => 'map.google.com', 'path' => '/', 'agent' => 'Macintosh; Intel Mac OS X 10_7_4', 'response_time' => 900000})
+        d.feed({'domain' => 'labs.google.com', 'path' => '/', 'agent' => 'Mozilla/5.0 Googlebot-FooBar/2.1', 'response_time' => 900000})
+        d.feed({'domain' => 'tagtest.google.com', 'path' => '/', 'agent' => 'Googlebot', 'response_time' => 900000})
+        d.feed({'domain' => 'noop.example.com'}) # to be ignored
+      end
+      events = d.events
+      assert_equal 5, events.length
+      assert_equal 'site.Google', events[0][0] # tag
+      assert_equal 'site.GoogleNews', events[1][0] # tag
+      assert_equal 'news.google.com', events[1][2]['domain']
+      assert_equal 'agent.MacOSX', events[2][0] #tag
+      assert_equal 'agent.Googlebot-FooBar', events[3][0] #tag
+      assert_equal 'site.input.access.tagtest', events[4][0] #tag
     end
-    events = d1.events
-    assert_equal 5, events.length
-    assert_equal 'site.Google', events[0][0] # tag
-    assert_equal 'site.GoogleNews', events[1][0] # tag
-    assert_equal 'news.google.com', events[1][2]['domain']
-    assert_equal 'agent.MacOSX', events[2][0] #tag
-    assert_equal 'agent.Googlebot-FooBar', events[3][0] #tag
-    assert_equal 'site.input.access.tagtest', events[4][0] #tag
-  end
 
-  def test_emit2_indent_and_capitalize_option
-    d1 = create_driver(CONFIG_INDENT_SPACE_AND_CAPITALIZE_OPTION)
-    d1.run(default_tag: "input.access") do
-      d1.feed({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
-      d1.feed({'domain' => 'news.google.com', 'path' => '/', 'agent' => 'Googlebot-Mobile', 'response_time' => 900000})
-      d1.feed({'domain' => 'map.google.com', 'path' => '/', 'agent' => 'Macintosh; Intel Mac OS X 10_7_4', 'response_time' => 900000})
-      d1.feed({'domain' => 'labs.google.com', 'path' => '/', 'agent' => 'Mozilla/5.0 Googlebot-FooBar/2.1', 'response_time' => 900000})
+    test "remove_tag_prefix" do
+      config = %[
+        remove_tag_prefix input
+        <rule>
+          key domain
+          pattern ^www\.google\.com$
+          tag ${tag}
+        </rule>
+      ]
+      d = create_driver(config)
+      d.run(default_tag: "input.access") do
+        d.feed({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
+      end
+      events = d.events
+      assert_equal 1, events.length
+      assert_equal 'access', events[0][0] # tag
     end
-    events = d1.events
-    assert_equal 4, events.length
-    assert_equal 'site.Google', events[0][0] # tag
-    assert_equal 'site.GoogleNews', events[1][0] # tag
-    assert_equal 'news.google.com', events[1][2]['domain']
-    assert_equal 'agent.MacOSX', events[2][0] #tag
-    assert_equal 'agent.Googlebot-Foobar', events[3][0] #tag
-  end
 
-  def test_emit3_remove_tag_prefix
-    d1 = create_driver(CONFIG_REMOVE_TAG_PREFIX)
-    d1.run(default_tag: "input.access") do
-      d1.feed({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
+    test "remove_tag_prefix with dot" do
+      config = %[
+        remove_tag_prefix input.
+        <rule>
+          key domain
+          pattern ^www\.google\.com$
+          tag ${tag}
+        </rule>
+      ]
+      d = create_driver(config)
+      d.run(default_tag: "input.access") do
+        d.feed({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
+      end
+      events = d.events
+      assert_equal 1, events.length
+      assert_equal 'access', events[0][0] # tag
     end
-    events = d1.events
-    assert_equal 1, events.length
-    assert_equal 'access', events[0][0] # tag
-  end
 
-  def test_emit4_remove_tag_prefix_with_dot
-    d1 = create_driver(CONFIG_REMOVE_TAG_PREFIX_WITH_DOT)
-    d1.run(default_tag: "input.access") do
-      d1.feed({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
+    test "short hostname" do
+      config = %[
+        remove_tag_prefix input
+        hostname_command hostname -s
+        <rule>
+          key domain
+          pattern ^www\.google\.com$
+          tag ${hostname}
+        </rule>
+      ]
+      d = create_driver(config)
+      d.run(default_tag: "input.access") do
+        d.feed({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
+      end
+      events = d.events
+      assert_equal 1, events.length
+      assert_equal `hostname -s`.chomp, events[0][0] # tag
     end
-    events = d1.events
-    assert_equal 1, events.length
-    assert_equal 'access', events[0][0] # tag
-  end
 
-  def test_emit5_short_hostname
-    d1 = create_driver(CONFIG_SHORT_HOSTNAME)
-    d1.run(default_tag: "input.access") do
-      d1.feed({'domain' => 'www.google.com', 'path' => '/foo/bar?key=value', 'agent' => 'Googlebot', 'response_time' => 1000000})
+    test "non matching" do
+      config = %[
+        <rule>
+          key domain
+          pattern ^www\..+$
+          tag not_start_with_www
+          invert true
+        </rule>
+        <rule>
+          key domain
+          pattern ^www\..+$
+          tag start_with_www
+        </rule>
+      ]
+      d = create_driver(config)
+      d.run(default_tag: "input.access") do
+        d.feed({'domain' => 'www.google.com'})
+        d.feed({'path' => '/'})
+        d.feed({'domain' => 'maps.google.com'})
+      end
+      events = d.events
+      assert_equal 3, events.length
+      assert_equal 'start_with_www', events[0][0] # tag
+      assert_equal 'not_start_with_www', events[1][0] # tag
+      assert_equal 'not_start_with_www', events[2][0] # tag
     end
-    events = d1.events
-    assert_equal 1, events.length
-    assert_equal `hostname -s`.chomp, events[0][0] # tag
-  end
 
-  def test_emit6_non_matching
-    d1 = create_driver(CONFIG_NON_MATCHING)
-    d1.run(default_tag: "input.access") do
-      d1.feed({'domain' => 'www.google.com'})
-      d1.feed({'path' => '/'})
-      d1.feed({'domain' => 'maps.google.com'})
+    test "split by tag" do
+      config = %[
+        <rule>
+          key user_name
+          pattern ^Lynn Minmay$
+          tag vip.${tag_parts[1]}.remember_love
+        </rule>
+        <rule>
+          key user_name
+          pattern ^Harlock$
+          tag ${tag_parts[2]}.${tag_parts[0]}.${tag_parts[1]}
+        </rule>
+        <rule>
+          key  world
+          pattern ^(alice|chaos)$
+          tag application.${tag_parts[0]}.$1_server
+        </rule>
+        <rule>
+          key world
+          pattern ^[a-z]+$
+          tag application.${tag_parts[1]}.future_server
+        </rule>
+      ]
+      d = create_driver(config)
+      d.run(default_tag: "game.production.api") do
+        d.feed({'user_id' => '10000', 'world' => 'chaos', 'user_name' => 'gamagoori'})
+        d.feed({'user_id' => '10001', 'world' => 'chaos', 'user_name' => 'sanageyama'})
+        d.feed({'user_id' => '10002', 'world' => 'nehan', 'user_name' => 'inumuta'})
+        d.feed({'user_id' => '77777', 'world' => 'space', 'user_name' => 'Lynn Minmay'})
+        d.feed({'user_id' => '99999', 'world' => 'space', 'user_name' => 'Harlock'})
+      end
+      events = d.events
+      assert_equal 5, events.length
+      assert_equal 'application.game.chaos_server', events[0][0]
+      assert_equal 'application.game.chaos_server', events[1][0]
+      assert_equal 'application.production.future_server', events[2][0]
+      assert_equal 'vip.production.remember_love', events[3][0]
+      assert_equal 'api.game.production', events[4][0]
     end
-    events = d1.events
-    assert_equal 3, events.length
-    assert_equal 'start_with_www', events[0][0] # tag
-    assert_equal 'not_start_with_www', events[1][0] # tag
-    assert_equal 'not_start_with_www', events[2][0] # tag
-  end
 
-  def test_emit7_jump_index
-    d1 = create_driver(CONFIG_JUMP_INDEX)
-    d1.run(default_tag: "input.access") do
-      d1.feed({'domain' => 'www.google.com', 'path' => '/', 'agent' => 'Googlebot', 'response_time' => 1000000})
-      d1.feed({'domain' => 'news.google.com', 'path' => '/', 'agent' => 'Googlebot', 'response_time' => 900000})
+    test "invalid_byte (UTF-8)" do
+      config = %[
+        <rule>
+          key client_name
+          pattern (.+)
+          tag app.$1
+        </rule>
+      ]
+      invalid_utf8 = "\xff".force_encoding('UTF-8')
+      d = create_driver(config)
+      d.run(default_tag: "input.activity") do
+        d.feed({'client_name' => invalid_utf8})
+      end
+      events = d.events
+      assert_equal 1, events.length
+      assert_equal "app.?", events[0][0]
+      assert_equal invalid_utf8, events[0][2]['client_name']
     end
-    events = d1.events
-    assert_equal 2, events.length
-    assert_equal 'site.Google', events[0][0] # tag
-    assert_equal 'site.GoogleNews', events[1][0] # tag
-  end
 
-  def test_emit8_split_by_tag
-    d1 = create_driver(CONFIG_SPLIT_BY_TAG)
-    d1.run(default_tag: "game.production.api") do
-      d1.feed({'user_id' => '10000', 'world' => 'chaos', 'user_name' => 'gamagoori'})
-      d1.feed({'user_id' => '10001', 'world' => 'chaos', 'user_name' => 'sanageyama'})
-      d1.feed({'user_id' => '10002', 'world' => 'nehan', 'user_name' => 'inumuta'})
-      d1.feed({'user_id' => '77777', 'world' => 'space', 'user_name' => 'Lynn Minmay'})
-      d1.feed({'user_id' => '99999', 'world' => 'space', 'user_name' => 'Harlock'})
+    test "invalid byte (US-ASCII)" do
+      config = %[
+        <rule>
+          key client_name
+          pattern (.+)
+          tag app.$1
+        </rule>
+      ]
+      invalid_ascii = "\xff".force_encoding('US-ASCII')
+      d = create_driver(config)
+      d.run(default_tag: "input.activity") do
+        d.feed({'client_name' => invalid_ascii})
+      end
+      events = d.events
+      assert_equal 1, events.length
+      assert_equal "app.?", events[0][0]
+      assert_equal invalid_ascii, events[0][2]['client_name']
     end
-    events = d1.events
-    assert_equal 5, events.length
-    assert_equal 'application.game.chaos_server', events[0][0]
-    assert_equal 'application.game.chaos_server', events[1][0]
-    assert_equal 'application.production.future_server', events[2][0]
-    assert_equal 'vip.production.remember_love', events[3][0]
-    assert_equal 'api.game.production', events[4][0]
-  end
 
-  def test_emit9_invalid_byte
-    invalid_utf8 = "\xff".force_encoding('UTF-8')
-    d1 = create_driver(CONFIG_INVALID_BYTE)
-    d1.run(default_tag: "input.activity") do
-      d1.feed({'client_name' => invalid_utf8})
+    test "nested key support with dot notation" do
+      conf = %[
+        <rule>
+          key $.email.domain
+          pattern ^(example)\.(com)$
+          tag $2.$1
+        </rule>
+      ]
+      d = create_driver(conf)
+      d.run(default_tag: "input") do
+        d.feed({ "email" => { "localpart" => "john", "domain" => "example.com" }})
+        d.feed({ "email" => { "localpart" => "doe", "domain" => "example.jp" }})
+      end
+      events = d.events
+      assert_equal "com.example", events[0][0]
     end
-    events = d1.events
-    assert_equal 1, events.length
-    assert_equal "app.?", events[0][0]
-    assert_equal invalid_utf8, events[0][2]['client_name']
 
-    invalid_ascii = "\xff".force_encoding('US-ASCII')
-    d1 = create_driver(CONFIG_INVALID_BYTE)
-    d1.run(default_tag: "input.activity") do
-      d1.feed({'client_name' => invalid_ascii})
+    test "nested key support with bracket notation" do
+      conf = %[
+        <rule>
+          key $['email']['domain']
+          pattern ^(example)\.(com)$
+          tag $2.$1
+        </rule>
+      ]
+      d = create_driver(conf)
+      d.run(default_tag: "input") do
+        d.feed({ "email" => { "localpart" => "john", "domain" => "example.com" }})
+        d.feed({ "email" => { "localpart" => "doe", "domain" => "example.jp" }})
+      end
+      events = d.events
+      assert_equal "com.example", events[0][0]
     end
-    events = d1.events
-    assert_equal 1, events.length
-    assert_equal "app.?", events[0][0]
-    assert_equal invalid_ascii, events[0][2]['client_name']
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-rewrite-tag-filter
 version: !ruby/object:Gem::Version
-  version: 2.0.0.rc1
+  version: 2.0.0
 platform: ruby
 authors:
 - Kentaro Yoshida
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-10-12 00:00:00.000000000 Z
+date: 2017-11-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: test-unit
@@ -86,9 +86,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.6.13