fluent-plugin-rename-key 0.3.4 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 385eb9972a3cac29dcee7801c0d64f8795347375
-  data.tar.gz: 4a57d056182bf5255a7cf67fad5069d0a8deff3d
+  metadata.gz: 74812e01d25afb405399759e5a33239c1c276965
+  data.tar.gz: e50fd65298761ece7e1b1ec7986b98a859e2fcac
 SHA512:
-  metadata.gz: efe4bbaa5cec4cb12f2df3375ca26a2d0112236ab597755e73c7aaf57a59239ecb960f542e61ad0bf4bb8950bf5f04fd4de2a7e9759029b954de9e9d06317d41
-  data.tar.gz: 7089ebf738857551dd5d713bca98da516d483467d9ebdf7a68aedc298aeac93deb1b8801614a6cef23f4cc7bd906ad5e675e50dfbe054878f6063fabf00fa01f
+  metadata.gz: 8dd4ac45f8169acb9a340cf7dd1677f7926b77e411404c8e19bef4d682a4936c65ca8c06470ff12c179d6e4dc87936d4dfced5cdb7a8eeac85369c344cb4270a
+  data.tar.gz: 0b566176c5d2c450e79a712087b16ed5bb9c1dbd9fb95a81f6cf3cb0954c9097bb57b521a005221f818bea5f4d819656d590d8f9c2c29e8fc7d2169c8446267d

data/.travis.yml

@@ -1,8 +1,9 @@
 language: ruby
 
 rvm:
-  - 2.3.0
+  - 2.3
   - 2.2
+  - 2.1
 
 # to avoid travis-ci issue since 2015-12-25
 before_install:

data/README.md

@@ -7,89 +7,83 @@
 
 ## Overview
 
-Fluentd output plugin. Renames keys matching the given regular expressions, assign new tags, and re-emits. This plugin resembles the implementation of [fluent-plugin-rewrite-tag-filter](https://github.com/y-ken/fluent-plugin-rewrite-tag-filter).
+This manual is for `~> 0.4.0`, which uses fluentd v0.14 API. For earlier version please see [here](https://github.com/shunwen/fluent-plugin-rename-key/tree/fluentd-v0.12).
 
-This plugin is created to resolve the invalid record problem while converting to BSON document before inserting to MongoDB, see [Restrictions on Field Names](http://docs.mongodb.org/manual/reference/limits/#Restrictions on Field Names) and [MongoDB Document Types](http://docs.mongodb.org/meta-driver/latest/legacy/bson/#mongodb-document-types) for more information.
+This plugin renames or replace portion of keys by regular expressions, assign new tags, and re-emits. 
 
-## Installation
-
-Install with gem or fluent-gem command as:
+It was created to work around the [field name restrictions](http://docs.mongodb.org/manual/reference/limits/#Restrictions on Field Names) of MongoDB BSON document. Also see [MongoDB Document Types](http://docs.mongodb.org/meta-driver/latest/legacy/bson/#mongodb-document-types) for more information.
 
-```
-# for fluentd
-$ gem install fluent-plugin-rename-key
+## Requirements
 
-# for td-agent OSX (Homebrew)
-$ /usr/local/Cellar/td-agent/1.1.17/bin/fluent-gem install fluent-plugin-rename-key
+For Fluentd earlier than v0.14.0, please use the earlier version 0.3.4. 
 
-# for td-agent
-$ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-rename-key
-```
+| fluent-plugin-rename-key  | Fluentd    | ruby   |
+|---------------------------|------------|--------|
+| ~> 0.3.4                  | >= v0.12.0 | >= 1.9 |
+| ~> 0.4.0                  | >= v0.14.0 | >= 2.1 |
 
-## Configuration
+## Installation
 
-### Notice
+See [Fluentd Installation Guide] (http://docs.fluentd.org/v0.12/categories/installation)
 
-v0.12's filter plugin does not have a functionality to modify tag and time. If you want to modify time in filter plugin, we recommends to migrate to depends on v0.14 Filter Plugin API.
-v0.14's filter API has `#filter_with_time` method.
+## Configuration
 
 ### Syntax
 
 ```
-# <num> is an integer, used to sort and apply the rules
-# <key_regexp> is the regular expression used to match the keys, whitespace is not allowed, use "\s" instead
-# <new_key> is the string with MatchData placeholder for creating the new key name, whitespace is allowed
+# <num> is an integer for ordering rules, rules are checked in ascending order. Only the first match is applied.
+# <key_regexp> is the regular expression to match keys, ' '(whitespace) is not allowed, use '\s' instead.
+# <new_key> is the new key name pattern, MatchData placeholder '${md[1]}' and whitespace are allowed.
 rename_rule<num> <key_regexp> <new_key>
 
-# <num> is an integer, used to sort and apply the rules
-# <key_regexp> is the regular expression used to match the keys, whitespace is not allowed, use "\s" instead
-# <new_key> is the string to replace the matches with, with MatchData placeholder for creating the new key name, whitespace is allowed. Optional, if missing then the matches are removed
+# <num> is an integer for ordering rules, rules are checked in ascending order. Only the first match is applied.
+# <key_regexp> is the regular expression to match keys, ' '(whitespace) is not allowed, use '\s' instead.
+# <new_key> is the pattern to replace the matches with, MatchData placeholder '${md[1]}' and whitespace are allowed. 
+#           This field is optional, if missing the matches will be replaced with ''(empty string).
 replace_rule<num> <key_regexp> <new_key>
 
-# Optional: remove tag prefix
+# Optional: dig into the hash structure and rename every matched key, or stop at the first level,
+# default is "true"
+deep_rename <bool>
+
+# Optional: remove tag prefix. This is only for <match>
 remove_tag_prefix <string>
 
-# Optional: append additional name to the original tag, default is "key_renamed"
+# Optional: append additional name to the original tag, default is 'key_renamed'. This is only for <match>
 append_tag <string>
-
-# Optional: dig into the hash structure and rename every matched key, or rename only keys at the first level,
-# default is "true"
-deep_rename <bool>
 ```
 
 ### Example
 
 Take this record as example: `'$url' => 'www.google.com', 'level2' => {'$1' => 'option1'}`.
-To successfully save it into MongoDB, we can use the following config to replace the keys starting with dollar sign.
+To save it to MongoDB, we can use the following config to replace the keys starting with dollar sign.
+
+For Fluentd v0.14 or later, use `rename_key` filter:
 
 ```
 # At rename_rule1, it matches the key starting the '$', say '$url',
 # and puts the following characters into match group 1.
 # Then uses the content in match group 1, ${md[1]} = 'url', to generate the new key name 'x$url'.
 
-<match input.test>
-  type rename_key
-  remove_tag_prefix input.test
-  append_tag renamed
+<filter input.test>
+  @type rename_key
   rename_rule1 ^\$(.+) x$${md[1]}
   rename_rule2 ^l(.{3})l(\d+) ${md[1]}_${md[2]}
 </match>
 ```
 
-The resulting record is `'x$url' => 'www.google.com', 'eve_2' => {'x$1' => 'option1'}` with new tag `renamed`.
+The result is `'x$url' => 'www.google.com', 'eve_2' => {'x$1' => 'option1'}`.
 
 ### MatchData placeholder
 
-This plugin uses Ruby's `String#match` to match the key to be replaced, and it is possible to refer to the contents of the resulting `MatchData` to create the new key name. `${md[0]}` refers to the matched string and `${md[1]}` refers to match group 1, and so on.
+This plugin uses `String#match` to match keys to be replaced. It is possible to reference the resulting `MatchData` in new key names. For example, `${md[0]}` is the matched string, `${md[1]}` is match group 1, and so on. 
 
-**Note:** Range expression `${md[0..2]}` is not supported.
+**Note:** This is done by matching `${md[0]}` string pattern, so array operations such as range `${md[0..2]}` is not supported.
 
-## TODO
-
-Pull requests are very welcome!!
+## Inspired by
+This plugin initially resembled the implementation of [fluent-plugin-rewrite-tag-filter](https://github.com/y-ken/fluent-plugin-rewrite-tag-filter).
 
 ## Copyright
 
 Copyright :  Copyright (c) 2013- Shunwen Hsiao (@hswtw)
 License   :  Apache License, Version 2.0
-

data/fluent-plugin-rename-key.gemspec

@@ -3,20 +3,20 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
   s.name        = 'fluent-plugin-rename-key'
-  s.version     = '0.3.4'
-  s.date        = '2016-11-29'
+  s.version     = '0.4.0'
+  s.date        = '2017-02-27'
   s.license     = 'Apache-2.0'
   s.authors     = ['Shunwen Hsiao', 'Julian Grinblat', 'Hiroshi Hatake']
   s.email       = ['hsiaoshunwen@gmail.com']
   s.homepage    = 'https://github.com/shunwen/fluent-plugin-rename-key'
   s.summary     = %q[Fluentd output plugin. Rename keys which match given regular expressions, assign new tags and re-emit the records.]
 
-  s.required_ruby_version = '>= 1.9.3'
+  s.required_ruby_version = '>= 2.1.0'
   s.files       = `git ls-files`.split("\n")
   s.test_files  = `git ls-files -- {test}/*`.split("\n")
   s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ['lib']
-  s.add_runtime_dependency 'fluentd', ">= 0.12.0", "< 2"
+  s.add_runtime_dependency 'fluentd', ">= 0.14.0", "< 2"
   s.add_development_dependency 'test-unit', '~> 3.2'
   s.add_development_dependency 'rake', '~> 11.3'
   s.add_development_dependency 'coveralls', '~> 0.8'

data/lib/fluent/plugin/filter_rename_key.rb

@@ -1,9 +1,10 @@
+require 'fluent/plugin/filter'
 require 'fluent/plugin/rename_key_util'
 
-class Fluent::RenameKeyFilter < Fluent::Filter
+class Fluent::Plugin::RenameKeyFilter < Fluent::Plugin::Filter
   Fluent::Plugin.register_filter 'rename_key', self
 
-  include Fluent::RenameKeyUtil
+  include Fluent::Plugin::RenameKeyUtil
 
   desc 'Deep rename/replace operation.'
   config_param :deep_rename, :bool, default: true
@@ -14,12 +15,12 @@ class Fluent::RenameKeyFilter < Fluent::Filter
     create_rename_rules(conf)
     create_replace_rules(conf)
 
-    raise Fluent::ConfigError, "No rename or replace rules are given" if @rename_rules.empty? && @replace_rules.empty?
+    if @rename_rules.empty? && @replace_rules.empty?
+      raise Fluent::ConfigError, 'No rename nor replace rule given'
+    end
   end
 
-  def filter tag, time, record
-    new_record = rename_key record
-    new_record = replace_key new_record
-    new_record
+  def filter _tag, _time, record
+    replace_key(rename_key(record))
   end
-end if defined?(Fluent::Filter)
+end

data/lib/fluent/plugin/out_rename_key.rb

@@ -1,27 +1,12 @@
+require 'fluent/plugin/output'
 require 'fluent/plugin/rename_key_util'
 
-class Fluent::RenameKeyOutput < Fluent::Output
+class Fluent::Plugin::RenameKeyOutput < Fluent::Plugin::Output
   Fluent::Plugin.register_output 'rename_key', self
 
-  include Fluent::RenameKeyUtil
+  helpers :event_emitter
 
-  # To support Fluentd v0.10.57 or earlier
-  unless method_defined?(:router)
-    define_method("router") { Fluent::Engine }
-  end
-
-  # Define `log` method for v0.10.42 or earlier
-  unless method_defined?(:log)
-    define_method("log") { $log }
-  end
-
-  # For fluentd v0.12.16 or earlier
-  class << self
-    unless method_defined?(:desc)
-      def desc(description)
-      end
-    end
-  end
+  include Fluent::Plugin::RenameKeyUtil
 
   DEFAULT_APPEND_TAG = 'key_renamed'
 
@@ -38,12 +23,12 @@ class Fluent::RenameKeyOutput < Fluent::Output
     create_rename_rules(conf)
     create_replace_rules(conf)
 
-    raise Fluent::ConfigError, "No rename or replace rules are given" if @rename_rules.empty? && @replace_rules.empty?
+    raise Fluent::ConfigError, 'No rename nor replace rules are given' if @rename_rules.empty? && @replace_rules.empty?
 
     @remove_tag_prefix = /^#{Regexp.escape @remove_tag_prefix}\.?/ if @remove_tag_prefix
   end
 
-  def emit tag, es, chain
+  def process tag, es
     es.each do |time, record|
       new_tag = @remove_tag_prefix ? tag.sub(@remove_tag_prefix, '') : tag
       new_tag = "#{new_tag}.#{@append_tag}".sub(/^\./, '')
@@ -51,7 +36,5 @@ class Fluent::RenameKeyOutput < Fluent::Output
       new_record = replace_key new_record
       router.emit new_tag, time, new_record
     end
-
-    chain.next
   end
 end

data/lib/fluent/plugin/rename_key_util.rb

@@ -1,55 +1,42 @@
-module Fluent
+module Fluent::Plugin
   module RenameKeyUtil
+    CONF_RENAME = 'rename_rule'
+    CONF_REPLACE = 'replace_rule'
+
     def create_rename_rules(conf)
       @rename_rules = []
-      conf_rename_rules = conf.keys.select { |k| k =~ /^rename_rule(\d+)$/ }
-      conf_rename_rules.sort_by { |r| r.sub('rename_rule', '').to_i }.each do |r|
-        key_regexp, new_key = parse_rename_rule conf[r]
+      rule_keys = conf.keys.select { |k| k.strip.start_with? CONF_RENAME }.
+          sort_by { |k| k.sub(CONF_RENAME, '').to_i }
 
-        if key_regexp.nil? || new_key.nil?
-          raise Fluent::ConfigError, "Failed to parse: #{r} #{conf[r]}"
-        end
+      rule_keys.each do |rule_key|
+        rule = parse_rename_rule conf[rule_key]
 
-        if @rename_rules.map { |r| r[:key_regexp] }.include? /#{key_regexp}/
-          raise Fluent::ConfigError, "Duplicated rules for key #{key_regexp}: #{@rename_rules}"
+        if @rename_rules.any? { |existing_rule| existing_rule[:key_regexp] == rule[:key_regexp] }
+          raise Fluent::ConfigError, "Duplicated rules for key #{rule[:key_regexp].source}: #{@rename_rules}"
         end
 
-        @rename_rules << { key_regexp: /#{key_regexp}/, new_key: new_key }
-        log.info "Added rename key rule: #{r} #{@rename_rules.last}"
+        @rename_rules << rule
+        log.info "Added rename key rule: #{rule_key} #{@rename_rules.last}"
       end
     end
 
     def create_replace_rules(conf)
       @replace_rules = []
-      conf_replace_rules = conf.keys.select { |k| k =~ /^replace_rule(\d+)$/ }
-      conf_replace_rules.sort_by { |r| r.sub('replace_rule', '').to_i }.each do |r|
-        key_regexp, replacement = parse_replace_rule conf[r]
+      rule_keys = conf.keys.select { |k| k.strip.start_with? CONF_REPLACE }.
+          sort_by { |k| k.sub(CONF_REPLACE, '').to_i }
 
-        if key_regexp.nil?
-          raise Fluent::ConfigError, "Failed to parse: #{r} #{conf[r]}"
-        end
-
-        if replacement.nil?
-          replacement = ""
-        end
+      rule_keys.each do |rule_key|
+        rule = parse_replace_rule conf[rule_key]
 
-        if @replace_rules.map { |r| r[:key_regexp] }.include? /#{key_regexp}/
-          raise Fluent::ConfigError, "Duplicated rules for key #{key_regexp}: #{@replace_rules}"
+        if @replace_rules.any? { |existing_rule| existing_rule[:key_regexp] == rule[:key_regexp] }
+          raise Fluent::ConfigError, "Duplicated rules for key #{rule[:key_regexp].source}: #{@replace_rules}"
         end
 
-        @replace_rules << { key_regexp: /#{key_regexp}/, replacement: replacement }
-        log.info "Added replace key rule: #{r} #{@replace_rules.last}"
+        @replace_rules << rule
+        log.info "Added replace key rule: #{rule_key} #{@replace_rules.last}"
       end
     end
 
-    def parse_rename_rule rule
-      $~.captures if rule.match /^([^\s]+)\s+(.+)$/
-    end
-
-    def parse_replace_rule rule
-      $~.captures if rule.match /^([^\s]+)(?:\s+(.+))?$/
-    end
-
     def rename_key record
       new_record = {}
 
@@ -106,6 +93,22 @@ module Fluent
       new_record
     end
 
+    private
+
+    def parse_rename_rule rule
+      m = rule.match(/^([^\s]+)\s+(.+)$/).captures
+      { key_regexp: /#{m[0]}/, new_key: m[1] }
+    rescue => e
+      raise Fluent::ConfigError, "Failed to parse rename rule #{rule} : #{e.message}"
+    end
+
+    def parse_replace_rule rule
+      m = rule.match(/^([^\s]+)(?:\s+(.+))?$/).captures
+      { key_regexp: /#{m[0]}/, replacement: m[1] || '' }
+    rescue => e
+      raise Fluent::ConfigError, "Failed to parse replace rule #{rule} : #{e.message}"
+    end
+
     def get_placeholder match_data
       placeholder = {}
 

data/test/helper.rb

@@ -1,31 +1,7 @@
-require 'rubygems'
-require 'bundler'
-begin
-  Bundler.setup(:default, :development)
-rescue Bundler::BundlerError => e
-  $stderr.puts e.message
-  $stderr.puts "Run `bundle install` to install missing gems"
-  exit e.status_code
-end
+require 'bundler/setup'
 require 'test/unit'
 
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
-$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(__dir__, '..', 'lib'))
+$LOAD_PATH.unshift(__dir__)
 require 'fluent/test'
-unless ENV.has_key?('VERBOSE')
-  nulllogger = Object.new
-  nulllogger.instance_eval {|obj|
-    def method_missing(method, *args)
-      # pass
-    end
-  }
-  $log = nulllogger
-end
-
-require 'fluent/plugin/out_rename_key'
-if defined?(Fluent::Filter)
-  require 'fluent/plugin/filter_rename_key'
-end
-
-class Test::Unit::TestCase
-end
+require 'fluent/test/helpers'

data/test/plugin/test_filter_rename_key.rb

@@ -1,4 +1,6 @@
 require 'helper'
+require 'fluent/test/driver/filter'
+require 'fluent/plugin/filter_rename_key'
 
 class RenameKeyFilterTest < Test::Unit::TestCase
   MATCH_TAG = 'incoming_tag'
@@ -6,12 +8,11 @@ class RenameKeyFilterTest < Test::Unit::TestCase
   REPLACE_RULE_CONFIG = 'replace_rule1 ^\$ x'
 
   def setup
-    omit("Fluentd v0.12 or later is required.") unless defined?(Fluent::Filter)
     Fluent::Test.setup
   end
 
-  def create_driver(conf, tag = MATCH_TAG)
-    Fluent::Test::FilterTestDriver.new(Fluent::RenameKeyFilter, tag).configure(conf)
+  def create_driver(conf)
+    Fluent::Test::Driver::Filter.new(Fluent::Plugin::RenameKeyFilter).configure(conf)
   end
 
   def test_config_error
@@ -39,50 +40,28 @@ class RenameKeyFilterTest < Test::Unit::TestCase
     assert_equal '^\$(.+)2(\d+) ${md[1]}_${md[2]}', d.instance.config['rename_rule2']
   end
 
-  def test_parse_rename_rule
-    parsed = Fluent::RenameKeyOutput.new.parse_rename_rule '(reg)(exp) ${md[1]} ${md[2]}'
-    assert_equal 2, parsed.length
-    assert_equal '(reg)(exp)', parsed[0]
-    assert_equal '${md[1]} ${md[2]}', parsed[1]
-  end
-
-  def test_parse_replace_rule_with_replacement
-    # Replace hyphens with underscores
-    parsed = Fluent::RenameKeyOutput.new.parse_replace_rule '- _'
-    assert_equal 2, parsed.length
-    assert_equal '-', parsed[0]
-    assert_equal '_', parsed[1]
-  end
-
-  def test_parse_replace_rule_without_replacement
-    # Remove all parenthesis hyphens and spaces
-    parsed = Fluent::RenameKeyOutput.new.parse_replace_rule '[()-\s]'
-    assert_equal 2, parsed.length
-    assert_equal '[()-\s]', parsed[0]
-    assert parsed[1].nil?
-  end
 
   def test_rename_rule_emit_deep_rename_hash
     d = create_driver RENAME_RULE_CONFIG
-    d.run do
-      d.emit '$key1' => 'value1', 'key2' => {'$key3' => 'value3', '$key4'=> {'$key5' => 'value5'} }
+    d.run default_tag: MATCH_TAG do
+      d.feed '$key1' => 'value1', 'key2' => {'$key3' => 'value3', '$key4'=> {'$key5' => 'value5'} }
     end
 
-    emits = d.emits
-    assert_equal %w[x$key1 key2], emits[0][2].keys
-    assert_equal %w[x$key3 x$key4], emits[0][2]['key2'].keys
-    assert_equal ['x$key5'], emits[0][2]['key2']['x$key4'].keys
+    records = d.filtered_records
+    assert_equal %w[x$key1 key2], records[0].keys
+    assert_equal %w[x$key3 x$key4], records[0]['key2'].keys
+    assert_equal ['x$key5'], records[0]['key2']['x$key4'].keys
   end
 
   def test_rename_rule_emit_deep_rename_array
     d = create_driver RENAME_RULE_CONFIG
-    d.run do
-      d.emit '$key1' => 'value1', 'key2' => [{'$key3' => 'value3'}, {'$key4'=> {'$key5' => 'value5'}}]
+    d.run default_tag: MATCH_TAG do
+      d.feed '$key1' => 'value1', 'key2' => [{'$key3' => 'value3'}, {'$key4'=> {'$key5' => 'value5'}}]
     end
 
-    emits = d.emits
-    assert_equal %w[x$key3 x$key4], emits[0][2]['key2'].flat_map(&:keys)
-    assert_equal ['x$key5'], emits[0][2]['key2'][1]['x$key4'].keys
+    records = d.filtered_records
+    assert_equal %w[x$key3 x$key4], records[0]['key2'].flat_map(&:keys)
+    assert_equal ['x$key5'], records[0]['key2'][1]['x$key4'].keys
   end
 
   def test_rename_rule_emit_deep_rename_off
@@ -92,22 +71,22 @@ class RenameKeyFilterTest < Test::Unit::TestCase
     ]
 
     d = create_driver config
-    d.run do
-      d.emit '$key1' => 'value1', 'key2' => {'$key3'=>'value3', '$key4'=> 'value4'}
+    d.run default_tag: MATCH_TAG do
+      d.feed '$key1' => 'value1', 'key2' => {'$key3'=>'value3', '$key4'=> 'value4'}
     end
 
-    emits = d.emits
-    assert_equal %w[$key3 $key4], emits[0][2]['key2'].keys
+    records = d.filtered_records
+    assert_equal %w[$key3 $key4], records[0]['key2'].keys
   end
 
   def test_rename_rule_emit_with_match_data
     d = create_driver 'rename_rule1 (\w+)\s(\w+)\s(\w+) ${md[3]} ${md[2]} ${md[1]}'
-    d.run do
-      d.emit 'key1 key2 key3' => 'value'
+    d.run default_tag: MATCH_TAG do
+      d.feed 'key1 key2 key3' => 'value'
     end
-    emits = d.emits
-    assert_equal 1, emits.length
-    assert_equal ['key3 key2 key1'], emits[0][2].keys
+    records = d.filtered_records
+    assert_equal 1, records.length
+    assert_equal ['key3 key2 key1'], records[0].keys
   end
 
   def test_multiple_rename_rules_emit
@@ -117,45 +96,45 @@ class RenameKeyFilterTest < Test::Unit::TestCase
     ]
 
     d = create_driver config_multiple_rules
-    d.run do
-      d.emit 'key 1' => 'value1', 'key 2' => 'value2'
+    d.run default_tag: MATCH_TAG do
+      d.feed 'key 1' => 'value1', 'key 2' => 'value2'
     end
 
-    emits = d.emits
-    assert_equal %w[key_1 key_2], emits[0][2].keys
+    records = d.filtered_records
+    assert_equal %w[key_1 key_2], records[0].keys
   end
 
   def test_replace_rule_emit_deep_rename_hash
     d = create_driver 'replace_rule1 ^(\$) x'
 
-    d.run do
-      d.emit '$key1' => 'value1', 'key2' => { 'key3' => 'value3', '$key4' => 'value4' }
+    d.run default_tag: MATCH_TAG do
+      d.feed '$key1' => 'value1', 'key2' => { 'key3' => 'value3', '$key4' => 'value4' }
     end
-    emits = d.emits
-    assert_equal %w[xkey1 key2], emits[0][2].keys
-    assert_equal %w[key3 xkey4], emits[0][2]['key2'].keys
+    records = d.filtered_records
+    assert_equal %w[xkey1 key2], records[0].keys
+    assert_equal %w[key3 xkey4], records[0]['key2'].keys
   end
 
   def test_replace_rule_emit_with_match_data
     d = create_driver 'rename_rule1 (\w+)\s(\w+)\s(\w+) ${md[3]} ${md[2]} ${md[1]}'
-    d.run do
-      d.emit 'key1 key2 key3' => 'value'
+    d.run default_tag: MATCH_TAG do
+      d.feed 'key1 key2 key3' => 'value'
     end
-    emits = d.emits
-    assert_equal 1, emits.length
-    assert_equal ['key3 key2 key1'], emits[0][2].keys
+    records = d.filtered_records
+    assert_equal 1, records.length
+    assert_equal ['key3 key2 key1'], records[0].keys
   end
 
   def test_replace_rule_emit_deep_rename_array
     d = create_driver 'replace_rule1 ^(\$) x${md[1]}'
 
-    d.run do
-      d.emit '$key1' => 'value1', 'key2' => [{'$key3' => 'value3'}, {'$key4' => {'$key5' => 'value5'}}]
+    d.run default_tag: MATCH_TAG do
+      d.feed '$key1' => 'value1', 'key2' => [{'$key3' => 'value3'}, {'$key4' => {'$key5' => 'value5'}}]
     end
 
-    emits = d.emits
-    assert_equal %w[x$key3 x$key4], emits[0][2]['key2'].flat_map(&:keys)
-    assert_equal %w[x$key5], emits[0][2]['key2'][1]['x$key4'].keys
+    records = d.filtered_records
+    assert_equal %w[x$key3 x$key4], records[0]['key2'].flat_map(&:keys)
+    assert_equal %w[x$key5], records[0]['key2'][1]['x$key4'].keys
   end
 
   def test_replace_rule_emit_deep_rename_off
@@ -165,51 +144,51 @@ class RenameKeyFilterTest < Test::Unit::TestCase
     ]
 
     d = create_driver config
-    d.run do
-      d.emit '$key1' => 'value1', 'key2' => {'$key3'=>'value3', '$key4'=> 'value4'}
+    d.run default_tag: MATCH_TAG do
+      d.feed '$key1' => 'value1', 'key2' => {'$key3'=>'value3', '$key4'=> 'value4'}
     end
 
-    emits = d.emits
-    assert_equal %w[$key3 $key4], emits[0][2]['key2'].keys
+    records = d.filtered_records
+    assert_equal %w[$key3 $key4], records[0]['key2'].keys
   end
 
   def test_replace_rule_emit_remove_matched_when_no_replacement
-    d = create_driver 'replace_rule1 [\s/()]'
-    d.run do
-      d.emit 'key (/1 )' => 'value1'
+    d = create_driver 'replace_rule1 "[\\\\s/()]"'
+    d.run default_tag: MATCH_TAG do
+      d.feed 'key (/1 )' => 'value1'
     end
 
-    emits = d.emits
-    assert_equal %w[key1], emits[0][2].keys
+    records = d.filtered_records
+    assert_equal %w[key1], records[0].keys
   end
 
   def test_multiple_replace_rules_emit
     config_multiple_rules = %q[
       replace_rule1 ^(\w+)\s(\d) ${md[1]}${md[2]}
-      replace_rule2 [\s()]
+      replace_rule2 "[\\\\s()]"
     ]
 
     d = create_driver config_multiple_rules
-    d.run do
-      d.emit 'key 1' => 'value1', 'key (2)' => 'value2'
+    d.run default_tag: MATCH_TAG do
+      d.feed 'key 1' => 'value1', 'key (2)' => 'value2'
     end
 
-    emits = d.emits
-    assert_equal %w[key1 key2], emits[0][2].keys
+    records = d.filtered_records
+    assert_equal %w[key1 key2], records[0].keys
   end
 
   def test_combined_rename_rule_and_replace_rule
     config_combined_rules = %q[
       rename_rule1 ^(.+)\s(one) ${md[1]}1
-      replace_rule2 [\s()]
+      replace_rule2 "[\\\\s()]"
     ]
 
     d = create_driver config_combined_rules
-    d.run do
-      d.emit '(key) one (x)' => 'value1', 'key (2)' => 'value2'
+    d.run default_tag: MATCH_TAG do
+      d.feed '(key) one (x)' => 'value1', 'key (2)' => 'value2'
     end
 
-    emits = d.emits
-    assert_equal %w[key1 key2], emits[0][2].keys
+    records = d.filtered_records
+    assert_equal %w[key1 key2], records[0].keys
   end
 end

data/test/plugin/test_out_rename_key.rb

@@ -1,4 +1,6 @@
 require 'helper'
+require 'fluent/plugin/out_rename_key'
+require 'fluent/test/driver/output'
 
 class RenameKeyOutputTest < Test::Unit::TestCase
   MATCH_TAG = 'incoming_tag'
@@ -9,15 +11,13 @@ class RenameKeyOutputTest < Test::Unit::TestCase
     Fluent::Test.setup
   end
 
-  def create_driver(conf, tag = MATCH_TAG)
-    Fluent::Test::OutputTestDriver.new(Fluent::RenameKeyOutput, tag).configure(conf)
+  def create_driver(conf)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::RenameKeyOutput).configure(conf)
   end
 
   def test_config_error
     assert_raise(Fluent::ConfigError) { create_driver('') }
-
     assert_raise(Fluent::ConfigError) { create_driver('rename_rule1 ^$(.+?) ') }
-
     assert_raise(Fluent::ConfigError) {
       config_dup_rules_for_a_key = %q[
         rename_rule1 ^\$(.+) ${md[1]}
@@ -38,39 +38,16 @@ class RenameKeyOutputTest < Test::Unit::TestCase
     assert_equal '^\$(.+)2(\d+) ${md[1]}_${md[2]}', d.instance.config['rename_rule2']
   end
 
-  def test_parse_rename_rule
-    parsed = Fluent::RenameKeyOutput.new.parse_rename_rule '(reg)(exp) ${md[1]} ${md[2]}'
-    assert_equal 2, parsed.length
-    assert_equal '(reg)(exp)', parsed[0]
-    assert_equal '${md[1]} ${md[2]}', parsed[1]
-  end
-
-  def test_parse_replace_rule_with_replacement
-    # Replace hyphens with underscores
-    parsed = Fluent::RenameKeyOutput.new.parse_replace_rule '- _'
-    assert_equal 2, parsed.length
-    assert_equal '-', parsed[0]
-    assert_equal '_', parsed[1]
-  end
-
-  def test_parse_replace_rule_without_replacement
-    # Remove all parethesis hyphens and spaces
-    parsed = Fluent::RenameKeyOutput.new.parse_replace_rule '[()-\s]'
-    assert_equal 2, parsed.length
-    assert_equal '[()-\s]', parsed[0]
-    assert parsed[1].nil?
-  end
-
   def test_emit_default_append_tag
-    append_tag = Fluent::RenameKeyOutput::DEFAULT_APPEND_TAG
+    append_tag = Fluent::Plugin::RenameKeyOutput::DEFAULT_APPEND_TAG
     d = create_driver RENAME_RULE_CONFIG
-    d.run do
-      d.emit '$key1' => 'value1', '%key2' => {'$key3'=>'123', '$key4'=> {'$key5' => 'value2'} }
+    d.run default_tag: MATCH_TAG do
+      d.feed '$key1' => 'value1', '%key2' => {'$key3'=>'123', '$key4'=> {'$key5' => 'value2'} }
     end
 
-    emits = d.emits
-    assert_equal 1, emits.length
-    assert_equal "#{MATCH_TAG}.#{append_tag}", emits[0][0]
+    events = d.events
+    assert_equal 1, events.length
+    assert_equal "#{MATCH_TAG}.#{append_tag}", events[0][0]
   end
 
   def test_emit_append_custom_tag
@@ -81,17 +58,17 @@ class RenameKeyOutputTest < Test::Unit::TestCase
     ]
     d = create_driver config
 
-    d.run do
-      d.emit '$key1' => 'value1', '%key2' => {'$key3'=>'123', '$key4'=> {'$key5' => 'value2'} }
+    d.run default_tag: MATCH_TAG do
+      d.feed '$key1' => 'value1', '%key2' => {'$key3'=>'123', '$key4'=> {'$key5' => 'value2'} }
     end
 
-    emits = d.emits
-    assert_equal 1, emits.length
-    assert_equal "#{MATCH_TAG}.#{custom_tag}", emits[0][0]
+    events = d.events
+    assert_equal 1, events.length
+    assert_equal "#{MATCH_TAG}.#{custom_tag}", events[0][0]
   end
 
   def test_remove_tag_prefix
-    append_tag = Fluent::RenameKeyOutput::DEFAULT_APPEND_TAG
+    append_tag = Fluent::Plugin::RenameKeyOutput::DEFAULT_APPEND_TAG
 
     config = %Q[
       #{RENAME_RULE_CONFIG}
@@ -99,38 +76,38 @@ class RenameKeyOutputTest < Test::Unit::TestCase
     ]
 
     d = create_driver config
-    d.run do
-      d.emit 'key1' => 'value1'
-      d.emit '$key2' => 'value2'
+    d.run default_tag: MATCH_TAG do
+      d.feed 'key1' => 'value1'
+      d.feed '$key2' => 'value2'
     end
 
-    emits = d.emits
-    assert_equal 2, emits.length
-    assert_equal append_tag, emits[0][0]
-    assert_equal append_tag, emits[1][0]
+    events = d.events
+    assert_equal 2, events.length
+    assert_equal append_tag, events[0][0]
+    assert_equal append_tag, events[1][0]
   end
 
   def test_rename_rule_emit_deep_rename_hash
     d = create_driver RENAME_RULE_CONFIG
-    d.run do
-      d.emit '$key1' => 'value1', 'key2' => {'$key3' => 'value3', '$key4'=> {'$key5' => 'value5'} }
+    d.run default_tag: MATCH_TAG do
+      d.feed '$key1' => 'value1', 'key2' => {'$key3' => 'value3', '$key4'=> {'$key5' => 'value5'} }
     end
 
-    emits = d.emits
-    assert_equal %w[x$key1 key2], emits[0][2].keys
-    assert_equal %w[x$key3 x$key4], emits[0][2]['key2'].keys
-    assert_equal ['x$key5'], emits[0][2]['key2']['x$key4'].keys
+    events = d.events
+    assert_equal %w[x$key1 key2], events[0][2].keys
+    assert_equal %w[x$key3 x$key4], events[0][2]['key2'].keys
+    assert_equal ['x$key5'], events[0][2]['key2']['x$key4'].keys
   end
 
   def test_rename_rule_emit_deep_rename_array
     d = create_driver RENAME_RULE_CONFIG
-    d.run do
-      d.emit '$key1' => 'value1', 'key2' => [{'$key3' => 'value3'}, {'$key4'=> {'$key5' => 'value5'}}]
+    d.run default_tag: MATCH_TAG do
+      d.feed '$key1' => 'value1', 'key2' => [{'$key3' => 'value3'}, {'$key4'=> {'$key5' => 'value5'}}]
     end
 
-    emits = d.emits
-    assert_equal %w[x$key3 x$key4], emits[0][2]['key2'].flat_map(&:keys)
-    assert_equal ['x$key5'], emits[0][2]['key2'][1]['x$key4'].keys
+    events = d.events
+    assert_equal %w[x$key3 x$key4], events[0][2]['key2'].flat_map(&:keys)
+    assert_equal ['x$key5'], events[0][2]['key2'][1]['x$key4'].keys
   end
 
   def test_rename_rule_emit_deep_rename_off
@@ -140,22 +117,22 @@ class RenameKeyOutputTest < Test::Unit::TestCase
     ]
 
     d = create_driver config
-    d.run do
-      d.emit '$key1' => 'value1', 'key2' => {'$key3'=>'value3', '$key4'=> 'value4'}
+    d.run default_tag: MATCH_TAG do
+      d.feed '$key1' => 'value1', 'key2' => {'$key3'=>'value3', '$key4'=> 'value4'}
     end
 
-    emits = d.emits
-    assert_equal %w[$key3 $key4], emits[0][2]['key2'].keys
+    events = d.events
+    assert_equal %w[$key3 $key4], events[0][2]['key2'].keys
   end
 
   def test_rename_rule_emit_with_match_data
     d = create_driver 'rename_rule1 (\w+)\s(\w+)\s(\w+) ${md[3]} ${md[2]} ${md[1]}'
-    d.run do
-      d.emit 'key1 key2 key3' => 'value'
+    d.run default_tag: MATCH_TAG do
+      d.feed 'key1 key2 key3' => 'value'
     end
-    emits = d.emits
-    assert_equal 1, emits.length
-    assert_equal ['key3 key2 key1'], emits[0][2].keys
+    events = d.events
+    assert_equal 1, events.length
+    assert_equal ['key3 key2 key1'], events[0][2].keys
   end
 
   def test_multiple_rename_rules_emit
@@ -165,45 +142,45 @@ class RenameKeyOutputTest < Test::Unit::TestCase
     ]
 
     d = create_driver config_multiple_rules
-    d.run do
-      d.emit 'key 1' => 'value1', 'key 2' => 'value2'
+    d.run default_tag: MATCH_TAG do
+      d.feed 'key 1' => 'value1', 'key 2' => 'value2'
     end
 
-    emits = d.emits
-    assert_equal %w[key_1 key_2], emits[0][2].keys
+    events = d.events
+    assert_equal %w[key_1 key_2], events[0][2].keys
   end
 
   def test_replace_rule_emit_deep_rename_hash
     d = create_driver 'replace_rule1 ^(\$) x'
 
-    d.run do
-      d.emit '$key1' => 'value1', 'key2' => { 'key3' => 'value3', '$key4' => 'value4' }
+    d.run default_tag: MATCH_TAG do
+      d.feed '$key1' => 'value1', 'key2' => { 'key3' => 'value3', '$key4' => 'value4' }
     end
-    emits = d.emits
-    assert_equal %w[xkey1 key2], emits[0][2].keys
-    assert_equal %w[key3 xkey4], emits[0][2]['key2'].keys
+    events = d.events
+    assert_equal %w[xkey1 key2], events[0][2].keys
+    assert_equal %w[key3 xkey4], events[0][2]['key2'].keys
   end
 
   def test_replace_rule_emit_with_match_data
     d = create_driver 'rename_rule1 (\w+)\s(\w+)\s(\w+) ${md[3]} ${md[2]} ${md[1]}'
-    d.run do
-      d.emit 'key1 key2 key3' => 'value'
+    d.run default_tag: MATCH_TAG do
+      d.feed 'key1 key2 key3' => 'value'
     end
-    emits = d.emits
-    assert_equal 1, emits.length
-    assert_equal ['key3 key2 key1'], emits[0][2].keys
+    events = d.events
+    assert_equal 1, events.length
+    assert_equal ['key3 key2 key1'], events[0][2].keys
   end
 
   def test_replace_rule_emit_deep_rename_array
     d = create_driver 'replace_rule1 ^(\$) x${md[1]}'
 
-    d.run do
-      d.emit '$key1' => 'value1', 'key2' => [{'$key3' => 'value3'}, {'$key4' => {'$key5' => 'value5'}}]
+    d.run default_tag: MATCH_TAG do
+      d.feed '$key1' => 'value1', 'key2' => [{'$key3' => 'value3'}, {'$key4' => {'$key5' => 'value5'}}]
     end
 
-    emits = d.emits
-    assert_equal %w[x$key3 x$key4], emits[0][2]['key2'].flat_map(&:keys)
-    assert_equal %w[x$key5], emits[0][2]['key2'][1]['x$key4'].keys
+    events = d.events
+    assert_equal %w[x$key3 x$key4], events[0][2]['key2'].flat_map(&:keys)
+    assert_equal %w[x$key5], events[0][2]['key2'][1]['x$key4'].keys
   end
 
   def test_replace_rule_emit_deep_rename_off
@@ -213,51 +190,51 @@ class RenameKeyOutputTest < Test::Unit::TestCase
     ]
 
     d = create_driver config
-    d.run do
-      d.emit '$key1' => 'value1', 'key2' => {'$key3'=>'value3', '$key4'=> 'value4'}
+    d.run default_tag: MATCH_TAG do
+      d.feed '$key1' => 'value1', 'key2' => {'$key3'=>'value3', '$key4'=> 'value4'}
     end
 
-    emits = d.emits
-    assert_equal %w[$key3 $key4], emits[0][2]['key2'].keys
+    events = d.events
+    assert_equal %w[$key3 $key4], events[0][2]['key2'].keys
   end
 
   def test_replace_rule_emit_remove_matched_when_no_replacement
-    d = create_driver 'replace_rule1 [\s/()]'
-    d.run do
-      d.emit 'key (/1 )' => 'value1'
+    d = create_driver 'replace_rule1 "[\\\\s/()]"'
+    d.run default_tag: MATCH_TAG do
+      d.feed 'key (/1 )' => 'value1'
     end
 
-    emits = d.emits
-    assert_equal %w[key1], emits[0][2].keys
+    events = d.events
+    assert_equal %w[key1], events[0][2].keys
   end
 
   def test_multiple_replace_rules_emit
     config_multiple_rules = %q[
       replace_rule1 ^(\w+)\s(\d) ${md[1]}${md[2]}
-      replace_rule2 [\s()]
+      replace_rule2 "[\\\\s()]"
     ]
 
     d = create_driver config_multiple_rules
-    d.run do
-      d.emit 'key 1' => 'value1', 'key (2)' => 'value2'
+    d.run default_tag: MATCH_TAG do
+      d.feed 'key 1' => 'value1', 'key (2)' => 'value2'
     end
 
-    emits = d.emits
-    assert_equal %w[key1 key2], emits[0][2].keys
+    events = d.events
+    assert_equal %w[key1 key2], events[0][2].keys
   end
 
   def test_combined_rename_rule_and_replace_rule
     config_combined_rules = %q[
       rename_rule1 ^(.+)\s(one) ${md[1]}1
-      replace_rule2 [\s()]
+      replace_rule2 "[\\\\s()]"
     ]
 
     d = create_driver config_combined_rules
-    d.run do
-      d.emit '(key) one (x)' => 'value1', 'key (2)' => 'value2'
+    d.run default_tag: MATCH_TAG do
+      d.feed '(key) one (x)' => 'value1', 'key (2)' => 'value2'
     end
 
-    emits = d.emits
-    assert_equal %w[key1 key2], emits[0][2].keys
+    events = d.events
+    assert_equal %w[key1 key2], events[0][2].keys
   end
 end

data/test/plugin/test_rename_key_util.rb

@@ -0,0 +1,32 @@
+require 'test/unit'
+require 'fluent/plugin/filter_rename_key'
+
+class TestClass
+  include Fluent::Plugin::RenameKeyUtil
+end
+
+class RenameKeyUtilTest < Test::Unit::TestCase
+
+  def test_parse_rename_rule
+    parsed = TestClass.new.send :parse_rename_rule, '(reg)(exp) ${md[1]} ${md[2]}'
+    assert_equal 2, parsed.length
+    assert_equal /(reg)(exp)/, parsed[:key_regexp]
+    assert_equal '${md[1]} ${md[2]}', parsed[:new_key]
+  end
+
+  def test_parse_replace_rule_with_replacement
+    # Replace hyphens with underscores
+    parsed = TestClass.new.send :parse_replace_rule, '- _'
+    assert_equal 2, parsed.length
+    assert_equal /-/, parsed[:key_regexp]
+    assert_equal '_', parsed[:replacement]
+  end
+
+  def test_parse_replace_rule_without_replacement
+    # Remove all parentheses, hyphens, and spaces
+    parsed = TestClass.new.send :parse_replace_rule, '[-\s()]'
+    assert_equal 2, parsed.length
+    assert_equal /[-\s()]/, parsed[:key_regexp]
+    assert_equal '', parsed[:replacement]
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-rename-key
 version: !ruby/object:Gem::Version
-  version: 0.3.4
+  version: 0.4.0
 platform: ruby
 authors:
 - Shunwen Hsiao
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-29 00:00:00.000000000 Z
+date: 2017-02-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -18,7 +18,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.12.0
+        version: 0.14.0
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -28,7 +28,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.12.0
+        version: 0.14.0
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -94,6 +94,7 @@ files:
 - test/helper.rb
 - test/plugin/test_filter_rename_key.rb
 - test/plugin/test_out_rename_key.rb
+- test/plugin/test_rename_key_util.rb
 homepage: https://github.com/shunwen/fluent-plugin-rename-key
 licenses:
 - Apache-2.0
@@ -106,7 +107,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 1.9.3
+      version: 2.1.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -114,10 +115,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.4
+rubygems_version: 2.6.10
 signing_key: 
 specification_version: 4
 summary: Fluentd output plugin. Rename keys which match given regular expressions,
   assign new tags and re-emit the records.
 test_files: []
-has_rdoc: