fluent-plugin-rename-key 0.1.4 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 50c9ff6f2cafefef879771c9ae58aa30370fc3b4
-  data.tar.gz: 753cd2ad645d97a697c797798802d536c561e0f8
+  metadata.gz: fe8b87f5ca1a68033f2d8d22ad283c097f63bca8
+  data.tar.gz: f7fd6a36cf2f4b85b02a3b626b28d2c986b40eb1
 SHA512:
-  metadata.gz: 460eb66318d43d083703f68058db5bbeb98d61926a4d3eeef53808babbbd8f6aab22a7637aa4fe4f73d2ffeb11a8b10c95bbb57ea4d57a1a886806505a093346
-  data.tar.gz: e5e733b1d8f6ce128f385d8b9d6ecc27c7ecbbc1f399e33d97c6d2943162f2315c03dfa8d3a436a56ada1fef3efc0e8f136d8af2a762267598e64d43925e06d2
+  metadata.gz: fd388ad6dd9c73618ee654dce8786dad13714f32cbf8f76706838050a4c28009c271faa90c73b82d57f5da52d4b1a27f16fd47fd8aba266f8ae07ddf7ca5abbc
+  data.tar.gz: c6eeab30ca1317114b157f9ddd6baf4041e330ed30ae289aeb44fad45ba72d8689af5041e0f6f79617d4def623d011a13c629d09201bcebbc8a6befc9f4208b1

data/README.md

@@ -1,10 +1,9 @@
 # fluent-plugin-rename-key, a plugin for [Fluentd](http://fluentd.org)
 
 ## Status
-[![Gem Version](https://badge.fury.io/rb/fluent-plugin-rename-key.png)](http://badge.fury.io/rb/fluent-plugin-rename-key)
-[![Build Status](https://travis-ci.org/shunwen/fluent-plugin-rename-key.png?branch=master)](https://travis-ci.org/shunwen/fluent-plugin-rename-key)
-[![Coverage Status](https://coveralls.io/repos/shunwen/fluent-plugin-rename-key/badge.png?branch=master)](https://coveralls.io/r/shunwen/fluent-plugin-rename-key?branch=master)
-[![Bitdeli Badge](https://d2weczhvl823v0.cloudfront.net/shunwen/fluent-plugin-rename-key/trend.png)](https://bitdeli.com/free "Bitdeli Badge")
+[![Gem Version](https://badge.fury.io/rb/fluent-plugin-rename-key.svg)](https://badge.fury.io/rb/fluent-plugin-rename-key)
+[![Build Status](https://travis-ci.org/shunwen/fluent-plugin-rename-key.svg?branch=master)](https://travis-ci.org/shunwen/fluent-plugin-rename-key)
+[![Coverage Status](https://coveralls.io/repos/shunwen/fluent-plugin-rename-key/badge.svg?branch=master)](https://coveralls.io/r/shunwen/fluent-plugin-rename-key?branch=master)
 
 ## Overview
 
@@ -37,6 +36,11 @@ $ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-rename-key
 # <new_key> is the string with MatchData placeholder for creating the new key name, whitespace is allowed
 rename_rule<num> <key_regexp> <new_key>
 
+# <num> is an integer, used to sort and apply the rules
+# <key_regexp> is the regular expression used to match the keys, whitespace is not allowed, use "\s" instead
+# <new_key> is the string to replace the matches with, with MatchData placeholder for creating the new key name, whitespace is allowed. Optional, if missing then the matches are removed
+replace_rule<num> <key_regexp> <new_key>
+
 # Optional: remove tag prefix
 remove_tag_prefix <string>
 

data/fluent-plugin-rename-key.gemspec

@@ -2,23 +2,23 @@
 $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |s|
-  s.name        = "fluent-plugin-rename-key"
-  s.version     = "0.1.4"
-  s.date        = "2016-04-04"
-  s.license     = "Apache-2.0"
-  s.authors     = ["Shunwen Hsiao"]
-  s.email       = "hsiaoshunwen@gmail.com"
-  s.homepage    = "https://github.com/shunwen/fluent-plugin-rename-key"
+  s.name        = 'fluent-plugin-rename-key'
+  s.version     = '0.2.0'
+  s.date        = '2016-04-05'
+  s.license     = 'Apache-2.0'
+  s.authors     = ['Shunwen Hsiao', 'Julian Grinblat']
+  s.email       = ['hsiaoshunwen@gmail.com']
+  s.homepage    = 'https://github.com/shunwen/fluent-plugin-rename-key'
   s.summary     = %q[Fluentd output plugin. Rename keys which match given regular expressions, assign new tags and re-emit the records.]
-  s.has_rdoc    = false
 
+  s.required_ruby_version = '>= 1.9.3'
   s.files       = `git ls-files`.split("\n")
-  s.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.test_files  = `git ls-files -- {test}/*`.split("\n")
   s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ['lib']
 
-  s.add_runtime_dependency "fluentd"
-  s.add_development_dependency "test-unit", ">= 3.1.0"
-  s.add_development_dependency "rake"
+  s.add_runtime_dependency 'fluentd'
+  s.add_development_dependency 'test-unit', '>= 3.1.0'
+  s.add_development_dependency 'rake'
   s.add_development_dependency 'coveralls'
 end

data/lib/fluent/plugin/out_rename_key.rb

@@ -1,16 +1,33 @@
 class Fluent::RenameKeyOutput < Fluent::Output
   Fluent::Plugin.register_output 'rename_key', self
 
+  # To support Fluentd v0.10.57 or earlier
+  unless method_defined?(:router)
+    define_method("router") { Fluent::Engine }
+  end
+
+  # Define `log` method for v0.10.42 or earlier
+  unless method_defined?(:log)
+    define_method("log") { $log }
+  end
+
+  # For fluentd v0.12.16 or earlier
+  class << self
+    unless method_defined?(:desc)
+      def desc(description)
+      end
+    end
+  end
+
   DEFAULT_APPEND_TAG = 'key_renamed'
 
+  desc 'Specify and remove tag prefix.'
   config_param :remove_tag_prefix, :string, default: nil
+  desc "Append custom tag postfix (default: #{DEFAULT_APPEND_TAG})."
   config_param :append_tag, :string, default: DEFAULT_APPEND_TAG
+  desc 'Deep rename/replace operation.'
   config_param :deep_rename, :bool, default: true
 
-  unless method_defined?(:router)
-    define_method("router") { Fluent::Engine }
-  end
-
   def configure conf
     super
 
@@ -28,10 +45,31 @@ class Fluent::RenameKeyOutput < Fluent::Output
       end
 
       @rename_rules << { key_regexp: /#{key_regexp}/, new_key: new_key }
-      $log.info "Added rename key rule: #{r} #{@rename_rules.last}"
+      log.info "Added rename key rule: #{r} #{@rename_rules.last}"
+    end
+
+    @replace_rules = []
+    conf_replace_rules = conf.keys.select { |k| k =~ /^replace_rule(\d+)$/ }
+    conf_replace_rules.sort_by { |r| r.sub('replace_rule', '').to_i }.each do |r|
+      key_regexp, replacement = parse_replace_rule conf[r]
+
+      if key_regexp.nil?
+        raise Fluent::ConfigError, "Failed to parse: #{r} #{conf[r]}"
+      end
+
+      if replacement.nil?
+          replacement = ""
+      end
+
+      if @replace_rules.map { |r| r[:key_regexp] }.include? /#{key_regexp}/
+        raise Fluent::ConfigError, "Duplicated rules for key #{key_regexp}: #{@replace_rules}"
+      end
+
+      @replace_rules << { key_regexp: /#{key_regexp}/, replacement: replacement }
+      log.info "Added replace key rule: #{r} #{@replace_rules.last}"
     end
 
-    raise Fluent::ConfigError, "No rename rules are given" if @rename_rules.empty?
+    raise Fluent::ConfigError, "No rename or replace rules are given" if @rename_rules.empty? && @replace_rules.empty?
 
     @remove_tag_prefix = /^#{Regexp.escape @remove_tag_prefix}\.?/ if @remove_tag_prefix
   end
@@ -41,6 +79,7 @@ class Fluent::RenameKeyOutput < Fluent::Output
       new_tag = @remove_tag_prefix ? tag.sub(@remove_tag_prefix, '') : tag
       new_tag = "#{new_tag}.#{@append_tag}".sub(/^\./, '')
       new_record = rename_key record
+      new_record = replace_key new_record
       router.emit new_tag, time, new_record
     end
 
@@ -50,9 +89,11 @@ class Fluent::RenameKeyOutput < Fluent::Output
   # private
 
   def parse_rename_rule rule
-    if rule.match /^([^\s]+)\s+(.+)$/
-      return $~.captures
-    end
+    $~.captures if rule.match /^([^\s]+)\s+(.+)$/
+  end
+
+  def parse_replace_rule rule
+    $~.captures if rule.match /^([^\s]+)(?:\s+(.+))?$/
   end
 
   def rename_key record
@@ -65,7 +106,7 @@ class Fluent::RenameKeyOutput < Fluent::Output
         next unless match_data # next rule
 
         placeholder = get_placeholder match_data
-        key = rule[:new_key].gsub /\${\w+\[\d+\]?}/, placeholder
+        key = rule[:new_key].gsub /\${md\[\d+\]}/, placeholder
         break
       end
 
@@ -83,11 +124,39 @@ class Fluent::RenameKeyOutput < Fluent::Output
     new_record
   end
 
+  def replace_key record
+    new_record = {}
+
+    record.each do |key, value|
+
+      @replace_rules.each do |rule|
+        match_data = key.match rule[:key_regexp]
+        next unless match_data # next rule
+
+        placeholder = get_placeholder match_data
+        key = key.gsub rule[:key_regexp], rule[:replacement].gsub(/\${md\[\d+\]}/, placeholder)
+        break
+      end
+
+      if @deep_rename
+        if value.is_a? Hash
+          value = replace_key value
+        elsif value.is_a? Array
+          value = value.map { |v| v.is_a?(Hash) ? replace_key(v) : v }
+        end
+      end
+
+      new_record[key] = value
+    end
+
+    new_record
+  end
+
   def get_placeholder match_data
     placeholder = {}
 
     match_data.to_a.each_with_index do |e, idx|
-      placeholder.store "${md[#{idx}]}", e
+      placeholder["${md[#{idx}]}"] = e
     end
 
     placeholder

data/test/plugin/test_out_rename_key.rb

@@ -2,13 +2,14 @@ require 'helper'
 
 class RenameKeyOutputTest < Test::Unit::TestCase
   MATCH_TAG = 'incoming_tag'
-  CONFIG = 'rename_rule1 ^\$(.+) x$${md[1]}'
+  RENAME_RULE_CONFIG = 'rename_rule1 ^\$(.+) x$${md[1]}'
+  REPLACE_RULE_CONFIG = 'replace_rule1 ^\$ x'
 
   def setup
     Fluent::Test.setup
   end
 
-  def create_driver(conf = CONFIG, tag = MATCH_TAG)
+  def create_driver(conf, tag = MATCH_TAG)
     Fluent::Test::OutputTestDriver.new(Fluent::RenameKeyOutput, tag).configure(conf)
   end
 
@@ -44,9 +45,25 @@ class RenameKeyOutputTest < Test::Unit::TestCase
     assert_equal '${md[1]} ${md[2]}', parsed[1]
   end
 
+  def test_parse_replace_rule_with_replacement
+    # Replace hyphens with underscores
+    parsed = Fluent::RenameKeyOutput.new.parse_replace_rule '- _'
+    assert_equal 2, parsed.length
+    assert_equal '-', parsed[0]
+    assert_equal '_', parsed[1]
+  end
+
+  def test_parse_replace_rule_without_replacement
+    # Remove all parethesis hyphens and spaces
+    parsed = Fluent::RenameKeyOutput.new.parse_replace_rule '[()-\s]'
+    assert_equal 2, parsed.length
+    assert_equal '[()-\s]', parsed[0]
+    assert parsed[1].nil?
+  end
+
   def test_emit_default_append_tag
     append_tag = Fluent::RenameKeyOutput::DEFAULT_APPEND_TAG
-    d = create_driver
+    d = create_driver RENAME_RULE_CONFIG
     d.run do
       d.emit '$key1' => 'value1', '%key2' => {'$key3'=>'123', '$key4'=> {'$key5' => 'value2'} }
     end
@@ -54,13 +71,12 @@ class RenameKeyOutputTest < Test::Unit::TestCase
     emits = d.emits
     assert_equal 1, emits.length
     assert_equal "#{MATCH_TAG}.#{append_tag}", emits[0][0]
-    assert_equal ['x$key1', '%key2'], emits[0][2].keys
   end
 
   def test_emit_append_custom_tag
     custom_tag = 'custom_tag'
     config = %Q[
-      #{CONFIG}
+      #{RENAME_RULE_CONFIG}
       append_tag #{custom_tag}
     ]
     d = create_driver config
@@ -72,67 +88,103 @@ class RenameKeyOutputTest < Test::Unit::TestCase
     emits = d.emits
     assert_equal 1, emits.length
     assert_equal "#{MATCH_TAG}.#{custom_tag}", emits[0][0]
-    assert_equal ['x$key1', '%key2'], emits[0][2].keys
   end
 
-  def test_emit_deep_rename_hash
-    d = create_driver
+  def test_remove_tag_prefix
+    append_tag = Fluent::RenameKeyOutput::DEFAULT_APPEND_TAG
+
+    config = %Q[
+      #{RENAME_RULE_CONFIG}
+      remove_tag_prefix #{MATCH_TAG}
+    ]
+
+    d = create_driver config
     d.run do
-      d.emit '$key1' => 'value1', '%key2' => {'$key3'=>'123', '$key4'=> {'$key5' => 'value2'} }
+      d.emit 'key1' => 'value1'
+      d.emit '$key2' => 'value2'
+    end
+
+    emits = d.emits
+    assert_equal 2, emits.length
+    assert_equal append_tag, emits[0][0]
+    assert_equal append_tag, emits[1][0]
+  end
+
+  def test_rename_rule_emit_deep_rename_hash
+    d = create_driver RENAME_RULE_CONFIG
+    d.run do
+      d.emit '$key1' => 'value1', 'key2' => {'$key3' => 'value3', '$key4'=> {'$key5' => 'value5'} }
     end
 
     emits = d.emits
-    assert_equal ['x$key3', 'x$key4'], emits[0][2]['%key2'].keys
-    assert_equal ['x$key5'], emits[0][2]['%key2']['x$key4'].keys
+    assert_equal %w[x$key1 key2], emits[0][2].keys
+    assert_equal %w[x$key3 x$key4], emits[0][2]['key2'].keys
+    assert_equal ['x$key5'], emits[0][2]['key2']['x$key4'].keys
   end
 
-  def test_emit_deep_rename_array
-    d = create_driver
+  def test_rename_rule_emit_deep_rename_array
+    d = create_driver RENAME_RULE_CONFIG
     d.run do
-      d.emit '$key1' => 'value1', '%key2' => [{'$key3'=>'123'}, {'$key4'=> {'$key5' => 'value2'}}]
+      d.emit '$key1' => 'value1', 'key2' => [{'$key3' => 'value3'}, {'$key4'=> {'$key5' => 'value5'}}]
     end
 
     emits = d.emits
-    assert_equal ['x$key3', 'x$key4'], emits[0][2]['%key2'].flat_map(&:keys)
-    assert_equal ['x$key5'], emits[0][2]['%key2'][1]['x$key4'].keys
+    assert_equal %w[x$key3 x$key4], emits[0][2]['key2'].flat_map(&:keys)
+    assert_equal ['x$key5'], emits[0][2]['key2'][1]['x$key4'].keys
   end
 
-  def test_emit_deep_rename_off
+  def test_rename_rule_emit_deep_rename_off
     config = %Q[
-      #{CONFIG}
+      #{RENAME_RULE_CONFIG}
       deep_rename false
     ]
 
     d = create_driver config
     d.run do
-      d.emit '$key1' => 'value1', '%key2' => {'$key3'=>'123', '$key4'=> {'$key5' => 'value2'} }
+      d.emit '$key1' => 'value1', 'key2' => {'$key3'=>'value3', '$key4'=> 'value4'}
     end
 
     emits = d.emits
-    assert_equal ['$key3', '$key4'], emits[0][2]['%key2'].keys
+    assert_equal %w[$key3 $key4], emits[0][2]['key2'].keys
   end
 
-  def test_remove_tag_prefix
-    append_tag = Fluent::RenameKeyOutput::DEFAULT_APPEND_TAG
+  def test_rename_rule_emit_with_match_data
+    d = create_driver 'rename_rule1 (\w+)\s(\w+)\s(\w+) ${md[3]} ${md[2]} ${md[1]}'
+    d.run do
+      d.emit 'key1 key2 key3' => 'value'
+    end
+    emits = d.emits
+    assert_equal 1, emits.length
+    assert_equal ['key3 key2 key1'], emits[0][2].keys
+  end
 
-    config = %Q[
-      #{CONFIG}
-      remove_tag_prefix #{MATCH_TAG}
+  def test_multiple_rename_rules_emit
+    config_multiple_rules = %q[
+      rename_rule1 ^(\w+)\s1 ${md[1]}_1
+      rename_rule2 ^(\w+)\s2 ${md[1]}_2
     ]
 
-    d = create_driver config
+    d = create_driver config_multiple_rules
     d.run do
-      d.emit 'key1' => 'value1'
-      d.emit '$key2' => 'value2'
+      d.emit 'key 1' => 'value1', 'key 2' => 'value2'
     end
 
     emits = d.emits
-    assert_equal 2, emits.length
-    assert_equal append_tag, emits[0][0]
-    assert_equal append_tag, emits[1][0]
+    assert_equal %w[key_1 key_2], emits[0][2].keys
+  end
+
+  def test_replace_rule_emit_deep_rename_hash
+    d = create_driver 'replace_rule1 ^(\$) x'
+
+    d.run do
+      d.emit '$key1' => 'value1', 'key2' => { 'key3' => 'value3', '$key4' => 'value4' }
+    end
+    emits = d.emits
+    assert_equal %w[xkey1 key2], emits[0][2].keys
+    assert_equal %w[key3 xkey4], emits[0][2]['key2'].keys
   end
 
-  def test_rename_with_match_data
+  def test_replace_rule_emit_with_match_data
     d = create_driver 'rename_rule1 (\w+)\s(\w+)\s(\w+) ${md[3]} ${md[2]} ${md[1]}'
     d.run do
       d.emit 'key1 key2 key3' => 'value'
@@ -142,18 +194,70 @@ class RenameKeyOutputTest < Test::Unit::TestCase
     assert_equal ['key3 key2 key1'], emits[0][2].keys
   end
 
-  def test_rename_with_multiple_rules
+  def test_replace_rule_emit_deep_rename_array
+    d = create_driver 'replace_rule1 ^(\$) x${md[1]}'
+
+    d.run do
+      d.emit '$key1' => 'value1', 'key2' => [{'$key3' => 'value3'}, {'$key4' => {'$key5' => 'value5'}}]
+    end
+
+    emits = d.emits
+    assert_equal %w[x$key3 x$key4], emits[0][2]['key2'].flat_map(&:keys)
+    assert_equal %w[x$key5], emits[0][2]['key2'][1]['x$key4'].keys
+  end
+
+  def test_replace_rule_emit_deep_rename_off
+    config = %Q[
+      #{REPLACE_RULE_CONFIG}
+      deep_rename false
+    ]
+
+    d = create_driver config
+    d.run do
+      d.emit '$key1' => 'value1', 'key2' => {'$key3'=>'value3', '$key4'=> 'value4'}
+    end
+
+    emits = d.emits
+    assert_equal %w[$key3 $key4], emits[0][2]['key2'].keys
+  end
+
+  def test_replace_rule_emit_remove_matched_when_no_replacement
+    d = create_driver 'replace_rule1 [\s/()]'
+    d.run do
+      d.emit 'key (/1 )' => 'value1'
+    end
+
+    emits = d.emits
+    assert_equal %w[key1], emits[0][2].keys
+  end
+
+  def test_multiple_replace_rules_emit
     config_multiple_rules = %q[
-      rename_rule1 ^(\w+)\s1 ${md[1]}_1
-      rename_rule2 ^(\w+)\s2 ${md[1]}_2
+      replace_rule1 ^(\w+)\s(\d) ${md[1]}${md[2]}
+      replace_rule2 [\s()]
     ]
 
     d = create_driver config_multiple_rules
     d.run do
-      d.emit 'key 1' => 'value1', 'key 2' => 'value2'
+      d.emit 'key 1' => 'value1', 'key (2)' => 'value2'
+    end
+
+    emits = d.emits
+    assert_equal %w[key1 key2], emits[0][2].keys
+  end
+
+  def test_combined_rename_rule_and_replace_rule
+    config_combined_rules = %q[
+      rename_rule1 ^(.+)\s(one) ${md[1]}1
+      replace_rule2 [\s()]
+    ]
+
+    d = create_driver config_combined_rules
+    d.run do
+      d.emit '(key) one (x)' => 'value1', 'key (2)' => 'value2'
     end
 
     emits = d.emits
-    assert_equal ['key_1', 'key_2'], emits[0][2].keys
+    assert_equal %w[key1 key2], emits[0][2].keys
   end
 end

metadata

@@ -1,14 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-rename-key
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.2.0
 platform: ruby
 authors:
 - Shunwen Hsiao
+- Julian Grinblat
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-04 00:00:00.000000000 Z
+date: 2016-04-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -67,7 +68,8 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 description: 
-email: hsiaoshunwen@gmail.com
+email:
+- hsiaoshunwen@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
@@ -75,7 +77,6 @@ files:
 - ".gitignore"
 - ".travis.yml"
 - Gemfile
-- Guardfile
 - LICENSE.txt
 - README.md
 - Rakefile
@@ -95,7 +96,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.9.3
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -108,6 +109,4 @@ signing_key:
 specification_version: 4
 summary: Fluentd output plugin. Rename keys which match given regular expressions,
   assign new tags and re-emit the records.
-test_files:
-- test/helper.rb
-- test/plugin/test_out_rename_key.rb
+test_files: []

data/Guardfile

@@ -1,9 +0,0 @@
-# A sample Guardfile
-# More info at https://github.com/guard/guard#readme
-
-guard :rspec do
-  watch(%r{^spec/.+_spec\.rb$})
-  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
-  watch('spec/spec_helper.rb')  { "spec" }
-end
-