fluent-plugin-rename-key 0.1.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 60e3765941b5e976e67523ec08d3339a331b3834
+  data.tar.gz: b28f691513dde3dd285b69067caf667a31e46beb
+SHA512:
+  metadata.gz: fa4c5cf24eca410e42877a6d62f1f1f48fcc45ee32b49264d4c46de9cd73c540e3596eabe4b55e51e40924a36742f09167f40526cd6a5f732990b822f32c0e79
+  data.tar.gz: a3e1c48fd89a1351a5145cc55ba1b24eca6ff507992c9fcb6c397ae73aad9e4fe5de60c65d4f2f686cca9d3b5254903f516b2839faa0c4cb646919266ac26392

data/.gitignore

@@ -0,0 +1,12 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+# For TextMate, emacs, vim
+*.tmproj
+tmtags
+*~
+\#*
+.\#*
+*.swp
+tmp/*

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+gemspec
+

data/Guardfile

@@ -0,0 +1,9 @@
+# A sample Guardfile
+# More info at https://github.com/guard/guard#readme
+
+guard :rspec do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+end
+

data/LICENSE.txt

@@ -0,0 +1,13 @@
+Copyright (c) 2013- Shunwen Hsiao
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -0,0 +1,73 @@
+# fluent-plugin-rename-key
+
+## Overview
+
+Fluentd Output filter plugin. It goes through each record, rename keys matching the given regular expressions, and re-emit the event with a new tag. This plugin resembles the implementation of [fluent-plugin-rewrite-tag-filter](https://github.com/y-ken/fluent-plugin-rewrite-tag-filter).
+
+This plugin is created to resolve the invalid record problem while converting to BSON document before inserting to MongoDB, see [restrictions on Field Names](http://docs.mongodb.org/manual/reference/limits/#Restrictions on Field Names) and [MongoDB Document Types](http://docs.mongodb.org/meta-driver/latest/legacy/bson/#mongodb-document-types).
+
+## Installation
+
+install with gem or fluent-gem command as:
+
+```
+# for fluentd
+$ gem install fluent-plugin-rename-key
+
+# for td-agent
+$ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-rename-key
+```
+
+## Configuration
+
+### Syntax
+
+```
+rename_rule<num> <key_regexp> <new_key>
+
+# Optional: remove tag prefix
+remove_tag_prefix <string>
+
+# Optional: append additional name to the original tag, default **key_renamed**
+append_tag <string>
+
+# Optional: dig into the hash structures and rename every matched key or rename only keys at the first level
+# default is true
+deep_rename <bool>
+```
+
+### Example
+
+Take this record as example: `'$url' => 'www.google.com', 'level2' => {'$1' => 'option1'}`.
+To successfully save it into MongoDB, we can use the following config to replace the keys starting with dollar sign.
+
+```
+# At rename_rule1, it matches the key starting the `$`, say `$url`, 
+# and puts the following characters into match group 1. 
+# Then uses the content in match group 1, `url`, to generate the new key name `x$url`.
+
+<match input.test>
+  type rename_key
+  remove_tag_prefix input.test
+  append_tag renamed
+  rename_rule1 ^\$(.+) x$${md[1]}
+  rename_rule2 ^l(eve)l(\d+) ${md[1]}_${md[2]}
+</match>
+```
+
+The resulting record will be `'x$url' => 'www.google.com', 'eve_2' => {'x$1' => 'option1'}`
+
+### MatchData placeholder
+
+This plugin uses Ruby's `String#match` to match the key to be replaced, and it is possible to refer to the contents of the resulting `MatchData` to create the new key name. `${md[0]}` refers to the matched string and `${md[1]}` refers to match group 1, and so on.
+
+**Note** Range expression ```${md[0..2]}``` is not supported.
+
+## TODO
+
+Pull requests are very welcome!!
+
+## Copyright
+
+Copyright :  Copyright (c) 2013- Shunwen Hsiao (@hswtw)
+License   :  Apache License, Version 2.0

data/fluent-plugin-rename-key.gemspec

@@ -0,0 +1,23 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-rename-key"
+  gem.version     = "0.1.2"
+  gem.license     = "Apache 2.0"
+  gem.authors     = ["Shunwen Hsiao"]
+  gem.email       = "hsiaoshunwen@gmail.com"
+  gem.homepage    = "https://github.com/shunwen/fluent-plugin-rename-key"
+  gem.summary     = %q[Fluentd Output filter plugin. It is designed to rename keys which match given patterns and assign new tags, and re-emit a record with the new tag.]
+  gem.has_rdoc    = false
+
+  gem.files       = `git ls-files`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "fluentd", "~> 0.10.9"
+  gem.add_development_dependency "rspec"
+  gem.add_development_dependency "bundler"
+
+end

data/lib/fluent/plugin/out_rename_key.rb

@@ -0,0 +1,83 @@
+class Fluent::RenameKeyOutput < Fluent::Output
+  Fluent::Plugin.register_output 'rename_key', self
+
+  config_param :remove_tag_prefix, :string, default: nil
+  config_param :append_tag, :string, default: 'key_renamed'
+  config_param :deep_rename, :bool, default: true
+
+  def configure conf
+    super
+
+    @rename_rules = []
+    conf_rename_rules = conf.keys.select { |k| k =~ /^rename_rule(\d+)$/ }
+    conf_rename_rules.sort_by { |r| r.sub('rename_rule', '').to_i }.each do |r|
+      key_regexp, new_key = parse_rename_rule conf[r]
+
+      if key_regexp.nil? || new_key.nil?
+        raise Fluent::ConfigError, "Failed to parse: #{r} #{conf[r]}"
+      end
+
+      if @rename_rules.map { |r| r[:key_regexp] }.include? /#{key_regexp}/
+        raise Fluent::ConfigError, "Duplicated rules for key #{key_regexp}: #{@rename_rules}"
+      end
+
+      @rename_rules << { key_regexp: /#{key_regexp}/, new_key: new_key }
+      $log.info "Added rename key rule: #{r} #{@rename_rules.last}"
+    end
+
+    raise Fluent::ConfigError, "No rename rules are given" if @rename_rules.empty?
+
+    @remove_tag_prefix = /^#{Regexp.escape @remove_tag_prefix}\.?/ if @remove_tag_prefix
+  end
+
+  def emit tag, es, chain
+    es.each do |time, record|
+      new_tag = @remove_tag_prefix ? tag.sub(@remove_tag_prefix, '') : tag
+      new_tag = "#{new_tag}.#{@append_tag}".sub(/^\./, '')
+      new_record = rename_key record
+      Fluent::Engine.emit new_tag, time, new_record
+    end
+
+    chain.next
+  end
+
+  # private
+
+  def parse_rename_rule rule
+    if rule.match /^([^\s]+)\s+(.+)$/
+      return $~.captures
+    end
+  end
+
+  def rename_key record
+    new_record = {}
+
+    record.each do |key, value|
+
+      @rename_rules.each do |rule|
+        match_data = key.match rule[:key_regexp]
+        next unless match_data # next rule
+
+        placeholder = get_placeholder match_data
+        key = rule[:new_key].gsub /\${\w+\[\d+\]?}/, placeholder
+        break
+      end
+
+      value = rename_key value if value.is_a? Hash and @deep_rename
+      new_record[key] = value
+    end
+
+    new_record
+  end
+
+  def get_placeholder match_data
+    placeholder = {}
+
+    match_data.to_a.each_with_index do |e, idx|
+      placeholder.store "${md[#{idx}]}", e
+    end
+
+    placeholder
+  end
+
+end

data/spec/plugin/out_rename_key_spec.rb

@@ -0,0 +1,124 @@
+require 'spec_helper'
+require 'fluent/plugin/out_rename_key'
+
+describe Fluent::RenameKeyOutput do
+  before :each do
+    Fluent::Test.setup
+  end
+
+  CONFIG = %q[
+    rename_rule1 ^\$(.+) x$${md[1]}
+    rename_rule2 ^l(eve)l(\d+) ${md[1]}_${md[2]}
+  ]
+
+  CONFIG_MULTI_RULES_FOR_A_KEY = %q[
+    rename_rule1 ^\$(.+) ${md[1]}
+    rename_rule2 ^\$(.+) ${md[1]} somthing
+  ]
+
+  CONFIG_REMOVE_TAG_PREFIX = %q[
+    rename_rule1 ^\$(.+) ${md[1]} somthing
+    remove_tag_prefix input
+  ]
+
+  CONFIG_APPEND_TAG = %q[
+    rename_rule1 ^\$(.+) ${md[1]} somthing
+    append_tag postfix
+  ]
+
+  def create_driver conf=CONFIG, tag='test'
+    Fluent::Test::OutputTestDriver.new(Fluent::RenameKeyOutput, tag).configure(conf)
+  end
+
+  context "configurations" do
+    it "raises error when no configuration" do
+      expect{create_driver ''}.to raise_error Fluent::ConfigError
+    end
+
+    it "raises error when rule is incomplete" do
+      expect{create_driver 'rename_rule1 ^$(.+?) '}.to raise_error Fluent::ConfigError
+    end
+
+    it "raises error when multiple rules are set for the same key pattern" do
+      expect{create_driver CONFIG_MULTI_RULES_FOR_A_KEY}.to raise_error Fluent::ConfigError
+    end
+
+    it "configures multiple rules" do
+      d = create_driver
+      expect(d.instance.config['rename_rule1']).to eq '^\$(.+) x$${md[1]}'
+      expect(d.instance.config['rename_rule2']).to eq '^l(eve)l(\d+) ${md[1]}_${md[2]}'
+    end
+  end
+
+  context "emits" do
+    it "removes tag prefix" do
+      d = create_driver CONFIG_REMOVE_TAG_PREFIX, 'input.test'
+      d.run { d.emit 'test' => 'data' }
+      expect(d.emits[0][0]).not_to start_with 'input'
+    end
+
+    it "appends additional tag" do
+      d = create_driver CONFIG_APPEND_TAG, 'input.test'
+      d.run { d.emit 'test' => 'data' }
+      expect(d.emits[0][0]).to eq 'input.test.postfix'
+    end
+  end
+
+  context "private methods" do
+    describe "#parse_rename_rule" do
+      let(:rename_rule_example) { '^\$(.+) ${md[1]}' }
+      let(:result) { Fluent::RenameKeyOutput.new.parse_rename_rule rename_rule_example }
+
+      it "captures 2 items, the key_regexp and new_name" do
+        expect(result).to have(2).items
+      end
+    end
+
+    describe "#rename_key" do
+      it "replace key name which matches the key_regexp at all level" do
+        d = create_driver %q[
+          rename_rule1 ^\$(.+) x$${md[1]}
+        ]
+        d.run do
+          d.emit '$url' => 'www.google.com', 'level2' => {'$1' => 'option1'}
+        end
+        result = d.emits[0][2]
+        expect(result).to have_key 'x$url'
+        expect(result['level2']).to have_key 'x$1'
+      end
+
+      it "replace key name only at the first level when deep_rename is false" do
+        d = create_driver %q[
+          rename_rule1 ^\$(.+) x$${md[1]}
+          deep_rename false
+        ]
+        d.run do
+          d.emit '$url' => 'www.google.com', 'level2' => {'$1' => 'option1'}
+        end
+        result = d.emits[0][2]
+        expect(result).to have_key 'x$url'
+        expect(result['level2']).to have_key '$1'
+      end
+
+      it "replaces key name using match data" do
+        d = create_driver 'rename_rule1 ^\$(.+)\s(\w+) x$${md[2]} ${md[1]}'
+        d.run do
+          d.emit '$url jump' => 'www.google.com', 'level2' => {'$1' => 'options1'}
+        end
+        result = d.emits[0][2]
+        expect(result).to have_key 'x$jump url'
+      end
+
+      it "replaces key using multiple rules" do
+        d = create_driver
+        d.run do
+          d.emit '$url jump' => 'www.google.com', 'level2' => {'$1' => 'options1'}
+        end
+        result = d.emits[0][2]
+        expect(result).to have_key 'eve_2'
+        expect(result['eve_2']).to have_key 'x$1'
+      end
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,24 @@
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), '..', 'lib')
+$LOAD_PATH.unshift File.dirname(__FILE__)
+
+require 'rspec'
+require 'fluent/test'
+
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end

metadata

@@ -0,0 +1,97 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-rename-key
+version: !ruby/object:Gem::Version
+  version: 0.1.2
+platform: ruby
+authors:
+- Shunwen Hsiao
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-11-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.9
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.10.9
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email: hsiaoshunwen@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Guardfile
+- LICENSE.txt
+- README.md
+- fluent-plugin-rename-key.gemspec
+- lib/fluent/plugin/out_rename_key.rb
+- spec/plugin/out_rename_key_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/shunwen/fluent-plugin-rename-key
+licenses:
+- Apache 2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Fluentd Output filter plugin. It is designed to rename keys which match given
+  patterns and assign new tags, and re-emit a record with the new tag.
+test_files:
+- spec/plugin/out_rename_key_spec.rb
+- spec/spec_helper.rb