fluent-plugin-rename-key 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +12 -0
- data/Gemfile +4 -0
- data/Guardfile +9 -0
- data/LICENSE.txt +13 -0
- data/README.md +73 -0
- data/fluent-plugin-rename-key.gemspec +23 -0
- data/lib/fluent/plugin/out_rename_key.rb +83 -0
- data/spec/plugin/out_rename_key_spec.rb +124 -0
- data/spec/spec_helper.rb +24 -0
- metadata +97 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 60e3765941b5e976e67523ec08d3339a331b3834
|
4
|
+
data.tar.gz: b28f691513dde3dd285b69067caf667a31e46beb
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: fa4c5cf24eca410e42877a6d62f1f1f48fcc45ee32b49264d4c46de9cd73c540e3596eabe4b55e51e40924a36742f09167f40526cd6a5f732990b822f32c0e79
|
7
|
+
data.tar.gz: a3e1c48fd89a1351a5145cc55ba1b24eca6ff507992c9fcb6c397ae73aad9e4fe5de60c65d4f2f686cca9d3b5254903f516b2839faa0c4cb646919266ac26392
|
data/.gitignore
ADDED
data/Gemfile
ADDED
data/Guardfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
Copyright (c) 2013- Shunwen Hsiao
|
2
|
+
|
3
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
you may not use this file except in compliance with the License.
|
5
|
+
You may obtain a copy of the License at
|
6
|
+
|
7
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
|
9
|
+
Unless required by applicable law or agreed to in writing, software
|
10
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
See the License for the specific language governing permissions and
|
13
|
+
limitations under the License.
|
data/README.md
ADDED
@@ -0,0 +1,73 @@
|
|
1
|
+
# fluent-plugin-rename-key
|
2
|
+
|
3
|
+
## Overview
|
4
|
+
|
5
|
+
Fluentd Output filter plugin. It goes through each record, rename keys matching the given regular expressions, and re-emit the event with a new tag. This plugin resembles the implementation of [fluent-plugin-rewrite-tag-filter](https://github.com/y-ken/fluent-plugin-rewrite-tag-filter).
|
6
|
+
|
7
|
+
This plugin is created to resolve the invalid record problem while converting to BSON document before inserting to MongoDB, see [restrictions on Field Names](http://docs.mongodb.org/manual/reference/limits/#Restrictions on Field Names) and [MongoDB Document Types](http://docs.mongodb.org/meta-driver/latest/legacy/bson/#mongodb-document-types).
|
8
|
+
|
9
|
+
## Installation
|
10
|
+
|
11
|
+
install with gem or fluent-gem command as:
|
12
|
+
|
13
|
+
```
|
14
|
+
# for fluentd
|
15
|
+
$ gem install fluent-plugin-rename-key
|
16
|
+
|
17
|
+
# for td-agent
|
18
|
+
$ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-rename-key
|
19
|
+
```
|
20
|
+
|
21
|
+
## Configuration
|
22
|
+
|
23
|
+
### Syntax
|
24
|
+
|
25
|
+
```
|
26
|
+
rename_rule<num> <key_regexp> <new_key>
|
27
|
+
|
28
|
+
# Optional: remove tag prefix
|
29
|
+
remove_tag_prefix <string>
|
30
|
+
|
31
|
+
# Optional: append additional name to the original tag, default **key_renamed**
|
32
|
+
append_tag <string>
|
33
|
+
|
34
|
+
# Optional: dig into the hash structures and rename every matched key or rename only keys at the first level
|
35
|
+
# default is true
|
36
|
+
deep_rename <bool>
|
37
|
+
```
|
38
|
+
|
39
|
+
### Example
|
40
|
+
|
41
|
+
Take this record as example: `'$url' => 'www.google.com', 'level2' => {'$1' => 'option1'}`.
|
42
|
+
To successfully save it into MongoDB, we can use the following config to replace the keys starting with dollar sign.
|
43
|
+
|
44
|
+
```
|
45
|
+
# At rename_rule1, it matches the key starting the `$`, say `$url`,
|
46
|
+
# and puts the following characters into match group 1.
|
47
|
+
# Then uses the content in match group 1, `url`, to generate the new key name `x$url`.
|
48
|
+
|
49
|
+
<match input.test>
|
50
|
+
type rename_key
|
51
|
+
remove_tag_prefix input.test
|
52
|
+
append_tag renamed
|
53
|
+
rename_rule1 ^\$(.+) x$${md[1]}
|
54
|
+
rename_rule2 ^l(eve)l(\d+) ${md[1]}_${md[2]}
|
55
|
+
</match>
|
56
|
+
```
|
57
|
+
|
58
|
+
The resulting record will be `'x$url' => 'www.google.com', 'eve_2' => {'x$1' => 'option1'}`
|
59
|
+
|
60
|
+
### MatchData placeholder
|
61
|
+
|
62
|
+
This plugin uses Ruby's `String#match` to match the key to be replaced, and it is possible to refer to the contents of the resulting `MatchData` to create the new key name. `${md[0]}` refers to the matched string and `${md[1]}` refers to match group 1, and so on.
|
63
|
+
|
64
|
+
**Note** Range expression ```${md[0..2]}``` is not supported.
|
65
|
+
|
66
|
+
## TODO
|
67
|
+
|
68
|
+
Pull requests are very welcome!!
|
69
|
+
|
70
|
+
## Copyright
|
71
|
+
|
72
|
+
Copyright : Copyright (c) 2013- Shunwen Hsiao (@hswtw)
|
73
|
+
License : Apache License, Version 2.0
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
$:.push File.expand_path('../lib', __FILE__)
|
3
|
+
|
4
|
+
Gem::Specification.new do |gem|
|
5
|
+
gem.name = "fluent-plugin-rename-key"
|
6
|
+
gem.version = "0.1.2"
|
7
|
+
gem.license = "Apache 2.0"
|
8
|
+
gem.authors = ["Shunwen Hsiao"]
|
9
|
+
gem.email = "hsiaoshunwen@gmail.com"
|
10
|
+
gem.homepage = "https://github.com/shunwen/fluent-plugin-rename-key"
|
11
|
+
gem.summary = %q[Fluentd Output filter plugin. It is designed to rename keys which match given patterns and assign new tags, and re-emit a record with the new tag.]
|
12
|
+
gem.has_rdoc = false
|
13
|
+
|
14
|
+
gem.files = `git ls-files`.split("\n")
|
15
|
+
gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
16
|
+
gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
17
|
+
gem.require_paths = ['lib']
|
18
|
+
|
19
|
+
gem.add_dependency "fluentd", "~> 0.10.9"
|
20
|
+
gem.add_development_dependency "rspec"
|
21
|
+
gem.add_development_dependency "bundler"
|
22
|
+
|
23
|
+
end
|
@@ -0,0 +1,83 @@
|
|
1
|
+
class Fluent::RenameKeyOutput < Fluent::Output
|
2
|
+
Fluent::Plugin.register_output 'rename_key', self
|
3
|
+
|
4
|
+
config_param :remove_tag_prefix, :string, default: nil
|
5
|
+
config_param :append_tag, :string, default: 'key_renamed'
|
6
|
+
config_param :deep_rename, :bool, default: true
|
7
|
+
|
8
|
+
def configure conf
|
9
|
+
super
|
10
|
+
|
11
|
+
@rename_rules = []
|
12
|
+
conf_rename_rules = conf.keys.select { |k| k =~ /^rename_rule(\d+)$/ }
|
13
|
+
conf_rename_rules.sort_by { |r| r.sub('rename_rule', '').to_i }.each do |r|
|
14
|
+
key_regexp, new_key = parse_rename_rule conf[r]
|
15
|
+
|
16
|
+
if key_regexp.nil? || new_key.nil?
|
17
|
+
raise Fluent::ConfigError, "Failed to parse: #{r} #{conf[r]}"
|
18
|
+
end
|
19
|
+
|
20
|
+
if @rename_rules.map { |r| r[:key_regexp] }.include? /#{key_regexp}/
|
21
|
+
raise Fluent::ConfigError, "Duplicated rules for key #{key_regexp}: #{@rename_rules}"
|
22
|
+
end
|
23
|
+
|
24
|
+
@rename_rules << { key_regexp: /#{key_regexp}/, new_key: new_key }
|
25
|
+
$log.info "Added rename key rule: #{r} #{@rename_rules.last}"
|
26
|
+
end
|
27
|
+
|
28
|
+
raise Fluent::ConfigError, "No rename rules are given" if @rename_rules.empty?
|
29
|
+
|
30
|
+
@remove_tag_prefix = /^#{Regexp.escape @remove_tag_prefix}\.?/ if @remove_tag_prefix
|
31
|
+
end
|
32
|
+
|
33
|
+
def emit tag, es, chain
|
34
|
+
es.each do |time, record|
|
35
|
+
new_tag = @remove_tag_prefix ? tag.sub(@remove_tag_prefix, '') : tag
|
36
|
+
new_tag = "#{new_tag}.#{@append_tag}".sub(/^\./, '')
|
37
|
+
new_record = rename_key record
|
38
|
+
Fluent::Engine.emit new_tag, time, new_record
|
39
|
+
end
|
40
|
+
|
41
|
+
chain.next
|
42
|
+
end
|
43
|
+
|
44
|
+
# private
|
45
|
+
|
46
|
+
def parse_rename_rule rule
|
47
|
+
if rule.match /^([^\s]+)\s+(.+)$/
|
48
|
+
return $~.captures
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
def rename_key record
|
53
|
+
new_record = {}
|
54
|
+
|
55
|
+
record.each do |key, value|
|
56
|
+
|
57
|
+
@rename_rules.each do |rule|
|
58
|
+
match_data = key.match rule[:key_regexp]
|
59
|
+
next unless match_data # next rule
|
60
|
+
|
61
|
+
placeholder = get_placeholder match_data
|
62
|
+
key = rule[:new_key].gsub /\${\w+\[\d+\]?}/, placeholder
|
63
|
+
break
|
64
|
+
end
|
65
|
+
|
66
|
+
value = rename_key value if value.is_a? Hash and @deep_rename
|
67
|
+
new_record[key] = value
|
68
|
+
end
|
69
|
+
|
70
|
+
new_record
|
71
|
+
end
|
72
|
+
|
73
|
+
def get_placeholder match_data
|
74
|
+
placeholder = {}
|
75
|
+
|
76
|
+
match_data.to_a.each_with_index do |e, idx|
|
77
|
+
placeholder.store "${md[#{idx}]}", e
|
78
|
+
end
|
79
|
+
|
80
|
+
placeholder
|
81
|
+
end
|
82
|
+
|
83
|
+
end
|
@@ -0,0 +1,124 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
require 'fluent/plugin/out_rename_key'
|
3
|
+
|
4
|
+
describe Fluent::RenameKeyOutput do
|
5
|
+
before :each do
|
6
|
+
Fluent::Test.setup
|
7
|
+
end
|
8
|
+
|
9
|
+
CONFIG = %q[
|
10
|
+
rename_rule1 ^\$(.+) x$${md[1]}
|
11
|
+
rename_rule2 ^l(eve)l(\d+) ${md[1]}_${md[2]}
|
12
|
+
]
|
13
|
+
|
14
|
+
CONFIG_MULTI_RULES_FOR_A_KEY = %q[
|
15
|
+
rename_rule1 ^\$(.+) ${md[1]}
|
16
|
+
rename_rule2 ^\$(.+) ${md[1]} somthing
|
17
|
+
]
|
18
|
+
|
19
|
+
CONFIG_REMOVE_TAG_PREFIX = %q[
|
20
|
+
rename_rule1 ^\$(.+) ${md[1]} somthing
|
21
|
+
remove_tag_prefix input
|
22
|
+
]
|
23
|
+
|
24
|
+
CONFIG_APPEND_TAG = %q[
|
25
|
+
rename_rule1 ^\$(.+) ${md[1]} somthing
|
26
|
+
append_tag postfix
|
27
|
+
]
|
28
|
+
|
29
|
+
def create_driver conf=CONFIG, tag='test'
|
30
|
+
Fluent::Test::OutputTestDriver.new(Fluent::RenameKeyOutput, tag).configure(conf)
|
31
|
+
end
|
32
|
+
|
33
|
+
context "configurations" do
|
34
|
+
it "raises error when no configuration" do
|
35
|
+
expect{create_driver ''}.to raise_error Fluent::ConfigError
|
36
|
+
end
|
37
|
+
|
38
|
+
it "raises error when rule is incomplete" do
|
39
|
+
expect{create_driver 'rename_rule1 ^$(.+?) '}.to raise_error Fluent::ConfigError
|
40
|
+
end
|
41
|
+
|
42
|
+
it "raises error when multiple rules are set for the same key pattern" do
|
43
|
+
expect{create_driver CONFIG_MULTI_RULES_FOR_A_KEY}.to raise_error Fluent::ConfigError
|
44
|
+
end
|
45
|
+
|
46
|
+
it "configures multiple rules" do
|
47
|
+
d = create_driver
|
48
|
+
expect(d.instance.config['rename_rule1']).to eq '^\$(.+) x$${md[1]}'
|
49
|
+
expect(d.instance.config['rename_rule2']).to eq '^l(eve)l(\d+) ${md[1]}_${md[2]}'
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
context "emits" do
|
54
|
+
it "removes tag prefix" do
|
55
|
+
d = create_driver CONFIG_REMOVE_TAG_PREFIX, 'input.test'
|
56
|
+
d.run { d.emit 'test' => 'data' }
|
57
|
+
expect(d.emits[0][0]).not_to start_with 'input'
|
58
|
+
end
|
59
|
+
|
60
|
+
it "appends additional tag" do
|
61
|
+
d = create_driver CONFIG_APPEND_TAG, 'input.test'
|
62
|
+
d.run { d.emit 'test' => 'data' }
|
63
|
+
expect(d.emits[0][0]).to eq 'input.test.postfix'
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
context "private methods" do
|
68
|
+
describe "#parse_rename_rule" do
|
69
|
+
let(:rename_rule_example) { '^\$(.+) ${md[1]}' }
|
70
|
+
let(:result) { Fluent::RenameKeyOutput.new.parse_rename_rule rename_rule_example }
|
71
|
+
|
72
|
+
it "captures 2 items, the key_regexp and new_name" do
|
73
|
+
expect(result).to have(2).items
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
describe "#rename_key" do
|
78
|
+
it "replace key name which matches the key_regexp at all level" do
|
79
|
+
d = create_driver %q[
|
80
|
+
rename_rule1 ^\$(.+) x$${md[1]}
|
81
|
+
]
|
82
|
+
d.run do
|
83
|
+
d.emit '$url' => 'www.google.com', 'level2' => {'$1' => 'option1'}
|
84
|
+
end
|
85
|
+
result = d.emits[0][2]
|
86
|
+
expect(result).to have_key 'x$url'
|
87
|
+
expect(result['level2']).to have_key 'x$1'
|
88
|
+
end
|
89
|
+
|
90
|
+
it "replace key name only at the first level when deep_rename is false" do
|
91
|
+
d = create_driver %q[
|
92
|
+
rename_rule1 ^\$(.+) x$${md[1]}
|
93
|
+
deep_rename false
|
94
|
+
]
|
95
|
+
d.run do
|
96
|
+
d.emit '$url' => 'www.google.com', 'level2' => {'$1' => 'option1'}
|
97
|
+
end
|
98
|
+
result = d.emits[0][2]
|
99
|
+
expect(result).to have_key 'x$url'
|
100
|
+
expect(result['level2']).to have_key '$1'
|
101
|
+
end
|
102
|
+
|
103
|
+
it "replaces key name using match data" do
|
104
|
+
d = create_driver 'rename_rule1 ^\$(.+)\s(\w+) x$${md[2]} ${md[1]}'
|
105
|
+
d.run do
|
106
|
+
d.emit '$url jump' => 'www.google.com', 'level2' => {'$1' => 'options1'}
|
107
|
+
end
|
108
|
+
result = d.emits[0][2]
|
109
|
+
expect(result).to have_key 'x$jump url'
|
110
|
+
end
|
111
|
+
|
112
|
+
it "replaces key using multiple rules" do
|
113
|
+
d = create_driver
|
114
|
+
d.run do
|
115
|
+
d.emit '$url jump' => 'www.google.com', 'level2' => {'$1' => 'options1'}
|
116
|
+
end
|
117
|
+
result = d.emits[0][2]
|
118
|
+
expect(result).to have_key 'eve_2'
|
119
|
+
expect(result['eve_2']).to have_key 'x$1'
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
require 'bundler'
|
2
|
+
begin
|
3
|
+
Bundler.setup(:default, :development)
|
4
|
+
rescue Bundler::BundlerError => e
|
5
|
+
$stderr.puts e.message
|
6
|
+
$stderr.puts "Run `bundle install` to install missing gems"
|
7
|
+
exit e.status_code
|
8
|
+
end
|
9
|
+
|
10
|
+
$LOAD_PATH.unshift File.join(File.dirname(__FILE__), '..', 'lib')
|
11
|
+
$LOAD_PATH.unshift File.dirname(__FILE__)
|
12
|
+
|
13
|
+
require 'rspec'
|
14
|
+
require 'fluent/test'
|
15
|
+
|
16
|
+
unless ENV.has_key?('VERBOSE')
|
17
|
+
nulllogger = Object.new
|
18
|
+
nulllogger.instance_eval {|obj|
|
19
|
+
def method_missing(method, *args)
|
20
|
+
# pass
|
21
|
+
end
|
22
|
+
}
|
23
|
+
$log = nulllogger
|
24
|
+
end
|
metadata
ADDED
@@ -0,0 +1,97 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: fluent-plugin-rename-key
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.2
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Shunwen Hsiao
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2013-11-14 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: fluentd
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - ~>
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 0.10.9
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ~>
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: 0.10.9
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rspec
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - '>='
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - '>='
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: bundler
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - '>='
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - '>='
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '0'
|
55
|
+
description:
|
56
|
+
email: hsiaoshunwen@gmail.com
|
57
|
+
executables: []
|
58
|
+
extensions: []
|
59
|
+
extra_rdoc_files: []
|
60
|
+
files:
|
61
|
+
- .gitignore
|
62
|
+
- Gemfile
|
63
|
+
- Guardfile
|
64
|
+
- LICENSE.txt
|
65
|
+
- README.md
|
66
|
+
- fluent-plugin-rename-key.gemspec
|
67
|
+
- lib/fluent/plugin/out_rename_key.rb
|
68
|
+
- spec/plugin/out_rename_key_spec.rb
|
69
|
+
- spec/spec_helper.rb
|
70
|
+
homepage: https://github.com/shunwen/fluent-plugin-rename-key
|
71
|
+
licenses:
|
72
|
+
- Apache 2.0
|
73
|
+
metadata: {}
|
74
|
+
post_install_message:
|
75
|
+
rdoc_options: []
|
76
|
+
require_paths:
|
77
|
+
- lib
|
78
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - '>='
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '0'
|
83
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
84
|
+
requirements:
|
85
|
+
- - '>='
|
86
|
+
- !ruby/object:Gem::Version
|
87
|
+
version: '0'
|
88
|
+
requirements: []
|
89
|
+
rubyforge_project:
|
90
|
+
rubygems_version: 2.0.3
|
91
|
+
signing_key:
|
92
|
+
specification_version: 4
|
93
|
+
summary: Fluentd Output filter plugin. It is designed to rename keys which match given
|
94
|
+
patterns and assign new tags, and re-emit a record with the new tag.
|
95
|
+
test_files:
|
96
|
+
- spec/plugin/out_rename_key_spec.rb
|
97
|
+
- spec/spec_helper.rb
|