fluent-plugin-rename-key 0.1.3 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +5 -2
- data/README.md +14 -12
- data/Rakefile +8 -8
- data/fluent-plugin-rename-key.gemspec +18 -19
- data/lib/fluent/plugin/out_rename_key.rb +8 -2
- data/{spec/spec_helper.rb → test/helper.rb} +9 -9
- data/test/plugin/test_out_rename_key.rb +159 -0
- metadata +26 -40
- data/spec/plugin/out_rename_key_spec.rb +0 -134
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 50c9ff6f2cafefef879771c9ae58aa30370fc3b4
|
4
|
+
data.tar.gz: 753cd2ad645d97a697c797798802d536c561e0f8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 460eb66318d43d083703f68058db5bbeb98d61926a4d3eeef53808babbbd8f6aab22a7637aa4fe4f73d2ffeb11a8b10c95bbb57ea4d57a1a886806505a093346
|
7
|
+
data.tar.gz: e5e733b1d8f6ce128f385d8b9d6ecc27c7ecbbc1f399e33d97c6d2943162f2315c03dfa8d3a436a56ada1fef3efc0e8f136d8af2a762267598e64d43925e06d2
|
data/.travis.yml
CHANGED
data/README.md
CHANGED
@@ -1,19 +1,20 @@
|
|
1
|
-
# fluent-plugin-rename-key
|
1
|
+
# fluent-plugin-rename-key, a plugin for [Fluentd](http://fluentd.org)
|
2
2
|
|
3
3
|
## Status
|
4
4
|
[](http://badge.fury.io/rb/fluent-plugin-rename-key)
|
5
5
|
[](https://travis-ci.org/shunwen/fluent-plugin-rename-key)
|
6
6
|
[](https://coveralls.io/r/shunwen/fluent-plugin-rename-key?branch=master)
|
7
|
+
[](https://bitdeli.com/free "Bitdeli Badge")
|
7
8
|
|
8
9
|
## Overview
|
9
10
|
|
10
|
-
Fluentd
|
11
|
+
Fluentd output plugin. Renames keys matching the given regular expressions, assign new tags, and re-emits. This plugin resembles the implementation of [fluent-plugin-rewrite-tag-filter](https://github.com/y-ken/fluent-plugin-rewrite-tag-filter).
|
11
12
|
|
12
|
-
This plugin is created to resolve the invalid record problem while converting to BSON document before inserting to MongoDB, see [Restrictions on Field Names](http://docs.mongodb.org/manual/reference/limits/#Restrictions on Field Names) and [MongoDB Document Types](http://docs.mongodb.org/meta-driver/latest/legacy/bson/#mongodb-document-types).
|
13
|
+
This plugin is created to resolve the invalid record problem while converting to BSON document before inserting to MongoDB, see [Restrictions on Field Names](http://docs.mongodb.org/manual/reference/limits/#Restrictions on Field Names) and [MongoDB Document Types](http://docs.mongodb.org/meta-driver/latest/legacy/bson/#mongodb-document-types) for more information.
|
13
14
|
|
14
15
|
## Installation
|
15
16
|
|
16
|
-
|
17
|
+
Install with gem or fluent-gem command as:
|
17
18
|
|
18
19
|
```
|
19
20
|
# for fluentd
|
@@ -31,7 +32,7 @@ $ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-rename-key
|
|
31
32
|
### Syntax
|
32
33
|
|
33
34
|
```
|
34
|
-
# <num> is
|
35
|
+
# <num> is an integer, used to sort and apply the rules
|
35
36
|
# <key_regexp> is the regular expression used to match the keys, whitespace is not allowed, use "\s" instead
|
36
37
|
# <new_key> is the string with MatchData placeholder for creating the new key name, whitespace is allowed
|
37
38
|
rename_rule<num> <key_regexp> <new_key>
|
@@ -39,11 +40,11 @@ rename_rule<num> <key_regexp> <new_key>
|
|
39
40
|
# Optional: remove tag prefix
|
40
41
|
remove_tag_prefix <string>
|
41
42
|
|
42
|
-
# Optional: append additional name to the original tag, default
|
43
|
+
# Optional: append additional name to the original tag, default is "key_renamed"
|
43
44
|
append_tag <string>
|
44
45
|
|
45
|
-
# Optional: dig into the hash
|
46
|
-
# default is true
|
46
|
+
# Optional: dig into the hash structure and rename every matched key, or rename only keys at the first level,
|
47
|
+
# default is "true"
|
47
48
|
deep_rename <bool>
|
48
49
|
```
|
49
50
|
|
@@ -53,9 +54,9 @@ Take this record as example: `'$url' => 'www.google.com', 'level2' => {'$1' => '
|
|
53
54
|
To successfully save it into MongoDB, we can use the following config to replace the keys starting with dollar sign.
|
54
55
|
|
55
56
|
```
|
56
|
-
# At rename_rule1, it matches the key starting the
|
57
|
+
# At rename_rule1, it matches the key starting the '$', say '$url',
|
57
58
|
# and puts the following characters into match group 1.
|
58
|
-
# Then uses the content in match group 1,
|
59
|
+
# Then uses the content in match group 1, ${md[1]} = 'url', to generate the new key name 'x$url'.
|
59
60
|
|
60
61
|
<match input.test>
|
61
62
|
type rename_key
|
@@ -66,13 +67,13 @@ To successfully save it into MongoDB, we can use the following config to replace
|
|
66
67
|
</match>
|
67
68
|
```
|
68
69
|
|
69
|
-
The resulting record
|
70
|
+
The resulting record is `'x$url' => 'www.google.com', 'eve_2' => {'x$1' => 'option1'}` with new tag `renamed`.
|
70
71
|
|
71
72
|
### MatchData placeholder
|
72
73
|
|
73
74
|
This plugin uses Ruby's `String#match` to match the key to be replaced, and it is possible to refer to the contents of the resulting `MatchData` to create the new key name. `${md[0]}` refers to the matched string and `${md[1]}` refers to match group 1, and so on.
|
74
75
|
|
75
|
-
**Note:** Range expression
|
76
|
+
**Note:** Range expression `${md[0..2]}` is not supported.
|
76
77
|
|
77
78
|
## TODO
|
78
79
|
|
@@ -82,3 +83,4 @@ Pull requests are very welcome!!
|
|
82
83
|
|
83
84
|
Copyright : Copyright (c) 2013- Shunwen Hsiao (@hswtw)
|
84
85
|
License : Apache License, Version 2.0
|
86
|
+
|
data/Rakefile
CHANGED
@@ -1,9 +1,9 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
end
|
8
|
-
rescue LoadError => e
|
1
|
+
require "bundler/gem_tasks"
|
2
|
+
require "rake/testtask"
|
3
|
+
Rake::TestTask.new(:test) do |test|
|
4
|
+
test.libs << 'lib' << 'test'
|
5
|
+
test.pattern = 'test/**/test_*.rb'
|
6
|
+
test.verbose = false
|
9
7
|
end
|
8
|
+
|
9
|
+
task :default => :test
|
@@ -1,25 +1,24 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
$:.push File.expand_path('../lib', __FILE__)
|
3
3
|
|
4
|
-
Gem::Specification.new do |
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
4
|
+
Gem::Specification.new do |s|
|
5
|
+
s.name = "fluent-plugin-rename-key"
|
6
|
+
s.version = "0.1.4"
|
7
|
+
s.date = "2016-04-04"
|
8
|
+
s.license = "Apache-2.0"
|
9
|
+
s.authors = ["Shunwen Hsiao"]
|
10
|
+
s.email = "hsiaoshunwen@gmail.com"
|
11
|
+
s.homepage = "https://github.com/shunwen/fluent-plugin-rename-key"
|
12
|
+
s.summary = %q[Fluentd output plugin. Rename keys which match given regular expressions, assign new tags and re-emit the records.]
|
13
|
+
s.has_rdoc = false
|
13
14
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
gem.add_dependency "fluentd", "~> 0.10.9"
|
20
|
-
gem.add_development_dependency "rspec"
|
21
|
-
gem.add_development_dependency "bundler"
|
22
|
-
gem.add_development_dependency "rake"
|
23
|
-
gem.add_development_dependency 'coveralls'
|
15
|
+
s.files = `git ls-files`.split("\n")
|
16
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
17
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
18
|
+
s.require_paths = ['lib']
|
24
19
|
|
20
|
+
s.add_runtime_dependency "fluentd"
|
21
|
+
s.add_development_dependency "test-unit", ">= 3.1.0"
|
22
|
+
s.add_development_dependency "rake"
|
23
|
+
s.add_development_dependency 'coveralls'
|
25
24
|
end
|
@@ -1,10 +1,16 @@
|
|
1
1
|
class Fluent::RenameKeyOutput < Fluent::Output
|
2
2
|
Fluent::Plugin.register_output 'rename_key', self
|
3
3
|
|
4
|
+
DEFAULT_APPEND_TAG = 'key_renamed'
|
5
|
+
|
4
6
|
config_param :remove_tag_prefix, :string, default: nil
|
5
|
-
config_param :append_tag, :string, default:
|
7
|
+
config_param :append_tag, :string, default: DEFAULT_APPEND_TAG
|
6
8
|
config_param :deep_rename, :bool, default: true
|
7
9
|
|
10
|
+
unless method_defined?(:router)
|
11
|
+
define_method("router") { Fluent::Engine }
|
12
|
+
end
|
13
|
+
|
8
14
|
def configure conf
|
9
15
|
super
|
10
16
|
|
@@ -35,7 +41,7 @@ class Fluent::RenameKeyOutput < Fluent::Output
|
|
35
41
|
new_tag = @remove_tag_prefix ? tag.sub(@remove_tag_prefix, '') : tag
|
36
42
|
new_tag = "#{new_tag}.#{@append_tag}".sub(/^\./, '')
|
37
43
|
new_record = rename_key record
|
38
|
-
|
44
|
+
router.emit new_tag, time, new_record
|
39
45
|
end
|
40
46
|
|
41
47
|
chain.next
|
@@ -1,3 +1,4 @@
|
|
1
|
+
require 'rubygems'
|
1
2
|
require 'bundler'
|
2
3
|
begin
|
3
4
|
Bundler.setup(:default, :development)
|
@@ -6,17 +7,11 @@ rescue Bundler::BundlerError => e
|
|
6
7
|
$stderr.puts "Run `bundle install` to install missing gems"
|
7
8
|
exit e.status_code
|
8
9
|
end
|
10
|
+
require 'test/unit'
|
9
11
|
|
10
|
-
$LOAD_PATH.unshift
|
11
|
-
$LOAD_PATH.unshift
|
12
|
-
|
13
|
-
require 'rspec'
|
14
|
-
require 'coveralls'
|
15
|
-
Coveralls.wear!
|
16
|
-
|
17
|
-
ARGV = []
|
12
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
|
13
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
18
14
|
require 'fluent/test'
|
19
|
-
|
20
15
|
unless ENV.has_key?('VERBOSE')
|
21
16
|
nulllogger = Object.new
|
22
17
|
nulllogger.instance_eval {|obj|
|
@@ -26,3 +21,8 @@ unless ENV.has_key?('VERBOSE')
|
|
26
21
|
}
|
27
22
|
$log = nulllogger
|
28
23
|
end
|
24
|
+
|
25
|
+
require 'fluent/plugin/out_rename_key'
|
26
|
+
|
27
|
+
class Test::Unit::TestCase
|
28
|
+
end
|
@@ -0,0 +1,159 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
class RenameKeyOutputTest < Test::Unit::TestCase
|
4
|
+
MATCH_TAG = 'incoming_tag'
|
5
|
+
CONFIG = 'rename_rule1 ^\$(.+) x$${md[1]}'
|
6
|
+
|
7
|
+
def setup
|
8
|
+
Fluent::Test.setup
|
9
|
+
end
|
10
|
+
|
11
|
+
def create_driver(conf = CONFIG, tag = MATCH_TAG)
|
12
|
+
Fluent::Test::OutputTestDriver.new(Fluent::RenameKeyOutput, tag).configure(conf)
|
13
|
+
end
|
14
|
+
|
15
|
+
def test_config_error
|
16
|
+
assert_raise(Fluent::ConfigError) { create_driver('') }
|
17
|
+
|
18
|
+
assert_raise(Fluent::ConfigError) { create_driver('rename_rule1 ^$(.+?) ') }
|
19
|
+
|
20
|
+
assert_raise(Fluent::ConfigError) {
|
21
|
+
config_dup_rules_for_a_key = %q[
|
22
|
+
rename_rule1 ^\$(.+) ${md[1]}
|
23
|
+
rename_rule2 ^\$(.+) ${md[1]} something
|
24
|
+
]
|
25
|
+
create_driver(config_dup_rules_for_a_key)
|
26
|
+
}
|
27
|
+
end
|
28
|
+
|
29
|
+
def test_config_success
|
30
|
+
config_multiple_rules = %q[
|
31
|
+
rename_rule1 ^\$(.+)1 x$${md[1]}
|
32
|
+
rename_rule2 ^\$(.+)2(\d+) ${md[1]}_${md[2]}
|
33
|
+
]
|
34
|
+
|
35
|
+
d = create_driver config_multiple_rules
|
36
|
+
assert_equal '^\$(.+)1 x$${md[1]}', d.instance.config['rename_rule1']
|
37
|
+
assert_equal '^\$(.+)2(\d+) ${md[1]}_${md[2]}', d.instance.config['rename_rule2']
|
38
|
+
end
|
39
|
+
|
40
|
+
def test_parse_rename_rule
|
41
|
+
parsed = Fluent::RenameKeyOutput.new.parse_rename_rule '(reg)(exp) ${md[1]} ${md[2]}'
|
42
|
+
assert_equal 2, parsed.length
|
43
|
+
assert_equal '(reg)(exp)', parsed[0]
|
44
|
+
assert_equal '${md[1]} ${md[2]}', parsed[1]
|
45
|
+
end
|
46
|
+
|
47
|
+
def test_emit_default_append_tag
|
48
|
+
append_tag = Fluent::RenameKeyOutput::DEFAULT_APPEND_TAG
|
49
|
+
d = create_driver
|
50
|
+
d.run do
|
51
|
+
d.emit '$key1' => 'value1', '%key2' => {'$key3'=>'123', '$key4'=> {'$key5' => 'value2'} }
|
52
|
+
end
|
53
|
+
|
54
|
+
emits = d.emits
|
55
|
+
assert_equal 1, emits.length
|
56
|
+
assert_equal "#{MATCH_TAG}.#{append_tag}", emits[0][0]
|
57
|
+
assert_equal ['x$key1', '%key2'], emits[0][2].keys
|
58
|
+
end
|
59
|
+
|
60
|
+
def test_emit_append_custom_tag
|
61
|
+
custom_tag = 'custom_tag'
|
62
|
+
config = %Q[
|
63
|
+
#{CONFIG}
|
64
|
+
append_tag #{custom_tag}
|
65
|
+
]
|
66
|
+
d = create_driver config
|
67
|
+
|
68
|
+
d.run do
|
69
|
+
d.emit '$key1' => 'value1', '%key2' => {'$key3'=>'123', '$key4'=> {'$key5' => 'value2'} }
|
70
|
+
end
|
71
|
+
|
72
|
+
emits = d.emits
|
73
|
+
assert_equal 1, emits.length
|
74
|
+
assert_equal "#{MATCH_TAG}.#{custom_tag}", emits[0][0]
|
75
|
+
assert_equal ['x$key1', '%key2'], emits[0][2].keys
|
76
|
+
end
|
77
|
+
|
78
|
+
def test_emit_deep_rename_hash
|
79
|
+
d = create_driver
|
80
|
+
d.run do
|
81
|
+
d.emit '$key1' => 'value1', '%key2' => {'$key3'=>'123', '$key4'=> {'$key5' => 'value2'} }
|
82
|
+
end
|
83
|
+
|
84
|
+
emits = d.emits
|
85
|
+
assert_equal ['x$key3', 'x$key4'], emits[0][2]['%key2'].keys
|
86
|
+
assert_equal ['x$key5'], emits[0][2]['%key2']['x$key4'].keys
|
87
|
+
end
|
88
|
+
|
89
|
+
def test_emit_deep_rename_array
|
90
|
+
d = create_driver
|
91
|
+
d.run do
|
92
|
+
d.emit '$key1' => 'value1', '%key2' => [{'$key3'=>'123'}, {'$key4'=> {'$key5' => 'value2'}}]
|
93
|
+
end
|
94
|
+
|
95
|
+
emits = d.emits
|
96
|
+
assert_equal ['x$key3', 'x$key4'], emits[0][2]['%key2'].flat_map(&:keys)
|
97
|
+
assert_equal ['x$key5'], emits[0][2]['%key2'][1]['x$key4'].keys
|
98
|
+
end
|
99
|
+
|
100
|
+
def test_emit_deep_rename_off
|
101
|
+
config = %Q[
|
102
|
+
#{CONFIG}
|
103
|
+
deep_rename false
|
104
|
+
]
|
105
|
+
|
106
|
+
d = create_driver config
|
107
|
+
d.run do
|
108
|
+
d.emit '$key1' => 'value1', '%key2' => {'$key3'=>'123', '$key4'=> {'$key5' => 'value2'} }
|
109
|
+
end
|
110
|
+
|
111
|
+
emits = d.emits
|
112
|
+
assert_equal ['$key3', '$key4'], emits[0][2]['%key2'].keys
|
113
|
+
end
|
114
|
+
|
115
|
+
def test_remove_tag_prefix
|
116
|
+
append_tag = Fluent::RenameKeyOutput::DEFAULT_APPEND_TAG
|
117
|
+
|
118
|
+
config = %Q[
|
119
|
+
#{CONFIG}
|
120
|
+
remove_tag_prefix #{MATCH_TAG}
|
121
|
+
]
|
122
|
+
|
123
|
+
d = create_driver config
|
124
|
+
d.run do
|
125
|
+
d.emit 'key1' => 'value1'
|
126
|
+
d.emit '$key2' => 'value2'
|
127
|
+
end
|
128
|
+
|
129
|
+
emits = d.emits
|
130
|
+
assert_equal 2, emits.length
|
131
|
+
assert_equal append_tag, emits[0][0]
|
132
|
+
assert_equal append_tag, emits[1][0]
|
133
|
+
end
|
134
|
+
|
135
|
+
def test_rename_with_match_data
|
136
|
+
d = create_driver 'rename_rule1 (\w+)\s(\w+)\s(\w+) ${md[3]} ${md[2]} ${md[1]}'
|
137
|
+
d.run do
|
138
|
+
d.emit 'key1 key2 key3' => 'value'
|
139
|
+
end
|
140
|
+
emits = d.emits
|
141
|
+
assert_equal 1, emits.length
|
142
|
+
assert_equal ['key3 key2 key1'], emits[0][2].keys
|
143
|
+
end
|
144
|
+
|
145
|
+
def test_rename_with_multiple_rules
|
146
|
+
config_multiple_rules = %q[
|
147
|
+
rename_rule1 ^(\w+)\s1 ${md[1]}_1
|
148
|
+
rename_rule2 ^(\w+)\s2 ${md[1]}_2
|
149
|
+
]
|
150
|
+
|
151
|
+
d = create_driver config_multiple_rules
|
152
|
+
d.run do
|
153
|
+
d.emit 'key 1' => 'value1', 'key 2' => 'value2'
|
154
|
+
end
|
155
|
+
|
156
|
+
emits = d.emits
|
157
|
+
assert_equal ['key_1', 'key_2'], emits[0][2].keys
|
158
|
+
end
|
159
|
+
end
|
metadata
CHANGED
@@ -1,83 +1,69 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-rename-key
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shunwen Hsiao
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2016-04-04 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- -
|
18
|
-
- !ruby/object:Gem::Version
|
19
|
-
version: 0.10.9
|
20
|
-
type: :runtime
|
21
|
-
prerelease: false
|
22
|
-
version_requirements: !ruby/object:Gem::Requirement
|
23
|
-
requirements:
|
24
|
-
- - ~>
|
25
|
-
- !ruby/object:Gem::Version
|
26
|
-
version: 0.10.9
|
27
|
-
- !ruby/object:Gem::Dependency
|
28
|
-
name: rspec
|
29
|
-
requirement: !ruby/object:Gem::Requirement
|
30
|
-
requirements:
|
31
|
-
- - '>='
|
17
|
+
- - ">="
|
32
18
|
- !ruby/object:Gem::Version
|
33
19
|
version: '0'
|
34
|
-
type: :
|
20
|
+
type: :runtime
|
35
21
|
prerelease: false
|
36
22
|
version_requirements: !ruby/object:Gem::Requirement
|
37
23
|
requirements:
|
38
|
-
- -
|
24
|
+
- - ">="
|
39
25
|
- !ruby/object:Gem::Version
|
40
26
|
version: '0'
|
41
27
|
- !ruby/object:Gem::Dependency
|
42
|
-
name:
|
28
|
+
name: test-unit
|
43
29
|
requirement: !ruby/object:Gem::Requirement
|
44
30
|
requirements:
|
45
|
-
- -
|
31
|
+
- - ">="
|
46
32
|
- !ruby/object:Gem::Version
|
47
|
-
version:
|
33
|
+
version: 3.1.0
|
48
34
|
type: :development
|
49
35
|
prerelease: false
|
50
36
|
version_requirements: !ruby/object:Gem::Requirement
|
51
37
|
requirements:
|
52
|
-
- -
|
38
|
+
- - ">="
|
53
39
|
- !ruby/object:Gem::Version
|
54
|
-
version:
|
40
|
+
version: 3.1.0
|
55
41
|
- !ruby/object:Gem::Dependency
|
56
42
|
name: rake
|
57
43
|
requirement: !ruby/object:Gem::Requirement
|
58
44
|
requirements:
|
59
|
-
- -
|
45
|
+
- - ">="
|
60
46
|
- !ruby/object:Gem::Version
|
61
47
|
version: '0'
|
62
48
|
type: :development
|
63
49
|
prerelease: false
|
64
50
|
version_requirements: !ruby/object:Gem::Requirement
|
65
51
|
requirements:
|
66
|
-
- -
|
52
|
+
- - ">="
|
67
53
|
- !ruby/object:Gem::Version
|
68
54
|
version: '0'
|
69
55
|
- !ruby/object:Gem::Dependency
|
70
56
|
name: coveralls
|
71
57
|
requirement: !ruby/object:Gem::Requirement
|
72
58
|
requirements:
|
73
|
-
- -
|
59
|
+
- - ">="
|
74
60
|
- !ruby/object:Gem::Version
|
75
61
|
version: '0'
|
76
62
|
type: :development
|
77
63
|
prerelease: false
|
78
64
|
version_requirements: !ruby/object:Gem::Requirement
|
79
65
|
requirements:
|
80
|
-
- -
|
66
|
+
- - ">="
|
81
67
|
- !ruby/object:Gem::Version
|
82
68
|
version: '0'
|
83
69
|
description:
|
@@ -86,8 +72,8 @@ executables: []
|
|
86
72
|
extensions: []
|
87
73
|
extra_rdoc_files: []
|
88
74
|
files:
|
89
|
-
- .gitignore
|
90
|
-
- .travis.yml
|
75
|
+
- ".gitignore"
|
76
|
+
- ".travis.yml"
|
91
77
|
- Gemfile
|
92
78
|
- Guardfile
|
93
79
|
- LICENSE.txt
|
@@ -95,11 +81,11 @@ files:
|
|
95
81
|
- Rakefile
|
96
82
|
- fluent-plugin-rename-key.gemspec
|
97
83
|
- lib/fluent/plugin/out_rename_key.rb
|
98
|
-
-
|
99
|
-
-
|
84
|
+
- test/helper.rb
|
85
|
+
- test/plugin/test_out_rename_key.rb
|
100
86
|
homepage: https://github.com/shunwen/fluent-plugin-rename-key
|
101
87
|
licenses:
|
102
|
-
- Apache
|
88
|
+
- Apache-2.0
|
103
89
|
metadata: {}
|
104
90
|
post_install_message:
|
105
91
|
rdoc_options: []
|
@@ -107,21 +93,21 @@ require_paths:
|
|
107
93
|
- lib
|
108
94
|
required_ruby_version: !ruby/object:Gem::Requirement
|
109
95
|
requirements:
|
110
|
-
- -
|
96
|
+
- - ">="
|
111
97
|
- !ruby/object:Gem::Version
|
112
98
|
version: '0'
|
113
99
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
114
100
|
requirements:
|
115
|
-
- -
|
101
|
+
- - ">="
|
116
102
|
- !ruby/object:Gem::Version
|
117
103
|
version: '0'
|
118
104
|
requirements: []
|
119
105
|
rubyforge_project:
|
120
|
-
rubygems_version: 2.
|
106
|
+
rubygems_version: 2.6.2
|
121
107
|
signing_key:
|
122
108
|
specification_version: 4
|
123
|
-
summary: Fluentd
|
124
|
-
|
109
|
+
summary: Fluentd output plugin. Rename keys which match given regular expressions,
|
110
|
+
assign new tags and re-emit the records.
|
125
111
|
test_files:
|
126
|
-
-
|
127
|
-
-
|
112
|
+
- test/helper.rb
|
113
|
+
- test/plugin/test_out_rename_key.rb
|
@@ -1,134 +0,0 @@
|
|
1
|
-
require 'spec_helper'
|
2
|
-
require 'fluent/plugin/out_rename_key'
|
3
|
-
|
4
|
-
describe Fluent::RenameKeyOutput do
|
5
|
-
before :each do
|
6
|
-
Fluent::Test.setup
|
7
|
-
end
|
8
|
-
|
9
|
-
CONFIG = %q[
|
10
|
-
rename_rule1 ^\$(.+) x$${md[1]}
|
11
|
-
rename_rule2 ^l(eve)l(\d+) ${md[1]}_${md[2]}
|
12
|
-
]
|
13
|
-
|
14
|
-
CONFIG_MULTI_RULES_FOR_A_KEY = %q[
|
15
|
-
rename_rule1 ^\$(.+) ${md[1]}
|
16
|
-
rename_rule2 ^\$(.+) ${md[1]} somthing
|
17
|
-
]
|
18
|
-
|
19
|
-
CONFIG_REMOVE_TAG_PREFIX = %q[
|
20
|
-
rename_rule1 ^\$(.+) ${md[1]} somthing
|
21
|
-
remove_tag_prefix input
|
22
|
-
]
|
23
|
-
|
24
|
-
CONFIG_APPEND_TAG = %q[
|
25
|
-
rename_rule1 ^\$(.+) ${md[1]} somthing
|
26
|
-
append_tag postfix
|
27
|
-
]
|
28
|
-
|
29
|
-
def create_driver conf=CONFIG, tag='test'
|
30
|
-
Fluent::Test::OutputTestDriver.new(Fluent::RenameKeyOutput, tag).configure(conf)
|
31
|
-
end
|
32
|
-
|
33
|
-
context "configurations" do
|
34
|
-
it "raises error when no configuration" do
|
35
|
-
expect{create_driver ''}.to raise_error Fluent::ConfigError
|
36
|
-
end
|
37
|
-
|
38
|
-
it "raises error when rule is incomplete" do
|
39
|
-
expect{create_driver 'rename_rule1 ^$(.+?) '}.to raise_error Fluent::ConfigError
|
40
|
-
end
|
41
|
-
|
42
|
-
it "raises error when multiple rules are set for the same key pattern" do
|
43
|
-
expect{create_driver CONFIG_MULTI_RULES_FOR_A_KEY}.to raise_error Fluent::ConfigError
|
44
|
-
end
|
45
|
-
|
46
|
-
it "configures multiple rules" do
|
47
|
-
d = create_driver
|
48
|
-
expect(d.instance.config['rename_rule1']).to eq '^\$(.+) x$${md[1]}'
|
49
|
-
expect(d.instance.config['rename_rule2']).to eq '^l(eve)l(\d+) ${md[1]}_${md[2]}'
|
50
|
-
end
|
51
|
-
end
|
52
|
-
|
53
|
-
context "emits" do
|
54
|
-
it "removes tag prefix" do
|
55
|
-
d = create_driver CONFIG_REMOVE_TAG_PREFIX, 'input.test'
|
56
|
-
d.run { d.emit 'test' => 'data' }
|
57
|
-
expect(d.emits[0][0]).not_to start_with 'input'
|
58
|
-
end
|
59
|
-
|
60
|
-
it "appends additional tag" do
|
61
|
-
d = create_driver CONFIG_APPEND_TAG, 'input.test'
|
62
|
-
d.run { d.emit 'test' => 'data' }
|
63
|
-
expect(d.emits[0][0]).to eq 'input.test.postfix'
|
64
|
-
end
|
65
|
-
end
|
66
|
-
|
67
|
-
context "private methods" do
|
68
|
-
describe "#parse_rename_rule" do
|
69
|
-
let(:rename_rule_example) { '^\$(.+) ${md[1]}' }
|
70
|
-
let(:result) { Fluent::RenameKeyOutput.new.parse_rename_rule rename_rule_example }
|
71
|
-
|
72
|
-
it "captures 2 items, the key_regexp and new_name" do
|
73
|
-
expect(result).to have(2).items
|
74
|
-
end
|
75
|
-
end
|
76
|
-
|
77
|
-
describe "#rename_key" do
|
78
|
-
it "replace key name which matches the key_regexp at all level" do
|
79
|
-
d = create_driver %q[
|
80
|
-
rename_rule1 ^\$(.+) x$${md[1]}
|
81
|
-
]
|
82
|
-
d.run do
|
83
|
-
d.emit '$url' => 'www.google.com', 'level2' => {'$1' => 'option1'}
|
84
|
-
end
|
85
|
-
result = d.emits[0][2]
|
86
|
-
expect(result).to have_key 'x$url'
|
87
|
-
expect(result['level2']).to have_key 'x$1'
|
88
|
-
end
|
89
|
-
|
90
|
-
it "replace key name only at the first level when deep_rename is false" do
|
91
|
-
d = create_driver %q[
|
92
|
-
rename_rule1 ^\$(.+) x$${md[1]}
|
93
|
-
deep_rename false
|
94
|
-
]
|
95
|
-
d.run do
|
96
|
-
d.emit '$url' => 'www.google.com', 'level2' => {'$1' => 'option1'}
|
97
|
-
end
|
98
|
-
result = d.emits[0][2]
|
99
|
-
expect(result).to have_key 'x$url'
|
100
|
-
expect(result['level2']).to have_key '$1'
|
101
|
-
end
|
102
|
-
|
103
|
-
it "replace key of hashes in an array" do
|
104
|
-
d = create_driver 'rename_rule1 ^\$(.+)\s(\w+) x$${md[2]} ${md[1]}'
|
105
|
-
d.run do
|
106
|
-
d.emit 'array' => [{'$url jump' => 'www.google.com'}, {'$url run' => 'www.google.com'}], 'level2' => {'$1' => 'options1'}
|
107
|
-
end
|
108
|
-
result = d.emits[0][2]
|
109
|
-
expect(result['array'][0]).to have_key 'x$jump url'
|
110
|
-
expect(result['array'][1]).to have_key 'x$run url'
|
111
|
-
end
|
112
|
-
|
113
|
-
it "replaces key name using match data" do
|
114
|
-
d = create_driver 'rename_rule1 ^\$(.+)\s(\w+) x$${md[2]} ${md[1]}'
|
115
|
-
d.run do
|
116
|
-
d.emit '$url jump' => 'www.google.com', 'level2' => {'$1' => 'options1'}
|
117
|
-
end
|
118
|
-
result = d.emits[0][2]
|
119
|
-
expect(result).to have_key 'x$jump url'
|
120
|
-
end
|
121
|
-
|
122
|
-
it "replaces key using multiple rules" do
|
123
|
-
d = create_driver
|
124
|
-
d.run do
|
125
|
-
d.emit '$url jump' => 'www.google.com', 'level2' => {'$1' => 'options1'}
|
126
|
-
end
|
127
|
-
result = d.emits[0][2]
|
128
|
-
expect(result).to have_key 'eve_2'
|
129
|
-
expect(result['eve_2']).to have_key 'x$1'
|
130
|
-
end
|
131
|
-
end
|
132
|
-
end
|
133
|
-
|
134
|
-
end
|