fluent-plugin-rename-key 0.2.0 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +0 -1
- data/README.md +5 -0
- data/fluent-plugin-rename-key.gemspec +8 -9
- data/lib/fluent/plugin/filter_rename_key.rb +25 -0
- data/lib/fluent/plugin/out_rename_key.rb +6 -114
- data/lib/fluent/plugin/rename_key_util.rb +119 -0
- data/test/helper.rb +3 -0
- data/test/plugin/test_filter_rename_key.rb +215 -0
- metadata +29 -18
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3594c5318a78874b31f44ee67dc0df53c93c3ded
|
4
|
+
data.tar.gz: 08e8678877a28546a04066094bf3c9c25dd131da
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d716e34b91a1386e0ef9507444b1f867c02f92f8d593dda28b3c0ab32deb9193bd87bb23fa8afc7f860aa6c96fa57b5b70aeebbe4fc49528d0d937ce5e98cba4
|
7
|
+
data.tar.gz: 1e23080c26b312eb4dd00d73ee88afeb4761d2090650a9c96f7baa7cf920b08606891885e18f48e41ac0367fc83338759b641f4e13e3474e69e8762fdac91145
|
data/.travis.yml
CHANGED
data/README.md
CHANGED
@@ -28,6 +28,11 @@ $ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-rename-key
|
|
28
28
|
|
29
29
|
## Configuration
|
30
30
|
|
31
|
+
### Notice
|
32
|
+
|
33
|
+
v0.12's filter plugin does not have a functionality to modify tag and time. If you want to modify time in filter plugin, we recommends to migrate to depends on v0.14 Filter Plugin API.
|
34
|
+
v0.14's filter API has `#filter_with_time` method.
|
35
|
+
|
31
36
|
### Syntax
|
32
37
|
|
33
38
|
```
|
@@ -3,22 +3,21 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = 'fluent-plugin-rename-key'
|
6
|
-
s.version = '0.
|
7
|
-
s.date = '2016-
|
6
|
+
s.version = '0.3.3'
|
7
|
+
s.date = '2016-11-29'
|
8
8
|
s.license = 'Apache-2.0'
|
9
|
-
s.authors = ['Shunwen Hsiao', 'Julian Grinblat']
|
9
|
+
s.authors = ['Shunwen Hsiao', 'Julian Grinblat', 'Hiroshi Hatake']
|
10
10
|
s.email = ['hsiaoshunwen@gmail.com']
|
11
11
|
s.homepage = 'https://github.com/shunwen/fluent-plugin-rename-key'
|
12
12
|
s.summary = %q[Fluentd output plugin. Rename keys which match given regular expressions, assign new tags and re-emit the records.]
|
13
13
|
|
14
|
-
s.required_ruby_version = '>=
|
14
|
+
s.required_ruby_version = '>= 2.2'
|
15
15
|
s.files = `git ls-files`.split("\n")
|
16
16
|
s.test_files = `git ls-files -- {test}/*`.split("\n")
|
17
17
|
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
18
18
|
s.require_paths = ['lib']
|
19
|
-
|
20
|
-
s.
|
21
|
-
s.add_development_dependency '
|
22
|
-
s.add_development_dependency '
|
23
|
-
s.add_development_dependency 'coveralls'
|
19
|
+
s.add_runtime_dependency 'fluentd', ">= 0.12.0", "< 2"
|
20
|
+
s.add_development_dependency 'test-unit', '~> 3.2'
|
21
|
+
s.add_development_dependency 'rake', '~> 11.3'
|
22
|
+
s.add_development_dependency 'coveralls', '~> 0.8'
|
24
23
|
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'fluent/plugin/rename_key_util'
|
2
|
+
|
3
|
+
class Fluent::RenameKeyFilter < Fluent::Filter
|
4
|
+
Fluent::Plugin.register_filter 'rename_key', self
|
5
|
+
|
6
|
+
include Fluent::RenameKeyUtil
|
7
|
+
|
8
|
+
desc 'Deep rename/replace operation.'
|
9
|
+
config_param :deep_rename, :bool, default: true
|
10
|
+
|
11
|
+
def configure conf
|
12
|
+
super
|
13
|
+
|
14
|
+
create_rename_rules(conf)
|
15
|
+
create_replace_rules(conf)
|
16
|
+
|
17
|
+
raise Fluent::ConfigError, "No rename or replace rules are given" if @rename_rules.empty? && @replace_rules.empty?
|
18
|
+
end
|
19
|
+
|
20
|
+
def filter tag, time, record
|
21
|
+
new_record = rename_key record
|
22
|
+
new_record = replace_key new_record
|
23
|
+
new_record
|
24
|
+
end
|
25
|
+
end if defined?(Fluent::Filter)
|
@@ -1,6 +1,10 @@
|
|
1
|
+
require 'fluent/plugin/rename_key_util'
|
2
|
+
|
1
3
|
class Fluent::RenameKeyOutput < Fluent::Output
|
2
4
|
Fluent::Plugin.register_output 'rename_key', self
|
3
5
|
|
6
|
+
include Fluent::RenameKeyUtil
|
7
|
+
|
4
8
|
# To support Fluentd v0.10.57 or earlier
|
5
9
|
unless method_defined?(:router)
|
6
10
|
define_method("router") { Fluent::Engine }
|
@@ -31,43 +35,8 @@ class Fluent::RenameKeyOutput < Fluent::Output
|
|
31
35
|
def configure conf
|
32
36
|
super
|
33
37
|
|
34
|
-
|
35
|
-
|
36
|
-
conf_rename_rules.sort_by { |r| r.sub('rename_rule', '').to_i }.each do |r|
|
37
|
-
key_regexp, new_key = parse_rename_rule conf[r]
|
38
|
-
|
39
|
-
if key_regexp.nil? || new_key.nil?
|
40
|
-
raise Fluent::ConfigError, "Failed to parse: #{r} #{conf[r]}"
|
41
|
-
end
|
42
|
-
|
43
|
-
if @rename_rules.map { |r| r[:key_regexp] }.include? /#{key_regexp}/
|
44
|
-
raise Fluent::ConfigError, "Duplicated rules for key #{key_regexp}: #{@rename_rules}"
|
45
|
-
end
|
46
|
-
|
47
|
-
@rename_rules << { key_regexp: /#{key_regexp}/, new_key: new_key }
|
48
|
-
log.info "Added rename key rule: #{r} #{@rename_rules.last}"
|
49
|
-
end
|
50
|
-
|
51
|
-
@replace_rules = []
|
52
|
-
conf_replace_rules = conf.keys.select { |k| k =~ /^replace_rule(\d+)$/ }
|
53
|
-
conf_replace_rules.sort_by { |r| r.sub('replace_rule', '').to_i }.each do |r|
|
54
|
-
key_regexp, replacement = parse_replace_rule conf[r]
|
55
|
-
|
56
|
-
if key_regexp.nil?
|
57
|
-
raise Fluent::ConfigError, "Failed to parse: #{r} #{conf[r]}"
|
58
|
-
end
|
59
|
-
|
60
|
-
if replacement.nil?
|
61
|
-
replacement = ""
|
62
|
-
end
|
63
|
-
|
64
|
-
if @replace_rules.map { |r| r[:key_regexp] }.include? /#{key_regexp}/
|
65
|
-
raise Fluent::ConfigError, "Duplicated rules for key #{key_regexp}: #{@replace_rules}"
|
66
|
-
end
|
67
|
-
|
68
|
-
@replace_rules << { key_regexp: /#{key_regexp}/, replacement: replacement }
|
69
|
-
log.info "Added replace key rule: #{r} #{@replace_rules.last}"
|
70
|
-
end
|
38
|
+
create_rename_rules(conf)
|
39
|
+
create_replace_rules(conf)
|
71
40
|
|
72
41
|
raise Fluent::ConfigError, "No rename or replace rules are given" if @rename_rules.empty? && @replace_rules.empty?
|
73
42
|
|
@@ -85,81 +54,4 @@ class Fluent::RenameKeyOutput < Fluent::Output
|
|
85
54
|
|
86
55
|
chain.next
|
87
56
|
end
|
88
|
-
|
89
|
-
# private
|
90
|
-
|
91
|
-
def parse_rename_rule rule
|
92
|
-
$~.captures if rule.match /^([^\s]+)\s+(.+)$/
|
93
|
-
end
|
94
|
-
|
95
|
-
def parse_replace_rule rule
|
96
|
-
$~.captures if rule.match /^([^\s]+)(?:\s+(.+))?$/
|
97
|
-
end
|
98
|
-
|
99
|
-
def rename_key record
|
100
|
-
new_record = {}
|
101
|
-
|
102
|
-
record.each do |key, value|
|
103
|
-
|
104
|
-
@rename_rules.each do |rule|
|
105
|
-
match_data = key.match rule[:key_regexp]
|
106
|
-
next unless match_data # next rule
|
107
|
-
|
108
|
-
placeholder = get_placeholder match_data
|
109
|
-
key = rule[:new_key].gsub /\${md\[\d+\]}/, placeholder
|
110
|
-
break
|
111
|
-
end
|
112
|
-
|
113
|
-
if @deep_rename
|
114
|
-
if value.is_a? Hash
|
115
|
-
value = rename_key value
|
116
|
-
elsif value.is_a? Array
|
117
|
-
value = value.map { |v| v.is_a?(Hash) ? rename_key(v) : v }
|
118
|
-
end
|
119
|
-
end
|
120
|
-
|
121
|
-
new_record[key] = value
|
122
|
-
end
|
123
|
-
|
124
|
-
new_record
|
125
|
-
end
|
126
|
-
|
127
|
-
def replace_key record
|
128
|
-
new_record = {}
|
129
|
-
|
130
|
-
record.each do |key, value|
|
131
|
-
|
132
|
-
@replace_rules.each do |rule|
|
133
|
-
match_data = key.match rule[:key_regexp]
|
134
|
-
next unless match_data # next rule
|
135
|
-
|
136
|
-
placeholder = get_placeholder match_data
|
137
|
-
key = key.gsub rule[:key_regexp], rule[:replacement].gsub(/\${md\[\d+\]}/, placeholder)
|
138
|
-
break
|
139
|
-
end
|
140
|
-
|
141
|
-
if @deep_rename
|
142
|
-
if value.is_a? Hash
|
143
|
-
value = replace_key value
|
144
|
-
elsif value.is_a? Array
|
145
|
-
value = value.map { |v| v.is_a?(Hash) ? replace_key(v) : v }
|
146
|
-
end
|
147
|
-
end
|
148
|
-
|
149
|
-
new_record[key] = value
|
150
|
-
end
|
151
|
-
|
152
|
-
new_record
|
153
|
-
end
|
154
|
-
|
155
|
-
def get_placeholder match_data
|
156
|
-
placeholder = {}
|
157
|
-
|
158
|
-
match_data.to_a.each_with_index do |e, idx|
|
159
|
-
placeholder["${md[#{idx}]}"] = e
|
160
|
-
end
|
161
|
-
|
162
|
-
placeholder
|
163
|
-
end
|
164
|
-
|
165
57
|
end
|
@@ -0,0 +1,119 @@
|
|
1
|
+
module Fluent
|
2
|
+
module RenameKeyUtil
|
3
|
+
def create_rename_rules(conf)
|
4
|
+
@rename_rules = []
|
5
|
+
conf_rename_rules = conf.keys.select { |k| k =~ /^rename_rule(\d+)$/ }
|
6
|
+
conf_rename_rules.sort_by { |r| r.sub('rename_rule', '').to_i }.each do |r|
|
7
|
+
key_regexp, new_key = parse_rename_rule conf[r]
|
8
|
+
|
9
|
+
if key_regexp.nil? || new_key.nil?
|
10
|
+
raise Fluent::ConfigError, "Failed to parse: #{r} #{conf[r]}"
|
11
|
+
end
|
12
|
+
|
13
|
+
if @rename_rules.map { |r| r[:key_regexp] }.include? /#{key_regexp}/
|
14
|
+
raise Fluent::ConfigError, "Duplicated rules for key #{key_regexp}: #{@rename_rules}"
|
15
|
+
end
|
16
|
+
|
17
|
+
@rename_rules << { key_regexp: /#{key_regexp}/, new_key: new_key }
|
18
|
+
log.info "Added rename key rule: #{r} #{@rename_rules.last}"
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def create_replace_rules(conf)
|
23
|
+
@replace_rules = []
|
24
|
+
conf_replace_rules = conf.keys.select { |k| k =~ /^replace_rule(\d+)$/ }
|
25
|
+
conf_replace_rules.sort_by { |r| r.sub('replace_rule', '').to_i }.each do |r|
|
26
|
+
key_regexp, replacement = parse_replace_rule conf[r]
|
27
|
+
|
28
|
+
if key_regexp.nil?
|
29
|
+
raise Fluent::ConfigError, "Failed to parse: #{r} #{conf[r]}"
|
30
|
+
end
|
31
|
+
|
32
|
+
if replacement.nil?
|
33
|
+
replacement = ""
|
34
|
+
end
|
35
|
+
|
36
|
+
if @replace_rules.map { |r| r[:key_regexp] }.include? /#{key_regexp}/
|
37
|
+
raise Fluent::ConfigError, "Duplicated rules for key #{key_regexp}: #{@replace_rules}"
|
38
|
+
end
|
39
|
+
|
40
|
+
@replace_rules << { key_regexp: /#{key_regexp}/, replacement: replacement }
|
41
|
+
log.info "Added replace key rule: #{r} #{@replace_rules.last}"
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def parse_rename_rule rule
|
46
|
+
$~.captures if rule.match /^([^\s]+)\s+(.+)$/
|
47
|
+
end
|
48
|
+
|
49
|
+
def parse_replace_rule rule
|
50
|
+
$~.captures if rule.match /^([^\s]+)(?:\s+(.+))?$/
|
51
|
+
end
|
52
|
+
|
53
|
+
def rename_key record
|
54
|
+
new_record = {}
|
55
|
+
|
56
|
+
record.each do |key, value|
|
57
|
+
|
58
|
+
@rename_rules.each do |rule|
|
59
|
+
match_data = key.match rule[:key_regexp]
|
60
|
+
next unless match_data # next rule
|
61
|
+
|
62
|
+
placeholder = get_placeholder match_data
|
63
|
+
key = rule[:new_key].gsub /\${md\[\d+\]}/, placeholder
|
64
|
+
break
|
65
|
+
end
|
66
|
+
|
67
|
+
if @deep_rename
|
68
|
+
if value.is_a? Hash
|
69
|
+
value = rename_key value
|
70
|
+
elsif value.is_a? Array
|
71
|
+
value = value.map { |v| v.is_a?(Hash) ? rename_key(v) : v }
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
new_record[key] = value
|
76
|
+
end
|
77
|
+
|
78
|
+
new_record
|
79
|
+
end
|
80
|
+
|
81
|
+
def replace_key record
|
82
|
+
new_record = {}
|
83
|
+
|
84
|
+
record.each do |key, value|
|
85
|
+
|
86
|
+
@replace_rules.each do |rule|
|
87
|
+
match_data = key.match rule[:key_regexp]
|
88
|
+
next unless match_data # next rule
|
89
|
+
|
90
|
+
placeholder = get_placeholder match_data
|
91
|
+
key = key.gsub rule[:key_regexp], rule[:replacement].gsub(/\${md\[\d+\]}/, placeholder)
|
92
|
+
break
|
93
|
+
end
|
94
|
+
|
95
|
+
if @deep_rename
|
96
|
+
if value.is_a? Hash
|
97
|
+
value = replace_key value
|
98
|
+
elsif value.is_a? Array
|
99
|
+
value = value.map { |v| v.is_a?(Hash) ? replace_key(v) : v }
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
new_record[key] = value
|
104
|
+
end
|
105
|
+
|
106
|
+
new_record
|
107
|
+
end
|
108
|
+
|
109
|
+
def get_placeholder match_data
|
110
|
+
placeholder = {}
|
111
|
+
|
112
|
+
match_data.to_a.each_with_index do |e, idx|
|
113
|
+
placeholder["${md[#{idx}]}"] = e
|
114
|
+
end
|
115
|
+
|
116
|
+
placeholder
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
data/test/helper.rb
CHANGED
@@ -0,0 +1,215 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
class RenameKeyFilterTest < Test::Unit::TestCase
|
4
|
+
MATCH_TAG = 'incoming_tag'
|
5
|
+
RENAME_RULE_CONFIG = 'rename_rule1 ^\$(.+) x$${md[1]}'
|
6
|
+
REPLACE_RULE_CONFIG = 'replace_rule1 ^\$ x'
|
7
|
+
|
8
|
+
def setup
|
9
|
+
omit("Fluentd v0.12 or later is required.") unless defined?(Fluent::Filter)
|
10
|
+
Fluent::Test.setup
|
11
|
+
end
|
12
|
+
|
13
|
+
def create_driver(conf, tag = MATCH_TAG)
|
14
|
+
Fluent::Test::FilterTestDriver.new(Fluent::RenameKeyFilter, tag).configure(conf)
|
15
|
+
end
|
16
|
+
|
17
|
+
def test_config_error
|
18
|
+
assert_raise(Fluent::ConfigError) { create_driver('') }
|
19
|
+
|
20
|
+
assert_raise(Fluent::ConfigError) { create_driver('rename_rule1 ^$(.+?) ') }
|
21
|
+
|
22
|
+
assert_raise(Fluent::ConfigError) {
|
23
|
+
config_dup_rules_for_a_key = %q[
|
24
|
+
rename_rule1 ^\$(.+) ${md[1]}
|
25
|
+
rename_rule2 ^\$(.+) ${md[1]} something
|
26
|
+
]
|
27
|
+
create_driver(config_dup_rules_for_a_key)
|
28
|
+
}
|
29
|
+
end
|
30
|
+
|
31
|
+
def test_config_success
|
32
|
+
config_multiple_rules = %q[
|
33
|
+
rename_rule1 ^\$(.+)1 x$${md[1]}
|
34
|
+
rename_rule2 ^\$(.+)2(\d+) ${md[1]}_${md[2]}
|
35
|
+
]
|
36
|
+
|
37
|
+
d = create_driver config_multiple_rules
|
38
|
+
assert_equal '^\$(.+)1 x$${md[1]}', d.instance.config['rename_rule1']
|
39
|
+
assert_equal '^\$(.+)2(\d+) ${md[1]}_${md[2]}', d.instance.config['rename_rule2']
|
40
|
+
end
|
41
|
+
|
42
|
+
def test_parse_rename_rule
|
43
|
+
parsed = Fluent::RenameKeyOutput.new.parse_rename_rule '(reg)(exp) ${md[1]} ${md[2]}'
|
44
|
+
assert_equal 2, parsed.length
|
45
|
+
assert_equal '(reg)(exp)', parsed[0]
|
46
|
+
assert_equal '${md[1]} ${md[2]}', parsed[1]
|
47
|
+
end
|
48
|
+
|
49
|
+
def test_parse_replace_rule_with_replacement
|
50
|
+
# Replace hyphens with underscores
|
51
|
+
parsed = Fluent::RenameKeyOutput.new.parse_replace_rule '- _'
|
52
|
+
assert_equal 2, parsed.length
|
53
|
+
assert_equal '-', parsed[0]
|
54
|
+
assert_equal '_', parsed[1]
|
55
|
+
end
|
56
|
+
|
57
|
+
def test_parse_replace_rule_without_replacement
|
58
|
+
# Remove all parenthesis hyphens and spaces
|
59
|
+
parsed = Fluent::RenameKeyOutput.new.parse_replace_rule '[()-\s]'
|
60
|
+
assert_equal 2, parsed.length
|
61
|
+
assert_equal '[()-\s]', parsed[0]
|
62
|
+
assert parsed[1].nil?
|
63
|
+
end
|
64
|
+
|
65
|
+
def test_rename_rule_emit_deep_rename_hash
|
66
|
+
d = create_driver RENAME_RULE_CONFIG
|
67
|
+
d.run do
|
68
|
+
d.emit '$key1' => 'value1', 'key2' => {'$key3' => 'value3', '$key4'=> {'$key5' => 'value5'} }
|
69
|
+
end
|
70
|
+
|
71
|
+
emits = d.emits
|
72
|
+
assert_equal %w[x$key1 key2], emits[0][2].keys
|
73
|
+
assert_equal %w[x$key3 x$key4], emits[0][2]['key2'].keys
|
74
|
+
assert_equal ['x$key5'], emits[0][2]['key2']['x$key4'].keys
|
75
|
+
end
|
76
|
+
|
77
|
+
def test_rename_rule_emit_deep_rename_array
|
78
|
+
d = create_driver RENAME_RULE_CONFIG
|
79
|
+
d.run do
|
80
|
+
d.emit '$key1' => 'value1', 'key2' => [{'$key3' => 'value3'}, {'$key4'=> {'$key5' => 'value5'}}]
|
81
|
+
end
|
82
|
+
|
83
|
+
emits = d.emits
|
84
|
+
assert_equal %w[x$key3 x$key4], emits[0][2]['key2'].flat_map(&:keys)
|
85
|
+
assert_equal ['x$key5'], emits[0][2]['key2'][1]['x$key4'].keys
|
86
|
+
end
|
87
|
+
|
88
|
+
def test_rename_rule_emit_deep_rename_off
|
89
|
+
config = %Q[
|
90
|
+
#{RENAME_RULE_CONFIG}
|
91
|
+
deep_rename false
|
92
|
+
]
|
93
|
+
|
94
|
+
d = create_driver config
|
95
|
+
d.run do
|
96
|
+
d.emit '$key1' => 'value1', 'key2' => {'$key3'=>'value3', '$key4'=> 'value4'}
|
97
|
+
end
|
98
|
+
|
99
|
+
emits = d.emits
|
100
|
+
assert_equal %w[$key3 $key4], emits[0][2]['key2'].keys
|
101
|
+
end
|
102
|
+
|
103
|
+
def test_rename_rule_emit_with_match_data
|
104
|
+
d = create_driver 'rename_rule1 (\w+)\s(\w+)\s(\w+) ${md[3]} ${md[2]} ${md[1]}'
|
105
|
+
d.run do
|
106
|
+
d.emit 'key1 key2 key3' => 'value'
|
107
|
+
end
|
108
|
+
emits = d.emits
|
109
|
+
assert_equal 1, emits.length
|
110
|
+
assert_equal ['key3 key2 key1'], emits[0][2].keys
|
111
|
+
end
|
112
|
+
|
113
|
+
def test_multiple_rename_rules_emit
|
114
|
+
config_multiple_rules = %q[
|
115
|
+
rename_rule1 ^(\w+)\s1 ${md[1]}_1
|
116
|
+
rename_rule2 ^(\w+)\s2 ${md[1]}_2
|
117
|
+
]
|
118
|
+
|
119
|
+
d = create_driver config_multiple_rules
|
120
|
+
d.run do
|
121
|
+
d.emit 'key 1' => 'value1', 'key 2' => 'value2'
|
122
|
+
end
|
123
|
+
|
124
|
+
emits = d.emits
|
125
|
+
assert_equal %w[key_1 key_2], emits[0][2].keys
|
126
|
+
end
|
127
|
+
|
128
|
+
def test_replace_rule_emit_deep_rename_hash
|
129
|
+
d = create_driver 'replace_rule1 ^(\$) x'
|
130
|
+
|
131
|
+
d.run do
|
132
|
+
d.emit '$key1' => 'value1', 'key2' => { 'key3' => 'value3', '$key4' => 'value4' }
|
133
|
+
end
|
134
|
+
emits = d.emits
|
135
|
+
assert_equal %w[xkey1 key2], emits[0][2].keys
|
136
|
+
assert_equal %w[key3 xkey4], emits[0][2]['key2'].keys
|
137
|
+
end
|
138
|
+
|
139
|
+
def test_replace_rule_emit_with_match_data
|
140
|
+
d = create_driver 'rename_rule1 (\w+)\s(\w+)\s(\w+) ${md[3]} ${md[2]} ${md[1]}'
|
141
|
+
d.run do
|
142
|
+
d.emit 'key1 key2 key3' => 'value'
|
143
|
+
end
|
144
|
+
emits = d.emits
|
145
|
+
assert_equal 1, emits.length
|
146
|
+
assert_equal ['key3 key2 key1'], emits[0][2].keys
|
147
|
+
end
|
148
|
+
|
149
|
+
def test_replace_rule_emit_deep_rename_array
|
150
|
+
d = create_driver 'replace_rule1 ^(\$) x${md[1]}'
|
151
|
+
|
152
|
+
d.run do
|
153
|
+
d.emit '$key1' => 'value1', 'key2' => [{'$key3' => 'value3'}, {'$key4' => {'$key5' => 'value5'}}]
|
154
|
+
end
|
155
|
+
|
156
|
+
emits = d.emits
|
157
|
+
assert_equal %w[x$key3 x$key4], emits[0][2]['key2'].flat_map(&:keys)
|
158
|
+
assert_equal %w[x$key5], emits[0][2]['key2'][1]['x$key4'].keys
|
159
|
+
end
|
160
|
+
|
161
|
+
def test_replace_rule_emit_deep_rename_off
|
162
|
+
config = %Q[
|
163
|
+
#{REPLACE_RULE_CONFIG}
|
164
|
+
deep_rename false
|
165
|
+
]
|
166
|
+
|
167
|
+
d = create_driver config
|
168
|
+
d.run do
|
169
|
+
d.emit '$key1' => 'value1', 'key2' => {'$key3'=>'value3', '$key4'=> 'value4'}
|
170
|
+
end
|
171
|
+
|
172
|
+
emits = d.emits
|
173
|
+
assert_equal %w[$key3 $key4], emits[0][2]['key2'].keys
|
174
|
+
end
|
175
|
+
|
176
|
+
def test_replace_rule_emit_remove_matched_when_no_replacement
|
177
|
+
d = create_driver 'replace_rule1 [\s/()]'
|
178
|
+
d.run do
|
179
|
+
d.emit 'key (/1 )' => 'value1'
|
180
|
+
end
|
181
|
+
|
182
|
+
emits = d.emits
|
183
|
+
assert_equal %w[key1], emits[0][2].keys
|
184
|
+
end
|
185
|
+
|
186
|
+
def test_multiple_replace_rules_emit
|
187
|
+
config_multiple_rules = %q[
|
188
|
+
replace_rule1 ^(\w+)\s(\d) ${md[1]}${md[2]}
|
189
|
+
replace_rule2 [\s()]
|
190
|
+
]
|
191
|
+
|
192
|
+
d = create_driver config_multiple_rules
|
193
|
+
d.run do
|
194
|
+
d.emit 'key 1' => 'value1', 'key (2)' => 'value2'
|
195
|
+
end
|
196
|
+
|
197
|
+
emits = d.emits
|
198
|
+
assert_equal %w[key1 key2], emits[0][2].keys
|
199
|
+
end
|
200
|
+
|
201
|
+
def test_combined_rename_rule_and_replace_rule
|
202
|
+
config_combined_rules = %q[
|
203
|
+
rename_rule1 ^(.+)\s(one) ${md[1]}1
|
204
|
+
replace_rule2 [\s()]
|
205
|
+
]
|
206
|
+
|
207
|
+
d = create_driver config_combined_rules
|
208
|
+
d.run do
|
209
|
+
d.emit '(key) one (x)' => 'value1', 'key (2)' => 'value2'
|
210
|
+
end
|
211
|
+
|
212
|
+
emits = d.emits
|
213
|
+
assert_equal %w[key1 key2], emits[0][2].keys
|
214
|
+
end
|
215
|
+
end
|
metadata
CHANGED
@@ -1,15 +1,16 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-rename-key
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shunwen Hsiao
|
8
8
|
- Julian Grinblat
|
9
|
+
- Hiroshi Hatake
|
9
10
|
autorequire:
|
10
11
|
bindir: bin
|
11
12
|
cert_chain: []
|
12
|
-
date: 2016-
|
13
|
+
date: 2016-11-29 00:00:00.000000000 Z
|
13
14
|
dependencies:
|
14
15
|
- !ruby/object:Gem::Dependency
|
15
16
|
name: fluentd
|
@@ -17,56 +18,62 @@ dependencies:
|
|
17
18
|
requirements:
|
18
19
|
- - ">="
|
19
20
|
- !ruby/object:Gem::Version
|
20
|
-
version:
|
21
|
+
version: 0.12.0
|
22
|
+
- - "<"
|
23
|
+
- !ruby/object:Gem::Version
|
24
|
+
version: '2'
|
21
25
|
type: :runtime
|
22
26
|
prerelease: false
|
23
27
|
version_requirements: !ruby/object:Gem::Requirement
|
24
28
|
requirements:
|
25
29
|
- - ">="
|
26
30
|
- !ruby/object:Gem::Version
|
27
|
-
version:
|
31
|
+
version: 0.12.0
|
32
|
+
- - "<"
|
33
|
+
- !ruby/object:Gem::Version
|
34
|
+
version: '2'
|
28
35
|
- !ruby/object:Gem::Dependency
|
29
36
|
name: test-unit
|
30
37
|
requirement: !ruby/object:Gem::Requirement
|
31
38
|
requirements:
|
32
|
-
- - "
|
39
|
+
- - "~>"
|
33
40
|
- !ruby/object:Gem::Version
|
34
|
-
version: 3.
|
41
|
+
version: '3.2'
|
35
42
|
type: :development
|
36
43
|
prerelease: false
|
37
44
|
version_requirements: !ruby/object:Gem::Requirement
|
38
45
|
requirements:
|
39
|
-
- - "
|
46
|
+
- - "~>"
|
40
47
|
- !ruby/object:Gem::Version
|
41
|
-
version: 3.
|
48
|
+
version: '3.2'
|
42
49
|
- !ruby/object:Gem::Dependency
|
43
50
|
name: rake
|
44
51
|
requirement: !ruby/object:Gem::Requirement
|
45
52
|
requirements:
|
46
|
-
- - "
|
53
|
+
- - "~>"
|
47
54
|
- !ruby/object:Gem::Version
|
48
|
-
version: '
|
55
|
+
version: '11.3'
|
49
56
|
type: :development
|
50
57
|
prerelease: false
|
51
58
|
version_requirements: !ruby/object:Gem::Requirement
|
52
59
|
requirements:
|
53
|
-
- - "
|
60
|
+
- - "~>"
|
54
61
|
- !ruby/object:Gem::Version
|
55
|
-
version: '
|
62
|
+
version: '11.3'
|
56
63
|
- !ruby/object:Gem::Dependency
|
57
64
|
name: coveralls
|
58
65
|
requirement: !ruby/object:Gem::Requirement
|
59
66
|
requirements:
|
60
|
-
- - "
|
67
|
+
- - "~>"
|
61
68
|
- !ruby/object:Gem::Version
|
62
|
-
version: '0'
|
69
|
+
version: '0.8'
|
63
70
|
type: :development
|
64
71
|
prerelease: false
|
65
72
|
version_requirements: !ruby/object:Gem::Requirement
|
66
73
|
requirements:
|
67
|
-
- - "
|
74
|
+
- - "~>"
|
68
75
|
- !ruby/object:Gem::Version
|
69
|
-
version: '0'
|
76
|
+
version: '0.8'
|
70
77
|
description:
|
71
78
|
email:
|
72
79
|
- hsiaoshunwen@gmail.com
|
@@ -81,8 +88,11 @@ files:
|
|
81
88
|
- README.md
|
82
89
|
- Rakefile
|
83
90
|
- fluent-plugin-rename-key.gemspec
|
91
|
+
- lib/fluent/plugin/filter_rename_key.rb
|
84
92
|
- lib/fluent/plugin/out_rename_key.rb
|
93
|
+
- lib/fluent/plugin/rename_key_util.rb
|
85
94
|
- test/helper.rb
|
95
|
+
- test/plugin/test_filter_rename_key.rb
|
86
96
|
- test/plugin/test_out_rename_key.rb
|
87
97
|
homepage: https://github.com/shunwen/fluent-plugin-rename-key
|
88
98
|
licenses:
|
@@ -96,7 +106,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
96
106
|
requirements:
|
97
107
|
- - ">="
|
98
108
|
- !ruby/object:Gem::Version
|
99
|
-
version:
|
109
|
+
version: '2.2'
|
100
110
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
101
111
|
requirements:
|
102
112
|
- - ">="
|
@@ -104,9 +114,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
104
114
|
version: '0'
|
105
115
|
requirements: []
|
106
116
|
rubyforge_project:
|
107
|
-
rubygems_version: 2.6.
|
117
|
+
rubygems_version: 2.6.4
|
108
118
|
signing_key:
|
109
119
|
specification_version: 4
|
110
120
|
summary: Fluentd output plugin. Rename keys which match given regular expressions,
|
111
121
|
assign new tags and re-emit the records.
|
112
122
|
test_files: []
|
123
|
+
has_rdoc:
|