fluent-plugin-rename-key 0.2.0 → 0.3.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +0 -1
- data/README.md +5 -0
- data/fluent-plugin-rename-key.gemspec +8 -9
- data/lib/fluent/plugin/filter_rename_key.rb +25 -0
- data/lib/fluent/plugin/out_rename_key.rb +6 -114
- data/lib/fluent/plugin/rename_key_util.rb +119 -0
- data/test/helper.rb +3 -0
- data/test/plugin/test_filter_rename_key.rb +215 -0
- metadata +29 -18
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 3594c5318a78874b31f44ee67dc0df53c93c3ded
|
4
|
+
data.tar.gz: 08e8678877a28546a04066094bf3c9c25dd131da
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d716e34b91a1386e0ef9507444b1f867c02f92f8d593dda28b3c0ab32deb9193bd87bb23fa8afc7f860aa6c96fa57b5b70aeebbe4fc49528d0d937ce5e98cba4
|
7
|
+
data.tar.gz: 1e23080c26b312eb4dd00d73ee88afeb4761d2090650a9c96f7baa7cf920b08606891885e18f48e41ac0367fc83338759b641f4e13e3474e69e8762fdac91145
|
data/.travis.yml
CHANGED
data/README.md
CHANGED
@@ -28,6 +28,11 @@ $ sudo /usr/lib64/fluent/ruby/bin/fluent-gem install fluent-plugin-rename-key
|
|
28
28
|
|
29
29
|
## Configuration
|
30
30
|
|
31
|
+
### Notice
|
32
|
+
|
33
|
+
v0.12's filter plugin does not have a functionality to modify tag and time. If you want to modify time in filter plugin, we recommends to migrate to depends on v0.14 Filter Plugin API.
|
34
|
+
v0.14's filter API has `#filter_with_time` method.
|
35
|
+
|
31
36
|
### Syntax
|
32
37
|
|
33
38
|
```
|
@@ -3,22 +3,21 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |s|
|
5
5
|
s.name = 'fluent-plugin-rename-key'
|
6
|
-
s.version = '0.
|
7
|
-
s.date = '2016-
|
6
|
+
s.version = '0.3.3'
|
7
|
+
s.date = '2016-11-29'
|
8
8
|
s.license = 'Apache-2.0'
|
9
|
-
s.authors = ['Shunwen Hsiao', 'Julian Grinblat']
|
9
|
+
s.authors = ['Shunwen Hsiao', 'Julian Grinblat', 'Hiroshi Hatake']
|
10
10
|
s.email = ['hsiaoshunwen@gmail.com']
|
11
11
|
s.homepage = 'https://github.com/shunwen/fluent-plugin-rename-key'
|
12
12
|
s.summary = %q[Fluentd output plugin. Rename keys which match given regular expressions, assign new tags and re-emit the records.]
|
13
13
|
|
14
|
-
s.required_ruby_version = '>=
|
14
|
+
s.required_ruby_version = '>= 2.2'
|
15
15
|
s.files = `git ls-files`.split("\n")
|
16
16
|
s.test_files = `git ls-files -- {test}/*`.split("\n")
|
17
17
|
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
18
18
|
s.require_paths = ['lib']
|
19
|
-
|
20
|
-
s.
|
21
|
-
s.add_development_dependency '
|
22
|
-
s.add_development_dependency '
|
23
|
-
s.add_development_dependency 'coveralls'
|
19
|
+
s.add_runtime_dependency 'fluentd', ">= 0.12.0", "< 2"
|
20
|
+
s.add_development_dependency 'test-unit', '~> 3.2'
|
21
|
+
s.add_development_dependency 'rake', '~> 11.3'
|
22
|
+
s.add_development_dependency 'coveralls', '~> 0.8'
|
24
23
|
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'fluent/plugin/rename_key_util'
|
2
|
+
|
3
|
+
class Fluent::RenameKeyFilter < Fluent::Filter
|
4
|
+
Fluent::Plugin.register_filter 'rename_key', self
|
5
|
+
|
6
|
+
include Fluent::RenameKeyUtil
|
7
|
+
|
8
|
+
desc 'Deep rename/replace operation.'
|
9
|
+
config_param :deep_rename, :bool, default: true
|
10
|
+
|
11
|
+
def configure conf
|
12
|
+
super
|
13
|
+
|
14
|
+
create_rename_rules(conf)
|
15
|
+
create_replace_rules(conf)
|
16
|
+
|
17
|
+
raise Fluent::ConfigError, "No rename or replace rules are given" if @rename_rules.empty? && @replace_rules.empty?
|
18
|
+
end
|
19
|
+
|
20
|
+
def filter tag, time, record
|
21
|
+
new_record = rename_key record
|
22
|
+
new_record = replace_key new_record
|
23
|
+
new_record
|
24
|
+
end
|
25
|
+
end if defined?(Fluent::Filter)
|
@@ -1,6 +1,10 @@
|
|
1
|
+
require 'fluent/plugin/rename_key_util'
|
2
|
+
|
1
3
|
class Fluent::RenameKeyOutput < Fluent::Output
|
2
4
|
Fluent::Plugin.register_output 'rename_key', self
|
3
5
|
|
6
|
+
include Fluent::RenameKeyUtil
|
7
|
+
|
4
8
|
# To support Fluentd v0.10.57 or earlier
|
5
9
|
unless method_defined?(:router)
|
6
10
|
define_method("router") { Fluent::Engine }
|
@@ -31,43 +35,8 @@ class Fluent::RenameKeyOutput < Fluent::Output
|
|
31
35
|
def configure conf
|
32
36
|
super
|
33
37
|
|
34
|
-
|
35
|
-
|
36
|
-
conf_rename_rules.sort_by { |r| r.sub('rename_rule', '').to_i }.each do |r|
|
37
|
-
key_regexp, new_key = parse_rename_rule conf[r]
|
38
|
-
|
39
|
-
if key_regexp.nil? || new_key.nil?
|
40
|
-
raise Fluent::ConfigError, "Failed to parse: #{r} #{conf[r]}"
|
41
|
-
end
|
42
|
-
|
43
|
-
if @rename_rules.map { |r| r[:key_regexp] }.include? /#{key_regexp}/
|
44
|
-
raise Fluent::ConfigError, "Duplicated rules for key #{key_regexp}: #{@rename_rules}"
|
45
|
-
end
|
46
|
-
|
47
|
-
@rename_rules << { key_regexp: /#{key_regexp}/, new_key: new_key }
|
48
|
-
log.info "Added rename key rule: #{r} #{@rename_rules.last}"
|
49
|
-
end
|
50
|
-
|
51
|
-
@replace_rules = []
|
52
|
-
conf_replace_rules = conf.keys.select { |k| k =~ /^replace_rule(\d+)$/ }
|
53
|
-
conf_replace_rules.sort_by { |r| r.sub('replace_rule', '').to_i }.each do |r|
|
54
|
-
key_regexp, replacement = parse_replace_rule conf[r]
|
55
|
-
|
56
|
-
if key_regexp.nil?
|
57
|
-
raise Fluent::ConfigError, "Failed to parse: #{r} #{conf[r]}"
|
58
|
-
end
|
59
|
-
|
60
|
-
if replacement.nil?
|
61
|
-
replacement = ""
|
62
|
-
end
|
63
|
-
|
64
|
-
if @replace_rules.map { |r| r[:key_regexp] }.include? /#{key_regexp}/
|
65
|
-
raise Fluent::ConfigError, "Duplicated rules for key #{key_regexp}: #{@replace_rules}"
|
66
|
-
end
|
67
|
-
|
68
|
-
@replace_rules << { key_regexp: /#{key_regexp}/, replacement: replacement }
|
69
|
-
log.info "Added replace key rule: #{r} #{@replace_rules.last}"
|
70
|
-
end
|
38
|
+
create_rename_rules(conf)
|
39
|
+
create_replace_rules(conf)
|
71
40
|
|
72
41
|
raise Fluent::ConfigError, "No rename or replace rules are given" if @rename_rules.empty? && @replace_rules.empty?
|
73
42
|
|
@@ -85,81 +54,4 @@ class Fluent::RenameKeyOutput < Fluent::Output
|
|
85
54
|
|
86
55
|
chain.next
|
87
56
|
end
|
88
|
-
|
89
|
-
# private
|
90
|
-
|
91
|
-
def parse_rename_rule rule
|
92
|
-
$~.captures if rule.match /^([^\s]+)\s+(.+)$/
|
93
|
-
end
|
94
|
-
|
95
|
-
def parse_replace_rule rule
|
96
|
-
$~.captures if rule.match /^([^\s]+)(?:\s+(.+))?$/
|
97
|
-
end
|
98
|
-
|
99
|
-
def rename_key record
|
100
|
-
new_record = {}
|
101
|
-
|
102
|
-
record.each do |key, value|
|
103
|
-
|
104
|
-
@rename_rules.each do |rule|
|
105
|
-
match_data = key.match rule[:key_regexp]
|
106
|
-
next unless match_data # next rule
|
107
|
-
|
108
|
-
placeholder = get_placeholder match_data
|
109
|
-
key = rule[:new_key].gsub /\${md\[\d+\]}/, placeholder
|
110
|
-
break
|
111
|
-
end
|
112
|
-
|
113
|
-
if @deep_rename
|
114
|
-
if value.is_a? Hash
|
115
|
-
value = rename_key value
|
116
|
-
elsif value.is_a? Array
|
117
|
-
value = value.map { |v| v.is_a?(Hash) ? rename_key(v) : v }
|
118
|
-
end
|
119
|
-
end
|
120
|
-
|
121
|
-
new_record[key] = value
|
122
|
-
end
|
123
|
-
|
124
|
-
new_record
|
125
|
-
end
|
126
|
-
|
127
|
-
def replace_key record
|
128
|
-
new_record = {}
|
129
|
-
|
130
|
-
record.each do |key, value|
|
131
|
-
|
132
|
-
@replace_rules.each do |rule|
|
133
|
-
match_data = key.match rule[:key_regexp]
|
134
|
-
next unless match_data # next rule
|
135
|
-
|
136
|
-
placeholder = get_placeholder match_data
|
137
|
-
key = key.gsub rule[:key_regexp], rule[:replacement].gsub(/\${md\[\d+\]}/, placeholder)
|
138
|
-
break
|
139
|
-
end
|
140
|
-
|
141
|
-
if @deep_rename
|
142
|
-
if value.is_a? Hash
|
143
|
-
value = replace_key value
|
144
|
-
elsif value.is_a? Array
|
145
|
-
value = value.map { |v| v.is_a?(Hash) ? replace_key(v) : v }
|
146
|
-
end
|
147
|
-
end
|
148
|
-
|
149
|
-
new_record[key] = value
|
150
|
-
end
|
151
|
-
|
152
|
-
new_record
|
153
|
-
end
|
154
|
-
|
155
|
-
def get_placeholder match_data
|
156
|
-
placeholder = {}
|
157
|
-
|
158
|
-
match_data.to_a.each_with_index do |e, idx|
|
159
|
-
placeholder["${md[#{idx}]}"] = e
|
160
|
-
end
|
161
|
-
|
162
|
-
placeholder
|
163
|
-
end
|
164
|
-
|
165
57
|
end
|
@@ -0,0 +1,119 @@
|
|
1
|
+
module Fluent
|
2
|
+
module RenameKeyUtil
|
3
|
+
def create_rename_rules(conf)
|
4
|
+
@rename_rules = []
|
5
|
+
conf_rename_rules = conf.keys.select { |k| k =~ /^rename_rule(\d+)$/ }
|
6
|
+
conf_rename_rules.sort_by { |r| r.sub('rename_rule', '').to_i }.each do |r|
|
7
|
+
key_regexp, new_key = parse_rename_rule conf[r]
|
8
|
+
|
9
|
+
if key_regexp.nil? || new_key.nil?
|
10
|
+
raise Fluent::ConfigError, "Failed to parse: #{r} #{conf[r]}"
|
11
|
+
end
|
12
|
+
|
13
|
+
if @rename_rules.map { |r| r[:key_regexp] }.include? /#{key_regexp}/
|
14
|
+
raise Fluent::ConfigError, "Duplicated rules for key #{key_regexp}: #{@rename_rules}"
|
15
|
+
end
|
16
|
+
|
17
|
+
@rename_rules << { key_regexp: /#{key_regexp}/, new_key: new_key }
|
18
|
+
log.info "Added rename key rule: #{r} #{@rename_rules.last}"
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
def create_replace_rules(conf)
|
23
|
+
@replace_rules = []
|
24
|
+
conf_replace_rules = conf.keys.select { |k| k =~ /^replace_rule(\d+)$/ }
|
25
|
+
conf_replace_rules.sort_by { |r| r.sub('replace_rule', '').to_i }.each do |r|
|
26
|
+
key_regexp, replacement = parse_replace_rule conf[r]
|
27
|
+
|
28
|
+
if key_regexp.nil?
|
29
|
+
raise Fluent::ConfigError, "Failed to parse: #{r} #{conf[r]}"
|
30
|
+
end
|
31
|
+
|
32
|
+
if replacement.nil?
|
33
|
+
replacement = ""
|
34
|
+
end
|
35
|
+
|
36
|
+
if @replace_rules.map { |r| r[:key_regexp] }.include? /#{key_regexp}/
|
37
|
+
raise Fluent::ConfigError, "Duplicated rules for key #{key_regexp}: #{@replace_rules}"
|
38
|
+
end
|
39
|
+
|
40
|
+
@replace_rules << { key_regexp: /#{key_regexp}/, replacement: replacement }
|
41
|
+
log.info "Added replace key rule: #{r} #{@replace_rules.last}"
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def parse_rename_rule rule
|
46
|
+
$~.captures if rule.match /^([^\s]+)\s+(.+)$/
|
47
|
+
end
|
48
|
+
|
49
|
+
def parse_replace_rule rule
|
50
|
+
$~.captures if rule.match /^([^\s]+)(?:\s+(.+))?$/
|
51
|
+
end
|
52
|
+
|
53
|
+
def rename_key record
|
54
|
+
new_record = {}
|
55
|
+
|
56
|
+
record.each do |key, value|
|
57
|
+
|
58
|
+
@rename_rules.each do |rule|
|
59
|
+
match_data = key.match rule[:key_regexp]
|
60
|
+
next unless match_data # next rule
|
61
|
+
|
62
|
+
placeholder = get_placeholder match_data
|
63
|
+
key = rule[:new_key].gsub /\${md\[\d+\]}/, placeholder
|
64
|
+
break
|
65
|
+
end
|
66
|
+
|
67
|
+
if @deep_rename
|
68
|
+
if value.is_a? Hash
|
69
|
+
value = rename_key value
|
70
|
+
elsif value.is_a? Array
|
71
|
+
value = value.map { |v| v.is_a?(Hash) ? rename_key(v) : v }
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
new_record[key] = value
|
76
|
+
end
|
77
|
+
|
78
|
+
new_record
|
79
|
+
end
|
80
|
+
|
81
|
+
def replace_key record
|
82
|
+
new_record = {}
|
83
|
+
|
84
|
+
record.each do |key, value|
|
85
|
+
|
86
|
+
@replace_rules.each do |rule|
|
87
|
+
match_data = key.match rule[:key_regexp]
|
88
|
+
next unless match_data # next rule
|
89
|
+
|
90
|
+
placeholder = get_placeholder match_data
|
91
|
+
key = key.gsub rule[:key_regexp], rule[:replacement].gsub(/\${md\[\d+\]}/, placeholder)
|
92
|
+
break
|
93
|
+
end
|
94
|
+
|
95
|
+
if @deep_rename
|
96
|
+
if value.is_a? Hash
|
97
|
+
value = replace_key value
|
98
|
+
elsif value.is_a? Array
|
99
|
+
value = value.map { |v| v.is_a?(Hash) ? replace_key(v) : v }
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
new_record[key] = value
|
104
|
+
end
|
105
|
+
|
106
|
+
new_record
|
107
|
+
end
|
108
|
+
|
109
|
+
def get_placeholder match_data
|
110
|
+
placeholder = {}
|
111
|
+
|
112
|
+
match_data.to_a.each_with_index do |e, idx|
|
113
|
+
placeholder["${md[#{idx}]}"] = e
|
114
|
+
end
|
115
|
+
|
116
|
+
placeholder
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
data/test/helper.rb
CHANGED
@@ -0,0 +1,215 @@
|
|
1
|
+
require 'helper'
|
2
|
+
|
3
|
+
class RenameKeyFilterTest < Test::Unit::TestCase
|
4
|
+
MATCH_TAG = 'incoming_tag'
|
5
|
+
RENAME_RULE_CONFIG = 'rename_rule1 ^\$(.+) x$${md[1]}'
|
6
|
+
REPLACE_RULE_CONFIG = 'replace_rule1 ^\$ x'
|
7
|
+
|
8
|
+
def setup
|
9
|
+
omit("Fluentd v0.12 or later is required.") unless defined?(Fluent::Filter)
|
10
|
+
Fluent::Test.setup
|
11
|
+
end
|
12
|
+
|
13
|
+
def create_driver(conf, tag = MATCH_TAG)
|
14
|
+
Fluent::Test::FilterTestDriver.new(Fluent::RenameKeyFilter, tag).configure(conf)
|
15
|
+
end
|
16
|
+
|
17
|
+
def test_config_error
|
18
|
+
assert_raise(Fluent::ConfigError) { create_driver('') }
|
19
|
+
|
20
|
+
assert_raise(Fluent::ConfigError) { create_driver('rename_rule1 ^$(.+?) ') }
|
21
|
+
|
22
|
+
assert_raise(Fluent::ConfigError) {
|
23
|
+
config_dup_rules_for_a_key = %q[
|
24
|
+
rename_rule1 ^\$(.+) ${md[1]}
|
25
|
+
rename_rule2 ^\$(.+) ${md[1]} something
|
26
|
+
]
|
27
|
+
create_driver(config_dup_rules_for_a_key)
|
28
|
+
}
|
29
|
+
end
|
30
|
+
|
31
|
+
def test_config_success
|
32
|
+
config_multiple_rules = %q[
|
33
|
+
rename_rule1 ^\$(.+)1 x$${md[1]}
|
34
|
+
rename_rule2 ^\$(.+)2(\d+) ${md[1]}_${md[2]}
|
35
|
+
]
|
36
|
+
|
37
|
+
d = create_driver config_multiple_rules
|
38
|
+
assert_equal '^\$(.+)1 x$${md[1]}', d.instance.config['rename_rule1']
|
39
|
+
assert_equal '^\$(.+)2(\d+) ${md[1]}_${md[2]}', d.instance.config['rename_rule2']
|
40
|
+
end
|
41
|
+
|
42
|
+
def test_parse_rename_rule
|
43
|
+
parsed = Fluent::RenameKeyOutput.new.parse_rename_rule '(reg)(exp) ${md[1]} ${md[2]}'
|
44
|
+
assert_equal 2, parsed.length
|
45
|
+
assert_equal '(reg)(exp)', parsed[0]
|
46
|
+
assert_equal '${md[1]} ${md[2]}', parsed[1]
|
47
|
+
end
|
48
|
+
|
49
|
+
def test_parse_replace_rule_with_replacement
|
50
|
+
# Replace hyphens with underscores
|
51
|
+
parsed = Fluent::RenameKeyOutput.new.parse_replace_rule '- _'
|
52
|
+
assert_equal 2, parsed.length
|
53
|
+
assert_equal '-', parsed[0]
|
54
|
+
assert_equal '_', parsed[1]
|
55
|
+
end
|
56
|
+
|
57
|
+
def test_parse_replace_rule_without_replacement
|
58
|
+
# Remove all parenthesis hyphens and spaces
|
59
|
+
parsed = Fluent::RenameKeyOutput.new.parse_replace_rule '[()-\s]'
|
60
|
+
assert_equal 2, parsed.length
|
61
|
+
assert_equal '[()-\s]', parsed[0]
|
62
|
+
assert parsed[1].nil?
|
63
|
+
end
|
64
|
+
|
65
|
+
def test_rename_rule_emit_deep_rename_hash
|
66
|
+
d = create_driver RENAME_RULE_CONFIG
|
67
|
+
d.run do
|
68
|
+
d.emit '$key1' => 'value1', 'key2' => {'$key3' => 'value3', '$key4'=> {'$key5' => 'value5'} }
|
69
|
+
end
|
70
|
+
|
71
|
+
emits = d.emits
|
72
|
+
assert_equal %w[x$key1 key2], emits[0][2].keys
|
73
|
+
assert_equal %w[x$key3 x$key4], emits[0][2]['key2'].keys
|
74
|
+
assert_equal ['x$key5'], emits[0][2]['key2']['x$key4'].keys
|
75
|
+
end
|
76
|
+
|
77
|
+
def test_rename_rule_emit_deep_rename_array
|
78
|
+
d = create_driver RENAME_RULE_CONFIG
|
79
|
+
d.run do
|
80
|
+
d.emit '$key1' => 'value1', 'key2' => [{'$key3' => 'value3'}, {'$key4'=> {'$key5' => 'value5'}}]
|
81
|
+
end
|
82
|
+
|
83
|
+
emits = d.emits
|
84
|
+
assert_equal %w[x$key3 x$key4], emits[0][2]['key2'].flat_map(&:keys)
|
85
|
+
assert_equal ['x$key5'], emits[0][2]['key2'][1]['x$key4'].keys
|
86
|
+
end
|
87
|
+
|
88
|
+
def test_rename_rule_emit_deep_rename_off
|
89
|
+
config = %Q[
|
90
|
+
#{RENAME_RULE_CONFIG}
|
91
|
+
deep_rename false
|
92
|
+
]
|
93
|
+
|
94
|
+
d = create_driver config
|
95
|
+
d.run do
|
96
|
+
d.emit '$key1' => 'value1', 'key2' => {'$key3'=>'value3', '$key4'=> 'value4'}
|
97
|
+
end
|
98
|
+
|
99
|
+
emits = d.emits
|
100
|
+
assert_equal %w[$key3 $key4], emits[0][2]['key2'].keys
|
101
|
+
end
|
102
|
+
|
103
|
+
def test_rename_rule_emit_with_match_data
|
104
|
+
d = create_driver 'rename_rule1 (\w+)\s(\w+)\s(\w+) ${md[3]} ${md[2]} ${md[1]}'
|
105
|
+
d.run do
|
106
|
+
d.emit 'key1 key2 key3' => 'value'
|
107
|
+
end
|
108
|
+
emits = d.emits
|
109
|
+
assert_equal 1, emits.length
|
110
|
+
assert_equal ['key3 key2 key1'], emits[0][2].keys
|
111
|
+
end
|
112
|
+
|
113
|
+
def test_multiple_rename_rules_emit
|
114
|
+
config_multiple_rules = %q[
|
115
|
+
rename_rule1 ^(\w+)\s1 ${md[1]}_1
|
116
|
+
rename_rule2 ^(\w+)\s2 ${md[1]}_2
|
117
|
+
]
|
118
|
+
|
119
|
+
d = create_driver config_multiple_rules
|
120
|
+
d.run do
|
121
|
+
d.emit 'key 1' => 'value1', 'key 2' => 'value2'
|
122
|
+
end
|
123
|
+
|
124
|
+
emits = d.emits
|
125
|
+
assert_equal %w[key_1 key_2], emits[0][2].keys
|
126
|
+
end
|
127
|
+
|
128
|
+
def test_replace_rule_emit_deep_rename_hash
|
129
|
+
d = create_driver 'replace_rule1 ^(\$) x'
|
130
|
+
|
131
|
+
d.run do
|
132
|
+
d.emit '$key1' => 'value1', 'key2' => { 'key3' => 'value3', '$key4' => 'value4' }
|
133
|
+
end
|
134
|
+
emits = d.emits
|
135
|
+
assert_equal %w[xkey1 key2], emits[0][2].keys
|
136
|
+
assert_equal %w[key3 xkey4], emits[0][2]['key2'].keys
|
137
|
+
end
|
138
|
+
|
139
|
+
def test_replace_rule_emit_with_match_data
|
140
|
+
d = create_driver 'rename_rule1 (\w+)\s(\w+)\s(\w+) ${md[3]} ${md[2]} ${md[1]}'
|
141
|
+
d.run do
|
142
|
+
d.emit 'key1 key2 key3' => 'value'
|
143
|
+
end
|
144
|
+
emits = d.emits
|
145
|
+
assert_equal 1, emits.length
|
146
|
+
assert_equal ['key3 key2 key1'], emits[0][2].keys
|
147
|
+
end
|
148
|
+
|
149
|
+
def test_replace_rule_emit_deep_rename_array
|
150
|
+
d = create_driver 'replace_rule1 ^(\$) x${md[1]}'
|
151
|
+
|
152
|
+
d.run do
|
153
|
+
d.emit '$key1' => 'value1', 'key2' => [{'$key3' => 'value3'}, {'$key4' => {'$key5' => 'value5'}}]
|
154
|
+
end
|
155
|
+
|
156
|
+
emits = d.emits
|
157
|
+
assert_equal %w[x$key3 x$key4], emits[0][2]['key2'].flat_map(&:keys)
|
158
|
+
assert_equal %w[x$key5], emits[0][2]['key2'][1]['x$key4'].keys
|
159
|
+
end
|
160
|
+
|
161
|
+
def test_replace_rule_emit_deep_rename_off
|
162
|
+
config = %Q[
|
163
|
+
#{REPLACE_RULE_CONFIG}
|
164
|
+
deep_rename false
|
165
|
+
]
|
166
|
+
|
167
|
+
d = create_driver config
|
168
|
+
d.run do
|
169
|
+
d.emit '$key1' => 'value1', 'key2' => {'$key3'=>'value3', '$key4'=> 'value4'}
|
170
|
+
end
|
171
|
+
|
172
|
+
emits = d.emits
|
173
|
+
assert_equal %w[$key3 $key4], emits[0][2]['key2'].keys
|
174
|
+
end
|
175
|
+
|
176
|
+
def test_replace_rule_emit_remove_matched_when_no_replacement
|
177
|
+
d = create_driver 'replace_rule1 [\s/()]'
|
178
|
+
d.run do
|
179
|
+
d.emit 'key (/1 )' => 'value1'
|
180
|
+
end
|
181
|
+
|
182
|
+
emits = d.emits
|
183
|
+
assert_equal %w[key1], emits[0][2].keys
|
184
|
+
end
|
185
|
+
|
186
|
+
def test_multiple_replace_rules_emit
|
187
|
+
config_multiple_rules = %q[
|
188
|
+
replace_rule1 ^(\w+)\s(\d) ${md[1]}${md[2]}
|
189
|
+
replace_rule2 [\s()]
|
190
|
+
]
|
191
|
+
|
192
|
+
d = create_driver config_multiple_rules
|
193
|
+
d.run do
|
194
|
+
d.emit 'key 1' => 'value1', 'key (2)' => 'value2'
|
195
|
+
end
|
196
|
+
|
197
|
+
emits = d.emits
|
198
|
+
assert_equal %w[key1 key2], emits[0][2].keys
|
199
|
+
end
|
200
|
+
|
201
|
+
def test_combined_rename_rule_and_replace_rule
|
202
|
+
config_combined_rules = %q[
|
203
|
+
rename_rule1 ^(.+)\s(one) ${md[1]}1
|
204
|
+
replace_rule2 [\s()]
|
205
|
+
]
|
206
|
+
|
207
|
+
d = create_driver config_combined_rules
|
208
|
+
d.run do
|
209
|
+
d.emit '(key) one (x)' => 'value1', 'key (2)' => 'value2'
|
210
|
+
end
|
211
|
+
|
212
|
+
emits = d.emits
|
213
|
+
assert_equal %w[key1 key2], emits[0][2].keys
|
214
|
+
end
|
215
|
+
end
|
metadata
CHANGED
@@ -1,15 +1,16 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-rename-key
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shunwen Hsiao
|
8
8
|
- Julian Grinblat
|
9
|
+
- Hiroshi Hatake
|
9
10
|
autorequire:
|
10
11
|
bindir: bin
|
11
12
|
cert_chain: []
|
12
|
-
date: 2016-
|
13
|
+
date: 2016-11-29 00:00:00.000000000 Z
|
13
14
|
dependencies:
|
14
15
|
- !ruby/object:Gem::Dependency
|
15
16
|
name: fluentd
|
@@ -17,56 +18,62 @@ dependencies:
|
|
17
18
|
requirements:
|
18
19
|
- - ">="
|
19
20
|
- !ruby/object:Gem::Version
|
20
|
-
version:
|
21
|
+
version: 0.12.0
|
22
|
+
- - "<"
|
23
|
+
- !ruby/object:Gem::Version
|
24
|
+
version: '2'
|
21
25
|
type: :runtime
|
22
26
|
prerelease: false
|
23
27
|
version_requirements: !ruby/object:Gem::Requirement
|
24
28
|
requirements:
|
25
29
|
- - ">="
|
26
30
|
- !ruby/object:Gem::Version
|
27
|
-
version:
|
31
|
+
version: 0.12.0
|
32
|
+
- - "<"
|
33
|
+
- !ruby/object:Gem::Version
|
34
|
+
version: '2'
|
28
35
|
- !ruby/object:Gem::Dependency
|
29
36
|
name: test-unit
|
30
37
|
requirement: !ruby/object:Gem::Requirement
|
31
38
|
requirements:
|
32
|
-
- - "
|
39
|
+
- - "~>"
|
33
40
|
- !ruby/object:Gem::Version
|
34
|
-
version: 3.
|
41
|
+
version: '3.2'
|
35
42
|
type: :development
|
36
43
|
prerelease: false
|
37
44
|
version_requirements: !ruby/object:Gem::Requirement
|
38
45
|
requirements:
|
39
|
-
- - "
|
46
|
+
- - "~>"
|
40
47
|
- !ruby/object:Gem::Version
|
41
|
-
version: 3.
|
48
|
+
version: '3.2'
|
42
49
|
- !ruby/object:Gem::Dependency
|
43
50
|
name: rake
|
44
51
|
requirement: !ruby/object:Gem::Requirement
|
45
52
|
requirements:
|
46
|
-
- - "
|
53
|
+
- - "~>"
|
47
54
|
- !ruby/object:Gem::Version
|
48
|
-
version: '
|
55
|
+
version: '11.3'
|
49
56
|
type: :development
|
50
57
|
prerelease: false
|
51
58
|
version_requirements: !ruby/object:Gem::Requirement
|
52
59
|
requirements:
|
53
|
-
- - "
|
60
|
+
- - "~>"
|
54
61
|
- !ruby/object:Gem::Version
|
55
|
-
version: '
|
62
|
+
version: '11.3'
|
56
63
|
- !ruby/object:Gem::Dependency
|
57
64
|
name: coveralls
|
58
65
|
requirement: !ruby/object:Gem::Requirement
|
59
66
|
requirements:
|
60
|
-
- - "
|
67
|
+
- - "~>"
|
61
68
|
- !ruby/object:Gem::Version
|
62
|
-
version: '0'
|
69
|
+
version: '0.8'
|
63
70
|
type: :development
|
64
71
|
prerelease: false
|
65
72
|
version_requirements: !ruby/object:Gem::Requirement
|
66
73
|
requirements:
|
67
|
-
- - "
|
74
|
+
- - "~>"
|
68
75
|
- !ruby/object:Gem::Version
|
69
|
-
version: '0'
|
76
|
+
version: '0.8'
|
70
77
|
description:
|
71
78
|
email:
|
72
79
|
- hsiaoshunwen@gmail.com
|
@@ -81,8 +88,11 @@ files:
|
|
81
88
|
- README.md
|
82
89
|
- Rakefile
|
83
90
|
- fluent-plugin-rename-key.gemspec
|
91
|
+
- lib/fluent/plugin/filter_rename_key.rb
|
84
92
|
- lib/fluent/plugin/out_rename_key.rb
|
93
|
+
- lib/fluent/plugin/rename_key_util.rb
|
85
94
|
- test/helper.rb
|
95
|
+
- test/plugin/test_filter_rename_key.rb
|
86
96
|
- test/plugin/test_out_rename_key.rb
|
87
97
|
homepage: https://github.com/shunwen/fluent-plugin-rename-key
|
88
98
|
licenses:
|
@@ -96,7 +106,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
96
106
|
requirements:
|
97
107
|
- - ">="
|
98
108
|
- !ruby/object:Gem::Version
|
99
|
-
version:
|
109
|
+
version: '2.2'
|
100
110
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
101
111
|
requirements:
|
102
112
|
- - ">="
|
@@ -104,9 +114,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
104
114
|
version: '0'
|
105
115
|
requirements: []
|
106
116
|
rubyforge_project:
|
107
|
-
rubygems_version: 2.6.
|
117
|
+
rubygems_version: 2.6.4
|
108
118
|
signing_key:
|
109
119
|
specification_version: 4
|
110
120
|
summary: Fluentd output plugin. Rename keys which match given regular expressions,
|
111
121
|
assign new tags and re-emit the records.
|
112
122
|
test_files: []
|
123
|
+
has_rdoc:
|