fluent-plugin-record-modifier 0.6.2 → 1.0.0.rc1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +4 -4
- data/ChangeLog +3 -13
- data/Gemfile +0 -2
- data/README.md +10 -15
- data/VERSION +1 -1
- data/fluent-plugin-record-modifier.gemspec +1 -1
- data/lib/fluent/plugin/filter_record_modifier.rb +25 -55
- data/lib/fluent/plugin/out_record_modifier.rb +9 -9
- data/test/test_filter_record_modifier.rb +38 -76
- data/test/test_out_record_modifier.rb +21 -22
- metadata +7 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: bd31ea6b4191c161f1518dbdfb7cbc57b9840902
|
4
|
+
data.tar.gz: ce1368e0f6b21845deafeb6c72ec60b06d9a2146
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 00b348bb146840c6e754d974e4400e5dee8444289f3f6bdee1e19b0932f76ece490639e1d0bf2ff0f90e4491b1fc127792b5fadf1130cf08c13e5627b4c30bf6
|
7
|
+
data.tar.gz: 4fcc2ab8f86c7e7e40df240fdbdde4e88c80fb4f2e1dd06ae45b766faac620c381e762946b0cbdb6c3795ead3e3e62b43219325a2f635d4b0bf6510fff3a586f
|
data/.travis.yml
CHANGED
@@ -1,25 +1,25 @@
|
|
1
1
|
language: ruby
|
2
2
|
|
3
3
|
rvm:
|
4
|
+
- 2.0.0
|
4
5
|
- 2.1
|
5
6
|
- 2.2.5
|
6
7
|
- 2.3.1
|
7
|
-
- 2.4.2
|
8
8
|
- ruby-head
|
9
|
+
- rbx
|
9
10
|
|
10
11
|
branches:
|
11
12
|
only:
|
12
13
|
- master
|
13
|
-
- v0.12
|
14
14
|
|
15
15
|
gemfile:
|
16
16
|
- Gemfile
|
17
17
|
|
18
|
-
sudo: false
|
19
|
-
|
20
18
|
matrix:
|
21
19
|
allow_failures:
|
22
20
|
- rvm: ruby-head
|
21
|
+
- rvm: rbx
|
22
|
+
- rvm: 2.0.0
|
23
23
|
|
24
24
|
before_install: gem update bundler
|
25
25
|
script: bundle exec rake test
|
data/ChangeLog
CHANGED
@@ -1,17 +1,7 @@
|
|
1
|
-
Release 0.
|
1
|
+
Release 1.0.0.rc1 - 2016/08/23
|
2
2
|
|
3
|
-
*
|
4
|
-
|
5
|
-
|
6
|
-
Release 0.6.1 - 2017/12/12
|
7
|
-
|
8
|
-
* Drop fluentd v0.10 support
|
9
|
-
* Add frozen object support
|
10
|
-
https://github.com/repeatedly/fluent-plugin-record-modifier/pull/26
|
11
|
-
|
12
|
-
Release 0.6.0 - 2017/06/06
|
13
|
-
|
14
|
-
* Drop fluentd v0.10 support
|
3
|
+
* Use v0.14 API
|
4
|
+
https://github.com/repeatedly/fluent-plugin-record-modifier/pull/14
|
15
5
|
|
16
6
|
Release 0.5.0 - 2016/08/23
|
17
7
|
|
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -5,6 +5,15 @@ Adding arbitary field to event record without customizing existence plugin.
|
|
5
5
|
For example, generated event from *in_tail* doesn't contain "hostname" of running machine.
|
6
6
|
In this case, you can use *record_modifier* to add "hostname" field to event record.
|
7
7
|
|
8
|
+
## Requirements
|
9
|
+
|
10
|
+
| fluent-plugin-record-modifier | fluentd | ruby |
|
11
|
+
|--------------------------------|---------|------|
|
12
|
+
| >= 1.0.0 | >= v0.14.0 | >= 2.1 |
|
13
|
+
| < 1.0.0 | >= v0.12.0 | >= 1.9 |
|
14
|
+
|
15
|
+
NOTE: fluent-plugin-record-modifier v1.0.0 is now RC. We will release stable v1.0.0 soon.
|
16
|
+
|
8
17
|
## Installation
|
9
18
|
|
10
19
|
Use RubyGems:
|
@@ -43,7 +52,7 @@ You can also use `record_transformer` like `${xxx}` placeholders and access `tag
|
|
43
52
|
|
44
53
|
<record>
|
45
54
|
tag ${tag}
|
46
|
-
tag_extract ${tag_parts[0]}-${
|
55
|
+
tag_extract ${tag_parts[0]}-${tag_parts[1]}-foo
|
47
56
|
formatted_time ${Time.at(time).to_s}
|
48
57
|
new_field foo:${record['key1'] + record['dict']['key']}
|
49
58
|
</record>
|
@@ -55,20 +64,6 @@ But unlike `record_transformer`, `record_modifier` doesn't support following fea
|
|
55
64
|
- tag_suffix and tag_prefix
|
56
65
|
- dynamic key placeholder
|
57
66
|
|
58
|
-
### prepare_value
|
59
|
-
|
60
|
-
Prepare values for filtering. This ruby code is evaluated in `configure` phase and prepared values can be used in `<record>`. Here is an example:
|
61
|
-
|
62
|
-
<filter pattern>
|
63
|
-
@type record_modifier
|
64
|
-
prepare_value require 'foo'; @foo = Foo.new
|
65
|
-
<record>
|
66
|
-
key ${@foo.method1}
|
67
|
-
</record>
|
68
|
-
</filter>
|
69
|
-
|
70
|
-
This feature is useful for using external library.
|
71
|
-
|
72
67
|
### char_encoding
|
73
68
|
|
74
69
|
Fluentd including some plugins treats logs as a BINARY by default to forward.
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
1.0.0.rc1
|
@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
|
|
17
17
|
gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
18
18
|
gem.require_paths = ['lib']
|
19
19
|
|
20
|
-
gem.add_dependency "fluentd", [">= 0.
|
20
|
+
gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
|
21
21
|
gem.add_development_dependency "rake", ">= 0.9.2"
|
22
22
|
gem.add_development_dependency("test-unit", ["~> 3.1.4"])
|
23
23
|
end
|
@@ -1,14 +1,9 @@
|
|
1
|
-
require 'fluent/filter'
|
1
|
+
require 'fluent/plugin/filter'
|
2
2
|
|
3
3
|
module Fluent
|
4
|
-
class RecordModifierFilter < Filter
|
4
|
+
class Plugin::RecordModifierFilter < Plugin::Filter
|
5
5
|
Fluent::Plugin.register_filter('record_modifier', self)
|
6
6
|
|
7
|
-
config_param :prepare_value, :string, default: nil,
|
8
|
-
desc: <<-DESC
|
9
|
-
Prepare values for filtering in configure phase. Prepared values can be used in <record>.
|
10
|
-
You can write any ruby code.
|
11
|
-
DESC
|
12
7
|
config_param :char_encoding, :string, default: nil,
|
13
8
|
desc: <<-DESC
|
14
9
|
Fluentd including some plugins treats the logs as a BINARY by default to forward.
|
@@ -31,7 +26,7 @@ Modified events will have only specified keys (if exist in original events).
|
|
31
26
|
This option is exclusive with `remove_keys`.
|
32
27
|
DESC
|
33
28
|
|
34
|
-
BUILTIN_CONFIGURATIONS = %W(type @type log_level @log_level id @id char_encoding remove_keys whitelist_keys
|
29
|
+
BUILTIN_CONFIGURATIONS = %W(type @type log_level @log_level id @id char_encoding remove_keys whitelist_keys)
|
35
30
|
|
36
31
|
def configure(conf)
|
37
32
|
super
|
@@ -50,6 +45,7 @@ DESC
|
|
50
45
|
end
|
51
46
|
}
|
52
47
|
|
48
|
+
@to_enc = nil
|
53
49
|
if @char_encoding
|
54
50
|
from, to = @char_encoding.split(':', 2)
|
55
51
|
@from_enc = Encoding.find(from)
|
@@ -72,7 +68,7 @@ DESC
|
|
72
68
|
check_config_placeholders(k, v)
|
73
69
|
element.has_key?(k) # to suppress unread configuration warning
|
74
70
|
@has_tag_parts = true if v.include?('tag_parts')
|
75
|
-
@map[k] = DynamicExpander.new(k, v
|
71
|
+
@map[k] = DynamicExpander.new(k, v)
|
76
72
|
end
|
77
73
|
end
|
78
74
|
|
@@ -88,31 +84,27 @@ DESC
|
|
88
84
|
GC.start
|
89
85
|
end
|
90
86
|
|
91
|
-
def
|
92
|
-
new_es = MultiEventStream.new
|
87
|
+
def filter(tag, time, record)
|
93
88
|
tag_parts = @has_tag_parts ? tag.split('.') : nil
|
94
89
|
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
}
|
90
|
+
@map.each_pair { |k, v|
|
91
|
+
record[k] = v.expand(tag, time, record, tag_parts)
|
92
|
+
}
|
99
93
|
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
end
|
109
|
-
record = modified
|
94
|
+
if @remove_keys
|
95
|
+
@remove_keys.each { |v|
|
96
|
+
record.delete(v)
|
97
|
+
}
|
98
|
+
elsif @whitelist_keys
|
99
|
+
modified = {}
|
100
|
+
record.each do |k, v|
|
101
|
+
modified[k] = v if @whitelist_keys.include?(k)
|
110
102
|
end
|
103
|
+
record = modified
|
104
|
+
end
|
111
105
|
|
112
|
-
|
113
|
-
|
114
|
-
}
|
115
|
-
new_es
|
106
|
+
record = change_encoding(record) if @char_encoding
|
107
|
+
record
|
116
108
|
end
|
117
109
|
|
118
110
|
private
|
@@ -121,17 +113,9 @@ DESC
|
|
121
113
|
if value.is_a?(String)
|
122
114
|
value.force_encoding(@from_enc)
|
123
115
|
elsif value.is_a?(Hash)
|
124
|
-
value.each_pair { |k, v|
|
125
|
-
if v.frozen? && v.is_a?(String)
|
126
|
-
value[k] = set_encoding(v.dup)
|
127
|
-
else
|
128
|
-
set_encoding(v)
|
129
|
-
end
|
130
|
-
}
|
116
|
+
value.each_pair { |k, v| set_encoding(v) }
|
131
117
|
elsif value.is_a?(Array)
|
132
118
|
value.each { |v| set_encoding(v) }
|
133
|
-
else
|
134
|
-
value
|
135
119
|
end
|
136
120
|
end
|
137
121
|
|
@@ -140,17 +124,9 @@ DESC
|
|
140
124
|
value.force_encoding(@from_enc) if value.encoding == Encoding::BINARY
|
141
125
|
value.encode!(@to_enc, @from_enc, :invalid => :replace, :undef => :replace)
|
142
126
|
elsif value.is_a?(Hash)
|
143
|
-
value.each_pair { |k, v|
|
144
|
-
if v.frozen? && v.is_a?(String)
|
145
|
-
value[k] = convert_encoding(v.dup)
|
146
|
-
else
|
147
|
-
convert_encoding(v)
|
148
|
-
end
|
149
|
-
}
|
127
|
+
value.each_pair { |k, v| convert_encoding(v) }
|
150
128
|
elsif value.is_a?(Array)
|
151
129
|
value.each { |v| convert_encoding(v) }
|
152
|
-
else
|
153
|
-
value
|
154
130
|
end
|
155
131
|
end
|
156
132
|
|
@@ -165,7 +141,7 @@ DESC
|
|
165
141
|
end
|
166
142
|
|
167
143
|
class DynamicExpander
|
168
|
-
def initialize(param_key, param_value
|
144
|
+
def initialize(param_key, param_value)
|
169
145
|
if param_value.include?('${')
|
170
146
|
__str_eval_code__ = parse_parameter(param_value)
|
171
147
|
|
@@ -181,12 +157,6 @@ DESC
|
|
181
157
|
@param_value = param_value
|
182
158
|
end
|
183
159
|
|
184
|
-
begin
|
185
|
-
eval prepare_value if prepare_value
|
186
|
-
rescue SyntaxError
|
187
|
-
raise ConfigError, "Pass invalid syntax parameter : key = prepare_value, value = #{prepare_value}"
|
188
|
-
end
|
189
|
-
|
190
160
|
begin
|
191
161
|
# check eval genarates wrong code or not
|
192
162
|
expand(nil, nil, nil, nil)
|
@@ -217,5 +187,5 @@ DESC
|
|
217
187
|
end
|
218
188
|
end
|
219
189
|
end
|
220
|
-
end
|
190
|
+
end
|
221
191
|
end
|
@@ -1,9 +1,11 @@
|
|
1
|
-
require 'fluent/output'
|
1
|
+
require 'fluent/plugin/output'
|
2
2
|
|
3
3
|
module Fluent
|
4
|
-
class RecordModifierOutput < Output
|
4
|
+
class Plugin::RecordModifierOutput < Plugin::Output
|
5
5
|
Fluent::Plugin.register_output('record_modifier', self)
|
6
6
|
|
7
|
+
helpers :event_emitter, :compat_parameters, :inject
|
8
|
+
|
7
9
|
config_param :tag, :string,
|
8
10
|
desc: "The output record tag name."
|
9
11
|
config_param :char_encoding, :string, default: nil,
|
@@ -29,11 +31,10 @@ Modified events will have only specified keys (if exist in original events).
|
|
29
31
|
This option is exclusive with `remove_keys`.
|
30
32
|
DESC
|
31
33
|
|
32
|
-
|
33
|
-
|
34
|
-
BUILTIN_CONFIGURATIONS = %W(type @type @id id @log_level log_level tag include_tag_key tag_key char_encoding remove_keys whitelist_keys)
|
34
|
+
BUILTIN_CONFIGURATIONS = %W(type @type log_level @log_level id @id tag include_tag_key tag_key char_encoding remove_keys whitelist_keys)
|
35
35
|
|
36
36
|
def configure(conf)
|
37
|
+
compat_parameters_convert(conf, :buffer, :inject)
|
37
38
|
super
|
38
39
|
|
39
40
|
@map = {}
|
@@ -45,6 +46,7 @@ DESC
|
|
45
46
|
end
|
46
47
|
}
|
47
48
|
|
49
|
+
@to_enc = nil
|
48
50
|
if @char_encoding
|
49
51
|
from, to = @char_encoding.split(':', 2)
|
50
52
|
@from_enc = Encoding.find(from)
|
@@ -70,15 +72,13 @@ DESC
|
|
70
72
|
end
|
71
73
|
end
|
72
74
|
|
73
|
-
def
|
75
|
+
def process(tag, es)
|
74
76
|
stream = MultiEventStream.new
|
75
77
|
es.each { |time, record|
|
76
|
-
|
78
|
+
record = inject_values_to_record(tag, time, record)
|
77
79
|
stream.add(time, modify_record(record))
|
78
80
|
}
|
79
81
|
router.emit_stream(@tag, stream)
|
80
|
-
|
81
|
-
chain.next
|
82
82
|
end
|
83
83
|
|
84
84
|
private
|
@@ -1,9 +1,9 @@
|
|
1
|
+
# coding: utf-8
|
1
2
|
require 'fluent/test'
|
3
|
+
require 'fluent/test/driver/filter'
|
2
4
|
require 'fluent/plugin/filter_record_modifier'
|
3
5
|
require 'test/unit'
|
4
6
|
|
5
|
-
exit unless defined?(Fluent::Filter)
|
6
|
-
|
7
7
|
class RecordModifierFilterTest < Test::Unit::TestCase
|
8
8
|
def setup
|
9
9
|
Fluent::Test.setup
|
@@ -11,6 +11,7 @@ class RecordModifierFilterTest < Test::Unit::TestCase
|
|
11
11
|
end
|
12
12
|
|
13
13
|
CONFIG = %q!
|
14
|
+
type record_modifier
|
14
15
|
remove_keys hoge
|
15
16
|
|
16
17
|
<record>
|
@@ -22,7 +23,7 @@ class RecordModifierFilterTest < Test::Unit::TestCase
|
|
22
23
|
!
|
23
24
|
|
24
25
|
def create_driver(conf = CONFIG)
|
25
|
-
Fluent::Test::
|
26
|
+
Fluent::Test::Driver::Filter.new(Fluent::Plugin::RecordModifierFilter).configure(conf)
|
26
27
|
end
|
27
28
|
|
28
29
|
def get_hostname
|
@@ -35,143 +36,104 @@ class RecordModifierFilterTest < Test::Unit::TestCase
|
|
35
36
|
map = d.instance.instance_variable_get(:@map)
|
36
37
|
|
37
38
|
map.each_pair { |k, v|
|
38
|
-
assert v.is_a?(Fluent::RecordModifierFilter::DynamicExpander)
|
39
|
+
assert v.is_a?(Fluent::Plugin::RecordModifierFilter::DynamicExpander)
|
39
40
|
}
|
40
41
|
end
|
41
42
|
|
42
43
|
def test_format
|
43
44
|
d = create_driver
|
44
45
|
|
45
|
-
d.run do
|
46
|
-
d.
|
47
|
-
d.
|
46
|
+
d.run(default_tag: @tag) do
|
47
|
+
d.feed("a" => 1)
|
48
|
+
d.feed("a" => 2)
|
48
49
|
end
|
49
50
|
|
50
51
|
mapped = {'gen_host' => get_hostname, 'foo' => 'bar', 'included_tag' => @tag, 'tag_wrap' => "-#{@tag.split('.')[0]}-#{@tag.split('.')[1]}-"}
|
51
52
|
assert_equal [
|
52
53
|
{"a" => 1}.merge(mapped),
|
53
54
|
{"a" => 2}.merge(mapped),
|
54
|
-
], d.
|
55
|
+
], d.filtered.map { |e| e.last }
|
55
56
|
end
|
56
57
|
|
57
58
|
def test_set_char_encoding
|
58
59
|
d = create_driver %[
|
60
|
+
type record_modifier
|
61
|
+
|
59
62
|
char_encoding utf-8
|
60
63
|
]
|
61
64
|
|
62
|
-
d.run do
|
63
|
-
d.
|
64
|
-
d.
|
65
|
-
d.
|
65
|
+
d.run(default_tag: @tag) do
|
66
|
+
d.feed("k" => 'v'.force_encoding('BINARY'))
|
67
|
+
d.feed("k" => %w(v ビ).map{|v| v.force_encoding('BINARY')})
|
68
|
+
d.feed("k" => {"l" => 'ビ'.force_encoding('BINARY')})
|
66
69
|
end
|
67
70
|
|
68
71
|
assert_equal [
|
69
72
|
{"k" => 'v'.force_encoding('UTF-8')},
|
70
73
|
{"k" => %w(v ビ).map{|v| v.force_encoding('UTF-8')}},
|
71
74
|
{"k" => {"l" => 'ビ'.force_encoding('UTF-8')}},
|
72
|
-
], d.
|
75
|
+
], d.filtered.map { |e| e.last }
|
73
76
|
end
|
74
77
|
|
75
78
|
def test_convert_char_encoding
|
76
79
|
d = create_driver %[
|
80
|
+
type record_modifier
|
81
|
+
|
77
82
|
char_encoding utf-8:cp932
|
78
83
|
]
|
79
84
|
|
80
|
-
d.run do
|
81
|
-
d.
|
82
|
-
d.
|
83
|
-
d.
|
85
|
+
d.run(default_tag: @tag) do
|
86
|
+
d.feed("k" => 'v'.force_encoding('utf-8'))
|
87
|
+
d.feed("k" => %w(v ビ).map{|v| v.force_encoding('utf-8')})
|
88
|
+
d.feed("k" => {"l" => 'ビ'.force_encoding('utf-8')})
|
84
89
|
end
|
85
90
|
|
86
91
|
assert_equal [
|
87
92
|
{"k" => 'v'.force_encoding('cp932')},
|
88
93
|
{"k" => %w(v ビ).map{|v| v.encode!('cp932')}},
|
89
94
|
{"k" => {"l" => 'ビ'.encode!('cp932')}},
|
90
|
-
], d.
|
95
|
+
], d.filtered.map { |e| e.last }
|
91
96
|
end
|
92
97
|
|
93
98
|
def test_remove_one_key
|
94
99
|
d = create_driver %[
|
100
|
+
type record_modifier
|
101
|
+
|
95
102
|
remove_keys k1
|
96
103
|
]
|
97
104
|
|
98
|
-
d.run do
|
99
|
-
d.
|
105
|
+
d.run(default_tag: @tag) do
|
106
|
+
d.feed("k1" => 'v', "k2" => 'v')
|
100
107
|
end
|
101
108
|
|
102
|
-
assert_equal [{"k2" => 'v'}], d.
|
109
|
+
assert_equal [{"k2" => 'v'}], d.filtered.map { |e| e.last }
|
103
110
|
end
|
104
111
|
|
105
112
|
def test_remove_multiple_keys
|
106
113
|
d = create_driver %[
|
114
|
+
type record_modifier
|
115
|
+
|
107
116
|
remove_keys k1, k2, k3
|
108
117
|
]
|
109
118
|
|
110
|
-
d.run do
|
111
|
-
d.
|
119
|
+
d.run(default_tag: @tag) do
|
120
|
+
d.feed({"k1" => 'v', "k2" => 'v', "k4" => 'v'})
|
112
121
|
end
|
113
122
|
|
114
|
-
assert_equal [{"k4" => 'v'}], d.
|
123
|
+
assert_equal [{"k4" => 'v'}], d.filtered.map { |e| e.last }
|
115
124
|
end
|
116
125
|
|
117
126
|
def test_remove_non_whitelist_keys
|
118
127
|
d = create_driver %[
|
119
|
-
|
120
|
-
]
|
121
|
-
|
122
|
-
d.run do
|
123
|
-
d.emit("k1" => 'v', "k2" => 'v', "k4" => 'v', "k5" => 'v')
|
124
|
-
end
|
125
|
-
|
126
|
-
assert_equal [{"k1" => 'v', "k2" => 'v'}], d.filtered_as_array.map(&:last)
|
127
|
-
end
|
128
|
+
type record_modifier
|
128
129
|
|
129
|
-
|
130
|
-
d = create_driver %[
|
131
|
-
prepare_value @foo = 'foo'
|
132
|
-
<record>
|
133
|
-
test_key ${@foo}
|
134
|
-
</record>
|
130
|
+
whitelist_keys k1, k2, k3
|
135
131
|
]
|
136
132
|
|
137
|
-
d.run do
|
138
|
-
d.
|
139
|
-
end
|
140
|
-
|
141
|
-
assert_equal [{"k1" => 'v', "test_key" => 'foo'}], d.filtered_as_array.map(&:last)
|
142
|
-
end
|
143
|
-
|
144
|
-
sub_test_case 'frozen check' do
|
145
|
-
def test_set_char_encoding
|
146
|
-
d = create_driver %[
|
147
|
-
char_encoding utf-8
|
148
|
-
]
|
149
|
-
|
150
|
-
d.run do
|
151
|
-
d.emit("k" => 'v'.force_encoding('BINARY').freeze, 'n' => 1)
|
152
|
-
d.emit("k" => {"l" => 'v'.force_encoding('BINARY').freeze, 'n' => 1})
|
153
|
-
end
|
154
|
-
|
155
|
-
assert_equal [
|
156
|
-
{"k" => 'v'.force_encoding('UTF-8'), 'n' => 1},
|
157
|
-
{"k" => {"l" => 'v'.force_encoding('UTF-8'), 'n' => 1}},
|
158
|
-
], d.filtered_as_array.map { |e| e.last }
|
133
|
+
d.run(default_tag: @tag) do
|
134
|
+
d.feed("k1" => 'v', "k2" => 'v', "k4" => 'v', "k5" => 'v')
|
159
135
|
end
|
160
136
|
|
161
|
-
|
162
|
-
d = create_driver %[
|
163
|
-
char_encoding utf-8:cp932
|
164
|
-
]
|
165
|
-
|
166
|
-
d.run do
|
167
|
-
d.emit("k" => 'v'.force_encoding('utf-8').freeze, 'n' => 1)
|
168
|
-
d.emit("k" => {"l" => 'v'.force_encoding('utf-8').freeze, 'n' => 1})
|
169
|
-
end
|
170
|
-
|
171
|
-
assert_equal [
|
172
|
-
{"k" => 'v'.force_encoding('cp932'), 'n' => 1},
|
173
|
-
{"k" => {"l" => 'v'.force_encoding('cp932'), 'n' => 1}},
|
174
|
-
], d.filtered_as_array.map { |e| e.last }
|
175
|
-
end
|
137
|
+
assert_equal [{"k1" => 'v', "k2" => 'v'}], d.filtered.map(&:last)
|
176
138
|
end
|
177
139
|
end
|
@@ -1,4 +1,4 @@
|
|
1
|
-
require 'fluent/test'
|
1
|
+
require 'fluent/test/driver/output'
|
2
2
|
require 'fluent/plugin/out_record_modifier'
|
3
3
|
|
4
4
|
|
@@ -19,7 +19,7 @@ class RecordModifierOutputTest < Test::Unit::TestCase
|
|
19
19
|
!
|
20
20
|
|
21
21
|
def create_driver(conf = CONFIG)
|
22
|
-
Fluent::Test::
|
22
|
+
Fluent::Test::Driver::Output.new(Fluent::Plugin::RecordModifierOutput).configure(conf)
|
23
23
|
end
|
24
24
|
|
25
25
|
def get_hostname
|
@@ -38,16 +38,16 @@ class RecordModifierOutputTest < Test::Unit::TestCase
|
|
38
38
|
def test_format
|
39
39
|
d = create_driver
|
40
40
|
|
41
|
-
d.run do
|
42
|
-
d.
|
43
|
-
d.
|
41
|
+
d.run(default_tag: 'test_tag') do
|
42
|
+
d.feed({"a" => 1})
|
43
|
+
d.feed({"a" => 2})
|
44
44
|
end
|
45
45
|
|
46
46
|
mapped = {'gen_host' => get_hostname, 'foo' => 'bar', 'included_tag' => 'test_tag'}
|
47
47
|
assert_equal [
|
48
48
|
{"a" => 1}.merge(mapped),
|
49
49
|
{"a" => 2}.merge(mapped),
|
50
|
-
], d.
|
50
|
+
], d.events.map { |e| e.last }
|
51
51
|
end
|
52
52
|
|
53
53
|
def test_set_char_encoding
|
@@ -58,12 +58,11 @@ class RecordModifierOutputTest < Test::Unit::TestCase
|
|
58
58
|
char_encoding utf-8
|
59
59
|
]
|
60
60
|
|
61
|
-
|
62
|
-
|
63
|
-
d.emit("k" => 'v'.force_encoding('BINARY'))
|
61
|
+
d.run(default_tag: 'test_tag') do
|
62
|
+
d.feed({"k" => 'v'.force_encoding('BINARY')})
|
64
63
|
end
|
65
64
|
|
66
|
-
assert_equal [{"k" => 'v'.force_encoding('UTF-8')}], d.
|
65
|
+
assert_equal [{"k" => 'v'.force_encoding('UTF-8')}], d.events.map { |e| e.last }
|
67
66
|
end
|
68
67
|
|
69
68
|
def test_convert_char_encoding
|
@@ -74,11 +73,11 @@ class RecordModifierOutputTest < Test::Unit::TestCase
|
|
74
73
|
char_encoding utf-8:cp932
|
75
74
|
]
|
76
75
|
|
77
|
-
d.run do
|
78
|
-
d.
|
76
|
+
d.run(default_tag: 'test_tag') do
|
77
|
+
d.feed("k" => 'v'.force_encoding('utf-8'))
|
79
78
|
end
|
80
79
|
|
81
|
-
assert_equal [{"k" => 'v'.force_encoding('cp932')}], d.
|
80
|
+
assert_equal [{"k" => 'v'.force_encoding('cp932')}], d.events.map { |e| e.last }
|
82
81
|
end
|
83
82
|
|
84
83
|
def test_remove_one_key
|
@@ -89,11 +88,11 @@ class RecordModifierOutputTest < Test::Unit::TestCase
|
|
89
88
|
remove_keys k1
|
90
89
|
]
|
91
90
|
|
92
|
-
d.run do
|
93
|
-
d.
|
91
|
+
d.run(default_tag: 'test_tag') do
|
92
|
+
d.feed({"k1" => 'v', "k2" => 'v'})
|
94
93
|
end
|
95
94
|
|
96
|
-
assert_equal [{"k2" => 'v'}], d.
|
95
|
+
assert_equal [{"k2" => 'v'}], d.events.map { |e| e.last }
|
97
96
|
end
|
98
97
|
|
99
98
|
def test_remove_multiple_keys
|
@@ -104,11 +103,11 @@ class RecordModifierOutputTest < Test::Unit::TestCase
|
|
104
103
|
remove_keys k1, k2, k3
|
105
104
|
]
|
106
105
|
|
107
|
-
d.run do
|
108
|
-
d.
|
106
|
+
d.run(default_tag: 'test_tag') do
|
107
|
+
d.feed({"k1" => 'v', "k2" => 'v', "k4" => 'v'})
|
109
108
|
end
|
110
109
|
|
111
|
-
assert_equal [{"k4" => 'v'}], d.
|
110
|
+
assert_equal [{"k4" => 'v'}], d.events.map { |e| e.last }
|
112
111
|
end
|
113
112
|
|
114
113
|
def test_remove_non_whitelist_keys
|
@@ -119,10 +118,10 @@ class RecordModifierOutputTest < Test::Unit::TestCase
|
|
119
118
|
whitelist_keys k1, k2, k3
|
120
119
|
]
|
121
120
|
|
122
|
-
d.run do
|
123
|
-
d.
|
121
|
+
d.run(default_tag: 'test_tag') do
|
122
|
+
d.feed({"k1" => 'v', "k2" => 'v', "k4" => 'v', "k5" => 'v'})
|
124
123
|
end
|
125
124
|
|
126
|
-
assert_equal [{"k1" => 'v', "k2" => 'v'}], d.
|
125
|
+
assert_equal [{"k1" => 'v', "k2" => 'v'}], d.events.map { |e| e.last }
|
127
126
|
end
|
128
127
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-record-modifier
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 1.0.0.rc1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Masahiro Nakagawa
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-06-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -16,7 +16,7 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
19
|
+
version: 0.10.58
|
20
20
|
- - "<"
|
21
21
|
- !ruby/object:Gem::Version
|
22
22
|
version: '2'
|
@@ -26,7 +26,7 @@ dependencies:
|
|
26
26
|
requirements:
|
27
27
|
- - ">="
|
28
28
|
- !ruby/object:Gem::Version
|
29
|
-
version:
|
29
|
+
version: 0.10.58
|
30
30
|
- - "<"
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: '2'
|
@@ -90,12 +90,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
90
90
|
version: '0'
|
91
91
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
92
92
|
requirements:
|
93
|
-
- - "
|
93
|
+
- - ">"
|
94
94
|
- !ruby/object:Gem::Version
|
95
|
-
version:
|
95
|
+
version: 1.3.1
|
96
96
|
requirements: []
|
97
97
|
rubyforge_project:
|
98
|
-
rubygems_version: 2.6.
|
98
|
+
rubygems_version: 2.6.11
|
99
99
|
signing_key:
|
100
100
|
specification_version: 4
|
101
101
|
summary: Filter plugin for modifying event record
|