fluent-plugin-record-modifier 0.6.2 → 1.0.0.rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +4 -4
- data/ChangeLog +3 -13
- data/Gemfile +0 -2
- data/README.md +10 -15
- data/VERSION +1 -1
- data/fluent-plugin-record-modifier.gemspec +1 -1
- data/lib/fluent/plugin/filter_record_modifier.rb +25 -55
- data/lib/fluent/plugin/out_record_modifier.rb +9 -9
- data/test/test_filter_record_modifier.rb +38 -76
- data/test/test_out_record_modifier.rb +21 -22
- metadata +7 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: bd31ea6b4191c161f1518dbdfb7cbc57b9840902
|
4
|
+
data.tar.gz: ce1368e0f6b21845deafeb6c72ec60b06d9a2146
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 00b348bb146840c6e754d974e4400e5dee8444289f3f6bdee1e19b0932f76ece490639e1d0bf2ff0f90e4491b1fc127792b5fadf1130cf08c13e5627b4c30bf6
|
7
|
+
data.tar.gz: 4fcc2ab8f86c7e7e40df240fdbdde4e88c80fb4f2e1dd06ae45b766faac620c381e762946b0cbdb6c3795ead3e3e62b43219325a2f635d4b0bf6510fff3a586f
|
data/.travis.yml
CHANGED
@@ -1,25 +1,25 @@
|
|
1
1
|
language: ruby
|
2
2
|
|
3
3
|
rvm:
|
4
|
+
- 2.0.0
|
4
5
|
- 2.1
|
5
6
|
- 2.2.5
|
6
7
|
- 2.3.1
|
7
|
-
- 2.4.2
|
8
8
|
- ruby-head
|
9
|
+
- rbx
|
9
10
|
|
10
11
|
branches:
|
11
12
|
only:
|
12
13
|
- master
|
13
|
-
- v0.12
|
14
14
|
|
15
15
|
gemfile:
|
16
16
|
- Gemfile
|
17
17
|
|
18
|
-
sudo: false
|
19
|
-
|
20
18
|
matrix:
|
21
19
|
allow_failures:
|
22
20
|
- rvm: ruby-head
|
21
|
+
- rvm: rbx
|
22
|
+
- rvm: 2.0.0
|
23
23
|
|
24
24
|
before_install: gem update bundler
|
25
25
|
script: bundle exec rake test
|
data/ChangeLog
CHANGED
@@ -1,17 +1,7 @@
|
|
1
|
-
Release 0.
|
1
|
+
Release 1.0.0.rc1 - 2016/08/23
|
2
2
|
|
3
|
-
*
|
4
|
-
|
5
|
-
|
6
|
-
Release 0.6.1 - 2017/12/12
|
7
|
-
|
8
|
-
* Drop fluentd v0.10 support
|
9
|
-
* Add frozen object support
|
10
|
-
https://github.com/repeatedly/fluent-plugin-record-modifier/pull/26
|
11
|
-
|
12
|
-
Release 0.6.0 - 2017/06/06
|
13
|
-
|
14
|
-
* Drop fluentd v0.10 support
|
3
|
+
* Use v0.14 API
|
4
|
+
https://github.com/repeatedly/fluent-plugin-record-modifier/pull/14
|
15
5
|
|
16
6
|
Release 0.5.0 - 2016/08/23
|
17
7
|
|
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -5,6 +5,15 @@ Adding arbitary field to event record without customizing existence plugin.
|
|
5
5
|
For example, generated event from *in_tail* doesn't contain "hostname" of running machine.
|
6
6
|
In this case, you can use *record_modifier* to add "hostname" field to event record.
|
7
7
|
|
8
|
+
## Requirements
|
9
|
+
|
10
|
+
| fluent-plugin-record-modifier | fluentd | ruby |
|
11
|
+
|--------------------------------|---------|------|
|
12
|
+
| >= 1.0.0 | >= v0.14.0 | >= 2.1 |
|
13
|
+
| < 1.0.0 | >= v0.12.0 | >= 1.9 |
|
14
|
+
|
15
|
+
NOTE: fluent-plugin-record-modifier v1.0.0 is now RC. We will release stable v1.0.0 soon.
|
16
|
+
|
8
17
|
## Installation
|
9
18
|
|
10
19
|
Use RubyGems:
|
@@ -43,7 +52,7 @@ You can also use `record_transformer` like `${xxx}` placeholders and access `tag
|
|
43
52
|
|
44
53
|
<record>
|
45
54
|
tag ${tag}
|
46
|
-
tag_extract ${tag_parts[0]}-${
|
55
|
+
tag_extract ${tag_parts[0]}-${tag_parts[1]}-foo
|
47
56
|
formatted_time ${Time.at(time).to_s}
|
48
57
|
new_field foo:${record['key1'] + record['dict']['key']}
|
49
58
|
</record>
|
@@ -55,20 +64,6 @@ But unlike `record_transformer`, `record_modifier` doesn't support following fea
|
|
55
64
|
- tag_suffix and tag_prefix
|
56
65
|
- dynamic key placeholder
|
57
66
|
|
58
|
-
### prepare_value
|
59
|
-
|
60
|
-
Prepare values for filtering. This ruby code is evaluated in `configure` phase and prepared values can be used in `<record>`. Here is an example:
|
61
|
-
|
62
|
-
<filter pattern>
|
63
|
-
@type record_modifier
|
64
|
-
prepare_value require 'foo'; @foo = Foo.new
|
65
|
-
<record>
|
66
|
-
key ${@foo.method1}
|
67
|
-
</record>
|
68
|
-
</filter>
|
69
|
-
|
70
|
-
This feature is useful for using external library.
|
71
|
-
|
72
67
|
### char_encoding
|
73
68
|
|
74
69
|
Fluentd including some plugins treats logs as a BINARY by default to forward.
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
1.0.0.rc1
|
@@ -17,7 +17,7 @@ Gem::Specification.new do |gem|
|
|
17
17
|
gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
18
18
|
gem.require_paths = ['lib']
|
19
19
|
|
20
|
-
gem.add_dependency "fluentd", [">= 0.
|
20
|
+
gem.add_dependency "fluentd", [">= 0.10.58", "< 2"]
|
21
21
|
gem.add_development_dependency "rake", ">= 0.9.2"
|
22
22
|
gem.add_development_dependency("test-unit", ["~> 3.1.4"])
|
23
23
|
end
|
@@ -1,14 +1,9 @@
|
|
1
|
-
require 'fluent/filter'
|
1
|
+
require 'fluent/plugin/filter'
|
2
2
|
|
3
3
|
module Fluent
|
4
|
-
class RecordModifierFilter < Filter
|
4
|
+
class Plugin::RecordModifierFilter < Plugin::Filter
|
5
5
|
Fluent::Plugin.register_filter('record_modifier', self)
|
6
6
|
|
7
|
-
config_param :prepare_value, :string, default: nil,
|
8
|
-
desc: <<-DESC
|
9
|
-
Prepare values for filtering in configure phase. Prepared values can be used in <record>.
|
10
|
-
You can write any ruby code.
|
11
|
-
DESC
|
12
7
|
config_param :char_encoding, :string, default: nil,
|
13
8
|
desc: <<-DESC
|
14
9
|
Fluentd including some plugins treats the logs as a BINARY by default to forward.
|
@@ -31,7 +26,7 @@ Modified events will have only specified keys (if exist in original events).
|
|
31
26
|
This option is exclusive with `remove_keys`.
|
32
27
|
DESC
|
33
28
|
|
34
|
-
BUILTIN_CONFIGURATIONS = %W(type @type log_level @log_level id @id char_encoding remove_keys whitelist_keys
|
29
|
+
BUILTIN_CONFIGURATIONS = %W(type @type log_level @log_level id @id char_encoding remove_keys whitelist_keys)
|
35
30
|
|
36
31
|
def configure(conf)
|
37
32
|
super
|
@@ -50,6 +45,7 @@ DESC
|
|
50
45
|
end
|
51
46
|
}
|
52
47
|
|
48
|
+
@to_enc = nil
|
53
49
|
if @char_encoding
|
54
50
|
from, to = @char_encoding.split(':', 2)
|
55
51
|
@from_enc = Encoding.find(from)
|
@@ -72,7 +68,7 @@ DESC
|
|
72
68
|
check_config_placeholders(k, v)
|
73
69
|
element.has_key?(k) # to suppress unread configuration warning
|
74
70
|
@has_tag_parts = true if v.include?('tag_parts')
|
75
|
-
@map[k] = DynamicExpander.new(k, v
|
71
|
+
@map[k] = DynamicExpander.new(k, v)
|
76
72
|
end
|
77
73
|
end
|
78
74
|
|
@@ -88,31 +84,27 @@ DESC
|
|
88
84
|
GC.start
|
89
85
|
end
|
90
86
|
|
91
|
-
def
|
92
|
-
new_es = MultiEventStream.new
|
87
|
+
def filter(tag, time, record)
|
93
88
|
tag_parts = @has_tag_parts ? tag.split('.') : nil
|
94
89
|
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
}
|
90
|
+
@map.each_pair { |k, v|
|
91
|
+
record[k] = v.expand(tag, time, record, tag_parts)
|
92
|
+
}
|
99
93
|
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
end
|
109
|
-
record = modified
|
94
|
+
if @remove_keys
|
95
|
+
@remove_keys.each { |v|
|
96
|
+
record.delete(v)
|
97
|
+
}
|
98
|
+
elsif @whitelist_keys
|
99
|
+
modified = {}
|
100
|
+
record.each do |k, v|
|
101
|
+
modified[k] = v if @whitelist_keys.include?(k)
|
110
102
|
end
|
103
|
+
record = modified
|
104
|
+
end
|
111
105
|
|
112
|
-
|
113
|
-
|
114
|
-
}
|
115
|
-
new_es
|
106
|
+
record = change_encoding(record) if @char_encoding
|
107
|
+
record
|
116
108
|
end
|
117
109
|
|
118
110
|
private
|
@@ -121,17 +113,9 @@ DESC
|
|
121
113
|
if value.is_a?(String)
|
122
114
|
value.force_encoding(@from_enc)
|
123
115
|
elsif value.is_a?(Hash)
|
124
|
-
value.each_pair { |k, v|
|
125
|
-
if v.frozen? && v.is_a?(String)
|
126
|
-
value[k] = set_encoding(v.dup)
|
127
|
-
else
|
128
|
-
set_encoding(v)
|
129
|
-
end
|
130
|
-
}
|
116
|
+
value.each_pair { |k, v| set_encoding(v) }
|
131
117
|
elsif value.is_a?(Array)
|
132
118
|
value.each { |v| set_encoding(v) }
|
133
|
-
else
|
134
|
-
value
|
135
119
|
end
|
136
120
|
end
|
137
121
|
|
@@ -140,17 +124,9 @@ DESC
|
|
140
124
|
value.force_encoding(@from_enc) if value.encoding == Encoding::BINARY
|
141
125
|
value.encode!(@to_enc, @from_enc, :invalid => :replace, :undef => :replace)
|
142
126
|
elsif value.is_a?(Hash)
|
143
|
-
value.each_pair { |k, v|
|
144
|
-
if v.frozen? && v.is_a?(String)
|
145
|
-
value[k] = convert_encoding(v.dup)
|
146
|
-
else
|
147
|
-
convert_encoding(v)
|
148
|
-
end
|
149
|
-
}
|
127
|
+
value.each_pair { |k, v| convert_encoding(v) }
|
150
128
|
elsif value.is_a?(Array)
|
151
129
|
value.each { |v| convert_encoding(v) }
|
152
|
-
else
|
153
|
-
value
|
154
130
|
end
|
155
131
|
end
|
156
132
|
|
@@ -165,7 +141,7 @@ DESC
|
|
165
141
|
end
|
166
142
|
|
167
143
|
class DynamicExpander
|
168
|
-
def initialize(param_key, param_value
|
144
|
+
def initialize(param_key, param_value)
|
169
145
|
if param_value.include?('${')
|
170
146
|
__str_eval_code__ = parse_parameter(param_value)
|
171
147
|
|
@@ -181,12 +157,6 @@ DESC
|
|
181
157
|
@param_value = param_value
|
182
158
|
end
|
183
159
|
|
184
|
-
begin
|
185
|
-
eval prepare_value if prepare_value
|
186
|
-
rescue SyntaxError
|
187
|
-
raise ConfigError, "Pass invalid syntax parameter : key = prepare_value, value = #{prepare_value}"
|
188
|
-
end
|
189
|
-
|
190
160
|
begin
|
191
161
|
# check eval genarates wrong code or not
|
192
162
|
expand(nil, nil, nil, nil)
|
@@ -217,5 +187,5 @@ DESC
|
|
217
187
|
end
|
218
188
|
end
|
219
189
|
end
|
220
|
-
end
|
190
|
+
end
|
221
191
|
end
|
@@ -1,9 +1,11 @@
|
|
1
|
-
require 'fluent/output'
|
1
|
+
require 'fluent/plugin/output'
|
2
2
|
|
3
3
|
module Fluent
|
4
|
-
class RecordModifierOutput < Output
|
4
|
+
class Plugin::RecordModifierOutput < Plugin::Output
|
5
5
|
Fluent::Plugin.register_output('record_modifier', self)
|
6
6
|
|
7
|
+
helpers :event_emitter, :compat_parameters, :inject
|
8
|
+
|
7
9
|
config_param :tag, :string,
|
8
10
|
desc: "The output record tag name."
|
9
11
|
config_param :char_encoding, :string, default: nil,
|
@@ -29,11 +31,10 @@ Modified events will have only specified keys (if exist in original events).
|
|
29
31
|
This option is exclusive with `remove_keys`.
|
30
32
|
DESC
|
31
33
|
|
32
|
-
|
33
|
-
|
34
|
-
BUILTIN_CONFIGURATIONS = %W(type @type @id id @log_level log_level tag include_tag_key tag_key char_encoding remove_keys whitelist_keys)
|
34
|
+
BUILTIN_CONFIGURATIONS = %W(type @type log_level @log_level id @id tag include_tag_key tag_key char_encoding remove_keys whitelist_keys)
|
35
35
|
|
36
36
|
def configure(conf)
|
37
|
+
compat_parameters_convert(conf, :buffer, :inject)
|
37
38
|
super
|
38
39
|
|
39
40
|
@map = {}
|
@@ -45,6 +46,7 @@ DESC
|
|
45
46
|
end
|
46
47
|
}
|
47
48
|
|
49
|
+
@to_enc = nil
|
48
50
|
if @char_encoding
|
49
51
|
from, to = @char_encoding.split(':', 2)
|
50
52
|
@from_enc = Encoding.find(from)
|
@@ -70,15 +72,13 @@ DESC
|
|
70
72
|
end
|
71
73
|
end
|
72
74
|
|
73
|
-
def
|
75
|
+
def process(tag, es)
|
74
76
|
stream = MultiEventStream.new
|
75
77
|
es.each { |time, record|
|
76
|
-
|
78
|
+
record = inject_values_to_record(tag, time, record)
|
77
79
|
stream.add(time, modify_record(record))
|
78
80
|
}
|
79
81
|
router.emit_stream(@tag, stream)
|
80
|
-
|
81
|
-
chain.next
|
82
82
|
end
|
83
83
|
|
84
84
|
private
|
@@ -1,9 +1,9 @@
|
|
1
|
+
# coding: utf-8
|
1
2
|
require 'fluent/test'
|
3
|
+
require 'fluent/test/driver/filter'
|
2
4
|
require 'fluent/plugin/filter_record_modifier'
|
3
5
|
require 'test/unit'
|
4
6
|
|
5
|
-
exit unless defined?(Fluent::Filter)
|
6
|
-
|
7
7
|
class RecordModifierFilterTest < Test::Unit::TestCase
|
8
8
|
def setup
|
9
9
|
Fluent::Test.setup
|
@@ -11,6 +11,7 @@ class RecordModifierFilterTest < Test::Unit::TestCase
|
|
11
11
|
end
|
12
12
|
|
13
13
|
CONFIG = %q!
|
14
|
+
type record_modifier
|
14
15
|
remove_keys hoge
|
15
16
|
|
16
17
|
<record>
|
@@ -22,7 +23,7 @@ class RecordModifierFilterTest < Test::Unit::TestCase
|
|
22
23
|
!
|
23
24
|
|
24
25
|
def create_driver(conf = CONFIG)
|
25
|
-
Fluent::Test::
|
26
|
+
Fluent::Test::Driver::Filter.new(Fluent::Plugin::RecordModifierFilter).configure(conf)
|
26
27
|
end
|
27
28
|
|
28
29
|
def get_hostname
|
@@ -35,143 +36,104 @@ class RecordModifierFilterTest < Test::Unit::TestCase
|
|
35
36
|
map = d.instance.instance_variable_get(:@map)
|
36
37
|
|
37
38
|
map.each_pair { |k, v|
|
38
|
-
assert v.is_a?(Fluent::RecordModifierFilter::DynamicExpander)
|
39
|
+
assert v.is_a?(Fluent::Plugin::RecordModifierFilter::DynamicExpander)
|
39
40
|
}
|
40
41
|
end
|
41
42
|
|
42
43
|
def test_format
|
43
44
|
d = create_driver
|
44
45
|
|
45
|
-
d.run do
|
46
|
-
d.
|
47
|
-
d.
|
46
|
+
d.run(default_tag: @tag) do
|
47
|
+
d.feed("a" => 1)
|
48
|
+
d.feed("a" => 2)
|
48
49
|
end
|
49
50
|
|
50
51
|
mapped = {'gen_host' => get_hostname, 'foo' => 'bar', 'included_tag' => @tag, 'tag_wrap' => "-#{@tag.split('.')[0]}-#{@tag.split('.')[1]}-"}
|
51
52
|
assert_equal [
|
52
53
|
{"a" => 1}.merge(mapped),
|
53
54
|
{"a" => 2}.merge(mapped),
|
54
|
-
], d.
|
55
|
+
], d.filtered.map { |e| e.last }
|
55
56
|
end
|
56
57
|
|
57
58
|
def test_set_char_encoding
|
58
59
|
d = create_driver %[
|
60
|
+
type record_modifier
|
61
|
+
|
59
62
|
char_encoding utf-8
|
60
63
|
]
|
61
64
|
|
62
|
-
d.run do
|
63
|
-
d.
|
64
|
-
d.
|
65
|
-
d.
|
65
|
+
d.run(default_tag: @tag) do
|
66
|
+
d.feed("k" => 'v'.force_encoding('BINARY'))
|
67
|
+
d.feed("k" => %w(v ビ).map{|v| v.force_encoding('BINARY')})
|
68
|
+
d.feed("k" => {"l" => 'ビ'.force_encoding('BINARY')})
|
66
69
|
end
|
67
70
|
|
68
71
|
assert_equal [
|
69
72
|
{"k" => 'v'.force_encoding('UTF-8')},
|
70
73
|
{"k" => %w(v ビ).map{|v| v.force_encoding('UTF-8')}},
|
71
74
|
{"k" => {"l" => 'ビ'.force_encoding('UTF-8')}},
|
72
|
-
], d.
|
75
|
+
], d.filtered.map { |e| e.last }
|
73
76
|
end
|
74
77
|
|
75
78
|
def test_convert_char_encoding
|
76
79
|
d = create_driver %[
|
80
|
+
type record_modifier
|
81
|
+
|
77
82
|
char_encoding utf-8:cp932
|
78
83
|
]
|
79
84
|
|
80
|
-
d.run do
|
81
|
-
d.
|
82
|
-
d.
|
83
|
-
d.
|
85
|
+
d.run(default_tag: @tag) do
|
86
|
+
d.feed("k" => 'v'.force_encoding('utf-8'))
|
87
|
+
d.feed("k" => %w(v ビ).map{|v| v.force_encoding('utf-8')})
|
88
|
+
d.feed("k" => {"l" => 'ビ'.force_encoding('utf-8')})
|
84
89
|
end
|
85
90
|
|
86
91
|
assert_equal [
|
87
92
|
{"k" => 'v'.force_encoding('cp932')},
|
88
93
|
{"k" => %w(v ビ).map{|v| v.encode!('cp932')}},
|
89
94
|
{"k" => {"l" => 'ビ'.encode!('cp932')}},
|
90
|
-
], d.
|
95
|
+
], d.filtered.map { |e| e.last }
|
91
96
|
end
|
92
97
|
|
93
98
|
def test_remove_one_key
|
94
99
|
d = create_driver %[
|
100
|
+
type record_modifier
|
101
|
+
|
95
102
|
remove_keys k1
|
96
103
|
]
|
97
104
|
|
98
|
-
d.run do
|
99
|
-
d.
|
105
|
+
d.run(default_tag: @tag) do
|
106
|
+
d.feed("k1" => 'v', "k2" => 'v')
|
100
107
|
end
|
101
108
|
|
102
|
-
assert_equal [{"k2" => 'v'}], d.
|
109
|
+
assert_equal [{"k2" => 'v'}], d.filtered.map { |e| e.last }
|
103
110
|
end
|
104
111
|
|
105
112
|
def test_remove_multiple_keys
|
106
113
|
d = create_driver %[
|
114
|
+
type record_modifier
|
115
|
+
|
107
116
|
remove_keys k1, k2, k3
|
108
117
|
]
|
109
118
|
|
110
|
-
d.run do
|
111
|
-
d.
|
119
|
+
d.run(default_tag: @tag) do
|
120
|
+
d.feed({"k1" => 'v', "k2" => 'v', "k4" => 'v'})
|
112
121
|
end
|
113
122
|
|
114
|
-
assert_equal [{"k4" => 'v'}], d.
|
123
|
+
assert_equal [{"k4" => 'v'}], d.filtered.map { |e| e.last }
|
115
124
|
end
|
116
125
|
|
117
126
|
def test_remove_non_whitelist_keys
|
118
127
|
d = create_driver %[
|
119
|
-
|
120
|
-
]
|
121
|
-
|
122
|
-
d.run do
|
123
|
-
d.emit("k1" => 'v', "k2" => 'v', "k4" => 'v', "k5" => 'v')
|
124
|
-
end
|
125
|
-
|
126
|
-
assert_equal [{"k1" => 'v', "k2" => 'v'}], d.filtered_as_array.map(&:last)
|
127
|
-
end
|
128
|
+
type record_modifier
|
128
129
|
|
129
|
-
|
130
|
-
d = create_driver %[
|
131
|
-
prepare_value @foo = 'foo'
|
132
|
-
<record>
|
133
|
-
test_key ${@foo}
|
134
|
-
</record>
|
130
|
+
whitelist_keys k1, k2, k3
|
135
131
|
]
|
136
132
|
|
137
|
-
d.run do
|
138
|
-
d.
|
139
|
-
end
|
140
|
-
|
141
|
-
assert_equal [{"k1" => 'v', "test_key" => 'foo'}], d.filtered_as_array.map(&:last)
|
142
|
-
end
|
143
|
-
|
144
|
-
sub_test_case 'frozen check' do
|
145
|
-
def test_set_char_encoding
|
146
|
-
d = create_driver %[
|
147
|
-
char_encoding utf-8
|
148
|
-
]
|
149
|
-
|
150
|
-
d.run do
|
151
|
-
d.emit("k" => 'v'.force_encoding('BINARY').freeze, 'n' => 1)
|
152
|
-
d.emit("k" => {"l" => 'v'.force_encoding('BINARY').freeze, 'n' => 1})
|
153
|
-
end
|
154
|
-
|
155
|
-
assert_equal [
|
156
|
-
{"k" => 'v'.force_encoding('UTF-8'), 'n' => 1},
|
157
|
-
{"k" => {"l" => 'v'.force_encoding('UTF-8'), 'n' => 1}},
|
158
|
-
], d.filtered_as_array.map { |e| e.last }
|
133
|
+
d.run(default_tag: @tag) do
|
134
|
+
d.feed("k1" => 'v', "k2" => 'v', "k4" => 'v', "k5" => 'v')
|
159
135
|
end
|
160
136
|
|
161
|
-
|
162
|
-
d = create_driver %[
|
163
|
-
char_encoding utf-8:cp932
|
164
|
-
]
|
165
|
-
|
166
|
-
d.run do
|
167
|
-
d.emit("k" => 'v'.force_encoding('utf-8').freeze, 'n' => 1)
|
168
|
-
d.emit("k" => {"l" => 'v'.force_encoding('utf-8').freeze, 'n' => 1})
|
169
|
-
end
|
170
|
-
|
171
|
-
assert_equal [
|
172
|
-
{"k" => 'v'.force_encoding('cp932'), 'n' => 1},
|
173
|
-
{"k" => {"l" => 'v'.force_encoding('cp932'), 'n' => 1}},
|
174
|
-
], d.filtered_as_array.map { |e| e.last }
|
175
|
-
end
|
137
|
+
assert_equal [{"k1" => 'v', "k2" => 'v'}], d.filtered.map(&:last)
|
176
138
|
end
|
177
139
|
end
|
@@ -1,4 +1,4 @@
|
|
1
|
-
require 'fluent/test'
|
1
|
+
require 'fluent/test/driver/output'
|
2
2
|
require 'fluent/plugin/out_record_modifier'
|
3
3
|
|
4
4
|
|
@@ -19,7 +19,7 @@ class RecordModifierOutputTest < Test::Unit::TestCase
|
|
19
19
|
!
|
20
20
|
|
21
21
|
def create_driver(conf = CONFIG)
|
22
|
-
Fluent::Test::
|
22
|
+
Fluent::Test::Driver::Output.new(Fluent::Plugin::RecordModifierOutput).configure(conf)
|
23
23
|
end
|
24
24
|
|
25
25
|
def get_hostname
|
@@ -38,16 +38,16 @@ class RecordModifierOutputTest < Test::Unit::TestCase
|
|
38
38
|
def test_format
|
39
39
|
d = create_driver
|
40
40
|
|
41
|
-
d.run do
|
42
|
-
d.
|
43
|
-
d.
|
41
|
+
d.run(default_tag: 'test_tag') do
|
42
|
+
d.feed({"a" => 1})
|
43
|
+
d.feed({"a" => 2})
|
44
44
|
end
|
45
45
|
|
46
46
|
mapped = {'gen_host' => get_hostname, 'foo' => 'bar', 'included_tag' => 'test_tag'}
|
47
47
|
assert_equal [
|
48
48
|
{"a" => 1}.merge(mapped),
|
49
49
|
{"a" => 2}.merge(mapped),
|
50
|
-
], d.
|
50
|
+
], d.events.map { |e| e.last }
|
51
51
|
end
|
52
52
|
|
53
53
|
def test_set_char_encoding
|
@@ -58,12 +58,11 @@ class RecordModifierOutputTest < Test::Unit::TestCase
|
|
58
58
|
char_encoding utf-8
|
59
59
|
]
|
60
60
|
|
61
|
-
|
62
|
-
|
63
|
-
d.emit("k" => 'v'.force_encoding('BINARY'))
|
61
|
+
d.run(default_tag: 'test_tag') do
|
62
|
+
d.feed({"k" => 'v'.force_encoding('BINARY')})
|
64
63
|
end
|
65
64
|
|
66
|
-
assert_equal [{"k" => 'v'.force_encoding('UTF-8')}], d.
|
65
|
+
assert_equal [{"k" => 'v'.force_encoding('UTF-8')}], d.events.map { |e| e.last }
|
67
66
|
end
|
68
67
|
|
69
68
|
def test_convert_char_encoding
|
@@ -74,11 +73,11 @@ class RecordModifierOutputTest < Test::Unit::TestCase
|
|
74
73
|
char_encoding utf-8:cp932
|
75
74
|
]
|
76
75
|
|
77
|
-
d.run do
|
78
|
-
d.
|
76
|
+
d.run(default_tag: 'test_tag') do
|
77
|
+
d.feed("k" => 'v'.force_encoding('utf-8'))
|
79
78
|
end
|
80
79
|
|
81
|
-
assert_equal [{"k" => 'v'.force_encoding('cp932')}], d.
|
80
|
+
assert_equal [{"k" => 'v'.force_encoding('cp932')}], d.events.map { |e| e.last }
|
82
81
|
end
|
83
82
|
|
84
83
|
def test_remove_one_key
|
@@ -89,11 +88,11 @@ class RecordModifierOutputTest < Test::Unit::TestCase
|
|
89
88
|
remove_keys k1
|
90
89
|
]
|
91
90
|
|
92
|
-
d.run do
|
93
|
-
d.
|
91
|
+
d.run(default_tag: 'test_tag') do
|
92
|
+
d.feed({"k1" => 'v', "k2" => 'v'})
|
94
93
|
end
|
95
94
|
|
96
|
-
assert_equal [{"k2" => 'v'}], d.
|
95
|
+
assert_equal [{"k2" => 'v'}], d.events.map { |e| e.last }
|
97
96
|
end
|
98
97
|
|
99
98
|
def test_remove_multiple_keys
|
@@ -104,11 +103,11 @@ class RecordModifierOutputTest < Test::Unit::TestCase
|
|
104
103
|
remove_keys k1, k2, k3
|
105
104
|
]
|
106
105
|
|
107
|
-
d.run do
|
108
|
-
d.
|
106
|
+
d.run(default_tag: 'test_tag') do
|
107
|
+
d.feed({"k1" => 'v', "k2" => 'v', "k4" => 'v'})
|
109
108
|
end
|
110
109
|
|
111
|
-
assert_equal [{"k4" => 'v'}], d.
|
110
|
+
assert_equal [{"k4" => 'v'}], d.events.map { |e| e.last }
|
112
111
|
end
|
113
112
|
|
114
113
|
def test_remove_non_whitelist_keys
|
@@ -119,10 +118,10 @@ class RecordModifierOutputTest < Test::Unit::TestCase
|
|
119
118
|
whitelist_keys k1, k2, k3
|
120
119
|
]
|
121
120
|
|
122
|
-
d.run do
|
123
|
-
d.
|
121
|
+
d.run(default_tag: 'test_tag') do
|
122
|
+
d.feed({"k1" => 'v', "k2" => 'v', "k4" => 'v', "k5" => 'v'})
|
124
123
|
end
|
125
124
|
|
126
|
-
assert_equal [{"k1" => 'v', "k2" => 'v'}], d.
|
125
|
+
assert_equal [{"k1" => 'v', "k2" => 'v'}], d.events.map { |e| e.last }
|
127
126
|
end
|
128
127
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-record-modifier
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 1.0.0.rc1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Masahiro Nakagawa
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-06-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -16,7 +16,7 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
19
|
+
version: 0.10.58
|
20
20
|
- - "<"
|
21
21
|
- !ruby/object:Gem::Version
|
22
22
|
version: '2'
|
@@ -26,7 +26,7 @@ dependencies:
|
|
26
26
|
requirements:
|
27
27
|
- - ">="
|
28
28
|
- !ruby/object:Gem::Version
|
29
|
-
version:
|
29
|
+
version: 0.10.58
|
30
30
|
- - "<"
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: '2'
|
@@ -90,12 +90,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
90
90
|
version: '0'
|
91
91
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
92
92
|
requirements:
|
93
|
-
- - "
|
93
|
+
- - ">"
|
94
94
|
- !ruby/object:Gem::Version
|
95
|
-
version:
|
95
|
+
version: 1.3.1
|
96
96
|
requirements: []
|
97
97
|
rubyforge_project:
|
98
|
-
rubygems_version: 2.6.
|
98
|
+
rubygems_version: 2.6.11
|
99
99
|
signing_key:
|
100
100
|
specification_version: 4
|
101
101
|
summary: Filter plugin for modifying event record
|