fluent-plugin-record-modifier 0.6.1 → 0.6.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/ChangeLog +5 -0
- data/README.md +14 -0
- data/VERSION +1 -1
- data/lib/fluent/plugin/filter_record_modifier.rb +20 -5
- data/test/test_filter_record_modifier.rb +23 -8
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d387097d02bdd7a23188147b637a43f66fe55698
|
4
|
+
data.tar.gz: 8e77cfa13ed400aba3ea7b6a974a71443a599742
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2b52745de9ae3a6a22d8cee708a2def04b70a96d61cc2221a99156031b9f6d849d2b49a561bdecc30807a3f027f77c6818bc3a784c9bd6836b8a31e8cfcaf944
|
7
|
+
data.tar.gz: '08a27b0a0475c8af1f6804318a22ec816a3084a30ac5047afed1dbf97c30c297f4d2f5d648cc112dc9f8d2eb06e9019bc8a9db75c79d0db912d67692ac85a2c7'
|
data/ChangeLog
CHANGED
data/README.md
CHANGED
@@ -55,6 +55,20 @@ But unlike `record_transformer`, `record_modifier` doesn't support following fea
|
|
55
55
|
- tag_suffix and tag_prefix
|
56
56
|
- dynamic key placeholder
|
57
57
|
|
58
|
+
### prepare_value
|
59
|
+
|
60
|
+
Prepare values for filtering. This ruby code is evaluated in `configure` phase and prepared values can be used in `<record>`. Here is an example:
|
61
|
+
|
62
|
+
<filter pattern>
|
63
|
+
@type record_modifier
|
64
|
+
prepare_value require 'foo'; @foo = Foo.new
|
65
|
+
<record>
|
66
|
+
key ${@foo.method1}
|
67
|
+
</record>
|
68
|
+
</filter>
|
69
|
+
|
70
|
+
This feature is useful for using external library.
|
71
|
+
|
58
72
|
### char_encoding
|
59
73
|
|
60
74
|
Fluentd including some plugins treats logs as a BINARY by default to forward.
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.6.
|
1
|
+
0.6.2
|
@@ -4,6 +4,11 @@ module Fluent
|
|
4
4
|
class RecordModifierFilter < Filter
|
5
5
|
Fluent::Plugin.register_filter('record_modifier', self)
|
6
6
|
|
7
|
+
config_param :prepare_value, :string, default: nil,
|
8
|
+
desc: <<-DESC
|
9
|
+
Prepare values for filtering in configure phase. Prepared values can be used in <record>.
|
10
|
+
You can write any ruby code.
|
11
|
+
DESC
|
7
12
|
config_param :char_encoding, :string, default: nil,
|
8
13
|
desc: <<-DESC
|
9
14
|
Fluentd including some plugins treats the logs as a BINARY by default to forward.
|
@@ -26,7 +31,7 @@ Modified events will have only specified keys (if exist in original events).
|
|
26
31
|
This option is exclusive with `remove_keys`.
|
27
32
|
DESC
|
28
33
|
|
29
|
-
BUILTIN_CONFIGURATIONS = %W(type @type log_level @log_level id @id char_encoding remove_keys whitelist_keys)
|
34
|
+
BUILTIN_CONFIGURATIONS = %W(type @type log_level @log_level id @id char_encoding remove_keys whitelist_keys prepare_value)
|
30
35
|
|
31
36
|
def configure(conf)
|
32
37
|
super
|
@@ -67,7 +72,7 @@ DESC
|
|
67
72
|
check_config_placeholders(k, v)
|
68
73
|
element.has_key?(k) # to suppress unread configuration warning
|
69
74
|
@has_tag_parts = true if v.include?('tag_parts')
|
70
|
-
@map[k] = DynamicExpander.new(k, v)
|
75
|
+
@map[k] = DynamicExpander.new(k, v, @prepare_value)
|
71
76
|
end
|
72
77
|
end
|
73
78
|
|
@@ -117,7 +122,7 @@ DESC
|
|
117
122
|
value.force_encoding(@from_enc)
|
118
123
|
elsif value.is_a?(Hash)
|
119
124
|
value.each_pair { |k, v|
|
120
|
-
if v.frozen?
|
125
|
+
if v.frozen? && v.is_a?(String)
|
121
126
|
value[k] = set_encoding(v.dup)
|
122
127
|
else
|
123
128
|
set_encoding(v)
|
@@ -125,6 +130,8 @@ DESC
|
|
125
130
|
}
|
126
131
|
elsif value.is_a?(Array)
|
127
132
|
value.each { |v| set_encoding(v) }
|
133
|
+
else
|
134
|
+
value
|
128
135
|
end
|
129
136
|
end
|
130
137
|
|
@@ -134,7 +141,7 @@ DESC
|
|
134
141
|
value.encode!(@to_enc, @from_enc, :invalid => :replace, :undef => :replace)
|
135
142
|
elsif value.is_a?(Hash)
|
136
143
|
value.each_pair { |k, v|
|
137
|
-
if v.frozen?
|
144
|
+
if v.frozen? && v.is_a?(String)
|
138
145
|
value[k] = convert_encoding(v.dup)
|
139
146
|
else
|
140
147
|
convert_encoding(v)
|
@@ -142,6 +149,8 @@ DESC
|
|
142
149
|
}
|
143
150
|
elsif value.is_a?(Array)
|
144
151
|
value.each { |v| convert_encoding(v) }
|
152
|
+
else
|
153
|
+
value
|
145
154
|
end
|
146
155
|
end
|
147
156
|
|
@@ -156,7 +165,7 @@ DESC
|
|
156
165
|
end
|
157
166
|
|
158
167
|
class DynamicExpander
|
159
|
-
def initialize(param_key, param_value)
|
168
|
+
def initialize(param_key, param_value, prepare_value)
|
160
169
|
if param_value.include?('${')
|
161
170
|
__str_eval_code__ = parse_parameter(param_value)
|
162
171
|
|
@@ -172,6 +181,12 @@ DESC
|
|
172
181
|
@param_value = param_value
|
173
182
|
end
|
174
183
|
|
184
|
+
begin
|
185
|
+
eval prepare_value if prepare_value
|
186
|
+
rescue SyntaxError
|
187
|
+
raise ConfigError, "Pass invalid syntax parameter : key = prepare_value, value = #{prepare_value}"
|
188
|
+
end
|
189
|
+
|
175
190
|
begin
|
176
191
|
# check eval genarates wrong code or not
|
177
192
|
expand(nil, nil, nil, nil)
|
@@ -126,6 +126,21 @@ class RecordModifierFilterTest < Test::Unit::TestCase
|
|
126
126
|
assert_equal [{"k1" => 'v', "k2" => 'v'}], d.filtered_as_array.map(&:last)
|
127
127
|
end
|
128
128
|
|
129
|
+
def test_prepare_values
|
130
|
+
d = create_driver %[
|
131
|
+
prepare_value @foo = 'foo'
|
132
|
+
<record>
|
133
|
+
test_key ${@foo}
|
134
|
+
</record>
|
135
|
+
]
|
136
|
+
|
137
|
+
d.run do
|
138
|
+
d.emit("k1" => 'v')
|
139
|
+
end
|
140
|
+
|
141
|
+
assert_equal [{"k1" => 'v', "test_key" => 'foo'}], d.filtered_as_array.map(&:last)
|
142
|
+
end
|
143
|
+
|
129
144
|
sub_test_case 'frozen check' do
|
130
145
|
def test_set_char_encoding
|
131
146
|
d = create_driver %[
|
@@ -133,13 +148,13 @@ class RecordModifierFilterTest < Test::Unit::TestCase
|
|
133
148
|
]
|
134
149
|
|
135
150
|
d.run do
|
136
|
-
d.emit("k" => 'v'.force_encoding('BINARY').freeze)
|
137
|
-
d.emit("k" => {"l" => 'v'.force_encoding('BINARY').freeze})
|
151
|
+
d.emit("k" => 'v'.force_encoding('BINARY').freeze, 'n' => 1)
|
152
|
+
d.emit("k" => {"l" => 'v'.force_encoding('BINARY').freeze, 'n' => 1})
|
138
153
|
end
|
139
154
|
|
140
155
|
assert_equal [
|
141
|
-
{"k" => 'v'.force_encoding('UTF-8')},
|
142
|
-
{"k" => {"l" => 'v'.force_encoding('UTF-8')}},
|
156
|
+
{"k" => 'v'.force_encoding('UTF-8'), 'n' => 1},
|
157
|
+
{"k" => {"l" => 'v'.force_encoding('UTF-8'), 'n' => 1}},
|
143
158
|
], d.filtered_as_array.map { |e| e.last }
|
144
159
|
end
|
145
160
|
|
@@ -149,13 +164,13 @@ class RecordModifierFilterTest < Test::Unit::TestCase
|
|
149
164
|
]
|
150
165
|
|
151
166
|
d.run do
|
152
|
-
d.emit("k" => 'v'.force_encoding('utf-8').freeze)
|
153
|
-
d.emit("k" => {"l" => 'v'.force_encoding('utf-8').freeze})
|
167
|
+
d.emit("k" => 'v'.force_encoding('utf-8').freeze, 'n' => 1)
|
168
|
+
d.emit("k" => {"l" => 'v'.force_encoding('utf-8').freeze, 'n' => 1})
|
154
169
|
end
|
155
170
|
|
156
171
|
assert_equal [
|
157
|
-
{"k" => 'v'.force_encoding('cp932')},
|
158
|
-
{"k" => {"l" => 'v'.force_encoding('cp932')}},
|
172
|
+
{"k" => 'v'.force_encoding('cp932'), 'n' => 1},
|
173
|
+
{"k" => {"l" => 'v'.force_encoding('cp932'), 'n' => 1}},
|
159
174
|
], d.filtered_as_array.map { |e| e.last }
|
160
175
|
end
|
161
176
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-record-modifier
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.
|
4
|
+
version: 0.6.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Masahiro Nakagawa
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-12-
|
11
|
+
date: 2017-12-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|