fluent-plugin-record-modifier 0.6.1 → 0.6.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ChangeLog +5 -0
- data/README.md +14 -0
- data/VERSION +1 -1
- data/lib/fluent/plugin/filter_record_modifier.rb +20 -5
- data/test/test_filter_record_modifier.rb +23 -8
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d387097d02bdd7a23188147b637a43f66fe55698
|
4
|
+
data.tar.gz: 8e77cfa13ed400aba3ea7b6a974a71443a599742
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2b52745de9ae3a6a22d8cee708a2def04b70a96d61cc2221a99156031b9f6d849d2b49a561bdecc30807a3f027f77c6818bc3a784c9bd6836b8a31e8cfcaf944
|
7
|
+
data.tar.gz: '08a27b0a0475c8af1f6804318a22ec816a3084a30ac5047afed1dbf97c30c297f4d2f5d648cc112dc9f8d2eb06e9019bc8a9db75c79d0db912d67692ac85a2c7'
|
data/ChangeLog
CHANGED
data/README.md
CHANGED
@@ -55,6 +55,20 @@ But unlike `record_transformer`, `record_modifier` doesn't support following fea
|
|
55
55
|
- tag_suffix and tag_prefix
|
56
56
|
- dynamic key placeholder
|
57
57
|
|
58
|
+
### prepare_value
|
59
|
+
|
60
|
+
Prepare values for filtering. This ruby code is evaluated in `configure` phase and prepared values can be used in `<record>`. Here is an example:
|
61
|
+
|
62
|
+
<filter pattern>
|
63
|
+
@type record_modifier
|
64
|
+
prepare_value require 'foo'; @foo = Foo.new
|
65
|
+
<record>
|
66
|
+
key ${@foo.method1}
|
67
|
+
</record>
|
68
|
+
</filter>
|
69
|
+
|
70
|
+
This feature is useful for using external library.
|
71
|
+
|
58
72
|
### char_encoding
|
59
73
|
|
60
74
|
Fluentd including some plugins treats logs as a BINARY by default to forward.
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.6.
|
1
|
+
0.6.2
|
@@ -4,6 +4,11 @@ module Fluent
|
|
4
4
|
class RecordModifierFilter < Filter
|
5
5
|
Fluent::Plugin.register_filter('record_modifier', self)
|
6
6
|
|
7
|
+
config_param :prepare_value, :string, default: nil,
|
8
|
+
desc: <<-DESC
|
9
|
+
Prepare values for filtering in configure phase. Prepared values can be used in <record>.
|
10
|
+
You can write any ruby code.
|
11
|
+
DESC
|
7
12
|
config_param :char_encoding, :string, default: nil,
|
8
13
|
desc: <<-DESC
|
9
14
|
Fluentd including some plugins treats the logs as a BINARY by default to forward.
|
@@ -26,7 +31,7 @@ Modified events will have only specified keys (if exist in original events).
|
|
26
31
|
This option is exclusive with `remove_keys`.
|
27
32
|
DESC
|
28
33
|
|
29
|
-
BUILTIN_CONFIGURATIONS = %W(type @type log_level @log_level id @id char_encoding remove_keys whitelist_keys)
|
34
|
+
BUILTIN_CONFIGURATIONS = %W(type @type log_level @log_level id @id char_encoding remove_keys whitelist_keys prepare_value)
|
30
35
|
|
31
36
|
def configure(conf)
|
32
37
|
super
|
@@ -67,7 +72,7 @@ DESC
|
|
67
72
|
check_config_placeholders(k, v)
|
68
73
|
element.has_key?(k) # to suppress unread configuration warning
|
69
74
|
@has_tag_parts = true if v.include?('tag_parts')
|
70
|
-
@map[k] = DynamicExpander.new(k, v)
|
75
|
+
@map[k] = DynamicExpander.new(k, v, @prepare_value)
|
71
76
|
end
|
72
77
|
end
|
73
78
|
|
@@ -117,7 +122,7 @@ DESC
|
|
117
122
|
value.force_encoding(@from_enc)
|
118
123
|
elsif value.is_a?(Hash)
|
119
124
|
value.each_pair { |k, v|
|
120
|
-
if v.frozen?
|
125
|
+
if v.frozen? && v.is_a?(String)
|
121
126
|
value[k] = set_encoding(v.dup)
|
122
127
|
else
|
123
128
|
set_encoding(v)
|
@@ -125,6 +130,8 @@ DESC
|
|
125
130
|
}
|
126
131
|
elsif value.is_a?(Array)
|
127
132
|
value.each { |v| set_encoding(v) }
|
133
|
+
else
|
134
|
+
value
|
128
135
|
end
|
129
136
|
end
|
130
137
|
|
@@ -134,7 +141,7 @@ DESC
|
|
134
141
|
value.encode!(@to_enc, @from_enc, :invalid => :replace, :undef => :replace)
|
135
142
|
elsif value.is_a?(Hash)
|
136
143
|
value.each_pair { |k, v|
|
137
|
-
if v.frozen?
|
144
|
+
if v.frozen? && v.is_a?(String)
|
138
145
|
value[k] = convert_encoding(v.dup)
|
139
146
|
else
|
140
147
|
convert_encoding(v)
|
@@ -142,6 +149,8 @@ DESC
|
|
142
149
|
}
|
143
150
|
elsif value.is_a?(Array)
|
144
151
|
value.each { |v| convert_encoding(v) }
|
152
|
+
else
|
153
|
+
value
|
145
154
|
end
|
146
155
|
end
|
147
156
|
|
@@ -156,7 +165,7 @@ DESC
|
|
156
165
|
end
|
157
166
|
|
158
167
|
class DynamicExpander
|
159
|
-
def initialize(param_key, param_value)
|
168
|
+
def initialize(param_key, param_value, prepare_value)
|
160
169
|
if param_value.include?('${')
|
161
170
|
__str_eval_code__ = parse_parameter(param_value)
|
162
171
|
|
@@ -172,6 +181,12 @@ DESC
|
|
172
181
|
@param_value = param_value
|
173
182
|
end
|
174
183
|
|
184
|
+
begin
|
185
|
+
eval prepare_value if prepare_value
|
186
|
+
rescue SyntaxError
|
187
|
+
raise ConfigError, "Pass invalid syntax parameter : key = prepare_value, value = #{prepare_value}"
|
188
|
+
end
|
189
|
+
|
175
190
|
begin
|
176
191
|
# check eval genarates wrong code or not
|
177
192
|
expand(nil, nil, nil, nil)
|
@@ -126,6 +126,21 @@ class RecordModifierFilterTest < Test::Unit::TestCase
|
|
126
126
|
assert_equal [{"k1" => 'v', "k2" => 'v'}], d.filtered_as_array.map(&:last)
|
127
127
|
end
|
128
128
|
|
129
|
+
def test_prepare_values
|
130
|
+
d = create_driver %[
|
131
|
+
prepare_value @foo = 'foo'
|
132
|
+
<record>
|
133
|
+
test_key ${@foo}
|
134
|
+
</record>
|
135
|
+
]
|
136
|
+
|
137
|
+
d.run do
|
138
|
+
d.emit("k1" => 'v')
|
139
|
+
end
|
140
|
+
|
141
|
+
assert_equal [{"k1" => 'v', "test_key" => 'foo'}], d.filtered_as_array.map(&:last)
|
142
|
+
end
|
143
|
+
|
129
144
|
sub_test_case 'frozen check' do
|
130
145
|
def test_set_char_encoding
|
131
146
|
d = create_driver %[
|
@@ -133,13 +148,13 @@ class RecordModifierFilterTest < Test::Unit::TestCase
|
|
133
148
|
]
|
134
149
|
|
135
150
|
d.run do
|
136
|
-
d.emit("k" => 'v'.force_encoding('BINARY').freeze)
|
137
|
-
d.emit("k" => {"l" => 'v'.force_encoding('BINARY').freeze})
|
151
|
+
d.emit("k" => 'v'.force_encoding('BINARY').freeze, 'n' => 1)
|
152
|
+
d.emit("k" => {"l" => 'v'.force_encoding('BINARY').freeze, 'n' => 1})
|
138
153
|
end
|
139
154
|
|
140
155
|
assert_equal [
|
141
|
-
{"k" => 'v'.force_encoding('UTF-8')},
|
142
|
-
{"k" => {"l" => 'v'.force_encoding('UTF-8')}},
|
156
|
+
{"k" => 'v'.force_encoding('UTF-8'), 'n' => 1},
|
157
|
+
{"k" => {"l" => 'v'.force_encoding('UTF-8'), 'n' => 1}},
|
143
158
|
], d.filtered_as_array.map { |e| e.last }
|
144
159
|
end
|
145
160
|
|
@@ -149,13 +164,13 @@ class RecordModifierFilterTest < Test::Unit::TestCase
|
|
149
164
|
]
|
150
165
|
|
151
166
|
d.run do
|
152
|
-
d.emit("k" => 'v'.force_encoding('utf-8').freeze)
|
153
|
-
d.emit("k" => {"l" => 'v'.force_encoding('utf-8').freeze})
|
167
|
+
d.emit("k" => 'v'.force_encoding('utf-8').freeze, 'n' => 1)
|
168
|
+
d.emit("k" => {"l" => 'v'.force_encoding('utf-8').freeze, 'n' => 1})
|
154
169
|
end
|
155
170
|
|
156
171
|
assert_equal [
|
157
|
-
{"k" => 'v'.force_encoding('cp932')},
|
158
|
-
{"k" => {"l" => 'v'.force_encoding('cp932')}},
|
172
|
+
{"k" => 'v'.force_encoding('cp932'), 'n' => 1},
|
173
|
+
{"k" => {"l" => 'v'.force_encoding('cp932'), 'n' => 1}},
|
159
174
|
], d.filtered_as_array.map { |e| e.last }
|
160
175
|
end
|
161
176
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-record-modifier
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.6.
|
4
|
+
version: 0.6.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Masahiro Nakagawa
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-12-
|
11
|
+
date: 2017-12-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|