fluent-plugin-record-modifier 0.4.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ChangeLog +7 -0
- data/README.md +25 -0
- data/VERSION +1 -1
- data/fluent-plugin-record-modifier.gemspec +1 -1
- data/lib/fluent/plugin/filter_record_modifier.rb +51 -18
- data/lib/fluent/plugin/out_record_modifier.rb +40 -7
- data/test/test_filter_record_modifier.rb +28 -3
- data/test/test_out_record_modifier.rb +15 -0
- metadata +4 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 393309787a689b84bc9282849cd03b6d50bf68f4
|
4
|
+
data.tar.gz: 24c92b2b9d47a616706701f4ec38c08ccbaccef7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 49ee18b86ca8e11cc803947c6dcc7f7beea0ce138d957ceaadb46f55053eb3ddf72d54b64bcbfd3b049bbc075823a54347acedb807cbb6b7ea972b8f6bce4eb2
|
7
|
+
data.tar.gz: b41c41bc4a2a972cadf7b078125d8866ba76b5324f5990c3cf7a412707e7a56c43c934d4cca85479ce6b82ec2473a786cf350ce004f1264fe7b98dbc1b12499c
|
data/ChangeLog
CHANGED
@@ -1,3 +1,10 @@
|
|
1
|
+
Release 0.4.1 - 2016/02/26
|
2
|
+
|
3
|
+
* Set/Convert encoding recursively
|
4
|
+
https://github.com/repeatedly/fluent-plugin-record-modifier/pull/9
|
5
|
+
* Add whitelist_keys option
|
6
|
+
https://github.com/repeatedly/fluent-plugin-record-modifier/pull/11
|
7
|
+
|
1
8
|
Release 0.4.0 - 2016/01/08
|
2
9
|
|
3
10
|
* Introduce <record> directive
|
data/README.md
CHANGED
@@ -112,6 +112,31 @@ then you got new record like below:
|
|
112
112
|
{"key3":"bar"}
|
113
113
|
```
|
114
114
|
|
115
|
+
### whitelist_keys
|
116
|
+
|
117
|
+
If you want to handle the set of explicitly specified keys, you can use `whitelist_keys` of this plugin. It's exclusive with `remove_keys`.
|
118
|
+
|
119
|
+
```conf
|
120
|
+
<filter pattern>
|
121
|
+
type record_modifier
|
122
|
+
|
123
|
+
# remove all keys except for key1 and key2
|
124
|
+
whitelist_keys key1,key2
|
125
|
+
</filter>
|
126
|
+
```
|
127
|
+
|
128
|
+
If following record is passed:
|
129
|
+
|
130
|
+
```js
|
131
|
+
{"key1":"hoge", "key2":"foo", "key3":"bar"}
|
132
|
+
```
|
133
|
+
|
134
|
+
then you got new record like below:
|
135
|
+
|
136
|
+
```js
|
137
|
+
{"key1":"hoge", "key2":"foo"}
|
138
|
+
```
|
139
|
+
|
115
140
|
### Mixins
|
116
141
|
|
117
142
|
* [fluent-mixin-config-placeholders](https://github.com/tagomoris/fluent-mixin-config-placeholders)
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.4.
|
1
|
+
0.4.1
|
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
|
|
3
3
|
|
4
4
|
Gem::Specification.new do |gem|
|
5
5
|
gem.name = "fluent-plugin-record-modifier"
|
6
|
-
gem.description = "
|
6
|
+
gem.description = "Filter plugin for modifying event record"
|
7
7
|
gem.homepage = "https://github.com/repeatedly/fluent-plugin-record-modifier"
|
8
8
|
gem.summary = gem.description
|
9
9
|
gem.version = File.read("VERSION").strip
|
@@ -4,12 +4,31 @@ module Fluent
|
|
4
4
|
class RecordModifierFilter < Filter
|
5
5
|
Fluent::Plugin.register_filter('record_modifier', self)
|
6
6
|
|
7
|
-
config_param :char_encoding, :string, :
|
8
|
-
|
7
|
+
config_param :char_encoding, :string, default: nil,
|
8
|
+
desc: <<-DESC
|
9
|
+
Fluentd including some plugins treats the logs as a BINARY by default to forward.
|
10
|
+
But an user sometimes processes the logs depends on their requirements,
|
11
|
+
e.g. handling char encoding correctly.
|
12
|
+
In more detail, please refer this section:
|
13
|
+
https://github.com/repeatedly/fluent-plugin-record-modifier#char_encoding.
|
14
|
+
DESC
|
15
|
+
config_param :remove_keys, :string, default: nil,
|
16
|
+
desc: <<-DESC
|
17
|
+
The logs include needless record keys in some cases.
|
18
|
+
You can remove it by using `remove_keys` parameter.
|
19
|
+
This option is exclusive with `whitelist_keys`.
|
20
|
+
DESC
|
21
|
+
|
22
|
+
config_param :whitelist_keys, :string, default: nil,
|
23
|
+
desc: <<-DESC
|
24
|
+
Specify `whitelist_keys` to remove all unexpected keys and values from events.
|
25
|
+
Modified events will have only specified keys (if exist in original events).
|
26
|
+
This option is exclusive with `remove_keys`.
|
27
|
+
DESC
|
9
28
|
|
10
29
|
include Fluent::Mixin::ConfigPlaceholders
|
11
30
|
|
12
|
-
BUILTIN_CONFIGURATIONS = %W(type @type log_level @log_level id @id char_encoding remove_keys)
|
31
|
+
BUILTIN_CONFIGURATIONS = %W(type @type log_level @log_level id @id char_encoding remove_keys whitelist_keys)
|
13
32
|
|
14
33
|
def configure(conf)
|
15
34
|
super
|
@@ -52,8 +71,12 @@ module Fluent
|
|
52
71
|
end
|
53
72
|
end
|
54
73
|
|
55
|
-
if @remove_keys
|
56
|
-
|
74
|
+
if @remove_keys and @whitelist_keys
|
75
|
+
raise Fluent::ConfigError, "remove_keys and whitelist_keys are exclusive with each other."
|
76
|
+
elsif @remove_keys
|
77
|
+
@remove_keys = @remove_keys.split(',').map(&:strip)
|
78
|
+
elsif @whitelist_keys
|
79
|
+
@whitelist_keys = @whitelist_keys.split(',').map(&:strip)
|
57
80
|
end
|
58
81
|
|
59
82
|
# Collect DynamicExpander related garbage instructions
|
@@ -73,6 +96,12 @@ module Fluent
|
|
73
96
|
@remove_keys.each { |v|
|
74
97
|
record.delete(v)
|
75
98
|
}
|
99
|
+
elsif @whitelist_keys
|
100
|
+
modified = {}
|
101
|
+
record.each do |k, v|
|
102
|
+
modified[k] = v if @whitelist_keys.include?(k)
|
103
|
+
end
|
104
|
+
record = modified
|
76
105
|
end
|
77
106
|
|
78
107
|
record = change_encoding(record) if @char_encoding
|
@@ -83,21 +112,25 @@ module Fluent
|
|
83
112
|
|
84
113
|
private
|
85
114
|
|
86
|
-
def set_encoding(
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
115
|
+
def set_encoding(value)
|
116
|
+
if value.is_a?(String)
|
117
|
+
value.force_encoding(@from_enc)
|
118
|
+
elsif value.is_a?(Hash)
|
119
|
+
value.each_pair { |k, v| set_encoding(v) }
|
120
|
+
elsif value.is_a?(Array)
|
121
|
+
value.each { |v| set_encoding(v) }
|
122
|
+
end
|
92
123
|
end
|
93
124
|
|
94
|
-
def convert_encoding(
|
95
|
-
|
96
|
-
if
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
125
|
+
def convert_encoding(value)
|
126
|
+
if value.is_a?(String)
|
127
|
+
value.force_encoding(@from_enc) if value.encoding == Encoding::BINARY
|
128
|
+
value.encode!(@to_enc, @from_enc, :invalid => :replace, :undef => :replace)
|
129
|
+
elsif value.is_a?(Hash)
|
130
|
+
value.each_pair { |k, v| convert_encoding(v) }
|
131
|
+
elsif value.is_a?(Array)
|
132
|
+
value.each { |v| convert_encoding(v) }
|
133
|
+
end
|
101
134
|
end
|
102
135
|
|
103
136
|
class DynamicExpander
|
@@ -4,14 +4,35 @@ module Fluent
|
|
4
4
|
class RecordModifierOutput < Output
|
5
5
|
Fluent::Plugin.register_output('record_modifier', self)
|
6
6
|
|
7
|
-
config_param :tag, :string
|
8
|
-
|
9
|
-
config_param :
|
7
|
+
config_param :tag, :string,
|
8
|
+
desc: "The output record tag name."
|
9
|
+
config_param :char_encoding, :string, default: nil,
|
10
|
+
desc: <<-DESC
|
11
|
+
Fluentd including some plugins treats the logs as a BINARY by default to forward.
|
12
|
+
But an user sometimes processes the logs depends on their requirements,
|
13
|
+
e.g. handling char encoding correctly.
|
14
|
+
In more detail, please refer this section:
|
15
|
+
https://github.com/repeatedly/fluent-plugin-record-modifier#char_encoding.
|
16
|
+
DESC
|
17
|
+
|
18
|
+
config_param :remove_keys, :string, default: nil,
|
19
|
+
desc: <<-DESC
|
20
|
+
The logs include needless record keys in some cases.
|
21
|
+
You can remove it by using `remove_keys` parameter.
|
22
|
+
This option is exclusive with `whitelist_keys`.
|
23
|
+
DESC
|
24
|
+
|
25
|
+
config_param :whitelist_keys, :string, default: nil,
|
26
|
+
desc: <<-DESC
|
27
|
+
Specify `whitelist_keys` to remove all unexpected keys and values from events.
|
28
|
+
Modified events will have only specified keys (if exist in original events).
|
29
|
+
This option is exclusive with `remove_keys`.
|
30
|
+
DESC
|
10
31
|
|
11
32
|
include SetTagKeyMixin
|
12
33
|
include Fluent::Mixin::ConfigPlaceholders
|
13
34
|
|
14
|
-
BUILTIN_CONFIGURATIONS = %W(type tag include_tag_key tag_key char_encoding remove_keys)
|
35
|
+
BUILTIN_CONFIGURATIONS = %W(type tag include_tag_key tag_key char_encoding remove_keys whitelist_keys)
|
15
36
|
|
16
37
|
def configure(conf)
|
17
38
|
super
|
@@ -40,16 +61,22 @@ module Fluent
|
|
40
61
|
end
|
41
62
|
end
|
42
63
|
|
43
|
-
if @remove_keys
|
44
|
-
|
64
|
+
if @remove_keys and @whitelist_keys
|
65
|
+
raise Fluent::ConfigError, "remove_keys and whitelist_keys are exclusive with each other."
|
66
|
+
elsif @remove_keys
|
67
|
+
@remove_keys = @remove_keys.split(',').map(&:strip)
|
68
|
+
elsif @whitelist_keys
|
69
|
+
@whitelist_keys = @whitelist_keys.split(',').map(&:strip)
|
45
70
|
end
|
46
71
|
end
|
47
72
|
|
48
73
|
def emit(tag, es, chain)
|
74
|
+
stream = MultiEventStream.new
|
49
75
|
es.each { |time, record|
|
50
76
|
filter_record(tag, time, record)
|
51
|
-
|
77
|
+
stream.add(time, modify_record(record))
|
52
78
|
}
|
79
|
+
Fluent::Engine.emit_stream(@tag, stream)
|
53
80
|
|
54
81
|
chain.next
|
55
82
|
end
|
@@ -65,6 +92,12 @@ module Fluent
|
|
65
92
|
@remove_keys.each { |v|
|
66
93
|
record.delete(v)
|
67
94
|
}
|
95
|
+
elsif @whitelist_keys
|
96
|
+
modified = {}
|
97
|
+
record.each do |k, v|
|
98
|
+
modified[k] = v if @whitelist_keys.include?(k)
|
99
|
+
end
|
100
|
+
record = modified
|
68
101
|
end
|
69
102
|
|
70
103
|
record = change_encoding(record) if @char_encoding
|
@@ -62,12 +62,17 @@ class RecordModifierFilterTest < Test::Unit::TestCase
|
|
62
62
|
char_encoding utf-8
|
63
63
|
]
|
64
64
|
|
65
|
-
|
66
65
|
d.run do
|
67
66
|
d.emit("k" => 'v'.force_encoding('BINARY'))
|
67
|
+
d.emit("k" => %w(v ビ).map{|v| v.force_encoding('BINARY')})
|
68
|
+
d.emit("k" => {"l" => 'ビ'.force_encoding('BINARY')})
|
68
69
|
end
|
69
70
|
|
70
|
-
assert_equal [
|
71
|
+
assert_equal [
|
72
|
+
{"k" => 'v'.force_encoding('UTF-8')},
|
73
|
+
{"k" => %w(v ビ).map{|v| v.force_encoding('UTF-8')}},
|
74
|
+
{"k" => {"l" => 'ビ'.force_encoding('UTF-8')}},
|
75
|
+
], d.filtered_as_array.map { |e| e.last }
|
71
76
|
end
|
72
77
|
|
73
78
|
def test_convert_char_encoding
|
@@ -79,9 +84,15 @@ class RecordModifierFilterTest < Test::Unit::TestCase
|
|
79
84
|
|
80
85
|
d.run do
|
81
86
|
d.emit("k" => 'v'.force_encoding('utf-8'))
|
87
|
+
d.emit("k" => %w(v ビ).map{|v| v.force_encoding('utf-8')})
|
88
|
+
d.emit("k" => {"l" => 'ビ'.force_encoding('utf-8')})
|
82
89
|
end
|
83
90
|
|
84
|
-
assert_equal [
|
91
|
+
assert_equal [
|
92
|
+
{"k" => 'v'.force_encoding('cp932')},
|
93
|
+
{"k" => %w(v ビ).map{|v| v.encode!('cp932')}},
|
94
|
+
{"k" => {"l" => 'ビ'.encode!('cp932')}},
|
95
|
+
], d.filtered_as_array.map { |e| e.last }
|
85
96
|
end
|
86
97
|
|
87
98
|
def test_remove_one_key
|
@@ -111,4 +122,18 @@ class RecordModifierFilterTest < Test::Unit::TestCase
|
|
111
122
|
|
112
123
|
assert_equal [{"k4" => 'v'}], d.filtered_as_array.map { |e| e.last }
|
113
124
|
end
|
125
|
+
|
126
|
+
def test_remove_non_whitelist_keys
|
127
|
+
d = create_driver %[
|
128
|
+
type record_modifier
|
129
|
+
|
130
|
+
whitelist_keys k1, k2, k3
|
131
|
+
]
|
132
|
+
|
133
|
+
d.run do
|
134
|
+
d.emit("k1" => 'v', "k2" => 'v', "k4" => 'v', "k5" => 'v')
|
135
|
+
end
|
136
|
+
|
137
|
+
assert_equal [{"k1" => 'v', "k2" => 'v'}], d.filtered_as_array.map(&:last)
|
138
|
+
end
|
114
139
|
end
|
@@ -110,4 +110,19 @@ class RecordModifierOutputTest < Test::Unit::TestCase
|
|
110
110
|
|
111
111
|
assert_equal [{"k4" => 'v'}], d.records
|
112
112
|
end
|
113
|
+
|
114
|
+
def test_remove_non_whitelist_keys
|
115
|
+
d = create_driver %[
|
116
|
+
type record_modifier
|
117
|
+
|
118
|
+
tag foo.filtered
|
119
|
+
whitelist_keys k1, k2, k3
|
120
|
+
]
|
121
|
+
|
122
|
+
d.run do
|
123
|
+
d.emit("k1" => 'v', "k2" => 'v', "k4" => 'v', "k5" => 'v')
|
124
|
+
end
|
125
|
+
|
126
|
+
assert_equal [{"k1" => 'v', "k2" => 'v'}], d.records
|
127
|
+
end
|
113
128
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fluent-plugin-record-modifier
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.4.
|
4
|
+
version: 0.4.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Masahiro Nakagawa
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-02-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: fluentd
|
@@ -72,7 +72,7 @@ dependencies:
|
|
72
72
|
- - "~>"
|
73
73
|
- !ruby/object:Gem::Version
|
74
74
|
version: 3.1.4
|
75
|
-
description:
|
75
|
+
description: Filter plugin for modifying event record
|
76
76
|
email: repeatedly@gmail.com
|
77
77
|
executables: []
|
78
78
|
extensions: []
|
@@ -112,7 +112,7 @@ rubyforge_project:
|
|
112
112
|
rubygems_version: 2.2.2
|
113
113
|
signing_key:
|
114
114
|
specification_version: 4
|
115
|
-
summary:
|
115
|
+
summary: Filter plugin for modifying event record
|
116
116
|
test_files:
|
117
117
|
- test/test_filter_record_modifier.rb
|
118
118
|
- test/test_out_record_modifier.rb
|