fluent-plugin-record-modifier 0.4.0 → 0.4.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 3ff1d6de463980c890752f4eb32719f3f160c026
4
- data.tar.gz: 4ed772c23ab2316e1bf064a9fd3c1ed498c1aae8
3
+ metadata.gz: 393309787a689b84bc9282849cd03b6d50bf68f4
4
+ data.tar.gz: 24c92b2b9d47a616706701f4ec38c08ccbaccef7
5
5
  SHA512:
6
- metadata.gz: 2477196fe7d51f5d26546107d4b9a01907e97e3dc38a0d5a2a1a4cc8182f298d24abb996b197e3609cae7e0a534781b93f0d5bd802d7afc31c0297f2f5e31924
7
- data.tar.gz: e0826bc7e9203ea5a6bf24f715b5b63d2b13a95c407a4d298cfd8b9f2f7c781afed10c69248774e8930b7a0f86cc1f569c75a9c2e3c00e689e3dffc4fe2f6e5d
6
+ metadata.gz: 49ee18b86ca8e11cc803947c6dcc7f7beea0ce138d957ceaadb46f55053eb3ddf72d54b64bcbfd3b049bbc075823a54347acedb807cbb6b7ea972b8f6bce4eb2
7
+ data.tar.gz: b41c41bc4a2a972cadf7b078125d8866ba76b5324f5990c3cf7a412707e7a56c43c934d4cca85479ce6b82ec2473a786cf350ce004f1264fe7b98dbc1b12499c
data/ChangeLog CHANGED
@@ -1,3 +1,10 @@
1
+ Release 0.4.1 - 2016/02/26
2
+
3
+ * Set/Convert encoding recursively
4
+ https://github.com/repeatedly/fluent-plugin-record-modifier/pull/9
5
+ * Add whitelist_keys option
6
+ https://github.com/repeatedly/fluent-plugin-record-modifier/pull/11
7
+
1
8
  Release 0.4.0 - 2016/01/08
2
9
 
3
10
  * Introduce <record> directive
data/README.md CHANGED
@@ -112,6 +112,31 @@ then you got new record like below:
112
112
  {"key3":"bar"}
113
113
  ```
114
114
 
115
+ ### whitelist_keys
116
+
117
+ If you want to handle the set of explicitly specified keys, you can use `whitelist_keys` of this plugin. It's exclusive with `remove_keys`.
118
+
119
+ ```conf
120
+ <filter pattern>
121
+ type record_modifier
122
+
123
+ # remove all keys except for key1 and key2
124
+ whitelist_keys key1,key2
125
+ </filter>
126
+ ```
127
+
128
+ If following record is passed:
129
+
130
+ ```js
131
+ {"key1":"hoge", "key2":"foo", "key3":"bar"}
132
+ ```
133
+
134
+ then you got new record like below:
135
+
136
+ ```js
137
+ {"key1":"hoge", "key2":"foo"}
138
+ ```
139
+
115
140
  ### Mixins
116
141
 
117
142
  * [fluent-mixin-config-placeholders](https://github.com/tagomoris/fluent-mixin-config-placeholders)
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.4.0
1
+ 0.4.1
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
3
3
 
4
4
  Gem::Specification.new do |gem|
5
5
  gem.name = "fluent-plugin-record-modifier"
6
- gem.description = "Output filter plugin for modifying each event record"
6
+ gem.description = "Filter plugin for modifying event record"
7
7
  gem.homepage = "https://github.com/repeatedly/fluent-plugin-record-modifier"
8
8
  gem.summary = gem.description
9
9
  gem.version = File.read("VERSION").strip
@@ -4,12 +4,31 @@ module Fluent
4
4
  class RecordModifierFilter < Filter
5
5
  Fluent::Plugin.register_filter('record_modifier', self)
6
6
 
7
- config_param :char_encoding, :string, :default => nil
8
- config_param :remove_keys, :string, :default => nil
7
+ config_param :char_encoding, :string, default: nil,
8
+ desc: <<-DESC
9
+ Fluentd including some plugins treats the logs as a BINARY by default to forward.
10
+ But an user sometimes processes the logs depends on their requirements,
11
+ e.g. handling char encoding correctly.
12
+ In more detail, please refer this section:
13
+ https://github.com/repeatedly/fluent-plugin-record-modifier#char_encoding.
14
+ DESC
15
+ config_param :remove_keys, :string, default: nil,
16
+ desc: <<-DESC
17
+ The logs include needless record keys in some cases.
18
+ You can remove it by using `remove_keys` parameter.
19
+ This option is exclusive with `whitelist_keys`.
20
+ DESC
21
+
22
+ config_param :whitelist_keys, :string, default: nil,
23
+ desc: <<-DESC
24
+ Specify `whitelist_keys` to remove all unexpected keys and values from events.
25
+ Modified events will have only specified keys (if exist in original events).
26
+ This option is exclusive with `remove_keys`.
27
+ DESC
9
28
 
10
29
  include Fluent::Mixin::ConfigPlaceholders
11
30
 
12
- BUILTIN_CONFIGURATIONS = %W(type @type log_level @log_level id @id char_encoding remove_keys)
31
+ BUILTIN_CONFIGURATIONS = %W(type @type log_level @log_level id @id char_encoding remove_keys whitelist_keys)
13
32
 
14
33
  def configure(conf)
15
34
  super
@@ -52,8 +71,12 @@ module Fluent
52
71
  end
53
72
  end
54
73
 
55
- if @remove_keys
56
- @remove_keys = @remove_keys.split(',').map { |e| e.strip }
74
+ if @remove_keys and @whitelist_keys
75
+ raise Fluent::ConfigError, "remove_keys and whitelist_keys are exclusive with each other."
76
+ elsif @remove_keys
77
+ @remove_keys = @remove_keys.split(',').map(&:strip)
78
+ elsif @whitelist_keys
79
+ @whitelist_keys = @whitelist_keys.split(',').map(&:strip)
57
80
  end
58
81
 
59
82
  # Collect DynamicExpander related garbage instructions
@@ -73,6 +96,12 @@ module Fluent
73
96
  @remove_keys.each { |v|
74
97
  record.delete(v)
75
98
  }
99
+ elsif @whitelist_keys
100
+ modified = {}
101
+ record.each do |k, v|
102
+ modified[k] = v if @whitelist_keys.include?(k)
103
+ end
104
+ record = modified
76
105
  end
77
106
 
78
107
  record = change_encoding(record) if @char_encoding
@@ -83,21 +112,25 @@ module Fluent
83
112
 
84
113
  private
85
114
 
86
- def set_encoding(record)
87
- record.each_pair { |k, v|
88
- if v.is_a?(String)
89
- v.force_encoding(@from_enc)
90
- end
91
- }
115
+ def set_encoding(value)
116
+ if value.is_a?(String)
117
+ value.force_encoding(@from_enc)
118
+ elsif value.is_a?(Hash)
119
+ value.each_pair { |k, v| set_encoding(v) }
120
+ elsif value.is_a?(Array)
121
+ value.each { |v| set_encoding(v) }
122
+ end
92
123
  end
93
124
 
94
- def convert_encoding(record)
95
- record.each_pair { |k, v|
96
- if v.is_a?(String)
97
- v.force_encoding(@from_enc) if v.encoding == Encoding::BINARY
98
- v.encode!(@to_enc, @from_enc, :invalid => :replace, :undef => :replace)
99
- end
100
- }
125
+ def convert_encoding(value)
126
+ if value.is_a?(String)
127
+ value.force_encoding(@from_enc) if value.encoding == Encoding::BINARY
128
+ value.encode!(@to_enc, @from_enc, :invalid => :replace, :undef => :replace)
129
+ elsif value.is_a?(Hash)
130
+ value.each_pair { |k, v| convert_encoding(v) }
131
+ elsif value.is_a?(Array)
132
+ value.each { |v| convert_encoding(v) }
133
+ end
101
134
  end
102
135
 
103
136
  class DynamicExpander
@@ -4,14 +4,35 @@ module Fluent
4
4
  class RecordModifierOutput < Output
5
5
  Fluent::Plugin.register_output('record_modifier', self)
6
6
 
7
- config_param :tag, :string
8
- config_param :char_encoding, :string, :default => nil
9
- config_param :remove_keys, :string, :default => nil
7
+ config_param :tag, :string,
8
+ desc: "The output record tag name."
9
+ config_param :char_encoding, :string, default: nil,
10
+ desc: <<-DESC
11
+ Fluentd including some plugins treats the logs as a BINARY by default to forward.
12
+ But an user sometimes processes the logs depends on their requirements,
13
+ e.g. handling char encoding correctly.
14
+ In more detail, please refer this section:
15
+ https://github.com/repeatedly/fluent-plugin-record-modifier#char_encoding.
16
+ DESC
17
+
18
+ config_param :remove_keys, :string, default: nil,
19
+ desc: <<-DESC
20
+ The logs include needless record keys in some cases.
21
+ You can remove it by using `remove_keys` parameter.
22
+ This option is exclusive with `whitelist_keys`.
23
+ DESC
24
+
25
+ config_param :whitelist_keys, :string, default: nil,
26
+ desc: <<-DESC
27
+ Specify `whitelist_keys` to remove all unexpected keys and values from events.
28
+ Modified events will have only specified keys (if exist in original events).
29
+ This option is exclusive with `remove_keys`.
30
+ DESC
10
31
 
11
32
  include SetTagKeyMixin
12
33
  include Fluent::Mixin::ConfigPlaceholders
13
34
 
14
- BUILTIN_CONFIGURATIONS = %W(type tag include_tag_key tag_key char_encoding remove_keys)
35
+ BUILTIN_CONFIGURATIONS = %W(type tag include_tag_key tag_key char_encoding remove_keys whitelist_keys)
15
36
 
16
37
  def configure(conf)
17
38
  super
@@ -40,16 +61,22 @@ module Fluent
40
61
  end
41
62
  end
42
63
 
43
- if @remove_keys
44
- @remove_keys = @remove_keys.split(',').map {|e| e.strip }
64
+ if @remove_keys and @whitelist_keys
65
+ raise Fluent::ConfigError, "remove_keys and whitelist_keys are exclusive with each other."
66
+ elsif @remove_keys
67
+ @remove_keys = @remove_keys.split(',').map(&:strip)
68
+ elsif @whitelist_keys
69
+ @whitelist_keys = @whitelist_keys.split(',').map(&:strip)
45
70
  end
46
71
  end
47
72
 
48
73
  def emit(tag, es, chain)
74
+ stream = MultiEventStream.new
49
75
  es.each { |time, record|
50
76
  filter_record(tag, time, record)
51
- Engine.emit(@tag, time, modify_record(record))
77
+ stream.add(time, modify_record(record))
52
78
  }
79
+ Fluent::Engine.emit_stream(@tag, stream)
53
80
 
54
81
  chain.next
55
82
  end
@@ -65,6 +92,12 @@ module Fluent
65
92
  @remove_keys.each { |v|
66
93
  record.delete(v)
67
94
  }
95
+ elsif @whitelist_keys
96
+ modified = {}
97
+ record.each do |k, v|
98
+ modified[k] = v if @whitelist_keys.include?(k)
99
+ end
100
+ record = modified
68
101
  end
69
102
 
70
103
  record = change_encoding(record) if @char_encoding
@@ -62,12 +62,17 @@ class RecordModifierFilterTest < Test::Unit::TestCase
62
62
  char_encoding utf-8
63
63
  ]
64
64
 
65
-
66
65
  d.run do
67
66
  d.emit("k" => 'v'.force_encoding('BINARY'))
67
+ d.emit("k" => %w(v ビ).map{|v| v.force_encoding('BINARY')})
68
+ d.emit("k" => {"l" => 'ビ'.force_encoding('BINARY')})
68
69
  end
69
70
 
70
- assert_equal [{"k" => 'v'.force_encoding('UTF-8')}], d.filtered_as_array.map { |e| e.last }
71
+ assert_equal [
72
+ {"k" => 'v'.force_encoding('UTF-8')},
73
+ {"k" => %w(v ビ).map{|v| v.force_encoding('UTF-8')}},
74
+ {"k" => {"l" => 'ビ'.force_encoding('UTF-8')}},
75
+ ], d.filtered_as_array.map { |e| e.last }
71
76
  end
72
77
 
73
78
  def test_convert_char_encoding
@@ -79,9 +84,15 @@ class RecordModifierFilterTest < Test::Unit::TestCase
79
84
 
80
85
  d.run do
81
86
  d.emit("k" => 'v'.force_encoding('utf-8'))
87
+ d.emit("k" => %w(v ビ).map{|v| v.force_encoding('utf-8')})
88
+ d.emit("k" => {"l" => 'ビ'.force_encoding('utf-8')})
82
89
  end
83
90
 
84
- assert_equal [{"k" => 'v'.force_encoding('cp932')}], d.filtered_as_array.map { |e| e.last }
91
+ assert_equal [
92
+ {"k" => 'v'.force_encoding('cp932')},
93
+ {"k" => %w(v ビ).map{|v| v.encode!('cp932')}},
94
+ {"k" => {"l" => 'ビ'.encode!('cp932')}},
95
+ ], d.filtered_as_array.map { |e| e.last }
85
96
  end
86
97
 
87
98
  def test_remove_one_key
@@ -111,4 +122,18 @@ class RecordModifierFilterTest < Test::Unit::TestCase
111
122
 
112
123
  assert_equal [{"k4" => 'v'}], d.filtered_as_array.map { |e| e.last }
113
124
  end
125
+
126
+ def test_remove_non_whitelist_keys
127
+ d = create_driver %[
128
+ type record_modifier
129
+
130
+ whitelist_keys k1, k2, k3
131
+ ]
132
+
133
+ d.run do
134
+ d.emit("k1" => 'v', "k2" => 'v', "k4" => 'v', "k5" => 'v')
135
+ end
136
+
137
+ assert_equal [{"k1" => 'v', "k2" => 'v'}], d.filtered_as_array.map(&:last)
138
+ end
114
139
  end
@@ -110,4 +110,19 @@ class RecordModifierOutputTest < Test::Unit::TestCase
110
110
 
111
111
  assert_equal [{"k4" => 'v'}], d.records
112
112
  end
113
+
114
+ def test_remove_non_whitelist_keys
115
+ d = create_driver %[
116
+ type record_modifier
117
+
118
+ tag foo.filtered
119
+ whitelist_keys k1, k2, k3
120
+ ]
121
+
122
+ d.run do
123
+ d.emit("k1" => 'v', "k2" => 'v', "k4" => 'v', "k5" => 'v')
124
+ end
125
+
126
+ assert_equal [{"k1" => 'v', "k2" => 'v'}], d.records
127
+ end
113
128
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-record-modifier
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.0
4
+ version: 0.4.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Masahiro Nakagawa
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-01-08 00:00:00.000000000 Z
11
+ date: 2016-02-26 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -72,7 +72,7 @@ dependencies:
72
72
  - - "~>"
73
73
  - !ruby/object:Gem::Version
74
74
  version: 3.1.4
75
- description: Output filter plugin for modifying each event record
75
+ description: Filter plugin for modifying event record
76
76
  email: repeatedly@gmail.com
77
77
  executables: []
78
78
  extensions: []
@@ -112,7 +112,7 @@ rubyforge_project:
112
112
  rubygems_version: 2.2.2
113
113
  signing_key:
114
114
  specification_version: 4
115
- summary: Output filter plugin for modifying each event record
115
+ summary: Filter plugin for modifying event record
116
116
  test_files:
117
117
  - test/test_filter_record_modifier.rb
118
118
  - test/test_out_record_modifier.rb