fluent-plugin-record-modifier 0.4.0 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 3ff1d6de463980c890752f4eb32719f3f160c026
4
- data.tar.gz: 4ed772c23ab2316e1bf064a9fd3c1ed498c1aae8
3
+ metadata.gz: 393309787a689b84bc9282849cd03b6d50bf68f4
4
+ data.tar.gz: 24c92b2b9d47a616706701f4ec38c08ccbaccef7
5
5
  SHA512:
6
- metadata.gz: 2477196fe7d51f5d26546107d4b9a01907e97e3dc38a0d5a2a1a4cc8182f298d24abb996b197e3609cae7e0a534781b93f0d5bd802d7afc31c0297f2f5e31924
7
- data.tar.gz: e0826bc7e9203ea5a6bf24f715b5b63d2b13a95c407a4d298cfd8b9f2f7c781afed10c69248774e8930b7a0f86cc1f569c75a9c2e3c00e689e3dffc4fe2f6e5d
6
+ metadata.gz: 49ee18b86ca8e11cc803947c6dcc7f7beea0ce138d957ceaadb46f55053eb3ddf72d54b64bcbfd3b049bbc075823a54347acedb807cbb6b7ea972b8f6bce4eb2
7
+ data.tar.gz: b41c41bc4a2a972cadf7b078125d8866ba76b5324f5990c3cf7a412707e7a56c43c934d4cca85479ce6b82ec2473a786cf350ce004f1264fe7b98dbc1b12499c
data/ChangeLog CHANGED
@@ -1,3 +1,10 @@
1
+ Release 0.4.1 - 2016/02/26
2
+
3
+ * Set/Convert encoding recursively
4
+ https://github.com/repeatedly/fluent-plugin-record-modifier/pull/9
5
+ * Add whitelist_keys option
6
+ https://github.com/repeatedly/fluent-plugin-record-modifier/pull/11
7
+
1
8
  Release 0.4.0 - 2016/01/08
2
9
 
3
10
  * Introduce <record> directive
data/README.md CHANGED
@@ -112,6 +112,31 @@ then you got new record like below:
112
112
  {"key3":"bar"}
113
113
  ```
114
114
 
115
+ ### whitelist_keys
116
+
117
+ If you want to handle the set of explicitly specified keys, you can use `whitelist_keys` of this plugin. It's exclusive with `remove_keys`.
118
+
119
+ ```conf
120
+ <filter pattern>
121
+ type record_modifier
122
+
123
+ # remove all keys except for key1 and key2
124
+ whitelist_keys key1,key2
125
+ </filter>
126
+ ```
127
+
128
+ If following record is passed:
129
+
130
+ ```js
131
+ {"key1":"hoge", "key2":"foo", "key3":"bar"}
132
+ ```
133
+
134
+ then you got new record like below:
135
+
136
+ ```js
137
+ {"key1":"hoge", "key2":"foo"}
138
+ ```
139
+
115
140
  ### Mixins
116
141
 
117
142
  * [fluent-mixin-config-placeholders](https://github.com/tagomoris/fluent-mixin-config-placeholders)
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.4.0
1
+ 0.4.1
@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
3
3
 
4
4
  Gem::Specification.new do |gem|
5
5
  gem.name = "fluent-plugin-record-modifier"
6
- gem.description = "Output filter plugin for modifying each event record"
6
+ gem.description = "Filter plugin for modifying event record"
7
7
  gem.homepage = "https://github.com/repeatedly/fluent-plugin-record-modifier"
8
8
  gem.summary = gem.description
9
9
  gem.version = File.read("VERSION").strip
@@ -4,12 +4,31 @@ module Fluent
4
4
  class RecordModifierFilter < Filter
5
5
  Fluent::Plugin.register_filter('record_modifier', self)
6
6
 
7
- config_param :char_encoding, :string, :default => nil
8
- config_param :remove_keys, :string, :default => nil
7
+ config_param :char_encoding, :string, default: nil,
8
+ desc: <<-DESC
9
+ Fluentd including some plugins treats the logs as a BINARY by default to forward.
10
+ But an user sometimes processes the logs depends on their requirements,
11
+ e.g. handling char encoding correctly.
12
+ In more detail, please refer this section:
13
+ https://github.com/repeatedly/fluent-plugin-record-modifier#char_encoding.
14
+ DESC
15
+ config_param :remove_keys, :string, default: nil,
16
+ desc: <<-DESC
17
+ The logs include needless record keys in some cases.
18
+ You can remove it by using `remove_keys` parameter.
19
+ This option is exclusive with `whitelist_keys`.
20
+ DESC
21
+
22
+ config_param :whitelist_keys, :string, default: nil,
23
+ desc: <<-DESC
24
+ Specify `whitelist_keys` to remove all unexpected keys and values from events.
25
+ Modified events will have only specified keys (if exist in original events).
26
+ This option is exclusive with `remove_keys`.
27
+ DESC
9
28
 
10
29
  include Fluent::Mixin::ConfigPlaceholders
11
30
 
12
- BUILTIN_CONFIGURATIONS = %W(type @type log_level @log_level id @id char_encoding remove_keys)
31
+ BUILTIN_CONFIGURATIONS = %W(type @type log_level @log_level id @id char_encoding remove_keys whitelist_keys)
13
32
 
14
33
  def configure(conf)
15
34
  super
@@ -52,8 +71,12 @@ module Fluent
52
71
  end
53
72
  end
54
73
 
55
- if @remove_keys
56
- @remove_keys = @remove_keys.split(',').map { |e| e.strip }
74
+ if @remove_keys and @whitelist_keys
75
+ raise Fluent::ConfigError, "remove_keys and whitelist_keys are exclusive with each other."
76
+ elsif @remove_keys
77
+ @remove_keys = @remove_keys.split(',').map(&:strip)
78
+ elsif @whitelist_keys
79
+ @whitelist_keys = @whitelist_keys.split(',').map(&:strip)
57
80
  end
58
81
 
59
82
  # Collect DynamicExpander related garbage instructions
@@ -73,6 +96,12 @@ module Fluent
73
96
  @remove_keys.each { |v|
74
97
  record.delete(v)
75
98
  }
99
+ elsif @whitelist_keys
100
+ modified = {}
101
+ record.each do |k, v|
102
+ modified[k] = v if @whitelist_keys.include?(k)
103
+ end
104
+ record = modified
76
105
  end
77
106
 
78
107
  record = change_encoding(record) if @char_encoding
@@ -83,21 +112,25 @@ module Fluent
83
112
 
84
113
  private
85
114
 
86
- def set_encoding(record)
87
- record.each_pair { |k, v|
88
- if v.is_a?(String)
89
- v.force_encoding(@from_enc)
90
- end
91
- }
115
+ def set_encoding(value)
116
+ if value.is_a?(String)
117
+ value.force_encoding(@from_enc)
118
+ elsif value.is_a?(Hash)
119
+ value.each_pair { |k, v| set_encoding(v) }
120
+ elsif value.is_a?(Array)
121
+ value.each { |v| set_encoding(v) }
122
+ end
92
123
  end
93
124
 
94
- def convert_encoding(record)
95
- record.each_pair { |k, v|
96
- if v.is_a?(String)
97
- v.force_encoding(@from_enc) if v.encoding == Encoding::BINARY
98
- v.encode!(@to_enc, @from_enc, :invalid => :replace, :undef => :replace)
99
- end
100
- }
125
+ def convert_encoding(value)
126
+ if value.is_a?(String)
127
+ value.force_encoding(@from_enc) if value.encoding == Encoding::BINARY
128
+ value.encode!(@to_enc, @from_enc, :invalid => :replace, :undef => :replace)
129
+ elsif value.is_a?(Hash)
130
+ value.each_pair { |k, v| convert_encoding(v) }
131
+ elsif value.is_a?(Array)
132
+ value.each { |v| convert_encoding(v) }
133
+ end
101
134
  end
102
135
 
103
136
  class DynamicExpander
@@ -4,14 +4,35 @@ module Fluent
4
4
  class RecordModifierOutput < Output
5
5
  Fluent::Plugin.register_output('record_modifier', self)
6
6
 
7
- config_param :tag, :string
8
- config_param :char_encoding, :string, :default => nil
9
- config_param :remove_keys, :string, :default => nil
7
+ config_param :tag, :string,
8
+ desc: "The output record tag name."
9
+ config_param :char_encoding, :string, default: nil,
10
+ desc: <<-DESC
11
+ Fluentd including some plugins treats the logs as a BINARY by default to forward.
12
+ But an user sometimes processes the logs depends on their requirements,
13
+ e.g. handling char encoding correctly.
14
+ In more detail, please refer this section:
15
+ https://github.com/repeatedly/fluent-plugin-record-modifier#char_encoding.
16
+ DESC
17
+
18
+ config_param :remove_keys, :string, default: nil,
19
+ desc: <<-DESC
20
+ The logs include needless record keys in some cases.
21
+ You can remove it by using `remove_keys` parameter.
22
+ This option is exclusive with `whitelist_keys`.
23
+ DESC
24
+
25
+ config_param :whitelist_keys, :string, default: nil,
26
+ desc: <<-DESC
27
+ Specify `whitelist_keys` to remove all unexpected keys and values from events.
28
+ Modified events will have only specified keys (if exist in original events).
29
+ This option is exclusive with `remove_keys`.
30
+ DESC
10
31
 
11
32
  include SetTagKeyMixin
12
33
  include Fluent::Mixin::ConfigPlaceholders
13
34
 
14
- BUILTIN_CONFIGURATIONS = %W(type tag include_tag_key tag_key char_encoding remove_keys)
35
+ BUILTIN_CONFIGURATIONS = %W(type tag include_tag_key tag_key char_encoding remove_keys whitelist_keys)
15
36
 
16
37
  def configure(conf)
17
38
  super
@@ -40,16 +61,22 @@ module Fluent
40
61
  end
41
62
  end
42
63
 
43
- if @remove_keys
44
- @remove_keys = @remove_keys.split(',').map {|e| e.strip }
64
+ if @remove_keys and @whitelist_keys
65
+ raise Fluent::ConfigError, "remove_keys and whitelist_keys are exclusive with each other."
66
+ elsif @remove_keys
67
+ @remove_keys = @remove_keys.split(',').map(&:strip)
68
+ elsif @whitelist_keys
69
+ @whitelist_keys = @whitelist_keys.split(',').map(&:strip)
45
70
  end
46
71
  end
47
72
 
48
73
  def emit(tag, es, chain)
74
+ stream = MultiEventStream.new
49
75
  es.each { |time, record|
50
76
  filter_record(tag, time, record)
51
- Engine.emit(@tag, time, modify_record(record))
77
+ stream.add(time, modify_record(record))
52
78
  }
79
+ Fluent::Engine.emit_stream(@tag, stream)
53
80
 
54
81
  chain.next
55
82
  end
@@ -65,6 +92,12 @@ module Fluent
65
92
  @remove_keys.each { |v|
66
93
  record.delete(v)
67
94
  }
95
+ elsif @whitelist_keys
96
+ modified = {}
97
+ record.each do |k, v|
98
+ modified[k] = v if @whitelist_keys.include?(k)
99
+ end
100
+ record = modified
68
101
  end
69
102
 
70
103
  record = change_encoding(record) if @char_encoding
@@ -62,12 +62,17 @@ class RecordModifierFilterTest < Test::Unit::TestCase
62
62
  char_encoding utf-8
63
63
  ]
64
64
 
65
-
66
65
  d.run do
67
66
  d.emit("k" => 'v'.force_encoding('BINARY'))
67
+ d.emit("k" => %w(v ビ).map{|v| v.force_encoding('BINARY')})
68
+ d.emit("k" => {"l" => 'ビ'.force_encoding('BINARY')})
68
69
  end
69
70
 
70
- assert_equal [{"k" => 'v'.force_encoding('UTF-8')}], d.filtered_as_array.map { |e| e.last }
71
+ assert_equal [
72
+ {"k" => 'v'.force_encoding('UTF-8')},
73
+ {"k" => %w(v ビ).map{|v| v.force_encoding('UTF-8')}},
74
+ {"k" => {"l" => 'ビ'.force_encoding('UTF-8')}},
75
+ ], d.filtered_as_array.map { |e| e.last }
71
76
  end
72
77
 
73
78
  def test_convert_char_encoding
@@ -79,9 +84,15 @@ class RecordModifierFilterTest < Test::Unit::TestCase
79
84
 
80
85
  d.run do
81
86
  d.emit("k" => 'v'.force_encoding('utf-8'))
87
+ d.emit("k" => %w(v ビ).map{|v| v.force_encoding('utf-8')})
88
+ d.emit("k" => {"l" => 'ビ'.force_encoding('utf-8')})
82
89
  end
83
90
 
84
- assert_equal [{"k" => 'v'.force_encoding('cp932')}], d.filtered_as_array.map { |e| e.last }
91
+ assert_equal [
92
+ {"k" => 'v'.force_encoding('cp932')},
93
+ {"k" => %w(v ビ).map{|v| v.encode!('cp932')}},
94
+ {"k" => {"l" => 'ビ'.encode!('cp932')}},
95
+ ], d.filtered_as_array.map { |e| e.last }
85
96
  end
86
97
 
87
98
  def test_remove_one_key
@@ -111,4 +122,18 @@ class RecordModifierFilterTest < Test::Unit::TestCase
111
122
 
112
123
  assert_equal [{"k4" => 'v'}], d.filtered_as_array.map { |e| e.last }
113
124
  end
125
+
126
+ def test_remove_non_whitelist_keys
127
+ d = create_driver %[
128
+ type record_modifier
129
+
130
+ whitelist_keys k1, k2, k3
131
+ ]
132
+
133
+ d.run do
134
+ d.emit("k1" => 'v', "k2" => 'v', "k4" => 'v', "k5" => 'v')
135
+ end
136
+
137
+ assert_equal [{"k1" => 'v', "k2" => 'v'}], d.filtered_as_array.map(&:last)
138
+ end
114
139
  end
@@ -110,4 +110,19 @@ class RecordModifierOutputTest < Test::Unit::TestCase
110
110
 
111
111
  assert_equal [{"k4" => 'v'}], d.records
112
112
  end
113
+
114
+ def test_remove_non_whitelist_keys
115
+ d = create_driver %[
116
+ type record_modifier
117
+
118
+ tag foo.filtered
119
+ whitelist_keys k1, k2, k3
120
+ ]
121
+
122
+ d.run do
123
+ d.emit("k1" => 'v', "k2" => 'v', "k4" => 'v', "k5" => 'v')
124
+ end
125
+
126
+ assert_equal [{"k1" => 'v', "k2" => 'v'}], d.records
127
+ end
113
128
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-record-modifier
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.4.0
4
+ version: 0.4.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Masahiro Nakagawa
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-01-08 00:00:00.000000000 Z
11
+ date: 2016-02-26 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: fluentd
@@ -72,7 +72,7 @@ dependencies:
72
72
  - - "~>"
73
73
  - !ruby/object:Gem::Version
74
74
  version: 3.1.4
75
- description: Output filter plugin for modifying each event record
75
+ description: Filter plugin for modifying event record
76
76
  email: repeatedly@gmail.com
77
77
  executables: []
78
78
  extensions: []
@@ -112,7 +112,7 @@ rubyforge_project:
112
112
  rubygems_version: 2.2.2
113
113
  signing_key:
114
114
  specification_version: 4
115
- summary: Output filter plugin for modifying each event record
115
+ summary: Filter plugin for modifying event record
116
116
  test_files:
117
117
  - test/test_filter_record_modifier.rb
118
118
  - test/test_out_record_modifier.rb