fluent-plugin-record-modifier 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3ff1d6de463980c890752f4eb32719f3f160c026
-  data.tar.gz: 4ed772c23ab2316e1bf064a9fd3c1ed498c1aae8
+  metadata.gz: 393309787a689b84bc9282849cd03b6d50bf68f4
+  data.tar.gz: 24c92b2b9d47a616706701f4ec38c08ccbaccef7
 SHA512:
-  metadata.gz: 2477196fe7d51f5d26546107d4b9a01907e97e3dc38a0d5a2a1a4cc8182f298d24abb996b197e3609cae7e0a534781b93f0d5bd802d7afc31c0297f2f5e31924
-  data.tar.gz: e0826bc7e9203ea5a6bf24f715b5b63d2b13a95c407a4d298cfd8b9f2f7c781afed10c69248774e8930b7a0f86cc1f569c75a9c2e3c00e689e3dffc4fe2f6e5d
+  metadata.gz: 49ee18b86ca8e11cc803947c6dcc7f7beea0ce138d957ceaadb46f55053eb3ddf72d54b64bcbfd3b049bbc075823a54347acedb807cbb6b7ea972b8f6bce4eb2
+  data.tar.gz: b41c41bc4a2a972cadf7b078125d8866ba76b5324f5990c3cf7a412707e7a56c43c934d4cca85479ce6b82ec2473a786cf350ce004f1264fe7b98dbc1b12499c

data/ChangeLog

@@ -1,3 +1,10 @@
+Release 0.4.1 - 2016/02/26
+
+  * Set/Convert encoding recursively
+    https://github.com/repeatedly/fluent-plugin-record-modifier/pull/9
+  * Add whitelist_keys option
+    https://github.com/repeatedly/fluent-plugin-record-modifier/pull/11
+
 Release 0.4.0 - 2016/01/08
 
   * Introduce <record> directive

data/README.md

@@ -112,6 +112,31 @@ then you got new record like below:
 {"key3":"bar"}
 ```
 
+### whitelist_keys
+
+If you want to handle the set of explicitly specified keys, you can use `whitelist_keys` of this plugin. It's exclusive with `remove_keys`.
+
+```conf
+<filter pattern>
+  type record_modifier
+
+  # remove all keys except for key1 and key2
+  whitelist_keys key1,key2
+</filter>
+```
+
+If following record is passed:
+
+```js
+{"key1":"hoge", "key2":"foo", "key3":"bar"}
+```
+
+then you got new record like below:
+
+```js
+{"key1":"hoge", "key2":"foo"}
+```
+
 ### Mixins
 
 * [fluent-mixin-config-placeholders](https://github.com/tagomoris/fluent-mixin-config-placeholders)

data/VERSION

@@ -1 +1 @@
-0.4.0
+0.4.1

data/fluent-plugin-record-modifier.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |gem|
   gem.name        = "fluent-plugin-record-modifier"
-  gem.description = "Output filter plugin for modifying each event record"
+  gem.description = "Filter plugin for modifying event record"
   gem.homepage    = "https://github.com/repeatedly/fluent-plugin-record-modifier"
   gem.summary     = gem.description
   gem.version     = File.read("VERSION").strip

data/lib/fluent/plugin/filter_record_modifier.rb

@@ -4,12 +4,31 @@ module Fluent
   class RecordModifierFilter < Filter
     Fluent::Plugin.register_filter('record_modifier', self)
 
-    config_param :char_encoding, :string, :default => nil
-    config_param :remove_keys, :string, :default => nil
+    config_param :char_encoding, :string, default: nil,
+                 desc: <<-DESC
+Fluentd including some plugins treats the logs as a BINARY by default to forward.
+But an user sometimes processes the logs depends on their requirements,
+e.g. handling char encoding correctly.
+In more detail, please refer this section:
+https://github.com/repeatedly/fluent-plugin-record-modifier#char_encoding.
+DESC
+    config_param :remove_keys, :string, default: nil,
+                 desc: <<-DESC
+The logs include needless record keys in some cases.
+You can remove it by using `remove_keys` parameter.
+This option is exclusive with `whitelist_keys`.
+DESC
+
+    config_param :whitelist_keys, :string, default: nil,
+                 desc: <<-DESC
+Specify `whitelist_keys` to remove all unexpected keys and values from events.
+Modified events will have only specified keys (if exist in original events).
+This option is exclusive with `remove_keys`.
+DESC
 
     include Fluent::Mixin::ConfigPlaceholders
 
-    BUILTIN_CONFIGURATIONS = %W(type @type log_level @log_level id @id char_encoding remove_keys)
+    BUILTIN_CONFIGURATIONS = %W(type @type log_level @log_level id @id char_encoding remove_keys whitelist_keys)
 
     def configure(conf)
       super
@@ -52,8 +71,12 @@ module Fluent
         end
       end
 
-      if @remove_keys
-        @remove_keys = @remove_keys.split(',').map { |e| e.strip }
+      if @remove_keys and @whitelist_keys
+        raise Fluent::ConfigError, "remove_keys and whitelist_keys are exclusive with each other."
+      elsif @remove_keys
+        @remove_keys = @remove_keys.split(',').map(&:strip)
+      elsif @whitelist_keys
+        @whitelist_keys = @whitelist_keys.split(',').map(&:strip)
       end
 
       # Collect DynamicExpander related garbage instructions
@@ -73,6 +96,12 @@ module Fluent
           @remove_keys.each { |v|
             record.delete(v)
           }
+        elsif @whitelist_keys
+          modified = {}
+          record.each do |k, v|
+            modified[k] = v if @whitelist_keys.include?(k)
+          end
+          record = modified
         end
 
         record = change_encoding(record) if @char_encoding
@@ -83,21 +112,25 @@ module Fluent
 
     private
 
-    def set_encoding(record)
-      record.each_pair { |k, v|
-        if v.is_a?(String)
-          v.force_encoding(@from_enc)
-        end
-      }
+    def set_encoding(value)
+      if value.is_a?(String)
+        value.force_encoding(@from_enc)
+      elsif value.is_a?(Hash)
+        value.each_pair { |k, v| set_encoding(v) }
+      elsif value.is_a?(Array)
+        value.each { |v| set_encoding(v) }
+      end
     end
 
-    def convert_encoding(record)
-      record.each_pair { |k, v|
-        if v.is_a?(String)
-          v.force_encoding(@from_enc) if v.encoding == Encoding::BINARY
-          v.encode!(@to_enc, @from_enc, :invalid => :replace, :undef => :replace)
-        end
-      }
+    def convert_encoding(value)
+      if value.is_a?(String)
+        value.force_encoding(@from_enc) if value.encoding == Encoding::BINARY
+        value.encode!(@to_enc, @from_enc, :invalid => :replace, :undef => :replace)
+      elsif value.is_a?(Hash)
+        value.each_pair { |k, v| convert_encoding(v) }
+      elsif value.is_a?(Array)
+        value.each { |v| convert_encoding(v) }
+      end
     end
 
     class DynamicExpander

data/lib/fluent/plugin/out_record_modifier.rb

@@ -4,14 +4,35 @@ module Fluent
   class RecordModifierOutput < Output
     Fluent::Plugin.register_output('record_modifier', self)
 
-    config_param :tag, :string
-    config_param :char_encoding, :string, :default => nil
-    config_param :remove_keys, :string, :default => nil
+    config_param :tag, :string,
+                 desc: "The output record tag name."
+    config_param :char_encoding, :string, default: nil,
+                 desc: <<-DESC
+Fluentd including some plugins treats the logs as a BINARY by default to forward.
+But an user sometimes processes the logs depends on their requirements,
+e.g. handling char encoding correctly.
+In more detail, please refer this section:
+https://github.com/repeatedly/fluent-plugin-record-modifier#char_encoding.
+DESC
+
+    config_param :remove_keys, :string, default: nil,
+                 desc: <<-DESC
+The logs include needless record keys in some cases.
+You can remove it by using `remove_keys` parameter.
+This option is exclusive with `whitelist_keys`.
+DESC
+
+    config_param :whitelist_keys, :string, default: nil,
+                 desc: <<-DESC
+Specify `whitelist_keys` to remove all unexpected keys and values from events.
+Modified events will have only specified keys (if exist in original events).
+This option is exclusive with `remove_keys`.
+DESC
 
     include SetTagKeyMixin
     include Fluent::Mixin::ConfigPlaceholders
 
-    BUILTIN_CONFIGURATIONS = %W(type tag include_tag_key tag_key char_encoding remove_keys)
+    BUILTIN_CONFIGURATIONS = %W(type tag include_tag_key tag_key char_encoding remove_keys whitelist_keys)
 
     def configure(conf)
       super
@@ -40,16 +61,22 @@ module Fluent
         end
       end
 
-      if @remove_keys
-        @remove_keys = @remove_keys.split(',').map {|e| e.strip }
+      if @remove_keys and @whitelist_keys
+        raise Fluent::ConfigError, "remove_keys and whitelist_keys are exclusive with each other."
+      elsif @remove_keys
+        @remove_keys = @remove_keys.split(',').map(&:strip)
+      elsif @whitelist_keys
+        @whitelist_keys = @whitelist_keys.split(',').map(&:strip)
       end
     end
 
     def emit(tag, es, chain)
+      stream = MultiEventStream.new
       es.each { |time, record|
         filter_record(tag, time, record)
-        Engine.emit(@tag, time, modify_record(record))
+        stream.add(time, modify_record(record))
       }
+      Fluent::Engine.emit_stream(@tag, stream)
 
       chain.next
     end
@@ -65,6 +92,12 @@ module Fluent
         @remove_keys.each { |v|
           record.delete(v)
         }
+      elsif @whitelist_keys
+        modified = {}
+        record.each do |k, v|
+          modified[k] = v if @whitelist_keys.include?(k)
+        end
+        record = modified
       end
 
       record = change_encoding(record) if @char_encoding

data/test/test_filter_record_modifier.rb

@@ -62,12 +62,17 @@ class RecordModifierFilterTest < Test::Unit::TestCase
       char_encoding utf-8
     ]
 
-
     d.run do
       d.emit("k" => 'v'.force_encoding('BINARY'))
+      d.emit("k" => %w(v ビ).map{|v| v.force_encoding('BINARY')})
+      d.emit("k" => {"l" => 'ビ'.force_encoding('BINARY')})
     end
 
-    assert_equal [{"k" => 'v'.force_encoding('UTF-8')}], d.filtered_as_array.map { |e| e.last }
+    assert_equal [
+      {"k" => 'v'.force_encoding('UTF-8')},
+      {"k" => %w(v ビ).map{|v| v.force_encoding('UTF-8')}},
+      {"k" => {"l" => 'ビ'.force_encoding('UTF-8')}},
+    ], d.filtered_as_array.map { |e| e.last }
   end
 
   def test_convert_char_encoding
@@ -79,9 +84,15 @@ class RecordModifierFilterTest < Test::Unit::TestCase
 
     d.run do
       d.emit("k" => 'v'.force_encoding('utf-8'))
+      d.emit("k" => %w(v ビ).map{|v| v.force_encoding('utf-8')})
+      d.emit("k" => {"l" => 'ビ'.force_encoding('utf-8')})
     end
 
-    assert_equal [{"k" => 'v'.force_encoding('cp932')}], d.filtered_as_array.map { |e| e.last }
+    assert_equal [
+      {"k" => 'v'.force_encoding('cp932')},
+      {"k" => %w(v ビ).map{|v| v.encode!('cp932')}},
+      {"k" => {"l" => 'ビ'.encode!('cp932')}},
+    ], d.filtered_as_array.map { |e| e.last }
   end
 
   def test_remove_one_key
@@ -111,4 +122,18 @@ class RecordModifierFilterTest < Test::Unit::TestCase
 
     assert_equal [{"k4" => 'v'}], d.filtered_as_array.map { |e| e.last }
   end
+
+  def test_remove_non_whitelist_keys
+    d = create_driver %[
+      type record_modifier
+
+      whitelist_keys k1, k2, k3
+    ]
+
+    d.run do
+      d.emit("k1" => 'v', "k2" => 'v', "k4" => 'v', "k5" => 'v')
+    end
+
+    assert_equal [{"k1" => 'v', "k2" => 'v'}], d.filtered_as_array.map(&:last)
+  end
 end

data/test/test_out_record_modifier.rb

@@ -110,4 +110,19 @@ class RecordModifierOutputTest < Test::Unit::TestCase
 
     assert_equal [{"k4" => 'v'}], d.records
   end
+
+  def test_remove_non_whitelist_keys
+    d = create_driver %[
+      type record_modifier
+
+      tag foo.filtered
+      whitelist_keys k1, k2, k3
+    ]
+
+    d.run do
+      d.emit("k1" => 'v', "k2" => 'v', "k4" => 'v', "k5" => 'v')
+    end
+
+    assert_equal [{"k1" => 'v', "k2" => 'v'}], d.records
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-record-modifier
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - Masahiro Nakagawa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-08 00:00:00.000000000 Z
+date: 2016-02-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -72,7 +72,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 3.1.4
-description: Output filter plugin for modifying each event record
+description: Filter plugin for modifying event record
 email: repeatedly@gmail.com
 executables: []
 extensions: []
@@ -112,7 +112,7 @@ rubyforge_project:
 rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
-summary: Output filter plugin for modifying each event record
+summary: Filter plugin for modifying event record
 test_files:
 - test/test_filter_record_modifier.rb
 - test/test_out_record_modifier.rb