fluent-plugin-record-modifier 1.0.1 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7f658bdc50758abb408ee1c2d61e3a982863564b
-  data.tar.gz: 03a81b1c111ebcd9f937764e63a5ef9736877c5d
+  metadata.gz: 290bea3c616897ac2b1125aabcebb421b3099863
+  data.tar.gz: f81e3ec261a75b79fd0ea5bac81698ffc6d9f357
 SHA512:
-  metadata.gz: 92bb89073f76c579f19c9fcc0af6bc1f2dabe67c367330fa178d69a6356c5fdc201cdc74c66ba1e3eb809fbc8bffcee289b95dc2d1f7434481c05c175f3b61d8
-  data.tar.gz: 2b22a75549e3d017f3bb928ac6247ba0f162042c237569227bff03cab4ccf19120159c931811e39ea21210dfe3f4bb51a862f04e6ce8c25b2ba0a821521f002a
+  metadata.gz: 64ae991840fb9cb6eab3bc2be79ccbe3c368fd59110a400fa4329dcffa14b852e7c4df18e816458f0c3308bf3937ef679e1f2939e067e2dc354900383e9e7a64
+  data.tar.gz: e82a147fa6b1408bf49f21853d888a3b616df5bf474d675952767aeb003bcc58f3761f9a60a8d3f74a007abab6604fbe7725da85de3123d90a5ac70a4593a332

data/ChangeLog

@@ -1,3 +1,9 @@
+Release 1.0.2 - 2017/12/27
+
+  * Add prepare_value parameter
+  * Fix non-string object handling
+  * Remove deprecated code
+
 Release 1.0.1 - 2017/12/16
 
   * Fix fluentd dependency

data/README.md

@@ -62,6 +62,20 @@ But unlike `record_transformer`, `record_modifier` doesn't support following fea
 - tag_suffix and tag_prefix
 - dynamic key placeholder
 
+### prepare_value
+
+Prepare values for filtering. This ruby code is evaluated in `configure` phase and prepared values can be used in `<record>`. Here is an example:
+
+    <filter pattern>
+      @type record_modifier
+      prepare_value require 'foo'; @foo = Foo.new
+      <record>
+        key ${@foo.method1}
+      </record>
+    </filter>
+
+This feature is useful for using external library.
+
 ### char_encoding
 
 Fluentd including some plugins treats logs as a BINARY by default to forward.

data/VERSION

@@ -1 +1 @@
-1.0.1
+1.0.2

data/lib/fluent/plugin/filter_record_modifier.rb

@@ -4,6 +4,11 @@ module Fluent
   class Plugin::RecordModifierFilter < Plugin::Filter
     Fluent::Plugin.register_filter('record_modifier', self)
 
+    config_param :prepare_value, :string, default: nil,
+                 desc: <<-DESC
+Prepare values for filtering in configure phase. Prepared values can be used in <record>.
+You can write any ruby code.
+DESC
     config_param :char_encoding, :string, default: nil,
                  desc: <<-DESC
 Fluentd including some plugins treats the logs as a BINARY by default to forward.
@@ -26,8 +31,6 @@ Modified events will have only specified keys (if exist in original events).
 This option is exclusive with `remove_keys`.
 DESC
 
-    BUILTIN_CONFIGURATIONS = %W(type @type log_level @log_level id @id char_encoding remove_keys whitelist_keys)
-
     def configure(conf)
       super
 
@@ -36,15 +39,6 @@ DESC
       end
 
       @map = {}
-      conf.each_pair { |k, v|
-        unless BUILTIN_CONFIGURATIONS.include?(k)
-          check_config_placeholders(k, v);
-          conf.has_key?(k)
-          $log.warn "top level definition is deprecated. Please put parameters inside <record>: '#{k} #{v}'"
-          @map[k] = DynamicExpander.new(k, v)
-        end
-      }
-
       @to_enc = nil
       if @char_encoding
         from, to = @char_encoding.split(':', 2)
@@ -68,7 +62,7 @@ DESC
           check_config_placeholders(k, v)
           element.has_key?(k) # to suppress unread configuration warning
           @has_tag_parts = true if v.include?('tag_parts')
-          @map[k] = DynamicExpander.new(k, v)
+          @map[k] = DynamicExpander.new(k, v, @prepare_value)
         end
       end
 
@@ -114,7 +108,7 @@ DESC
         value.force_encoding(@from_enc)
       elsif value.is_a?(Hash)
         value.each_pair { |k, v|
-          if v.frozen?
+          if v.frozen? && v.is_a?(String)
             value[k] = set_encoding(v.dup)
           else
             set_encoding(v)
@@ -122,6 +116,8 @@ DESC
         }
       elsif value.is_a?(Array)
         value.each { |v| set_encoding(v) }
+      else
+        value
       end
     end
 
@@ -131,7 +127,7 @@ DESC
         value.encode!(@to_enc, @from_enc, :invalid => :replace, :undef => :replace)
       elsif value.is_a?(Hash)
         value.each_pair { |k, v|
-          if v.frozen?
+          if v.frozen? && v.is_a?(String)
             value[k] = convert_encoding(v.dup)
           else
             convert_encoding(v)
@@ -139,6 +135,8 @@ DESC
         }
       elsif value.is_a?(Array)
         value.each { |v| convert_encoding(v) }
+      else
+        value
       end
     end
 
@@ -153,7 +151,7 @@ DESC
     end
 
     class DynamicExpander
-      def initialize(param_key, param_value)
+      def initialize(param_key, param_value, prepare_value)
         if param_value.include?('${')
           __str_eval_code__ = parse_parameter(param_value)
 
@@ -169,6 +167,12 @@ DESC
           @param_value = param_value
         end
 
+        begin
+          eval prepare_value if prepare_value
+        rescue SyntaxError
+          raise ConfigError, "Pass invalid syntax parameter : key = prepare_value, value = #{prepare_value}"
+        end
+
         begin
           # check eval genarates wrong code or not
           expand(nil, nil, nil, nil)

data/test/test_filter_record_modifier.rb

@@ -126,6 +126,21 @@ class RecordModifierFilterTest < Test::Unit::TestCase
     assert_equal [{"k1" => 'v', "k2" => 'v'}], d.filtered.map(&:last)
   end
 
+  def test_prepare_values
+    d = create_driver %[
+      prepare_value @foo = 'foo'
+      <record>
+        test_key ${@foo}
+      </record>
+    ]
+
+    d.run(default_tag: @tag) do
+      d.feed("k1" => 'v')
+    end
+
+    assert_equal [{"k1" => 'v', "test_key" => 'foo'}], d.filtered.map(&:last)
+  end
+
   sub_test_case 'frozen check' do
     def test_set_char_encoding
       d = create_driver %[
@@ -133,13 +148,13 @@ class RecordModifierFilterTest < Test::Unit::TestCase
       ]
 
       d.run(default_tag: @tag) do
-        d.feed("k" => 'v'.force_encoding('BINARY').freeze)
-        d.feed("k" => {"l" => 'v'.force_encoding('BINARY').freeze})
+        d.feed("k" => 'v'.force_encoding('BINARY').freeze, 'n' => 1)
+        d.feed("k" => {"l" => 'v'.force_encoding('BINARY').freeze, 'n' => 1})
       end
 
       assert_equal [
-        {"k" => 'v'.force_encoding('UTF-8')},
-        {"k" => {"l" => 'v'.force_encoding('UTF-8')}},
+        {"k" => 'v'.force_encoding('UTF-8'), 'n' => 1},
+        {"k" => {"l" => 'v'.force_encoding('UTF-8'), 'n' => 1}},
       ], d.filtered.map { |e| e.last }
     end
 
@@ -149,13 +164,13 @@ class RecordModifierFilterTest < Test::Unit::TestCase
       ]
 
       d.run(default_tag: @tag) do
-        d.feed("k" => 'v'.force_encoding('utf-8').freeze)
-        d.feed("k" => {"l" => 'v'.force_encoding('utf-8').freeze})
+        d.feed("k" => 'v'.force_encoding('utf-8').freeze, 'n' => 1)
+        d.feed("k" => {"l" => 'v'.force_encoding('utf-8').freeze, 'n' => 1})
       end
 
       assert_equal [
-        {"k" => 'v'.force_encoding('cp932')},
-        {"k" => {"l" => 'v'.force_encoding('cp932')}},
+        {"k" => 'v'.force_encoding('cp932'), 'n' => 1},
+        {"k" => {"l" => 'v'.force_encoding('cp932'), 'n' => 1}},
       ], d.filtered.map { |e| e.last }
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-record-modifier
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.0.2
 platform: ruby
 authors:
 - Masahiro Nakagawa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-12-16 00:00:00.000000000 Z
+date: 2017-12-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd