fluent-plugin-record-modifier 1.1.0 → 2.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: 3b44157dfddde7eefc70b4798674e499834fad6f52fafc8da7289696944ec9ca
-  data.tar.gz: faa3f952b7bb41eaa46719494d3aed58bf8f59ed0d7fa476ff2491c700c5ff73
+SHA1:
+  metadata.gz: 50fbb4b964b12f2b00fcb947614ba55e7d5df199
+  data.tar.gz: 1184b66de40e482d7cdc882b903b60656b8f323d
 SHA512:
-  metadata.gz: 0e69f8b91068b1998c60a41449d97471aca548fd7eb002bf8e26048895b95af6cd64c17c771d4a85ef1bcecf63138d404bc1a56ed6a6c052047f5d85db57820f
-  data.tar.gz: b47c9e3a4f054a663e96b7b7ac548ba2877edc941cfcc2e123a7d832239ae1be136baf8a330cdebed391523b92f2b737a68ceb6187211b79ec15e3f17843471f
+  metadata.gz: ca78925ef98fca002b037e3e4e250563eec3d1782e998ae3e8594ca72c363d1c510909a4ad44d7b56c3d6b1f274ef43c0438e515f2c984fcd9994ab9f6faa458
+  data.tar.gz: 03bc94a47d90ad10e3fa96931d9e2d315e3a3fdee6ccc901a7b233f5c927e7adf8d667580a333a9ec749adc1eb41885fb0b2529a0fd868a5e7663984ba070384

data/ChangeLog

@@ -1,4 +1,9 @@
-Release 1.1.0 - 2017/06/01
+Release 2.0.0 - 2019/01/23
+
+  * Update output plugin to follow filter plugin
+  * Remove backward compatibility check for 0.5.0 or earlier version
+
+Release 1.1.0 - 2018/06/01
 
   * Add `replace` config sections
     https://github.com/repeatedly/fluent-plugin-record-modifier/pull/33

data/README.md

@@ -9,6 +9,7 @@ In this case, you can use *record_modifier* to add "hostname" field to event rec
 
 | fluent-plugin-record-modifier  | fluentd | ruby |
 |--------------------------------|---------|------|
+| >= 2.0.0 | >= v1.0.0  | >= 2.1 |
 | >= 1.0.0 | >= v0.14.0 | >= 2.1 |
 |  < 1.0.0 | >= v0.12.0 | >= 1.9 |
 
@@ -16,7 +17,7 @@ In this case, you can use *record_modifier* to add "hostname" field to event rec
 
 Use RubyGems:
 
-    gem install fluent-plugin-record-modifier --no-document
+    fluent-gem install fluent-plugin-record-modifier --no-document
 
 ## Configuration
 
@@ -206,14 +207,16 @@ If you need own complex logic in filter, writing filter plugin is better. But if
 
 ### record_modifier output
 
-In v0.10, you can use `record_modifier` output to emulate filter. `record_modifier` output doesn't support `<record>` way.
+Output plugin version of `record_modifier` filter. If you want to process events and change tag at the same time, this plugin is useful.
 
     <match pattern>
-      type record_modifier
-      tag foo.filtered
+      @type record_modifier
+      tag foo.${record["field1"]}
 
-      gen_host "#{Socket.gethostname}"
-      foo bar
+      <record>
+        gen_host "#{Socket.gethostname}"
+        foo bar
+      </record>
     </match>
 
 ## Copyright

data/VERSION

@@ -1 +1 @@
-1.1.0
+2.0.0

data/fluent-plugin-record-modifier.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |gem|
   gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   gem.require_paths = ['lib']
 
-  gem.add_dependency "fluentd", [">= 0.14.0", "< 2"]
+  gem.add_dependency "fluentd", [">= 1.0", "< 2"]
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency("test-unit", ["~> 3.1.4"])
 end

data/lib/fluent/plugin/filter_record_modifier.rb

@@ -50,10 +50,6 @@ DESC
     def configure(conf)
       super
 
-      if conf.has_key?('include_tag_key')
-        raise ConfigError, "include_tag_key and tag_key parameters are removed. Use 'tag ${tag}' in <record> section"
-      end
-
       @map = {}
       @to_enc = nil
       if @char_encoding
@@ -75,7 +71,6 @@ DESC
       @has_tag_parts = false
       conf.elements.select { |element| element.name == 'record' }.each do |element|
         element.each_pair do |k, v|
-          check_config_placeholders(k, v)
           element.has_key?(k) # to suppress unread configuration warning
           @has_tag_parts = true if v.include?('tag_parts')
           @map[k] = DynamicExpander.new(k, v, @prepare_value)
@@ -165,16 +160,6 @@ DESC
       end
     end
 
-    HOSTNAME_PLACEHOLDERS = %W(__HOSTNAME__ ${hostname})
-
-    def check_config_placeholders(k, v)
-      HOSTNAME_PLACEHOLDERS.each { |ph|
-        if v.include?(ph)
-          raise ConfigError, %!#{ph} placeholder in #{k} is removed. Use "\#{Socket.gethostname}" instead.!
-        end
-      }
-    end
-
     class DynamicExpander
       def initialize(param_key, param_value, prepare_value)
         if param_value.include?('${')

data/lib/fluent/plugin/out_record_modifier.rb

@@ -4,10 +4,16 @@ module Fluent
   class Plugin::RecordModifierOutput < Plugin::Output
     Fluent::Plugin.register_output('record_modifier', self)
 
-    helpers :event_emitter, :compat_parameters, :inject
+    helpers :event_emitter
 
     config_param :tag, :string,
                  desc: "The output record tag name."
+
+    config_param :prepare_value, :string, default: nil,
+                 desc: <<-DESC
+Prepare values for filtering in configure phase. Prepared values can be used in <record>.
+You can write any ruby code.
+DESC
     config_param :char_encoding, :string, default: nil,
                  desc: <<-DESC
 Fluentd including some plugins treats the logs as a BINARY by default to forward.
@@ -16,7 +22,6 @@ e.g. handling char encoding correctly.
 In more detail, please refer this section:
 https://github.com/repeatedly/fluent-plugin-record-modifier#char_encoding.
 DESC
-
     config_param :remove_keys, :string, default: nil,
                  desc: <<-DESC
 The logs include needless record keys in some cases.
@@ -31,21 +36,26 @@ Modified events will have only specified keys (if exist in original events).
 This option is exclusive with `remove_keys`.
 DESC
 
-    BUILTIN_CONFIGURATIONS = %W(type @type log_level @log_level id @id tag include_tag_key tag_key char_encoding remove_keys whitelist_keys)
+    config_section :replace, param_name: :replaces, multi: true do
+      desc "The field name to which the regular expression is applied"
+      config_param :key, :string
+      desc "The regular expression"
+      config_param :expression do |value|
+        if value.start_with?("/") && value.end_with?("/")
+          Regexp.compile(value[1..-2])
+        else
+          $log.warn "You should use \"pattern /#{value}/\" instead of \"pattern #{value}\""
+          Regexp.compile(value)
+        end
+      end
+      desc "The replacement string"
+      config_param :replace, :string
+    end
 
     def configure(conf)
-      compat_parameters_convert(conf, :buffer, :inject)
       super
 
       @map = {}
-      conf.each_pair { |k, v|
-        unless BUILTIN_CONFIGURATIONS.include?(k)
-          check_config_placeholders(k, v)
-          conf.has_key?(k)
-          @map[k] = v
-        end
-      }
-
       @to_enc = nil
       if @char_encoding
         from, to = @char_encoding.split(':', 2)
@@ -63,6 +73,15 @@ DESC
         end
       end
 
+      @has_tag_parts = false
+      conf.elements.select { |element| element.name == 'record' }.each do |element|
+        element.each_pair do |k, v|
+          element.has_key?(k) # to suppress unread configuration warning
+          @has_tag_parts = true if v.include?('tag_parts')
+          @map[k] = DynamicExpander.new(k, v, @prepare_value)
+        end
+      end
+
       if @remove_keys and @whitelist_keys
         raise Fluent::ConfigError, "remove_keys and whitelist_keys are exclusive with each other."
       elsif @remove_keys
@@ -70,32 +89,42 @@ DESC
       elsif @whitelist_keys
         @whitelist_keys = @whitelist_keys.split(',').map(&:strip)
       end
+
+      @has_tag_parts = true if @tag.include?('tag_parts')
+      @tag_ex = DynamicExpander.new('tag', @tag, @prepare_value)
+
+        # Collect DynamicExpander related garbage instructions
+      GC.start
     end
 
     def process(tag, es)
-      stream = MultiEventStream.new
-      es.each { |time, record|
-        record = inject_values_to_record(tag, time, record)
-        stream.add(time, modify_record(record))
-      }
-      router.emit_stream(@tag, stream)
+      tag_parts = @has_tag_parts ? tag.split('.') : nil
+      if @tag_ex.param_value.nil?
+        result = {}
+        es.each { |time, record|
+          new_record = modify_record(tag, time, record, tag_parts)
+          new_tag = @tag_ex.expand(tag, time, new_record, tag_parts)
+          result[new_tag] ||= MultiEventStream.new
+          result[new_tag].add(time, new_record)
+        }
+        result.each { |tag, stream|
+          router.emit_stream(tag, stream)
+        }
+      else
+        stream = MultiEventStream.new
+        es.each { |time, record|
+          new_record = modify_record(tag, time, record, tag_parts)
+          stream.add(time, new_record)
+        }
+        router.emit_stream(@tag, stream)
+      end
     end
 
     private
 
-    HOSTNAME_PLACEHOLDERS = %W(__HOSTNAME__ ${hostname})
-
-    def check_config_placeholders(k, v)
-      HOSTNAME_PLACEHOLDERS.each { |ph|
-        if v.include?(ph)
-          raise ConfigError, %!#{ph} placeholder in #{k} is removed. Use "\#{Socket.gethostname}" instead.!
-        end
-      }
-    end
-
-    def modify_record(record)
+    def modify_record(tag, time, record, tag_parts)
       @map.each_pair { |k, v|
-        record[k] = v
+        record[k] = v.expand(tag, time, record, tag_parts)
       }
 
       if @remove_keys
@@ -110,25 +139,110 @@ DESC
         record = modified
       end
 
+      unless @replaces.empty?
+        @replaces.each { |replace|
+          target_key = replace.key
+          if record.include?(target_key) && replace.expression.match(record[target_key])
+            record[target_key] = record[target_key].gsub(replace.expression, replace.replace)
+          end
+        }
+      end
+
       record = change_encoding(record) if @char_encoding
       record
     end
 
-    def set_encoding(record)
-      record.each_pair { |k, v|
-        if v.is_a?(String)
-          v.force_encoding(@from_enc)
-        end
-      }
+    def set_encoding(value)
+      if value.is_a?(String)
+        value.force_encoding(@from_enc)
+      elsif value.is_a?(Hash)
+        value.each_pair { |k, v|
+          if v.frozen? && v.is_a?(String)
+            value[k] = set_encoding(v.dup)
+          else
+            set_encoding(v)
+          end
+        }
+      elsif value.is_a?(Array)
+        value.each { |v| set_encoding(v) }
+      else
+        value
+      end
     end
 
-    def convert_encoding(record)
-      record.each_pair { |k, v|
-        if v.is_a?(String)
-          v.force_encoding(@from_enc) if v.encoding == Encoding::BINARY
-          v.encode!(@to_enc, @from_enc, :invalid => :replace, :undef => :replace)
+    def convert_encoding(value)
+      if value.is_a?(String)
+        value.force_encoding(@from_enc) if value.encoding == Encoding::BINARY
+        value.encode!(@to_enc, @from_enc, :invalid => :replace, :undef => :replace)
+      elsif value.is_a?(Hash)
+        value.each_pair { |k, v|
+          if v.frozen? && v.is_a?(String)
+            value[k] = convert_encoding(v.dup)
+          else
+            convert_encoding(v)
+          end
+        }
+      elsif value.is_a?(Array)
+        value.each { |v| convert_encoding(v) }
+      else
+        value
+      end
+    end
+
+    class DynamicExpander
+      attr_reader :param_value
+
+      def initialize(param_key, param_value, prepare_value)
+        if param_value.include?('${')
+          __str_eval_code__ = parse_parameter(param_value)
+
+          # Use class_eval with string instead of define_method for performance.
+          # It can't share instructions but this is 2x+ faster than define_method in filter case.
+          # Refer: http://tenderlovemaking.com/2013/03/03/dynamic_method_definitions.html
+          (class << self; self; end).class_eval <<-EORUBY,  __FILE__, __LINE__ + 1
+            def expand(tag, time, record, tag_parts)
+              #{__str_eval_code__}
+            end
+          EORUBY
+        else
+          @param_value = param_value
         end
-      }
+
+        begin
+          eval prepare_value if prepare_value
+        rescue SyntaxError
+          raise ConfigError, "Pass invalid syntax parameter : key = prepare_value, value = #{prepare_value}"
+        end
+
+        begin
+          # check eval genarates wrong code or not
+          expand(nil, nil, nil, nil)
+        rescue SyntaxError
+          raise ConfigError, "Pass invalid syntax parameter : key = #{param_key}, value = #{param_value}"
+        rescue
+          # Ignore other runtime errors
+        end
+      end
+
+      # Default implementation for fixed value. This is overwritten when parameter contains '${xxx}' placeholder
+      def expand(tag, time, record, tag_parts)
+        @param_value
+      end
+
+      private
+
+      def parse_parameter(value)
+        num_placeholders = value.scan('${').size
+        if num_placeholders == 1
+          if value.start_with?('${') && value.end_with?('}')
+            return value[2..-2]
+          else
+            "\"#{value.gsub('${', '#{')}\""
+          end
+        else
+          "\"#{value.gsub('${', '#{')}\""
+        end
+      end
     end
   end
 end

data/test/test_out_record_modifier.rb

@@ -8,13 +8,11 @@ class RecordModifierOutputTest < Test::Unit::TestCase
   end
 
   CONFIG = %q!
-    type record_modifier
     tag foo.filtered
-
-    gen_host "#{Socket.gethostname}"
-    foo bar
-    include_tag_key
-    tag_key included_tag
+    <record>
+      gen_host "#{Socket.gethostname}"
+      foo bar
+    </record>
     remove_keys hoge
   !
 
@@ -31,8 +29,8 @@ class RecordModifierOutputTest < Test::Unit::TestCase
     d = create_driver
     map = d.instance.instance_variable_get(:@map)
 
-    assert_equal get_hostname, map['gen_host']
-    assert_equal 'bar', map['foo']
+    assert_equal get_hostname, map['gen_host'].param_value
+    assert_equal 'bar', map['foo'].param_value
   end
 
   def test_format
@@ -43,17 +41,39 @@ class RecordModifierOutputTest < Test::Unit::TestCase
       d.feed({"a" => 2})
     end
 
-    mapped = {'gen_host' => get_hostname, 'foo' => 'bar', 'included_tag' => 'test_tag'}
+    mapped = {'gen_host' => get_hostname, 'foo' => 'bar'}
     assert_equal [
       {"a" => 1}.merge(mapped),
       {"a" => 2}.merge(mapped),
     ], d.events.map { |e| e.last }
   end
 
-  def test_set_char_encoding
+  def test_dynamic_tag_with_tag
     d = create_driver %[
-      type record_modifier
+      tag foo.${tag}
+    ]
+
+    d.run(default_tag: 'test_tag') do
+      d.feed({"k" => 'v'})
+    end
+
+    assert_equal 'foo.test_tag', d.events.first.first
+  end
+
+  def test_dynamic_tag_with_record_field
+    d = create_driver %[
+      tag foo.${record["k"]}
+    ]
 
+    d.run(default_tag: 'test_tag') do
+      d.feed({"k" => 'v'})
+    end
+
+    assert_equal 'foo.v', d.events.first.first
+  end
+
+  def test_set_char_encoding
+    d = create_driver %[
       tag foo.filtered
       char_encoding utf-8
     ]
@@ -67,8 +87,6 @@ class RecordModifierOutputTest < Test::Unit::TestCase
 
   def test_convert_char_encoding
     d = create_driver %[
-      type record_modifier
-
       tag foo.filtered
       char_encoding utf-8:cp932
     ]
@@ -82,8 +100,6 @@ class RecordModifierOutputTest < Test::Unit::TestCase
 
   def test_remove_one_key
     d = create_driver %[
-      type record_modifier
-
       tag foo.filtered
       remove_keys k1
     ]
@@ -97,8 +113,6 @@ class RecordModifierOutputTest < Test::Unit::TestCase
 
   def test_remove_multiple_keys
     d = create_driver %[
-      type record_modifier
-
       tag foo.filtered
       remove_keys k1, k2, k3
     ]
@@ -112,8 +126,6 @@ class RecordModifierOutputTest < Test::Unit::TestCase
 
   def test_remove_non_whitelist_keys
     d = create_driver %[
-      type record_modifier
-
       tag foo.filtered
       whitelist_keys k1, k2, k3
     ]

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-record-modifier
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 2.0.0
 platform: ruby
 authors:
 - Masahiro Nakagawa
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-06-01 00:00:00.000000000 Z
+date: 2019-01-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -16,7 +16,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.0
+        version: '1.0'
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -26,7 +26,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.0
+        version: '1.0'
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -95,7 +95,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 2.6.14.1
 signing_key: 
 specification_version: 4
 summary: Filter plugin for modifying event record