fluent-plugin-record-reformer 0.7.2 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a489c46189c76ae3c606242b7b5d2a955d044f7c
-  data.tar.gz: 72dc9bea1cab4b0153f7d8b2847a5f75334a2e64
+  metadata.gz: bf0c0a50d2b8509f3fa6917e0f700abe30e5249a
+  data.tar.gz: fba94b0ab7c6259354c70be688ef5ad81cc29efe
 SHA512:
-  metadata.gz: 43d512aeb2f8e63779ba894716e17b5c1c0c666a7860497da43c1566f2e00265faa182166272ce9d65e22b83a3a878e7e206cbb72664aa6203588fa8dd34e0ac
-  data.tar.gz: a29e0f8fd38e867411fe2787d1074fabc280b73328c19b7df49d38f81808d97392cd98b1ce988bd517085f7c8aa673a088d66bfc3f8eef72d3b5cb5187d21399
+  metadata.gz: 132876d93292dd82888bbf7a34010dbc3c9e99ea746952165c919a71be3ad8118b3572009f5a8c5deb18654b714142a9b100b39c07ff2e3e40ae9d1e1c014ca9
+  data.tar.gz: 98e35238beb1848b8076ab6c66095a9cb15eb91301bd21f8eed18ad4f70e2eae2b932ad6c525c516f57794d0fee651a162d25a2b519c9fb19bceda90947aaad7

data/.travis.yml

@@ -1,8 +1,9 @@
 rvm:
-  - 1.9.3
   - 2.0.0
   - 2.1.*
   - 2.2.*
+  - 2.3.0
 gemfile:
   - Gemfile
+  - Gemfile.fluentd.0.10
 before_install: gem update bundler

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 0.8.0 (2016/01/28)
+
+Enhancements
+
+* Support `${record["key"]}` placeholder
+* Speed up `enable_ruby true`
+
 ## 0.7.2 (2015/12/29)
 
 Enhancements

data/Gemfile.fluentd.0.10

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+gem 'fluentd', '~> 0.10.43'
+gemspec

data/README.md

@@ -126,14 +126,10 @@ This results in same, but please note that following option parameters are reser
 
 ## Placeholders
 
-The keys of input json are available as placeholders. In the above example, 
-
-* ${remove_me}
-* ${not_remove_me}
-* ${message}
-
-shall be available. In addition, following placeholders are reserved: 
+Following placeholders are available:
 
+* ${record["key"]} Record value of `key` such as `${record["message"]}` in the above example (available from v0.8.0).
+  * Originally, record placeholders were available as `${key}` such as `${message}`. This is still kept for the backward compatibility, but would be removed in the future.
 * ${hostname} Hostname of the running machine
 * ${tag} Input tag
 * ${time} Time of the event

data/fluent-plugin-record-reformer.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |gem|
   gem.name        = "fluent-plugin-record-reformer"
-  gem.version     = "0.7.2"
+  gem.version     = "0.8.0"
   gem.authors     = ["Naotoshi Seo"]
   gem.email       = "sonots@gmail.com"
   gem.homepage    = "https://github.com/sonots/fluent-plugin-record-reformer"

data/lib/fluent/plugin/out_record_reformer.rb

@@ -10,21 +10,21 @@ module Fluent
     end
 
     config_param :output_tag, :string, :default => nil, # obsolete
-                 :desc => 'The output tag name. This option is deprecated. Use `tag` option instead.'
+      :desc => 'The output tag name. This option is deprecated. Use `tag` option instead.'
     config_param :tag, :string, :default => nil,
-                 :desc => 'The output tag name.'
+      :desc => 'The output tag name.'
     config_param :remove_keys, :string, :default => nil,
-                 :desc => 'Specify record keys to be removed by a string separated by , (comma).'
+      :desc => 'Specify record keys to be removed by a string separated by , (comma).'
     config_param :keep_keys, :string, :default => nil,
-                 :desc => 'Specify record keys to be kept by a string separated by , (comma).'
+      :desc => 'Specify record keys to be kept by a string separated by , (comma).'
     config_param :renew_record, :bool, :default => false,
-                 :desc => 'Creates an output record newly without extending (merging) the input record fields.'
+      :desc => 'Creates an output record newly without extending (merging) the input record fields.'
     config_param :renew_time_key, :string, :default => nil,
-                 :desc => 'Overwrites the time of events with a value of the record field.'
+      :desc => 'Overwrites the time of events with a value of the record field.'
     config_param :enable_ruby, :bool, :default => true, # true for lower version compatibility
-                 :desc => 'Enable to use ruby codes in placeholders.'
+      :desc => 'Enable to use ruby codes in placeholders.'
     config_param :auto_typecast, :bool, :default => false, # false for lower version compatibility
-                 :desc => 'Automatically cast the field types.'
+      :desc => 'Automatically cast the field types.'
 
     BUILTIN_CONFIGURATIONS = %W(@id @type @label type tag output_tag remove_keys renew_record keep_keys enable_ruby renew_time_key auto_typecast)
 
@@ -41,17 +41,17 @@ module Fluent
     def configure(conf)
       super
 
-      @map = {}
+      map = {}
       conf.each_pair { |k, v|
         next if BUILTIN_CONFIGURATIONS.include?(k)
         conf.has_key?(k) # to suppress unread configuration warning
-        @map[k] = parse_value(v)
+        map[k] = parse_value(v)
       }
       # <record></record> directive
       conf.elements.select { |element| element.name == 'record' }.each { |element|
         element.each_pair { |k, v|
           element.has_key?(k) # to suppress unread configuration warning
-          @map[k] = parse_value(v)
+          map[k] = parse_value(v)
         }
       }
 
@@ -86,6 +86,8 @@ module Fluent
         else
           PlaceholderExpander.new(placeholder_expander_params)
         end
+      @map = @placeholder_expander.preprocess_map(map)
+      @tag = @placeholder_expander.preprocess_map(@tag)
 
       @hostname = Socket.gethostname
     end
@@ -94,18 +96,22 @@ module Fluent
       tag_parts = tag.split('.')
       tag_prefix = tag_prefix(tag_parts)
       tag_suffix = tag_suffix(tag_parts)
-      placeholders = {
-        'tag' => tag,
-        'tags' => tag_parts,
-        'tag_parts' => tag_parts,
+      placeholder_values = {
+        'tag'        => tag,
+        'tags'       => tag_parts, # for old version compatibility
+        'tag_parts'  => tag_parts,
         'tag_prefix' => tag_prefix,
         'tag_suffix' => tag_suffix,
-        'hostname' => @hostname,
+        'hostname'   => @hostname,
       }
       last_record = nil
       es.each {|time, record|
         last_record = record # for debug log
-        new_tag, new_record = reform(@tag, time, record, placeholders)
+        placeholder_values.merge!({
+          'time'     => @placeholder_expander.time_value(time),
+          'record'   => record,
+        })
+        new_tag, new_record = reform(@tag, record, placeholder_values)
         if new_tag
           if @renew_time_key && new_record.has_key?(@renew_time_key)
             time = new_record[@renew_time_key].to_i
@@ -116,7 +122,7 @@ module Fluent
       chain.next
     rescue => e
       log.warn "record_reformer: #{e.class} #{e.message} #{e.backtrace.first}"
-      log.debug "record_reformer: tag:#{@tag} map:#{@map} record:#{last_record} placeholders:#{placeholders}"
+      log.debug "record_reformer: tag:#{@tag} map:#{@map} record:#{last_record} placeholder_values:#{placeholder_values}"
     end
 
     private
@@ -132,9 +138,10 @@ module Fluent
       value_str # emit as string
     end
 
-    def reform(tag, time, record, opts)
-      @placeholder_expander.prepare_placeholders(time, record, opts)
-      new_tag = @placeholder_expander.expand(tag)
+    def reform(tag, record, placeholder_values)
+      @placeholder_expander.prepare_placeholders(placeholder_values)
+
+      new_tag = expand_placeholders(tag)
 
       new_record = @renew_record ? {} : record.dup
       @keep_keys.each {|k| new_record[k] = record[k]} if @keep_keys and @renew_record
@@ -190,17 +197,29 @@ module Fluent
         @auto_typecast = params[:auto_typecast]
       end
 
-      def prepare_placeholders(time, record, opts)
-        placeholders = { '${time}' => Time.at(time).to_s }
-        record.each {|key, value| placeholders.store("${#{key}}", value) }
+      def time_value(time)
+        Time.at(time).to_s
+      end
+
+      def preprocess_map(value, force_stringify = false)
+        value
+      end
 
-        opts.each do |key, value|
+      def prepare_placeholders(placeholder_values)
+        placeholders = {}
+
+        placeholder_values.each do |key, value|
           if value.kind_of?(Array) # tag_parts, etc
             size = value.size
-            value.each_with_index { |v, idx|
+            value.each_with_index do |v, idx|
               placeholders.store("${#{key}[#{idx}]}", v)
               placeholders.store("${#{key}[#{idx-size}]}", v) # support [-1]
-            }
+            end
+          elsif value.kind_of?(Hash) # record, etc
+            value.each do |k, v|
+              placeholders.store("${#{k}}", v) # foo
+              placeholders.store(%Q[${#{key}["#{k}"]}], v) # record["foo"]
+            end
           else # string, interger, float, and others?
             placeholders.store("${#{key}}", value)
           end
@@ -209,22 +228,27 @@ module Fluent
         @placeholders = placeholders
       end
 
-      def expand(str, force_stringify=false)
+      # Expand string with placeholders
+      #
+      # @param [String] str
+      # @param [Boolean] force_stringify the value must be string, used for hash key
+      def expand(str, force_stringify = false)
         if @auto_typecast and !force_stringify
           single_placeholder_matched = str.match(/\A(\${[^}]+}|__[A-Z_]+__)\z/)
           if single_placeholder_matched
-            log_unknown_placeholder($1)
+            log_if_unknown_placeholder($1)
             return @placeholders[single_placeholder_matched[1]]
           end
         end
         str.gsub(/(\${[^}]+}|__[A-Z_]+__)/) {
-          log_unknown_placeholder($1)
+          log_if_unknown_placeholder($1)
           @placeholders[$1]
         }
       end
 
       private
-      def log_unknown_placeholder(placeholder)
+
+      def log_if_unknown_placeholder(placeholder)
         unless @placeholders.include?(placeholder)
           log.warn "record_reformer: unknown placeholder `#{placeholder}` found"
         end
@@ -232,45 +256,96 @@ module Fluent
     end
 
     class RubyPlaceholderExpander
-      attr_reader :placeholders, :log
+      attr_reader :log
 
       def initialize(params)
         @log = params[:log]
         @auto_typecast = params[:auto_typecast]
+        @cleanroom_expander = CleanroomExpander.new
       end
 
-      # Get placeholders as a struct
-      #
-      # @param [Time]   time        the time
-      # @param [Hash]   record      the record
-      # @param [Hash]   opts        others
-      def prepare_placeholders(time, record, opts)
-        struct = UndefOpenStruct.new(record)
-        struct.time = Time.at(time)
-        opts.each {|key, value| struct.__send__("#{key}=", value) }
-        @placeholders = struct
+      def time_value(time)
+        Time.at(time)
       end
 
-      # Replace placeholders in a string
+      # Preprocess record map to convert into ruby string expansion
       #
-      # @param [String] str         the string to be replaced
-      def expand(str, force_stringify=false)
-        if @auto_typecast and !force_stringify
-          single_placeholder_matched = str.match(/\A\${([^}]+)}\z/)
-          if single_placeholder_matched
-            code = single_placeholder_matched[1]
-            return eval code, @placeholders.instance_eval { binding }
+      # @param [Hash|String|Array] value record map config
+      # @param [Boolean] force_stringify the value must be string, used for hash key
+      def preprocess_map(value, force_stringify = false)
+        new_value = nil
+        if value.is_a?(String)
+          if @auto_typecast and !force_stringify
+            if single_placeholder_matched = value.match(/\A\${([^}]+)}\z/) # ${..} => ..
+              new_value = single_placeholder_matched[1]
+            end
+          end
+          unless new_value
+            new_value = %Q{%Q[#{value.gsub(/\$\{([^}]+)\}/, '#{\1}')}]} # xx${..}xx => %Q[xx#{..}xx]
+          end
+        elsif value.is_a?(Hash)
+          new_value = {}
+          value.each_pair do |k, v|
+            new_value[preprocess_map(k, true)] = preprocess_map(v)
           end
+        elsif value.is_a?(Array)
+          new_value = []
+          value.each_with_index do |v, i|
+            new_value[i] = preprocess_map(v)
+          end
+        else
+          new_value = value
         end
-        interpolated = str.gsub(/\$\{([^}]+)\}/, '#{\1}') # ${..} => #{..}
-        eval "\"#{interpolated}\"", @placeholders.instance_eval { binding }
+        new_value
+      end
+
+      def prepare_placeholders(placeholder_values)
+        @tag = placeholder_values['tag']
+        @time = placeholder_values['time']
+        @record = placeholder_values['record']
+        @tag_parts = placeholder_values['tag_parts']
+        @tag_prefix = placeholder_values['tag_prefix']
+        @tag_suffix = placeholder_values['tag_suffix']
+        @hostname = placeholder_values['hostname']
+      end
+
+      # Expand string with placeholders
+      #
+      # @param [String] str
+      def expand(str, force_stringify = false)
+        @cleanroom_expander.expand(
+          str,
+          @tag,
+          @time,
+          @record,
+          @tag_parts,
+          @tag_prefix,
+          @tag_suffix,
+          @hostname,
+        )
       rescue => e
         log.warn "record_reformer: failed to expand `#{str}`", :error_class => e.class, :error => e.message
         log.warn_backtrace
         nil
       end
 
-      class UndefOpenStruct < OpenStruct
+      class CleanroomExpander
+        def expand(__str_to_eval__, tag, time, record, tag_parts, tag_prefix, tag_suffix, hostname)
+          tags = tag_parts # for old version compatibility
+          @record = record # for old version compatibility
+          instance_eval(__str_to_eval__)
+        end
+
+        # for old version compatibility
+        def method_missing(name)
+          key = name.to_s
+          if @record.has_key?(key)
+            @record[key]
+          else
+            raise NameError, "undefined local variable or method `#{key}'"
+          end
+        end
+
         (Object.instance_methods).each do |m|
           undef_method m unless m.to_s =~ /^__|respond_to_missing\?|object_id|public_methods|instance_eval|method_missing|define_singleton_method|respond_to\?|new_ostruct_member/
         end

data/test/helper.rb

@@ -7,16 +7,17 @@ unless defined?(Test::Unit::AssertionFailedError)
   end
 end
 
-# Stop non required sleep at
-# https://github.com/fluent/fluentd/blob/018791f6b1b0400b71e37df2fb3ad80e456d2c11/lib/fluent/test/base.rb#L56
+# Reduce sleep period at
+# https://github.com/fluent/fluentd/blob/a271b3ec76ab7cf89ebe4012aa5b3912333dbdb7/lib/fluent/test/base.rb#L81
 module Fluent
   module Test
     class TestDriver
-      def run(&block)
+      def run(num_waits = 10, &block)
         @instance.start
         begin
           # wait until thread starts
-          # 10.times { sleep 0.05 }
+          # num_waits.times { sleep 0.05 }
+          sleep 0.05
           return yield
         ensure
           @instance.shutdown

data/test/test_out_record_reformer.rb

@@ -470,6 +470,28 @@ EOC
           end
           assert_equal(expected_results, actual_results)
         end
+
+        test %Q[record["key"] with enable_ruby #{enable_ruby}] do
+          config = %[
+            tag tag
+            enable_ruby #{enable_ruby}
+            auto_typecast true
+            <record>
+              _timestamp ${record["@timestamp"]}
+              _foo_bar   ${record["foo.bar"]}
+            </record>
+          ]
+          d = create_driver(config, use_v1)
+          record = {
+            "foo.bar"    => "foo.bar",
+            "@timestamp" => 10,
+          }
+          es = emit(config, use_v1, [record])
+          es.each_with_index do |(tag, time, r), i|
+            assert { r['_timestamp'] == record['@timestamp'] }
+            assert { r['_foo_bar'] == record['foo.bar'] }
+          end
+        end
       end
 
       test 'unknown placeholder (enable_ruby no)' do
@@ -495,7 +517,7 @@ EOC
           </record>
         ]
         d = create_driver(config, use_v1)
-        mock(d.instance.log).warn("record_reformer: failed to expand `${unknown['bar']}`", anything)
+        mock(d.instance.log).warn("record_reformer: failed to expand `%Q[\#{unknown['bar']}]`", anything)
         d.run { d.emit({}, @time) }
         # emit, but nil value
         assert_equal 1, d.emits.size
@@ -510,7 +532,7 @@ EOC
           enable_ruby yes
         ]
         d = create_driver(config, use_v1)
-        mock(d.instance.log).warn("record_reformer: failed to expand `${unknown['bar']}`", anything)
+        mock(d.instance.log).warn("record_reformer: failed to expand `%Q[\#{unknown['bar']}]`", anything)
         d.run { d.emit({}, @time) }
         # nil tag message should not be emitted
         assert_equal 0, d.emits.size
@@ -548,5 +570,38 @@ EOC
         end
       end
     end
+
+    test "compatibility test (enable_ruby yes) (use_v1 #{use_v1})" do
+      config = %[
+        tag tag
+        enable_ruby yes
+        auto_typecast yes
+        <record>
+          _message   prefix-${message}-suffix
+          _time      ${Time.at(time)}
+          _number    ${number == '-' ? 0 : number}
+          _match     ${/0x[0-9a-f]+/.match(hex)[0]}
+          _timestamp ${__send__("@timestamp")}
+          _foo_bar   ${__send__('foo.bar')}
+        </record>
+      ]
+      d = create_driver(config, use_v1)
+      record = {
+        "number"     => "-",
+        "hex"        => "0x10",
+        "foo.bar"    => "foo.bar",
+        "@timestamp" => 10,
+        "message"    => "10",
+      }
+      es = emit(config, use_v1, [record])
+      es.each_with_index do |(tag, time, r), i|
+        assert { r['_message'] == "prefix-#{record['message']}-suffix" }
+        assert { r['_time'] == Time.at(@time) }
+        assert { r['_number'] == 0 }
+        assert { r['_match'] == record['hex'] }
+        assert { r['_timestamp'] == record['@timestamp'] }
+        assert { r['_foo_bar'] == record['foo.bar'] }
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-record-reformer
 version: !ruby/object:Gem::Version
-  version: 0.7.2
+  version: 0.8.0
 platform: ruby
 authors:
 - Naotoshi Seo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-12-29 00:00:00.000000000 Z
+date: 2016-01-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -119,6 +119,7 @@ files:
 - ".travis.yml"
 - CHANGELOG.md
 - Gemfile
+- Gemfile.fluentd.0.10
 - LICENSE
 - README.md
 - Rakefile