fluent-plugin-record-reformer 0.8.0 → 0.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bf0c0a50d2b8509f3fa6917e0f700abe30e5249a
-  data.tar.gz: fba94b0ab7c6259354c70be688ef5ad81cc29efe
+  metadata.gz: 5c12a8d7d13aa7b7cd2774128c13b1ce7ed4473a
+  data.tar.gz: 7bf4dd5a13506bd3d9e89acf739d27322bb737e4
 SHA512:
-  metadata.gz: 132876d93292dd82888bbf7a34010dbc3c9e99ea746952165c919a71be3ad8118b3572009f5a8c5deb18654b714142a9b100b39c07ff2e3e40ae9d1e1c014ca9
-  data.tar.gz: 98e35238beb1848b8076ab6c66095a9cb15eb91301bd21f8eed18ad4f70e2eae2b932ad6c525c516f57794d0fee651a162d25a2b519c9fb19bceda90947aaad7
+  metadata.gz: 54d23d63dca07dad339103db47a0b7374ef6aef190f4d8e64b984a625ac40b1e158a46f2c9b6aa8c735f1701a19fe0fc16418ba39d4e06052307756d1e0f96b3
+  data.tar.gz: fda86acdd87b031272f229b8162a2b2b77caa07788e0db8076fb9e84f8cb74ab865aa6a15be579928aba42e38c904b7ae5eeddabb377596a50dbcf96b9a31e40

data/CHANGELOG.md

@@ -1,3 +1,13 @@
+## 0.8.1 (2016/03/09)
+
+Fixes
+
+* Fix to be thread-safe
+
+Changes
+
+* Relax conditions which auto_typecast is applied for enable_ruby yes
+
 ## 0.8.0 (2016/01/28)
 
 Enhancements

data/fluent-plugin-record-reformer.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |gem|
   gem.name        = "fluent-plugin-record-reformer"
-  gem.version     = "0.8.0"
+  gem.version     = "0.8.1"
   gem.authors     = ["Naotoshi Seo"]
   gem.email       = "sonots@gmail.com"
   gem.homepage    = "https://github.com/sonots/fluent-plugin-record-reformer"

data/lib/fluent/plugin/out_record_reformer.rb

@@ -139,30 +139,31 @@ module Fluent
     end
 
     def reform(tag, record, placeholder_values)
-      @placeholder_expander.prepare_placeholders(placeholder_values)
+      placeholders = @placeholder_expander.prepare_placeholders(placeholder_values)
 
-      new_tag = expand_placeholders(tag)
+      new_tag = expand_placeholders(tag, placeholders)
 
       new_record = @renew_record ? {} : record.dup
       @keep_keys.each {|k| new_record[k] = record[k]} if @keep_keys and @renew_record
-      new_record.merge!(expand_placeholders(@map))
+      new_record.merge!(expand_placeholders(@map, placeholders))
       @remove_keys.each {|k| new_record.delete(k) } if @remove_keys
 
       [new_tag, new_record]
     end
 
-    def expand_placeholders(value)
+    def expand_placeholders(value, placeholders)
       if value.is_a?(String)
-        new_value = @placeholder_expander.expand(value)
+        new_value = @placeholder_expander.expand(value, placeholders)
       elsif value.is_a?(Hash)
         new_value = {}
         value.each_pair do |k, v|
-          new_value[@placeholder_expander.expand(k, true)] = expand_placeholders(v)
+          new_key = @placeholder_expander.expand(k, placeholders, true)
+          new_value[new_key] = expand_placeholders(v, placeholders)
         end
       elsif value.is_a?(Array)
         new_value = []
         value.each_with_index do |v, i|
-          new_value[i] = expand_placeholders(v)
+          new_value[i] = expand_placeholders(v, placeholders)
         end
       else
         new_value = value
@@ -189,6 +190,7 @@ module Fluent
       rev_tag_suffix.reverse!
     end
 
+    # THIS CLASS MUST BE THREAD-SAFE
     class PlaceholderExpander
       attr_reader :placeholders, :log
 
@@ -225,36 +227,37 @@ module Fluent
           end
         end
 
-        @placeholders = placeholders
+        placeholders
       end
 
       # Expand string with placeholders
       #
       # @param [String] str
       # @param [Boolean] force_stringify the value must be string, used for hash key
-      def expand(str, force_stringify = false)
+      def expand(str, placeholders, force_stringify = false)
         if @auto_typecast and !force_stringify
           single_placeholder_matched = str.match(/\A(\${[^}]+}|__[A-Z_]+__)\z/)
           if single_placeholder_matched
-            log_if_unknown_placeholder($1)
-            return @placeholders[single_placeholder_matched[1]]
+            log_if_unknown_placeholder($1, placeholders)
+            return placeholders[single_placeholder_matched[1]]
           end
         end
         str.gsub(/(\${[^}]+}|__[A-Z_]+__)/) {
-          log_if_unknown_placeholder($1)
-          @placeholders[$1]
+          log_if_unknown_placeholder($1, placeholders)
+          placeholders[$1]
         }
       end
 
       private
 
-      def log_if_unknown_placeholder(placeholder)
-        unless @placeholders.include?(placeholder)
+      def log_if_unknown_placeholder(placeholder, placeholders)
+        unless placeholders.include?(placeholder)
           log.warn "record_reformer: unknown placeholder `#{placeholder}` found"
         end
       end
     end
 
+    # THIS CLASS MUST BE THREAD-SAFE
     class RubyPlaceholderExpander
       attr_reader :log
 
@@ -276,12 +279,13 @@ module Fluent
         new_value = nil
         if value.is_a?(String)
           if @auto_typecast and !force_stringify
-            if single_placeholder_matched = value.match(/\A\${([^}]+)}\z/) # ${..} => ..
-              new_value = single_placeholder_matched[1]
+            num_placeholders = value.scan('${').size
+            if num_placeholders == 1 and value.start_with?('${') && value.end_with?('}')
+              new_value = value[2..-2] # ${..} => ..
             end
           end
           unless new_value
-            new_value = %Q{%Q[#{value.gsub(/\$\{([^}]+)\}/, '#{\1}')}]} # xx${..}xx => %Q[xx#{..}xx]
+            new_value = "%Q[#{value.gsub('${', '#{')}]" # xx${..}xx => %Q[xx#{..}xx]
           end
         elsif value.is_a?(Hash)
           new_value = {}
@@ -300,28 +304,22 @@ module Fluent
       end
 
       def prepare_placeholders(placeholder_values)
-        @tag = placeholder_values['tag']
-        @time = placeholder_values['time']
-        @record = placeholder_values['record']
-        @tag_parts = placeholder_values['tag_parts']
-        @tag_prefix = placeholder_values['tag_prefix']
-        @tag_suffix = placeholder_values['tag_suffix']
-        @hostname = placeholder_values['hostname']
+        placeholder_values
       end
 
       # Expand string with placeholders
       #
       # @param [String] str
-      def expand(str, force_stringify = false)
+      def expand(str, placeholders, force_stringify = false)
         @cleanroom_expander.expand(
           str,
-          @tag,
-          @time,
-          @record,
-          @tag_parts,
-          @tag_prefix,
-          @tag_suffix,
-          @hostname,
+          placeholders['tag'],
+          placeholders['time'],
+          placeholders['record'],
+          placeholders['tag_parts'],
+          placeholders['tag_prefix'],
+          placeholders['tag_suffix'],
+          placeholders['hostname'],
         )
       rescue => e
         log.warn "record_reformer: failed to expand `#{str}`", :error_class => e.class, :error => e.message
@@ -332,15 +330,16 @@ module Fluent
       class CleanroomExpander
         def expand(__str_to_eval__, tag, time, record, tag_parts, tag_prefix, tag_suffix, hostname)
           tags = tag_parts # for old version compatibility
-          @record = record # for old version compatibility
+          Thread.current[:record_reformer_record] = record # for old version compatibility
           instance_eval(__str_to_eval__)
         end
 
         # for old version compatibility
         def method_missing(name)
           key = name.to_s
-          if @record.has_key?(key)
-            @record[key]
+          record = Thread.current[:record_reformer_record]
+          if record.has_key?(key)
+            record[key]
           else
             raise NameError, "undefined local variable or method `#{key}'"
           end

data/test/test_out_record_reformer.rb

@@ -554,6 +554,24 @@ EOC
         end
       end
 
+      # https://github.com/sonots/fluent-plugin-record-reformer/issues/35
+      test 'auto_typecast placeholder containing {} (enable_ruby yes)' do
+        config = %[
+          tag tag
+          enable_ruby yes
+          auto_typecast yes
+          <record>
+            foo ${record.map{|k,v|v}}
+          </record>
+        ]
+        d = create_driver(config, use_v1)
+        message = {"@timestamp" => "foo"}
+        d.run { d.emit(message, @time) }
+        d.emits.each do |(tag, time, record)|
+          assert_equal [message["@timestamp"]], record["foo"]
+        end
+      end
+
       test 'expand fields starting with @ (enable_ruby yes)' do
         config = %[
           tag tag

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-record-reformer
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.8.1
 platform: ruby
 authors:
 - Naotoshi Seo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-27 00:00:00.000000000 Z
+date: 2016-03-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd