fluent-plugin-record-reformer 0.6.3 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: beaea68d997ac0ad945eed7538383648b364365a
-  data.tar.gz: a09b4a368bda8e78d1d9b302710eb2d64ab0982b
+  metadata.gz: db209a964b20147c182f8bb1276a2d867c470859
+  data.tar.gz: 78013a0f27ac7e155be2465afaa4d71b8f02a679
 SHA512:
-  metadata.gz: 2860304c62904d3805abd0bbae0ae41015c22c046a37219c4834c3d08d986c59c3c53c51c3e1998cbbdb9f169e0e0bcab38a1a567d4dd848c2887b36f3186b69
-  data.tar.gz: b629578c947c7b663277eafdfecc36bcd034c7d64b8d3c3b44f9940e23907a8bcfe9ab3943e08707aa51a141310f7304346b92a8598ac2e8344c269f0a16e10e
+  metadata.gz: 11dd3f611b7534410ca27776b703268d2ffe84db4f4b576d228629e268c2b0d168fbd7e0a96d638dd07a6ef70cbfe153dbfd98912c76d01694b5fe550e28084b
+  data.tar.gz: 399ebe1048befb5e396f86d55858e2eec000035dddc327218115bfc262e7fbf31c341d48449f8eedb30775a71ff453f9966cd9265fde9f10dbb0b88538b5c95a

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.7.0 (2015/06/19)
+
+Enhancements
+
+* Add `auto_typecast` option (thanks to @piroor)
+
 ## 0.6.3 (2015/05/27)
 
 Fixes:

data/README.md

@@ -92,7 +92,7 @@ This results in same, but please note that following option parameters are reser
 
 - renew\_time\_key *string*
 
-    `renew_time_key foo` overwrite time of events with value of field `foo` if exist. The value of `foo` must be unix time.
+    `renew_time_key foo` overwrites the time of events with a value of the record field `foo` if exists. The value of `foo` must be a unix time.
 
 - keep_keys
 
@@ -105,6 +105,25 @@ This results in same, but please note that following option parameters are reser
     Enable to use ruby codes in placeholders. See `Placeholders` section.
     Default is `true` (just for lower version compatibility). 
 
+- auto_typecast *bool*
+
+    Automatically cast the field types. Default is false.
+    NOTE: This option is effective only for field values comprised of a single placeholder. 
+
+    Effective Examples:
+    
+        foo ${foo}
+    
+    Non-Effective Examples:
+    
+        foo ${foo}${bar}
+        foo ${foo}bar
+        foo 1
+    
+    Internally, this **keeps** the type of value if the value text is comprised of a single placeholder, otherwise, values are treated as strings. 
+    
+    When you need to cast field types manually, [out_typecast](https://github.com/tarom/fluent-plugin-typecast) and [filter_typecast](https://github.com/sonots/fluent-plugin-filter_typecast) are available. 
+
 ## Placeholders
 
 The keys of input json are available as placeholders. In the above example, 

data/example.conf

@@ -0,0 +1,20 @@
+<source>
+  type dummy
+  tag dummy
+  dummy {"message":"foo","time":1432732710,"members":["Alice"]}
+</source>
+
+<match dummy>
+  type record_reformer
+  renew_time_key time
+  tag reformed.${tag}
+  enable_ruby true
+  auto_typecast true
+  <record>
+    members ${members + ["Bob"]}
+  </record>
+</match>
+
+<match reformed.**>
+  type stdout
+</match>

data/fluent-plugin-record-reformer.gemspec

@@ -3,7 +3,7 @@ $:.push File.expand_path('../lib', __FILE__)
 
 Gem::Specification.new do |gem|
   gem.name        = "fluent-plugin-record-reformer"
-  gem.version     = "0.6.3"
+  gem.version     = "0.7.0"
   gem.authors     = ["Naotoshi Seo"]
   gem.email       = "sonots@gmail.com"
   gem.homepage    = "https://github.com/sonots/fluent-plugin-record-reformer"

data/lib/fluent/plugin/out_record_reformer.rb

@@ -16,8 +16,9 @@ module Fluent
     config_param :renew_record, :bool, :default => false
     config_param :renew_time_key, :string, :default => nil
     config_param :enable_ruby, :bool, :default => true # true for lower version compatibility
+    config_param :auto_typecast, :bool, :default => false # false for lower version compatibility
 
-    BUILTIN_CONFIGURATIONS = %W(type tag output_tag remove_keys renew_record keep_keys enable_ruby renew_time_key)
+    BUILTIN_CONFIGURATIONS = %W(type tag output_tag remove_keys renew_record keep_keys enable_ruby renew_time_key auto_typecast)
 
     # To support log_level option implemented by Fluentd v0.10.43
     unless method_defined?(:log)
@@ -63,15 +64,19 @@ module Fluent
         raise Fluent::ConfigError, "out_record_reformer: `tag` must be specified"
       end
 
+      placeholder_expander_params = {
+        :log           => log,
+        :auto_typecast => @auto_typecast,
+      }
       @placeholder_expander =
         if @enable_ruby
           # require utilities which would be used in ruby placeholders
           require 'pathname'
           require 'uri'
           require 'cgi'
-          RubyPlaceholderExpander.new(log)
+          RubyPlaceholderExpander.new(placeholder_expander_params)
         else
-          PlaceholderExpander.new(log)
+          PlaceholderExpander.new(placeholder_expander_params)
         end
 
       @hostname = Socket.gethostname
@@ -137,7 +142,7 @@ module Fluent
       elsif value.is_a?(Hash)
         new_value = {}
         value.each_pair do |k, v|
-          new_value[@placeholder_expander.expand(k)] = expand_placeholders(v)
+          new_value[@placeholder_expander.expand(k, true)] = expand_placeholders(v)
         end
       elsif value.is_a?(Array)
         new_value = []
@@ -172,8 +177,9 @@ module Fluent
     class PlaceholderExpander
       attr_reader :placeholders, :log
 
-      def initialize(log)
-        @log = log
+      def initialize(params)
+        @log = params[:log]
+        @auto_typecast = params[:auto_typecast]
       end
 
       def prepare_placeholders(time, record, opts)
@@ -195,19 +201,34 @@ module Fluent
         @placeholders = placeholders
       end
 
-      def expand(str)
+      def expand(str, force_stringify=false)
+        if @auto_typecast and !force_stringify
+          single_placeholder_matched = str.match(/\A(\${[^}]+}|__[A-Z_]+__)\z/)
+          if single_placeholder_matched
+            log_unknown_placeholder($1)
+            return @placeholders[single_placeholder_matched[1]]
+          end
+        end
         str.gsub(/(\${[^}]+}|__[A-Z_]+__)/) {
-          log.warn "record_reformer: unknown placeholder `#{$1}` found" unless @placeholders.include?($1)
+          log_unknown_placeholder($1)
           @placeholders[$1]
         }
       end
+
+      private
+      def log_unknown_placeholder(placeholder)
+        unless @placeholders.include?(placeholder)
+          log.warn "record_reformer: unknown placeholder `#{placeholder}` found"
+        end
+      end
     end
 
     class RubyPlaceholderExpander
       attr_reader :placeholders, :log
 
-      def initialize(log)
-        @log = log
+      def initialize(params)
+        @log = params[:log]
+        @auto_typecast = params[:auto_typecast]
       end
 
       # Get placeholders as a struct
@@ -225,7 +246,14 @@ module Fluent
       # Replace placeholders in a string
       #
       # @param [String] str         the string to be replaced
-      def expand(str)
+      def expand(str, force_stringify=false)
+        if @auto_typecast and !force_stringify
+          single_placeholder_matched = str.match(/\A\${([^}]+)}\z/)
+          if single_placeholder_matched
+            code = single_placeholder_matched[1]
+            return eval code, @placeholders.instance_eval { binding }
+          end
+        end
         interpolated = str.gsub(/\$\{([^}]+)\}/, '#{\1}') # ${..} => #{..}
         eval "\"#{interpolated}\"", @placeholders.instance_eval { binding }
       rescue => e

data/test/test_out_record_reformer.rb

@@ -27,7 +27,12 @@ class RecordReformerOutputTest < Test::Unit::TestCase
     d = create_driver(config, use_v1)
     d.run do
       msgs.each do |msg|
-        d.emit({'eventType0' => 'bar', 'message' => msg}, @time)
+        record = {
+          'eventType0' => 'bar',
+          'message'    => msg,
+        }
+        record = record.merge(msg) if msg.is_a?(Hash)
+        d.emit(record, @time)
       end
     end
 
@@ -357,6 +362,114 @@ EOC
             end
           end
         end
+
+        test "disabled autodetectction of value type with enable_ruby #{enable_ruby}" do
+          config = %[
+            tag tag
+            enable_ruby #{enable_ruby}
+            auto_typecast false
+            <record>
+              single      ${source}
+              multiple    ${source}${source}
+              with_prefix prefix-${source}
+              with_suffix ${source}-suffix
+            </record>
+          ]
+          msgs = [
+            { "source" => "string" },
+            { "source" => 123 },
+            { "source" => [1, 2] },
+            { "source" => {a:1, b:2} },
+            { "source" => nil },
+          ]
+          expected_results = [
+            { :single      => "string",
+              :multiple    => "stringstring",
+              :with_prefix => "prefix-string",
+              :with_suffix => "string-suffix" },
+            { :single      => 123.to_s,
+              :multiple    => "#{123.to_s}#{123.to_s}",
+              :with_prefix => "prefix-#{123.to_s}",
+              :with_suffix => "#{123.to_s}-suffix" },
+            { :single      => [1, 2].to_s,
+              :multiple    => "#{[1, 2].to_s}#{[1, 2].to_s}",
+              :with_prefix => "prefix-#{[1, 2].to_s}",
+              :with_suffix => "#{[1, 2].to_s}-suffix" },
+            { :single      => {a:1, b:2}.to_s,
+              :multiple    => "#{{a:1, b:2}.to_s}#{{a:1, b:2}.to_s}",
+              :with_prefix => "prefix-#{{a:1, b:2}.to_s}",
+              :with_suffix => "#{{a:1, b:2}.to_s}-suffix" },
+            { :single      => nil.to_s,
+              :multiple    => "#{nil.to_s}#{nil.to_s}",
+              :with_prefix => "prefix-#{nil.to_s}",
+              :with_suffix => "#{nil.to_s}-suffix" },
+          ]
+          actual_results = []
+          es = emit(config, use_v1, msgs)
+          es.each_with_index do |(tag, time, record), i|
+            actual_results << {
+              :single      => record["single"],
+              :multiple    => record["multiple"],
+              :with_prefix => record["with_prefix"],
+              :with_suffix => record["with_suffix"],
+            }
+          end
+          assert_equal(expected_results, actual_results)
+        end
+
+        test "enabled autodetectction of value type with enable_ruby #{enable_ruby}" do
+          config = %[
+            tag tag
+            enable_ruby #{enable_ruby}
+            auto_typecast true
+            <record>
+              single      ${source}
+              multiple    ${source}${source}
+              with_prefix prefix-${source}
+              with_suffix ${source}-suffix
+            </record>
+          ]
+          msgs = [
+            { "source" => "string" },
+            { "source" => 123 },
+            { "source" => [1, 2] },
+            { "source" => {a:1, b:2} },
+            { "source" => nil },
+          ]
+          expected_results = [
+            { :single      => "string",
+              :multiple    => "stringstring",
+              :with_prefix => "prefix-string",
+              :with_suffix => "string-suffix" },
+            { :single      => 123,
+              :multiple    => "#{123.to_s}#{123.to_s}",
+              :with_prefix => "prefix-#{123.to_s}",
+              :with_suffix => "#{123.to_s}-suffix" },
+            { :single      => [1, 2],
+              :multiple    => "#{[1, 2].to_s}#{[1, 2].to_s}",
+              :with_prefix => "prefix-#{[1, 2].to_s}",
+              :with_suffix => "#{[1, 2].to_s}-suffix" },
+            { :single      => {a:1, b:2},
+              :multiple    => "#{{a:1, b:2}.to_s}#{{a:1, b:2}.to_s}",
+              :with_prefix => "prefix-#{{a:1, b:2}.to_s}",
+              :with_suffix => "#{{a:1, b:2}.to_s}-suffix" },
+            { :single      => nil,
+              :multiple    => "#{nil.to_s}#{nil.to_s}",
+              :with_prefix => "prefix-#{nil.to_s}",
+              :with_suffix => "#{nil.to_s}-suffix" },
+          ]
+          actual_results = []
+          es = emit(config, use_v1, msgs)
+          es.each_with_index do |(tag, time, record), i|
+            actual_results << {
+              :single      => record["single"],
+              :multiple    => record["multiple"],
+              :with_prefix => record["with_prefix"],
+              :with_suffix => record["with_suffix"],
+            }
+          end
+          assert_equal(expected_results, actual_results)
+        end
       end
 
       test 'unknown placeholder (enable_ruby no)' do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-record-reformer
 version: !ruby/object:Gem::Version
-  version: 0.6.3
+  version: 0.7.0
 platform: ruby
 authors:
 - Naotoshi Seo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-27 00:00:00.000000000 Z
+date: 2015-06-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -122,6 +122,7 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- example.conf
 - fluent-plugin-record-reformer.gemspec
 - lib/fluent/plugin/out_record_reformer.rb
 - test/bench_out_record_reformer.rb