fluentd 0.12.40 → 1.6.2

data/test/plugin/test_filter_record_transformer.rb

@@ -1,16 +1,19 @@
 require_relative '../helper'
 require 'timecop'
+require 'fluent/test/driver/filter'
 require 'fluent/plugin/filter_record_transformer'
+require 'flexmock/test_unit'
 
 class RecordTransformerFilterTest < Test::Unit::TestCase
   include Fluent
+  include FlexMock::TestCase
 
   setup do
     Test.setup
     @hostname = Socket.gethostname.chomp
     @tag = 'test.tag'
     @tag_parts = @tag.split('.')
-    @time = Time.utc(1,2,3,4,5,2010,nil,nil,nil,nil)
+    @time = event_time('2010-05-04 03:02:01 UTC')
     Timecop.freeze(@time)
   end
 
@@ -19,7 +22,7 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
   end
 
   def create_driver(conf = '')
-    Test::FilterTestDriver.new(RecordTransformerFilter, @tag).configure(conf, true)
+    Fluent::Test::Driver::Filter.new(Fluent::Plugin::RecordTransformerFilter).configure(conf)
   end
 
   sub_test_case 'configure' do
@@ -37,13 +40,14 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
   end
 
   sub_test_case "test options" do
-    def emit(config, msgs = [''])
+    def filter(config, msgs = [''])
       d = create_driver(config)
       d.run {
         msgs.each { |msg|
-          d.emit({'foo' => 'bar', 'message' => msg}, @time)
+          d.feed(@tag, @time, {'foo' => 'bar', 'message' => msg, 'nest' => {'k1' => 'v1', 'k2' => 'v2'}})
         }
-      }.filtered
+      }
+      d.filtered
     end
 
     CONFIG = %[
@@ -51,43 +55,52 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
         hostname ${hostname}
         tag ${tag}
         time ${time}
-        message ${hostname} ${tag_parts[-1]} ${message}
+        message ${hostname} ${tag_parts[-1]} ${record["message"]}
       </record>
     ]
 
     test 'typical usage' do
       msgs = ['1', '2']
-      es = emit(CONFIG, msgs)
-      es.each_with_index do |(t, r), i|
+      filtered = filter(CONFIG, msgs)
+      filtered.each_with_index do |(_t, r), i|
         assert_equal('bar', r['foo'])
         assert_equal(@hostname, r['hostname'])
         assert_equal(@tag, r['tag'])
-        assert_equal(@time.to_s, r['time'])
+        assert_equal(Time.at(@time).localtime.to_s, r['time'])
         assert_equal("#{@hostname} #{@tag_parts[-1]} #{msgs[i]}", r['message'])
+        assert_equal({'k1' => 'v1', 'k2' => 'v2'}, r['nest'])
       end
     end
 
     test 'remove_keys' do
       config = CONFIG + %[remove_keys foo,message]
-      es = emit(config)
-      es.each_with_index do |(t, r), i|
+      filtered = filter(config)
+      filtered.each_with_index do |(_t, r), i|
         assert_not_include(r, 'foo')
         assert_equal(@hostname, r['hostname'])
         assert_equal(@tag, r['tag'])
-        assert_equal(@time.to_s, r['time'])
+        assert_equal(Time.at(@time).localtime.to_s, r['time'])
         assert_not_include(r, 'message')
       end
     end
 
+    test 'remove_keys with nested key' do
+      config = CONFIG + %[remove_keys $.nest.k1]
+      filtered = filter(config)
+      filtered.each_with_index do |(_t, r), i|
+        assert_not_include(r['nest'], 'k1')
+      end
+    end
+
     test 'renew_record' do
       config = CONFIG + %[renew_record true]
       msgs = ['1', '2']
-      es = emit(config, msgs)
-      es.each_with_index do |(t, r), i|
+      filtered = filter(config, msgs)
+      filtered.each_with_index do |(_t, r), i|
         assert_not_include(r, 'foo')
         assert_equal(@hostname, r['hostname'])
         assert_equal(@tag, r['tag'])
-        assert_equal(@time.to_s, r['time'])
+        assert_equal(Time.at(@time).localtime.to_s, r['time'])
         assert_equal("#{@hostname} #{@tag_parts[-1]} #{msgs[i]}", r['message'])
       end
     end
@@ -95,11 +108,12 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
     test 'renew_time_key' do
       config = %[renew_time_key message]
       times = [ Time.local(2,2,3,4,5,2010,nil,nil,nil,nil), Time.local(3,2,3,4,5,2010,nil,nil,nil,nil) ]
-      msgs = times.map{|t| t.to_i.to_s }
-      es = emit(config, msgs)
-      es.each_with_index do |(time, record), i|
+      msgs = times.map{|t| t.to_f.to_s }
+      filtered = filter(config, msgs)
+      filtered.each_with_index do |(time, _record), i|
         assert_equal(times[i].to_i, time)
-        assert_true(record.has_key?('message'))
+        assert(time.is_a?(Fluent::EventTime))
+        assert_true(_record.has_key?('message'))
       end
     end
 
@@ -114,9 +128,10 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
                ]
       times = [Time.local(2, 2, 3, 4, 5, 2010, nil, nil, nil, nil), Time.local(3, 2, 3, 4, 5, 2010, nil, nil, nil, nil)]
       msgs = times.map { |t| t.to_f.to_s }
-      filtered = emit(config, msgs)
+      filtered = filter(config, msgs)
       filtered.each_with_index do |(time, _record), i|
         assert_equal(times[i].to_i, time)
+        assert(time.is_a?(Fluent::EventTime))
         assert_false(_record.has_key?('event_time_key'))
       end
     end
@@ -124,8 +139,8 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
     test 'keep_keys' do
       config = %[renew_record true\nkeep_keys foo,message]
       msgs = ['1', '2']
-      es = emit(config, msgs)
-      es.each_with_index do |(t, r), i|
+      filtered = filter(config, msgs)
+      filtered.each_with_index do |(_t, r), i|
         assert_equal('bar', r['foo'])
         assert_equal(msgs[i], r['message'])
       end
@@ -134,8 +149,8 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
     test 'keep_keys that are not present in the original record should not be included in the result record' do
       config = %[renew_record true\nkeep_keys foo, bar, baz, message]
       msgs = ['1', '2', nil]
-      es = emit(config, msgs)
-      es.each_with_index do |(t, r), i|
+      filtered = filter(config, msgs)
+      filtered.each_with_index do |(_t, r), i|
         assert_equal('bar', r['foo'])
         assert_equal(msgs[i], r['message'])
         assert_equal(false, r.has_key?('bar'))
@@ -147,13 +162,13 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
       config = %[
         enable_ruby yes
         <record>
-          message ${hostname} ${tag_parts.last} ${URI.encode(message)}
+          message ${hostname} ${tag_parts.last} ${"'" + record["message"] + "'"}
         </record>
       ]
       msgs = ['1', '2']
-      es = emit(config, msgs)
-      es.each_with_index do |(t, r), i|
-        assert_equal("#{@hostname} #{@tag_parts[-1]} #{msgs[i]}", r['message'])
+      filtered = filter(config, msgs)
+      filtered.each_with_index do |(_t, r), i|
+        assert_equal("#{@hostname} #{@tag_parts[-1]} '#{msgs[i]}'", r['message'])
       end
     end
 
@@ -164,8 +179,8 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
         </record>
       %]
       msgs = ['1', '2']
-      es = emit(config, msgs)
-      es.each_with_index do |(t, r), i|
+      filtered = filter(config, msgs)
+      filtered.each_with_index do |(_t, r), i|
         assert_equal({"k1"=>100, "k2"=>"foobar"}, r['hash_field'])
       end
     end
@@ -177,8 +192,8 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
         </record>
       %]
       msgs = ['1', '2']
-      es = emit(config, msgs)
-      es.each_with_index do |(t, r), i|
+      filtered = filter(config, msgs)
+      filtered.each_with_index do |(_t, r), i|
         assert_equal([1,2,3], r['array_field'])
       end
     end
@@ -190,36 +205,39 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
         </record>
       %]
       msgs = ['1', '2']
-      es = emit(config, msgs)
-      es.each_with_index do |(t, r), i|
+      filtered = filter(config, msgs)
+      filtered.each_with_index do |(_t, r), i|
         assert_equal({"hello"=>[1,2,3], "world"=>{"foo"=>"bar"}}, r['mixed_field'])
       end
     end
   end
 
   sub_test_case 'test placeholders' do
-    def emit(config, msgs = [''])
+    def filter(config, msgs = [''])
       d = create_driver(config)
       yield d if block_given?
       d.run {
-        msgs.each do |msg|
-          record = {'eventType0' => 'bar', 'message' => msg}
-          record = record.merge(msg) if msg.is_a?(Hash)
-          d.emit(record, @time)
+        records = msgs.map do |msg|
+          next msg if msg.is_a?(Hash)
+          { 'eventType0' => 'bar', 'message' => msg }
         end
-      }.filtered
+        records.each do |record|
+          d.feed(@tag, @time, record)
+        end
+      }
+      d.filtered
     end
 
     %w[yes no].each do |enable_ruby|
-      test "hostname with enble_ruby #{enable_ruby}" do
+      test "hostname with enable_ruby #{enable_ruby}" do
         config = %[
           enable_ruby #{enable_ruby}
           <record>
             message ${hostname}
           </record>
         ]
-        es = emit(config)
-        es.each do |t, r|
+        filtered = filter(config)
+        filtered.each do |t, r|
           assert_equal(@hostname, r['message'])
         end
       end
@@ -231,8 +249,8 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
             message ${tag}
           </record>
         ]
-        es = emit(config)
-        es.each do |t, r|
+        filtered = filter(config)
+        filtered.each do |t, r|
           assert_equal(@tag, r['message'])
         end
       end
@@ -245,8 +263,8 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
           </record>
         ]
         expected = "#{@tag.split('.').first} #{@tag.split('.').last}"
-        es = emit(config)
-        es.each do |t, r|
+        filtered = filter(config)
+        filtered.each do |t, r|
           assert_equal(expected, r['message'])
         end
       end
@@ -260,8 +278,8 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
         ]
         @tag = 'prefix.test.tag.suffix'
         expected = "prefix.test prefix.test.tag tag.suffix test.tag.suffix"
-        es = emit(config)
-        es.each do |t, r|
+        filtered = filter(config)
+        filtered.each do |t, r|
           assert_equal(expected, r['message'])
         end
       end
@@ -273,9 +291,13 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
             message ${time}
           </record>
         ]
-        es = emit(config)
-        es.each do |t, r|
-          assert_equal(@time.to_s, r['message'])
+        filtered = filter(config)
+        filtered.each do |t, r|
+          if enable_ruby == "yes"
+            assert_equal(Time.at(@time).localtime, r['message'])
+          else
+            assert_equal(Time.at(@time).localtime.to_s, r['message'])
+          end
         end
       end
 
@@ -284,13 +306,13 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
           enable_ruby #{enable_ruby}
           remove_keys eventType0
           <record>
-            message bar ${message}
-            eventtype ${eventType0}
+            message bar ${record["message"]}
+            eventtype ${record["eventType0"]}
           </record>
         ]
         msgs = ['1', '2']
-        es = emit(config, msgs)
-        es.each_with_index do |(t, r), i|
+        filtered = filter(config, msgs)
+        filtered.each_with_index do |(_t, r), i|
           assert_not_include(r, 'eventType0')
           assert_equal("bar", r['eventtype'])
           assert_equal("bar #{msgs[i]}", r['message'])
@@ -306,7 +328,7 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
           </record>
         ]
         records = [{'tag' => 'tag', 'time' => 'time'}]
-        filtered = emit(config, records)
+        filtered = filter(config, records)
         filtered.each_with_index do |(_t, r), i|
           assert_not_equal('tag', r['new_tag'])
           assert_equal(@tag, r['new_tag'])
@@ -326,8 +348,8 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
           </record>
         ]
         msgs = ['1', '2']
-        es = emit(config, msgs)
-        es.each_with_index do |(t, r), i|
+        filtered = filter(config, msgs)
+        filtered.each_with_index do |(_t, r), i|
           assert_equal({"hostname" => @hostname, "tag" => @tag, "#{@tag}" => 100}, r['hash_field'])
         end
       end
@@ -340,8 +362,8 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
           </record>
         ]
         msgs = ['1', '2']
-        es = emit(config, msgs)
-        es.each_with_index do |(t, r), i|
+        filtered = filter(config, msgs)
+        filtered.each_with_index do |(_t, r), i|
           assert_equal([@hostname, @tag], r['array_field'])
         end
       end
@@ -354,8 +376,8 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
           </record>
         ]
         msgs = ['1', '2']
-        es = emit(config, msgs)
-        es.each_with_index do |(t, r), i|
+        filtered = filter(config, msgs)
+        filtered.each_with_index do |(_t, r), i|
           assert_equal([{"tag" => @tag}], r['mixed_field'])
         end
       end
@@ -370,8 +392,8 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
           </record>
         ]
         msgs = ['1', '2']
-        es = emit(config, msgs)
-        es.each_with_index do |(t, r), i|
+        filtered = filter(config, msgs)
+        filtered.each_with_index do |(_t, r), i|
           assert_equal({@hostname=>'hostname',"foo.#{@tag}"=>'tag'}, r)
         end
       end
@@ -381,11 +403,11 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
           auto_typecast false
           enable_ruby #{enable_ruby}
           <record>
-            single      ${source}
-            multiple    ${source}${source}
-            with_prefix prefix-${source}
-            with_suffix ${source}-suffix
-            with_quote  source[""]
+            single      ${record["source"]}
+            multiple    ${record["source"]}${record["source"]}
+            with_prefix prefix-${record["source"]}
+            with_suffix ${record["source"]}-suffix
+            with_quote  record["source"][""]
           </record>
         ]
         msgs = [
@@ -400,31 +422,31 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
             multiple: "stringstring",
             with_prefix: "prefix-string",
             with_suffix: "string-suffix",
-            with_quote: %Q{source[""]} },
+            with_quote: %Q{record["source"][""]} },
           { single: 123.to_s,
             multiple: "#{123.to_s}#{123.to_s}",
             with_prefix: "prefix-#{123.to_s}",
             with_suffix: "#{123.to_s}-suffix",
-            with_quote: %Q{source[""]} },
+            with_quote: %Q{record["source"][""]} },
           { single: [1, 2].to_s,
             multiple: "#{[1, 2].to_s}#{[1, 2].to_s}",
             with_prefix: "prefix-#{[1, 2].to_s}",
             with_suffix: "#{[1, 2].to_s}-suffix",
-            with_quote: %Q{source[""]} },
+            with_quote: %Q{record["source"][""]} },
           { single: {a:1, b:2}.to_s,
             multiple: "#{{a:1, b:2}.to_s}#{{a:1, b:2}.to_s}",
             with_prefix: "prefix-#{{a:1, b:2}.to_s}",
             with_suffix: "#{{a:1, b:2}.to_s}-suffix",
-            with_quote: %Q{source[""]} },
+            with_quote: %Q{record["source"][""]} },
           { single: nil.to_s,
             multiple: "#{nil.to_s}#{nil.to_s}",
             with_prefix: "prefix-#{nil.to_s}",
             with_suffix: "#{nil.to_s}-suffix",
-            with_quote: %Q{source[""]} },
+            with_quote: %Q{record["source"][""]} },
         ]
         actual_results = []
-        es = emit(config, msgs)
-        es.each_with_index do |(t, r), i|
+        filtered = filter(config, msgs)
+        filtered.each_with_index do |(_t, r), i|
           actual_results << {
             single: r["single"],
             multiple: r["multiple"],
@@ -441,10 +463,10 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
           auto_typecast yes
           enable_ruby #{enable_ruby}
           <record>
-            single      ${source}
-            multiple    ${source}${source}
-            with_prefix prefix-${source}
-            with_suffix ${source}-suffix
+            single      ${record["source"]}
+            multiple    ${record["source"]}${record["source"]}
+            with_prefix prefix-${record["source"]}
+            with_suffix ${record["source"]}-suffix
           </record>
         ]
         msgs = [
@@ -477,8 +499,8 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
             with_suffix: "#{nil.to_s}-suffix" },
         ]
         actual_results = []
-        es = emit(config, msgs)
-        es.each_with_index do |(t, r), i|
+        filtered = filter(config, msgs)
+        filtered.each_with_index do |(_t, r), i|
           actual_results << {
             single: r["single"],
             multiple: r["multiple"],
@@ -488,27 +510,6 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
         end
         assert_equal(expected_results, actual_results)
       end
-
-      test %Q[record["key"] with enable_ruby #{enable_ruby}] do
-        config = %[
-          enable_ruby #{enable_ruby}
-          auto_typecast yes
-          <record>
-            _timestamp ${record["@timestamp"]}
-            _foo_bar   ${record["foo.bar"]}
-          </record>
-        ]
-        d = create_driver(config)
-        record = {
-          "foo.bar"    => "foo.bar",
-          "@timestamp" => 10,
-        }
-        es = d.run { d.emit(record, @time) }.filtered
-        es.each do |t, r|
-          assert { r['_timestamp'] == record['@timestamp'] }
-          assert { r['_foo_bar'] == record['foo.bar'] }
-        end
-      end
     end
 
     test 'unknown placeholder (enable_ruby no)' do
@@ -518,37 +519,23 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
           message ${unknown}
         </record>
       ]
-      emit(config) { |d|
+      filter(config) { |d|
         mock(d.instance.log).warn("unknown placeholder `${unknown}` found")
       }
     end
 
-    test 'failed to expand (enable_ruby yes)' do
-      config = %[
-        enable_ruby yes
-        <record>
-          message ${unknown['bar']}
-        </record>
-      ]
-      es = emit(config) { |d|
-        mock(d.instance.log).warn("failed to expand `%Q[\#{unknown['bar']}]`", anything)
-      }
-      es.each do |t, r|
-        assert_nil(r['message'])
-      end
-    end
-
     test 'expand fields starting with @ (enable_ruby no)' do
       config = %[
         enable_ruby no
         <record>
-          foo ${@timestamp}
+          foo ${record["@timestamp"]}
         </record>
       ]
       d = create_driver(config)
       message = {"@timestamp" => "foo"}
-      es = d.run { d.emit(message, @time) }.filtered
-      es.each do |t, r|
+      d.run { d.feed(@tag, @time, message) }
+      filtered = d.filtered
+      filtered.each do |t, r|
         assert_equal(message["@timestamp"], r['foo'])
       end
     end
@@ -564,57 +551,27 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
       ]
       d = create_driver(config)
       message = {"@timestamp" => "foo"}
-      es = d.run { d.emit(message, @time) }.filtered
-      es.each do |t, r|
+      d.run { d.feed(@tag, @time, message) }
+      filtered = d.filtered
+      filtered.each do |t, r|
         assert_equal([message["@timestamp"]], r['foo'])
       end
     end
+  end # test placeholders
 
-    test 'expand fields starting with @ (enable_ruby yes)' do
-      config = %[
+  sub_test_case 'test error record' do
+    test 'invalid record for placeholders' do
+      d = create_driver(%[
         enable_ruby yes
         <record>
-          foo ${__send__("@timestamp")}
+          foo ${record["unknown"]["key"]}
         </record>
-      ]
-      d = create_driver(config)
-      message = {"@timestamp" => "foo"}
-      es = d.run { d.emit(message, @time) }.filtered
-      es.each do |t, r|
-        assert_equal(message["@timestamp"], r['foo'])
+      ])
+      flexmock(d.instance.router).should_receive(:emit_error_event).
+        with(String, Fluent::EventTime, Hash, RuntimeError).once
+      d.run do
+        d.feed(@tag, Fluent::EventTime.now, {'key' => 'value'})
       end
     end
-  end # test placeholders
-
-  test "compatibility test (enable_ruby yes)" do
-    config = %[
-      enable_ruby yes
-      auto_typecast yes
-      <record>
-        _message   prefix-${message}-suffix
-        _time      ${Time.at(time)}
-        _number    ${number == '-' ? 0 : number}
-        _match     ${/0x[0-9a-f]+/.match(hex)[0]}
-        _timestamp ${__send__("@timestamp")}
-        _foo_bar   ${__send__('foo.bar')}
-      </record>
-    ]
-    d = create_driver(config)
-    record = {
-      "number"     => "-",
-      "hex"        => "0x10",
-      "foo.bar"    => "foo.bar",
-      "@timestamp" => 10,
-      "message"    => "10",
-    }
-    es = d.run { d.emit(record, @time) }.filtered
-    es.each do |t, r|
-      assert { r['_message'] == "prefix-#{record['message']}-suffix" }
-      assert { r['_time'] == Time.at(@time) }
-      assert { r['_number'] == 0 }
-      assert { r['_match'] == record['hex'] }
-      assert { r['_timestamp'] == record['@timestamp'] }
-      assert { r['_foo_bar'] == record['foo.bar'] }
-    end
   end
 end