fluentd 0.12.40 → 0.14.0

data/test/plugin/test_filter_grep.rb

@@ -22,51 +22,12 @@ class GrepFilterTest < Test::Unit::TestCase
 
     test "regexpN can contain a space" do
       d = create_driver(%[regexp1 message  foo])
-      assert_equal(Regexp.compile(/ foo/), d.instance._regexps['message'])
+      assert_equal(Regexp.compile(/ foo/), d.instance.regexps['message'])
     end
 
     test "excludeN can contain a space" do
       d = create_driver(%[exclude1 message  foo])
-      assert_equal(Regexp.compile(/ foo/), d.instance._excludes['message'])
-    end
-
-    sub_test_case "duplicate key" do
-      test "flat" do
-        conf = %[
-          regexp1 message test
-          regexp2 message test2
-        ]
-        assert_raise(Fluent::ConfigError) do
-          create_driver(conf)
-        end
-      end
-      test "section" do
-        conf = %[
-          <regexp>
-            key message
-            pattern test
-          </regexp>
-          <regexp>
-            key message
-            pattern test2
-          </regexp>
-        ]
-        assert_raise(Fluent::ConfigError) do
-          create_driver(conf)
-        end
-      end
-      test "mix" do
-        conf = %[
-          regexp1 message test
-          <regexp>
-            key message
-            pattern test
-          </regexp>
-        ]
-        assert_raise(Fluent::ConfigError) do
-          create_driver(conf)
-        end
-      end
+      assert_equal(Regexp.compile(/ foo/), d.instance.excludes['message'])
     end
   end
 
@@ -114,38 +75,6 @@ class GrepFilterTest < Test::Unit::TestCase
       end
     end
 
-    test 'regexps' do
-      conf = %[
-        <regexp>
-          key message
-          pattern WARN
-        </regexp>
-      ]
-      es = emit(conf, messages)
-      assert_equal(3, es.instance_variable_get(:@record_array).size)
-      assert_block('only WARN logs') do
-        es.all? { |t, r|
-          !r['message'].include?('INFO')
-        }
-      end
-    end
-
-    test 'excludes' do
-      conf = %[
-        <exclude>
-          key message
-          pattern favicon
-        </exclude>
-      ]
-      es = emit(conf, messages)
-      assert_equal(3, es.instance_variable_get(:@record_array).size)
-      assert_block('remove favicon logs') do
-        es.all? { |t, r|
-          !r['message'].include?('favicon')
-        }
-      end
-    end
-
     sub_test_case 'with invalid sequence' do
       def messages
         [

data/test/plugin/test_filter_record_transformer.rb

@@ -58,7 +58,7 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
     test 'typical usage' do
       msgs = ['1', '2']
       es = emit(CONFIG, msgs)
-      es.each_with_index do |(t, r), i|
+      es.each_with_index do |(_t, r), i|
         assert_equal('bar', r['foo'])
         assert_equal(@hostname, r['hostname'])
         assert_equal(@tag, r['tag'])
@@ -70,7 +70,7 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
     test 'remove_keys' do
       config = CONFIG + %[remove_keys foo,message]
       es = emit(config)
-      es.each_with_index do |(t, r), i|
+      es.each_with_index do |(_t, r), i|
         assert_not_include(r, 'foo')
         assert_equal(@hostname, r['hostname'])
         assert_equal(@tag, r['tag'])
@@ -83,7 +83,7 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
       config = CONFIG + %[renew_record true]
       msgs = ['1', '2']
       es = emit(config, msgs)
-      es.each_with_index do |(t, r), i|
+      es.each_with_index do |(_t, r), i|
         assert_not_include(r, 'foo')
         assert_equal(@hostname, r['hostname'])
         assert_equal(@tag, r['tag'])
@@ -95,29 +95,11 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
     test 'renew_time_key' do
       config = %[renew_time_key message]
       times = [ Time.local(2,2,3,4,5,2010,nil,nil,nil,nil), Time.local(3,2,3,4,5,2010,nil,nil,nil,nil) ]
-      msgs = times.map{|t| t.to_i.to_s }
+      msgs = times.map{|t| t.to_f.to_s }
       es = emit(config, msgs)
-      es.each_with_index do |(time, record), i|
+      es.each_with_index do |(time, _record), i|
         assert_equal(times[i].to_i, time)
-        assert_true(record.has_key?('message'))
-      end
-    end
-
-    test 'renew_time_key and remove_keys' do
-      config = %[
-                 renew_time_key event_time_key
-                 remove_keys event_time_key
-                 auto_typecast true
-                 <record>
-                   event_time_key ${record["message"]}
-                 </record>
-               ]
-      times = [Time.local(2, 2, 3, 4, 5, 2010, nil, nil, nil, nil), Time.local(3, 2, 3, 4, 5, 2010, nil, nil, nil, nil)]
-      msgs = times.map { |t| t.to_f.to_s }
-      filtered = emit(config, msgs)
-      filtered.each_with_index do |(time, _record), i|
-        assert_equal(times[i].to_i, time)
-        assert_false(_record.has_key?('event_time_key'))
+        assert(time.is_a?(Fluent::EventTime))
       end
     end
 
@@ -125,21 +107,9 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
       config = %[renew_record true\nkeep_keys foo,message]
       msgs = ['1', '2']
       es = emit(config, msgs)
-      es.each_with_index do |(t, r), i|
-        assert_equal('bar', r['foo'])
-        assert_equal(msgs[i], r['message'])
-      end
-    end
-
-    test 'keep_keys that are not present in the original record should not be included in the result record' do
-      config = %[renew_record true\nkeep_keys foo, bar, baz, message]
-      msgs = ['1', '2', nil]
-      es = emit(config, msgs)
-      es.each_with_index do |(t, r), i|
+      es.each_with_index do |(_t, r), i|
         assert_equal('bar', r['foo'])
         assert_equal(msgs[i], r['message'])
-        assert_equal(false, r.has_key?('bar'))
-        assert_equal(false, r.has_key?('baz'))
       end
     end
 
@@ -147,13 +117,13 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
       config = %[
         enable_ruby yes
         <record>
-          message ${hostname} ${tag_parts.last} ${URI.encode(message)}
+          message ${hostname} ${tag_parts.last} ${"'" + message + "'"}
         </record>
       ]
       msgs = ['1', '2']
       es = emit(config, msgs)
-      es.each_with_index do |(t, r), i|
-        assert_equal("#{@hostname} #{@tag_parts[-1]} #{msgs[i]}", r['message'])
+      es.each_with_index do |(_t, r), i|
+        assert_equal("#{@hostname} #{@tag_parts[-1]} '#{msgs[i]}'", r['message'])
       end
     end
 
@@ -165,7 +135,7 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
       %]
       msgs = ['1', '2']
       es = emit(config, msgs)
-      es.each_with_index do |(t, r), i|
+      es.each_with_index do |(_t, r), i|
         assert_equal({"k1"=>100, "k2"=>"foobar"}, r['hash_field'])
       end
     end
@@ -178,7 +148,7 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
       %]
       msgs = ['1', '2']
       es = emit(config, msgs)
-      es.each_with_index do |(t, r), i|
+      es.each_with_index do |(_t, r), i|
         assert_equal([1,2,3], r['array_field'])
       end
     end
@@ -191,7 +161,7 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
       %]
       msgs = ['1', '2']
       es = emit(config, msgs)
-      es.each_with_index do |(t, r), i|
+      es.each_with_index do |(_t, r), i|
         assert_equal({"hello"=>[1,2,3], "world"=>{"foo"=>"bar"}}, r['mixed_field'])
       end
     end
@@ -203,7 +173,10 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
       yield d if block_given?
       d.run {
         msgs.each do |msg|
-          record = {'eventType0' => 'bar', 'message' => msg}
+          record = {
+            'eventType0' => 'bar',
+            'message'    => msg,
+          }
           record = record.merge(msg) if msg.is_a?(Hash)
           d.emit(record, @time)
         end
@@ -290,30 +263,13 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
         ]
         msgs = ['1', '2']
         es = emit(config, msgs)
-        es.each_with_index do |(t, r), i|
+        es.each_with_index do |(_t, r), i|
           assert_not_include(r, 'eventType0')
           assert_equal("bar", r['eventtype'])
           assert_equal("bar #{msgs[i]}", r['message'])
         end
       end
 
-      test "Prevent overwriting reserved keys such as tag with enable_ruby #{enable_ruby}" do
-        config = %[
-          enable_ruby #{enable_ruby}
-          <record>
-            new_tag ${tag}
-            new_record_tag ${record["tag"]}
-          </record>
-        ]
-        records = [{'tag' => 'tag', 'time' => 'time'}]
-        filtered = emit(config, records)
-        filtered.each_with_index do |(_t, r), i|
-          assert_not_equal('tag', r['new_tag'])
-          assert_equal(@tag, r['new_tag'])
-          assert_equal('tag', r['new_record_tag'])
-        end
-      end
-
       test "hash values with placeholders with enable_ruby #{enable_ruby}" do
         config = %[
           enable_ruby #{enable_ruby}
@@ -327,7 +283,7 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
         ]
         msgs = ['1', '2']
         es = emit(config, msgs)
-        es.each_with_index do |(t, r), i|
+        es.each_with_index do |(_t, r), i|
           assert_equal({"hostname" => @hostname, "tag" => @tag, "#{@tag}" => 100}, r['hash_field'])
         end
       end
@@ -341,7 +297,7 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
         ]
         msgs = ['1', '2']
         es = emit(config, msgs)
-        es.each_with_index do |(t, r), i|
+        es.each_with_index do |(_t, r), i|
           assert_equal([@hostname, @tag], r['array_field'])
         end
       end
@@ -355,7 +311,7 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
         ]
         msgs = ['1', '2']
         es = emit(config, msgs)
-        es.each_with_index do |(t, r), i|
+        es.each_with_index do |(_t, r), i|
           assert_equal([{"tag" => @tag}], r['mixed_field'])
         end
       end
@@ -371,7 +327,7 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
         ]
         msgs = ['1', '2']
         es = emit(config, msgs)
-        es.each_with_index do |(t, r), i|
+        es.each_with_index do |(_t, r), i|
           assert_equal({@hostname=>'hostname',"foo.#{@tag}"=>'tag'}, r)
         end
       end
@@ -424,7 +380,7 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
         ]
         actual_results = []
         es = emit(config, msgs)
-        es.each_with_index do |(t, r), i|
+        es.each_with_index do |(_t, r), i|
           actual_results << {
             single: r["single"],
             multiple: r["multiple"],
@@ -478,7 +434,7 @@ class RecordTransformerFilterTest < Test::Unit::TestCase
         ]
         actual_results = []
         es = emit(config, msgs)
-        es.each_with_index do |(t, r), i|
+        es.each_with_index do |(_t, r), i|
           actual_results << {
             single: r["single"],
             multiple: r["multiple"],

data/test/plugin/test_filter_stdout.rb

@@ -34,7 +34,7 @@ class StdoutFilterTest < Test::Unit::TestCase
   def test_through_record
     d = create_driver
     time = Time.now
-    filtered = emit(d, {'test' => 'test'}, time)
+    filtered = emit(d, {'test' => 'test'}, Fluent::EventTime.from_time(time))
     assert_equal({'test' => 'test'}, filtered)
   end
 
@@ -61,7 +61,7 @@ class StdoutFilterTest < Test::Unit::TestCase
   def test_output_type_json
     d = create_driver(CONFIG + "\noutput_type json")
     time = Time.now
-    out = capture_log(d) { emit(d, {'test' => 'test'}, time) }
+    out = capture_log(d) { emit(d, {'test' => 'test'}, Fluent::EventTime.from_time(time)) }
     assert_equal "#{time.localtime} filter.test: {\"test\":\"test\"}\n", out
 
     # NOTE: Float::NAN is not jsonable
@@ -73,12 +73,12 @@ class StdoutFilterTest < Test::Unit::TestCase
   def test_output_type_hash
     d = create_driver(CONFIG + "\noutput_type hash")
     time = Time.now
-    out = capture_log(d) { emit(d, {'test' => 'test'}, time) }
+    out = capture_log(d) { emit(d, {'test' => 'test'}, Fluent::EventTime.from_time(time)) }
     assert_equal "#{time.localtime} filter.test: {\"test\"=>\"test\"}\n", out
 
     # NOTE: Float::NAN is not jsonable, but hash string can output it.
     d = create_driver(CONFIG + "\noutput_type hash")
-    out = capture_log(d) { emit(d, {'test' => Float::NAN}, time) }
+    out = capture_log(d) { emit(d, {'test' => Float::NAN}, Fluent::EventTime.from_time(time)) }
     assert_equal "#{time.localtime} filter.test: {\"test\"=>NaN}\n", out
   end
 
@@ -86,7 +86,7 @@ class StdoutFilterTest < Test::Unit::TestCase
   def test_include_time_key
     d = create_driver(CONFIG + "\noutput_type json\ninclude_time_key true\nutc")
     time = Time.now
-    message_time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+    message_time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
     out = capture_log(d) { emit(d, {'test' => 'test'}, message_time) }
     assert_equal "#{time.localtime} filter.test: {\"test\":\"test\",\"time\":\"2011-01-02T13:14:15Z\"}\n", out
   end
@@ -95,7 +95,7 @@ class StdoutFilterTest < Test::Unit::TestCase
   def test_format_json
     d = create_driver(CONFIG + "\nformat json")
     time = Time.now
-    out = capture_log(d) { emit(d, {'test' => 'test'}, time) }
+    out = capture_log(d) { emit(d, {'test' => 'test'}, Fluent::EventTime.from_time(time)) }
     assert_equal "{\"test\":\"test\"}\n", out
   end
 

data/test/plugin/test_in_debug_agent.rb

@@ -1,4 +1,6 @@
 require_relative '../helper'
+require 'fluent/test'
+require 'fluent/plugin/in_debug_agent'
 require 'fileutils'
 
 class DebugAgentInputTest < Test::Unit::TestCase

data/test/plugin/test_in_dummy.rb

@@ -1,5 +1,6 @@
 require_relative '../helper'
-require 'fluent/test'
+require 'fluent/test/driver/input'
+require 'fluent/plugin/in_dummy'
 
 class DummyTest < Test::Unit::TestCase
   def setup
@@ -7,7 +8,7 @@ class DummyTest < Test::Unit::TestCase
   end
 
   def create_driver(conf)
-    Fluent::Test::InputTestDriver.new(Fluent::DummyInput).configure(conf)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::DummyInput).configure(conf)
   end
 
   sub_test_case 'configure' do
@@ -42,10 +43,6 @@ class DummyTest < Test::Unit::TestCase
       assert_equal 10, d.instance.rate
     end
 
-    def json_error_messages_regexp
-      /JSON::ParserError|got incomplete JSON|0th element of dummy, foo, is not a hash/
-    end
-
     test 'dummy' do
       # hash is okay
       d = create_driver(config + %[dummy {"foo":"bar"}])
@@ -55,7 +52,7 @@ class DummyTest < Test::Unit::TestCase
       d = create_driver(config + %[dummy [{"foo":"bar"}]])
       assert_equal [{"foo"=>"bar"}], d.instance.dummy
 
-      assert_raise_message(json_error_messages_regexp) do
+      assert_raise_message(/JSON::ParserError|got incomplete JSON/) do
         create_driver(config + %[dummy "foo"])
       end
 
@@ -74,23 +71,20 @@ class DummyTest < Test::Unit::TestCase
 
     test 'simple' do
       d = create_driver(config)
-      d.run {
-        # d.run sleeps 0.5 sec
-      }
-      emits = d.emits
-      emits.each do |tag, time, record|
+      d.run(timeout: 0.5)
+
+      d.events.each do |tag, time, record|
         assert_equal("dummy", tag)
         assert_equal({"foo"=>"bar"}, record)
+        assert(time.is_a?(Fluent::EventTime))
       end
     end
 
     test 'with auto_increment_key' do
       d = create_driver(config + %[auto_increment_key id])
-      d.run {
-        # d.run sleeps 0.5 sec
-      }
-      emits = d.emits
-      emits.each_with_index do |(tag, time, record), i|
+      d.run(timeout: 0.5)
+
+      d.events.each_with_index do |(tag, _time, record), i|
         assert_equal("dummy", tag)
         assert_equal({"foo"=>"bar", "id"=>i}, record)
       end

data/test/plugin/test_in_exec.rb

@@ -1,11 +1,12 @@
 require_relative '../helper'
 require 'fluent/test'
+require 'fluent/plugin/in_exec'
 require 'net/http'
 
 class ExecInputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
-    @test_time = Time.parse("2011-01-02 13:14:15").to_i
+    @test_time = Fluent::EventTime.parse("2011-01-02 13:14:15")
     @script = File.expand_path(File.join(File.dirname(__FILE__), '..', 'scripts', 'exec_script.rb'))
   end
 
@@ -53,17 +54,6 @@ class ExecInputTest < Test::Unit::TestCase
     ]
   end
 
-  def invalid_json_config
-    # For counting command execution, redirect stderr to file
-    %[
-      command ruby #{@script} #{@test_time} 4
-      format json
-      tag_key tag
-      time_key time
-      run_interval 0.5
-    ]
-  end
-
   def test_configure
     d = create_driver
     assert_equal 'tsv', d.instance.format
@@ -103,6 +93,7 @@ class ExecInputTest < Test::Unit::TestCase
     emits = d.emits
     assert_equal true, emits.length > 0
     assert_equal ["tag1", @test_time, {"k1"=>"ok"}], emits[0]
+    assert_equal_event_time(@test_time, emits[0][1])
   end
 
   def test_emit_json
@@ -115,6 +106,7 @@ class ExecInputTest < Test::Unit::TestCase
     emits = d.emits
     assert_equal true, emits.length > 0
     assert_equal ["tag1", @test_time, {"k1"=>"ok"}], emits[0]
+    assert_equal_event_time(@test_time, emits[0][1])
   end
 
   def test_emit_msgpack
@@ -127,6 +119,7 @@ class ExecInputTest < Test::Unit::TestCase
     emits = d.emits
     assert_equal true, emits.length > 0
     assert_equal ["tag1", @test_time, {"k1"=>"ok"}], emits[0]
+    assert_equal_event_time(@test_time, emits[0][1])
   end
 
   def test_emit_regexp
@@ -139,18 +132,6 @@ class ExecInputTest < Test::Unit::TestCase
     emits = d.emits
     assert_equal true, emits.length > 0
     assert_equal ["regex_tag", @test_time, {"message"=>"hello"}], emits[0]
-    assert_equal @test_time, emits[0][1]
-  end
-
-  def test_emit_with_invalid_script
-    d = create_driver invalid_json_config
-
-    d.run do
-      sleep 2
-    end
-
-    assert_equal true, d.emits.empty?
-    logs = d.instance.log.logs
-    assert_equal true, logs.count { |line| line =~ /exec failed to run or shutdown child process/ }.between?(1, 4)
+    assert_equal_event_time(@test_time, emits[0][1])
   end
 end