fluentd 0.12.43 → 0.14.0

data/test/test_mixin.rb

@@ -208,7 +208,7 @@ module MixinTest
 
     def test_timezone_invalid
       assert_raise(Fluent::ConfigError) do
-        d = create_driver(Fluent::SetTimeKeyMixin, %[
+        create_driver(Fluent::SetTimeKeyMixin, %[
         include_time_key true
         timezone Invalid/Invalid
         ])

data/test/test_output.rb

@@ -1,8 +1,9 @@
 require_relative 'helper'
 require 'fluent/test'
 require 'fluent/output'
+require 'fluent/output_chain'
 require 'timecop'
-require 'flexmock'
+require 'flexmock/test_unit'
 
 module FluentOutputTest
   include Fluent
@@ -47,7 +48,6 @@ module FluentOutputTest
       assert_equal nil, d.instance.max_retry_wait
       assert_equal 1.0, d.instance.retry_wait
       assert_equal 1, d.instance.num_threads
-      assert_equal 20.0, d.instance.slow_flush_log_threshold
       assert_equal 1, d.instance.queued_chunk_flush_interval
 
       # max_retry_wait
@@ -58,12 +58,15 @@ module FluentOutputTest
       d = create_driver(CONFIG + %[disable_retry_limit true])
       assert_equal true, d.instance.disable_retry_limit
 
-      # retry_wait is converted to Float for calc_retry_wait
-      d = create_driver(CONFIG + %[retry_wait 1s])
-      assert_equal Float, d.instance.retry_wait.class
+      #### retry_state cares it
+      # # retry_wait is converted to Float for calc_retry_wait
+      # d = create_driver(CONFIG + %[retry_wait 1s])
+      # assert_equal Float, d.instance.retry_wait.class
     end
 
     def test_calc_retry_wait
+      omit "too internal test"
+
       # default
       d = create_driver
       d.instance.retry_limit.times {
@@ -82,6 +85,8 @@ module FluentOutputTest
     end
 
     def test_calc_retry_wait_with_integer_retry_wait
+      omit "too internal test"
+
       d = create_driver(CONFIG + %[retry_wait 2s])
       d.instance.retry_limit.times {
         d.instance.instance_eval { @num_errors += 1 }
@@ -90,6 +95,8 @@ module FluentOutputTest
     end
 
     def test_large_num_retries
+      omit "too internal test"
+
       # Test that everything works properly after a very large number of
       # retries and we hit the expected max_retry_wait.
       exp_max_retry_wait = 300
@@ -135,6 +142,8 @@ module FluentOutputTest
     end
 
     def test_submit_flush_target
+      omit "too internal test"
+
       # default
       d = create_mock_driver
       d.instance.start_mock
@@ -188,6 +197,8 @@ module FluentOutputTest
       end
 
       test "force_flush works on retrying" do
+        omit "too internal test"
+
         d = create_driver(CONFIG)
         d.instance.start
         buffer = d.instance.instance_variable_get(:@buffer)
@@ -202,67 +213,25 @@ module FluentOutputTest
         10.times { sleep 0.05 }
       end
     end
+  end
 
-    sub_test_case "test slow_flush_log_threshold" do
-      def create_slow_driver(conf, sleep_time)
-        d = Fluent::Test::BufferedOutputTestDriver.new(Fluent::BufferedOutput) do
-          attr_accessor :sleep_time
-
-          def initialize
-            super
-            @_written_check_mutex = Mutex.new
-            @_written_check = false
-          end
-
-          def written_check
-            @_written_check_mutex.synchronize { @_written_check }
-          end
-
-          def write(chunk)
-            sleep @sleep_time
-          end
-
-          def try_flush
-            super
-            @_written_check_mutex.synchronize { @_written_check = true }
-          end
-        end.configure(conf)
-        d.instance.sleep_time = sleep_time
-        d
-      end
-
-      def setup
-        # output#try_flush uses $log, so it should be replaced
-        @orig_log = $log
-        $log = Fluent::Test::TestLogger.new
-      end
+  class ObjectBufferedOutputTest < ::Test::Unit::TestCase
+    include FluentOutputTest
 
-      def teardown
-        $log = @orig_log
-      end
+    def setup
+      Fluent::Test.setup
+    end
 
-      test "flush took longer time than slow_flush_log_threshold" do
-        d = create_slow_driver(%[
-          slow_flush_log_threshold 0.5
-        ], 1.5)
-        run_driver(d)
-        assert_equal 1, $log.logs.count { |line| line =~ /buffer flush took longer time than slow_flush_log_threshold:/ }
-      end
+    CONFIG = %[]
 
-      test "flush took lower time than slow_flush_log_threshold. No logs" do
-        d = create_slow_driver(%[
-          slow_flush_log_threshold 5.0
-        ], 0)
-        run_driver(d)
-        assert_equal 0, $log.logs.count { |line| line =~ /buffer flush took longer time than slow_flush_log_threshold:/ }
-      end
+    def create_driver(conf=CONFIG)
+      Fluent::Test::OutputTestDriver.new(Fluent::ObjectBufferedOutput).configure(conf, true)
+    end
 
-      def run_driver(d)
-        d.instance.start
-        d.instance.emit('test', OneEventStream.new(@time.to_i, {"message" => "foo"}), NullOutputChain.instance)
-        d.instance.force_flush
-        until d.instance.written_check; end
-      end
+    def test_configure
+      # default
+      d = create_driver
+      assert_equal true, d.instance.time_as_integer
     end
   end
 
@@ -278,10 +247,19 @@ module FluentOutputTest
 
     TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/tmp/time_sliced_output")
 
-    CONFIG = %[buffer_path #{TMP_DIR}/foo]
+    CONFIG = %[
+      buffer_path #{TMP_DIR}/foo
+      time_slice_format %Y%m%d
+    ]
+
+    class TimeSlicedOutputTestPlugin < Fluent::TimeSlicedOutput
+      def format(tag, time, record)
+        ''
+      end
+    end
 
     def create_driver(conf=CONFIG)
-      Fluent::Test::TimeSlicedOutputTestDriver.new(Fluent::TimeSlicedOutput).configure(conf, true)
+      Fluent::Test::TimeSlicedOutputTestDriver.new(TimeSlicedOutputTestPlugin).configure(conf, true)
     end
 
     sub_test_case "force_flush test" do
@@ -296,6 +274,8 @@ module FluentOutputTest
       end
 
       test "force_flush immediately flushes" do
+        omit "too internal test"
+
         d = create_driver(CONFIG + %[
           time_format %Y%m%d%H%M%S
         ])
@@ -320,17 +300,24 @@ module FluentOutputTest
       end
 
       test "emit with valid event" do
+        omit "there's no #emit method anymore in output plugins"
+
         d = create_driver
         d.instance.start
-        d.instance.emit('test', OneEventStream.new(@time.to_i, {"message" => "foo"}), NullOutputChain.instance)
+        if d.instance.method(:emit).arity == 3
+          d.instance.emit('test', OneEventStream.new(@time.to_i, {"message" => "foo"}), NullOutputChain.instance)
+        else
+          d.instance.emit('test', OneEventStream.new(@time.to_i, {"message" => "foo"}))
+        end
         assert_equal 0, d.instance.log.logs.size
       end
 
       test "emit with invalid event" do
         d = create_driver
         d.instance.start
-        d.instance.emit('test', OneEventStream.new('string', 10), NullOutputChain.instance)
-        assert_equal 1, d.instance.log.logs.count { |line| line =~ /dump an error event/ }
+        assert_raise ArgumentError, "time must be a Fluent::EventTime (or Integer)" do
+          d.instance.emit_events('test', OneEventStream.new('string', 10))
+        end
       end
     end
   end

data/test/test_parser.rb

@@ -1,11 +1,22 @@
 require_relative 'helper'
 require 'fluent/test'
 require 'fluent/parser'
-require 'fluent/plugin/string_util'
 
 module ParserTest
   include Fluent
 
+  def setup
+    Fluent::Test.setup
+  end
+
+  def str2time(str_time, format = nil)
+    if format
+      Fluent::EventTime.from_time(Time.strptime(str_time, format))
+    else
+      Fluent::EventTime.parse(str_time)
+    end
+  end
+
   class BaseParserTest < ::Test::Unit::TestCase
     include ParserTest
 
@@ -63,24 +74,28 @@ module ParserTest
     def test_call_with_parse
       parser = TextParser::TimeParser.new(nil)
 
-      time = event_time('2013-09-18 12:00:00 +0900')
-      assert_equal_event_time(time, parser.parse('2013-09-18 12:00:00 +0900'))
+      assert(parser.parse('2013-09-18 12:00:00 +0900').is_a?(Fluent::EventTime))
+
+      time = str2time('2013-09-18 12:00:00 +0900')
+      assert_equal(time, parser.parse('2013-09-18 12:00:00 +0900'))
     end
 
     def test_parse_with_strptime
       parser = TextParser::TimeParser.new('%d/%b/%Y:%H:%M:%S %z')
 
-      time = event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z')
-      assert_equal_event_time(time, parser.parse('28/Feb/2013:12:00:00 +0900'))
+      assert(parser.parse('28/Feb/2013:12:00:00 +0900').is_a?(Fluent::EventTime))
+
+      time = str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z')
+      assert_equal(time, parser.parse('28/Feb/2013:12:00:00 +0900'))
     end
 
-    data('with sec' => '2017-01-01T12:00:00+09:00',
-         'with msec' => '2017-01-01T12:00:00.123+09:00')
-    def test_parse_with_iso8601(data)
-      parser = TextParser::TimeParser.new('%iso8601')
+    def test_parse_nsec_with_strptime
+      parser = TextParser::TimeParser.new('%d/%b/%Y:%H:%M:%S:%N %z')
 
-      time = event_time(data)
-      assert_equal_event_time(time, parser.parse(data))
+      assert(parser.parse('28/Feb/2013:12:00:00:123456789 +0900').is_a?(Fluent::EventTime))
+
+      time = str2time('28/Feb/2013:12:00:00:123456789 +0900', '%d/%b/%Y:%H:%M:%S:%N %z')
+      assert_equal_event_time(time, parser.parse('28/Feb/2013:12:00:00:123456789 +0900'))
     end
 
     def test_parse_with_invalid_argument
@@ -99,14 +114,14 @@ module ParserTest
 
     def internal_test_case(parser)
       text = '192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] [14/Feb/2013:12:00:00 +0900] "true /,/user HTTP/1.1" 200 777'
-      [parser.parse(text), parser.parse(text) { |time, record| return time, record}].each { |time, record|
-        assert_equal_event_time(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
+      parser.parse(text) { |time, record|
+        assert_equal(str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'), time)
         assert_equal({
           'user' => '-',
           'flag' => true,
           'code' => 200.0,
           'size' => 777,
-          'date' => event_time('14/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'),
+          'date' => str2time('14/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'),
           'host' => '192.168.0.1',
           'path' => ['/', '/user']
         }, record)
@@ -140,18 +155,18 @@ module ParserTest
       )
       text = '2013-02-28 12:00:00 +0900'
       parser.parse(text) do |time, record|
-        assert_equal event_time(text), time
+        assert_equal Fluent::EventTime.parse(text), time
       end
     end
 
     def test_parse_without_time
-      time_at_start = event_time()
+      time_at_start = Time.now.to_i
       text = "tagomori_satoshi tagomoris 34\n"
 
       parser = TextParser::RegexpParser.new(Regexp.new(%q!^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$!))
-      parser.configure('types'=>'name:string,user:string,age:bool')
+      parser.configure('types'=>'name:string,user:string,age:integer')
 
-      [parser.parse(text), parser.parse(text) { |time, record| return time, record}].each { |time, record|
+      parser.parse(text) { |time, record|
         assert time && time >= time_at_start, "parser puts current time without time input"
         assert_equal "tagomori_satoshi", record["name"]
         assert_equal "tagomoris", record["user"]
@@ -161,7 +176,8 @@ module ParserTest
       parser2 = Fluent::Test::ParserTestDriver.new(TextParser::RegexpParser, Regexp.new(%q!^(?<name>[^ ]*) (?<user>[^ ]*) (?<age>\d*)$!))
       parser2.configure('types'=>'name:string,user:string,age:integer')
       parser2.instance.estimate_current_event = false
-      [parser2.parse(text), parser2.parse(text) { |time, record| return time, record}].each { |time, record|
+
+      parser2.parse(text) { |time, record|
         assert_equal "tagomori_satoshi", record["name"]
         assert_equal "tagomoris", record["user"]
         assert_equal 34, record["age"]
@@ -200,14 +216,14 @@ module ParserTest
     include ParserTest
 
     def setup
-      @parser = TextParser::TEMPLATE_REGISTRY.lookup('apache').call
+      @parser = Fluent::Plugin.new_parser('apache')
     end
 
     data('parse' => :parse, 'call' => :call)
     def test_call(method_name)
       m = @parser.method(method_name)
       m.call('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777') { |time, record|
-        assert_equal_event_time(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
+        assert_equal(str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'), time)
         assert_equal({
           'user'    => '-',
           'method'  => 'GET',
@@ -236,7 +252,8 @@ module ParserTest
     include ParserTest
 
     def setup
-      @parser = TextParser::TEMPLATE_REGISTRY.lookup('apache_error').call
+      @parser = Fluent::Plugin.new_parser('apache_error')
+      @parser.configure({})
       @expected = {
         'level' => 'error',
         'client' => '127.0.0.1',
@@ -246,21 +263,21 @@ module ParserTest
 
     def test_parse
       @parser.parse('[Wed Oct 11 14:32:52 2000] [error] [client 127.0.0.1] client denied by server configuration') { |time, record|
-        assert_equal_event_time(event_time('Wed Oct 11 14:32:52 2000'), time)
+        assert_equal(str2time('Wed Oct 11 14:32:52 2000'), time)
         assert_equal(@expected, record)
       }
     end
 
     def test_parse_with_pid
       @parser.parse('[Wed Oct 11 14:32:52 2000] [error] [pid 1000] [client 127.0.0.1] client denied by server configuration') { |time, record|
-        assert_equal_event_time(event_time('Wed Oct 11 14:32:52 2000'), time)
+        assert_equal(str2time('Wed Oct 11 14:32:52 2000'), time)
         assert_equal(@expected.merge('pid' => '1000'), record)
       }
     end
 
     def test_parse_without_client
       @parser.parse('[Wed Oct 11 14:32:52 2000] [notice] Apache/2.2.15 (Unix) DAV/2 configured -- resuming normal operations') { |time, record|
-        assert_equal_event_time(event_time('Wed Oct 11 14:32:52 2000'), time)
+        assert_equal(str2time('Wed Oct 11 14:32:52 2000'), time)
         assert_equal({
           'level' => 'notice',
           'message' => 'Apache/2.2.15 (Unix) DAV/2 configured -- resuming normal operations'
@@ -288,7 +305,7 @@ module ParserTest
 
     def test_parse
       @parser.parse('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"') { |time, record|
-        assert_equal_event_time(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
+        assert_equal(str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'), time)
         assert_equal(@expected, record)
       }
       assert_equal(TextParser::ApacheParser::REGEXP, @parser.patterns['format'])
@@ -297,18 +314,10 @@ module ParserTest
 
     def test_parse_without_http_version
       @parser.parse('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET /" 200 777 "-" "Opera/12.0"') { |time, record|
-        assert_equal_event_time(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
+        assert_equal(str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'), time)
         assert_equal(@expected, record)
       }
     end
-
-    def test_parse_with_escape_sequence
-      @parser.parse('192.168.0.1 - - [28/Feb/2013:12:00:00 +0900] "GET /\" HTTP/1.1" 200 777 "referer \\\ \"" "user agent \\\ \""') { |_, record|
-        assert_equal('/\"', record['path'])
-        assert_equal('referer \\\ \"', record['referer'])
-        assert_equal('user agent \\\ \"', record['agent'])
-      }
-    end
   end
 
   class SyslogParserTest < ::Test::Unit::TestCase
@@ -327,7 +336,7 @@ module ParserTest
     def test_parse
       @parser.configure({})
       @parser.parse('Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
-        assert_equal_event_time(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
+        assert_equal(str2time('Feb 28 12:00:00', '%b %d %H:%M:%S'), time)
         assert_equal(@expected, record)
       }
       assert_equal(TextParser::SyslogParser::REGEXP, @parser.instance.patterns['format'])
@@ -337,7 +346,7 @@ module ParserTest
     def test_parse_with_time_format
       @parser.configure('time_format' => '%b %d %M:%S:%H')
       @parser.parse('Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
-        assert_equal_event_time(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
+        assert_equal(str2time('Feb 28 12:00:00', '%b %d %H:%M:%S'), time)
         assert_equal(@expected, record)
       }
       assert_equal('%b %d %M:%S:%H', @parser.instance.patterns['time_format'])
@@ -346,7 +355,7 @@ module ParserTest
     def test_parse_with_priority
       @parser.configure('with_priority' => true)
       @parser.parse('<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test') { |time, record|
-        assert_equal_event_time(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
+        assert_equal(str2time('Feb 28 12:00:00', '%b %d %H:%M:%S'), time)
         assert_equal(@expected.merge('pri' => 6), record)
       }
       assert_equal(TextParser::SyslogParser::REGEXP_WITH_PRI, @parser.instance.patterns['format'])
@@ -356,7 +365,7 @@ module ParserTest
     def test_parse_without_colon
       @parser.configure({})
       @parser.parse('Feb 28 12:00:00 192.168.0.1 fluentd[11111] [error] Syslog test') { |time, record|
-        assert_equal_event_time(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
+        assert_equal(str2time('Feb 28 12:00:00', '%b %d %H:%M:%S'), time)
         assert_equal(@expected, record)
       }
       assert_equal(TextParser::SyslogParser::REGEXP, @parser.instance.patterns['format'])
@@ -373,256 +382,6 @@ module ParserTest
         assert_equal "Feb 28 00:00:12", record['time']
       end
     end
-
-    def test_parse_various_characters_for_tag
-      ident = '~!@#$%^&*()_+=-`]{};"\'/?\\,.<>'
-      @parser.configure({})
-      @parser.parse("Feb 28 12:00:00 192.168.0.1 #{ident}[11111]: [error] Syslog test") { |time, record|
-        assert_equal_event_time(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
-        assert_equal(@expected.merge('ident' => ident), record)
-      }
-    end
-
-    def test_parse_various_characters_for_tag_with_priority
-      ident = '~!@#$%^&*()_+=-`]{};"\'/?\\,.<>'
-      @parser.configure({'with_priority' => true})
-      @parser.parse("<6>Feb 28 12:00:00 192.168.0.1 #{ident}[11111]: [error] Syslog test") { |time, record|
-        assert_equal_event_time(event_time('Feb 28 12:00:00', format: '%b %d %H:%M:%S'), time)
-        assert_equal(@expected.merge('pri' => 6, 'ident' => ident), record)
-      }
-    end
-
-    class TestRFC5424Regexp < self
-      def test_parse_with_rfc5424_message
-        @parser.configure(
-          'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
-          'message_format' => 'rfc5424',
-          'with_priority' => true,
-        )
-        text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
-        @parser.parse(text) do |time, record|
-          assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
-          assert_equal "-", record["pid"]
-          assert_equal "-", record["msgid"]
-          assert_equal "-", record["extradata"]
-          assert_equal "Hi, from Fluentd!", record["message"]
-        end
-        assert_equal(TextParser::SyslogParser::REGEXP_RFC5424_WITH_PRI,
-                     @parser.instance.patterns['format'])
-
-      end
-
-      def test_parse_with_rfc5424_message_and_without_priority
-        @parser.configure(
-           'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
-           'message_format' => 'rfc5424',
-        )
-        text = '2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
-        @parser.instance.parse(text) do |time, record|
-          assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
-          assert_equal "-", record["pid"]
-          assert_equal "-", record["msgid"]
-          assert_equal "-", record["extradata"]
-          assert_equal "Hi, from Fluentd!", record["message"]
-        end
-        assert_equal(TextParser::SyslogParser::REGEXP_RFC5424,
-                     @parser.instance.patterns['format'])
-      end
-
-      def test_parse_with_rfc5424_message_without_time_format
-        @parser.configure(
-          'message_format' => 'rfc5424',
-          'with_priority' => true,
-        )
-        text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
-        @parser.instance.parse(text) do |time, record|
-          assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
-          assert_equal "-", record["pid"]
-          assert_equal "-", record["msgid"]
-          assert_equal "-", record["extradata"]
-          assert_equal "Hi, from Fluentd!", record["message"]
-        end
-      end
-
-      def test_parse_with_rfc5424_structured_message
-        @parser.configure(
-          'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
-          'message_format' => 'rfc5424',
-          'with_priority' => true,
-        )
-        text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
-        @parser.parse(text) do |time, record|
-          assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
-          assert_equal "11111", record["pid"]
-          assert_equal "ID24224", record["msgid"]
-          assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
-                       record["extradata"]
-          assert_equal "Hi, from Fluentd!", record["message"]
-        end
-      end
-    end
-
-    class TestAutoRegexp < self
-      def test_auto_with_legacy_syslog_message
-        @parser.configure(
-          'time_format' => '%b %d %M:%S:%H',
-          'mseeage_format' => 'auto',
-        )
-        text = 'Feb 28 00:00:12 192.168.0.1 fluentd[11111]: [error] Syslog test'
-        @parser.parse(text) do |time, record|
-          assert_equal(event_time("Feb 28 00:00:12", format: '%b %d %M:%S:%H'), time)
-          assert_equal(@expected, record)
-        end
-      end
-
-      def test_auto_with_legacy_syslog_priority_message
-        @parser.configure(
-          'time_format' => '%b %d %M:%S:%H',
-          'with_priority' => true,
-          'mseeage_format' => 'auto',
-        )
-        text = '<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
-        @parser.parse(text) do |time, record|
-          assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
-          assert_equal(@expected.merge('pri' => 6), record)
-        end
-        assert_equal(TextParser::SyslogParser::REGEXP_WITH_PRI,
-                     @parser.instance.patterns['format'])
-      end
-
-      def test_parse_with_rfc5424_message
-        @parser.configure(
-          'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
-          'message_format' => 'auto',
-          'with_priority' => true,
-        )
-        text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
-        @parser.parse(text) do |time, record|
-          assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
-          assert_equal "-", record["pid"]
-          assert_equal "-", record["msgid"]
-          assert_equal "-", record["extradata"]
-          assert_equal "Hi, from Fluentd!", record["message"]
-        end
-        assert_equal(TextParser::SyslogParser::REGEXP_RFC5424_WITH_PRI,
-                     @parser.instance.patterns['format'])
-      end
-
-      def test_parse_with_rfc5424_structured_message
-        @parser.configure(
-          'time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
-          'message_format' => 'auto',
-          'with_priority' => true,
-        )
-        text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
-        @parser.parse(text) do |time, record|
-          assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
-          assert_equal "11111", record["pid"]
-          assert_equal "ID24224", record["msgid"]
-          assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
-                       record["extradata"]
-          assert_equal "Hi, from Fluentd!", record["message"]
-        end
-        assert_equal(TextParser::SyslogParser::REGEXP_RFC5424_WITH_PRI,
-                     @parser.instance.patterns['format'])
-      end
-
-      def test_parse_with_both_message_type
-        @parser.configure(
-          'time_format' => '%b %d %M:%S:%H',
-          'rfc5424_time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
-          'message_format' => 'auto',
-          'with_priority' => true,
-        )
-        text = '<1>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
-        @parser.parse(text) do |time, record|
-          assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
-          assert_equal(@expected.merge('pri' => 1), record)
-        end
-        assert_equal(TextParser::SyslogParser::REGEXP_WITH_PRI,
-                     @parser.instance.patterns['format'])
-
-        text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
-        @parser.parse(text) do |time, record|
-          assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
-          assert_equal "11111", record["pid"]
-          assert_equal "ID24224", record["msgid"]
-          assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
-                       record["extradata"]
-          assert_equal "Hi, from Fluentd!", record["message"]
-        end
-        assert_equal(TextParser::SyslogParser::REGEXP_RFC5424_WITH_PRI,
-                   @parser.instance.patterns['format'])
-
-        text = '<1>Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'
-        @parser.parse(text) do |time, record|
-          assert_equal(event_time("Feb 28 12:00:02", format: '%b %d %M:%S:%H'), time)
-          assert_equal(@expected.merge('pri' => 1), record)
-        end
-        assert_equal(TextParser::SyslogParser::REGEXP_WITH_PRI,
-                   @parser.instance.patterns['format'])
-
-        text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
-        @parser.parse(text) do |time, record|
-          assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
-          assert_equal "-", record["pid"]
-          assert_equal "-", record["msgid"]
-          assert_equal "-", record["extradata"]
-          assert_equal "Hi, from Fluentd!", record["message"]
-        end
-        assert_equal(TextParser::SyslogParser::REGEXP_RFC5424_WITH_PRI,
-                     @parser.instance.patterns['format'])
-
-      end
-
-      def test_parse_with_both_message_type_and_priority
-        @parser.configure(
-          'time_format' => '%b %d %M:%S:%H',
-          'rfc5424_time_format' => '%Y-%m-%dT%H:%M:%S.%L%z',
-          'with_priority' => true,
-          'message_format' => 'auto',
-        )
-        text = '<6>Feb 28 12:00:00 192.168.0.1 fluentd[11111]: [error] Syslog test'
-        @parser.parse(text) do |time, record|
-          assert_equal(event_time("Feb 28 12:00:00", format: '%b %d %M:%S:%H'), time)
-          assert_equal(@expected.merge('pri' => 6), record)
-        end
-        assert_equal(TextParser::SyslogParser::REGEXP_WITH_PRI,
-                     @parser.instance.patterns['format'])
-
-        text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd 11111 ID24224 [exampleSDID@20224 iut="3" eventSource="Application" eventID="11211"] Hi, from Fluentd!'
-        @parser.parse(text) do |time, record|
-          assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
-          assert_equal "11111", record["pid"]
-          assert_equal "ID24224", record["msgid"]
-          assert_equal "[exampleSDID@20224 iut=\"3\" eventSource=\"Application\" eventID=\"11211\"]",
-                       record["extradata"]
-          assert_equal "Hi, from Fluentd!", record["message"]
-        end
-        assert_equal(TextParser::SyslogParser::REGEXP_RFC5424_WITH_PRI,
-                     @parser.instance.patterns['format'])
-
-        text = '<16>Feb 28 12:00:02 192.168.0.1 fluentd[11111]: [error] Syslog test'
-        @parser.parse(text) do |time, record|
-          assert_equal(event_time("Feb 28 12:00:02", format: '%b %d %M:%S:%H'), time)
-          assert_equal(@expected.merge('pri' => 16), record)
-        end
-        assert_equal(TextParser::SyslogParser::REGEXP_WITH_PRI,
-                     @parser.instance.patterns['format'])
-
-        text = '<16>1 2017-02-06T13:14:15.003Z 192.168.0.1 fluentd - - - Hi, from Fluentd!'
-        @parser.parse(text) do |time, record|
-          assert_equal(event_time("2017-02-06T13:14:15.003Z", format: '%Y-%m-%dT%H:%M:%S.%L%z'), time)
-          assert_equal "-", record["pid"]
-          assert_equal "-", record["msgid"]
-          assert_equal "-", record["extradata"]
-          assert_equal "Hi, from Fluentd!", record["message"]
-        end
-        assert_equal(TextParser::SyslogParser::REGEXP_RFC5424_WITH_PRI,
-                     @parser.instance.patterns['format'])
-
-      end
-    end
   end
 
   class JsonParserTest < ::Test::Unit::TestCase
@@ -636,7 +395,7 @@ module ParserTest
     def test_parse(data)
       @parser.configure('json_parser' => data)
       @parser.parse('{"time":1362020400,"host":"192.168.0.1","size":777,"method":"PUT"}') { |time, record|
-        assert_equal_event_time(event_time('2013-02-28 12:00:00 +0900'), time)
+        assert_equal(str2time('2013-02-28 12:00:00 +0900').to_i, time)
         assert_equal({
           'host'   => '192.168.0.1',
           'size'   => 777,
@@ -680,14 +439,6 @@ module ParserTest
       }
     end
 
-    data('oj' => 'oj', 'yajl' => 'yajl')
-    def test_parse_with_colon_string(data)
-      @parser.configure('json_parser' => data)
-      @parser.parse('{"time":1362020400,"log":":message"}') { |time, record|
-        assert_equal(record['log'], ':message')
-      }
-    end
-
     data('oj' => 'oj', 'yajl' => 'yajl')
     def test_parse_with_invalid_time(data)
       @parser.configure('json_parser' => data)
@@ -696,16 +447,43 @@ module ParserTest
       end
     end
 
+    data('oj' => 'oj', 'yajl' => 'yajl')
+    def test_parse_float_time(data)
+      parser = TextParser::JSONParser.new
+      parser.configure('json_parser' => data)
+      text = "100.1"
+      parser.parse("{\"time\":\"#{text}\"}") do |time, record|
+        assert_equal Time.at(text.to_f).to_i, time.sec
+        assert_equal Time.at(text.to_f).nsec, time.nsec
+      end
+    end
+
     data('oj' => 'oj', 'yajl' => 'yajl')
     def test_parse_with_keep_time_key(data)
       parser = TextParser::JSONParser.new
+      format = "%d/%b/%Y:%H:%M:%S %z"
       parser.configure(
-        'time_format' => "%d/%b/%Y:%H:%M:%S %z",
+        'time_format' => format,
         'keep_time_key' => 'true',
         'json_parser' => data
       )
       text = "28/Feb/2013:12:00:00 +0900"
       parser.parse("{\"time\":\"#{text}\"}") do |time, record|
+        assert_equal Time.strptime(text, format).to_i, time.sec
+        assert_equal text, record['time']
+      end
+    end
+
+    data('oj' => 'oj', 'yajl' => 'yajl')
+    def test_parse_with_keep_time_key_without_time_format(data)
+      parser = TextParser::JSONParser.new
+      parser.configure(
+        'keep_time_key' => 'true',
+        'json_parser' => data
+      )
+      text = "100"
+      parser.parse("{\"time\":\"#{text}\"}") do |time, record|
+        assert_equal text.to_i, time.sec
         assert_equal text, record['time']
       end
     end
@@ -715,7 +493,7 @@ module ParserTest
     include ParserTest
 
     def setup
-      @parser = TextParser::TEMPLATE_REGISTRY.lookup('nginx').call
+      @parser = Fluent::Plugin.new_parser('nginx')
       @expected = {
         'remote'  => '127.0.0.1',
         'host'    => '192.168.0.1',
@@ -731,21 +509,21 @@ module ParserTest
 
     def test_parse
       @parser.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET / HTTP/1.1" 200 777 "-" "Opera/12.0"') { |time, record|
-        assert_equal_event_time(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
+        assert_equal(str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'), time)
         assert_equal(@expected, record)
       }
     end
 
     def test_parse_with_empty_included_path
       @parser.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET /a[ ]b HTTP/1.1" 200 777 "-" "Opera/12.0"') { |time, record|
-        assert_equal_event_time(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
+        assert_equal(str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'), time)
         assert_equal(@expected.merge('path' => '/a[ ]b'), record)
       }
     end
 
     def test_parse_without_http_version
       @parser.parse('127.0.0.1 192.168.0.1 - [28/Feb/2013:12:00:00 +0900] "GET /" 200 777 "-" "Opera/12.0"') { |time, record|
-        assert_equal_event_time(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
+        assert_equal(str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'), time)
         assert_equal(@expected, record)
       }
     end
@@ -774,7 +552,7 @@ module ParserTest
       parser = TextParser::TSVParser.new
       parser.configure('keys' => param, 'time_key' => 'time')
       parser.parse("2013/02/28 12:00:00\t192.168.0.1\t111") { |time, record|
-        assert_equal_event_time(event_time('2013/02/28 12:00:00', format: '%Y/%m/%d %H:%M:%S'), time)
+        assert_equal(str2time('2013/02/28 12:00:00', '%Y/%m/%d %H:%M:%S'), time)
         assert_equal({
           'a' => '192.168.0.1',
           'b' => '111',
@@ -839,10 +617,11 @@ module ParserTest
     end
 
     data('array param' => '["a","b","c","d","e","f"]', 'string param' => 'a,b,c,d,e,f')
-    def test_parse_with_null_value_pattern(param)
+    def test_parse_with_null_value_pattern
       parser = TextParser::TSVParser.new
       parser.configure(
         'keys'=>param,
+        'time_key'=>'time',
         'null_value_pattern'=>'^(-|null|NULL)$'
       )
       parser.parse("-\tnull\tNULL\t\t--\tnuLL") do |time, record|
@@ -856,10 +635,11 @@ module ParserTest
     end
 
     data('array param' => '["a","b"]', 'string param' => 'a,b')
-    def test_parse_with_null_empty_string(param)
+    def test_parse_with_null_empty_string
       parser = TextParser::TSVParser.new
       parser.configure(
         'keys'=>param,
+        'time_key'=>'time',
         'null_empty_string'=>true
       )
       parser.parse("\t ") do |time, record|
@@ -877,7 +657,7 @@ module ParserTest
       parser = TextParser::CSVParser.new
       parser.configure('keys' => param, 'time_key' => 'time')
       parser.parse("2013/02/28 12:00:00,192.168.0.1,111") { |time, record|
-        assert_equal_event_time(event_time('2013/02/28 12:00:00', format: '%Y/%m/%d %H:%M:%S'), time)
+        assert_equal(str2time('2013/02/28 12:00:00', '%Y/%m/%d %H:%M:%S'), time)
         assert_equal({
           'c' => '192.168.0.1',
           'd' => '111',
@@ -926,27 +706,29 @@ module ParserTest
     end
 
     data('array param' => '["a","b","c","d","e","f"]', 'string param' => 'a,b,c,d,e,f')
-    def test_parse_with_null_value_pattern(param)
+    def test_parse_with_null_value_pattern
       parser = TextParser::CSVParser.new
       parser.configure(
         'keys'=>param,
+        'time_key'=>'time',
         'null_value_pattern'=>'^(-|null|NULL)$'
       )
       parser.parse("-,null,NULL,,--,nuLL") do |time, record|
         assert_nil record['a']
         assert_nil record['b']
         assert_nil record['c']
-        assert_nil record['d']
+        assert_equal record['d'], ''
         assert_equal record['e'], '--'
         assert_equal record['f'], 'nuLL'
       end
     end
 
     data('array param' => '["a","b"]', 'string param' => 'a,b')
-    def test_parse_with_null_empty_string(param)
+    def test_parse_with_null_empty_string
       parser = TextParser::CSVParser.new
       parser.configure(
         'keys'=>param,
+        'time_key'=>'time',
         'null_empty_string'=>true
       )
       parser.parse(", ") do |time, record|
@@ -978,7 +760,7 @@ module ParserTest
       parser = TextParser::LabeledTSVParser.new
       parser.configure({})
       parser.parse("time:2013/02/28 12:00:00\thost:192.168.0.1\treq_id:111") { |time, record|
-        assert_equal_event_time(event_time('2013/02/28 12:00:00', format: '%Y/%m/%d %H:%M:%S'), time)
+        assert_equal(str2time('2013/02/28 12:00:00', '%Y/%m/%d %H:%M:%S'), time)
         assert_equal({
           'host'   => '192.168.0.1',
           'req_id' => '111',
@@ -993,7 +775,7 @@ module ParserTest
         'label_delimiter' => '=',
       )
       parser.parse('time=2013/02/28 12:00:00,host=192.168.0.1,req_id=111') { |time, record|
-        assert_equal_event_time(event_time('2013/02/28 12:00:00', format: '%Y/%m/%d %H:%M:%S'), time)
+        assert_equal(str2time('2013/02/28 12:00:00', '%Y/%m/%d %H:%M:%S'), time)
         assert_equal({
           'host'   => '192.168.0.1',
           'req_id' => '111',
@@ -1008,7 +790,7 @@ module ParserTest
         'time_format' => '%d/%b/%Y:%H:%M:%S %z',
       )
       parser.parse("mytime:28/Feb/2013:12:00:00 +0900\thost:192.168.0.1\treq_id:111") { |time, record|
-        assert_equal_event_time(event_time('28/Feb/2013:12:00:00 +0900', format: '%d/%b/%Y:%H:%M:%S %z'), time)
+        assert_equal(str2time('28/Feb/2013:12:00:00 +0900', '%d/%b/%Y:%H:%M:%S %z'), time)
         assert_equal({
           'host'   => '192.168.0.1',
           'req_id' => '111',
@@ -1093,7 +875,7 @@ module ParserTest
     end
 
     def test_parse
-      parser = TextParser::TEMPLATE_REGISTRY.lookup('none').call
+      parser = Fluent::Plugin.new_parser('none')
       parser.configure({})
       parser.parse('log message!') { |time, record|
         assert_equal({'message' => 'log message!'}, record)
@@ -1111,14 +893,14 @@ module ParserTest
     def test_parse_without_default_time
       time_at_start = Time.now.to_i
 
-      parser = TextParser::TEMPLATE_REGISTRY.lookup('none').call
+      parser = Fluent::Plugin.new_parser('none')
       parser.configure({})
       parser.parse('log message!') { |time, record|
         assert time && time >= time_at_start, "parser puts current time without time input"
         assert_equal({'message' => 'log message!'}, record)
       }
 
-      parser = TextParser::TEMPLATE_REGISTRY.lookup('none').call
+      parser = Fluent::Plugin.new_parser('none')
       parser.estimate_current_event = false
       parser.configure({})
       parser.parse('log message!') { |time, record|
@@ -1132,7 +914,7 @@ module ParserTest
     include ParserTest
 
     def create_parser(conf)
-      parser = TextParser::TEMPLATE_REGISTRY.lookup('multiline').call
+      parser = Fluent::Plugin.new_parser('multiline')
       parser.configure(conf)
       parser
     end
@@ -1153,7 +935,7 @@ javax.management.RuntimeErrorException: null
 \tat Main.main(Main.java:16) ~[bin/:na]
 EOS
 
-        assert_equal_event_time(event_time('2013-3-03 14:27:33'), time)
+        assert_equal(str2time('2013-3-03 14:27:33').to_i, time)
         assert_equal({
           "thread"  => "main",
           "level"   => "ERROR",
@@ -1171,7 +953,7 @@ message=test1
 EOS
 
         assert(parser.firstline?('----'))
-        assert_equal_event_time(event_time('2013-3-03 14:27:33'), time)
+        assert_equal(str2time('2013-3-03 14:27:33').to_i, time)
         assert_equal({"message" => "test1"}, record)
       }
     end
@@ -1193,7 +975,7 @@ Completed 200 OK in 4ms (Views: 3.2ms | ActiveRecord: 0.0ms)
 EOS
 
         assert(parser.firstline?('Started GET "/users/123/" for 127.0.0.1...'))
-        assert_equal_event_time(event_time('2013-06-14 12:00:11 +0900'), time)
+        assert_equal(str2time('2013-06-14 12:00:11 +0900').to_i, time)
         assert_equal({
           "method" => "GET",
           "path" => "/users/123/",
@@ -1245,7 +1027,7 @@ EOS
 
     def test_lookup_unknown_format
       assert_raise ConfigError do
-        TextParser::TEMPLATE_REGISTRY.lookup('unknown')
+        Fluent::Plugin.new_parser('unknown')
       end
     end
 
@@ -1253,7 +1035,7 @@ EOS
     def test_lookup_known_parser(data)
       $LOAD_PATH.unshift(File.join(File.expand_path(File.dirname(__FILE__)), 'scripts'))
       assert_nothing_raised ConfigError do
-        TextParser::TEMPLATE_REGISTRY.lookup(data)
+        Fluent::Plugin.new_parser(data)
       end
       $LOAD_PATH.shift
     end
@@ -1261,7 +1043,7 @@ EOS
     def test_parse_with_return
       parser = TextParser.new
       parser.configure('format' => 'none')
-      time, record = parser.parse('log message!')
+      _time, record = parser.parse('log message!')
       assert_equal({'message' => 'log message!'}, record)
     end