fluentd 0.12.43 → 0.14.0

data/test/plugin/test_out_file.rb

@@ -1,5 +1,6 @@
 require_relative '../helper'
 require 'fluent/test'
+require 'fluent/plugin/out_file'
 require 'fileutils'
 require 'time'
 
@@ -62,7 +63,7 @@ class FileOutputTest < Test::Unit::TestCase
     d = create_driver(%[path #{TMP_DIR}/out_file_test])
     time = Time.parse("2011-01-02 13:14:15 UTC").to_i
 
-    with_timezone('Asia/Taipei') do
+    with_timezone(Fluent.windows? ? 'NST-8' : 'Asia/Taipei') do
       d.emit({"a"=>1}, time)
       d.expect_format %[2011-01-02T21:14:15+08:00\ttest\t{"a":1}\n]
       d.run
@@ -121,7 +122,7 @@ class FileOutputTest < Test::Unit::TestCase
     # Zlib::GzipReader has a bug of concatenated file: https://bugs.ruby-lang.org/issues/9790
     # Following code from https://www.ruby-forum.com/topic/971591#979520
     result = ''
-    File.open(path) { |io|
+    File.open(path, "rb") { |io|
       loop do
         gzr = Zlib::GzipReader.new(io)
         result << gzr.read
@@ -150,6 +151,54 @@ class FileOutputTest < Test::Unit::TestCase
     check_gzipped_result(paths[0], %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n])
   end
 
+  class TestWithSystem < self
+    TMP_DIR_WITH_SYSTEM = File.expand_path(File.dirname(__FILE__) + "/../tmp/out_file_system#{ENV['TEST_ENV_NUMBER']}")
+    # 0750 interprets as "488". "488".to_i(8) # => 4. So, it makes wrong permission. Umm....
+    OVERRIDE_DIR_PERMISSION = 750
+    OVERRIDE_FILE_PERMISSION = 0620
+    CONFIG_WITH_SYSTEM = %[
+      path #{TMP_DIR_WITH_SYSTEM}/out_file_test
+      compress gz
+      utc
+      <system>
+        file_permission #{OVERRIDE_FILE_PERMISSION}
+        dir_permission #{OVERRIDE_DIR_PERMISSION}
+      </system>
+    ]
+
+    def setup
+      omit "NTFS doesn't support UNIX like permissions" if Fluent.windows?
+      FileUtils.rm_rf(TMP_DIR_WITH_SYSTEM)
+    end
+
+    def parse_system(text)
+      basepath = File.expand_path(File.dirname(__FILE__) + '/../../')
+      Fluent::Config.parse(text, '(test)', basepath, true).elements.find { |e| e.name == 'system' }
+    end
+
+    def test_write_with_system
+      system_conf = parse_system(CONFIG_WITH_SYSTEM)
+      sc = Fluent::SystemConfig.new(system_conf)
+      Fluent::Engine.init(sc)
+      d = create_driver CONFIG_WITH_SYSTEM
+
+      time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+      d.emit({"a"=>1}, time)
+      d.emit({"a"=>2}, time)
+
+      # FileOutput#write returns path
+      paths = d.run
+      expect_paths = ["#{TMP_DIR_WITH_SYSTEM}/out_file_test.20110102_0.log.gz"]
+      assert_equal expect_paths, paths
+
+      check_gzipped_result(paths[0], %[2011-01-02T13:14:15Z\ttest\t{"a":1}\n] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}\n])
+      dir_mode = "%o" % File::stat(TMP_DIR_WITH_SYSTEM).mode
+      assert_equal(OVERRIDE_DIR_PERMISSION, dir_mode[-3, 3].to_i)
+      file_mode = "%o" % File::stat(paths[0]).mode
+      assert_equal(OVERRIDE_FILE_PERMISSION, file_mode[-3, 3].to_i)
+    end
+  end
+
   def test_write_with_format_json
     d = create_driver [CONFIG, 'format json', 'include_time_key true', 'time_as_epoch'].join("\n")
 
@@ -245,28 +294,33 @@ class FileOutputTest < Test::Unit::TestCase
   end
 
   def test_write_with_symlink
+    omit "Windows doesn't support symlink" if Fluent.windows?
     conf = CONFIG + %[
       symlink_path #{SYMLINK_PATH}
     ]
     symlink_path = "#{SYMLINK_PATH}"
 
-    Fluent::FileBuffer.clear_buffer_paths
     d = Fluent::Test::TestDriver.new(Fluent::FileOutput).configure(conf)
 
     begin
       d.instance.start
       10.times { sleep 0.05 }
+
       time = Time.parse("2011-01-02 13:14:15 UTC").to_i
       es = Fluent::OneEventStream.new(time, {"a"=>1})
-      d.instance.emit('tag', es, Fluent::NullOutputChain.instance)
+      d.instance.emit_events('tag', es)
 
-      assert File.exists?(symlink_path)
+      assert File.exist?(symlink_path)
       assert File.symlink?(symlink_path)
 
-      d.instance.enqueue_buffer
+      es = Fluent::OneEventStream.new(event_time("2011-01-03 14:15:16 UTC"), {"a"=>2})
+      d.instance.emit_events('tag', es)
 
-      assert !File.exists?(symlink_path)
+      assert File.exist?(symlink_path)
       assert File.symlink?(symlink_path)
+
+      meta = d.instance.metadata('tag', event_time("2011-01-03 14:15:16 UTC"), {})
+      assert_equal d.instance.buffer.instance_eval{ @stage[meta].path }, File.readlink(symlink_path)
     ensure
       d.instance.shutdown
       FileUtils.rm_rf(symlink_path)

data/test/plugin/test_out_forward.rb

@@ -1,5 +1,6 @@
 require_relative '../helper'
 require 'fluent/test'
+require 'fluent/plugin/out_forward'
 
 class ForwardOutputTest < Test::Unit::TestCase
   def setup
@@ -10,6 +11,7 @@ class ForwardOutputTest < Test::Unit::TestCase
   TARGET_PORT = 13999
   CONFIG = %[
     send_timeout 51
+    heartbeat_type udp
     <server>
       name test
       host #{TARGET_HOST}
@@ -39,21 +41,20 @@ class ForwardOutputTest < Test::Unit::TestCase
         @exceptions << e
         raise e
       end
-
-      # Fluentd v0.12 BufferedOutputTestDriver calls this method.
-      # BufferedOutput#format_stream calls format method, but ForwardOutput#format is not defined.
-      # Because ObjectBufferedOutput#emit calls es.to_msgpack_stream directly.
-      def format_stream(tag, es)
-        es.to_msgpack_stream
-      end
     }.configure(conf)
   end
 
   def test_configure
-    d = create_driver
+    d = create_driver(%[
+      <server>
+        name test
+        host #{TARGET_HOST}
+        port #{TARGET_PORT}
+      </server>
+    ])
     nodes = d.instance.nodes
-    assert_equal 51, d.instance.send_timeout
-    assert_equal :udp, d.instance.heartbeat_type
+    assert_equal 60, d.instance.send_timeout
+    assert_equal :tcp, d.instance.heartbeat_type
     assert_equal 1, nodes.length
     node = nodes.first
     assert_equal "test", node.name
@@ -61,9 +62,9 @@ class ForwardOutputTest < Test::Unit::TestCase
     assert_equal 13999, node.port
   end
 
-  def test_configure_tcp_heartbeat
-    d = create_driver(CONFIG + "\nheartbeat_type tcp")
-    assert_equal :tcp, d.instance.heartbeat_type
+  def test_configure_udp_heartbeat
+    d = create_driver(CONFIG + "\nheartbeat_type udp")
+    assert_equal :udp, d.instance.heartbeat_type
   end
 
   def test_configure_none_heartbeat
@@ -119,6 +120,75 @@ class ForwardOutputTest < Test::Unit::TestCase
     assert_equal 2, d.instance.ack_response_timeout
   end
 
+  def test_send_with_time_as_integer
+    target_input_driver = create_target_input_driver
+
+    d = create_driver(CONFIG + %[flush_interval 1s])
+
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+
+    records = [
+      {"a" => 1},
+      {"a" => 2}
+    ]
+    d.register_run_post_condition do
+      d.instance.responses.length == 1
+    end
+
+    target_input_driver.run do
+      d.run do
+        records.each do |record|
+          d.emit record, time
+        end
+      end
+    end
+
+    emits = target_input_driver.emits
+    assert_equal ['test', time, records[0]], emits[0]
+    assert_equal ['test', time, records[1]], emits[1]
+    assert(emits[0][1].is_a?(Integer))
+    assert(emits[1][1].is_a?(Integer))
+
+    assert_equal [nil], d.instance.responses # not attempt to receive responses, so nil is returned
+    assert_empty d.instance.exceptions
+  end
+
+  def test_send_without_time_as_integer
+    target_input_driver = create_target_input_driver
+
+    d = create_driver(CONFIG + %[
+      flush_interval 1s
+      time_as_integer false
+    ])
+
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+
+    records = [
+      {"a" => 1},
+      {"a" => 2}
+    ]
+    d.register_run_post_condition do
+      d.instance.responses.length == 1
+    end
+
+    target_input_driver.run do
+      d.run do
+        records.each do |record|
+          d.emit record, time
+        end
+      end
+    end
+
+    emits = target_input_driver.emits
+    assert_equal ['test', time, records[0]], emits[0]
+    assert_equal ['test', time, records[1]], emits[1]
+    assert_equal_event_time(time, emits[0][1])
+    assert_equal_event_time(time, emits[1][1])
+
+    assert_equal [nil], d.instance.responses # not attempt to receive responses, so nil is returned
+    assert_empty d.instance.exceptions
+  end
+
   def test_send_to_a_node_supporting_responses
     target_input_driver = create_target_input_driver(true)
 
@@ -305,6 +375,7 @@ class ForwardOutputTest < Test::Unit::TestCase
   def create_target_input_driver(do_respond=false, disconnect=false, conf=TARGET_CONFIG)
     require 'fluent/plugin/in_forward'
 
+    # TODO: Support actual TCP heartbeat test
     DummyEngineDriver.new(Fluent::ForwardInput) {
       handler_class = Class.new(Fluent::ForwardInput::Handler) { |klass|
         attr_reader :chunk_counter # for checking if received data is successfully deserialized
@@ -314,12 +385,14 @@ class ForwardOutputTest < Test::Unit::TestCase
           @log = log
           @chunk_counter = 0
           @on_message = on_message
+          @source = nil
         end
 
         if do_respond
           def write(data)
             @sock.write data
-          rescue => e
+          rescue
+            @sock.close_write
             @sock.close
           end
         else
@@ -338,7 +411,7 @@ class ForwardOutputTest < Test::Unit::TestCase
 
       define_method(:start) do
         @thread = Thread.new do
-          Socket.tcp_server_loop(@host, @port) do |sock, client_addrinfo|
+          Socket.tcp_server_loop(@bind, @port) do |sock, client_addrinfo|
             begin
               handler = handler_class.new(sock, @log, method(:on_message))
               loop do
@@ -416,6 +489,10 @@ class ForwardOutputTest < Test::Unit::TestCase
         @emit_streams ||= []
       end
 
+      def clear!
+        @emit_streams = []
+      end
+
       def emit_stream(tag, es)
         @emit_streams << [tag, es.to_a]
       end

data/test/plugin/test_out_roundrobin.rb

@@ -1,5 +1,6 @@
 require_relative '../helper'
 require 'fluent/test'
+require 'fluent/plugin/out_roundrobin'
 
 class RoundRobinOutputTest < Test::Unit::TestCase
   class << self

data/test/plugin/test_out_stdout.rb

@@ -1,5 +1,6 @@
 require_relative '../helper'
-require 'fluent/test'
+require 'fluent/test/driver/output'
+require 'fluent/plugin/out_stdout'
 
 class StdoutOutputTest < Test::Unit::TestCase
   def setup
@@ -10,7 +11,7 @@ class StdoutOutputTest < Test::Unit::TestCase
   ]
 
   def create_driver(conf = CONFIG)
-    Fluent::Test::OutputTestDriver.new(Fluent::StdoutOutput).configure(conf)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::StdoutOutput).configure(conf)
   end
 
   def test_configure
@@ -33,28 +34,36 @@ class StdoutOutputTest < Test::Unit::TestCase
   data('oj' => 'oj', 'yajl' => 'yajl')
   def test_emit_json(data)
     d = create_driver(CONFIG + "\noutput_type json\njson_parser #{data}")
-    time = Time.now
-    out = capture_log { d.emit({'test' => 'test'}, time) }
-    assert_equal "#{time.localtime} test: {\"test\":\"test\"}\n", out
+    time = event_time()
+    out = capture_log do
+      d.run(default_tag: 'test') do
+        d.feed(time, {'test' => 'test'})
+      end
+    end
+    assert_equal "#{Time.at(time).localtime} test: {\"test\":\"test\"}\n", out
 
     if data == 'yajl'
       # NOTE: Float::NAN is not jsonable
-      assert_raise(Yajl::EncodeError) { d.emit({'test' => Float::NAN}, time) }
+      assert_raise(Yajl::EncodeError) { d.feed('test', time, {'test' => Float::NAN}) }
     else
-      out = capture_log { d.emit({'test' => Float::NAN}, time) }
-      assert_equal "#{time.localtime} test: {\"test\":NaN}\n", out
+      out = capture_log { d.feed('test', time, {'test' => Float::NAN}) }
+      assert_equal "#{Time.at(time).localtime} test: {\"test\":NaN}\n", out
     end
   end
 
   def test_emit_hash
     d = create_driver(CONFIG + "\noutput_type hash")
-    time = Time.now
-    out = capture_log { d.emit({'test' => 'test'}, time) }
-    assert_equal "#{time.localtime} test: {\"test\"=>\"test\"}\n", out
+    time = event_time()
+    out = capture_log do
+      d.run(default_tag: 'test') do
+        d.feed(time, {'test' => 'test'})
+      end
+    end
+    assert_equal "#{Time.at(time).localtime} test: {\"test\"=>\"test\"}\n", out
 
     # NOTE: Float::NAN is not jsonable, but hash string can output it.
-    out = capture_log { d.emit({'test' => Float::NAN}, time) }
-    assert_equal "#{time.localtime} test: {\"test\"=>NaN}\n", out
+    out = capture_log { d.feed('test', time, {'test' => Float::NAN}) }
+    assert_equal "#{Time.at(time).localtime} test: {\"test\"=>NaN}\n", out
   end
 
   private

data/test/plugin/test_out_stream.rb

@@ -1,5 +1,6 @@
 require_relative '../helper'
 require 'fluent/test'
+require 'fluent/plugin/out_stream'
 
 module StreamOutputTest
   def setup
@@ -22,6 +23,23 @@ module StreamOutputTest
     assert_equal(expect, result)
   end
 
+  def test_write_event_time
+    d = create_driver
+
+    time = Fluent::EventTime.parse("2011-01-02 13:14:15 UTC")
+    d.emit({"a"=>1}, time)
+    d.emit({"a"=>2}, time)
+
+    expect = ["test",
+        Fluent::Engine.msgpack_factory.packer.write([time,{"a"=>1}]).to_s +
+        Fluent::Engine.msgpack_factory.packer.write([time,{"a"=>2}]).to_s
+      ]
+    expect = Fluent::Engine.msgpack_factory.packer.write(expect).to_s
+
+    result = d.run
+    assert_equal(expect, result)
+  end
+
   def create_driver(klass, conf)
     Fluent::Test::BufferedOutputTestDriver.new(klass) do
       def write(chunk)

data/test/plugin/test_output.rb

@@ -0,0 +1,515 @@
+require_relative '../helper'
+require 'fluent/plugin/output'
+require 'fluent/plugin/buffer'
+require 'fluent/event'
+
+require 'json'
+require 'time'
+require 'timeout'
+
+module FluentPluginOutputTest
+  class DummyBareOutput < Fluent::Plugin::Output
+    def register(name, &block)
+      instance_variable_set("@#{name}", block)
+    end
+  end
+  class DummySyncOutput < DummyBareOutput
+    def initialize
+      super
+      @process = nil
+    end
+    def process(tag, es)
+      @process ? @process.call(tag, es) : nil
+    end
+  end
+  class DummyAsyncOutput < DummyBareOutput
+    def initialize
+      super
+      @format = nil
+      @write = nil
+    end
+    def format(tag, time, record)
+      @format ? @format.call(tag, time, record) : [tag, time, record].to_json
+    end
+    def write(chunk)
+      @write ? @write.call(chunk) : nil
+    end
+  end
+  class DummyAsyncStandardOutput < DummyBareOutput
+    def initialize
+      super
+      @write = nil
+    end
+    def write(chunk)
+      @write ? @write.call(chunk) : nil
+    end
+  end
+  class DummyDelayedOutput < DummyBareOutput
+    def initialize
+      super
+      @format = nil
+      @try_write = nil
+    end
+    def format(tag, time, record)
+      @format ? @format.call(tag, time, record) : [tag, time, record].to_json
+    end
+    def try_write(chunk)
+      @try_write ? @try_write.call(chunk) : nil
+    end
+  end
+  class DummyDelayedStandardOutput < DummyBareOutput
+    def initialize
+      super
+      @try_write = nil
+    end
+    def try_write(chunk)
+      @try_write ? @try_write.call(chunk) : nil
+    end
+  end
+  class DummyFullFeatureOutput < DummyBareOutput
+    def initialize
+      super
+      @prefer_buffered_processing = nil
+      @prefer_delayed_commit = nil
+      @process = nil
+      @format = nil
+      @write = nil
+      @try_write = nil
+    end
+    def prefer_buffered_processing
+      @prefer_buffered_processing ? @prefer_buffered_processing.call : false
+    end
+    def prefer_delayed_commit
+      @prefer_delayed_commit ? @prefer_delayed_commit.call : false
+    end
+    def process(tag, es)
+      @process ? @process.call(tag, es) : nil
+    end
+    def format(tag, time, record)
+      @format ? @format.call(tag, time, record) : [tag, time, record].to_json
+    end
+    def write(chunk)
+      @write ? @write.call(chunk) : nil
+    end
+    def try_write(chunk)
+      @try_write ? @try_write.call(chunk) : nil
+    end
+  end
+end
+
+class OutputTest < Test::Unit::TestCase
+  def create_output(type=:full)
+    case type
+    when :bare     then FluentPluginOutputTest::DummyBareOutput.new
+    when :sync     then FluentPluginOutputTest::DummySyncOutput.new
+    when :buffered then FluentPluginOutputTest::DummyAsyncOutput.new
+    when :standard then FluentPluginOutputTest::DummyAsyncStandardOutput.new
+    when :delayed  then FluentPluginOutputTest::DummyDelayedOutput.new
+    when :sdelayed then FluentPluginOutputTest::DummyDelayedStandardOutput.new
+    when :full     then FluentPluginOutputTest::DummyFullFeatureOutput.new
+    else
+      raise ArgumentError, "unknown type: #{type}"
+    end
+  end
+  def create_metadata(timekey: nil, tag: nil, variables: nil)
+    Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)
+  end
+  def waiting(seconds)
+    begin
+      Timeout.timeout(seconds) do
+        yield
+      end
+    rescue Timeout::Error
+      STDERR.print(*@i.log.out.logs)
+      raise
+    end
+  end
+
+  sub_test_case 'basic output feature' do
+    setup do
+      @i = create_output(:full)
+    end
+
+    test '#implement? can return features for plugin instances' do
+      i1 = FluentPluginOutputTest::DummyBareOutput.new
+      assert !i1.implement?(:synchronous)
+      assert !i1.implement?(:buffered)
+      assert !i1.implement?(:delayed_commit)
+      assert !i1.implement?(:custom_format)
+
+      i2 = FluentPluginOutputTest::DummySyncOutput.new
+      assert i2.implement?(:synchronous)
+      assert !i2.implement?(:buffered)
+      assert !i2.implement?(:delayed_commit)
+      assert !i2.implement?(:custom_format)
+
+      i3 = FluentPluginOutputTest::DummyAsyncOutput.new
+      assert !i3.implement?(:synchronous)
+      assert i3.implement?(:buffered)
+      assert !i3.implement?(:delayed_commit)
+      assert i3.implement?(:custom_format)
+
+      i4 = FluentPluginOutputTest::DummyAsyncStandardOutput.new
+      assert !i4.implement?(:synchronous)
+      assert i4.implement?(:buffered)
+      assert !i4.implement?(:delayed_commit)
+      assert !i4.implement?(:custom_format)
+
+      i5 = FluentPluginOutputTest::DummyDelayedOutput.new
+      assert !i5.implement?(:synchronous)
+      assert !i5.implement?(:buffered)
+      assert i5.implement?(:delayed_commit)
+      assert i5.implement?(:custom_format)
+
+      i6 = FluentPluginOutputTest::DummyDelayedStandardOutput.new
+      assert !i6.implement?(:synchronous)
+      assert !i6.implement?(:buffered)
+      assert i6.implement?(:delayed_commit)
+      assert !i6.implement?(:custom_format)
+
+      i6 = FluentPluginOutputTest::DummyFullFeatureOutput.new
+      assert i6.implement?(:synchronous)
+      assert i6.implement?(:buffered)
+      assert i6.implement?(:delayed_commit)
+      assert i6.implement?(:custom_format)
+    end
+
+    test 'plugin lifecycle for configure/start/stop/before_shutdown/shutdown/after_shutdown/close/terminate' do
+      assert !@i.configured?
+      @i.configure(config_element())
+      assert @i.configured?
+      assert !@i.started?
+      @i.start
+      assert @i.started?
+      assert !@i.stopped?
+      @i.stop
+      assert @i.stopped?
+      assert !@i.before_shutdown?
+      @i.before_shutdown
+      assert @i.before_shutdown?
+      assert !@i.shutdown?
+      @i.shutdown
+      assert @i.shutdown?
+      assert !@i.after_shutdown?
+      @i.after_shutdown
+      assert @i.after_shutdown?
+      assert !@i.closed?
+      @i.close
+      assert @i.closed?
+      assert !@i.terminated?
+      @i.terminate
+      assert @i.terminated?
+    end
+
+    test '#extract_placeholders does nothing if chunk key is not specified' do
+      @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
+      assert !@i.chunk_key_time
+      assert !@i.chunk_key_tag
+      assert_equal [], @i.chunk_keys
+      tmpl = "/mypath/%Y/%m/%d/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail"
+      t = event_time('2016-04-11 20:30:00 +0900')
+      v = {key1: "value1", key2: "value2"}
+      m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
+      assert_equal tmpl, @i.extract_placeholders(tmpl, m)
+    end
+
+    test '#extract_placeholders can extract time if time key and range are configured' do
+      @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time', {'timekey' => 60*30, 'timekey_zone' => "+0900"})]))
+      assert @i.chunk_key_time
+      assert !@i.chunk_key_tag
+      assert_equal [], @i.chunk_keys
+      tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail"
+      t = event_time('2016-04-11 20:30:00 +0900')
+      v = {key1: "value1", key2: "value2"}
+      m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
+      assert_equal "/mypath/2016/04/11/20-30/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail", @i.extract_placeholders(tmpl, m)
+    end
+
+    test '#extract_placeholders can extract tag and parts of tag if tag is configured' do
+      @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'tag', {})]))
+      assert !@i.chunk_key_time
+      assert @i.chunk_key_tag
+      assert_equal [], @i.chunk_keys
+      tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail"
+      t = event_time('2016-04-11 20:30:00 +0900')
+      v = {key1: "value1", key2: "value2"}
+      m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
+      assert_equal "/mypath/%Y/%m/%d/%H-%M/fluentd.test.output/test/output/${key1}/${key2}/tail", @i.extract_placeholders(tmpl, m)
+    end
+
+    test '#extract_placeholders can extract variables if variables are configured' do
+      @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'key1,key2', {})]))
+      assert !@i.chunk_key_time
+      assert !@i.chunk_key_tag
+      assert_equal ['key1','key2'], @i.chunk_keys
+      tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail"
+      t = event_time('2016-04-11 20:30:00 +0900')
+      v = {key1: "value1", key2: "value2"}
+      m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
+      assert_equal "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/value1/value2/tail", @i.extract_placeholders(tmpl, m)
+    end
+
+    test '#extract_placeholders can extract all chunk keys if configured' do
+      @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time,tag,key1,key2', {'timekey' => 60*30, 'timekey_zone' => "+0900"})]))
+      assert @i.chunk_key_time
+      assert @i.chunk_key_tag
+      assert_equal ['key1','key2'], @i.chunk_keys
+      tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[1]}/${tag[2]}/${key1}/${key2}/tail"
+      t = event_time('2016-04-11 20:30:00 +0900')
+      v = {key1: "value1", key2: "value2"}
+      m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
+      assert_equal "/mypath/2016/04/11/20-30/fluentd.test.output/test/output/value1/value2/tail", @i.extract_placeholders(tmpl, m)
+    end
+
+    test '#extract_placeholders removes out-of-range tag part and unknown variable placeholders' do
+      @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time,tag,key1,key2', {'timekey' => 60*30, 'timekey_zone' => "+0900"})]))
+      assert @i.chunk_key_time
+      assert @i.chunk_key_tag
+      assert_equal ['key1','key2'], @i.chunk_keys
+      tmpl = "/mypath/%Y/%m/%d/%H-%M/${tag}/${tag[3]}/${tag[4]}/${key3}/${key4}/tail"
+      t = event_time('2016-04-11 20:30:00 +0900')
+      v = {key1: "value1", key2: "value2"}
+      m = create_metadata(timekey: t, tag: 'fluentd.test.output', variables: v)
+      assert_equal "/mypath/2016/04/11/20-30/fluentd.test.output/////tail", @i.extract_placeholders(tmpl, m)
+    end
+
+    test '#metadata returns object which contains tag/timekey/variables from records as specified in configuration' do
+      tag = 'test.output'
+      time = event_time('2016-04-12 15:31:23 -0700')
+      timekey = event_time('2016-04-12 15:00:00 -0700')
+      record = {"key1" => "value1", "num1" => 1, "message" => "my message"}
+
+      i1 = create_output(:buffered)
+      i1.configure(config_element('ROOT','',{},[config_element('buffer', '')]))
+      assert_equal create_metadata(), i1.metadata(tag, time, record)
+
+      i2 = create_output(:buffered)
+      i2.configure(config_element('ROOT','',{},[config_element('buffer', 'tag')]))
+      assert_equal create_metadata(tag: tag), i2.metadata(tag, time, record)
+
+      i3 = create_output(:buffered)
+      i3.configure(config_element('ROOT','',{},[config_element('buffer', 'time', {"timekey" => 3600, "timekey_zone" => "-0700"})]))
+      assert_equal create_metadata(timekey: timekey), i3.metadata(tag, time, record)
+
+      i4 = create_output(:buffered)
+      i4.configure(config_element('ROOT','',{},[config_element('buffer', 'key1', {})]))
+      assert_equal create_metadata(variables: {key1: "value1"}), i4.metadata(tag, time, record)
+
+      i5 = create_output(:buffered)
+      i5.configure(config_element('ROOT','',{},[config_element('buffer', 'key1,num1', {})]))
+      assert_equal create_metadata(variables: {key1: "value1", num1: 1}), i5.metadata(tag, time, record)
+
+      i6 = create_output(:buffered)
+      i6.configure(config_element('ROOT','',{},[config_element('buffer', 'tag,time', {"timekey" => 3600, "timekey_zone" => "-0700"})]))
+      assert_equal create_metadata(timekey: timekey, tag: tag), i6.metadata(tag, time, record)
+
+      i7 = create_output(:buffered)
+      i7.configure(config_element('ROOT','',{},[config_element('buffer', 'tag,num1', {"timekey" => 3600, "timekey_zone" => "-0700"})]))
+      assert_equal create_metadata(tag: tag, variables: {num1: 1}), i7.metadata(tag, time, record)
+
+      i8 = create_output(:buffered)
+      i8.configure(config_element('ROOT','',{},[config_element('buffer', 'time,tag,key1', {"timekey" => 3600, "timekey_zone" => "-0700"})]))
+      assert_equal create_metadata(timekey: timekey, tag: tag, variables: {key1: "value1"}), i8.metadata(tag, time, record)
+    end
+
+    test '#emit calls #process via #emit_sync for non-buffered output' do
+      i = create_output(:sync)
+      process_called = false
+      i.register(:process){|tag, es| process_called = true }
+      i.configure(config_element())
+      i.start
+
+      t = event_time()
+      i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
+
+      assert process_called
+
+      i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
+    end
+
+    test '#emit calls #format for buffered output' do
+      i = create_output(:buffered)
+      format_called_times = 0
+      i.register(:format){|tag, time, record| format_called_times += 1; '' }
+      i.configure(config_element())
+      i.start
+
+      t = event_time()
+      i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
+
+      assert_equal 2, format_called_times
+
+      i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
+    end
+
+    test '#prefer_buffered_processing (returns false) decides non-buffered without <buffer> section' do
+      i = create_output(:full)
+
+      process_called = false
+      format_called_times = 0
+      i.register(:process){|tag, es| process_called = true }
+      i.register(:format){|tag, time, record| format_called_times += 1; '' }
+
+      i.configure(config_element())
+      i.register(:prefer_buffered_processing){ false } # delayed decision is possible to change after (output's) configure
+      i.start
+
+      assert !i.prefer_buffered_processing
+
+      t = event_time()
+      i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
+
+      waiting(4){ Thread.pass until process_called }
+
+      assert process_called
+      assert_equal 0, format_called_times
+
+      i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
+    end
+
+    test '#prefer_buffered_processing (returns true) decides buffered without <buffer> section' do
+      i = create_output(:full)
+
+      process_called = false
+      format_called_times = 0
+      i.register(:process){|tag, es| process_called = true }
+      i.register(:format){|tag, time, record| format_called_times += 1; '' }
+
+      i.configure(config_element())
+      i.register(:prefer_buffered_processing){ true } # delayed decision is possible to change after (output's) configure
+      i.start
+
+      assert i.prefer_buffered_processing
+
+      t = event_time()
+      i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
+
+      assert !process_called
+      assert_equal 2, format_called_times
+
+      i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
+    end
+
+    test 'output plugin will call #write for normal buffered plugin to flush buffer chunks' do
+      i = create_output(:buffered)
+      write_called = false
+      i.register(:write){ |chunk| write_called = true }
+
+      i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {"flush_mode" => "immediate"})]))
+      i.start
+
+      t = event_time()
+      i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
+      i.force_flush
+
+      waiting(4){ Thread.pass until write_called }
+
+      assert write_called
+
+      i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
+    end
+
+    test 'output plugin will call #try_write for plugin supports delayed commit only to flush buffer chunks' do
+      i = create_output(:delayed)
+      try_write_called = false
+      i.register(:try_write){|chunk| try_write_called = true; commit_write(chunk.unique_id) }
+
+      i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {"flush_mode" => "immediate"})]))
+      i.start
+
+      t = event_time()
+      i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
+      i.force_flush
+
+      waiting(4){ Thread.pass until try_write_called }
+
+      assert try_write_called
+
+      i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
+    end
+
+    test '#prefer_delayed_commit (returns false) decides delayed commit is disabled if both are implemented' do
+      i = create_output(:full)
+      write_called = false
+      try_write_called = false
+      i.register(:write){ |chunk| write_called = true }
+      i.register(:try_write){|chunk| try_write_called = true; commit_write(chunk.unique_id) }
+
+      i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {"flush_mode" => "immediate"})]))
+      i.register(:prefer_delayed_commit){ false } # delayed decision is possible to change after (output's) configure
+      i.start
+
+      assert !i.prefer_delayed_commit
+
+      t = event_time()
+      i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
+      i.force_flush
+
+      waiting(4){ Thread.pass until write_called || try_write_called }
+
+      assert write_called
+      assert !try_write_called
+
+      i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
+    end
+
+    test '#prefer_delayed_commit (returns true) decides delayed commit is enabled if both are implemented' do
+      i = create_output(:full)
+      write_called = false
+      try_write_called = false
+      i.register(:write){ |chunk| write_called = true }
+      i.register(:try_write){|chunk| try_write_called = true; commit_write(chunk.unique_id) }
+
+      i.configure(config_element('ROOT', '', {}, [config_element('buffer', '', {"flush_mode" => "immediate"})]))
+      i.register(:prefer_delayed_commit){ true } # delayed decision is possible to change after (output's) configure
+      i.start
+
+      assert i.prefer_delayed_commit
+
+      t = event_time()
+      i.emit_events('tag', Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ]))
+      i.force_flush
+
+      waiting(4){ Thread.pass until write_called || try_write_called }
+
+      assert !write_called
+      assert try_write_called
+
+      i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
+    end
+  end
+
+  sub_test_case 'sync output feature' do
+    setup do
+      @i = create_output(:sync)
+    end
+
+    test 'raises configuration error if <buffer> section is specified' do
+      assert_raise Fluent::ConfigError do
+        @i.configure(config_element('ROOT','',{},[config_element('buffer', '')]))
+      end
+    end
+
+    test 'raises configuration error if <secondary> section is specified' do
+      assert_raise Fluent::ConfigError do
+        @i.configure(config_element('ROOT','',{},[config_element('secondary','')]))
+      end
+    end
+
+    test '#process is called for each event streams' do
+      ary = []
+      @i.register(:process){|tag, es| ary << [tag, es] }
+      @i.configure(config_element())
+      @i.start
+
+      t = event_time()
+      es = Fluent::ArrayEventStream.new([ [t, {"key" => "value1"}], [t, {"key" => "value2"}] ])
+      5.times do
+        @i.emit_events('tag', es)
+      end
+      assert_equal 5, ary.size
+
+      @i.stop; @i.before_shutdown; @i.shutdown; @i.after_shutdown; @i.close; @i.terminate
+    end
+  end
+end