fluentd-hubspot 0.14.14.1

data/test/plugin/test_bare_output.rb

@@ -0,0 +1,118 @@
+require_relative '../helper'
+require 'fluent/plugin/bare_output'
+require 'fluent/event'
+
+module FluentPluginBareOutputTest
+  class DummyPlugin < Fluent::Plugin::BareOutput
+    attr_reader :store
+    def initialize
+      super
+      @store = []
+    end
+    def process(tag, es)
+      es.each do |time, record|
+        @store << [tag, time, record]
+      end
+    end
+  end
+end
+
+class BareOutputTest < Test::Unit::TestCase
+  setup do
+    Fluent::Test.setup
+    @p = FluentPluginBareOutputTest::DummyPlugin.new
+  end
+
+  test 'has healthy lifecycle' do
+    assert !@p.configured?
+    @p.configure(config_element())
+    assert @p.configured?
+
+    assert !@p.started?
+    @p.start
+    assert @p.start
+
+    assert !@p.stopped?
+    @p.stop
+    assert @p.stopped?
+
+    assert !@p.before_shutdown?
+    @p.before_shutdown
+    assert @p.before_shutdown?
+
+    assert !@p.shutdown?
+    @p.shutdown
+    assert @p.shutdown?
+
+    assert !@p.after_shutdown?
+    @p.after_shutdown
+    assert @p.after_shutdown?
+
+    assert !@p.closed?
+    @p.close
+    assert @p.closed?
+
+    assert !@p.terminated?
+    @p.terminate
+    assert @p.terminated?
+  end
+
+  test 'has plugin_id automatically generated' do
+    assert @p.respond_to?(:plugin_id_configured?)
+    assert @p.respond_to?(:plugin_id)
+
+    @p.configure(config_element())
+
+    assert !@p.plugin_id_configured?
+    assert @p.plugin_id
+    assert{ @p.plugin_id != 'mytest' }
+  end
+
+  test 'has plugin_id manually configured' do
+    @p.configure(config_element('ROOT', '', {'@id' => 'mytest'}))
+    assert @p.plugin_id_configured?
+    assert_equal 'mytest', @p.plugin_id
+  end
+
+  test 'has plugin logger' do
+    assert @p.respond_to?(:log)
+    assert @p.log
+
+    # default logger
+    original_logger = @p.log
+
+    @p.configure(config_element('ROOT', '', {'@log_level' => 'debug'}))
+
+    assert{ @p.log.object_id != original_logger.object_id }
+    assert_equal Fluent::Log::LEVEL_DEBUG, @p.log.level
+  end
+
+  test 'can load plugin helpers' do
+    assert_nothing_raised do
+      class FluentPluginBareOutputTest::DummyPlugin2 < Fluent::Plugin::BareOutput
+        helpers :storage
+      end
+    end
+  end
+
+  test 'can get input event stream to write' do
+    @p.configure(config_element('ROOT'))
+    @p.start
+
+    es1 = Fluent::OneEventStream.new(event_time('2016-05-21 18:37:31 +0900'), {'k1' => 'v1'})
+    es2 = Fluent::ArrayEventStream.new([
+        [event_time('2016-05-21 18:38:33 +0900'), {'k2' => 'v2'}],
+        [event_time('2016-05-21 18:39:10 +0900'), {'k3' => 'v3'}],
+      ])
+    @p.emit_events('mytest1', es1)
+    @p.emit_events('mytest2', es2)
+
+    all_events = [
+      ['mytest1', event_time('2016-05-21 18:37:31 +0900'), {'k1' => 'v1'}],
+      ['mytest2', event_time('2016-05-21 18:38:33 +0900'), {'k2' => 'v2'}],
+      ['mytest2', event_time('2016-05-21 18:39:10 +0900'), {'k3' => 'v3'}],
+    ]
+
+    assert_equal all_events, @p.store
+  end
+end

data/test/plugin/test_base.rb

@@ -0,0 +1,115 @@
+require_relative '../helper'
+require 'fluent/plugin/base'
+
+module FluentPluginBaseTest
+  class DummyPlugin < Fluent::Plugin::Base
+  end
+end
+
+class BaseTest < Test::Unit::TestCase
+  setup do
+    @p = FluentPluginBaseTest::DummyPlugin.new
+  end
+
+  test 'has methods for phases of plugin life cycle, and methods to know "super"s were correctly called or not' do
+    assert !@p.configured?
+    @p.configure(config_element())
+    assert @p.configured?
+
+    assert !@p.started?
+    @p.start
+    assert @p.start
+
+    assert !@p.stopped?
+    @p.stop
+    assert @p.stopped?
+
+    assert !@p.before_shutdown?
+    @p.before_shutdown
+    assert @p.before_shutdown?
+
+    assert !@p.shutdown?
+    @p.shutdown
+    assert @p.shutdown?
+
+    assert !@p.after_shutdown?
+    @p.after_shutdown
+    assert @p.after_shutdown?
+
+    assert !@p.closed?
+    @p.close
+    assert @p.closed?
+
+    assert !@p.terminated?
+    @p.terminate
+    assert @p.terminated?
+  end
+
+  test 'can access system config' do
+    assert @p.system_config
+
+    @p.system_config_override({'process_name' => 'mytest'})
+    assert_equal 'mytest', @p.system_config.process_name
+  end
+
+  test 'does not have router in default' do
+    assert !@p.has_router?
+  end
+
+  sub_test_case '#fluentd_worker_id' do
+    test 'returns 0 in default' do
+      assert_equal 0, @p.fluentd_worker_id
+    end
+
+    test 'returns the value specified via SERVERENGINE_WORKER_ID env variable' do
+      pre_value = ENV['SERVERENGINE_WORKER_ID']
+      begin
+        ENV['SERVERENGINE_WORKER_ID'] = 7.to_s
+        assert_equal 7, @p.fluentd_worker_id
+      ensure
+        ENV['SERVERENGINE_WORKER_ID'] = pre_value
+      end
+    end
+  end
+
+  test 'does not have root dir in default' do
+    assert_nil @p.plugin_root_dir
+  end
+
+  test 'is configurable by config_param and config_section' do
+    assert_nothing_raised do
+      class FluentPluginBaseTest::DummyPlugin2 < Fluent::Plugin::TestBase
+        config_param :myparam1, :string
+        config_section :mysection, multi: false do
+          config_param :myparam2, :integer
+        end
+      end
+    end
+    p2 = FluentPluginBaseTest::DummyPlugin2.new
+    assert_nothing_raised do
+      p2.configure(config_element('ROOT', '', {'myparam1' => 'myvalue1'}, [config_element('mysection', '', {'myparam2' => 99})]))
+    end
+    assert_equal 'myvalue1', p2.myparam1
+    assert_equal 99, p2.mysection.myparam2
+  end
+
+  test 'plugins are available with multi worker configuration in default' do
+    assert @p.multi_workers_ready?
+  end
+
+  test 'provides #string_safe_encoding to scrub invalid sequence string with info logging' do
+    logger = Fluent::Test::TestLogger.new
+    m = Module.new do
+      define_method(:log) do
+        logger
+      end
+    end
+    @p.extend m
+    assert_equal [], logger.logs
+
+    ret = @p.string_safe_encoding("abc\xff.\x01f"){|s| s.split(/\./) }
+    assert_equal ['abc?', "\u0001f"], ret
+    assert_equal 1, logger.logs.size
+    assert{ logger.logs.first.include?("invalid byte sequence is replaced in ") }
+  end
+end

data/test/plugin/test_buf_file.rb

@@ -0,0 +1,843 @@
+require_relative '../helper'
+require 'fluent/plugin/buf_file'
+require 'fluent/plugin/output'
+require 'fluent/unique_id'
+require 'fluent/system_config'
+require 'fluent/env'
+
+require 'msgpack'
+
+module FluentPluginFileBufferTest
+  class DummyOutputPlugin < Fluent::Plugin::Output
+    Fluent::Plugin.register_output('buffer_file_test_output', self)
+    config_section :buffer do
+      config_set_default :@type, 'file'
+    end
+    def multi_workers_ready?
+      true
+    end
+    def write(chunk)
+      # drop
+    end
+  end
+end
+
+class FileBufferTest < Test::Unit::TestCase
+  def metadata(timekey: nil, tag: nil, variables: nil)
+    Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)
+  end
+
+  def write_metadata(path, chunk_id, metadata, size, ctime, mtime)
+    metadata = {
+      timekey: metadata.timekey, tag: metadata.tag, variables: metadata.variables,
+      id: chunk_id,
+      s: size,
+      c: ctime,
+      m: mtime,
+    }
+    File.open(path, 'wb') do |f|
+      f.write metadata.to_msgpack
+    end
+  end
+
+  sub_test_case 'non configured buffer plugin instance' do
+    setup do
+      Fluent::Test.setup
+
+      @dir = File.expand_path('../../tmp/buffer_file_dir', __FILE__)
+      FileUtils.rm_rf @dir
+      FileUtils.mkdir_p @dir
+    end
+
+    test 'path should include * normally' do
+      d = FluentPluginFileBufferTest::DummyOutputPlugin.new
+      p = Fluent::Plugin::FileBuffer.new
+      p.owner = d
+      p.configure(config_element('buffer', '', {'path' => File.join(@dir, 'buffer.*.file')}))
+      assert_equal File.join(@dir, 'buffer.*.file'), p.path
+    end
+
+    test 'existing directory will be used with additional default file name' do
+      d = FluentPluginFileBufferTest::DummyOutputPlugin.new
+      p = Fluent::Plugin::FileBuffer.new
+      p.owner = d
+      p.configure(config_element('buffer', '', {'path' => @dir}))
+      assert_equal File.join(@dir, 'buffer.*.log'), p.path
+    end
+
+    test 'unexisting path without * handled as directory' do
+      d = FluentPluginFileBufferTest::DummyOutputPlugin.new
+      p = Fluent::Plugin::FileBuffer.new
+      p.owner = d
+      p.configure(config_element('buffer', '', {'path' => File.join(@dir, 'buffer')}))
+      assert_equal File.join(@dir, 'buffer', 'buffer.*.log'), p.path
+    end
+  end
+
+  sub_test_case 'buffer configurations and workers' do
+    setup do
+      @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__)
+      FileUtils.rm_rf @bufdir
+      Fluent::Test.setup
+
+      @d = FluentPluginFileBufferTest::DummyOutputPlugin.new
+      @p = Fluent::Plugin::FileBuffer.new
+      @p.owner = @d
+    end
+
+    test 'raise error if configured path is of existing file' do
+      @bufpath = File.join(@bufdir, 'buf')
+      FileUtils.mkdir_p @bufdir
+      File.open(@bufpath, 'w'){|f| } # create and close the file
+      assert File.exist?(@bufpath)
+      assert File.file?(@bufpath)
+
+      buf_conf = config_element('buffer', '', {'path' => @bufpath})
+      assert_raise Fluent::ConfigError.new("Plugin 'file' does not support multi workers configuration (Fluent::Plugin::FileBuffer)") do
+        Fluent::SystemConfig.overwrite_system_config('workers' => 4) do
+          @d.configure(config_element('ROOT', '', {'@id' => 'dummy_output_with_buf'}, [buf_conf]))
+        end
+      end
+    end
+
+    test 'raise error if fluentd is configured to use file path pattern and multi workers' do
+      @bufpath = File.join(@bufdir, 'testbuf.*.log')
+      buf_conf = config_element('buffer', '', {'path' => @bufpath})
+      assert_raise Fluent::ConfigError.new("Plugin 'file' does not support multi workers configuration (Fluent::Plugin::FileBuffer)") do
+        Fluent::SystemConfig.overwrite_system_config('workers' => 4) do
+          @d.configure(config_element('ROOT', '', {'@id' => 'dummy_output_with_buf'}, [buf_conf]))
+        end
+      end
+    end
+
+    test 'enables multi worker configuration with unexisting directory path' do
+      assert_false File.exist?(@bufdir)
+      buf_conf = config_element('buffer', '', {'path' => @bufdir})
+      assert_nothing_raised do
+        Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir, 'workers' => 4) do
+          @d.configure(config_element('ROOT', '', {}, [buf_conf]))
+        end
+      end
+    end
+
+    test 'enables multi worker configuration with existing directory path' do
+      FileUtils.mkdir_p @bufdir
+      buf_conf = config_element('buffer', '', {'path' => @bufdir})
+      assert_nothing_raised do
+        Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir, 'workers' => 4) do
+          @d.configure(config_element('ROOT', '', {}, [buf_conf]))
+        end
+      end
+    end
+
+    test 'enables multi worker configuration with root dir' do
+      buf_conf = config_element('buffer', '')
+      assert_nothing_raised do
+        Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir, 'workers' => 4) do
+          @d.configure(config_element('ROOT', '', {'@id' => 'dummy_output_with_buf'}, [buf_conf]))
+        end
+      end
+    end
+  end
+
+  sub_test_case 'buffer plugin configured only with path' do
+    setup do
+      @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__)
+      @bufpath = File.join(@bufdir, 'testbuf.*.log')
+      FileUtils.rm_r @bufdir if File.exist?(@bufdir)
+
+      Fluent::Test.setup
+      @d = FluentPluginFileBufferTest::DummyOutputPlugin.new
+      @p = Fluent::Plugin::FileBuffer.new
+      @p.owner = @d
+      @p.configure(config_element('buffer', '', {'path' => @bufpath}))
+      @p.start
+    end
+
+    teardown do
+      if @p
+        @p.stop unless @p.stopped?
+        @p.before_shutdown unless @p.before_shutdown?
+        @p.shutdown unless @p.shutdown?
+        @p.after_shutdown unless @p.after_shutdown?
+        @p.close unless @p.closed?
+        @p.terminate unless @p.terminated?
+      end
+      if @bufdir
+        Dir.glob(File.join(@bufdir, '*')).each do |path|
+          next if ['.', '..'].include?(File.basename(path))
+          File.delete(path)
+        end
+      end
+    end
+
+    test 'this is persistent plugin' do
+      assert @p.persistent?
+    end
+
+    test '#start creates directory for buffer chunks' do
+      plugin = Fluent::Plugin::FileBuffer.new
+      plugin.owner = @d
+      rand_num = rand(0..100)
+      bufpath = File.join(File.expand_path("../../tmp/buffer_file_#{rand_num}", __FILE__), 'testbuf.*.log')
+      bufdir = File.dirname(bufpath)
+
+      FileUtils.rm_r bufdir if File.exist?(bufdir)
+      assert !File.exist?(bufdir)
+
+      plugin.configure(config_element('buffer', '', {'path' => bufpath}))
+      assert !File.exist?(bufdir)
+
+      plugin.start
+      assert File.exist?(bufdir)
+      assert{ File.stat(bufdir).mode.to_s(8).end_with?('755') }
+
+      plugin.stop; plugin.before_shutdown; plugin.shutdown; plugin.after_shutdown; plugin.close; plugin.terminate
+      FileUtils.rm_r bufdir
+    end
+
+    test '#start creates directory for buffer chunks with specified permission' do
+      omit "NTFS doesn't support UNIX like permissions" if Fluent.windows?
+
+      plugin = Fluent::Plugin::FileBuffer.new
+      plugin.owner = @d
+      rand_num = rand(0..100)
+      bufpath = File.join(File.expand_path("../../tmp/buffer_file_#{rand_num}", __FILE__), 'testbuf.*.log')
+      bufdir = File.dirname(bufpath)
+
+      FileUtils.rm_r bufdir if File.exist?(bufdir)
+      assert !File.exist?(bufdir)
+
+      plugin.configure(config_element('buffer', '', {'path' => bufpath, 'dir_permission' => '0700'}))
+      assert !File.exist?(bufdir)
+
+      plugin.start
+      assert File.exist?(bufdir)
+      assert{ File.stat(bufdir).mode.to_s(8).end_with?('700') }
+
+      plugin.stop; plugin.before_shutdown; plugin.shutdown; plugin.after_shutdown; plugin.close; plugin.terminate
+      FileUtils.rm_r bufdir
+    end
+
+    test '#start creates directory for buffer chunks with specified permission via system config' do
+      omit "NTFS doesn't support UNIX like permissions" if Fluent.windows?
+
+      sysconf = {'dir_permission' => '700'}
+      Fluent::SystemConfig.overwrite_system_config(sysconf) do
+        plugin = Fluent::Plugin::FileBuffer.new
+        plugin.owner = @d
+        rand_num = rand(0..100)
+        bufpath = File.join(File.expand_path("../../tmp/buffer_file_#{rand_num}", __FILE__), 'testbuf.*.log')
+        bufdir = File.dirname(bufpath)
+
+        FileUtils.rm_r bufdir if File.exist?(bufdir)
+        assert !File.exist?(bufdir)
+
+        plugin.configure(config_element('buffer', '', {'path' => bufpath}))
+        assert !File.exist?(bufdir)
+
+        plugin.start
+        assert File.exist?(bufdir)
+        assert{ File.stat(bufdir).mode.to_s(8).end_with?('700') }
+
+        plugin.stop; plugin.before_shutdown; plugin.shutdown; plugin.after_shutdown; plugin.close; plugin.terminate
+        FileUtils.rm_r bufdir
+      end
+    end
+
+    test '#generate_chunk generates blank file chunk on path from unique_id of metadata' do
+      m1 = metadata()
+      c1 = @p.generate_chunk(m1)
+      assert c1.is_a? Fluent::Plugin::Buffer::FileChunk
+      assert_equal m1, c1.metadata
+      assert c1.empty?
+      assert_equal :unstaged, c1.state
+      assert_equal Fluent::Plugin::Buffer::FileChunk::FILE_PERMISSION, c1.permission
+      assert_equal @bufpath.gsub('.*.', ".b#{Fluent::UniqueId.hex(c1.unique_id)}."), c1.path
+      assert{ File.stat(c1.path).mode.to_s(8).end_with?('644') }
+
+      m2 = metadata(timekey: event_time('2016-04-17 11:15:00 -0700').to_i)
+      c2 = @p.generate_chunk(m2)
+      assert c2.is_a? Fluent::Plugin::Buffer::FileChunk
+      assert_equal m2, c2.metadata
+      assert c2.empty?
+      assert_equal :unstaged, c2.state
+      assert_equal Fluent::Plugin::Buffer::FileChunk::FILE_PERMISSION, c2.permission
+      assert_equal @bufpath.gsub('.*.', ".b#{Fluent::UniqueId.hex(c2.unique_id)}."), c2.path
+      assert{ File.stat(c2.path).mode.to_s(8).end_with?('644') }
+
+      c1.purge
+      c2.purge
+    end
+
+    test '#generate_chunk generates blank file chunk with specified permission' do
+      omit "NTFS doesn't support UNIX like permissions" if Fluent.windows?
+
+      plugin = Fluent::Plugin::FileBuffer.new
+      plugin.owner = @d
+      rand_num = rand(0..100)
+      bufpath = File.join(File.expand_path("../../tmp/buffer_file_#{rand_num}", __FILE__), 'testbuf.*.log')
+      bufdir = File.dirname(bufpath)
+
+      FileUtils.rm_r bufdir if File.exist?(bufdir)
+      assert !File.exist?(bufdir)
+
+      plugin.configure(config_element('buffer', '', {'path' => bufpath, 'file_permission' => '0600'}))
+      assert !File.exist?(bufdir)
+      plugin.start
+
+      m = metadata()
+      c = plugin.generate_chunk(m)
+      assert c.is_a? Fluent::Plugin::Buffer::FileChunk
+      assert_equal m, c.metadata
+      assert c.empty?
+      assert_equal :unstaged, c.state
+      assert_equal 0600, c.permission
+      assert_equal bufpath.gsub('.*.', ".b#{Fluent::UniqueId.hex(c.unique_id)}."), c.path
+      assert{ File.stat(c.path).mode.to_s(8).end_with?('600') }
+
+      c.purge
+
+      plugin.stop; plugin.before_shutdown; plugin.shutdown; plugin.after_shutdown; plugin.close; plugin.terminate
+      FileUtils.rm_r bufdir
+    end
+  end
+
+  sub_test_case 'configured with system root directory and plugin @id' do
+    setup do
+      @root_dir = File.expand_path('../../tmp/buffer_file_root', __FILE__)
+      FileUtils.rm_rf @root_dir
+
+      Fluent::Test.setup
+      @d = FluentPluginFileBufferTest::DummyOutputPlugin.new
+      @p = Fluent::Plugin::FileBuffer.new
+      @p.owner = @d
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => @root_dir) do
+        @d.configure(config_element('ROOT', '', {'@id' => 'dummy_output_with_buf'}))
+        @p.configure(config_element('buffer', ''))
+      end
+    end
+
+    teardown do
+      if @p
+        @p.stop unless @p.stopped?
+        @p.before_shutdown unless @p.before_shutdown?
+        @p.shutdown unless @p.shutdown?
+        @p.after_shutdown unless @p.after_shutdown?
+        @p.close unless @p.closed?
+        @p.terminate unless @p.terminated?
+      end
+    end
+
+    test '#start creates directory for buffer chunks' do
+      expected_buffer_path = File.join(@root_dir, 'worker0', 'dummy_output_with_buf', 'buffer', 'buffer.*.log')
+      expected_buffer_dir = File.dirname(expected_buffer_path)
+      assert_equal expected_buffer_path, @p.path
+      assert_false Dir.exist?(expected_buffer_dir)
+
+      @p.start
+
+      assert Dir.exist?(expected_buffer_dir)
+    end
+  end
+
+  sub_test_case 'there are no existing file chunks' do
+    setup do
+      @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__)
+      @bufpath = File.join(@bufdir, 'testbuf.*.log')
+      FileUtils.rm_r @bufdir if File.exist?(@bufdir)
+
+      Fluent::Test.setup
+      @d = FluentPluginFileBufferTest::DummyOutputPlugin.new
+      @p = Fluent::Plugin::FileBuffer.new
+      @p.owner = @d
+      @p.configure(config_element('buffer', '', {'path' => @bufpath}))
+      @p.start
+    end
+    teardown do
+      if @p
+        @p.stop unless @p.stopped?
+        @p.before_shutdown unless @p.before_shutdown?
+        @p.shutdown unless @p.shutdown?
+        @p.after_shutdown unless @p.after_shutdown?
+        @p.close unless @p.closed?
+        @p.terminate unless @p.terminated?
+      end
+      if @bufdir
+        Dir.glob(File.join(@bufdir, '*')).each do |path|
+          next if ['.', '..'].include?(File.basename(path))
+          File.delete(path)
+        end
+      end
+    end
+
+    test '#resume returns empty buffer state' do
+      ary = @p.resume
+      assert_equal({}, ary[0])
+      assert_equal([], ary[1])
+    end
+  end
+
+  sub_test_case 'there are some existing file chunks' do
+    setup do
+      @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__)
+      FileUtils.mkdir_p @bufdir unless File.exist?(@bufdir)
+
+      @c1id = Fluent::UniqueId.generate
+      p1 = File.join(@bufdir, "etest.q#{Fluent::UniqueId.hex(@c1id)}.log")
+      File.open(p1, 'wb') do |f|
+        f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t2.test", event_time('2016-04-17 13:58:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t3.test", event_time('2016-04-17 13:58:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t4.test", event_time('2016-04-17 13:58:22 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+      end
+      write_metadata(
+        p1 + '.meta', @c1id, metadata(timekey: event_time('2016-04-17 13:58:00 -0700').to_i),
+        4, event_time('2016-04-17 13:58:00 -0700').to_i, event_time('2016-04-17 13:58:22 -0700').to_i
+      )
+
+      @c2id = Fluent::UniqueId.generate
+      p2 = File.join(@bufdir, "etest.q#{Fluent::UniqueId.hex(@c2id)}.log")
+      File.open(p2, 'wb') do |f|
+        f.write ["t1.test", event_time('2016-04-17 13:59:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t2.test", event_time('2016-04-17 13:59:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t3.test", event_time('2016-04-17 13:59:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+      end
+      write_metadata(
+        p2 + '.meta', @c2id, metadata(timekey: event_time('2016-04-17 13:59:00 -0700').to_i),
+        3, event_time('2016-04-17 13:59:00 -0700').to_i, event_time('2016-04-17 13:59:23 -0700').to_i
+      )
+
+      @c3id = Fluent::UniqueId.generate
+      p3 = File.join(@bufdir, "etest.b#{Fluent::UniqueId.hex(@c3id)}.log")
+      File.open(p3, 'wb') do |f|
+        f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t2.test", event_time('2016-04-17 14:00:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t3.test", event_time('2016-04-17 14:00:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t4.test", event_time('2016-04-17 14:00:28 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+      end
+      write_metadata(
+        p3 + '.meta', @c3id, metadata(timekey: event_time('2016-04-17 14:00:00 -0700').to_i),
+        4, event_time('2016-04-17 14:00:00 -0700').to_i, event_time('2016-04-17 14:00:28 -0700').to_i
+      )
+
+      @c4id = Fluent::UniqueId.generate
+      p4 = File.join(@bufdir, "etest.b#{Fluent::UniqueId.hex(@c4id)}.log")
+      File.open(p4, 'wb') do |f|
+        f.write ["t1.test", event_time('2016-04-17 14:01:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t2.test", event_time('2016-04-17 14:01:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t3.test", event_time('2016-04-17 14:01:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+      end
+      write_metadata(
+        p4 + '.meta', @c4id, metadata(timekey: event_time('2016-04-17 14:01:00 -0700').to_i),
+        3, event_time('2016-04-17 14:01:00 -0700').to_i, event_time('2016-04-17 14:01:25 -0700').to_i
+      )
+
+      @bufpath = File.join(@bufdir, 'etest.*.log')
+
+      Fluent::Test.setup
+      @d = FluentPluginFileBufferTest::DummyOutputPlugin.new
+      @p = Fluent::Plugin::FileBuffer.new
+      @p.owner = @d
+      @p.configure(config_element('buffer', '', {'path' => @bufpath}))
+      @p.start
+    end
+
+    teardown do
+      if @p
+        @p.stop unless @p.stopped?
+        @p.before_shutdown unless @p.before_shutdown?
+        @p.shutdown unless @p.shutdown?
+        @p.after_shutdown unless @p.after_shutdown?
+        @p.close unless @p.closed?
+        @p.terminate unless @p.terminated?
+      end
+      if @bufdir
+        Dir.glob(File.join(@bufdir, '*')).each do |path|
+          next if ['.', '..'].include?(File.basename(path))
+          File.delete(path)
+        end
+      end
+    end
+
+    test '#resume returns staged/queued chunks with metadata' do
+      assert_equal 2, @p.stage.size
+      assert_equal 2, @p.queue.size
+
+      stage = @p.stage
+
+      m3 = metadata(timekey: event_time('2016-04-17 14:00:00 -0700').to_i)
+      assert_equal @c3id, stage[m3].unique_id
+      assert_equal 4, stage[m3].size
+      assert_equal :staged, stage[m3].state
+
+      m4 = metadata(timekey: event_time('2016-04-17 14:01:00 -0700').to_i)
+      assert_equal @c4id, stage[m4].unique_id
+      assert_equal 3, stage[m4].size
+      assert_equal :staged, stage[m4].state
+    end
+
+    test '#resume returns queued chunks ordered by last modified time (FIFO)' do
+      assert_equal 2, @p.stage.size
+      assert_equal 2, @p.queue.size
+
+      queue = @p.queue
+
+      assert{ queue[0].modified_at < queue[1].modified_at }
+
+      assert_equal @c1id, queue[0].unique_id
+      assert_equal :queued, queue[0].state
+      assert_equal event_time('2016-04-17 13:58:00 -0700').to_i, queue[0].metadata.timekey
+      assert_nil queue[0].metadata.tag
+      assert_nil queue[0].metadata.variables
+      assert_equal Time.parse('2016-04-17 13:58:00 -0700').localtime, queue[0].created_at
+      assert_equal Time.parse('2016-04-17 13:58:22 -0700').localtime, queue[0].modified_at
+      assert_equal 4, queue[0].size
+
+      assert_equal @c2id, queue[1].unique_id
+      assert_equal :queued, queue[1].state
+      assert_equal event_time('2016-04-17 13:59:00 -0700').to_i, queue[1].metadata.timekey
+      assert_nil queue[1].metadata.tag
+      assert_nil queue[1].metadata.variables
+      assert_equal Time.parse('2016-04-17 13:59:00 -0700').localtime, queue[1].created_at
+      assert_equal Time.parse('2016-04-17 13:59:23 -0700').localtime, queue[1].modified_at
+      assert_equal 3, queue[1].size
+    end
+  end
+
+  sub_test_case 'there are some existing file chunks, both in specified path and per-worker directory under specified path, configured as multi workers' do
+    setup do
+      @bufdir = File.expand_path('../../tmp/buffer_file/path', __FILE__)
+      @worker0_dir = File.join(@bufdir, "worker0")
+      @worker1_dir = File.join(@bufdir, "worker1")
+      FileUtils.rm_rf @bufdir
+      FileUtils.mkdir_p @worker0_dir
+      FileUtils.mkdir_p @worker1_dir
+
+      @bufdir_chunk_1 = Fluent::UniqueId.generate
+      bc1 = File.join(@bufdir, "buffer.q#{Fluent::UniqueId.hex(@bufdir_chunk_1)}.log")
+      File.open(bc1, 'wb') do |f|
+        f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t2.test", event_time('2016-04-17 13:58:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t3.test", event_time('2016-04-17 13:58:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t4.test", event_time('2016-04-17 13:58:22 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+      end
+      write_metadata(
+        bc1 + '.meta', @bufdir_chunk_1, metadata(timekey: event_time('2016-04-17 13:58:00 -0700').to_i),
+        4, event_time('2016-04-17 13:58:00 -0700').to_i, event_time('2016-04-17 13:58:22 -0700').to_i
+      )
+
+      @bufdir_chunk_2 = Fluent::UniqueId.generate
+      bc2 = File.join(@bufdir, "buffer.q#{Fluent::UniqueId.hex(@bufdir_chunk_2)}.log")
+      File.open(bc2, 'wb') do |f|
+        f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t2.test", event_time('2016-04-17 13:58:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t3.test", event_time('2016-04-17 13:58:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t4.test", event_time('2016-04-17 13:58:22 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+      end
+      write_metadata(
+        bc2 + '.meta', @bufdir_chunk_2, metadata(timekey: event_time('2016-04-17 13:58:00 -0700').to_i),
+        4, event_time('2016-04-17 13:58:00 -0700').to_i, event_time('2016-04-17 13:58:22 -0700').to_i
+      )
+
+      @worker_dir_chunk_1 = Fluent::UniqueId.generate
+      wc0_1 = File.join(@worker0_dir, "buffer.q#{Fluent::UniqueId.hex(@worker_dir_chunk_1)}.log")
+      wc1_1 = File.join(@worker1_dir, "buffer.q#{Fluent::UniqueId.hex(@worker_dir_chunk_1)}.log")
+      [wc0_1, wc1_1].each do |chunk_path|
+        File.open(chunk_path, 'wb') do |f|
+          f.write ["t1.test", event_time('2016-04-17 13:59:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+          f.write ["t2.test", event_time('2016-04-17 13:59:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+          f.write ["t3.test", event_time('2016-04-17 13:59:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        end
+        write_metadata(
+          chunk_path + '.meta', @worker_dir_chunk_1, metadata(timekey: event_time('2016-04-17 13:59:00 -0700').to_i),
+          3, event_time('2016-04-17 13:59:00 -0700').to_i, event_time('2016-04-17 13:59:23 -0700').to_i
+        )
+      end
+
+      @worker_dir_chunk_2 = Fluent::UniqueId.generate
+      wc0_2 = File.join(@worker0_dir, "buffer.b#{Fluent::UniqueId.hex(@worker_dir_chunk_2)}.log")
+      wc1_2 = File.join(@worker1_dir, "buffer.b#{Fluent::UniqueId.hex(@worker_dir_chunk_2)}.log")
+      [wc0_2, wc1_2].each do |chunk_path|
+        File.open(chunk_path, 'wb') do |f|
+          f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+          f.write ["t2.test", event_time('2016-04-17 14:00:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+          f.write ["t3.test", event_time('2016-04-17 14:00:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+          f.write ["t4.test", event_time('2016-04-17 14:00:28 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        end
+        write_metadata(
+          chunk_path + '.meta', @worker_dir_chunk_2, metadata(timekey: event_time('2016-04-17 14:00:00 -0700').to_i),
+          4, event_time('2016-04-17 14:00:00 -0700').to_i, event_time('2016-04-17 14:00:28 -0700').to_i
+        )
+      end
+
+      @worker_dir_chunk_3 = Fluent::UniqueId.generate
+      wc0_3 = File.join(@worker0_dir, "buffer.b#{Fluent::UniqueId.hex(@worker_dir_chunk_3)}.log")
+      wc1_3 = File.join(@worker1_dir, "buffer.b#{Fluent::UniqueId.hex(@worker_dir_chunk_3)}.log")
+      [wc0_3, wc1_3].each do |chunk_path|
+        File.open(chunk_path, 'wb') do |f|
+          f.write ["t1.test", event_time('2016-04-17 14:01:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+          f.write ["t2.test", event_time('2016-04-17 14:01:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+          f.write ["t3.test", event_time('2016-04-17 14:01:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        end
+        write_metadata(
+          chunk_path + '.meta', @worker_dir_chunk_3, metadata(timekey: event_time('2016-04-17 14:01:00 -0700').to_i),
+          3, event_time('2016-04-17 14:01:00 -0700').to_i, event_time('2016-04-17 14:01:25 -0700').to_i
+        )
+      end
+
+      Fluent::Test.setup
+    end
+
+    teardown do
+      if @p
+        @p.stop unless @p.stopped?
+        @p.before_shutdown unless @p.before_shutdown?
+        @p.shutdown unless @p.shutdown?
+        @p.after_shutdown unless @p.after_shutdown?
+        @p.close unless @p.closed?
+        @p.terminate unless @p.terminated?
+      end
+    end
+
+    test 'worker(id=0) #resume returns staged/queued chunks with metadata, not only in worker dir, including the directory specified by path' do
+      ENV['SERVERENGINE_WORKER_ID'] = '0'
+
+      buf_conf = config_element('buffer', '', {'path' => @bufdir})
+      @d = FluentPluginFileBufferTest::DummyOutputPlugin.new
+      with_worker_config(workers: 2, worker_id: 0) do
+        @d.configure(config_element('output', '', {}, [buf_conf]))
+      end
+
+      @d.start
+      @p = @d.buffer
+
+      assert_equal 2, @p.stage.size
+      assert_equal 3, @p.queue.size
+
+      stage = @p.stage
+
+      m1 = metadata(timekey: event_time('2016-04-17 14:00:00 -0700').to_i)
+      assert_equal @worker_dir_chunk_2, stage[m1].unique_id
+      assert_equal 4, stage[m1].size
+      assert_equal :staged, stage[m1].state
+
+      m2 = metadata(timekey: event_time('2016-04-17 14:01:00 -0700').to_i)
+      assert_equal @worker_dir_chunk_3, stage[m2].unique_id
+      assert_equal 3, stage[m2].size
+      assert_equal :staged, stage[m2].state
+
+      queue = @p.queue
+
+      assert_equal [@bufdir_chunk_1, @bufdir_chunk_2, @worker_dir_chunk_1].sort, queue.map(&:unique_id).sort
+      assert_equal [3, 4, 4], queue.map(&:size).sort
+      assert_equal [:queued, :queued, :queued], queue.map(&:state)
+    end
+
+    test 'worker(id=1) #resume returns staged/queued chunks with metadata, only in worker dir' do
+      buf_conf = config_element('buffer', '', {'path' => @bufdir})
+      @d = FluentPluginFileBufferTest::DummyOutputPlugin.new
+      with_worker_config(workers: 2, worker_id: 1) do
+        @d.configure(config_element('output', '', {}, [buf_conf]))
+      end
+
+      @d.start
+      @p = @d.buffer
+
+      assert_equal 2, @p.stage.size
+      assert_equal 1, @p.queue.size
+
+      stage = @p.stage
+
+      m1 = metadata(timekey: event_time('2016-04-17 14:00:00 -0700').to_i)
+      assert_equal @worker_dir_chunk_2, stage[m1].unique_id
+      assert_equal 4, stage[m1].size
+      assert_equal :staged, stage[m1].state
+
+      m2 = metadata(timekey: event_time('2016-04-17 14:01:00 -0700').to_i)
+      assert_equal @worker_dir_chunk_3, stage[m2].unique_id
+      assert_equal 3, stage[m2].size
+      assert_equal :staged, stage[m2].state
+
+      queue = @p.queue
+
+      assert_equal @worker_dir_chunk_1, queue[0].unique_id
+      assert_equal 3, queue[0].size
+      assert_equal :queued, queue[0].state
+    end
+  end
+
+  sub_test_case 'there are some existing file chunks without metadata file' do
+    setup do
+      @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__)
+
+      @c1id = Fluent::UniqueId.generate
+      p1 = File.join(@bufdir, "etest.201604171358.q#{Fluent::UniqueId.hex(@c1id)}.log")
+      File.open(p1, 'wb') do |f|
+        f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t2.test", event_time('2016-04-17 13:58:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t3.test", event_time('2016-04-17 13:58:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t4.test", event_time('2016-04-17 13:58:22 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+      end
+      FileUtils.touch(p1, mtime: Time.parse('2016-04-17 13:58:28 -0700'))
+
+      @c2id = Fluent::UniqueId.generate
+      p2 = File.join(@bufdir, "etest.201604171359.q#{Fluent::UniqueId.hex(@c2id)}.log")
+      File.open(p2, 'wb') do |f|
+        f.write ["t1.test", event_time('2016-04-17 13:59:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t2.test", event_time('2016-04-17 13:59:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t3.test", event_time('2016-04-17 13:59:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+      end
+      FileUtils.touch(p2, mtime: Time.parse('2016-04-17 13:59:30 -0700'))
+
+      @c3id = Fluent::UniqueId.generate
+      p3 = File.join(@bufdir, "etest.201604171400.b#{Fluent::UniqueId.hex(@c3id)}.log")
+      File.open(p3, 'wb') do |f|
+        f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t2.test", event_time('2016-04-17 14:00:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t3.test", event_time('2016-04-17 14:00:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t4.test", event_time('2016-04-17 14:00:28 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+      end
+      FileUtils.touch(p3, mtime: Time.parse('2016-04-17 14:00:29 -0700'))
+
+      @c4id = Fluent::UniqueId.generate
+      p4 = File.join(@bufdir, "etest.201604171401.b#{Fluent::UniqueId.hex(@c4id)}.log")
+      File.open(p4, 'wb') do |f|
+        f.write ["t1.test", event_time('2016-04-17 14:01:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t2.test", event_time('2016-04-17 14:01:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t3.test", event_time('2016-04-17 14:01:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+      end
+      FileUtils.touch(p4, mtime: Time.parse('2016-04-17 14:01:22 -0700'))
+
+      @bufpath = File.join(@bufdir, 'etest.*.log')
+
+      Fluent::Test.setup
+      @d = FluentPluginFileBufferTest::DummyOutputPlugin.new
+      @p = Fluent::Plugin::FileBuffer.new
+      @p.owner = @d
+      @p.configure(config_element('buffer', '', {'path' => @bufpath}))
+      @p.start
+    end
+
+    teardown do
+      if @p
+        @p.stop unless @p.stopped?
+        @p.before_shutdown unless @p.before_shutdown?
+        @p.shutdown unless @p.shutdown?
+        @p.after_shutdown unless @p.after_shutdown?
+        @p.close unless @p.closed?
+        @p.terminate unless @p.terminated?
+      end
+      if @bufdir
+        Dir.glob(File.join(@bufdir, '*')).each do |path|
+          next if ['.', '..'].include?(File.basename(path))
+          File.delete(path)
+        end
+      end
+    end
+
+    test '#resume returns queued chunks for files without metadata' do
+      assert_equal 0, @p.stage.size
+      assert_equal 4, @p.queue.size
+
+      queue = @p.queue
+
+      m = metadata()
+
+      assert_equal @c1id, queue[0].unique_id
+      assert_equal m, queue[0].metadata
+      assert_equal 0, queue[0].size
+      assert_equal :queued, queue[0].state
+      assert_equal Time.parse('2016-04-17 13:58:28 -0700'), queue[0].modified_at
+
+      assert_equal @c2id, queue[1].unique_id
+      assert_equal m, queue[1].metadata
+      assert_equal 0, queue[1].size
+      assert_equal :queued, queue[1].state
+      assert_equal Time.parse('2016-04-17 13:59:30 -0700'), queue[1].modified_at
+
+      assert_equal @c3id, queue[2].unique_id
+      assert_equal m, queue[2].metadata
+      assert_equal 0, queue[2].size
+      assert_equal :queued, queue[2].state
+      assert_equal Time.parse('2016-04-17 14:00:29 -0700'), queue[2].modified_at
+
+      assert_equal @c4id, queue[3].unique_id
+      assert_equal m, queue[3].metadata
+      assert_equal 0, queue[3].size
+      assert_equal :queued, queue[3].state
+      assert_equal Time.parse('2016-04-17 14:01:22 -0700'), queue[3].modified_at
+    end
+  end
+
+  sub_test_case 'there are some non-buffer chunk files, with a path without buffer chunk ids' do
+    setup do
+      @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__)
+
+      FileUtils.rm_rf @bufdir
+      FileUtils.mkdir_p @bufdir
+
+      @c1id = Fluent::UniqueId.generate
+      p1 = File.join(@bufdir, "etest.201604171358.q#{Fluent::UniqueId.hex(@c1id)}.log")
+      File.open(p1, 'wb') do |f|
+        f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t2.test", event_time('2016-04-17 13:58:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t3.test", event_time('2016-04-17 13:58:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t4.test", event_time('2016-04-17 13:58:22 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+      end
+      FileUtils.touch(p1, mtime: Time.parse('2016-04-17 13:58:28 -0700'))
+
+      @not_chunk = File.join(@bufdir, 'etest.20160416.log')
+      File.open(@not_chunk, 'wb') do |f|
+        f.write ["t1.test", event_time('2016-04-16 23:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t2.test", event_time('2016-04-16 23:58:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t3.test", event_time('2016-04-16 23:58:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t4.test", event_time('2016-04-16 23:58:22 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+      end
+      FileUtils.touch(@not_chunk, mtime: Time.parse('2016-04-17 00:00:00 -0700'))
+
+      @bufpath = File.join(@bufdir, 'etest.*.log')
+
+      Fluent::Test.setup
+      @d = FluentPluginFileBufferTest::DummyOutputPlugin.new
+      @p = Fluent::Plugin::FileBuffer.new
+      @p.owner = @d
+      @p.configure(config_element('buffer', '', {'path' => @bufpath}))
+      @p.start
+    end
+
+    teardown do
+      if @p
+        @p.stop unless @p.stopped?
+        @p.before_shutdown unless @p.before_shutdown?
+        @p.shutdown unless @p.shutdown?
+        @p.after_shutdown unless @p.after_shutdown?
+        @p.close unless @p.closed?
+        @p.terminate unless @p.terminated?
+      end
+      if @bufdir
+        Dir.glob(File.join(@bufdir, '*')).each do |path|
+          next if ['.', '..'].include?(File.basename(path))
+          File.delete(path)
+        end
+      end
+    end
+
+    test '#resume returns queued chunks for files without metadata, while ignoring non-chunk looking files' do
+      assert_equal 0, @p.stage.size
+      assert_equal 1, @p.queue.size
+
+      queue = @p.queue
+
+      m = metadata()
+
+      assert_equal @c1id, queue[0].unique_id
+      assert_equal m, queue[0].metadata
+      assert_equal 0, queue[0].size
+      assert_equal :queued, queue[0].state
+      assert_equal Time.parse('2016-04-17 13:58:28 -0700'), queue[0].modified_at
+
+      assert File.exist?(@not_chunk)
+    end
+  end
+end