fluentd 1.15.2-x64-mingw32 → 1.16.0-x64-mingw32

data/test/plugin/test_buf_file.rb

@@ -1151,15 +1151,13 @@ class FileBufferTest < Test::Unit::TestCase
 
   sub_test_case 'there are existing broken file chunks' do
     setup do
+      @id_output = 'backup_test'
       @bufdir = File.expand_path('../../tmp/broken_buffer_file', __FILE__)
-      FileUtils.mkdir_p @bufdir unless File.exist?(@bufdir)
+      FileUtils.rm_rf @bufdir rescue nil
+      FileUtils.mkdir_p @bufdir
       @bufpath = File.join(@bufdir, 'broken_test.*.log')
 
       Fluent::Test.setup
-      @d = FluentPluginFileBufferTest::DummyOutputPlugin.new
-      @p = Fluent::Plugin::FileBuffer.new
-      @p.owner = @d
-      @p.configure(config_element('buffer', '', {'path' => @bufpath}))
     end
 
     teardown do
@@ -1171,12 +1169,12 @@ class FileBufferTest < Test::Unit::TestCase
         @p.close unless @p.closed?
         @p.terminate unless @p.terminated?
       end
-      if @bufdir
-        Dir.glob(File.join(@bufdir, '*')).each do |path|
-          next if ['.', '..'].include?(File.basename(path))
-          File.delete(path)
-        end
-      end
+    end
+
+    def setup_plugins(buf_conf)
+      @d = FluentPluginFileBufferTest::DummyOutputPlugin.new
+      @d.configure(config_element('ROOT', '', {'@id' => @id_output}, [config_element('buffer', '', buf_conf)]))
+      @p = @d.buffer
     end
 
     def create_first_chunk(mode)
@@ -1232,44 +1230,85 @@ class FileBufferTest < Test::Unit::TestCase
       assert { logs.any? { |log| log.include?(msg) } }
     end
 
-    test '#resume ignores staged empty chunk' do
-      _, p1 = create_first_chunk('b')
+    test '#resume backups staged empty chunk' do
+      setup_plugins({'path' => @bufpath})
+      c1id, p1 = create_first_chunk('b')
       File.open(p1, 'wb') { |f| } # create staged empty chunk file
       c2id, _ = create_second_chunk('b')
 
-      @p.start
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
+        @p.start
+      end
+
       compare_staged_chunk(@p.stage, c2id, '2016-04-17 14:01:00 -0700', 3, :staged)
       compare_log(@p, 'staged file chunk is empty')
+      assert { not File.exist?(p1) }
+      assert { File.exist?("#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c1id)}.log") }
     end
 
-    test '#resume ignores staged broken metadata' do
+    test '#resume backups staged broken metadata' do
+      setup_plugins({'path' => @bufpath})
       c1id, _ = create_first_chunk('b')
-      _, p2 = create_second_chunk('b')
+      c2id, p2 = create_second_chunk('b')
       File.open(p2 + '.meta', 'wb') { |f| f.write("\0" * 70) } # create staged broken meta file
 
-      @p.start
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
+        @p.start
+      end
+
       compare_staged_chunk(@p.stage, c1id, '2016-04-17 14:00:00 -0700', 4, :staged)
       compare_log(@p, 'staged meta file is broken')
+      assert { not File.exist?(p2) }
+      assert { File.exist?("#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c2id)}.log") }
     end
 
-    test '#resume ignores enqueued empty chunk' do
-      _, p1 = create_first_chunk('q')
+    test '#resume backups enqueued empty chunk' do
+      setup_plugins({'path' => @bufpath})
+      c1id, p1 = create_first_chunk('q')
       File.open(p1, 'wb') { |f| } # create enqueued empty chunk file
       c2id, _ = create_second_chunk('q')
 
-      @p.start
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
+        @p.start
+      end
+
       compare_queued_chunk(@p.queue, c2id, 3, :queued)
       compare_log(@p, 'enqueued file chunk is empty')
+      assert { not File.exist?(p1) }
+      assert { File.exist?("#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c1id)}.log") }
     end
 
-    test '#resume ignores enqueued broken metadata' do
+    test '#resume backups enqueued broken metadata' do
+      setup_plugins({'path' => @bufpath})
       c1id, _ = create_first_chunk('q')
-      _, p2 = create_second_chunk('q')
+      c2id, p2 = create_second_chunk('q')
       File.open(p2 + '.meta', 'wb') { |f| f.write("\0" * 70) } # create enqueued broken meta file
 
-      @p.start
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
+        @p.start
+      end
+
       compare_queued_chunk(@p.queue, c1id, 4, :queued)
       compare_log(@p, 'enqueued meta file is broken')
+      assert { not File.exist?(p2) }
+      assert { File.exist?("#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c2id)}.log") }
+    end
+
+    test '#resume throws away broken chunk with disable_chunk_backup' do
+      setup_plugins({'path' => @bufpath, 'disable_chunk_backup' => true})
+      c1id, _ = create_first_chunk('b')
+      c2id, p2 = create_second_chunk('b')
+      File.open(p2 + '.meta', 'wb') { |f| f.write("\0" * 70) } # create staged broken meta file
+
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
+        @p.start
+      end
+
+      compare_staged_chunk(@p.stage, c1id, '2016-04-17 14:00:00 -0700', 4, :staged)
+      compare_log(@p, 'staged meta file is broken')
+      compare_log(@p, 'disable_chunk_backup is true')
+      assert { not File.exist?(p2) }
+      assert { not File.exist?("#{@bufdir}/backup/worker0/#{@id_output}/#{@d.dump_unique_id_hex(c2id)}.log") }
     end
   end
 end

data/test/plugin/test_buf_file_single.rb

@@ -830,4 +830,69 @@ class FileSingleBufferTest < Test::Unit::TestCase
       assert_equal :queued, queue[0].state
     end
   end
+
+  sub_test_case 'there are existing broken file chunks' do
+    setup do
+      FileUtils.rm_rf(@bufdir) rescue nil
+      FileUtils.mkdir_p(@bufdir)
+    end
+
+    teardown do
+      return unless @p
+
+      @p.stop unless @p.stopped?
+      @p.before_shutdown unless @p.before_shutdown?
+      @p.shutdown unless @p.shutdown?
+      @p.after_shutdown unless @p.after_shutdown?
+      @p.close unless @p.closed?
+      @p.terminate unless @p.terminated?
+    end
+
+    test '#resume backups empty chunk' do
+      id_output = 'backup_test'
+      @d = create_driver(%[
+        @id #{id_output}
+        <buffer tag>
+         @type file_single
+         path #{PATH}
+        </buffer>
+      ])
+      @p = @d.instance.buffer
+
+      c1id = Fluent::UniqueId.generate
+      p1 = File.join(@bufdir, "fsb.foo.b#{Fluent::UniqueId.hex(c1id)}.buf")
+      File.open(p1, 'wb') { |f| } # create empty chunk file
+
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
+        @p.start
+      end
+
+      assert { not File.exist?(p1) }
+      assert { File.exist?("#{@bufdir}/backup/worker0/#{id_output}/#{@d.instance.dump_unique_id_hex(c1id)}.log") }
+    end
+
+    test '#resume throws away broken chunk with disable_chunk_backup' do
+      id_output = 'backup_test'
+      @d = create_driver(%[
+        @id #{id_output}
+        <buffer tag>
+         @type file_single
+         path #{PATH}
+         disable_chunk_backup true
+        </buffer>
+      ])
+      @p = @d.instance.buffer
+
+      c1id = Fluent::UniqueId.generate
+      p1 = File.join(@bufdir, "fsb.foo.b#{Fluent::UniqueId.hex(c1id)}.buf")
+      File.open(p1, 'wb') { |f| } # create empty chunk file
+
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir) do
+        @p.start
+      end
+
+      assert { not File.exist?(p1) }
+      assert { not File.exist?("#{@bufdir}/backup/worker0/#{id_output}/#{@d.instance.dump_unique_id_hex(c1id)}.log") }
+    end
+  end
 end

data/test/plugin/test_in_http.rb

@@ -7,9 +7,8 @@ require 'timecop'
 class HttpInputTest < Test::Unit::TestCase
   class << self
     def startup
-      socket_manager_path = ServerEngine::SocketManager::Server.generate_path
-      @server = ServerEngine::SocketManager::Server.open(socket_manager_path)
-      ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
+      @server = ServerEngine::SocketManager::Server.open
+      ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @server.path.to_s
     end
 
     def shutdown

data/test/plugin/test_in_monitor_agent.rb

@@ -355,14 +355,13 @@ EOC
 
     test "fluentd opts" do
       d = create_driver
-      opts = Fluent::Supervisor.default_options
 
       filepath = nil
       begin
         FileUtils.mkdir_p(CONFIG_DIR)
         filepath = File.expand_path('fluentd.conf', CONFIG_DIR)
         FileUtils.touch(filepath)
-        s = Fluent::Supervisor.new(opts.merge(config_path: filepath))
+        s = Fluent::Supervisor.new({config_path: filepath})
         s.configure
       ensure
         FileUtils.rm_r(CONFIG_DIR) rescue _
@@ -502,7 +501,7 @@ EOC
           v.puts(conf)
         end
 
-        @supervisor = Fluent::Supervisor.new(Fluent::Supervisor.default_options.merge(config_path: @filepath))
+        @supervisor = Fluent::Supervisor.new({config_path: @filepath})
         @supervisor.configure
       ensure
         FileUtils.rm_r(CONFIG_DIR) rescue _