fluentd 0.14.11-x86-mingw32 → 0.14.12-x86-mingw32

data/test/plugin/test_buf_file.rb

@@ -10,6 +10,12 @@ require 'msgpack'
 module FluentPluginFileBufferTest
   class DummyOutputPlugin < Fluent::Plugin::Output
     Fluent::Plugin.register_output('buffer_file_test_output', self)
+    config_section :buffer do
+      config_set_default :@type, 'file'
+    end
+    def multi_workers_ready?
+      true
+    end
     def write(chunk)
       # drop
     end
@@ -39,13 +45,8 @@ class FileBufferTest < Test::Unit::TestCase
       Fluent::Test.setup
 
       @dir = File.expand_path('../../tmp/buffer_file_dir', __FILE__)
-      unless File.exist?(@dir)
-        FileUtils.mkdir_p @dir
-      end
-      Dir.glob(File.join(@dir, '*')).each do |path|
-        next if ['.', '..'].include?(File.basename(path))
-        File.delete(path)
-      end
+      FileUtils.rm_rf @dir
+      FileUtils.mkdir_p @dir
     end
 
     test 'path should include * normally' do
@@ -73,6 +74,72 @@ class FileBufferTest < Test::Unit::TestCase
     end
   end
 
+  sub_test_case 'buffer configurations and workers' do
+    setup do
+      @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__)
+      FileUtils.rm_rf @bufdir
+      Fluent::Test.setup
+
+      @d = FluentPluginFileBufferTest::DummyOutputPlugin.new
+      @p = Fluent::Plugin::FileBuffer.new
+      @p.owner = @d
+    end
+
+    test 'raise error if configured path is of existing file' do
+      @bufpath = File.join(@bufdir, 'buf')
+      FileUtils.mkdir_p @bufdir
+      File.open(@bufpath, 'w'){|f| } # create and close the file
+      assert File.exist?(@bufpath)
+      assert File.file?(@bufpath)
+
+      buf_conf = config_element('buffer', '', {'path' => @bufpath})
+      assert_raise Fluent::ConfigError.new("Plugin 'file' does not support multi workers configuration (Fluent::Plugin::FileBuffer)") do
+        Fluent::SystemConfig.overwrite_system_config('workers' => 4) do
+          @d.configure(config_element('ROOT', '', {'@id' => 'dummy_output_with_buf'}, [buf_conf]))
+        end
+      end
+    end
+
+    test 'raise error if fluentd is configured to use file path pattern and multi workers' do
+      @bufpath = File.join(@bufdir, 'testbuf.*.log')
+      buf_conf = config_element('buffer', '', {'path' => @bufpath})
+      assert_raise Fluent::ConfigError.new("Plugin 'file' does not support multi workers configuration (Fluent::Plugin::FileBuffer)") do
+        Fluent::SystemConfig.overwrite_system_config('workers' => 4) do
+          @d.configure(config_element('ROOT', '', {'@id' => 'dummy_output_with_buf'}, [buf_conf]))
+        end
+      end
+    end
+
+    test 'enables multi worker configuration with unexisting directory path' do
+      assert_false File.exist?(@bufdir)
+      buf_conf = config_element('buffer', '', {'path' => @bufdir})
+      assert_nothing_raised do
+        Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir, 'workers' => 4) do
+          @d.configure(config_element('ROOT', '', {}, [buf_conf]))
+        end
+      end
+    end
+
+    test 'enables multi worker configuration with existing directory path' do
+      FileUtils.mkdir_p @bufdir
+      buf_conf = config_element('buffer', '', {'path' => @bufdir})
+      assert_nothing_raised do
+        Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir, 'workers' => 4) do
+          @d.configure(config_element('ROOT', '', {}, [buf_conf]))
+        end
+      end
+    end
+
+    test 'enables multi worker configuration with root dir' do
+      buf_conf = config_element('buffer', '')
+      assert_nothing_raised do
+        Fluent::SystemConfig.overwrite_system_config('root_dir' => @bufdir, 'workers' => 4) do
+          @d.configure(config_element('ROOT', '', {'@id' => 'dummy_output_with_buf'}, [buf_conf]))
+        end
+      end
+    end
+  end
+
   sub_test_case 'buffer plugin configured only with path' do
     setup do
       @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__)
@@ -141,7 +208,7 @@ class FileBufferTest < Test::Unit::TestCase
       FileUtils.rm_r bufdir if File.exist?(bufdir)
       assert !File.exist?(bufdir)
 
-      plugin.configure(config_element('buffer', '', {'path' => bufpath, 'dir_permission' => 0700}))
+      plugin.configure(config_element('buffer', '', {'path' => bufpath, 'dir_permission' => '0700'}))
       assert !File.exist?(bufdir)
 
       plugin.start
@@ -215,7 +282,7 @@ class FileBufferTest < Test::Unit::TestCase
       FileUtils.rm_r bufdir if File.exist?(bufdir)
       assert !File.exist?(bufdir)
 
-      plugin.configure(config_element('buffer', '', {'path' => bufpath, 'file_permission' => 0600}))
+      plugin.configure(config_element('buffer', '', {'path' => bufpath, 'file_permission' => '0600'}))
       assert !File.exist?(bufdir)
       plugin.start
 
@@ -438,6 +505,168 @@ class FileBufferTest < Test::Unit::TestCase
     end
   end
 
+  sub_test_case 'there are some existing file chunks, both in specified path and per-worker directory under specified path, configured as multi workers' do
+    setup do
+      @bufdir = File.expand_path('../../tmp/buffer_file/path', __FILE__)
+      @worker0_dir = File.join(@bufdir, "worker0")
+      @worker1_dir = File.join(@bufdir, "worker1")
+      FileUtils.rm_rf @bufdir
+      FileUtils.mkdir_p @worker0_dir
+      FileUtils.mkdir_p @worker1_dir
+
+      @bufdir_chunk_1 = Fluent::UniqueId.generate
+      bc1 = File.join(@bufdir, "buffer.q#{Fluent::UniqueId.hex(@bufdir_chunk_1)}.log")
+      File.open(bc1, 'wb') do |f|
+        f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t2.test", event_time('2016-04-17 13:58:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t3.test", event_time('2016-04-17 13:58:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t4.test", event_time('2016-04-17 13:58:22 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+      end
+      write_metadata(
+        bc1 + '.meta', @bufdir_chunk_1, metadata(timekey: event_time('2016-04-17 13:58:00 -0700').to_i),
+        4, event_time('2016-04-17 13:58:00 -0700').to_i, event_time('2016-04-17 13:58:22 -0700').to_i
+      )
+
+      @bufdir_chunk_2 = Fluent::UniqueId.generate
+      bc2 = File.join(@bufdir, "buffer.q#{Fluent::UniqueId.hex(@bufdir_chunk_2)}.log")
+      File.open(bc2, 'wb') do |f|
+        f.write ["t1.test", event_time('2016-04-17 13:58:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t2.test", event_time('2016-04-17 13:58:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t3.test", event_time('2016-04-17 13:58:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        f.write ["t4.test", event_time('2016-04-17 13:58:22 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+      end
+      write_metadata(
+        bc2 + '.meta', @bufdir_chunk_2, metadata(timekey: event_time('2016-04-17 13:58:00 -0700').to_i),
+        4, event_time('2016-04-17 13:58:00 -0700').to_i, event_time('2016-04-17 13:58:22 -0700').to_i
+      )
+
+      @worker_dir_chunk_1 = Fluent::UniqueId.generate
+      wc0_1 = File.join(@worker0_dir, "buffer.q#{Fluent::UniqueId.hex(@worker_dir_chunk_1)}.log")
+      wc1_1 = File.join(@worker1_dir, "buffer.q#{Fluent::UniqueId.hex(@worker_dir_chunk_1)}.log")
+      [wc0_1, wc1_1].each do |chunk_path|
+        File.open(chunk_path, 'wb') do |f|
+          f.write ["t1.test", event_time('2016-04-17 13:59:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+          f.write ["t2.test", event_time('2016-04-17 13:59:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+          f.write ["t3.test", event_time('2016-04-17 13:59:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        end
+        write_metadata(
+          chunk_path + '.meta', @worker_dir_chunk_1, metadata(timekey: event_time('2016-04-17 13:59:00 -0700').to_i),
+          3, event_time('2016-04-17 13:59:00 -0700').to_i, event_time('2016-04-17 13:59:23 -0700').to_i
+        )
+      end
+
+      @worker_dir_chunk_2 = Fluent::UniqueId.generate
+      wc0_2 = File.join(@worker0_dir, "buffer.b#{Fluent::UniqueId.hex(@worker_dir_chunk_2)}.log")
+      wc1_2 = File.join(@worker1_dir, "buffer.b#{Fluent::UniqueId.hex(@worker_dir_chunk_2)}.log")
+      [wc0_2, wc1_2].each do |chunk_path|
+        File.open(chunk_path, 'wb') do |f|
+          f.write ["t1.test", event_time('2016-04-17 14:00:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+          f.write ["t2.test", event_time('2016-04-17 14:00:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+          f.write ["t3.test", event_time('2016-04-17 14:00:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+          f.write ["t4.test", event_time('2016-04-17 14:00:28 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        end
+        write_metadata(
+          chunk_path + '.meta', @worker_dir_chunk_2, metadata(timekey: event_time('2016-04-17 14:00:00 -0700').to_i),
+          4, event_time('2016-04-17 14:00:00 -0700').to_i, event_time('2016-04-17 14:00:28 -0700').to_i
+        )
+      end
+
+      @worker_dir_chunk_3 = Fluent::UniqueId.generate
+      wc0_3 = File.join(@worker0_dir, "buffer.b#{Fluent::UniqueId.hex(@worker_dir_chunk_3)}.log")
+      wc1_3 = File.join(@worker1_dir, "buffer.b#{Fluent::UniqueId.hex(@worker_dir_chunk_3)}.log")
+      [wc0_3, wc1_3].each do |chunk_path|
+        File.open(chunk_path, 'wb') do |f|
+          f.write ["t1.test", event_time('2016-04-17 14:01:15 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+          f.write ["t2.test", event_time('2016-04-17 14:01:17 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+          f.write ["t3.test", event_time('2016-04-17 14:01:21 -0700').to_i, {"message" => "yay"}].to_json + "\n"
+        end
+        write_metadata(
+          chunk_path + '.meta', @worker_dir_chunk_3, metadata(timekey: event_time('2016-04-17 14:01:00 -0700').to_i),
+          3, event_time('2016-04-17 14:01:00 -0700').to_i, event_time('2016-04-17 14:01:25 -0700').to_i
+        )
+      end
+
+      Fluent::Test.setup
+    end
+
+    teardown do
+      if @p
+        @p.stop unless @p.stopped?
+        @p.before_shutdown unless @p.before_shutdown?
+        @p.shutdown unless @p.shutdown?
+        @p.after_shutdown unless @p.after_shutdown?
+        @p.close unless @p.closed?
+        @p.terminate unless @p.terminated?
+      end
+    end
+
+    test 'worker(id=0) #resume returns staged/queued chunks with metadata, not only in worker dir, including the directory specified by path' do
+      ENV['SERVERENGINE_WORKER_ID'] = '0'
+
+      buf_conf = config_element('buffer', '', {'path' => @bufdir})
+      @d = FluentPluginFileBufferTest::DummyOutputPlugin.new
+      with_worker_config(workers: 2, worker_id: 0) do
+        @d.configure(config_element('output', '', {}, [buf_conf]))
+      end
+
+      @d.start
+      @p = @d.buffer
+
+      assert_equal 2, @p.stage.size
+      assert_equal 3, @p.queue.size
+
+      stage = @p.stage
+
+      m1 = metadata(timekey: event_time('2016-04-17 14:00:00 -0700').to_i)
+      assert_equal @worker_dir_chunk_2, stage[m1].unique_id
+      assert_equal 4, stage[m1].size
+      assert_equal :staged, stage[m1].state
+
+      m2 = metadata(timekey: event_time('2016-04-17 14:01:00 -0700').to_i)
+      assert_equal @worker_dir_chunk_3, stage[m2].unique_id
+      assert_equal 3, stage[m2].size
+      assert_equal :staged, stage[m2].state
+
+      queue = @p.queue
+
+      assert_equal [@bufdir_chunk_1, @bufdir_chunk_2, @worker_dir_chunk_1].sort, queue.map(&:unique_id).sort
+      assert_equal [3, 4, 4], queue.map(&:size).sort
+      assert_equal [:queued, :queued, :queued], queue.map(&:state)
+    end
+
+    test 'worker(id=1) #resume returns staged/queued chunks with metadata, only in worker dir' do
+      buf_conf = config_element('buffer', '', {'path' => @bufdir})
+      @d = FluentPluginFileBufferTest::DummyOutputPlugin.new
+      with_worker_config(workers: 2, worker_id: 1) do
+        @d.configure(config_element('output', '', {}, [buf_conf]))
+      end
+
+      @d.start
+      @p = @d.buffer
+
+      assert_equal 2, @p.stage.size
+      assert_equal 1, @p.queue.size
+
+      stage = @p.stage
+
+      m1 = metadata(timekey: event_time('2016-04-17 14:00:00 -0700').to_i)
+      assert_equal @worker_dir_chunk_2, stage[m1].unique_id
+      assert_equal 4, stage[m1].size
+      assert_equal :staged, stage[m1].state
+
+      m2 = metadata(timekey: event_time('2016-04-17 14:01:00 -0700').to_i)
+      assert_equal @worker_dir_chunk_3, stage[m2].unique_id
+      assert_equal 3, stage[m2].size
+      assert_equal :staged, stage[m2].state
+
+      queue = @p.queue
+
+      assert_equal @worker_dir_chunk_1, queue[0].unique_id
+      assert_equal 3, queue[0].size
+      assert_equal :queued, queue[0].state
+    end
+  end
+
   sub_test_case 'there are some existing file chunks without metadata file' do
     setup do
       @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__)
@@ -545,6 +774,9 @@ class FileBufferTest < Test::Unit::TestCase
     setup do
       @bufdir = File.expand_path('../../tmp/buffer_file', __FILE__)
 
+      FileUtils.rm_rf @bufdir
+      FileUtils.mkdir_p @bufdir
+
       @c1id = Fluent::UniqueId.generate
       p1 = File.join(@bufdir, "etest.201604171358.q#{Fluent::UniqueId.hex(@c1id)}.log")
       File.open(p1, 'wb') do |f|

data/test/plugin/test_buffer.rb

@@ -138,7 +138,7 @@ class BufferTest < Test::Unit::TestCase
     end
 
     test 'chunk records limit is ignored in default' do
-      assert_nil @p.chunk_records_limit
+      assert_nil @p.chunk_limit_records
     end
 
     test '#storable? checks total size of staged and enqueued(includes dequeued chunks) against total_limit_size' do
@@ -631,7 +631,7 @@ class BufferTest < Test::Unit::TestCase
         define_method(:commit){ raise "yay" }
       end
 
-      assert_raise "yay" do
+      assert_raise RuntimeError.new("yay") do
         @p.write({m => [row]})
       end
 
@@ -810,7 +810,7 @@ class BufferTest < Test::Unit::TestCase
           [event_time('2016-04-11 16:40:04 +0000'), {"message" => "z" * 1024 * 128}],
         ]
       )
-      assert_raise "yay" do
+      assert_raise RuntimeError.new("yay") do
         @p.write({m => es2}, format: ->(e){e.to_msgpack_stream})
       end
 
@@ -1113,9 +1113,9 @@ class BufferTest < Test::Unit::TestCase
     end
   end
 
-  sub_test_case 'with configuration includes chunk_records_limit' do
+  sub_test_case 'with configuration includes chunk_limit_records' do
     setup do
-      @p = create_buffer({"chunk_limit_size" => 1024, "total_limit_size" => 10240, "chunk_records_limit" => 6})
+      @p = create_buffer({"chunk_limit_size" => 1024, "total_limit_size" => 10240, "chunk_limit_records" => 6})
       @dm0 = dm0 = create_metadata(Time.parse('2016-04-11 16:00:00 +0000').to_i, nil, nil)
       @dm1 = dm1 = create_metadata(Time.parse('2016-04-11 16:10:00 +0000').to_i, nil, nil)
       @dm2 = dm2 = create_metadata(Time.parse('2016-04-11 16:20:00 +0000').to_i, nil, nil)
@@ -1139,7 +1139,7 @@ class BufferTest < Test::Unit::TestCase
     end
 
     test '#chunk_size_over? returns true if too many records exists in a chunk even if its bytes is less than limit' do
-      assert_equal 6, @p.chunk_records_limit
+      assert_equal 6, @p.chunk_limit_records
 
       m = create_metadata(Time.parse('2016-04-11 16:40:00 +0000').to_i)
 
@@ -1155,7 +1155,7 @@ class BufferTest < Test::Unit::TestCase
     end
 
     test '#chunk_size_full? returns true if enough many records exists in a chunk even if its bytes is less than limit' do
-      assert_equal 6, @p.chunk_records_limit
+      assert_equal 6, @p.chunk_limit_records
 
       m = create_metadata(Time.parse('2016-04-11 16:40:00 +0000').to_i)
 
@@ -1171,9 +1171,9 @@ class BufferTest < Test::Unit::TestCase
     end
   end
 
-  sub_test_case 'with configuration includes queue_length_limit' do
+  sub_test_case 'with configuration includes queue_limit_length' do
     setup do
-      @p = create_buffer({"chunk_limit_size" => 1024, "total_limit_size" => 10240, "queue_length_limit" => 5})
+      @p = create_buffer({"chunk_limit_size" => 1024, "total_limit_size" => 10240, "queue_limit_length" => 5})
       @dm0 = dm0 = create_metadata(Time.parse('2016-04-11 16:00:00 +0000').to_i, nil, nil)
       @dm1 = dm1 = create_metadata(Time.parse('2016-04-11 16:10:00 +0000').to_i, nil, nil)
       @dm2 = dm2 = create_metadata(Time.parse('2016-04-11 16:20:00 +0000').to_i, nil, nil)
@@ -1196,9 +1196,9 @@ class BufferTest < Test::Unit::TestCase
       @p.start
     end
 
-    test '#configure will overwrite standard configuration if queue_length_limit' do
+    test '#configure will overwrite standard configuration if queue_limit_length' do
       assert_equal 1024, @p.chunk_limit_size
-      assert_equal 5, @p.queue_length_limit
+      assert_equal 5, @p.queue_limit_length
       assert_equal (1024*5), @p.total_limit_size
     end
   end

data/test/plugin/test_buffer_file_chunk.rb

@@ -63,13 +63,13 @@ class BufferFileChunkTest < Test::Unit::TestCase
 
     test '.generate_stage_chunk_path generates path with staged mark & chunk unique_id' do
       assert_equal gen_path("mychunk.b52fde6425d7406bdb19b936e1a1ba98c.log"), @klass.generate_stage_chunk_path(gen_path("mychunk.*.log"), gen_test_chunk_id)
-      assert_raise "BUG: buffer chunk path on stage MUST have '.*.'" do
+      assert_raise RuntimeError.new("BUG: buffer chunk path on stage MUST have '.*.'") do
         @klass.generate_stage_chunk_path(gen_path("mychunk.log"), gen_test_chunk_id)
       end
-      assert_raise "BUG: buffer chunk path on stage MUST have '.*.'" do
+      assert_raise RuntimeError.new("BUG: buffer chunk path on stage MUST have '.*.'") do
         @klass.generate_stage_chunk_path(gen_path("mychunk.*"), gen_test_chunk_id)
       end
-      assert_raise "BUG: buffer chunk path on stage MUST have '.*.'" do
+      assert_raise RuntimeError.new("BUG: buffer chunk path on stage MUST have '.*.'") do
         @klass.generate_stage_chunk_path(gen_path("*.log"), gen_test_chunk_id)
       end
     end
@@ -679,7 +679,7 @@ class BufferFileChunkTest < Test::Unit::TestCase
       assert_equal @d.bytesize, @c.bytesize
       assert_equal @d, @c.read
 
-      assert_raise "BUG: appending to non-staged chunk, now 'queued'" do
+      assert_raise RuntimeError.new("BUG: concatenating to unwritable chunk, now 'queued'") do
         @c.append(["queued chunk is read only"])
       end
       assert_raise IOError do
@@ -721,7 +721,7 @@ class BufferFileChunkTest < Test::Unit::TestCase
       assert_equal 0, @c.size
       assert_equal @d, @c.read
 
-      assert_raise "BUG: appending to non-staged chunk, now 'queued'" do
+      assert_raise RuntimeError.new("BUG: concatenating to unwritable chunk, now 'queued'") do
         @c.append(["queued chunk is read only"])
       end
       assert_raise IOError do
@@ -763,7 +763,7 @@ class BufferFileChunkTest < Test::Unit::TestCase
       assert_equal 0, @c.size
       assert_equal @d, @c.read
 
-      assert_raise "BUG: appending to non-staged chunk, now 'queued'" do
+      assert_raise RuntimeError.new("BUG: concatenating to unwritable chunk, now 'queued'") do
         @c.append(["queued chunk is read only"])
       end
       assert_raise IOError do

data/test/plugin/test_compressable.rb

@@ -18,6 +18,7 @@ class CompressableTest < Test::Unit::TestCase
     test 'write compressed data to IO with output_io option' do
       io = StringIO.new
       compress(@src, output_io: io)
+      waiting(10){ sleep 0.1 until @gzipped_src == io.string }
       assert_equal @gzipped_src, io.string
     end
   end
@@ -35,6 +36,7 @@ class CompressableTest < Test::Unit::TestCase
     test 'write decompressed data to IO with output_io option' do
       io = StringIO.new
       decompress(@gzipped_src, output_io: io)
+      waiting(10){ sleep 0.1 until @src == io.string }
       assert_equal @src, io.string
     end
 
@@ -56,6 +58,7 @@ class CompressableTest < Test::Unit::TestCase
       output_io = StringIO.new
 
       decompress(input_io: input_io, output_io: output_io)
+      waiting(10){ sleep 0.1 until @src == output_io.string }
       assert_equal @src, output_io.string
     end
 

data/test/plugin/test_filter.rb

@@ -165,6 +165,10 @@ class FilterPluginTest < Test::Unit::TestCase
       end
     end
 
+    test 'are available with multi worker configuration in default' do
+      assert @p.multi_workers_ready?
+    end
+
     test 'filters events correctly' do
       test_es = [
         [event_time('2016-04-19 13:01:00 -0700'), {"num" => "1", "message" => "Hello filters!"}],