fluentd 0.14.17-x64-mingw32 → 1.3.1-x64-mingw32

data/test/plugin/test_output_as_buffered.rb

@@ -219,6 +219,24 @@ class BufferedOutputTest < Test::Unit::TestCase
     Timecop.return
   end
 
+  test 'queued_chunks_limit_size is same as flush_thread_count by default' do
+    hash = {'flush_thread_count' => 4}
+    i = create_output
+    i.register(:prefer_buffered_processing) { true }
+    i.configure(config_element('ROOT', '', {}, [config_element('buffer','tag',hash)]))
+
+    assert_equal 4, i.buffer.queued_chunks_limit_size
+  end
+
+  test 'prefer queued_chunks_limit_size parameter than flush_thread_count' do
+    hash = {'flush_thread_count' => 4, 'queued_chunks_limit_size' => 2}
+    i = create_output
+    i.register(:prefer_buffered_processing) { true }
+    i.configure(config_element('ROOT', '', {}, [config_element('buffer','tag',hash)]))
+
+    assert_equal 2, i.buffer.queued_chunks_limit_size
+  end
+
   sub_test_case 'chunk feature in #write for output plugins' do
     setup do
       @stored_global_logger = $log
@@ -1064,6 +1082,7 @@ class BufferedOutputTest < Test::Unit::TestCase
         'flush_thread_count' => 1,
         'flush_thread_burst_interval' => 0.1,
         'chunk_limit_size' => 1024,
+        'queued_chunks_limit_size' => 100
       }
       @i = create_output(:buffered)
       @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))

data/test/plugin/test_output_as_buffered_backup.rb

@@ -0,0 +1,307 @@
+require_relative '../helper'
+require 'fluent/plugin/output'
+require 'fluent/plugin/buffer'
+require 'fluent/event'
+require 'fluent/error'
+
+require 'json'
+require 'time'
+require 'timeout'
+require 'timecop'
+
+
+class BufferedOutputBackupTest < Test::Unit::TestCase
+  class BareOutput < Fluent::Plugin::Output
+    def register(name, &block)
+      instance_variable_set("@#{name}", block)
+    end
+  end
+  class DummyOutput < BareOutput
+    def initialize
+      super
+      @process = nil
+      @format = nil
+      @write = nil
+      @try_write = nil
+    end
+    def prefer_buffered_processing
+      true
+    end
+    def prefer_delayed_commit
+      false
+    end
+    def process(tag, es)
+      @process ? @process.call(tag, es) : nil
+    end
+    def format(tag, time, record)
+      [tag, time.to_i, record].to_json + "\n"
+    end
+    def write(chunk)
+      @write ? @write.call(chunk) : nil
+    end
+    def try_write(chunk)
+      @try_write ? @try_write.call(chunk) : nil
+    end
+  end
+  class DummyOutputForSecondary < BareOutput
+    def initialize
+      super
+      @process = nil
+      @format = nil
+      @write = nil
+      @try_write = nil
+    end
+    def prefer_buffered_processing
+      true
+    end
+    def prefer_delayed_commit
+      false
+    end
+    def process(tag, es)
+      @process ? @process.call(tag, es) : nil
+    end
+    def format(tag, time, record)
+      [tag, time.to_i, record].to_json + "\n"
+    end
+    def write(chunk)
+      @write ? @write.call(chunk) : nil
+    end
+    def try_write(chunk)
+      @try_write ? @try_write.call(chunk) : nil
+    end
+  end
+  class DummyAsyncOutputForSecondary < BareOutput
+    def initialize
+      super
+      @process = nil
+      @format = nil
+      @write = nil
+      @try_write = nil
+    end
+    def prefer_buffered_processing
+      true
+    end
+    def prefer_delayed_commit
+      true
+    end
+    def process(tag, es)
+      @process ? @process.call(tag, es) : nil
+    end
+    def format(tag, time, record)
+      [tag, time.to_i, record].to_json + "\n"
+    end
+    def write(chunk)
+      @write ? @write.call(chunk) : nil
+    end
+    def try_write(chunk)
+      @try_write ? @try_write.call(chunk) : nil
+    end
+  end
+
+  TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/../tmp/bu#{ENV['TEST_ENV_NUMBER']}")
+
+  def create_output
+    DummyOutput.new
+  end
+  def create_metadata(timekey: nil, tag: nil, variables: nil)
+    Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)
+  end
+  def waiting(seconds)
+    begin
+      Timeout.timeout(seconds) do
+        yield
+      end
+    rescue Timeout::Error
+      STDERR.print(*@i.log.out.logs)
+      raise
+    end
+  end
+
+  def dummy_event_stream
+    Fluent::ArrayEventStream.new([
+      [ event_time('2016-04-13 18:33:00'), {"name" => "moris", "age" => 36, "message" => "data1"} ],
+      [ event_time('2016-04-13 18:33:13'), {"name" => "moris", "age" => 36, "message" => "data2"} ],
+      [ event_time('2016-04-13 18:33:32'), {"name" => "moris", "age" => 36, "message" => "data3"} ],
+    ])
+  end
+
+  setup do
+    @i = create_output
+    FileUtils.rm_rf(TMP_DIR)
+    FileUtils.mkdir_p(TMP_DIR)
+
+    Fluent::Plugin.register_output('backup_output', DummyOutput)
+    Fluent::Plugin.register_output('backup_output2', DummyOutputForSecondary)
+    Fluent::Plugin.register_output('backup_async_output', DummyAsyncOutputForSecondary)
+  end
+
+  teardown do
+    if @i
+      @i.stop unless @i.stopped?
+      @i.before_shutdown unless @i.before_shutdown?
+      @i.shutdown unless @i.shutdown?
+      @i.after_shutdown unless @i.after_shutdown?
+      @i.close unless @i.closed?
+      @i.terminate unless @i.terminated?
+    end
+    Timecop.return
+  end
+
+  sub_test_case 'buffered output for broken chunks' do
+    def flush_chunks
+      @i.start
+      @i.after_start
+
+      @i.interrupt_flushes
+
+      now = Time.parse('2016-04-13 18:33:30 -0700')
+      Timecop.freeze(now)
+      @i.emit_events("test.tag.1", dummy_event_stream())
+      now = Time.parse('2016-04-13 18:33:32 -0700')
+      Timecop.freeze(now)
+
+      @i.enqueue_thread_wait
+      @i.flush_thread_wakeup
+      waiting(4) { Thread.pass until @i.write_count > 0 }
+
+      assert { @i.write_count > 0 }
+      Timecop.freeze(now)
+      @i.flush_thread_wakeup
+    end
+
+    def wait_flush(target_file)
+      waiting(5) {
+        target_dir = File.join(File.dirname(target_file), "*")
+        while Dir.glob(target_dir).size.zero?
+        end
+      }
+    end
+
+    test 'backup chunk without secondary' do
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR) do
+        id = 'backup_test'
+        hash = {
+          'flush_interval' => 1,
+          'flush_thread_burst_interval' => 0.1,
+        }
+        chunk_id = nil
+        @i.configure(config_element('ROOT', '', {'@id' => id}, [config_element('buffer', 'tag', hash)]))
+        @i.register(:write) { |chunk|
+          chunk_id = chunk.unique_id;
+          raise Fluent::UnrecoverableError, "yay, your #write must fail"
+        }
+
+        flush_chunks
+
+        target = "#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log"
+        wait_flush(target)
+        assert_true File.exist?(target)
+        logs = @i.log.out.logs
+        assert { logs.any? { |l| l.include?("got unrecoverable error in primary and no secondary") } }
+      end
+    end
+
+    test 'backup chunk with same type secondary' do
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR) do
+        id = 'backup_test_with_same_secondary'
+        hash = {
+          'flush_interval' => 1,
+          'flush_thread_burst_interval' => 0.1,
+        }
+        chunk_id = nil
+        secconf = config_element('secondary','',{'@type' => 'backup_output'})
+        @i.configure(config_element('ROOT', '', {'@id' => id}, [config_element('buffer', 'tag', hash), secconf]))
+        @i.register(:write) { |chunk|
+          chunk_id = chunk.unique_id;
+          raise Fluent::UnrecoverableError, "yay, your #write must fail"
+        }
+
+        flush_chunks
+
+        target = "#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log"
+        wait_flush(target)
+        assert_true File.exist?(target)
+        logs = @i.log.out.logs
+        assert { logs.any? { |l| l.include?("got unrecoverable error in primary and secondary type is same as primary") } }
+      end
+    end
+
+    test 'backup chunk with different type secondary' do
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR) do
+        id = 'backup_test_with_diff_secondary'
+        hash = {
+          'flush_interval' => 1,
+          'flush_thread_burst_interval' => 0.1,
+        }
+        chunk_id = nil
+        secconf = config_element('secondary','',{'@type' => 'backup_output2'})
+        @i.configure(config_element('ROOT', '', {'@id' => id}, [config_element('buffer', 'tag', hash), secconf]))
+        @i.register(:write) { |chunk|
+          chunk_id = chunk.unique_id;
+          raise Fluent::UnrecoverableError, "yay, your #write must fail"
+        }
+        @i.secondary.register(:write) { |chunk|
+          raise Fluent::UnrecoverableError, "yay, your secondary #write must fail"
+        }
+
+        flush_chunks
+
+        target = "#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log"
+        wait_flush(target)
+        assert_true File.exist?(target)
+        logs = @i.log.out.logs
+        assert { logs.any? { |l| l.include?("got unrecoverable error in primary. Skip retry and flush chunk to secondary") } }
+        assert { logs.any? { |l| l.include?("got an error in secondary for unrecoverable error") } }
+      end
+    end
+
+    test 'backup chunk with async secondary' do
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR) do
+        id = 'backup_test_with_diff_secondary'
+        hash = {
+          'flush_interval' => 1,
+          'flush_thread_burst_interval' => 0.1,
+        }
+        chunk_id = nil
+        secconf = config_element('secondary','',{'@type' => 'backup_async_output'})
+        @i.configure(config_element('ROOT', '', {'@id' => id}, [config_element('buffer', 'tag', hash), secconf]))
+        @i.register(:write) { |chunk|
+          chunk_id = chunk.unique_id;
+          raise Fluent::UnrecoverableError, "yay, your #write must fail"
+        }
+
+        flush_chunks
+
+        target = "#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log"
+        wait_flush(target)
+        assert_true File.exist?(target)
+        logs = @i.log.out.logs
+        assert { logs.any? { |l| l.include?("got unrecoverable error in primary and secondary is async output") } }
+      end
+    end
+
+    test 'chunk is thrown away when disable_chunk_backup is true' do
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR) do
+        id = 'backup_test'
+        hash = {
+          'flush_interval' => 1,
+          'flush_thread_burst_interval' => 0.1,
+          'disable_chunk_backup' => true
+        }
+        chunk_id = nil
+        @i.configure(config_element('ROOT', '', {'@id' => id}, [config_element('buffer', 'tag', hash)]))
+        @i.register(:write) { |chunk|
+          chunk_id = chunk.unique_id;
+          raise Fluent::UnrecoverableError, "yay, your #write must fail"
+        }
+
+        flush_chunks
+
+        target = "#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log"
+        assert_false File.exist?(target)
+        logs = @i.log.out.logs
+        assert { logs.any? { |l| l.include?("disable_chunk_backup is true") } }
+      end
+    end
+  end
+end

data/test/plugin/test_output_as_buffered_retries.rb

@@ -123,6 +123,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
         'flush_interval' => 1,
         'flush_thread_burst_interval' => 0.1,
         'retry_randomize' => false,
+        'queued_chunks_limit_size' => 100
       }
       @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
       @i.register(:prefer_buffered_processing){ true }
@@ -252,6 +253,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
         'flush_thread_burst_interval' => 0.1,
         'retry_randomize' => false,
         'retry_timeout' => 3600,
+        'queued_chunks_limit_size' => 100
       }
       @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
       @i.register(:prefer_buffered_processing){ true }
@@ -342,6 +344,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
         'flush_thread_burst_interval' => 0.1,
         'retry_randomize' => false,
         'retry_max_times' => 10,
+        'queued_chunks_limit_size' => 100
       }
       @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
       @i.register(:prefer_buffered_processing){ true }
@@ -412,6 +415,68 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
       assert{ @i.buffer.stage.size == 1 }
       assert{ chunks.all?{|c| c.empty? } }
     end
+
+    test 'output plugin limits queued chunks via queued_chunks_limit_size' do
+      chunk_key = 'tag'
+      hash = {
+        'flush_interval' => 1,
+        'flush_thread_burst_interval' => 0.1,
+        'retry_randomize' => false,
+        'retry_max_times' => 7,
+        'queued_chunks_limit_size' => 2,
+      }
+      @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
+      @i.register(:prefer_buffered_processing) { true }
+      @i.register(:format) { |tag,time,record| [tag,time.to_i,record].to_json + "\n" }
+      @i.register(:write) { |chunk| raise "yay, your #write must fail" }
+      @i.start
+      @i.after_start
+
+      @i.interrupt_flushes
+
+      now = Time.parse('2016-04-13 18:33:30 -0700')
+      Timecop.freeze(now)
+
+      @i.emit_events("test.tag.1", dummy_event_stream())
+
+      now = Time.parse('2016-04-13 18:33:31 -0700')
+      Timecop.freeze(now)
+
+      @i.emit_events("test.tag.2", dummy_event_stream())
+
+      @i.enqueue_thread_wait
+      @i.flush_thread_wakeup
+      waiting(4) { Thread.pass until @i.write_count > 0 && @i.num_errors > 0 }
+
+      assert { @i.buffer.queue.size > 0 }
+      assert { @i.buffer.queue.first.metadata.tag == 'test.tag.1' }
+
+      assert { @i.write_count > 0 }
+      assert { @i.num_errors > 0 }
+
+      prev_write_count = @i.write_count
+      prev_num_errors = @i.num_errors
+
+      chunks = @i.buffer.queue.dup
+
+      20.times do |i| # large times enough
+        now = @i.next_flush_time
+
+        Timecop.freeze(now)
+        @i.enqueue_thread_wait
+        @i.flush_thread_wakeup
+        waiting(4) { Thread.pass until @i.write_count > prev_write_count && @i.num_errors > prev_num_errors }
+
+        @i.emit_events("test.tag.1", dummy_event_stream())
+        assert { @i.buffer.queue.size <= 2 }
+        assert { @i.buffer.stage.size == 1 } # all new data is stored into staged chunk
+
+        break if @i.buffer.queue.size == 0
+
+        prev_write_count = @i.write_count
+        prev_num_errors = @i.num_errors
+      end
+    end
   end
 
   sub_test_case 'bufferd output for retries with periodical retry' do
@@ -423,6 +488,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
         'retry_type' => :periodic,
         'retry_wait' => 3,
         'retry_randomize' => false,
+        'queued_chunks_limit_size' => 100
       }
       @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
       @i.register(:prefer_buffered_processing){ true }
@@ -469,6 +535,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
         'retry_wait' => 30,
         'retry_randomize' => false,
         'retry_timeout' => 120,
+        'queued_chunks_limit_size' => 100
       }
       @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
       @i.register(:prefer_buffered_processing){ true }
@@ -564,6 +631,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
         'retry_wait' => 3,
         'retry_randomize' => false,
         'retry_max_times' => 10,
+        'queued_chunks_limit_size' => 100
       }
       @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
       @i.register(:prefer_buffered_processing){ true }
@@ -662,6 +730,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
         'retry_randomize' => false,
         'retry_timeout' => 3600,
         'retry_max_times' => 10,
+        'queued_chunks_limit_size' => 100
       }
       @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
       @i.register(:prefer_buffered_processing){ true }
@@ -733,6 +802,7 @@ class BufferedOutputRetryTest < Test::Unit::TestCase
         'retry_wait' => 30,
         'retry_timeout' => 360,
         'retry_max_times' => 10,
+        'queued_chunks_limit_size' => 100
       }
       @i.configure(config_element('ROOT','',{},[config_element('buffer',chunk_key,hash)]))
       @i.register(:prefer_buffered_processing){ true }