dtomasgu-fluentd 1.14.7.pre.dev

data/test/plugin/test_output_as_buffered_backup.rb

@@ -0,0 +1,363 @@
+require_relative '../helper'
+require 'fluent/plugin/output'
+require 'fluent/plugin/buffer'
+require 'fluent/event'
+require 'fluent/error'
+
+require 'json'
+require 'time'
+require 'timeout'
+require 'timecop'
+
+
+class BufferedOutputBackupTest < Test::Unit::TestCase
+  class BareOutput < Fluent::Plugin::Output
+    def register(name, &block)
+      instance_variable_set("@#{name}", block)
+    end
+  end
+  class DummyOutput < BareOutput
+    def initialize
+      super
+      @process = nil
+      @format = nil
+      @write = nil
+      @try_write = nil
+    end
+    def prefer_buffered_processing
+      true
+    end
+    def prefer_delayed_commit
+      false
+    end
+    def process(tag, es)
+      @process ? @process.call(tag, es) : nil
+    end
+    def format(tag, time, record)
+      [tag, time.to_i, record].to_json + "\n"
+    end
+    def write(chunk)
+      @write ? @write.call(chunk) : nil
+    end
+    def try_write(chunk)
+      @try_write ? @try_write.call(chunk) : nil
+    end
+  end
+  class DummyOutputForSecondary < BareOutput
+    def initialize
+      super
+      @process = nil
+      @format = nil
+      @write = nil
+      @try_write = nil
+    end
+    def prefer_buffered_processing
+      true
+    end
+    def prefer_delayed_commit
+      false
+    end
+    def process(tag, es)
+      @process ? @process.call(tag, es) : nil
+    end
+    def format(tag, time, record)
+      [tag, time.to_i, record].to_json + "\n"
+    end
+    def write(chunk)
+      @write ? @write.call(chunk) : nil
+    end
+    def try_write(chunk)
+      @try_write ? @try_write.call(chunk) : nil
+    end
+  end
+  class DummyAsyncOutputForSecondary < BareOutput
+    def initialize
+      super
+      @process = nil
+      @format = nil
+      @write = nil
+      @try_write = nil
+    end
+    def prefer_buffered_processing
+      true
+    end
+    def prefer_delayed_commit
+      true
+    end
+    def process(tag, es)
+      @process ? @process.call(tag, es) : nil
+    end
+    def format(tag, time, record)
+      [tag, time.to_i, record].to_json + "\n"
+    end
+    def write(chunk)
+      @write ? @write.call(chunk) : nil
+    end
+    def try_write(chunk)
+      @try_write ? @try_write.call(chunk) : nil
+    end
+  end
+
+  TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/../tmp/bu#{ENV['TEST_ENV_NUMBER']}")
+
+  def create_output
+    DummyOutput.new
+  end
+  def create_metadata(timekey: nil, tag: nil, variables: nil)
+    Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)
+  end
+  def waiting(seconds)
+    begin
+      Timeout.timeout(seconds) do
+        yield
+      end
+    rescue Timeout::Error
+      STDERR.print(*@i.log.out.logs)
+      raise
+    end
+  end
+
+  def dummy_event_stream
+    Fluent::ArrayEventStream.new([
+      [ event_time('2016-04-13 18:33:00'), {"name" => "moris", "age" => 36, "message" => "data1"} ],
+      [ event_time('2016-04-13 18:33:13'), {"name" => "moris", "age" => 36, "message" => "data2"} ],
+      [ event_time('2016-04-13 18:33:32'), {"name" => "moris", "age" => 36, "message" => "data3"} ],
+    ])
+  end
+
+  setup do
+    @i = create_output
+    FileUtils.rm_rf(TMP_DIR)
+    FileUtils.mkdir_p(TMP_DIR)
+
+    Fluent::Plugin.register_output('backup_output', DummyOutput)
+    Fluent::Plugin.register_output('backup_output2', DummyOutputForSecondary)
+    Fluent::Plugin.register_output('backup_async_output', DummyAsyncOutputForSecondary)
+  end
+
+  teardown do
+    if @i
+      @i.stop unless @i.stopped?
+      @i.before_shutdown unless @i.before_shutdown?
+      @i.shutdown unless @i.shutdown?
+      @i.after_shutdown unless @i.after_shutdown?
+      @i.close unless @i.closed?
+      @i.terminate unless @i.terminated?
+    end
+    Timecop.return
+  end
+
+  sub_test_case 'buffered output for broken chunks' do
+    def flush_chunks
+      @i.start
+      @i.after_start
+
+      @i.interrupt_flushes
+
+      now = Time.parse('2016-04-13 18:33:30 -0700')
+      Timecop.freeze(now)
+      @i.emit_events("test.tag.1", dummy_event_stream())
+      now = Time.parse('2016-04-13 18:33:32 -0700')
+      Timecop.freeze(now)
+
+      @i.enqueue_thread_wait
+      @i.flush_thread_wakeup
+      waiting(4) { Thread.pass until @i.write_count > 0 }
+
+      assert { @i.write_count > 0 }
+      Timecop.freeze(now)
+      @i.flush_thread_wakeup
+    end
+
+    def wait_flush(target_file)
+      waiting(5) {
+        target_dir = File.join(File.dirname(target_file), "*")
+        while Dir.glob(target_dir).size.zero?
+          # Avoid to lose globbed entries on Windows in busy loop
+          sleep 0.1 if Fluent.windows?
+        end
+      }
+    end
+
+    data('unrecoverable error' => Fluent::UnrecoverableError,
+         'type error' => TypeError,
+         'argument error' => ArgumentError,
+         'no method error' => NoMethodError,
+         'msgpack unpack error' => MessagePack::UnpackError,
+         'encoding error' => EncodingError)
+    test 'backup chunk without secondary' do |error_class|
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR) do
+        id = 'backup_test'
+        hash = {
+          'flush_interval' => 1,
+          'flush_thread_burst_interval' => 0.1,
+        }
+        chunk_id = nil
+        @i.configure(config_element('ROOT', '', {'@id' => id}, [config_element('buffer', 'tag', hash)]))
+        @i.register(:write) { |chunk|
+          chunk_id = chunk.unique_id;
+          raise error_class, "yay, your #write must fail"
+        }
+
+        flush_chunks
+
+        target = "#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log"
+        wait_flush(target)
+        assert_true File.exist?(target)
+        logs = @i.log.out.logs
+        assert { logs.any? { |l| l.include?("got unrecoverable error in primary and no secondary") } }
+      end
+    end
+
+    test 'backup chunk with same type secondary' do
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR) do
+        id = 'backup_test_with_same_secondary'
+        hash = {
+          'flush_interval' => 1,
+          'flush_thread_burst_interval' => 0.1,
+        }
+        chunk_id = nil
+        secconf = config_element('secondary','',{'@type' => 'backup_output'})
+        @i.configure(config_element('ROOT', '', {'@id' => id}, [config_element('buffer', 'tag', hash), secconf]))
+        @i.register(:write) { |chunk|
+          chunk_id = chunk.unique_id;
+          raise Fluent::UnrecoverableError, "yay, your #write must fail"
+        }
+
+        flush_chunks
+
+        target = "#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log"
+        wait_flush(target)
+        assert_true File.exist?(target)
+        logs = @i.log.out.logs
+        assert { logs.any? { |l| l.include?("got unrecoverable error in primary and secondary type is same as primary") } }
+      end
+    end
+
+    test 'create directory' do
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR) do
+        id = 'backup_test_with_same_secondary'
+        hash = { 'flush_interval' => 1, 'flush_thread_burst_interval' => 0.1 }
+        chunk_id = nil
+        secconf = config_element('secondary', '', { '@type' => 'backup_output' })
+        @i.configure(config_element('ROOT', '', { '@id' => id }, [config_element('buffer', 'tag', hash), secconf]))
+        @i.register(:write) { |chunk|
+          chunk_id = chunk.unique_id;
+          raise Fluent::UnrecoverableError, "yay, your #write must fail"
+        }
+
+        flush_chunks
+
+        target = "#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log"
+        target_dir = File.dirname(target)
+        wait_flush(target)
+
+        assert_path_exist(target_dir)
+        assert_equal '755', File.stat(target_dir).mode.to_s(8)[-3, 3]
+      end
+    end
+
+    test 'create directory with specific mode' do
+      omit "NTFS doesn't support UNIX like permissions" if Fluent.windows?
+
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR, 'dir_permission' => '744') do
+        id = 'backup_test_with_same_secondary'
+        hash = { 'flush_interval' => 1, 'flush_thread_burst_interval' => 0.1 }
+        chunk_id = nil
+        secconf = config_element('secondary', '', { '@type' => 'backup_output' })
+        @i.configure(config_element('ROOT', '', { '@id' => id }, [config_element('buffer', 'tag', hash), secconf]))
+        @i.register(:write) { |chunk|
+          chunk_id = chunk.unique_id;
+          raise Fluent::UnrecoverableError, "yay, your #write must fail"
+        }
+
+        flush_chunks
+
+        target = "#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log"
+        target_dir = File.dirname(target)
+        wait_flush(target)
+
+        assert_path_exist(target_dir)
+        assert_equal '744', File.stat(target_dir).mode.to_s(8)[-3, 3]
+      end
+    end
+
+    test 'backup chunk with different type secondary' do
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR) do
+        id = 'backup_test_with_diff_secondary'
+        hash = {
+          'flush_interval' => 1,
+          'flush_thread_burst_interval' => 0.1,
+        }
+        chunk_id = nil
+        secconf = config_element('secondary','',{'@type' => 'backup_output2'})
+        @i.configure(config_element('ROOT', '', {'@id' => id}, [config_element('buffer', 'tag', hash), secconf]))
+        @i.register(:write) { |chunk|
+          chunk_id = chunk.unique_id;
+          raise Fluent::UnrecoverableError, "yay, your #write must fail"
+        }
+        @i.secondary.register(:write) { |chunk|
+          raise Fluent::UnrecoverableError, "yay, your secondary #write must fail"
+        }
+
+        flush_chunks
+
+        target = "#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log"
+        wait_flush(target)
+        assert_true File.exist?(target)
+        logs = @i.log.out.logs
+        assert { logs.any? { |l| l.include?("got unrecoverable error in primary. Skip retry and flush chunk to secondary") } }
+        assert { logs.any? { |l| l.include?("got an error in secondary for unrecoverable error") } }
+      end
+    end
+
+    test 'backup chunk with async secondary' do
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR) do
+        id = 'backup_test_with_diff_secondary'
+        hash = {
+          'flush_interval' => 1,
+          'flush_thread_burst_interval' => 0.1,
+        }
+        chunk_id = nil
+        secconf = config_element('secondary','',{'@type' => 'backup_async_output'})
+        @i.configure(config_element('ROOT', '', {'@id' => id}, [config_element('buffer', 'tag', hash), secconf]))
+        @i.register(:write) { |chunk|
+          chunk_id = chunk.unique_id;
+          raise Fluent::UnrecoverableError, "yay, your #write must fail"
+        }
+
+        flush_chunks
+
+        target = "#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log"
+        wait_flush(target)
+        assert_true File.exist?(target)
+        logs = @i.log.out.logs
+        assert { logs.any? { |l| l.include?("got unrecoverable error in primary and secondary is async output") } }
+      end
+    end
+
+    test 'chunk is thrown away when disable_chunk_backup is true' do
+      Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR) do
+        id = 'backup_test'
+        hash = {
+          'flush_interval' => 1,
+          'flush_thread_burst_interval' => 0.1,
+          'disable_chunk_backup' => true
+        }
+        chunk_id = nil
+        @i.configure(config_element('ROOT', '', {'@id' => id}, [config_element('buffer', 'tag', hash)]))
+        @i.register(:write) { |chunk|
+          chunk_id = chunk.unique_id;
+          raise Fluent::UnrecoverableError, "yay, your #write must fail"
+        }
+
+        flush_chunks
+
+        target = "#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log"
+        assert_false File.exist?(target)
+        logs = @i.log.out.logs
+        assert { logs.any? { |l| l.include?("disable_chunk_backup is true") } }
+      end
+    end
+  end
+end

data/test/plugin/test_output_as_buffered_compress.rb

@@ -0,0 +1,165 @@
+require_relative '../helper'
+require 'fluent/plugin/output'
+require 'fluent/plugin/buffer'
+require 'fluent/plugin/compressable'
+require 'fluent/event'
+
+require 'timeout'
+
+module FluentPluginOutputAsBufferedCompressTest
+  class DummyBareOutput < Fluent::Plugin::Output
+    def register(name, &block)
+      instance_variable_set("@#{name}", block)
+    end
+  end
+
+  class DummyAsyncOutput < DummyBareOutput
+    def initialize
+      super
+      @format = @write = nil
+    end
+    def write(chunk)
+      @write ? @write.call(chunk) : nil
+    end
+  end
+
+  class DummyAsyncOutputWithFormat < DummyBareOutput
+    def initialize
+      super
+      @format = nil
+    end
+    def write(chunk)
+      @write ? @write.call(chunk) : nil
+    end
+    def format(tag, time, record)
+      @format ? @format.call(tag, time, record) : [tag, time, record].to_json
+    end
+  end
+end
+
+class BufferedOutputCompressTest < Test::Unit::TestCase
+  include Fluent::Plugin::Compressable
+
+  def create_output(type=:async)
+    case type
+    when :async then FluentPluginOutputAsBufferedCompressTest::DummyAsyncOutput.new
+    when :async_with_format then FluentPluginOutputAsBufferedCompressTest::DummyAsyncOutputWithFormat.new
+    else
+      raise ArgumentError, "unknown type: #{type}"
+    end
+  end
+
+  def waiting(seconds)
+    begin
+      Timeout.timeout(seconds) do
+        yield
+      end
+    rescue Timeout::Error
+      STDERR.print(*@i.log.out.logs)
+      raise
+    end
+  end
+
+  def dummy_event_stream
+    Fluent::ArrayEventStream.new(
+      [
+        [event_time('2016-04-13 18:33:00'), { 'name' => 'moris', 'age' => 36, 'message' => 'data1' }],
+        [event_time('2016-04-13 18:33:13'), { 'name' => 'moris', 'age' => 36, 'message' => 'data2' }],
+        [event_time('2016-04-13 18:33:32'), { 'name' => 'moris', 'age' => 36, 'message' => 'data3' }],
+      ]
+    )
+  end
+
+  TMP_DIR = File.expand_path('../../tmp/test_output_as_buffered_compress', __FILE__)
+
+  setup do
+    FileUtils.rm_r TMP_DIR rescue nil
+    FileUtils.mkdir_p TMP_DIR
+  end
+
+  teardown do
+    if @i
+      @i.stop unless @i.stopped?
+      @i.before_shutdown unless @i.before_shutdown?
+      @i.shutdown unless @i.shutdown?
+      @i.after_shutdown unless @i.after_shutdown?
+      @i.close unless @i.closed?
+      @i.terminate unless @i.terminated?
+    end
+  end
+
+  data(
+    handle_simple_stream: config_element('buffer', '', { 'flush_interval' => 1, 'compress' => 'gzip' }),
+    handle_stream_with_standard_format:  config_element('buffer', 'tag', { 'flush_interval' => 1, 'compress' => 'gzip' }),
+    handle_simple_stream_and_file_chunk: config_element('buffer', '', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
+    handle_stream_with_standard_format_and_file_chunk:  config_element('buffer', 'tag', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
+  )
+  test 'call a standard format when output plugin adds data to chunk' do |buffer_config|
+    @i = create_output(:async)
+    @i.configure(config_element('ROOT','', {}, [buffer_config]))
+    @i.start
+    @i.after_start
+
+    io = StringIO.new
+    es = dummy_event_stream
+    expected = es.map { |e| e }
+    compressed_data = ''
+
+    assert_equal :gzip, @i.buffer.compress
+
+    @i.register(:write) do |c|
+      compressed_data = c.instance_variable_get(:@chunk)
+      if compressed_data.is_a?(File)
+        compressed_data.seek(0, IO::SEEK_SET)
+        compressed_data = compressed_data.read
+      end
+      c.write_to(io)
+    end
+
+    @i.emit_events('tag', es)
+    @i.enqueue_thread_wait
+    @i.flush_thread_wakeup
+    waiting(4) { Thread.pass until io.size > 0 }
+
+    assert_equal expected, Fluent::MessagePackEventStream.new(decompress(compressed_data)).map { |t, r| [t, r] }
+    assert_equal expected, Fluent::MessagePackEventStream.new(io.string).map { |t, r| [t, r] }
+  end
+
+  data(
+    handle_simple_stream: config_element('buffer', '', { 'flush_interval' => 1, 'compress' => 'gzip' }),
+    handle_stream_with_custom_format:  config_element('buffer', 'tag', { 'flush_interval' => 1, 'compress' => 'gzip' }),
+    handle_simple_stream_and_file_chunk: config_element('buffer', '', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
+    handle_stream_with_custom_format_and_file_chunk:  config_element('buffer', 'tag', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
+  )
+  test 'call a custom format when output plugin adds data to chunk' do |buffer_config|
+    @i = create_output(:async_with_format)
+    @i.configure(config_element('ROOT','', {}, [buffer_config]))
+    @i.start
+    @i.after_start
+
+    io = StringIO.new
+    es = dummy_event_stream
+    expected = es.map { |e| "#{e[1]}\n" }.join # e[1] is record
+    compressed_data = ''
+
+    assert_equal :gzip, @i.buffer.compress
+
+    @i.register(:format) { |tag, time, record| "#{record}\n" }
+    @i.register(:write) { |c|
+      compressed_data = c.instance_variable_get(:@chunk)
+      if compressed_data.is_a?(File)
+        compressed_data.seek(0, IO::SEEK_SET)
+        compressed_data = compressed_data.read
+      end
+      c.write_to(io)
+    }
+
+    @i.emit_events('tag', es)
+    @i.enqueue_thread_wait
+    @i.flush_thread_wakeup
+    waiting(4) { sleep 0.1 until io.size > 0 }
+
+    assert_equal expected, decompress(compressed_data)
+    assert_equal expected, io.string
+  end
+end