fluentd 1.17.0 → 1.17.1

data/test/plugin/test_output_as_buffered_backup.rb

@@ -1,363 +0,0 @@
-require_relative '../helper'
-require 'fluent/plugin/output'
-require 'fluent/plugin/buffer'
-require 'fluent/event'
-require 'fluent/error'
-
-require 'json'
-require 'time'
-require 'timeout'
-require 'timecop'
-
-
-class BufferedOutputBackupTest < Test::Unit::TestCase
-  class BareOutput < Fluent::Plugin::Output
-    def register(name, &block)
-      instance_variable_set("@#{name}", block)
-    end
-  end
-  class DummyOutput < BareOutput
-    def initialize
-      super
-      @process = nil
-      @format = nil
-      @write = nil
-      @try_write = nil
-    end
-    def prefer_buffered_processing
-      true
-    end
-    def prefer_delayed_commit
-      false
-    end
-    def process(tag, es)
-      @process ? @process.call(tag, es) : nil
-    end
-    def format(tag, time, record)
-      [tag, time.to_i, record].to_json + "\n"
-    end
-    def write(chunk)
-      @write ? @write.call(chunk) : nil
-    end
-    def try_write(chunk)
-      @try_write ? @try_write.call(chunk) : nil
-    end
-  end
-  class DummyOutputForSecondary < BareOutput
-    def initialize
-      super
-      @process = nil
-      @format = nil
-      @write = nil
-      @try_write = nil
-    end
-    def prefer_buffered_processing
-      true
-    end
-    def prefer_delayed_commit
-      false
-    end
-    def process(tag, es)
-      @process ? @process.call(tag, es) : nil
-    end
-    def format(tag, time, record)
-      [tag, time.to_i, record].to_json + "\n"
-    end
-    def write(chunk)
-      @write ? @write.call(chunk) : nil
-    end
-    def try_write(chunk)
-      @try_write ? @try_write.call(chunk) : nil
-    end
-  end
-  class DummyAsyncOutputForSecondary < BareOutput
-    def initialize
-      super
-      @process = nil
-      @format = nil
-      @write = nil
-      @try_write = nil
-    end
-    def prefer_buffered_processing
-      true
-    end
-    def prefer_delayed_commit
-      true
-    end
-    def process(tag, es)
-      @process ? @process.call(tag, es) : nil
-    end
-    def format(tag, time, record)
-      [tag, time.to_i, record].to_json + "\n"
-    end
-    def write(chunk)
-      @write ? @write.call(chunk) : nil
-    end
-    def try_write(chunk)
-      @try_write ? @try_write.call(chunk) : nil
-    end
-  end
-
-  TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/../tmp/bu#{ENV['TEST_ENV_NUMBER']}")
-
-  def create_output
-    DummyOutput.new
-  end
-  def create_metadata(timekey: nil, tag: nil, variables: nil)
-    Fluent::Plugin::Buffer::Metadata.new(timekey, tag, variables)
-  end
-  def waiting(seconds)
-    begin
-      Timeout.timeout(seconds) do
-        yield
-      end
-    rescue Timeout::Error
-      STDERR.print(*@i.log.out.logs)
-      raise
-    end
-  end
-
-  def dummy_event_stream
-    Fluent::ArrayEventStream.new([
-      [ event_time('2016-04-13 18:33:00'), {"name" => "moris", "age" => 36, "message" => "data1"} ],
-      [ event_time('2016-04-13 18:33:13'), {"name" => "moris", "age" => 36, "message" => "data2"} ],
-      [ event_time('2016-04-13 18:33:32'), {"name" => "moris", "age" => 36, "message" => "data3"} ],
-    ])
-  end
-
-  setup do
-    @i = create_output
-    FileUtils.rm_rf(TMP_DIR)
-    FileUtils.mkdir_p(TMP_DIR)
-
-    Fluent::Plugin.register_output('backup_output', DummyOutput)
-    Fluent::Plugin.register_output('backup_output2', DummyOutputForSecondary)
-    Fluent::Plugin.register_output('backup_async_output', DummyAsyncOutputForSecondary)
-  end
-
-  teardown do
-    if @i
-      @i.stop unless @i.stopped?
-      @i.before_shutdown unless @i.before_shutdown?
-      @i.shutdown unless @i.shutdown?
-      @i.after_shutdown unless @i.after_shutdown?
-      @i.close unless @i.closed?
-      @i.terminate unless @i.terminated?
-    end
-    Timecop.return
-  end
-
-  sub_test_case 'buffered output for broken chunks' do
-    def flush_chunks
-      @i.start
-      @i.after_start
-
-      @i.interrupt_flushes
-
-      now = Time.parse('2016-04-13 18:33:30 -0700')
-      Timecop.freeze(now)
-      @i.emit_events("test.tag.1", dummy_event_stream())
-      now = Time.parse('2016-04-13 18:33:32 -0700')
-      Timecop.freeze(now)
-
-      @i.enqueue_thread_wait
-      @i.flush_thread_wakeup
-      waiting(4) { Thread.pass until @i.write_count > 0 }
-
-      assert { @i.write_count > 0 }
-      Timecop.freeze(now)
-      @i.flush_thread_wakeup
-    end
-
-    def wait_flush(target_file)
-      waiting(5) {
-        target_dir = File.join(File.dirname(target_file), "*")
-        while Dir.glob(target_dir).size.zero?
-          # Avoid to lose globbed entries on Windows in busy loop
-          sleep 0.1 if Fluent.windows?
-        end
-      }
-    end
-
-    data('unrecoverable error' => Fluent::UnrecoverableError,
-         'type error' => TypeError,
-         'argument error' => ArgumentError,
-         'no method error' => NoMethodError,
-         'msgpack unpack error' => MessagePack::UnpackError,
-         'encoding error' => EncodingError)
-    test 'backup chunk without secondary' do |error_class|
-      Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR) do
-        id = 'backup_test'
-        hash = {
-          'flush_interval' => 1,
-          'flush_thread_burst_interval' => 0.1,
-        }
-        chunk_id = nil
-        @i.configure(config_element('ROOT', '', {'@id' => id}, [config_element('buffer', 'tag', hash)]))
-        @i.register(:write) { |chunk|
-          chunk_id = chunk.unique_id;
-          raise error_class, "yay, your #write must fail"
-        }
-
-        flush_chunks
-
-        target = "#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log"
-        wait_flush(target)
-        assert_true File.exist?(target)
-        logs = @i.log.out.logs
-        assert { logs.any? { |l| l.include?("got unrecoverable error in primary and no secondary") } }
-      end
-    end
-
-    test 'backup chunk with same type secondary' do
-      Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR) do
-        id = 'backup_test_with_same_secondary'
-        hash = {
-          'flush_interval' => 1,
-          'flush_thread_burst_interval' => 0.1,
-        }
-        chunk_id = nil
-        secconf = config_element('secondary','',{'@type' => 'backup_output'})
-        @i.configure(config_element('ROOT', '', {'@id' => id}, [config_element('buffer', 'tag', hash), secconf]))
-        @i.register(:write) { |chunk|
-          chunk_id = chunk.unique_id;
-          raise Fluent::UnrecoverableError, "yay, your #write must fail"
-        }
-
-        flush_chunks
-
-        target = "#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log"
-        wait_flush(target)
-        assert_true File.exist?(target)
-        logs = @i.log.out.logs
-        assert { logs.any? { |l| l.include?("got unrecoverable error in primary and secondary type is same as primary") } }
-      end
-    end
-
-    test 'create directory' do
-      Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR) do
-        id = 'backup_test_with_same_secondary'
-        hash = { 'flush_interval' => 1, 'flush_thread_burst_interval' => 0.1 }
-        chunk_id = nil
-        secconf = config_element('secondary', '', { '@type' => 'backup_output' })
-        @i.configure(config_element('ROOT', '', { '@id' => id }, [config_element('buffer', 'tag', hash), secconf]))
-        @i.register(:write) { |chunk|
-          chunk_id = chunk.unique_id;
-          raise Fluent::UnrecoverableError, "yay, your #write must fail"
-        }
-
-        flush_chunks
-
-        target = "#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log"
-        target_dir = File.dirname(target)
-        wait_flush(target)
-
-        assert_path_exist(target_dir)
-        assert_equal '755', File.stat(target_dir).mode.to_s(8)[-3, 3]
-      end
-    end
-
-    test 'create directory with specific mode' do
-      omit "NTFS doesn't support UNIX like permissions" if Fluent.windows?
-
-      Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR, 'dir_permission' => '744') do
-        id = 'backup_test_with_same_secondary'
-        hash = { 'flush_interval' => 1, 'flush_thread_burst_interval' => 0.1 }
-        chunk_id = nil
-        secconf = config_element('secondary', '', { '@type' => 'backup_output' })
-        @i.configure(config_element('ROOT', '', { '@id' => id }, [config_element('buffer', 'tag', hash), secconf]))
-        @i.register(:write) { |chunk|
-          chunk_id = chunk.unique_id;
-          raise Fluent::UnrecoverableError, "yay, your #write must fail"
-        }
-
-        flush_chunks
-
-        target = "#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log"
-        target_dir = File.dirname(target)
-        wait_flush(target)
-
-        assert_path_exist(target_dir)
-        assert_equal '744', File.stat(target_dir).mode.to_s(8)[-3, 3]
-      end
-    end
-
-    test 'backup chunk with different type secondary' do
-      Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR) do
-        id = 'backup_test_with_diff_secondary'
-        hash = {
-          'flush_interval' => 1,
-          'flush_thread_burst_interval' => 0.1,
-        }
-        chunk_id = nil
-        secconf = config_element('secondary','',{'@type' => 'backup_output2'})
-        @i.configure(config_element('ROOT', '', {'@id' => id}, [config_element('buffer', 'tag', hash), secconf]))
-        @i.register(:write) { |chunk|
-          chunk_id = chunk.unique_id;
-          raise Fluent::UnrecoverableError, "yay, your #write must fail"
-        }
-        @i.secondary.register(:write) { |chunk|
-          raise Fluent::UnrecoverableError, "yay, your secondary #write must fail"
-        }
-
-        flush_chunks
-
-        target = "#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log"
-        wait_flush(target)
-        assert_true File.exist?(target)
-        logs = @i.log.out.logs
-        assert { logs.any? { |l| l.include?("got unrecoverable error in primary. Skip retry and flush chunk to secondary") } }
-        assert { logs.any? { |l| l.include?("got an error in secondary for unrecoverable error") } }
-      end
-    end
-
-    test 'backup chunk with async secondary' do
-      Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR) do
-        id = 'backup_test_with_diff_secondary'
-        hash = {
-          'flush_interval' => 1,
-          'flush_thread_burst_interval' => 0.1,
-        }
-        chunk_id = nil
-        secconf = config_element('secondary','',{'@type' => 'backup_async_output'})
-        @i.configure(config_element('ROOT', '', {'@id' => id}, [config_element('buffer', 'tag', hash), secconf]))
-        @i.register(:write) { |chunk|
-          chunk_id = chunk.unique_id;
-          raise Fluent::UnrecoverableError, "yay, your #write must fail"
-        }
-
-        flush_chunks
-
-        target = "#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log"
-        wait_flush(target)
-        assert_true File.exist?(target)
-        logs = @i.log.out.logs
-        assert { logs.any? { |l| l.include?("got unrecoverable error in primary and secondary is async output") } }
-      end
-    end
-
-    test 'chunk is thrown away when disable_chunk_backup is true' do
-      Fluent::SystemConfig.overwrite_system_config('root_dir' => TMP_DIR) do
-        id = 'backup_test'
-        hash = {
-          'flush_interval' => 1,
-          'flush_thread_burst_interval' => 0.1,
-          'disable_chunk_backup' => true
-        }
-        chunk_id = nil
-        @i.configure(config_element('ROOT', '', {'@id' => id}, [config_element('buffer', 'tag', hash)]))
-        @i.register(:write) { |chunk|
-          chunk_id = chunk.unique_id;
-          raise Fluent::UnrecoverableError, "yay, your #write must fail"
-        }
-
-        flush_chunks
-
-        target = "#{TMP_DIR}/backup/worker0/#{id}/#{@i.dump_unique_id_hex(chunk_id)}.log"
-        assert_false File.exist?(target)
-        logs = @i.log.out.logs
-        assert { logs.any? { |l| l.include?("disable_chunk_backup is true") } }
-      end
-    end
-  end
-end

data/test/plugin/test_output_as_buffered_compress.rb

@@ -1,179 +0,0 @@
-require_relative '../helper'
-require 'fluent/plugin/output'
-require 'fluent/plugin/buffer'
-require 'fluent/plugin/compressable'
-require 'fluent/event'
-
-require 'timeout'
-
-module FluentPluginOutputAsBufferedCompressTest
-  class DummyBareOutput < Fluent::Plugin::Output
-    def register(name, &block)
-      instance_variable_set("@#{name}", block)
-    end
-  end
-
-  class DummyAsyncOutput < DummyBareOutput
-    def initialize
-      super
-      @format = @write = nil
-    end
-    def write(chunk)
-      @write ? @write.call(chunk) : nil
-    end
-  end
-
-  class DummyAsyncOutputWithFormat < DummyBareOutput
-    def initialize
-      super
-      @format = nil
-    end
-    def write(chunk)
-      @write ? @write.call(chunk) : nil
-    end
-    def format(tag, time, record)
-      @format ? @format.call(tag, time, record) : [tag, time, record].to_json
-    end
-  end
-
-  def self.dummy_event_stream
-    Fluent::ArrayEventStream.new(
-      [
-        [event_time('2016-04-13 18:33:00'), { 'name' => 'moris', 'age' => 36, 'message' => 'data1' }],
-        [event_time('2016-04-13 18:33:13'), { 'name' => 'moris', 'age' => 36, 'message' => 'data2' }],
-        [event_time('2016-04-13 18:33:32'), { 'name' => 'moris', 'age' => 36, 'message' => 'data3' }],
-      ]
-    )
-  end
-end
-
-class BufferedOutputCompressTest < Test::Unit::TestCase
-  include Fluent::Plugin::Compressable
-
-  def create_output(type=:async)
-    case type
-    when :async then FluentPluginOutputAsBufferedCompressTest::DummyAsyncOutput.new
-    when :async_with_format then FluentPluginOutputAsBufferedCompressTest::DummyAsyncOutputWithFormat.new
-    else
-      raise ArgumentError, "unknown type: #{type}"
-    end
-  end
-
-  def waiting(seconds)
-    begin
-      Timeout.timeout(seconds) do
-        yield
-      end
-    rescue Timeout::Error
-      STDERR.print(*@i.log.out.logs)
-      raise
-    end
-  end
-
-  TMP_DIR = File.expand_path('../../tmp/test_output_as_buffered_compress', __FILE__)
-
-  setup do
-    FileUtils.rm_r TMP_DIR rescue nil
-    FileUtils.mkdir_p TMP_DIR
-  end
-
-  teardown do
-    if @i
-      @i.stop unless @i.stopped?
-      @i.before_shutdown unless @i.before_shutdown?
-      @i.shutdown unless @i.shutdown?
-      @i.after_shutdown unless @i.after_shutdown?
-      @i.close unless @i.closed?
-      @i.terminate unless @i.terminated?
-    end
-  end
-
-  data(
-    :buffer_config,
-    [
-      config_element('buffer', '', { 'flush_interval' => 1, 'compress' => 'gzip' }),
-      config_element('buffer', 'tag', { 'flush_interval' => 1, 'compress' => 'gzip' }),
-      config_element('buffer', '', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
-      config_element('buffer', 'tag', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
-    ],
-  )
-  data(
-    :input_es,
-    [
-      FluentPluginOutputAsBufferedCompressTest.dummy_event_stream,
-      # If already compressed data is incoming, it must be written as is (i.e. without decompressed).
-      # https://github.com/fluent/fluentd/issues/4146
-      Fluent::CompressedMessagePackEventStream.new(FluentPluginOutputAsBufferedCompressTest.dummy_event_stream.to_compressed_msgpack_stream),
-    ],
-  )
-  test 'call a standard format when output plugin adds data to chunk' do |data|
-    buffer_config = data[:buffer_config]
-    es = data[:input_es].dup # Note: the data matrix is shared in all patterns, so we need `dup` here.
-
-    @i = create_output(:async)
-    @i.configure(config_element('ROOT','', {}, [buffer_config]))
-    @i.start
-    @i.after_start
-
-    io = StringIO.new
-    expected = es.dup.map { |t, r| [t, r] }
-    compressed_data = ''
-
-    assert_equal :gzip, @i.buffer.compress
-
-    @i.register(:write) do |c|
-      compressed_data = c.instance_variable_get(:@chunk)
-      if compressed_data.is_a?(File)
-        compressed_data.seek(0, IO::SEEK_SET)
-        compressed_data = compressed_data.read
-      end
-      c.write_to(io)
-    end
-
-    @i.emit_events('tag', es)
-    @i.enqueue_thread_wait
-    @i.flush_thread_wakeup
-    waiting(4) { Thread.pass until io.size > 0 }
-
-    assert_equal expected, Fluent::MessagePackEventStream.new(decompress(compressed_data)).map { |t, r| [t, r] }
-    assert_equal expected, Fluent::MessagePackEventStream.new(io.string).map { |t, r| [t, r] }
-  end
-
-  data(
-    handle_simple_stream: config_element('buffer', '', { 'flush_interval' => 1, 'compress' => 'gzip' }),
-    handle_stream_with_custom_format:  config_element('buffer', 'tag', { 'flush_interval' => 1, 'compress' => 'gzip' }),
-    handle_simple_stream_and_file_chunk: config_element('buffer', '', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
-    handle_stream_with_custom_format_and_file_chunk:  config_element('buffer', 'tag', { '@type' => 'file', 'path' => File.join(TMP_DIR,'test.*.log'), 'flush_interval' => 1, 'compress' => 'gzip' }),
-  )
-  test 'call a custom format when output plugin adds data to chunk' do |buffer_config|
-    @i = create_output(:async_with_format)
-    @i.configure(config_element('ROOT','', {}, [buffer_config]))
-    @i.start
-    @i.after_start
-
-    io = StringIO.new
-    es = FluentPluginOutputAsBufferedCompressTest.dummy_event_stream
-    expected = es.map { |e| "#{e[1]}\n" }.join # e[1] is record
-    compressed_data = ''
-
-    assert_equal :gzip, @i.buffer.compress
-
-    @i.register(:format) { |tag, time, record| "#{record}\n" }
-    @i.register(:write) { |c|
-      compressed_data = c.instance_variable_get(:@chunk)
-      if compressed_data.is_a?(File)
-        compressed_data.seek(0, IO::SEEK_SET)
-        compressed_data = compressed_data.read
-      end
-      c.write_to(io)
-    }
-
-    @i.emit_events('tag', es)
-    @i.enqueue_thread_wait
-    @i.flush_thread_wakeup
-    waiting(4) { sleep 0.1 until io.size > 0 }
-
-    assert_equal expected, decompress(compressed_data)
-    assert_equal expected, io.string
-  end
-end