fluentd 1.17.0 → 1.17.1

data/test/command/test_binlog_reader.rb

@@ -1,362 +0,0 @@
-require_relative '../helper'
-
-require 'yajl'
-require 'flexmock/test_unit'
-
-require 'fluent/command/binlog_reader'
-require 'fluent/event'
-
-class TestFluentBinlogReader < ::Test::Unit::TestCase
-  module ::BinlogReaderCommand
-    class Dummy < Base
-      def call; end
-    end
-  end
-
-  def suppress_stdout
-    out = StringIO.new
-    $stdout = out
-    yield
-  ensure
-    $stdout = STDOUT
-  end
-
-  sub_test_case 'call' do
-    data(
-      empty: [],
-      invalid: %w(invalid packed.log),
-    )
-    test 'should fail when invalid command' do |argv|
-      fu = FluentBinlogReader.new(argv)
-
-      assert_raise(SystemExit) do
-        suppress_stdout { fu.call }
-      end
-    end
-
-    data(
-      cat: %w(cat packed.log),
-      head: %w(head packed.log),
-      formats: %w(formats packed.log)
-    )
-    test 'should succeed when valid command' do |argv|
-      fu = FluentBinlogReader.new(argv)
-
-      flexstub(::BinlogReaderCommand) do |command|
-        command.should_receive(:const_get).once.and_return(::BinlogReaderCommand::Dummy)
-        assert_nothing_raised do
-          fu.call
-        end
-      end
-    end
-  end
-end
-
-class TestBaseCommand < ::Test::Unit::TestCase
-  TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/../tmp/command/binlog_reader#{ENV['TEST_ENV_NUMBER']}")
-
-  def create_message_packed_file(path, times = [event_time], records = [{ 'message' => 'dummy' }])
-    es = Fluent::MultiEventStream.new(times, records)
-    v = es.to_msgpack_stream
-    out_path = "#{TMP_DIR}/#{path}"
-    File.open(out_path, 'wb') do |f|
-      f.print(v)
-    end
-    waiting(5) do
-      sleep 0.5 until File.size(out_path) == v.bytesize
-    end
-  end
-
-  def setup
-    FileUtils.rm_rf(TMP_DIR)
-    FileUtils.mkdir_p(TMP_DIR)
-  end
-
-  def timezone(timezone = 'UTC')
-    old = ENV['TZ']
-    ENV['TZ'] = timezone
-    yield
-  ensure
-    ENV['TZ'] = old
-  end
-end
-
-class TestHead < TestBaseCommand
-  setup do
-    @default_newline = if Fluent.windows?
-                         "\r\n"
-                       else
-                         "\n"
-                       end
-  end
-
-  sub_test_case 'initialize' do
-    data(
-      'file is not passed' => %w(),
-      'file is not found' => %w(invalid_path.log)
-    )
-    test 'should fail if file is invalid' do |argv|
-      assert_raise(SystemExit) do
-        capture_stdout { BinlogReaderCommand::Head.new(argv) }
-      end
-    end
-
-    test 'should succeed if a file is valid' do
-      file_name = 'packed.log'
-      argv = ["#{TMP_DIR}/#{file_name}"]
-      create_message_packed_file(file_name)
-
-      assert_nothing_raised do
-        BinlogReaderCommand::Head.new(argv)
-      end
-    end
-
-    test 'should fail when config_params format is invalid' do
-      file_name = 'packed.log'
-      argv = ["#{TMP_DIR}/#{file_name}", '--format=csv', '-e', 'only_key']
-      create_message_packed_file(file_name)
-
-      assert_raise(SystemExit) do
-        capture_stdout { BinlogReaderCommand::Head.new(argv) }
-      end
-    end
-
-    test 'should succeed if config_params format is valid' do
-      file_name = 'packed.log'
-      argv = ["#{TMP_DIR}/#{file_name}", '--format=csv', '-e', 'fields=message']
-      create_message_packed_file(file_name)
-
-      assert_nothing_raised do
-        capture_stdout { BinlogReaderCommand::Head.new(argv) }
-      end
-    end
-  end
-
-  sub_test_case 'call' do
-    setup do
-      @file_name = 'packed.log'
-      @t = '2011-01-02 13:14:15 UTC'
-      @record = { 'message' => 'dummy' }
-    end
-
-    test 'should output the beginning of the file with default format (out_file)' do
-      argv = ["#{TMP_DIR}/#{@file_name}"]
-
-      timezone do
-        create_message_packed_file(@file_name, [event_time(@t).to_i] * 6, [@record] * 6)
-        head = BinlogReaderCommand::Head.new(argv)
-        out = capture_stdout { head.call }
-        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}#{@default_newline}" * 5, out
-      end
-    end
-
-    test 'should set the number of lines to display' do
-      argv = ["#{TMP_DIR}/#{@file_name}", '-n', '1']
-
-      timezone do
-        create_message_packed_file(@file_name, [event_time(@t).to_i] * 6, [@record] * 6)
-        head = BinlogReaderCommand::Head.new(argv)
-        out = capture_stdout { head.call }
-        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}#{@default_newline}", out
-      end
-    end
-
-    test 'should fail when the number of lines is invalid' do
-      argv = ["#{TMP_DIR}/#{@file_name}", '-n', '0']
-
-      create_message_packed_file(@file_name)
-      assert_raise(SystemExit) do
-        capture_stdout { BinlogReaderCommand::Head.new(argv) }
-      end
-    end
-
-    test 'should output content of a file with json format' do
-      argv = ["#{TMP_DIR}/#{@file_name}", '--format=json']
-
-      timezone do
-        create_message_packed_file(@file_name, [event_time(@t).to_i], [@record])
-        head = BinlogReaderCommand::Head.new(argv)
-        out = capture_stdout { head.call }
-        assert_equal "#{Yajl.dump(@record)}#{@default_newline}", out
-      end
-    end
-
-    test 'should fail with an invalid format' do
-      argv = ["#{TMP_DIR}/#{@file_name}", '--format=invalid']
-
-      timezone do
-        create_message_packed_file(@file_name, [event_time(@t).to_i], [@record])
-        head = BinlogReaderCommand::Head.new(argv)
-
-        assert_raise(SystemExit) do
-          capture_stdout { head.call }
-        end
-      end
-    end
-
-    test 'should succeed if multiple config_params format' do
-      file_name = 'packed.log'
-      argv = ["#{TMP_DIR}/#{file_name}", '--format=csv', '-e', 'fields=message,fo', '-e', 'delimiter=|']
-      create_message_packed_file(file_name, [event_time], [{ 'message' => 'dummy', 'fo' => 'dummy2' }])
-
-      head = BinlogReaderCommand::Head.new(argv)
-      assert_equal "\"dummy\"|\"dummy2\"\n", capture_stdout { head.call }
-    end
-  end
-end
-
-class TestCat < TestBaseCommand
-  setup do
-    @default_newline = if Fluent.windows?
-                         "\r\n"
-                       else
-                         "\n"
-                       end
-  end
-
-  sub_test_case 'initialize' do
-    data(
-      'file is not passed' => [],
-      'file is not found' => %w(invalid_path.log)
-    )
-    test 'should fail if a file is invalid' do |argv|
-      assert_raise(SystemExit) do
-        capture_stdout { BinlogReaderCommand::Head.new(argv) }
-      end
-    end
-
-    test 'should succeed if a file is valid' do
-      file_name = 'packed.log'
-      argv = ["#{TMP_DIR}/#{file_name}"]
-      create_message_packed_file(file_name)
-
-      assert_nothing_raised do
-        BinlogReaderCommand::Cat.new(argv)
-      end
-    end
-
-    test 'should fail when config_params format is invalid' do
-      file_name = 'packed.log'
-      argv = ["#{TMP_DIR}/#{file_name}", '--format=json', '-e', 'only_key']
-      create_message_packed_file(file_name)
-
-      assert_raise(SystemExit) do
-        capture_stdout { BinlogReaderCommand::Cat.new(argv) }
-      end
-    end
-
-    test 'should succeed when config_params format is valid' do
-      file_name = 'packed.log'
-      argv = ["#{TMP_DIR}/#{file_name}", '--format=csv', '-e', 'fields=message']
-      create_message_packed_file(file_name)
-
-      assert_nothing_raised do
-        capture_stdout { BinlogReaderCommand::Cat.new(argv) }
-      end
-    end
-  end
-
-  sub_test_case 'call' do
-    setup do
-      @file_name = 'packed.log'
-      @t = '2011-01-02 13:14:15 UTC'
-      @record = { 'message' => 'dummy' }
-    end
-
-    test 'should output the file with default format(out_file)' do
-      argv = ["#{TMP_DIR}/#{@file_name}"]
-
-      timezone do
-        create_message_packed_file(@file_name, [event_time(@t).to_i] * 6, [@record] * 6)
-        head = BinlogReaderCommand::Cat.new(argv)
-        out = capture_stdout { head.call }
-        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}#{@default_newline}" * 6, out
-      end
-    end
-
-    test 'should set the number of lines to display' do
-      argv = ["#{TMP_DIR}/#{@file_name}", '-n', '1']
-
-      timezone do
-        create_message_packed_file(@file_name, [event_time(@t).to_i] * 6, [@record] * 6)
-        head = BinlogReaderCommand::Cat.new(argv)
-        out = capture_stdout { head.call }
-        assert_equal "2011-01-02T13:14:15+00:00\t#{TMP_DIR}/#{@file_name}\t#{Yajl.dump(@record)}#{@default_newline}", out
-      end
-    end
-
-    test 'should output content of a file with json format' do
-      argv = ["#{TMP_DIR}/#{@file_name}", '--format=json']
-
-      timezone do
-        create_message_packed_file(@file_name, [event_time(@t).to_i], [@record])
-        head = BinlogReaderCommand::Cat.new(argv)
-        out = capture_stdout { head.call }
-        assert_equal "#{Yajl.dump(@record)}#{@default_newline}", out
-      end
-    end
-
-    test 'should fail with an invalid format' do
-      argv = ["#{TMP_DIR}/#{@file_name}", '--format=invalid']
-
-      timezone do
-        create_message_packed_file(@file_name, [event_time(@t).to_i], [@record])
-        head = BinlogReaderCommand::Cat.new(argv)
-
-        assert_raise(SystemExit) do
-          capture_stdout { head.call }
-        end
-      end
-    end
-
-    test 'should succeed if multiple config_params format' do
-      file_name = 'packed.log'
-      argv = ["#{TMP_DIR}/#{file_name}", '--format=csv', '-e', 'fields=message,fo', '-e', 'delimiter=|']
-      create_message_packed_file(file_name, [event_time], [{ 'message' => 'dummy', 'fo' => 'dummy2' }])
-
-      head = BinlogReaderCommand::Cat.new(argv)
-      assert_equal "\"dummy\"|\"dummy2\"\n", capture_stdout { head.call }
-    end
-  end
-end
-
-class TestFormats < TestBaseCommand
-  test 'parse_option!' do
-    assert_raise(SystemExit) do
-      capture_stdout do
-        BinlogReaderCommand::Formats.new(['--plugin=invalid_dir_path'])
-      end
-    end
-  end
-
-  sub_test_case 'call' do
-    test 'display available plugins' do
-      f = BinlogReaderCommand::Formats.new
-      out = capture_stdout { f.call }
-      assert out.include?('json')
-      assert out.include?('csv')
-    end
-
-    test 'add new plugins using --plugin option' do
-      dir_path = File.expand_path(File.dirname(__FILE__) + '/../scripts/fluent/plugin/formatter1')
-
-      f = BinlogReaderCommand::Formats.new(["--plugin=#{dir_path}"])
-      out = capture_stdout { f.call }
-      assert out.include?('json')
-      assert out.include?('csv')
-      assert out.include?('test1')
-    end
-
-    test 'add multiple plugins using --plugin option' do
-      dir_path1 = File.expand_path(File.dirname(__FILE__) + '/../scripts/fluent/plugin/formatter1')
-      dir_path2 = File.expand_path(File.dirname(__FILE__) + '/../scripts/fluent/plugin/formatter2')
-
-      f = BinlogReaderCommand::Formats.new(["--plugin=#{dir_path1}", '-p', dir_path2])
-      out = capture_stdout { f.call }
-      assert out.include?('json')
-      assert out.include?('csv')
-      assert out.include?('test1')
-      assert out.include?('test2')
-    end
-  end
-end

data/test/command/test_ca_generate.rb

@@ -1,70 +0,0 @@
-require_relative '../helper'
-
-require 'yajl'
-require 'flexmock/test_unit'
-require 'tmpdir'
-
-require 'fluent/command/ca_generate'
-require 'fluent/event'
-
-class TestFluentCaGenerate < ::Test::Unit::TestCase
-  def test_generate_ca_pair
-    cert, key = Fluent::CaGenerate.generate_ca_pair(Fluent::CaGenerate::DEFAULT_OPTIONS)
-    assert_equal(OpenSSL::X509::Certificate, cert.class)
-    assert_true(key.private?)
-  end
-
-  def test_ca_generate
-    dumped_output = capture_stdout do
-      Dir.mktmpdir do |dir|
-        Fluent::CaGenerate.new([dir, "fluentd"]).call
-        assert_true(File.exist?(File.join(dir, "ca_key.pem")))
-        assert_true(File.exist?(File.join(dir, "ca_cert.pem")))
-      end
-    end
-    expected = <<TEXT
-successfully generated: ca_key.pem, ca_cert.pem
-copy and use ca_cert.pem to client(out_forward)
-TEXT
-    assert_equal(expected, dumped_output)
-  end
-
-  sub_test_case "configure options" do
-    test "should respond multiple options" do
-      dumped_output = capture_stdout do
-        Dir.mktmpdir do |dir|
-          Fluent::CaGenerate.new([dir, "fluentd",
-                                  "--country", "JP", "--key-length", "4096",
-                                  "--state", "Tokyo", "--locality", "Chiyoda-ku",
-                                  "--common-name", "Forward CA"]).call
-          assert_true(File.exist?(File.join(dir, "ca_key.pem")))
-          assert_true(File.exist?(File.join(dir, "ca_cert.pem")))
-        end
-      end
-      expected = <<TEXT
-successfully generated: ca_key.pem, ca_cert.pem
-copy and use ca_cert.pem to client(out_forward)
-TEXT
-      assert_equal(expected, dumped_output)
-    end
-
-    test "invalid options" do
-      Dir.mktmpdir do |dir|
-        assert_raise(OptionParser::InvalidOption) do
-          Fluent::CaGenerate.new([dir, "fluentd",
-                                  "--invalid"]).call
-        end
-        assert_false(File.exist?(File.join(dir, "ca_key.pem")))
-        assert_false(File.exist?(File.join(dir, "ca_cert.pem")))
-      end
-    end
-
-    test "empty options" do
-      assert_raise(SystemExit) do
-        capture_stdout do
-          Fluent::CaGenerate.new([]).call
-        end
-      end
-    end
-  end
-end

data/test/command/test_cap_ctl.rb

@@ -1,100 +0,0 @@
-require_relative '../helper'
-
-require 'tempfile'
-require 'fluent/command/cap_ctl'
-
-class TestFluentCapCtl < Test::Unit::TestCase
-  setup do
-    omit "This environment does not handle Linux capability" unless defined?(CapNG)
-  end
-
-  sub_test_case "success" do
-    test "clear capability" do
-      logs = capture_stdout do
-        Fluent::CapCtl.new(["--clear"]).call
-      end
-      expression = /\AClear capabilities .*\n/m
-      assert_match expression, logs
-    end
-
-    test "add capability" do
-      logs = capture_stdout do
-        Fluent::CapCtl.new(["--add", "dac_override"]).call
-      end
-      expression = /\AUpdating .* done.\nAdding .*\n/m
-      assert_match expression, logs
-    end
-
-    test "drop capability" do
-      logs = capture_stdout do
-        Fluent::CapCtl.new(["--drop", "chown"]).call
-      end
-      expression = /\AUpdating .* done.\nDropping .*\n/m
-      assert_match expression, logs
-    end
-
-    test "get capability" do
-      logs = capture_stdout do
-        Fluent::CapCtl.new(["--get"]).call
-      end
-      expression = /\ACapabilities in .*,\nEffective:   .*\nInheritable: .*\nPermitted:   .*/m
-      assert_match expression, logs
-    end
-  end
-
-  sub_test_case "success with file" do
-    test "clear capability" do
-      logs = capture_stdout do
-        Tempfile.create("fluent-cap-") do |tempfile|
-          Fluent::CapCtl.new(["--clear-cap", "-f", tempfile.path]).call
-        end
-      end
-      expression = /\AClear capabilities .*\n/m
-      assert_match expression, logs
-    end
-
-    test "add capability" do
-      logs = capture_stdout do
-        Tempfile.create("fluent-cap-") do |tempfile|
-          Fluent::CapCtl.new(["--add", "dac_override", "-f", tempfile.path]).call
-        end
-      end
-      expression = /\AUpdating .* done.\nAdding .*\n/m
-      assert_match expression, logs
-    end
-
-    test "drop capability" do
-      logs = capture_stdout do
-        Tempfile.create("fluent-cap-") do |tempfile|
-          Fluent::CapCtl.new(["--drop", "chown", "-f", tempfile.path]).call
-        end
-      end
-      expression = /\AUpdating .* done.\nDropping .*\n/m
-      assert_match expression, logs
-    end
-
-    test "get capability" do
-      logs = capture_stdout do
-        Tempfile.create("fluent-cap-") do |tempfile|
-          Fluent::CapCtl.new(["--get", "-f", tempfile.path]).call
-        end
-      end
-      expression = /\ACapabilities in .*,\nEffective:   .*\nInheritable: .*\nPermitted:   .*/m
-      assert_match expression, logs
-    end
-  end
-
-  sub_test_case "invalid" do
-    test "add capability" do
-      assert_raise(ArgumentError) do
-        Fluent::CapCtl.new(["--add", "nonexitent"]).call
-      end
-    end
-
-    test "drop capability" do
-      assert_raise(ArgumentError) do
-        Fluent::CapCtl.new(["--drop", "invalid"]).call
-      end
-    end
-  end
-end

data/test/command/test_cat.rb

@@ -1,128 +0,0 @@
-require_relative '../helper'
-
-require 'test-unit'
-require 'open3'
-require 'fluent/plugin/output'
-require 'fluent/plugin/in_forward'
-require 'fluent/plugin/out_secondary_file'
-require 'fluent/test/driver/output'
-require 'fluent/test/driver/input'
-
-class TestFluentCat < ::Test::Unit::TestCase
-  def setup
-    Fluent::Test.setup
-    FileUtils.mkdir_p(TMP_DIR)
-    @record = { 'key' => 'value' }
-    @time = event_time
-    @es = Fluent::OneEventStream.new(@time, @record)
-    @primary = create_primary
-    metadata = @primary.buffer.new_metadata
-    @chunk = create_chunk(@primary, metadata, @es)
-    @port = unused_port
-  end
-
-  def teardown
-    FileUtils.rm_rf(TMP_DIR)
-    @port = nil
-  end
-
-  TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/../tmp/command/fluent_cat#{ENV['TEST_ENV_NUMBER']}")
-  FLUENT_CAT_COMMAND = File.expand_path(File.dirname(__FILE__) + "/../../bin/fluent-cat")
-
-  def config
-    %[
-      port #{@port}
-      bind 127.0.0.1
-    ]
-  end
-
-  SECONDARY_CONFIG = %[
-    directory #{TMP_DIR}
-  ]
-
-  class DummyOutput < Fluent::Plugin::Output
-    def write(chunk); end
-  end
-
-  def create_driver(conf=config)
-    Fluent::Test::Driver::Input.new(Fluent::Plugin::ForwardInput).configure(conf)
-  end
-
-  def create_primary(buffer_cofig = config_element('buffer'))
-    DummyOutput.new.configure(config_element('ROOT','',{}, [buffer_cofig]))
-  end
-
-  def create_secondary_driver(conf=SECONDARY_CONFIG)
-    c = Fluent::Test::Driver::Output.new(Fluent::Plugin::SecondaryFileOutput)
-    c.instance.acts_as_secondary(@primary)
-    c.configure(conf)
-  end
-
-  def create_chunk(primary, metadata, es)
-    primary.buffer.generate_chunk(metadata).tap do |c|
-      c.concat(es.to_msgpack_stream, es.size)
-      c.commit
-    end
-  end
-
-  sub_test_case "json" do
-    def test_cat_json
-      d = create_driver
-      d.run(expect_records: 1) do
-        Open3.pipeline_w("#{ServerEngine.ruby_bin_path} #{FLUENT_CAT_COMMAND} --port #{@port} json") do |stdin|
-          stdin.puts('{"key":"value"}')
-          stdin.close
-        end
-      end
-      event = d.events.first
-      assert_equal([1, "json", @record],
-                   [d.events.size, event.first, event.last])
-    end
-  end
-
-  sub_test_case "msgpack" do
-    def test_cat_secondary_file
-      d = create_secondary_driver
-      path = d.instance.write(@chunk)
-      d = create_driver
-      d.run(expect_records: 1) do
-        Open3.pipeline_w("#{ServerEngine.ruby_bin_path} #{FLUENT_CAT_COMMAND} --port #{@port} --format msgpack secondary") do |stdin|
-          stdin.write(File.read(path))
-          stdin.close
-        end
-      end
-      event = d.events.first
-      assert_equal([1, "secondary", @record],
-                   [d.events.size, event.first, event.last])
-    end
-  end
-
-  sub_test_case "send specific event time" do
-    def test_without_event_time
-      event_time = Fluent::EventTime.now
-      d = create_driver
-      d.run(expect_records: 1) do
-        Open3.pipeline_w("#{ServerEngine.ruby_bin_path} #{FLUENT_CAT_COMMAND} --port #{@port} tag") do |stdin|
-          stdin.puts('{"key":"value"}')
-          stdin.close
-        end
-      end
-      event = d.events.first
-      assert_in_delta(event_time.to_f, event[1].to_f, 3.0) # expect command to be finished in 3 seconds
-      assert_equal([1, "tag", true, @record],
-                   [d.events.size, event.first, event_time.to_f < event[1].to_f, event.last])
-    end
-
-    def test_with_event_time
-      event_time = "2021-01-02 13:14:15.0+00:00"
-      d = create_driver
-      d.run(expect_records: 1) do
-        Open3.pipeline_w("#{ServerEngine.ruby_bin_path} #{FLUENT_CAT_COMMAND} --port #{@port} --event-time '#{event_time}' tag") do |stdin|
-          stdin.puts('{"key":"value"}')
-          stdin.close
-        end
-      end
-      assert_equal([["tag", Fluent::EventTime.parse(event_time), @record]], d.events)
-    end
-  end
-end