fluentd 1.15.2-x86-mingw32 → 1.16.1-x86-mingw32

data/test/plugin_helper/test_server.rb

@@ -16,14 +16,20 @@ class ServerPluginHelperTest < Test::Unit::TestCase
 
   setup do
     @port = unused_port
-    @socket_manager_path = ServerEngine::SocketManager::Server.generate_path
-    if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
-      FileUtils.rm_f @socket_manager_path
+    if Fluent.windows?
+      @socket_manager_server = ServerEngine::SocketManager::Server.open
+      @socket_manager_path = @socket_manager_server.path
+    else
+      @socket_manager_path = ServerEngine::SocketManager::Server.generate_path
+      if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
+        FileUtils.rm_f @socket_manager_path
+      end
+      @socket_manager_server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
     end
-    @socket_manager_server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
     ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @socket_manager_path.to_s
 
     @d = Dummy.new
+    @d.under_plugin_development = true
     @d.start
     @d.after_start
   end
@@ -37,7 +43,7 @@ class ServerPluginHelperTest < Test::Unit::TestCase
     (@d.terminated? || @d.terminate) rescue nil
 
     @socket_manager_server.close
-    if @socket_manager_server.is_a?(String) && File.exist?(@socket_manager_path)
+    if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
       FileUtils.rm_f @socket_manager_path
     end
   end
@@ -789,6 +795,50 @@ class ServerPluginHelperTest < Test::Unit::TestCase
         end
       end
     end
+
+    sub_test_case 'over max_bytes' do
+      data("cut off on Non-Windows", { max_bytes: 32, records: ["a" * 40], expected: ["a" * 32] }, keep: true) unless Fluent.windows?
+      data("drop on Windows", { max_bytes: 32, records: ["a" * 40], expected: [] }, keep: true) if Fluent.windows?
+      test 'with sock' do |data|
+        max_bytes, records, expected = data.values
+
+        actual_records = []
+        @d.server_create_udp(:myserver, @port, max_bytes: max_bytes) do |data, sock|
+          actual_records << data
+        end
+
+        open_client(:udp, "127.0.0.1", @port) do |sock|
+          records.each do |record|
+            sock.send(record, 0)
+          end
+        end
+
+        waiting(10) { sleep 0.1 until actual_records.size >= expected.size }
+        sleep 1 if expected.size == 0 # To confirm no record recieved.
+
+        assert_equal expected, actual_records
+      end
+
+      test 'without sock' do |data|
+        max_bytes, records, expected = data.values
+
+        actual_records = []
+        @d.server_create_udp(:myserver, @port, max_bytes: max_bytes) do |data|
+          actual_records << data
+        end
+
+        open_client(:udp, "127.0.0.1", @port) do |sock|
+          records.each do |record|
+            sock.send(record, 0)
+          end
+        end
+
+        waiting(10) { sleep 0.1 until actual_records.size >= expected.size }
+        sleep 1 if expected.size == 0 # To confirm no record recieved.
+
+        assert_equal expected, actual_records
+      end
+    end
   end
 
   module CertUtil
@@ -1570,6 +1620,10 @@ class ServerPluginHelperTest < Test::Unit::TestCase
 
   def open_client(proto, addr, port)
     client = case proto
+             when :udp
+               c = UDPSocket.open
+               c.connect(addr, port)
+               c
              when :tcp
                TCPSocket.open(addr, port)
              when :tls

data/test/test_config.rb

@@ -237,6 +237,63 @@ class ConfigTest < Test::Unit::TestCase
         ])
     end
 
+    def test_included_glob
+      write_config "#{TMP_DIR}/config.yaml", <<-EOS
+      config:
+        - !include "include/*.yaml"
+      EOS
+      write_config "#{TMP_DIR}/include/02_source2.yaml", <<-EOS
+      - source:
+          $type: dummy
+          tag: tag.dummy
+      EOS
+      write_config "#{TMP_DIR}/include/01_source1.yaml", <<-EOS
+      - source:
+          $type: tcp
+          tag: tag.tcp
+          parse:
+            $arg:
+              - why.parse.section.doesnot.have.arg
+              - huh
+            $type: none
+      EOS
+      write_config "#{TMP_DIR}/include/03_match1.yaml", <<-EOS
+      - match:
+          $tag: tag.*
+          $type: stdout
+          buffer:
+            $type: memory
+            flush_interval: 1s
+      EOS
+      root_conf = read_config("#{TMP_DIR}/config.yaml", use_yaml: true)
+      tcp_source_conf = root_conf.elements.first
+      dummy_source_conf = root_conf.elements[1]
+      parse_tcp_conf = tcp_source_conf.elements.first
+      match_conf = root_conf.elements[2]
+
+      assert_equal(
+        [
+          'tcp',
+          'tag.tcp',
+          'none',
+          'why.parse.section.doesnot.have.arg,huh',
+          'dummy',
+          'tag.dummy',
+          'stdout',
+          'tag.*',
+        ],
+        [
+          tcp_source_conf['@type'],
+          tcp_source_conf['tag'],
+          parse_tcp_conf['@type'],
+          parse_tcp_conf.arg,
+          dummy_source_conf['@type'],
+          dummy_source_conf['tag'],
+          match_conf['@type'],
+          match_conf.arg,
+        ])
+    end
+
     def test_check_not_fetchd
       write_config "#{TMP_DIR}/config_test_not_fetched.yaml", <<-EOS
       config:
@@ -287,27 +344,6 @@ class ConfigTest < Test::Unit::TestCase
     File.open(path, "w:#{encoding}:utf-8") {|f| f.write data }
   end
 
-  def test_inline
-    prepare_config
-    opts = {
-      :config_path => "#{TMP_DIR}/config_test_1.conf",
-      :inline_config => "<source>\n  type http\n  port 2222\n </source>",
-      :use_v1_config => false
-    }
-    assert_nothing_raised do
-      Fluent::Supervisor.new(opts)
-    end
-    create_warn_dummy_logger
-  end
-
-  def create_warn_dummy_logger
-    dl_opts = {}
-    dl_opts[:log_level] = ServerEngine::DaemonLogger::WARN
-    logdev = Fluent::Test::DummyLogDevice.new
-    logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)
-    $log = Fluent::Log.new(logger)
-  end
-
   sub_test_case '.build' do
     test 'read config' do
       write_config("#{TMP_DIR}/build/config_build.conf", 'key value')

data/test/{plugin/test_file_wrapper.rb → test_file_wrapper.rb}

@@ -1,5 +1,5 @@
-require_relative '../helper'
-require 'fluent/plugin/file_wrapper'
+require_relative 'helper'
+require 'fluent/file_wrapper'
 
 class FileWrapperTest < Test::Unit::TestCase
   TMP_DIR = File.dirname(__FILE__) + "/../tmp/file_wrapper#{ENV['TEST_ENV_NUMBER']}"

data/test/test_formatter.rb

@@ -18,7 +18,7 @@ module FormatterTest
 
     def test_call
       formatter = Formatter.new
-      formatter.configure({})
+      formatter.configure(config_element())
       assert_raise NotImplementedError do
         formatter.format('tag', Engine.now, {})
       end
@@ -130,7 +130,7 @@ module FormatterTest
     include FormatterTest
 
     def setup
-      @formatter = TextFormatter::MessagePackFormatter.new
+      @formatter = Fluent::Test::FormatterTestDriver.new(TextFormatter::MessagePackFormatter)
       @time = Engine.now
     end
 
@@ -146,7 +146,7 @@ module FormatterTest
     include FormatterTest
 
     def setup
-      @formatter = TextFormatter::LabeledTSVFormatter.new
+      @formatter = Fluent::Test::FormatterTestDriver.new(TextFormatter::LabeledTSVFormatter)
       @time = Engine.now
       @newline = if Fluent.windows?
                    "\r\n"
@@ -156,16 +156,16 @@ module FormatterTest
     end
 
     def test_config_params
-      assert_equal "\t", @formatter.delimiter
-      assert_equal  ":", @formatter.label_delimiter
+      assert_equal "\t", @formatter.instance.delimiter
+      assert_equal  ":", @formatter.instance.label_delimiter
 
       @formatter.configure(
         'delimiter'       => ',',
         'label_delimiter' => '=',
       )
 
-      assert_equal ",", @formatter.delimiter
-      assert_equal "=", @formatter.label_delimiter
+      assert_equal ",", @formatter.instance.delimiter
+      assert_equal "=", @formatter.instance.label_delimiter
     end
 
     def test_format
@@ -220,14 +220,14 @@ module FormatterTest
     include FormatterTest
 
     def setup
-      @formatter = TextFormatter::CsvFormatter.new
+      @formatter = Fluent::Test::FormatterTestDriver.new(TextFormatter::CsvFormatter)
       @time = Engine.now
     end
 
     def test_config_params
-      assert_equal ',', @formatter.delimiter
-      assert_equal true, @formatter.force_quotes
-      assert_nil @formatter.fields
+      assert_equal ',', @formatter.instance.delimiter
+      assert_equal true, @formatter.instance.force_quotes
+      assert_nil @formatter.instance.fields
     end
 
     data(
@@ -237,8 +237,8 @@ module FormatterTest
     def test_config_params_with_customized_delimiters(data)
       expected, target = data
       @formatter.configure('delimiter' => target, 'fields' => 'a,b,c')
-      assert_equal expected, @formatter.delimiter
-      assert_equal ['a', 'b', 'c'], @formatter.fields
+      assert_equal expected, @formatter.instance.delimiter
+      assert_equal ['a', 'b', 'c'], @formatter.instance.fields
     end
 
     def test_format
@@ -299,7 +299,7 @@ module FormatterTest
       'blank' => 'one,,two,three')
     def test_config_params_with_fields(data)
       @formatter.configure('fields' => data)
-      assert_equal %w(one two three), @formatter.fields
+      assert_equal %w(one two three), @formatter.instance.fields
     end
   end
 
@@ -313,31 +313,34 @@ module FormatterTest
                  end
     end
 
+    def create_driver(klass_or_str)
+      Fluent::Test::FormatterTestDriver.new(klass_or_str)
+    end
 
     def test_config_params
-      formatter = TextFormatter::SingleValueFormatter.new
-      assert_equal "message", formatter.message_key
+      formatter = create_driver(TextFormatter::SingleValueFormatter)
+      assert_equal "message", formatter.instance.message_key
 
       formatter.configure('message_key' => 'foobar')
-      assert_equal "foobar", formatter.message_key
+      assert_equal "foobar", formatter.instance.message_key
     end
 
     def test_format
-      formatter = Fluent::Plugin.new_formatter('single_value')
+      formatter = create_driver('single_value')
       formatter.configure({})
       formatted = formatter.format('tag', Engine.now, {'message' => 'awesome'})
       assert_equal("awesome#{@newline}", formatted)
     end
 
     def test_format_without_newline
-      formatter = Fluent::Plugin.new_formatter('single_value')
+      formatter = create_driver('single_value')
       formatter.configure('add_newline' => 'false')
       formatted = formatter.format('tag', Engine.now, {'message' => 'awesome'})
       assert_equal("awesome", formatted)
     end
 
     def test_format_with_message_key
-      formatter = TextFormatter::SingleValueFormatter.new
+      formatter = create_driver(TextFormatter::SingleValueFormatter)
       formatter.configure('message_key' => 'foobar')
       formatted = formatter.format('tag', Engine.now, {'foobar' => 'foo'})
 

data/test/test_log.rb

@@ -4,13 +4,17 @@ require 'fluent/engine'
 require 'fluent/log'
 require 'timecop'
 require 'logger'
+require 'securerandom'
+require 'pathname'
 
 class LogTest < Test::Unit::TestCase
-  TMP_DIR = File.expand_path(File.dirname(__FILE__) + "/tmp/log/#{ENV['TEST_ENV_NUMBER']}")
+  def tmp_dir
+    File.join(File.dirname(__FILE__), "tmp", "log", "#{ENV['TEST_ENV_NUMBER']}", SecureRandom.hex(10))
+  end
 
   def setup
-    FileUtils.rm_rf(TMP_DIR)
-    FileUtils.mkdir_p(TMP_DIR)
+    @tmp_dir = tmp_dir
+    FileUtils.mkdir_p(@tmp_dir)
     @log_device = Fluent::Test::DummyLogDevice.new
     @timestamp = Time.parse("2016-04-21 02:58:41 +0000")
     @timestamp_str = @timestamp.strftime("%Y-%m-%d %H:%M:%S %z")
@@ -21,6 +25,21 @@ class LogTest < Test::Unit::TestCase
     @log_device.reset
     Timecop.return
     Thread.current[:last_repeated_stacktrace] = nil
+    begin
+      FileUtils.rm_rf(@tmp_dir)
+    rescue Errno::EACCES
+      # It may occur on Windows because of delete pending state due to delayed GC.
+      # Ruby 3.2 or later doesn't ignore Errno::EACCES:
+      # https://github.com/ruby/ruby/commit/983115cf3c8f75b1afbe3274f02c1529e1ce3a81
+    end
+  end
+
+  def test_per_process_path
+    path = Fluent::Log.per_process_path("C:/tmp/test.log", :supervisor, 0)
+    assert_equal(path, "C:/tmp/test-supervisor-0.log")
+
+    path = Fluent::Log.per_process_path("C:/tmp/test.log", :worker, 1)
+    assert_equal(path, "C:/tmp/test-1.log")
   end
 
   sub_test_case "log level" do
@@ -560,7 +579,7 @@ class LogTest < Test::Unit::TestCase
       Timecop.freeze(@timestamp)
 
       rotate_age, rotate_size, travel_term = expected
-      path = "#{TMP_DIR}/log-dev-io-#{rotate_size}-#{rotate_age}"
+      path = "#{@tmp_dir}/log-dev-io-#{rotate_size}-#{rotate_age}"
 
       logdev = Fluent::LogDeviceIO.new(path, shift_age: rotate_age, shift_size: rotate_size)
       logger = ServerEngine::DaemonLogger.new(logdev)
@@ -583,45 +602,71 @@ class LogTest < Test::Unit::TestCase
 
   def test_log_rotates_specified_size_with_logdevio
     with_timezone('utc') do
-      rotate_age = 2
-      rotate_size = 100
-      path = "#{TMP_DIR}/log-dev-io-#{rotate_size}-#{rotate_age}"
-      path0 = path + '.0'
-      path1 = path + '.1'
+      begin
+        rotate_age = 2
+        rotate_size = 100
+        path = "#{@tmp_dir}/log-dev-io-#{rotate_size}-#{rotate_age}"
+        path0 = path + '.0'
+        path1 = path + '.1'
+
+        logdev = Fluent::LogDeviceIO.new(path, shift_age: rotate_age, shift_size: rotate_size)
+        logger = ServerEngine::DaemonLogger.new(logdev)
+        log = Fluent::Log.new(logger)
+
+        msg = 'a' * 101
+        log.info msg
+        assert_match msg, File.read(path)
+        assert_true File.exist?(path)
+        assert_true !File.exist?(path0)
+        assert_true !File.exist?(path1)
+
+        # create log.0
+        msg2 = 'b' * 101
+        log.info msg2
+        c = File.read(path)
+        c0 = File.read(path0)
+        assert_match msg2, c
+        assert_match msg, c0
+        assert_true File.exist?(path)
+        assert_true File.exist?(path0)
+        assert_true !File.exist?(path1)
+
+        # rotate
+        msg3 = 'c' * 101
+        log.info msg3
+        c = File.read(path)
+        c0 = File.read(path0)
+        assert_match msg3, c
+        assert_match msg2, c0
+        assert_true File.exist?(path)
+        assert_true File.exist?(path0)
+        assert_true !File.exist?(path1)
+      ensure
+       logdev&.close
+     end
+    end
+  end
 
-      logdev = Fluent::LogDeviceIO.new(path, shift_age: rotate_age, shift_size: rotate_size)
-      logger = ServerEngine::DaemonLogger.new(logdev)
-      log = Fluent::Log.new(logger)
+  def test_reopen
+    path = Pathname(@tmp_dir) + "fluent.log"
 
-      msg = 'a' * 101
-      log.info msg
-      assert_match msg, File.read(path)
-      assert_true File.exist?(path)
-      assert_true !File.exist?(path0)
-      assert_true !File.exist?(path1)
+    logdev = Fluent::LogDeviceIO.new(path.to_s)
+    logger = ServerEngine::DaemonLogger.new(logdev)
+    log = Fluent::Log.new(logger, path: path)
 
-      # create log.0
-      msg2 = 'b' * 101
-      log.info msg2
-      c = File.read(path)
-      c0 = File.read(path0)
-      assert_match msg2, c
-      assert_match msg, c0
-      assert_true File.exist?(path)
-      assert_true File.exist?(path0)
-      assert_true !File.exist?(path1)
-
-      # rotate
-      msg3 = 'c' * 101
-      log.info msg3
-      c = File.read(path)
-      c0 = File.read(path0)
-      assert_match msg3, c
-      assert_match msg2, c0
-      assert_true File.exist?(path)
-      assert_true File.exist?(path0)
-      assert_true !File.exist?(path1)
-    end
+    message = "This is test message."
+
+    log.info message
+    log.reopen!
+    log.info message
+
+    assert { path.read.lines.select{ |line| line.include?(message) }.size == 2 }
+    # Assert reopening the same file.
+    # Especially, on Windows, the filepath is fixed for each process with rotate,
+    # so we need to care about this.
+    assert { path.parent.entries.size == 3 } # [".", "..", "fluent.log"]
+  ensure
+    logdev&.close
   end
 end