fluentd 1.15.3 → 1.16.2

data/lib/fluent/test/driver/base.rb

@@ -16,6 +16,7 @@
 
 require 'fluent/config'
 require 'fluent/config/element'
+require 'fluent/env'
 require 'fluent/log'
 require 'fluent/clock'
 
@@ -102,11 +103,16 @@ module Fluent
 
         def instance_start
           if @instance.respond_to?(:server_wait_until_start)
-            @socket_manager_path = ServerEngine::SocketManager::Server.generate_path
-            if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
-              FileUtils.rm_f @socket_manager_path
+            if Fluent.windows?
+              @socket_manager_server = ServerEngine::SocketManager::Server.open
+              @socket_manager_path = @socket_manager_server.path
+            else
+              @socket_manager_path = ServerEngine::SocketManager::Server.generate_path
+              if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
+                FileUtils.rm_f @socket_manager_path
+              end
+              @socket_manager_server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
             end
-            @socket_manager_server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
             ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @socket_manager_path.to_s
           end
 
@@ -173,7 +179,7 @@ module Fluent
 
           if @socket_manager_server
             @socket_manager_server.close
-            if @socket_manager_server.is_a?(String) && File.exist?(@socket_manager_path)
+            if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
               FileUtils.rm_f @socket_manager_path
             end
           end

data/lib/fluent/test/driver/filter.rb

@@ -37,6 +37,10 @@ module Fluent
           @filtered.map {|_time, record| record }
         end
 
+        def filtered_time
+          @filtered.map {|time, _record| time }
+        end
+
         def instance_hook_after_started
           super
           filter_hook = ->(time, record) { @filtered << [time, record] }

data/lib/fluent/test/startup_shutdown.rb

@@ -21,9 +21,8 @@ module Fluent
   module Test
     module StartupShutdown
       def startup
-        socket_manager_path = ServerEngine::SocketManager::Server.generate_path
-        @server = ServerEngine::SocketManager::Server.open(socket_manager_path)
-        ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = socket_manager_path.to_s
+        @server = ServerEngine::SocketManager::Server.open
+        ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @server.path.to_s
       end
 
       def shutdown
@@ -31,15 +30,14 @@ module Fluent
       end
 
       def self.setup
-        @socket_manager_path = ServerEngine::SocketManager::Server.generate_path
-        @server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
-        ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @socket_manager_path.to_s
+        @server = ServerEngine::SocketManager::Server.open
+        ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @server.path.to_s
       end
 
       def self.teardown
         @server.close
-        # on Windows, socket_manager_path is a TCP port number
-        FileUtils.rm_f @socket_manager_path unless Fluent.windows?
+        # on Windows, the path is a TCP port number
+        FileUtils.rm_f @server.path unless Fluent.windows?
       end
     end
   end

data/lib/fluent/version.rb

@@ -16,6 +16,6 @@
 
 module Fluent
 
-  VERSION = '1.15.3'
+  VERSION = '1.16.2'
 
 end

data/templates/new_gem/test/helper.rb.erb

@@ -1,4 +1,3 @@
-$LOAD_PATH.unshift(File.expand_path("../../", __FILE__))
 require "test-unit"
 require "fluent/test"
 require "fluent/test/driver/<%= type %>"

data/test/command/test_ctl.rb

@@ -35,7 +35,7 @@ class TestFluentdCtl < ::Test::Unit::TestCase
       Signal.trap(signal) do
         got_signal = true
       end
-      Fluent::Ctl.new([command, "#{$$}"]).call
+      Fluent::Ctl.new([command, Process.pid.to_s]).call
       assert_true(got_signal)
     end
   end

data/test/command/test_fluentd.rb

@@ -71,6 +71,20 @@ class TestFluentdCommand < ::Test::Unit::TestCase
     end
   end
 
+  def process_kill(pid)
+    if Fluent.windows?
+      Process.kill(:KILL, pid) rescue nil
+      return
+    end
+
+    begin
+      Process.kill(:TERM, pid) rescue nil
+      Timeout.timeout(10){ sleep 0.1 while process_exist?(pid) }
+    rescue Timeout::Error
+      Process.kill(:KILL, pid) rescue nil
+    end
+  end
+
   def execute_command(cmdline, chdir=@tmp_dir, env = {})
     null_stream = Fluent::FileWrapper.open(File::NULL, 'w')
     gemfile_path = File.expand_path(File.dirname(__FILE__) + "../../../Gemfile")
@@ -85,12 +99,12 @@ class TestFluentdCommand < ::Test::Unit::TestCase
         yield pid, io
         # p(here: "execute command", pid: pid, worker_pids: @worker_pids)
       ensure
-        Process.kill(:KILL, pid) rescue nil
+        process_kill(pid)
         if @supervisor_pid
-          Process.kill(:KILL, @supervisor_pid) rescue nil
+          process_kill(@supervisor_pid)
         end
         @worker_pids.each do |cpid|
-          Process.kill(:KILL, cpid) rescue nil
+          process_kill(cpid)
         end
         # p(here: "execute command", pid: pid, exist: process_exist?(pid), worker_pids: @worker_pids, exists: @worker_pids.map{|i| process_exist?(i) })
         Timeout.timeout(10){ sleep 0.1 while process_exist?(pid) }
@@ -112,7 +126,9 @@ class TestFluentdCommand < ::Test::Unit::TestCase
     end
   end
 
-  def assert_log_matches(cmdline, *pattern_list, patterns_not_match: [], timeout: 10, env: {})
+  # ATTENTION: This stops taking logs when all `pattern_list` match or timeout,
+  # so `patterns_not_match` can test only logs up to that point.
+  def assert_log_matches(cmdline, *pattern_list, patterns_not_match: [], timeout: 20, env: {})
     matched = false
     matched_wrongly = false
     assert_error_msg = ""
@@ -121,7 +137,7 @@ class TestFluentdCommand < ::Test::Unit::TestCase
       execute_command(cmdline, @tmp_dir, env) do |pid, stdout|
         begin
           waiting(timeout) do
-            while process_exist?(pid) && !matched
+            while process_exist?(pid)
               readables, _, _ = IO.select([stdout], nil, nil, 1)
               next unless readables
               break if readables.first.eof?
@@ -133,6 +149,18 @@ class TestFluentdCommand < ::Test::Unit::TestCase
               if pattern_list.all?{|ptn| lines.any?{|line| ptn.is_a?(Regexp) ? ptn.match(line) : line.include?(ptn) } }
                 matched = true
               end
+
+              if Fluent.windows?
+                # https://github.com/fluent/fluentd/issues/4095
+                # On Windows, the initial process is different from the supervisor process,
+                # so we need to wait until `SUPERVISOR_PID_PATTERN` appears in the logs to get the pid.
+                # (Worker processes will be killed by the supervisor process, so we don't need it-)
+                break if matched && SUPERVISOR_PID_PATTERN =~ stdio_buf
+              else
+                # On Non-Windows, the initial process is the supervisor process,
+                # so we don't need to wait `SUPERVISOR_PID_PATTERN`.
+                break if matched
+              end
             end
           end
         ensure
@@ -146,6 +174,10 @@ class TestFluentdCommand < ::Test::Unit::TestCase
       end
     rescue Timeout::Error
       assert_error_msg = "execution timeout"
+      # https://github.com/fluent/fluentd/issues/4095
+      # On Windows, timeout without `@supervisor_pid` means that the test is invalid,
+      # since the supervisor process will survive without being killed correctly.
+      flunk("Invalid test: The pid of supervisor could not be taken, which is necessary on Windows.") if Fluent.windows? && @supervisor_pid.nil?
     rescue => e
       assert_error_msg = "unexpected error in launching fluentd: #{e.inspect}"
     else
@@ -177,7 +209,7 @@ class TestFluentdCommand < ::Test::Unit::TestCase
     assert matched && !matched_wrongly, assert_error_msg
   end
 
-  def assert_fluentd_fails_to_start(cmdline, *pattern_list, timeout: 10)
+  def assert_fluentd_fails_to_start(cmdline, *pattern_list, timeout: 20)
     # empty_list.all?{ ... } is always true
     matched = false
     running = false
@@ -213,6 +245,10 @@ class TestFluentdCommand < ::Test::Unit::TestCase
       end
     rescue Timeout::Error
       assert_error_msg = "execution timeout with command out:\n" + stdio_buf
+      # https://github.com/fluent/fluentd/issues/4095
+      # On Windows, timeout without `@supervisor_pid` means that the test is invalid,
+      # since the supervisor process will survive without being killed correctly.
+      flunk("Invalid test: The pid of supervisor could not be taken, which is necessary on Windows.") if Fluent.windows? && @supervisor_pid.nil?
     rescue => e
       assert_error_msg = "unexpected error in launching fluentd: #{e.inspect}\n" + stdio_buf
       assert false, assert_error_msg
@@ -1151,4 +1187,99 @@ CONF
                          "shared socket for multiple workers is disabled",)
     end
   end
+
+  # TODO: `patterns_not_match` can test only logs up to `pattern_list`,
+  # so we need to fix some meaningless `patterns_not_match` conditions.
+  sub_test_case 'log_level by command line option' do
+    test 'info' do
+      conf = ""
+      conf_path = create_conf_file('empty.conf', conf)
+      assert File.exist?(conf_path)
+      assert_log_matches(create_cmdline(conf_path),
+                         "[info]",
+                         patterns_not_match: ["[debug]"])
+    end
+
+    test 'debug' do
+      conf = ""
+      conf_path = create_conf_file('empty.conf', conf)
+      assert File.exist?(conf_path)
+      assert_log_matches(create_cmdline(conf_path, "-v"),
+                         "[debug]",
+                         patterns_not_match: ["[trace]"])
+    end
+
+    data("Trace" => "-vv")
+    data("Invalid low level should be treated as Trace level": "-vvv")
+    test 'trace' do |option|
+      conf = <<CONF
+<source>
+  @type sample
+  tag test
+</source>
+CONF
+      conf_path = create_conf_file('sample.conf', conf)
+      assert File.exist?(conf_path)
+      assert_log_matches(create_cmdline(conf_path, option),
+                         "[trace]",)
+    end
+
+    test 'warn' do
+      omit "Can't run on Windows since there is no way to take pid of the supervisor." if Fluent.windows?
+      conf = <<CONF
+<source>
+  @type sample
+  tag test
+</source>
+CONF
+      conf_path = create_conf_file('sample.conf', conf)
+      assert File.exist?(conf_path)
+      assert_log_matches(create_cmdline(conf_path, "-q"),
+                         "[warn]",
+                         patterns_not_match: ["[info]"])
+    end
+
+    data("Error" => "-qq")
+    data("Fatal should be treated as Error level" => "-qqq")
+    data("Invalid high level should be treated as Error level": "-qqqq")
+    test 'error' do |option|
+      # This test can run on Windows correctly,
+      # since the process will stop automatically with an error.
+      conf = <<CONF
+<source>
+  @type plugin_not_found
+  tag test
+</source>
+CONF
+      conf_path = create_conf_file('plugin_not_found.conf', conf)
+      assert File.exist?(conf_path)
+      assert_log_matches(create_cmdline(conf_path, option),
+                         "[error]",
+                         patterns_not_match: ["[warn]"])
+    end
+
+    test 'system config one should not be overwritten when cmd line one is not specified' do
+      conf = <<CONF
+<system>
+  log_level debug
+</system>
+CONF
+      conf_path = create_conf_file('debug.conf', conf)
+      assert File.exist?(conf_path)
+      assert_log_matches(create_cmdline(conf_path),
+                         "[debug]")
+    end
+  end
+
+  sub_test_case "inline_config" do
+    test "can change log_level by --inline-config" do
+      # Since we can't define multiple `<system>` directives, this use-case is not recommended.
+      # This is just for this test.
+      inline_conf = '<system>\nlog_level debug\n</system>'
+      conf_path = create_conf_file('test.conf', "")
+      assert File.exist?(conf_path)
+      assert_log_matches(create_cmdline(conf_path, "--inline-config", inline_conf),
+                         "[debug]")
+    end
+  end
 end

data/test/command/test_plugin_config_formatter.rb

@@ -188,7 +188,6 @@ slow_flush_log_threshold: float: (20.0)
  retry_exponential_backoff_base: float: (2)
  retry_max_interval: time: (nil)
  retry_randomize: bool: (true)
- disable_chunk_backup: bool: (false)
 <secondary>: optional, single
  @type: string: (nil)
  <buffer>: optional, single

data/test/compat/test_parser.rb

@@ -38,14 +38,14 @@ class TextParserTest < ::Test::Unit::TestCase
 
   def test_parse_with_return
     parser = Fluent::TextParser.new
-    parser.configure('format' => 'none')
+    parser.configure(config_element('test', '', 'format' => 'none'))
     _time, record = parser.parse('log message!')
     assert_equal({'message' => 'log message!'}, record)
   end
 
   def test_parse_with_block
     parser = Fluent::TextParser.new
-    parser.configure('format' => 'none')
+    parser.configure(config_element('test', '', 'format' => 'none'))
     parser.parse('log message!') { |time, record|
       assert_equal({'message' => 'log message!'}, record)
     }
@@ -53,7 +53,7 @@ class TextParserTest < ::Test::Unit::TestCase
 
   def test_multi_event_parser
     parser = Fluent::TextParser.new
-    parser.configure('format' => 'multi_event_test')
+    parser.configure(config_element('test', '', 'format' => 'multi_event_test'))
     i = 0
     parser.parse('log message!') { |time, record|
       assert_equal('log message!', record['message'])
@@ -67,7 +67,7 @@ class TextParserTest < ::Test::Unit::TestCase
     assert_nil p1.estimate_current_event
     assert_nil p1.parser
 
-    p1.configure('format' => 'none')
+    p1.configure(config_element('test', '', 'format' => 'none'))
     assert_equal true, p1.parser.estimate_current_event
 
     p2 = Fluent::TextParser.new
@@ -76,7 +76,7 @@ class TextParserTest < ::Test::Unit::TestCase
 
     p2.estimate_current_event = false
 
-    p2.configure('format' => 'none')
+    p2.configure(config_element('test', '', 'format' => 'none'))
     assert_equal false, p2.parser.estimate_current_event
   end
 

data/test/config/test_system_config.rb

@@ -5,13 +5,6 @@ require 'fluent/config/section'
 require 'fluent/system_config'
 
 module Fluent::Config
-  class FakeLoggerInitializer
-    attr_accessor :level
-    def initialize
-      @level = nil
-    end
-  end
-
   class FakeSupervisor
     attr_writer :log_level
 
@@ -21,7 +14,6 @@ module Fluent::Config
         wokers: nil,
         restart_worker_interval: nil,
         root_dir: nil,
-        log: FakeLoggerInitializer.new,
         log_level: Fluent::Log::LEVEL_INFO,
         suppress_interval: nil,
         suppress_config_dump: nil,

data/test/log/test_console_adapter.rb

@@ -0,0 +1,110 @@
+require_relative '../helper'
+
+require 'fluent/log'
+require 'fluent/log/console_adapter'
+
+class ConsoleAdapterTest < Test::Unit::TestCase
+  def setup
+    @timestamp = Time.parse("2023-01-01 15:32:41 +0000")
+    @timestamp_str = @timestamp.strftime("%Y-%m-%d %H:%M:%S %z")
+    Timecop.freeze(@timestamp)
+
+    @logdev = Fluent::Test::DummyLogDevice.new
+    @logger = ServerEngine::DaemonLogger.new(@logdev)
+    @fluent_log = Fluent::Log.new(@logger)
+    @console_logger = Fluent::Log::ConsoleAdapter.wrap(@fluent_log)
+  end
+
+  def teardown
+    Timecop.return
+  end
+
+  def test_expected_log_levels
+    assert_equal({debug: 0, info: 1, warn: 2, error: 3, fatal: 4},
+                 Console::Logger::LEVELS)
+  end
+
+  data(trace: [Fluent::Log::LEVEL_TRACE, :debug],
+       debug: [Fluent::Log::LEVEL_DEBUG, :debug],
+       info: [Fluent::Log::LEVEL_INFO, :info],
+       warn: [Fluent::Log::LEVEL_WARN, :warn],
+       error: [Fluent::Log::LEVEL_ERROR, :error],
+       fatal: [Fluent::Log::LEVEL_FATAL, :fatal])
+  def test_reflect_log_level(data)
+    level, expected = data
+    @fluent_log.level = level
+    console_logger = Fluent::Log::ConsoleAdapter.wrap(@fluent_log)
+    assert_equal(Console::Logger::LEVELS[expected],
+                 console_logger.level)
+  end
+
+  data(debug: :debug,
+       info: :info,
+       warn: :warn,
+       error: :error,
+       fatal: :fatal)
+  def test_string_subject(level)
+    @console_logger.send(level, "subject")
+    assert_equal(["#{@timestamp_str} [#{level}]:   0.0s: subject\n"],
+                 @logdev.logs)
+  end
+
+  data(debug: :debug,
+       info: :info,
+       warn: :warn,
+       error: :error,
+       fatal: :fatal)
+  def test_args(level)
+    @console_logger.send(level, "subject", 1, 2, 3)
+    assert_equal([
+                   "#{@timestamp_str} [#{level}]:   0.0s: subject\n" +
+                   "      | 1\n" +
+                   "      | 2\n" +
+                   "      | 3\n"
+                 ],
+                 @logdev.logs)
+  end
+
+  data(debug: :debug,
+       info: :info,
+       warn: :warn,
+       error: :error,
+       fatal: :fatal)
+  def test_options(level)
+    @console_logger.send(level, "subject", kwarg1: "opt1", kwarg2: "opt2")
+    assert_equal([
+                   "#{@timestamp_str} [#{level}]:   0.0s: subject\n" +
+                   "      | {\"kwarg1\":\"opt1\",\"kwarg2\":\"opt2\"}\n"
+                 ],
+                 @logdev.logs)
+  end
+
+  data(debug: :debug,
+       info: :info,
+       warn: :warn,
+       error: :error,
+       fatal: :fatal)
+  def test_block(level)
+    @console_logger.send(level, "subject") { "block message" }
+    assert_equal([
+                   "#{@timestamp_str} [#{level}]:   0.0s: subject\n" +
+                   "      | block message\n"
+                 ],
+                 @logdev.logs)
+  end
+
+  data(debug: :debug,
+       info: :info,
+       warn: :warn,
+       error: :error,
+       fatal: :fatal)
+  def test_multiple_entries(level)
+    @console_logger.send(level, "subject1")
+    @console_logger.send(level, "line2")
+    assert_equal([
+                   "#{@timestamp_str} [#{level}]:   0.0s: subject1\n",
+                   "#{@timestamp_str} [#{level}]:   0.0s: line2\n"
+                 ],
+                 @logdev.logs)
+  end
+end

data/test/plugin/in_tail/test_position_file.rb

@@ -26,6 +26,10 @@ class IntailPositionFileTest < Test::Unit::TestCase
     "valid_path" => Fluent::Plugin::TailInput::TargetInfo.new("valid_path", 1),
     "inode23bit" => Fluent::Plugin::TailInput::TargetInfo.new("inode23bit", 0),
   }
+  TEST_CONTENT_INODES = {
+    1 => Fluent::Plugin::TailInput::TargetInfo.new("valid_path", 1),
+    0 => Fluent::Plugin::TailInput::TargetInfo.new("inode23bit", 0),
+  }
 
   def write_data(f, content)
     f.write(content)
@@ -221,7 +225,7 @@ class IntailPositionFileTest < Test::Unit::TestCase
   end
 
   sub_test_case '#unwatch' do
-    test 'deletes entry by path' do
+    test 'unwatch entry by path' do
       write_data(@file, TEST_CONTENT)
       pf = Fluent::Plugin::TailInput::PositionFile.load(@file, false, {}, logger: $log)
       inode1 = File.stat(@file).ino
@@ -239,6 +243,32 @@ class IntailPositionFileTest < Test::Unit::TestCase
 
       assert_not_equal p1, p2
     end
+
+    test 'unwatch entries by inode' do
+      write_data(@file, TEST_CONTENT)
+      pf = Fluent::Plugin::TailInput::PositionFile.load(@file, true, TEST_CONTENT_INODES, logger: $log)
+
+      existing_targets = TEST_CONTENT_INODES.select do |inode, target_info|
+        inode == 1
+      end
+      pe_to_unwatch = pf[TEST_CONTENT_INODES[0]]
+
+      pf.unwatch_removed_targets(existing_targets)
+
+      assert_equal(
+        {
+          map_keys: [TEST_CONTENT_INODES[1].ino],
+          unwatched_pe_pos: Fluent::Plugin::TailInput::PositionFile::UNWATCHED_POSITION,
+        },
+        {
+          map_keys: pf.instance_variable_get(:@map).keys,
+          unwatched_pe_pos: pe_to_unwatch.read_pos,
+        }
+      )
+
+      unwatched_pe_retaken = pf[TEST_CONTENT_INODES[0]]
+      assert_not_equal pe_to_unwatch, unwatched_pe_retaken
+    end
   end
 
   sub_test_case 'FilePositionEntry' do

data/test/plugin/out_forward/test_ack_handler.rb

@@ -98,4 +98,43 @@ class AckHandlerTest < Test::Unit::TestCase
       w.close rescue nil
     end
   end
+
+  # ForwardOutput uses AckHandler in multiple threads, so we need to assume this case.
+  # If exclusive control for this case is implemented, this test may not be necessary.
+  test 'raises no error when another thread closes a socket' do
+    ack_handler = Fluent::Plugin::ForwardOutput::AckHandler.new(timeout: 10, log: $log, read_length: 100)
+
+    node = flexmock('node', host: '127.0.0.1', port: '1000') # for log
+    chunk_id = 'chunk_id 111'
+    ack = ack_handler.create_ack(chunk_id, node)
+
+    r, w = IO.pipe
+    begin
+      w.write(chunk_id)
+      stub(r).recv { |_|
+        sleep(1) # To ensure that multiple threads select the socket before closing.
+        raise IOError, 'stream closed in another thread' if r.closed?
+        MessagePack.pack({ 'ack' => Base64.encode64('chunk_id 111') })
+      }
+      ack.enqueue(r)
+
+      threads = []
+      2.times do
+        threads << Thread.new do
+          ack_handler.collect_response(1) do |cid, n, s, ret|
+            s&.close
+          end
+        end
+      end
+
+      assert_true threads.map{ |t| t.join(10) }.all?
+      assert_false(
+        $log.out.logs.any? { |log| log.include?('[error]') },
+        $log.out.logs.select{ |log| log.include?('[error]') }.join('\n')
+      )
+    ensure
+      r.close rescue nil
+      w.close rescue nil
+    end
+  end
 end