fluentd 1.15.2-x64-mingw32 → 1.16.0-x64-mingw32

data/test/plugin/test_in_tcp.rb

@@ -156,6 +156,19 @@ class TcpInputTest < Test::Unit::TestCase
     assert_equal hostname, event[2]['host']
   end
 
+  test "send_keepalive_packet_can_be_enabled" do
+    d = create_driver(base_config + %!
+      format none
+      send_keepalive_packet true
+    !)
+    assert_true d.instance.send_keepalive_packet
+
+    d = create_driver(base_config + %!
+      format none
+    !)
+    assert_false d.instance.send_keepalive_packet
+  end
+
   test 'source_address_key' do
     d = create_driver(base_config + %!
       format none
@@ -213,6 +226,8 @@ class TcpInputTest < Test::Unit::TestCase
 
       assert_equal 1, d.instance.log.logs.count { |l| l =~ /anonymous client/ }
       assert_equal 0, d.events.size
+    ensure
+      d.instance_shutdown if d&.instance
     end
   end
 

data/test/plugin/test_out_file.rb

@@ -5,6 +5,7 @@ require 'fileutils'
 require 'time'
 require 'timecop'
 require 'zlib'
+require 'fluent/file_wrapper'
 
 class FileOutputTest < Test::Unit::TestCase
   def setup
@@ -1016,7 +1017,7 @@ class FileOutputTest < Test::Unit::TestCase
 
     test 'returns filepath with index which does not exist yet' do
       5.times do |i|
-        File.open(File.join(@tmp, "exist_#{i}.log"), 'a'){|f| } # open(create) and close
+        Fluent::FileWrapper.open(File.join(@tmp, "exist_#{i}.log"), 'a'){|f| } # open(create) and close
       end
       @i.find_filepath_available(File.join(@tmp, "exist_**.log")) do |path|
         assert_equal File.join(@tmp, "exist_5.log"), path
@@ -1025,7 +1026,7 @@ class FileOutputTest < Test::Unit::TestCase
 
     test 'creates lock directory when with_lock is true to exclude operations of other worker process' do
       5.times do |i|
-        File.open(File.join(@tmp, "exist_#{i}.log"), 'a')
+        Fluent::FileWrapper.open(File.join(@tmp, "exist_#{i}.log"), 'a')
       end
       Dir.mkdir(File.join(@tmp, "exist_5.log.lock"))
       @i.find_filepath_available(File.join(@tmp, "exist_**.log"), with_lock: true) do |path|

data/test/plugin/test_out_forward.rb

@@ -1331,26 +1331,22 @@ EOL
         d = create_driver(output_conf)
         d.instance_start
 
-        begin
-          chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
-          mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,
-                                                   linger_timeout: anything,
-                                                   send_timeout: anything,
-                                                   recv_timeout: anything,
-                                                   connect_timeout: anything) { |sock|
-            mock(sock).close.once; sock
-          }.twice
-
-          target_input_driver.run(timeout: 15) do
-            d.run(shutdown: false) do
-              node = d.instance.nodes.first
-              2.times do
-                node.send_data('test', chunk) rescue nil
-              end
+        chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
+        mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,
+                                                 linger_timeout: anything,
+                                                 send_timeout: anything,
+                                                 recv_timeout: anything,
+                                                 connect_timeout: anything) { |sock|
+          mock(sock).close.once; sock
+        }.twice
+
+        target_input_driver.run(timeout: 15) do
+          d.run do
+            node = d.instance.nodes.first
+            2.times do
+              node.send_data('test', chunk) rescue nil
             end
           end
-        ensure
-          d.instance_shutdown
         end
       end
     end

data/test/plugin/test_out_http.rb

@@ -378,6 +378,7 @@ class HTTPOutputTest < Test::Unit::TestCase
           password hello?
         </auth>
       ])
+      d.instance.system_config_override(root_dir: TMP_DIR) # Backup files are generated in TMP_DIR.
       d.run(default_tag: 'test.http', shutdown: false) do
         test_events.each { |event|
           d.feed(event)

data/test/plugin/test_output.rb

@@ -803,7 +803,10 @@ class OutputTest < Test::Unit::TestCase
     end
 
     test 'output plugin will call #try_write for plugin supports delayed commit only to flush buffer chunks' do
+      tmp_dir = File.join(__dir__, '../tmp/test_output')
+
       i = create_output(:delayed)
+      i.system_config_override(root_dir: tmp_dir) # Backup files are generated in `tmp_dir`.
       try_write_called = false
       i.register(:try_write){|chunk| try_write_called = true; commit_write(chunk.unique_id) }
 
@@ -820,6 +823,8 @@ class OutputTest < Test::Unit::TestCase
       assert try_write_called
 
       i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
+    ensure
+      FileUtils.rm_rf(tmp_dir)
     end
 
     test '#prefer_delayed_commit (returns false) decides delayed commit is disabled if both are implemented' do
@@ -849,7 +854,10 @@ class OutputTest < Test::Unit::TestCase
     end
 
     test '#prefer_delayed_commit (returns true) decides delayed commit is enabled if both are implemented' do
+      tmp_dir = File.join(__dir__, '../tmp/test_output')
+
       i = create_output(:full)
+      i.system_config_override(root_dir: tmp_dir) # Backup files are generated in `tmp_dir`.
       write_called = false
       try_write_called = false
       i.register(:write){ |chunk| write_called = true }
@@ -872,6 +880,8 @@ class OutputTest < Test::Unit::TestCase
       assert try_write_called
 
       i.stop; i.before_shutdown; i.shutdown; i.after_shutdown; i.close; i.terminate
+    ensure
+      FileUtils.rm_rf(tmp_dir)
     end
 
     test 'flush_interval is ignored when flush_mode is not interval' do
@@ -1062,4 +1072,263 @@ class OutputTest < Test::Unit::TestCase
       }
     end
   end
+
+  sub_test_case "actual_flush_thread_count" do
+    data(
+      "Not buffered",
+      {
+        output_type: :sync,
+        config: config_element(),
+        expected: 0,
+      }
+    )
+    data(
+      "Buffered with singile thread",
+      {
+        output_type: :full,
+        config: config_element("ROOT", "", {}, [config_element("buffer", "", {})]),
+        expected: 1,
+      }
+    )
+    data(
+      "Buffered with multiple threads",
+      {
+        output_type: :full,
+        config: config_element("ROOT", "", {}, [config_element("buffer", "", {"flush_thread_count" => 8})]),
+        expected: 8,
+      }
+    )
+    test "actual_flush_thread_count" do |data|
+      o = create_output(data[:output_type])
+      o.configure(data[:config])
+      assert_equal data[:expected], o.actual_flush_thread_count
+    end
+
+    data(
+      "Buffered with single thread",
+      {
+        output_type: :full,
+        config: config_element(
+          "ROOT", "", {},
+          [
+            config_element("buffer", "", {}), 
+            config_element("secondary", "", {"@type" => "test", "name" => "test"}),
+          ]
+        ),
+        expected: 1,
+      }
+    )
+    data(
+      "Buffered with multiple threads",
+      {
+        output_type: :full,
+        config: config_element(
+          "ROOT", "", {},
+          [
+            config_element("buffer", "", {"flush_thread_count" => 8}),
+            config_element("secondary", "", {"@type" => "test", "name" => "test"}),
+          ]
+        ),
+        expected: 8,
+      }
+    )
+    test "actual_flush_thread_count for secondary" do |data|
+      primary = create_output(data[:output_type])
+      primary.configure(data[:config])
+      assert_equal data[:expected], primary.secondary.actual_flush_thread_count
+    end
+  end
+
+  sub_test_case "synchronize_path" do
+    def setup
+      Dir.mktmpdir do |lock_dir|
+        ENV['FLUENTD_LOCK_DIR'] = lock_dir
+        yield
+      end
+    end
+
+    def assert_worker_lock(lock_path, expect_locked)
+      # With LOCK_NB set, flock() returns:
+      #   * `false` when the file is already locked.
+      #   * `0` when the file is not locked.
+      File.open(lock_path, "w") do |f|
+        if expect_locked
+          assert_equal false, f.flock(File::LOCK_EX|File::LOCK_NB)
+        else
+          assert_equal 0, f.flock(File::LOCK_EX|File::LOCK_NB)
+        end
+      end
+    end
+
+    def assert_thread_lock(output_plugin, expect_locked)
+      t = Thread.new do
+        output_plugin.synchronize_path("test") do
+        end
+      end
+      if expect_locked
+        assert_nil t.join(3)
+      else
+        assert_not_nil t.join(3)
+      end
+    end
+
+    data(
+      "Not buffered with single worker",
+      {
+        output_type: :sync,
+        config: config_element(),
+        workers: 1,
+        expect_worker_lock: false,
+        expect_thread_lock: false,
+      }
+    )
+    data(
+      "Not buffered with multiple workers",
+      {
+        output_type: :sync,
+        config: config_element(),
+        workers: 4,
+        expect_worker_lock: true,
+        expect_thread_lock: false,
+      }
+    )
+    data(
+      "Buffered with single thread and single worker",
+      {
+        output_type: :full,
+        config: config_element("ROOT", "", {}, [config_element("buffer", "", {})]),
+        workers: 1,
+        expect_worker_lock: false,
+        expect_thread_lock: false,
+      }
+    )
+    data(
+      "Buffered with multiple threads and single worker",
+      {
+        output_type: :full,
+        config: config_element("ROOT", "", {}, [config_element("buffer", "", {"flush_thread_count" => 8})]),
+        workers: 1,
+        expect_worker_lock: false,
+        expect_thread_lock: true,
+      }
+    )
+    data(
+      "Buffered with single thread and multiple workers",
+      {
+        output_type: :full,
+        config: config_element("ROOT", "", {}, [config_element("buffer", "", {})]),
+        workers: 4,
+        expect_worker_lock: true,
+        expect_thread_lock: false,
+      }
+    )
+    data(
+      "Buffered with multiple threads and multiple workers",
+      {
+        output_type: :full,
+        config: config_element("ROOT", "", {}, [config_element("buffer", "", {"flush_thread_count" => 8})]),
+        workers: 4,
+        expect_worker_lock: true,
+        expect_thread_lock: true,
+      }
+    )
+    test "synchronize_path" do |data|
+      o = create_output(data[:output_type])
+      o.configure(data[:config])
+      o.system_config_override(workers: data[:workers])
+
+      test_lock_name = "test_lock_name"
+      lock_path = o.get_lock_path(test_lock_name)
+
+      o.synchronize_path(test_lock_name) do
+        assert_worker_lock(lock_path, data[:expect_worker_lock])
+        assert_thread_lock(o, data[:expect_thread_lock])
+      end
+
+      assert_worker_lock(lock_path, false)
+      assert_thread_lock(o, false)
+    end
+
+    data(
+      "Buffered with single thread and single worker",
+      {
+        output_type: :full,
+        config: config_element(
+          "ROOT", "", {},
+          [
+            config_element("buffer", "", {}),
+            config_element("secondary", "", {"@type" => "test", "name" => "test"}),
+          ]
+        ),
+        workers: 1,
+        expect_worker_lock: false,
+        expect_thread_lock: false,
+      }
+    )
+    data(
+      "Buffered with multiple threads and single worker",
+      {
+        output_type: :full,
+        config: config_element(
+          "ROOT", "", {},
+          [
+            config_element("buffer", "", {"flush_thread_count" => 8}),
+            config_element("secondary", "", {"@type" => "test", "name" => "test"}),
+          ]
+        ),
+        workers: 1,
+        expect_worker_lock: false,
+        expect_thread_lock: true,
+      }
+    )
+    data(
+      "Buffered with single thread and multiple workers",
+      {
+        output_type: :full,
+        config: config_element(
+          "ROOT", "", {},
+          [
+            config_element("buffer", "", {}),
+            config_element("secondary", "", {"@type" => "test", "name" => "test"}),
+          ]
+        ),
+        workers: 4,
+        expect_worker_lock: true,
+        expect_thread_lock: false,
+      }
+    )
+    data(
+      "Buffered with multiple threads and multiple workers",
+      {
+        output_type: :full,
+        config: config_element(
+          "ROOT", "", {},
+          [
+            config_element("buffer", "", {"flush_thread_count" => 8}),
+            config_element("secondary", "", {"@type" => "test", "name" => "test"}),
+          ]
+        ),
+        workers: 4,
+        expect_worker_lock: true,
+        expect_thread_lock: true,
+      }
+    )
+    test "synchronize_path for secondary" do |data|
+      primary = create_output(data[:output_type])
+      primary.configure(data[:config])
+      secondary = primary.secondary
+      secondary.system_config_override(workers: data[:workers])
+
+      test_lock_name = "test_lock_name"
+      lock_path = secondary.get_lock_path(test_lock_name)
+
+      secondary.synchronize_path(test_lock_name) do
+        assert_worker_lock(lock_path, data[:expect_worker_lock])
+        assert_thread_lock(secondary, data[:expect_thread_lock])
+      end
+
+      assert_worker_lock(lock_path, false)
+      assert_thread_lock(secondary, false)
+    end
+  end
 end

data/test/plugin/test_parser_regexp.rb

@@ -28,12 +28,7 @@ class RegexpParserTest < ::Test::Unit::TestCase
       if initialize_conf
         Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp, conf))
       else
-        # Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp)).configure(conf)
-        instance = Fluent::Compat::TextParser::RegexpParser.new(regexp)
-        instance.configure(conf)
-        d = Struct.new(:instance).new
-        d.instance = instance
-        d
+        Fluent::Test::Driver::Parser.new(Fluent::Compat::TextParser::RegexpParser.new(regexp)).configure(conf)
       end
     end
 

data/test/plugin_helper/test_http_server_helper.rb

@@ -127,7 +127,7 @@ class HttpHelperTest < Test::Unit::TestCase
     end
 
     client = Async::HTTP::Client.new(Async::HTTP::Endpoint.parse("https://#{addr}:#{port}", ssl_context: context))
-    reactor = Async::Reactor.new(nil, logger: NULL_LOGGER)
+    reactor = Async::Reactor.new(nil, logger: Fluent::Log::ConsoleAdapter.wrap(NULL_LOGGER))
 
     resp = nil
     error = nil

data/test/plugin_helper/test_server.rb

@@ -16,11 +16,16 @@ class ServerPluginHelperTest < Test::Unit::TestCase
 
   setup do
     @port = unused_port
-    @socket_manager_path = ServerEngine::SocketManager::Server.generate_path
-    if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
-      FileUtils.rm_f @socket_manager_path
+    if Fluent.windows?
+      @socket_manager_server = ServerEngine::SocketManager::Server.open
+      @socket_manager_path = @socket_manager_server.path
+    else
+      @socket_manager_path = ServerEngine::SocketManager::Server.generate_path
+      if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
+        FileUtils.rm_f @socket_manager_path
+      end
+      @socket_manager_server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
     end
-    @socket_manager_server = ServerEngine::SocketManager::Server.open(@socket_manager_path)
     ENV['SERVERENGINE_SOCKETMANAGER_PATH'] = @socket_manager_path.to_s
 
     @d = Dummy.new
@@ -37,7 +42,7 @@ class ServerPluginHelperTest < Test::Unit::TestCase
     (@d.terminated? || @d.terminate) rescue nil
 
     @socket_manager_server.close
-    if @socket_manager_server.is_a?(String) && File.exist?(@socket_manager_path)
+    if @socket_manager_path.is_a?(String) && File.exist?(@socket_manager_path)
       FileUtils.rm_f @socket_manager_path
     end
   end

data/test/test_config.rb

@@ -237,6 +237,63 @@ class ConfigTest < Test::Unit::TestCase
         ])
     end
 
+    def test_included_glob
+      write_config "#{TMP_DIR}/config.yaml", <<-EOS
+      config:
+        - !include "include/*.yaml"
+      EOS
+      write_config "#{TMP_DIR}/include/02_source2.yaml", <<-EOS
+      - source:
+          $type: dummy
+          tag: tag.dummy
+      EOS
+      write_config "#{TMP_DIR}/include/01_source1.yaml", <<-EOS
+      - source:
+          $type: tcp
+          tag: tag.tcp
+          parse:
+            $arg:
+              - why.parse.section.doesnot.have.arg
+              - huh
+            $type: none
+      EOS
+      write_config "#{TMP_DIR}/include/03_match1.yaml", <<-EOS
+      - match:
+          $tag: tag.*
+          $type: stdout
+          buffer:
+            $type: memory
+            flush_interval: 1s
+      EOS
+      root_conf = read_config("#{TMP_DIR}/config.yaml", use_yaml: true)
+      tcp_source_conf = root_conf.elements.first
+      dummy_source_conf = root_conf.elements[1]
+      parse_tcp_conf = tcp_source_conf.elements.first
+      match_conf = root_conf.elements[2]
+
+      assert_equal(
+        [
+          'tcp',
+          'tag.tcp',
+          'none',
+          'why.parse.section.doesnot.have.arg,huh',
+          'dummy',
+          'tag.dummy',
+          'stdout',
+          'tag.*',
+        ],
+        [
+          tcp_source_conf['@type'],
+          tcp_source_conf['tag'],
+          parse_tcp_conf['@type'],
+          parse_tcp_conf.arg,
+          dummy_source_conf['@type'],
+          dummy_source_conf['tag'],
+          match_conf['@type'],
+          match_conf.arg,
+        ])
+    end
+
     def test_check_not_fetchd
       write_config "#{TMP_DIR}/config_test_not_fetched.yaml", <<-EOS
       config:
@@ -287,27 +344,6 @@ class ConfigTest < Test::Unit::TestCase
     File.open(path, "w:#{encoding}:utf-8") {|f| f.write data }
   end
 
-  def test_inline
-    prepare_config
-    opts = {
-      :config_path => "#{TMP_DIR}/config_test_1.conf",
-      :inline_config => "<source>\n  type http\n  port 2222\n </source>",
-      :use_v1_config => false
-    }
-    assert_nothing_raised do
-      Fluent::Supervisor.new(opts)
-    end
-    create_warn_dummy_logger
-  end
-
-  def create_warn_dummy_logger
-    dl_opts = {}
-    dl_opts[:log_level] = ServerEngine::DaemonLogger::WARN
-    logdev = Fluent::Test::DummyLogDevice.new
-    logger = ServerEngine::DaemonLogger.new(logdev, dl_opts)
-    $log = Fluent::Log.new(logger)
-  end
-
   sub_test_case '.build' do
     test 'read config' do
       write_config("#{TMP_DIR}/build/config_build.conf", 'key value')

data/test/{plugin/test_file_wrapper.rb → test_file_wrapper.rb}

@@ -1,5 +1,5 @@
-require_relative '../helper'
-require 'fluent/plugin/file_wrapper'
+require_relative 'helper'
+require 'fluent/file_wrapper'
 
 class FileWrapperTest < Test::Unit::TestCase
   TMP_DIR = File.dirname(__FILE__) + "/../tmp/file_wrapper#{ENV['TEST_ENV_NUMBER']}"