fluentd 1.16.7-x86-mingw32 → 1.17.0-x86-mingw32

data/test/plugin/test_in_tail.rb

@@ -1538,6 +1538,147 @@ class TailInputTest < Test::Unit::TestCase
       assert_equal(ex_paths - [ex_paths.last], plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
     end
 
+    sub_test_case "expand_paths with glob" do |data|
+      sub_test_case "extended_glob" do
+        data("curly braces"          => [true, "always", "test/plugin/data/log_numeric/{0,1}*.log"],
+             "square brackets"       => [true, "always", "test/plugin/data/log_numeric/[0-1][2-4].log"],
+             "asterisk"              => [true, "always", "test/plugin/data/log/*.log"],
+             "one character matcher" => [true, "always", "test/plugin/data/log/tes?.log"],
+            )
+        def test_expand_paths_with_use_glob_p_and_almost_set_of_patterns
+          result, option, path = data
+          config = config_element("", "", {
+                                    "tag" => "tail",
+                                    "path" => path,
+                                    "format" => "none",
+                                    "pos_file" => "#{@tmp_dir}/tail.pos",
+                                    "read_from_head" => true,
+                                    "refresh_interval" => 30,
+                                    "glob_policy" => option,
+                                    "path_delimiter" => "|",
+                                    "rotate_wait" => "#{EX_ROTATE_WAIT}s",
+                                    "follow_inodes" => "#{EX_FOLLOW_INODES}",
+                                  })
+          plugin = create_driver(config, false).instance
+          assert_equal(result, !!plugin.use_glob?(path))
+        end
+
+        data("curly braces"          => [true, false, "extended", "test/plugin/data/log_numeric/{0,1}*.log"],
+             "square brackets"       => [false, true, "extended", "test/plugin/data/log_numeric/[0-1][2-4].log"],
+             "asterisk"              => [false, true, "extended", "test/plugin/data/log/*.log"],
+             "one character matcher" => [false, true, "extended", "test/plugin/data/log/tes?.log"],
+            )
+        def test_expand_paths_with_use_glob_p
+          emit_exception_p, result, option, path = data
+          config = config_element("", "", {
+                                    "tag" => "tail",
+                                    "path" => path,
+                                    "format" => "none",
+                                    "pos_file" => "#{@tmp_dir}/tail.pos",
+                                    "read_from_head" => true,
+                                    "refresh_interval" => 30,
+                                    "glob_policy" => option,
+                                    "rotate_wait" => "#{EX_ROTATE_WAIT}s",
+                                    "follow_inodes" => "#{EX_FOLLOW_INODES}",
+                                  })
+          if emit_exception_p
+            assert_raise(Fluent::ConfigError) do
+              plugin = create_driver(config, false).instance
+            end
+          else
+            plugin = create_driver(config, false).instance
+            assert_equal(result, !!plugin.use_glob?(path))
+          end
+        end
+      end
+
+      sub_test_case "only_use_backward_compatible" do
+        data("square brackets"       => [false, "backward_compatible", "test/plugin/data/log_numeric/[0-1][2-4].log"],
+             "asterisk"              => [true,  "backward_compatible", "test/plugin/data/log/*.log"],
+             "one character matcher" => [false, "backward_compatible", "test/plugin/data/log/tes?.log"],
+            )
+        def test_expand_paths_with_use_glob_p
+          result, option, path = data
+          config = config_element("", "", {
+                                    "tag" => "tail",
+                                    "path" => path,
+                                    "format" => "none",
+                                    "pos_file" => "#{@tmp_dir}/tail.pos",
+                                    "read_from_head" => true,
+                                    "refresh_interval" => 30,
+                                    "glob_policy" => option,
+                                    "rotate_wait" => "#{EX_ROTATE_WAIT}s",
+                                    "follow_inodes" => "#{EX_FOLLOW_INODES}",
+                                  })
+          plugin = create_driver(config, false).instance
+          assert_equal(result, !!plugin.use_glob?(path))
+        end
+      end
+    end
+
+    def ex_config_with_brackets
+      config_element("", "", {
+                     "tag" => "tail",
+                     "path" => "test/plugin/data/log_numeric/[0-1][2-4].log",
+                     "format" => "none",
+                     "pos_file" => "#{@tmp_dir}/tail.pos",
+                     "read_from_head" => true,
+                     "refresh_interval" => 30,
+                     "glob_policy" => "extended",
+                     "rotate_wait" => "#{EX_ROTATE_WAIT}s",
+                     "follow_inodes" => "#{EX_FOLLOW_INODES}",
+                   })
+    end
+
+    def test_config_with_always_with_default_delimiter
+      assert_raise(Fluent::ConfigError) do
+        config = config_element("", "", {
+                         "tag" => "tail",
+                         "path" => "test/plugin/data/log_numeric/[0-1][2-4].log",
+                         "format" => "none",
+                         "pos_file" => "#{@tmp_dir}/tail.pos",
+                         "read_from_head" => true,
+                         "refresh_interval" => 30,
+                         "glob_policy" => "always",
+                         "rotate_wait" => "#{EX_ROTATE_WAIT}s",
+                         "follow_inodes" => "#{EX_FOLLOW_INODES}",
+                       })
+
+        create_driver(config, false).instance
+      end
+    end
+
+    def test_config_with_always_with_custom_delimiter
+      assert_nothing_raised do
+        config = config_element("", "", {
+                         "tag" => "tail",
+                         "path" => "test/plugin/data/log_numeric/[0-1][2-4].log",
+                         "format" => "none",
+                         "pos_file" => "#{@tmp_dir}/tail.pos",
+                         "read_from_head" => true,
+                         "refresh_interval" => 30,
+                         "glob_policy" => "always",
+                         "path_delimiter" => "|",
+                         "rotate_wait" => "#{EX_ROTATE_WAIT}s",
+                         "follow_inodes" => "#{EX_FOLLOW_INODES}",
+                       })
+
+        create_driver(config, false).instance
+      end
+    end
+
+    def test_expand_paths_with_brackets
+      expanded_paths = [
+        create_target_info('test/plugin/data/log_numeric/01.log'),
+        create_target_info('test/plugin/data/log_numeric/02.log'),
+        create_target_info('test/plugin/data/log_numeric/12.log'),
+        create_target_info('test/plugin/data/log_numeric/14.log'),
+      ]
+
+      plugin = create_driver(ex_config_with_brackets, false).instance
+      assert_equal(expanded_paths - [expanded_paths.first], plugin.expand_paths.values.sort_by { |path_ino| path_ino.path })
+    end
+
     def test_expand_paths_with_duplicate_configuration
       expanded_paths = [
         create_target_info('test/plugin/data/log/foo/bar.log'),
@@ -3172,7 +3313,9 @@ class TailInputTest < Test::Unit::TestCase
         Fluent::FileWrapper.open("#{@tmp_dir}/tail.txt0", "ab") {|f| f.puts "file3 log2"}
       end
 
-      pos_file_inode = tail_watchers[2].pe.read_inode
+      inode_0 = tail_watchers[0]&.ino
+      inode_1 = tail_watchers[1]&.ino
+      inode_2 = tail_watchers[2]&.ino
       record_values = d.events.collect { |event| event[2]["message"] }.sort
       position_entries = []
       Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r") do |f|
@@ -3186,15 +3329,17 @@ class TailInputTest < Test::Unit::TestCase
         {
           record_values: ["file1 log1", "file1 log2", "file2 log1", "file2 log2", "file3 log1", "file3 log2"],
           tail_watcher_paths: ["#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0"],
+          tail_watcher_inodes: [inode_0, inode_1, inode_2],
           tail_watcher_io_handler_opened_statuses: [false, false, false],
           position_entries: [
             # The recorded path is old, but it is no problem. The path is not used when using follow_inodes.
-            ["#{@tmp_dir}/tail.txt0", "0000000000000016", pos_file_inode],
+            ["#{@tmp_dir}/tail.txt0", "0000000000000016", inode_2],
           ],
         },
         {
           record_values: record_values,
           tail_watcher_paths: tail_watchers.collect { |tw| tw.path },
+          tail_watcher_inodes: tail_watchers.collect { |tw| tw.ino },
           tail_watcher_io_handler_opened_statuses: tail_watchers.collect { |tw| tw.instance_variable_get(:@io_handler)&.opened? || false },
           position_entries: position_entries
         },
@@ -3249,7 +3394,9 @@ class TailInputTest < Test::Unit::TestCase
         sleep 4
       end
 
-      pos_file_inode = tail_watchers[2].pe.read_inode
+      inode_0 = tail_watchers[0]&.ino
+      inode_1 = tail_watchers[1]&.ino
+      inode_2 = tail_watchers[2]&.ino
       record_values = d.events.collect { |event| event[2]["message"] }.sort
       position_entries = []
       Fluent::FileWrapper.open("#{@tmp_dir}/tail.pos", "r") do |f|
@@ -3263,14 +3410,16 @@ class TailInputTest < Test::Unit::TestCase
         {
           record_values: ["file1 log1", "file1 log2", "file2 log1", "file2 log2", "file3 log1", "file3 log2"],
           tail_watcher_paths: ["#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0", "#{@tmp_dir}/tail.txt0"],
+          tail_watcher_inodes: [inode_0, inode_1, inode_2],
           tail_watcher_io_handler_opened_statuses: [false, false, false],
           position_entries: [
-            ["#{@tmp_dir}/tail.txt0", "0000000000000016", pos_file_inode],
+            ["#{@tmp_dir}/tail.txt0", "0000000000000016", inode_2],
           ],
         },
         {
           record_values: record_values,
           tail_watcher_paths: tail_watchers.collect { |tw| tw.path },
+          tail_watcher_inodes: tail_watchers.collect { |tw| tw.ino },
           tail_watcher_io_handler_opened_statuses: tail_watchers.collect { |tw| tw.instance_variable_get(:@io_handler)&.opened? || false },
           position_entries: position_entries
         },

data/test/plugin/test_in_tcp.rb

@@ -5,7 +5,7 @@ require 'fluent/plugin/in_tcp'
 class TcpInputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
-    @port = unused_port(protocol: :tcp)
+    @port = unused_port
   end
 
   def teardown

data/test/plugin/test_in_udp.rb

@@ -5,7 +5,7 @@ require 'fluent/plugin/in_udp'
 class UdpInputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
-    @port = unused_port(protocol: :udp)
+    @port = unused_port
   end
 
   def teardown
@@ -268,31 +268,25 @@ class UdpInputTest < Test::Unit::TestCase
 
   test 'message_length_limit' do
     message_length_limit = 32
-
-    if Fluent.windows?
-      expected_records = ["0" * 30, "4" * 30]
-    else
-      expected_records = 1.upto(3).collect do |i|
-        "#{i}" * message_length_limit
-      end
-      expected_records.prepend("0" * 30)
-      expected_records.append("4" * 30)
-    end
-
     d = create_driver(base_config + %!
       format none
       message_length_limit #{message_length_limit}
     !)
-    d.run(expect_records: expected_records.size, timeout: 5) do
+    d.run(expect_records: 3) do
       create_udp_socket('127.0.0.1', @port) do |u|
-        u.send("0" * 30 + "\n", 0)
-        1.upto(3) do |i|
+        3.times do |i|
           u.send("#{i}" * 40 + "\n", 0)
         end
-        u.send("4" * 30 + "\n", 0)
       end
     end
 
+    if Fluent.windows?
+      expected_records = []
+    else
+      expected_records = 3.times.collect do |i|
+        "#{i}" * message_length_limit
+      end
+    end
     actual_records = d.events.collect do |event|
       event[2]["message"]
     end

data/test/plugin/test_out_exec_filter.rb

@@ -500,18 +500,10 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     d = create_driver(conf)
     time = event_time('2011-01-02 13:14:15')
 
-    d.run(default_tag: 'test', expect_emits: 4) do
-      d.feed(time, {"k1" => 0})
-      d.flush
-      sleep 0.5
-      d.feed(time, {"k1" => 1})
-      d.flush
-      sleep 0.5
-      d.feed(time, {"k1" => 2})
-      d.flush
-      sleep 0.5
-      d.feed(time, {"k1" => 3})
-    end
+    d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: true,  shutdown: false){ d.feed(time, {"k1" => 0}) }
+    d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: false, shutdown: false){ d.feed(time, {"k1" => 1}) }
+    d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: false, shutdown: false){ d.feed(time, {"k1" => 2}) }
+    d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: false, shutdown: false){ d.feed(time, {"k1" => 3}) }
 
     assert_equal "2011-01-02 13:14:15\ttest\t0\n", d.formatted[0]
     assert_equal "2011-01-02 13:14:15\ttest\t1\n", d.formatted[1]
@@ -532,6 +524,9 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     assert_equal pid_list[1], events[1][2]['child_pid']
     assert_equal pid_list[0], events[2][2]['child_pid']
     assert_equal pid_list[1], events[3][2]['child_pid']
+
+  ensure
+    d.run(start: false, shutdown: true)
   end
 
   # child process exits per 3 lines

data/test/plugin/test_out_file.rb

@@ -130,7 +130,7 @@ class FileOutputTest < Test::Unit::TestCase
           'path' => "#{TMP_DIR}/${tag}/${type}/conf_test.%Y%m%d.%H%M.log",
           'add_path_suffix' => 'false',
           'append' => "true",
-          'symlink_path' => "#{TMP_DIR}/${tag}/conf_test.current.log",
+          'symlink_path' => "#{TMP_DIR}/conf_test.current.log",
           'compress' => 'gzip',
           'recompress' => 'true',
         }, [
@@ -183,26 +183,6 @@ class FileOutputTest < Test::Unit::TestCase
         Fluent::Test::Driver::Output.new(Fluent::Plugin::NullOutput).configure(conf)
       end
     end
-
-    test 'warning for symlink_path not including correct placeholders corresponding to chunk keys' do
-      omit "Windows doesn't support symlink" if Fluent.windows?
-      conf = config_element('match', '**', {
-          'path' => "#{TMP_DIR}/${tag}/${key1}/${key2}/conf_test.%Y%m%d.%H%M.log",
-          'symlink_path' => "#{TMP_DIR}/conf_test.current.log",
-        }, [
-          config_element('buffer', 'time,tag,key1,key2', {
-              '@type' => 'file',
-              'timekey' => '1d',
-              'path' => "#{TMP_DIR}/buf_conf_test",
-          }),
-      ])
-      assert_nothing_raised do
-        d = create_driver(conf)
-        assert do
-          d.logs.count { |log| log.include?("multiple chunks are competing for a single symlink_path") } == 2
-        end
-      end
-    end
   end
 
   sub_test_case 'fully configured output' do
@@ -334,7 +314,7 @@ class FileOutputTest < Test::Unit::TestCase
       assert_equal r5, d.formatted[4]
 
       read_gunzip = ->(path){
-        File.open(path, 'rb'){ |fio|
+        File.open(path){ |fio|
           Zlib::GzipReader.new(StringIO.new(fio.read)).read
         }
       }

data/test/plugin/test_out_forward.rb

@@ -12,8 +12,7 @@ class ForwardOutputTest < Test::Unit::TestCase
     FileUtils.rm_rf(TMP_DIR)
     FileUtils.mkdir_p(TMP_DIR)
     @d = nil
-    # forward plugin uses TCP and UDP sockets on the same port number
-    @target_port = unused_port(protocol: :all)
+    @target_port = unused_port
   end
 
   def teardown
@@ -611,6 +610,7 @@ EOL
 
     @d = d = create_driver(config + %[
       require_ack_response true
+      ack_response_timeout 1s
       <buffer tag>
         flush_mode immediate
         retry_type periodic
@@ -658,6 +658,7 @@ EOL
 
     @d = d = create_driver(config + %[
       require_ack_response true
+      ack_response_timeout 10s
       <buffer tag>
         flush_mode immediate
         retry_type periodic

data/test/plugin/test_out_http.rb

@@ -7,6 +7,7 @@ require 'webrick/https'
 require 'net/http'
 require 'uri'
 require 'json'
+require 'aws-sdk-core'
 
 # WEBrick's ProcHandler doesn't handle PUT by default
 module WEBrick::HTTPServlet
@@ -390,6 +391,97 @@ class HTTPOutputTest < Test::Unit::TestCase
     end
   end
 
+
+  sub_test_case 'aws sigv4 auth' do
+    setup do
+      @@fake_aws_credentials = Aws::Credentials.new(
+        'fakeaccess',
+        'fakesecret',
+        'fake session token'
+      )
+    end
+
+    def server_port
+      19883
+    end
+
+    def test_aws_sigv4_sts_role_arn
+      stub(Aws::AssumeRoleCredentials).new do |credentials_provider|
+        stub(credentials_provider).credentials {
+          @@fake_aws_credentials
+        }
+        credentials_provider
+      end
+
+      d = create_driver(config + %[
+          <auth>
+            method aws_sigv4
+            aws_service someservice
+            aws_region my-region-1
+            aws_role_arn arn:aws:iam::123456789012:role/MyRole
+          </auth>
+        ])
+      d.run(default_tag: 'test.http') do
+        test_events.each { |event|
+          d.feed(event)
+        }
+      end
+
+      result = @@result
+      assert_equal 'POST', result.method
+      assert_equal 'application/x-ndjson', result.content_type
+      assert_equal test_events, result.data
+      assert_not_empty result.headers
+      assert_not_nil result.headers['authorization']
+      assert_match /AWS4-HMAC-SHA256 Credential=[a-zA-Z0-9]*\/\d+\/my-region-1\/someservice\/aws4_request/, result.headers['authorization']
+      assert_match /SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token/, result.headers['authorization']
+      assert_equal @@fake_aws_credentials.session_token, result.headers['x-amz-security-token']
+      assert_not_nil result.headers['x-amz-content-sha256']
+      assert_not_empty result.headers['x-amz-content-sha256']
+      assert_not_nil result.headers['x-amz-security-token']
+      assert_not_empty result.headers['x-amz-security-token']
+      assert_not_nil result.headers['x-amz-date']
+      assert_not_empty result.headers['x-amz-date']
+    end
+
+    def test_aws_sigv4_no_role
+      stub(Aws::CredentialProviderChain).new do |provider_chain|
+        stub(provider_chain).resolve {
+          @@fake_aws_credentials
+        }
+        provider_chain
+      end
+      d = create_driver(config + %[
+          <auth>
+            method aws_sigv4
+            aws_service someservice
+            aws_region my-region-1
+          </auth>
+        ])
+      d.run(default_tag: 'test.http') do
+        test_events.each { |event|
+          d.feed(event)
+        }
+      end
+
+      result = @@result
+      assert_equal 'POST', result.method
+      assert_equal 'application/x-ndjson', result.content_type
+      assert_equal test_events, result.data
+      assert_not_empty result.headers
+      assert_not_nil result.headers['authorization']
+      assert_match /AWS4-HMAC-SHA256 Credential=[a-zA-Z0-9]*\/\d+\/my-region-1\/someservice\/aws4_request/, result.headers['authorization']
+      assert_match /SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date;x-amz-security-token/, result.headers['authorization']
+      assert_equal @@fake_aws_credentials.session_token, result.headers['x-amz-security-token']
+      assert_not_nil result.headers['x-amz-content-sha256']
+      assert_not_empty result.headers['x-amz-content-sha256']
+      assert_not_nil result.headers['x-amz-security-token']
+      assert_not_empty result.headers['x-amz-security-token']
+      assert_not_nil result.headers['x-amz-date']
+      assert_not_empty result.headers['x-amz-date']
+    end
+  end
+
   sub_test_case 'HTTPS' do
     def server_port
       19882
@@ -426,4 +518,40 @@ class HTTPOutputTest < Test::Unit::TestCase
       assert_not_empty result.headers
     end
   end
+
+  sub_test_case 'connection_reuse' do
+    def server_port
+      19883
+    end
+
+    def test_connection_recreation
+      d = create_driver(%[
+        endpoint http://127.0.0.1:#{server_port}/test
+        reuse_connections true
+      ])
+
+      d.run(default_tag: 'test.http', shutdown: false) do
+        d.feed(test_events[0])
+      end
+
+      data = @@result.data
+
+      # Restart server to simulate connection loss
+      @@http_server_thread.kill
+      @@http_server_thread.join
+      @@http_server_thread = Thread.new do
+        run_http_server
+      end
+
+      d.run(default_tag: 'test.http') do
+        d.feed(test_events[1])
+      end
+
+      result = @@result
+      assert_equal 'POST', result.method
+      assert_equal 'application/x-ndjson', result.content_type
+      assert_equal test_events, data.concat(result.data)
+      assert_not_empty result.headers
+    end
+  end
 end

data/test/plugin/test_out_stream.rb

@@ -54,7 +54,7 @@ class TcpOutputTest < Test::Unit::TestCase
 
   def setup
     super
-    @port = unused_port(protocol: :tcp)
+    @port = unused_port
   end
 
   def teardown

data/test/plugin/test_owned_by.rb

@@ -26,7 +26,6 @@ class OwnedByMixinTest < Test::Unit::TestCase
 
       assert_equal parent.object_id, child.owner.object_id
 
-      assert child.instance_eval{ @_plugin_id_configured }
       assert_equal 'my_parent_id', child.instance_eval{ @_plugin_id }
 
       assert_equal Fluent::Log::LEVEL_TRACE, child.log.level