fluentd 1.16.0 → 1.16.7

data/test/plugin/test_in_tcp.rb

@@ -5,7 +5,7 @@ require 'fluent/plugin/in_tcp'
 class TcpInputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
-    @port = unused_port
+    @port = unused_port(protocol: :tcp)
   end
 
   def teardown
@@ -218,16 +218,14 @@ class TcpInputTest < Test::Unit::TestCase
           </client>
         </security>
       !)
-      d.run(shutdown: false, expect_records: 1, timeout: 2) do
+      d.run(expect_records: 1, timeout: 2) do
         create_tcp_socket('127.0.0.1', @port) do |sock|
           sock.send("hello\n", 0)
         end
       end
 
-      assert_equal 1, d.instance.log.logs.count { |l| l =~ /anonymous client/ }
+      assert_equal 1, d.logs.count { |l| l =~ /anonymous client/ }
       assert_equal 0, d.events.size
-    ensure
-      d.instance_shutdown if d&.instance
     end
   end
 
@@ -255,4 +253,76 @@ class TcpInputTest < Test::Unit::TestCase
       assert_equal 'hello', event[2]['msg']
     end
   end
+
+  sub_test_case "message_length_limit" do
+    data("batch_emit", { extract: "" }, keep: true)
+    data("single_emit", { extract: "<extract>\ntag_key tag\n</extract>\n" }, keep: true)
+    test "drop records exceeding limit" do |data|
+      message_length_limit = 10
+      d = create_driver(base_config + %!
+        message_length_limit #{message_length_limit}
+        <parse>
+          @type none
+        </parse>
+        #{data[:extract]}
+      !)
+      d.run(expect_records: 2, timeout: 10) do
+        create_tcp_socket('127.0.0.1', @port) do |sock|
+          sock.send("a" * message_length_limit + "\n", 0)
+          sock.send("b" * (message_length_limit + 1) + "\n", 0)
+          sock.send("c" * (message_length_limit - 1) + "\n", 0)
+        end
+      end
+
+      expected_records = [
+        "a" * message_length_limit,
+        "c" * (message_length_limit - 1)
+      ]
+      actual_records = d.events.collect do |event|
+        event[2]["message"]
+      end
+
+      assert_equal expected_records, actual_records
+    end
+
+    test "clear buffer and discard the subsequent data until the next delimiter" do |data|
+      message_length_limit = 12
+      d = create_driver(base_config + %!
+        message_length_limit #{message_length_limit}
+        delimiter ";"
+        <parse>
+          @type json
+        </parse>
+        #{data[:extract]}
+      !)
+      d.run(expect_records: 1, timeout: 10) do
+        create_tcp_socket('127.0.0.1', @port) do |sock|
+          sock.send('{"message":', 0)
+          sock.send('"hello', 0)
+          sleep 1 # To make the server read data and clear the buffer here.
+          sock.send('world!"};', 0) # This subsequent data must be discarded so that a parsing failure doesn't occur.
+          sock.send('{"k":"v"};', 0) # This will succeed to parse.
+        end
+      end
+
+      logs = d.logs.collect do |log|
+        log.gsub(/\A\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [-+]\d{4} /, "")
+      end
+      actual_records = d.events.collect do |event|
+        event[2]
+      end
+
+      assert_equal(
+        {
+          # Asserting that '[warn]: pattern not matched message="world!\"}"' warning does not occur.
+          logs: ['[info]: The buffer size exceeds \'message_length_limit\', cleared: limit=12 size=17 head="{\"message\":\"hello"' + "\n"],
+          records: [{"k" => "v"}],
+        },
+        {
+          logs: logs[1..],
+          records: actual_records,
+        }
+      )
+    end
+  end
 end

data/test/plugin/test_in_udp.rb

@@ -5,7 +5,7 @@ require 'fluent/plugin/in_udp'
 class UdpInputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
-    @port = unused_port
+    @port = unused_port(protocol: :udp)
   end
 
   def teardown
@@ -265,4 +265,38 @@ class UdpInputTest < Test::Unit::TestCase
       end
     end
   end
+
+  test 'message_length_limit' do
+    message_length_limit = 32
+
+    if Fluent.windows?
+      expected_records = ["0" * 30, "4" * 30]
+    else
+      expected_records = 1.upto(3).collect do |i|
+        "#{i}" * message_length_limit
+      end
+      expected_records.prepend("0" * 30)
+      expected_records.append("4" * 30)
+    end
+
+    d = create_driver(base_config + %!
+      format none
+      message_length_limit #{message_length_limit}
+    !)
+    d.run(expect_records: expected_records.size, timeout: 5) do
+      create_udp_socket('127.0.0.1', @port) do |u|
+        u.send("0" * 30 + "\n", 0)
+        1.upto(3) do |i|
+          u.send("#{i}" * 40 + "\n", 0)
+        end
+        u.send("4" * 30 + "\n", 0)
+      end
+    end
+
+    actual_records = d.events.collect do |event|
+      event[2]["message"]
+    end
+
+    assert_equal expected_records, actual_records
+  end
 end

data/test/plugin/test_in_unix.rb

@@ -174,8 +174,8 @@ class UnixInputTest < Test::Unit::TestCase
     assert_equal 0, @d.events.size
 
     logs = @d.instance.log.logs
-    assert_equal 1, logs.select { |line|
+    assert_equal 1, logs.count { |line|
       line =~ / \[warn\]: incoming data is broken: msg=#{data.inspect}/
-    }.size, "should not accept broken chunk"
+    }, "should not accept broken chunk"
   end
 end unless Fluent.windows?

data/test/plugin/test_multi_output.rb

@@ -150,7 +150,7 @@ class MultiOutputTest < Test::Unit::TestCase
       log_size_for_metrics_plugin_helper = 4
       expected_warn_log_size = log_size_for_multi_output_itself + log_size_for_metrics_plugin_helper
       logs = @i.log.out.logs
-      assert{ logs.select{|log| log.include?('[warn]') && log.include?("'type' is deprecated parameter name. use '@type' instead.") }.size == expected_warn_log_size }
+      assert{ logs.count{|log| log.include?('[warn]') && log.include?("'type' is deprecated parameter name. use '@type' instead.") } == expected_warn_log_size }
     end
 
     test '#emit_events calls #process always' do

data/test/plugin/test_out_exec_filter.rb

@@ -500,10 +500,18 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     d = create_driver(conf)
     time = event_time('2011-01-02 13:14:15')
 
-    d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: true,  shutdown: false){ d.feed(time, {"k1" => 0}) }
-    d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: false, shutdown: false){ d.feed(time, {"k1" => 1}) }
-    d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: false, shutdown: false){ d.feed(time, {"k1" => 2}) }
-    d.run(default_tag: 'test', expect_emits: 1, timeout: 10, start: false, shutdown: false){ d.feed(time, {"k1" => 3}) }
+    d.run(default_tag: 'test', expect_emits: 4) do
+      d.feed(time, {"k1" => 0})
+      d.flush
+      sleep 0.5
+      d.feed(time, {"k1" => 1})
+      d.flush
+      sleep 0.5
+      d.feed(time, {"k1" => 2})
+      d.flush
+      sleep 0.5
+      d.feed(time, {"k1" => 3})
+    end
 
     assert_equal "2011-01-02 13:14:15\ttest\t0\n", d.formatted[0]
     assert_equal "2011-01-02 13:14:15\ttest\t1\n", d.formatted[1]
@@ -524,9 +532,6 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     assert_equal pid_list[1], events[1][2]['child_pid']
     assert_equal pid_list[0], events[2][2]['child_pid']
     assert_equal pid_list[1], events[3][2]['child_pid']
-
-  ensure
-    d.run(start: false, shutdown: true)
   end
 
   # child process exits per 3 lines
@@ -597,8 +602,8 @@ class ExecFilterOutputTest < Test::Unit::TestCase
     # the number of pids should be same with number of child processes
     assert_equal 2, pid_list.size
     logs = d.instance.log.out.logs
-    assert_equal 2, logs.select { |l| l.include?('child process exits with error code') }.size
-    assert_equal 2, logs.select { |l| l.include?('respawning child process') }.size
+    assert_equal 2, logs.count { |l| l.include?('child process exits with error code') }
+    assert_equal 2, logs.count { |l| l.include?('respawning child process') }
 
   ensure
     d.run(start: false, shutdown: true)

data/test/plugin/test_out_file.rb

@@ -130,7 +130,7 @@ class FileOutputTest < Test::Unit::TestCase
           'path' => "#{TMP_DIR}/${tag}/${type}/conf_test.%Y%m%d.%H%M.log",
           'add_path_suffix' => 'false',
           'append' => "true",
-          'symlink_path' => "#{TMP_DIR}/conf_test.current.log",
+          'symlink_path' => "#{TMP_DIR}/${tag}/conf_test.current.log",
           'compress' => 'gzip',
           'recompress' => 'true',
         }, [
@@ -183,6 +183,26 @@ class FileOutputTest < Test::Unit::TestCase
         Fluent::Test::Driver::Output.new(Fluent::Plugin::NullOutput).configure(conf)
       end
     end
+
+    test 'warning for symlink_path not including correct placeholders corresponding to chunk keys' do
+      omit "Windows doesn't support symlink" if Fluent.windows?
+      conf = config_element('match', '**', {
+          'path' => "#{TMP_DIR}/${tag}/${key1}/${key2}/conf_test.%Y%m%d.%H%M.log",
+          'symlink_path' => "#{TMP_DIR}/conf_test.current.log",
+        }, [
+          config_element('buffer', 'time,tag,key1,key2', {
+              '@type' => 'file',
+              'timekey' => '1d',
+              'path' => "#{TMP_DIR}/buf_conf_test",
+          }),
+      ])
+      assert_nothing_raised do
+        d = create_driver(conf)
+        assert do
+          d.logs.count { |log| log.include?("multiple chunks are competing for a single symlink_path") } == 2
+        end
+      end
+    end
   end
 
   sub_test_case 'fully configured output' do
@@ -264,8 +284,8 @@ class FileOutputTest < Test::Unit::TestCase
       assert !(Dir.exist?("#{TMP_DIR}/my.data/a"))
       assert !(Dir.exist?("#{TMP_DIR}/your.data/a"))
       buffer_files = Dir.entries("#{TMP_DIR}/buf_full").reject{|e| e =~ /^\.+$/ }
-      assert_equal 2, buffer_files.select{|n| n.end_with?('.meta') }.size
-      assert_equal 2, buffer_files.select{|n| !n.end_with?('.meta') }.size
+      assert_equal 2, buffer_files.count{|n| n.end_with?('.meta') }
+      assert_equal 2, buffer_files.count{|n| !n.end_with?('.meta') }
 
       m1 = d.instance.metadata('my.data', t1, {"type" => "a"})
       m2 = d.instance.metadata('your.data', t3, {"type" => "a"})
@@ -314,7 +334,7 @@ class FileOutputTest < Test::Unit::TestCase
       assert_equal r5, d.formatted[4]
 
       read_gunzip = ->(path){
-        File.open(path){ |fio|
+        File.open(path, 'rb'){ |fio|
           Zlib::GzipReader.new(StringIO.new(fio.read)).read
         }
       }

data/test/plugin/test_out_forward.rb

@@ -12,7 +12,8 @@ class ForwardOutputTest < Test::Unit::TestCase
     FileUtils.rm_rf(TMP_DIR)
     FileUtils.mkdir_p(TMP_DIR)
     @d = nil
-    @target_port = unused_port
+    # forward plugin uses TCP and UDP sockets on the same port number
+    @target_port = unused_port(protocol: :all)
   end
 
   def teardown
@@ -156,7 +157,14 @@ EOL
     normal_conf = config_element('match', '**', {}, [
         config_element('server', '', {'name' => 'test', 'host' => 'unexisting.yaaaaaaaaaaaaaay.host.example.com'})
       ])
-    assert_raise SocketError do
+
+    if Socket.const_defined?(:ResolutionError) # as of Ruby 3.3
+      error_class = Socket::ResolutionError
+    else
+      error_class = SocketError
+    end
+
+    assert_raise error_class do
       create_driver(normal_conf)
     end
 
@@ -165,7 +173,7 @@ EOL
       ])
     @d = d = create_driver(conf)
     expected_log = "failed to resolve node name when configured"
-    expected_detail = 'server="test" error_class=SocketError'
+    expected_detail = "server=\"test\" error_class=#{error_class.name}"
     logs = d.logs
     assert{ logs.any?{|log| log.include?(expected_log) && log.include?(expected_detail) } }
   end
@@ -603,7 +611,6 @@ EOL
 
     @d = d = create_driver(config + %[
       require_ack_response true
-      ack_response_timeout 1s
       <buffer tag>
         flush_mode immediate
         retry_type periodic
@@ -651,7 +658,6 @@ EOL
 
     @d = d = create_driver(config + %[
       require_ack_response true
-      ack_response_timeout 10s
       <buffer tag>
         flush_mode immediate
         retry_type periodic
@@ -1241,27 +1247,22 @@ EOL
     target_input_driver = create_target_input_driver(conf: target_config)
     output_conf = config
     d = create_driver(output_conf)
-    d.instance_start
 
-    begin
-      chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
-      mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,
-                                               linger_timeout: anything,
-                                               send_timeout: anything,
-                                               recv_timeout: anything,
-                                               connect_timeout: anything
-                                              ) { |sock| mock(sock).close.once; sock }.twice
+    chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
+    mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,
+                                             linger_timeout: anything,
+                                             send_timeout: anything,
+                                             recv_timeout: anything,
+                                             connect_timeout: anything
+                                            ) { |sock| mock(sock).close.once; sock }.twice
 
-      target_input_driver.run(timeout: 15) do
-        d.run(shutdown: false) do
-          node = d.instance.nodes.first
-          2.times do
-            node.send_data('test', chunk) rescue nil
-          end
+    target_input_driver.run(timeout: 15) do
+      d.run do
+        node = d.instance.nodes.first
+        2.times do
+          node.send_data('test', chunk) rescue nil
         end
       end
-    ensure
-      d.instance_shutdown
     end
   end
 
@@ -1275,7 +1276,6 @@ EOL
       port #{@target_port}
     </server>
     ])
-    d.instance_start
     assert_nothing_raised { d.run }
   end
 
@@ -1287,33 +1287,28 @@ EOL
         keepalive_timeout 2
       ]
       d = create_driver(output_conf)
-      d.instance_start
 
-      begin
-        chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
-        mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,
-                                                 linger_timeout: anything,
-                                                 send_timeout: anything,
-                                                 recv_timeout: anything,
-                                                 connect_timeout: anything
-                                                ) { |sock| mock(sock).close.once; sock }.once
+      chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
+      mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,
+                                               linger_timeout: anything,
+                                               send_timeout: anything,
+                                               recv_timeout: anything,
+                                               connect_timeout: anything
+                                              ) { |sock| mock(sock).close.once; sock }.once
 
-        target_input_driver.run(timeout: 15) do
-          d.run(shutdown: false) do
-            node = d.instance.nodes.first
-            2.times do
-              node.send_data('test', chunk) rescue nil
-            end
+      target_input_driver.run(timeout: 15) do
+        d.run do
+          node = d.instance.nodes.first
+          2.times do
+            node.send_data('test', chunk) rescue nil
           end
         end
-      ensure
-        d.instance_shutdown
       end
     end
 
     test 'create timer of purging obsolete sockets' do
       output_conf = config + %[keepalive true]
-      d = create_driver(output_conf)
+      @d = d = create_driver(output_conf)
 
       mock(d.instance).timer_execute(:out_forward_heartbeat_request, 1).once
       mock(d.instance).timer_execute(:out_forward_keep_alived_socket_watcher, 5).once
@@ -1329,7 +1324,6 @@ EOL
           keepalive_timeout 2
         ]
         d = create_driver(output_conf)
-        d.instance_start
 
         chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
         mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,

data/test/plugin/test_out_stream.rb

@@ -54,7 +54,7 @@ class TcpOutputTest < Test::Unit::TestCase
 
   def setup
     super
-    @port = unused_port
+    @port = unused_port(protocol: :tcp)
   end
 
   def teardown

data/test/plugin/test_output.rb

@@ -447,25 +447,25 @@ class OutputTest < Test::Unit::TestCase
         @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
         validators = @i.placeholder_validators(:path, "/my/path/${tag}/${username}/file.%Y%m%d_%H%M.log")
         assert_equal 3, validators.size
-        assert_equal 1, validators.select(&:time?).size
-        assert_equal 1, validators.select(&:tag?).size
-        assert_equal 1, validators.select(&:keys?).size
+        assert_equal 1, validators.count(&:time?)
+        assert_equal 1, validators.count(&:tag?)
+        assert_equal 1, validators.count(&:keys?)
       end
 
       test 'returns validators for time, tag and keys when a plugin is configured with these keys even if a template does not have placeholders' do
         @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time,tag,username', {'timekey' => 60})]))
         validators = @i.placeholder_validators(:path, "/my/path/file.log")
         assert_equal 3, validators.size
-        assert_equal 1, validators.select(&:time?).size
-        assert_equal 1, validators.select(&:tag?).size
-        assert_equal 1, validators.select(&:keys?).size
+        assert_equal 1, validators.count(&:time?)
+        assert_equal 1, validators.count(&:tag?)
+        assert_equal 1, validators.count(&:keys?)
       end
 
       test 'returns a validator for time if a template has timestamp placeholders' do
         @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
         validators = @i.placeholder_validators(:path, "/my/path/file.%Y-%m-%d.log")
         assert_equal 1, validators.size
-        assert_equal 1, validators.select(&:time?).size
+        assert_equal 1, validators.count(&:time?)
         assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.%Y-%m-%d.log' has timestamp placeholders, but chunk key 'time' is not configured") do
           validators.first.validate!
         end
@@ -475,7 +475,7 @@ class OutputTest < Test::Unit::TestCase
         @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'time', {'timekey' => '30'})]))
         validators = @i.placeholder_validators(:path, "/my/path/to/file.log")
         assert_equal 1, validators.size
-        assert_equal 1, validators.select(&:time?).size
+        assert_equal 1, validators.count(&:time?)
         assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/to/file.log' doesn't have timestamp placeholders for timekey 30") do
           validators.first.validate!
         end
@@ -485,7 +485,7 @@ class OutputTest < Test::Unit::TestCase
         @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
         validators = @i.placeholder_validators(:path, "/my/path/${tag}/file.log")
         assert_equal 1, validators.size
-        assert_equal 1, validators.select(&:tag?).size
+        assert_equal 1, validators.count(&:tag?)
         assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/${tag}/file.log' has tag placeholders, but chunk key 'tag' is not configured") do
           validators.first.validate!
         end
@@ -495,7 +495,7 @@ class OutputTest < Test::Unit::TestCase
         @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'tag')]))
         validators = @i.placeholder_validators(:path, "/my/path/file.log")
         assert_equal 1, validators.size
-        assert_equal 1, validators.select(&:tag?).size
+        assert_equal 1, validators.count(&:tag?)
         assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.log' doesn't have tag placeholder") do
           validators.first.validate!
         end
@@ -505,7 +505,7 @@ class OutputTest < Test::Unit::TestCase
         @i.configure(config_element('ROOT', '', {}, [config_element('buffer', '')]))
         validators = @i.placeholder_validators(:path, "/my/path/${username}/file.${group}.log")
         assert_equal 1, validators.size
-        assert_equal 1, validators.select(&:keys?).size
+        assert_equal 1, validators.count(&:keys?)
         assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/${username}/file.${group}.log' has placeholders, but chunk keys doesn't have keys group,username") do
           validators.first.validate!
         end
@@ -515,7 +515,7 @@ class OutputTest < Test::Unit::TestCase
         @i.configure(config_element('ROOT', '', {}, [config_element('buffer', 'username,group')]))
         validators = @i.placeholder_validators(:path, "/my/path/file.log")
         assert_equal 1, validators.size
-        assert_equal 1, validators.select(&:keys?).size
+        assert_equal 1, validators.count(&:keys?)
         assert_raise Fluent::ConfigError.new("Parameter 'path: /my/path/file.log' doesn't have enough placeholders for keys group,username") do
           validators.first.validate!
         end