fluentd 1.13.3 → 1.16.5

data/test/plugin/test_out_file.rb

@@ -5,6 +5,7 @@ require 'fileutils'
 require 'time'
 require 'timecop'
 require 'zlib'
+require 'fluent/file_wrapper'
 
 class FileOutputTest < Test::Unit::TestCase
   def setup
@@ -263,8 +264,8 @@ class FileOutputTest < Test::Unit::TestCase
       assert !(Dir.exist?("#{TMP_DIR}/my.data/a"))
       assert !(Dir.exist?("#{TMP_DIR}/your.data/a"))
       buffer_files = Dir.entries("#{TMP_DIR}/buf_full").reject{|e| e =~ /^\.+$/ }
-      assert_equal 2, buffer_files.select{|n| n.end_with?('.meta') }.size
-      assert_equal 2, buffer_files.select{|n| !n.end_with?('.meta') }.size
+      assert_equal 2, buffer_files.count{|n| n.end_with?('.meta') }
+      assert_equal 2, buffer_files.count{|n| !n.end_with?('.meta') }
 
       m1 = d.instance.metadata('my.data', t1, {"type" => "a"})
       m2 = d.instance.metadata('your.data', t3, {"type" => "a"})
@@ -394,6 +395,11 @@ class FileOutputTest < Test::Unit::TestCase
     assert_equal expect, result
   end
 
+  def check_result(path, expect)
+    result = File.read(path, mode: "rb")
+    assert_equal expect, result
+  end
+
   sub_test_case 'write' do
     test 'basic case' do
       d = create_driver
@@ -535,20 +541,27 @@ class FileOutputTest < Test::Unit::TestCase
     assert_equal 3, Dir.glob("#{TMP_DIR}/out_file_test.*").size
   end
 
-  test 'append' do
+  data(
+    "with compression" => true,
+    "without compression" => false,
+  )
+  test 'append' do |compression|
     time = event_time("2011-01-02 13:14:15 UTC")
     formatted_lines = %[2011-01-02T13:14:15Z\ttest\t{"a":1}#{@default_newline}] + %[2011-01-02T13:14:15Z\ttest\t{"a":2}#{@default_newline}]
 
     write_once = ->(){
-      d = create_driver %[
+      config = %[
         path #{TMP_DIR}/out_file_test
-        compress gz
         utc
         append true
         <buffer>
           timekey_use_utc true
         </buffer>
       ]
+      if compression
+        config << "        compress gz"
+      end
+      d = create_driver(config)
       d.run(default_tag: 'test'){
         d.feed(time, {"a"=>1})
         d.feed(time, {"a"=>2})
@@ -556,17 +569,21 @@ class FileOutputTest < Test::Unit::TestCase
       d.instance.last_written_path
     }
 
-    path = write_once.call
-    assert_equal "#{TMP_DIR}/out_file_test.20110102.log.gz", path
-    check_gzipped_result(path, formatted_lines)
-
-    path = write_once.call
-    assert_equal "#{TMP_DIR}/out_file_test.20110102.log.gz", path
-    check_gzipped_result(path, formatted_lines * 2)
+    log_file_name = "out_file_test.20110102.log"
+    if compression
+      log_file_name << ".gz"
+    end
 
-    path = write_once.call
-    assert_equal "#{TMP_DIR}/out_file_test.20110102.log.gz", path
-    check_gzipped_result(path, formatted_lines * 3)
+    1.upto(3) do |i|
+      path = write_once.call
+      assert_equal "#{TMP_DIR}/#{log_file_name}", path
+      expect = formatted_lines * i
+      if compression
+        check_gzipped_result(path, expect)
+      else
+        check_result(path, expect)
+      end
+    end
   end
 
   test 'append when JST' do
@@ -1000,7 +1017,7 @@ class FileOutputTest < Test::Unit::TestCase
 
     test 'returns filepath with index which does not exist yet' do
       5.times do |i|
-        File.open(File.join(@tmp, "exist_#{i}.log"), 'a'){|f| } # open(create) and close
+        Fluent::FileWrapper.open(File.join(@tmp, "exist_#{i}.log"), 'a'){|f| } # open(create) and close
       end
       @i.find_filepath_available(File.join(@tmp, "exist_**.log")) do |path|
         assert_equal File.join(@tmp, "exist_5.log"), path
@@ -1009,7 +1026,7 @@ class FileOutputTest < Test::Unit::TestCase
 
     test 'creates lock directory when with_lock is true to exclude operations of other worker process' do
       5.times do |i|
-        File.open(File.join(@tmp, "exist_#{i}.log"), 'a')
+        Fluent::FileWrapper.open(File.join(@tmp, "exist_#{i}.log"), 'a')
       end
       Dir.mkdir(File.join(@tmp, "exist_5.log.lock"))
       @i.find_filepath_available(File.join(@tmp, "exist_**.log"), with_lock: true) do |path|

data/test/plugin/test_out_forward.rb

@@ -156,7 +156,14 @@ EOL
     normal_conf = config_element('match', '**', {}, [
         config_element('server', '', {'name' => 'test', 'host' => 'unexisting.yaaaaaaaaaaaaaay.host.example.com'})
       ])
-    assert_raise SocketError do
+
+    if Socket.const_defined?(:ResolutionError) # as of Ruby 3.3
+      error_class = Socket::ResolutionError
+    else
+      error_class = SocketError
+    end
+
+    assert_raise error_class do
       create_driver(normal_conf)
     end
 
@@ -165,7 +172,7 @@ EOL
       ])
     @d = d = create_driver(conf)
     expected_log = "failed to resolve node name when configured"
-    expected_detail = 'server="test" error_class=SocketError'
+    expected_detail = "server=\"test\" error_class=#{error_class.name}"
     logs = d.logs
     assert{ logs.any?{|log| log.include?(expected_log) && log.include?(expected_detail) } }
   end
@@ -431,10 +438,12 @@ EOL
     ]
 
     stub(d.instance.ack_handler).read_ack_from_sock(anything).never
-    target_input_driver.run(expect_records: 2) do
-      d.run do
-        emit_events.each do |tag, t, record|
-          d.feed(tag, t, record)
+    assert_rr do
+      target_input_driver.run(expect_records: 2) do
+        d.run do
+          emit_events.each do |tag, t, record|
+            d.feed(tag, t, record)
+          end
         end
       end
     end
@@ -461,10 +470,12 @@ EOL
     ]
 
     stub(d.instance.ack_handler).read_ack_from_sock(anything).never
-    target_input_driver.run(expect_records: 2) do
-      d.run(default_tag: 'test') do
-        records.each do |record|
-          d.feed(time, record)
+    assert_rr do
+      target_input_driver.run(expect_records: 2) do
+        d.run(default_tag: 'test') do
+          records.each do |record|
+            d.feed(time, record)
+          end
         end
       end
     end
@@ -491,10 +502,12 @@ EOL
       {"a" => 2}
     ]
     stub(d.instance.ack_handler).read_ack_from_sock(anything).never
-    target_input_driver.run(expect_records: 2) do
-      d.run(default_tag: 'test') do
-        records.each do |record|
-          d.feed(time, record)
+    assert_rr do
+      target_input_driver.run(expect_records: 2) do
+        d.run(default_tag: 'test') do
+          records.each do |record|
+            d.feed(time, record)
+          end
         end
       end
     end
@@ -549,10 +562,12 @@ EOL
     ]
     # not attempt to receive responses
     stub(d.instance.ack_handler).read_ack_from_sock(anything).never
-    target_input_driver.run(expect_records: 2) do
-      d.run(default_tag: 'test') do
-        records.each do |record|
-          d.feed(time, record)
+    assert_rr do
+      target_input_driver.run(expect_records: 2) do
+        d.run(default_tag: 'test') do
+          records.each do |record|
+            d.feed(time, record)
+          end
         end
       end
     end
@@ -575,10 +590,12 @@ EOL
     ]
     # not attempt to receive responses
     stub(d.instance.ack_handler).read_ack_from_sock(anything).never
-    target_input_driver.run(expect_records: 2) do
-      d.run(default_tag: 'test') do
-        records.each do |record|
-          d.feed(time, record)
+    assert_rr do
+      target_input_driver.run(expect_records: 2) do
+        d.run(default_tag: 'test') do
+          records.each do |record|
+            d.feed(time, record)
+          end
         end
       end
     end
@@ -1231,27 +1248,22 @@ EOL
     target_input_driver = create_target_input_driver(conf: target_config)
     output_conf = config
     d = create_driver(output_conf)
-    d.instance_start
 
-    begin
-      chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
-      mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,
-                                               linger_timeout: anything,
-                                               send_timeout: anything,
-                                               recv_timeout: anything,
-                                               connect_timeout: anything
-                                              ) { |sock| mock(sock).close.once; sock }.twice
+    chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
+    mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,
+                                             linger_timeout: anything,
+                                             send_timeout: anything,
+                                             recv_timeout: anything,
+                                             connect_timeout: anything
+                                            ) { |sock| mock(sock).close.once; sock }.twice
 
-      target_input_driver.run(timeout: 15) do
-        d.run(shutdown: false) do
-          node = d.instance.nodes.first
-          2.times do
-            node.send_data('test', chunk) rescue nil
-          end
+    target_input_driver.run(timeout: 15) do
+      d.run do
+        node = d.instance.nodes.first
+        2.times do
+          node.send_data('test', chunk) rescue nil
         end
       end
-    ensure
-      d.instance_shutdown
     end
   end
 
@@ -1265,7 +1277,6 @@ EOL
       port #{@target_port}
     </server>
     ])
-    d.instance_start
     assert_nothing_raised { d.run }
   end
 
@@ -1277,33 +1288,28 @@ EOL
         keepalive_timeout 2
       ]
       d = create_driver(output_conf)
-      d.instance_start
 
-      begin
-        chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
-        mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,
-                                                 linger_timeout: anything,
-                                                 send_timeout: anything,
-                                                 recv_timeout: anything,
-                                                 connect_timeout: anything
-                                                ) { |sock| mock(sock).close.once; sock }.once
+      chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
+      mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,
+                                               linger_timeout: anything,
+                                               send_timeout: anything,
+                                               recv_timeout: anything,
+                                               connect_timeout: anything
+                                              ) { |sock| mock(sock).close.once; sock }.once
 
-        target_input_driver.run(timeout: 15) do
-          d.run(shutdown: false) do
-            node = d.instance.nodes.first
-            2.times do
-              node.send_data('test', chunk) rescue nil
-            end
+      target_input_driver.run(timeout: 15) do
+        d.run do
+          node = d.instance.nodes.first
+          2.times do
+            node.send_data('test', chunk) rescue nil
           end
         end
-      ensure
-        d.instance_shutdown
       end
     end
 
     test 'create timer of purging obsolete sockets' do
       output_conf = config + %[keepalive true]
-      d = create_driver(output_conf)
+      @d = d = create_driver(output_conf)
 
       mock(d.instance).timer_execute(:out_forward_heartbeat_request, 1).once
       mock(d.instance).timer_execute(:out_forward_keep_alived_socket_watcher, 5).once
@@ -1319,28 +1325,23 @@ EOL
           keepalive_timeout 2
         ]
         d = create_driver(output_conf)
-        d.instance_start
 
-        begin
-          chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
-          mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,
-                                                   linger_timeout: anything,
-                                                   send_timeout: anything,
-                                                   recv_timeout: anything,
-                                                   connect_timeout: anything) { |sock|
-            mock(sock).close.once; sock
-          }.twice
-
-          target_input_driver.run(timeout: 15) do
-            d.run(shutdown: false) do
-              node = d.instance.nodes.first
-              2.times do
-                node.send_data('test', chunk) rescue nil
-              end
+        chunk = Fluent::Plugin::Buffer::MemoryChunk.new(Fluent::Plugin::Buffer::Metadata.new(nil, nil, nil))
+        mock.proxy(d.instance).socket_create_tcp(TARGET_HOST, @target_port,
+                                                 linger_timeout: anything,
+                                                 send_timeout: anything,
+                                                 recv_timeout: anything,
+                                                 connect_timeout: anything) { |sock|
+          mock(sock).close.once; sock
+        }.twice
+
+        target_input_driver.run(timeout: 15) do
+          d.run do
+            node = d.instance.nodes.first
+            2.times do
+              node.send_data('test', chunk) rescue nil
             end
           end
-        ensure
-          d.instance_shutdown
         end
       end
     end

data/test/plugin/test_out_http.rb

@@ -378,6 +378,7 @@ class HTTPOutputTest < Test::Unit::TestCase
           password hello?
         </auth>
       ])
+      d.instance.system_config_override(root_dir: TMP_DIR) # Backup files are generated in TMP_DIR.
       d.run(default_tag: 'test.http', shutdown: false) do
         test_events.each { |event|
           d.feed(event)

data/test/plugin/test_out_stdout.rb

@@ -32,7 +32,7 @@ class StdoutOutputTest < Test::Unit::TestCase
       assert_kind_of Fluent::Plugin::StdoutFormatter, d.instance.formatter
       assert_equal 'hash', d.instance.formatter.output_type
 
-      assert_raise(Fluent::ConfigError) do
+      assert_raise(Fluent::NotFoundPluginError) do
         d = create_driver(CONFIG + "\noutput_type foo")
       end
     end
@@ -126,7 +126,7 @@ class StdoutOutputTest < Test::Unit::TestCase
       assert_kind_of Fluent::Plugin::StdoutFormatter, d.instance.formatter
       assert_equal 'hash', d.instance.formatter.output_type
 
-      assert_raise(Fluent::ConfigError) do
+      assert_raise(Fluent::NotFoundPluginError) do
         create_driver(config_element("ROOT", "", {"output_type" => "foo"}, [config_element("buffer")]))
       end
     end