fluentd 1.12.1-x64-mingw32 → 1.13.1-x64-mingw32

data/test/plugin/test_in_tail.rb

@@ -6,6 +6,8 @@ require 'fluent/system_config'
 require 'net/http'
 require 'flexmock/test_unit'
 require 'timecop'
+require 'tmpdir'
+require 'securerandom'
 
 class TailInputTest < Test::Unit::TestCase
   include FlexMock::TestCase
@@ -22,28 +24,64 @@ class TailInputTest < Test::Unit::TestCase
   end
 
   def cleanup_directory(path)
-    begin
-      FileUtils.rm_f(path, secure: true)
-    rescue ArgumentError
-      FileUtils.rm_f(path) # For Ruby 2.6 or before.
+    unless Dir.exist?(path)
+      FileUtils.mkdir_p(path)
+      return
     end
-    if File.exist?(path)
-      FileUtils.remove_entry_secure(path, true)
+
+    if Fluent.windows?
+      Dir.glob("*", base: path).each do |name|
+        begin
+          cleanup_file(File.join(path, name))
+        rescue
+          # expect test driver block release already owned file handle.
+        end
+      end
+    else
+      begin
+        FileUtils.rm_f(path, secure:true)
+      rescue ArgumentError
+        FileUtils.rm_f(path) # For Ruby 2.6 or before.
+      end
+      if File.exist?(path)
+        FileUtils.remove_entry_secure(path, true)
+      end
     end
     FileUtils.mkdir_p(path)
   end
 
   def cleanup_file(path)
-    begin
-      FileUtils.rm_f(path, secure: true)
-    rescue ArgumentError
-      FileUtils.rm_f(path) # For Ruby 2.6 or before.
-    end
-    if File.exist?(path)
-      # ensure files are closed for Windows, on which deleted files
-      # are still visible from filesystem
-      GC.start(full_mark: true, immediate_mark: true, immediate_sweep: true)
-      FileUtils.remove_entry_secure(path, true)
+    if Fluent.windows?
+      # On Windows, when the file or directory is removed and created
+      # frequently, there is a case that creating file or directory will
+      # fail. This situation is caused by pending file or directory
+      # deletion which is mentioned on win32 API document [1]
+      # As a workaround, execute rename and remove method.
+      #
+      # [1] https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-createfilea#files
+      #
+      file = File.join(Dir.tmpdir, SecureRandom.hex(10))
+      begin
+        FileUtils.mv(path, file)
+        FileUtils.rm_rf(file, secure: true)
+      rescue ArgumentError
+        FileUtils.rm_rf(file) # For Ruby 2.6 or before.
+      end
+      if File.exist?(file)
+        # ensure files are closed for Windows, on which deleted files
+        # are still visible from filesystem
+        GC.start(full_mark: true, immediate_mark: true, immediate_sweep: true)
+        FileUtils.remove_entry_secure(file, true)
+      end
+    else
+      begin
+        FileUtils.rm_f(path, secure: true)
+      rescue ArgumentError
+        FileUtils.rm_f(path) # For Ruby 2.6 or before.
+      end
+      if File.exist?(path)
+        FileUtils.remove_entry_secure(path, true)
+      end
     end
   end
 
@@ -118,6 +156,7 @@ class TailInputTest < Test::Unit::TestCase
       assert_equal 2, d.instance.rotate_wait
       assert_equal "#{TMP_DIR}/tail.pos", d.instance.pos_file
       assert_equal 1000, d.instance.read_lines_limit
+      assert_equal -1, d.instance.read_bytes_limit_per_second
       assert_equal false, d.instance.ignore_repeated_permission_error
       assert_nothing_raised do
         d.instance.have_read_capability?
@@ -157,6 +196,22 @@ class TailInputTest < Test::Unit::TestCase
       end
     end
 
+    sub_test_case "log throttling per file" do
+      test "w/o watcher timer is invalid" do
+        conf = CONFIG_ENABLE_WATCH_TIMER + config_element("ROOT", "", {"read_bytes_limit_per_second" => "8k"})
+        assert_raise(Fluent::ConfigError) do
+          create_driver(conf)
+        end
+      end
+
+      test "valid" do
+        conf = config_element("ROOT", "", {"read_bytes_limit_per_second" => "8k"})
+        assert_raise(Fluent::ConfigError) do
+          create_driver(conf)
+        end
+      end
+    end
+
     test "both enable_watch_timer and enable_stat_watcher are false" do
       assert_raise(Fluent::ConfigError) do
         create_driver(CONFIG_ENABLE_WATCH_TIMER + CONFIG_DISABLE_STAT_WATCHER + PARSE_SINGLE_LINE_CONFIG)
@@ -285,6 +340,124 @@ class TailInputTest < Test::Unit::TestCase
       assert num_events <= d.emit_count
     end
 
+    sub_test_case "log throttling per file" do
+      teardown do
+        cleanup_file("#{TMP_DIR}/tail.txt")
+      end
+
+      sub_test_case "reads_bytes_per_second w/o throttled" do
+        data("flat 8192 bytes, 2 events"        => [:flat, 100, 8192, 2],
+             "flat 8192 bytes, 2 events w/o stat watcher" => [:flat_without_stat, 100, 8192, 2],
+             "flat #{8192*10} bytes, 20 events"  => [:flat, 100, (8192 * 10), 20],
+             "flat #{8192*10} bytes, 20 events w/o stat watcher"  => [:flat_without_stat, 100, (8192 * 10), 20],
+             "parse #{8192*4} bytes, 8 events"  => [:parse, 100, (8192 * 4), 8],
+             "parse #{8192*4} bytes, 8 events w/o stat watcher"  => [:parse_without_stat, 100, (8192 * 4), 8],
+             "parse #{8192*10} bytes, 20 events" => [:parse, 100, (8192 * 10), 20],
+             "parse #{8192*10} bytes, 20 events w/o stat watcher" => [:parse_without_stat, 100, (8192 * 10), 20],
+             "flat 8k bytes with unit, 2 events" => [:flat, 100, "8k", 2])
+        def test_emit_with_read_bytes_limit_per_second(data)
+          config_style, limit, limit_bytes, num_events = data
+          case config_style
+          when :flat
+            config = CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes })
+          when :parse
+            config = CONFIG_READ_FROM_HEAD + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes }) + PARSE_SINGLE_LINE_CONFIG
+          when :flat_without_stat
+            config = CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + CONFIG_DISABLE_STAT_WATCHER + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes })
+          when :parse_without_stat
+            config = CONFIG_READ_FROM_HEAD + CONFIG_DISABLE_STAT_WATCHER + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes }) + PARSE_SINGLE_LINE_CONFIG
+          end
+
+          msg = 'test' * 2000 # in_tail reads 8192 bytes at once.
+          start_time = Fluent::Clock.now
+
+          d = create_driver(config)
+          d.run(expect_emits: 2) do
+            File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
+              100.times do
+                f.puts msg
+              end
+            }
+          end
+
+          assert_true(Fluent::Clock.now - start_time > 1)
+          assert_equal(num_events.times.map { {"message" => msg} },
+                       d.events.collect { |event| event[2] })
+        end
+
+        def test_read_bytes_limit_precede_read_lines_limit
+          config = CONFIG_READ_FROM_HEAD +
+                   SINGLE_LINE_CONFIG +
+                   config_element("", "", {
+                                    "read_lines_limit" => 1000,
+                                    "read_bytes_limit_per_second" => 8192
+                                  })
+          msg = 'abc'
+          start_time = Fluent::Clock.now
+          d = create_driver(config)
+          d.run(expect_emits: 2) do
+            File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
+              8000.times do
+                f.puts msg
+              end
+            }
+          end
+
+          assert_true(Fluent::Clock.now - start_time > 1)
+          assert_equal(4096.times.map { {"message" => msg} },
+                       d.events.collect { |event| event[2] })
+        end
+      end
+
+      sub_test_case "reads_bytes_per_second w/ throttled already" do
+        data("flat 8192 bytes"  => [:flat, 100, 8192],
+             "parse 8192 bytes" => [:parse, 100, 8192])
+        def test_emit_with_read_bytes_limit_per_second(data)
+          config_style, limit, limit_bytes = data
+          case config_style
+          when :flat
+            config = CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes })
+          when :parse
+            config = CONFIG_READ_FROM_HEAD + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes }) + PARSE_SINGLE_LINE_CONFIG
+          end
+          d = create_driver(config)
+          msg = 'test' * 2000 # in_tail reads 8192 bytes at once.
+
+          mock.proxy(d.instance).io_handler(anything, anything) do |io_handler|
+            require 'fluent/config/types'
+            limit_bytes_value = Fluent::Config.size_value(limit_bytes)
+            io_handler.instance_variable_set(:@number_bytes_read, limit_bytes_value)
+            if Fluent.linux?
+              mock.proxy(io_handler).handle_notify.at_least(5)
+            else
+              mock.proxy(io_handler).handle_notify.twice
+            end
+            io_handler
+          end
+
+          File.open("#{TMP_DIR}/tail.txt", "ab") do |f|
+            100.times do
+              f.puts msg
+            end
+          end
+
+          # We should not do shutdown here due to hard timeout.
+          d.run do
+            start_time = Fluent::Clock.now
+            while Fluent::Clock.now - start_time < 0.8 do
+              File.open("#{TMP_DIR}/tail.txt", "ab") do |f|
+                f.puts msg
+                f.flush
+              end
+              sleep 0.05
+            end
+          end
+
+          assert_equal([], d.events)
+        end
+      end
+    end
+
     data(flat: CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG,
          parse: CONFIG_READ_FROM_HEAD + PARSE_SINGLE_LINE_CONFIG)
     def test_emit_with_read_from_head(data)
@@ -551,6 +724,7 @@ class TailInputTest < Test::Unit::TestCase
     File.open("#{TMP_DIR}/tail.txt", "wb") {|f|
       f.puts "test1"
       f.puts "test2"
+      f.flush
     }
 
     d = create_driver(config)
@@ -560,19 +734,23 @@ class TailInputTest < Test::Unit::TestCase
         f.puts "test3\ntest4"
         f.flush
       }
-      sleep 1
+      waiting(2) { sleep 0.1 until d.events.length == 2 }
       File.truncate("#{TMP_DIR}/tail.txt", 6)
     end
 
-    events = d.events
-    assert_equal(3, events.length)
-    assert_equal({"message" => "test3"}, events[0][2])
-    assert_equal({"message" => "test4"}, events[1][2])
-    assert_equal({"message" => "test1"}, events[2][2])
-    assert(events[0][1].is_a?(Fluent::EventTime))
-    assert(events[1][1].is_a?(Fluent::EventTime))
-    assert(events[2][1].is_a?(Fluent::EventTime))
-    assert_equal(2, d.emit_count)
+    expected = {
+      emit_count: 2,
+      events: [
+        [Fluent::EventTime, {"message" => "test3"}],
+        [Fluent::EventTime, {"message" => "test4"}],
+        [Fluent::EventTime, {"message" => "test1"}],
+      ]
+    }
+    actual = {
+      emit_count: d.emit_count,
+      events: d.events.collect{|event| [event[1].class, event[2]]}
+    }
+    assert_equal(expected, actual)
   end
 
   def test_move_truncate_move_back
@@ -1876,4 +2054,98 @@ class TailInputTest < Test::Unit::TestCase
     waiting(5) { sleep 0.1 until d.instance.instance_variable_get(:@tails).keys.size == 1 }
     d.instance_shutdown
   end
+
+  def test_ENOENT_error_after_setup_watcher
+    path = "#{TMP_DIR}/tail.txt"
+    FileUtils.touch(path)
+    config = config_element('', '', {
+                              'format' => 'none',
+                            })
+    d = create_driver(config)
+    mock.proxy(d.instance).setup_watcher(anything, anything) do |tw|
+      cleanup_file(path)
+      tw
+    end
+    assert_nothing_raised do
+      d.run(shutdown: false) {}
+    end
+    d.instance_shutdown
+    assert($log.out.logs.any?{|log| log.include?("stat() for #{path} failed with Errno::ENOENT. Drop tail watcher for now.\n") })
+  end
+
+  def test_EACCES_error_after_setup_watcher
+    omit "Cannot test with root user" if Process::UID.eid == 0
+    path = "#{TMP_DIR}/noaccess/tail.txt"
+    begin
+      FileUtils.mkdir_p("#{TMP_DIR}/noaccess")
+      FileUtils.chmod(0755, "#{TMP_DIR}/noaccess")
+      FileUtils.touch(path)
+      config = config_element('', '', {
+                                'tag' => "tail",
+                                'path' => path,
+                                'format' => 'none',
+                              })
+      d = create_driver(config, false)
+      mock.proxy(d.instance).setup_watcher(anything, anything) do |tw|
+        FileUtils.chmod(0000, "#{TMP_DIR}/noaccess")
+        tw
+      end
+      assert_nothing_raised do
+        d.run(shutdown: false) {}
+      end
+      d.instance_shutdown
+      assert($log.out.logs.any?{|log| log.include?("stat() for #{path} failed with Errno::EACCES. Drop tail watcher for now.\n") })
+    end
+  ensure
+    if File.exist?("#{TMP_DIR}/noaccess")
+      FileUtils.chmod(0755, "#{TMP_DIR}/noaccess")
+      FileUtils.rm_rf("#{TMP_DIR}/noaccess")
+    end
+  end unless Fluent.windows?
+
+  def test_EACCES
+    path = "#{TMP_DIR}/tail.txt"
+    FileUtils.touch(path)
+    config = config_element('', '', {
+                              'format' => 'none',
+                            })
+    d = create_driver(config)
+    mock.proxy(Fluent::FileWrapper).stat(path) do |stat|
+      raise Errno::EACCES
+    end.at_least(1)
+    assert_nothing_raised do
+      d.run(shutdown: false) {}
+    end
+    d.instance_shutdown
+    assert($log.out.logs.any?{|log| log.include?("expand_paths: stat() for #{path} failed with Errno::EACCES. Skip file.\n") })
+  end
+
+  def test_shutdown_timeout
+    path = "#{TMP_DIR}/tail.txt"
+    File.open("#{TMP_DIR}/tail.txt", "wb") do |f|
+      (1024 * 1024 * 5).times do
+        f.puts "{\"test\":\"fizzbuzz\"}"
+      end
+    end
+
+    config =
+      CONFIG_READ_FROM_HEAD +
+      config_element('', '', {
+                              'format' => 'json',
+                              'skip_refresh_on_startup' => true,
+                            })
+    d = create_driver(config)
+    mock.proxy(d.instance).io_handler(anything, anything) do |io_handler|
+      io_handler.shutdown_timeout = 0.5
+      io_handler
+    end
+
+    start_time = Fluent::Clock.now
+    assert_nothing_raised do
+      d.run(expect_emits: 1)
+    end
+
+    elapsed = Fluent::Clock.now - start_time
+    assert_true(elapsed > 0.5 && elapsed < 2.5)
+  end
 end

data/test/plugin/test_in_tcp.rb

@@ -5,21 +5,33 @@ require 'fluent/plugin/in_tcp'
 class TcpInputTest < Test::Unit::TestCase
   def setup
     Fluent::Test.setup
+    @port = unused_port
   end
 
-  PORT = unused_port
-  BASE_CONFIG = %[
-    port #{PORT}
-    tag tcp
-  ]
-  CONFIG = BASE_CONFIG + %[
-    bind 127.0.0.1
-    format none
-  ]
-  IPv6_CONFIG = BASE_CONFIG + %[
-    bind ::1
-    format none
-  ]
+  def teardown
+    @port = nil
+  end
+
+  def base_config
+    %[
+      port #{@port}
+      tag tcp
+    ]
+  end
+
+  def ipv4_config
+    base_config + %[
+      bind 127.0.0.1
+      format none
+    ]
+  end
+
+  def ipv6_config
+    base_config + %[
+      bind ::1
+      format none
+    ]
+  end
 
   def create_driver(conf)
     Fluent::Test::Driver::Input.new(Fluent::Plugin::TcpInput).configure(conf)
@@ -35,22 +47,23 @@ class TcpInputTest < Test::Unit::TestCase
 
 
   data(
-    'ipv4' => [CONFIG, '127.0.0.1', :ipv4],
-    'ipv6' => [IPv6_CONFIG, '::1', :ipv6],
+    'ipv4' => ['127.0.0.1', :ipv4],
+    'ipv6' => ['::1', :ipv6],
   )
   test 'configure' do |data|
-    conf, bind, protocol = data
+    bind, protocol = data
+    conf = send("#{protocol}_config")
     omit "IPv6 is not supported on this environment" if protocol == :ipv6 && !ipv6_enabled?
 
     d = create_driver(conf)
-    assert_equal PORT, d.instance.port
+    assert_equal @port, d.instance.port
     assert_equal bind, d.instance.bind
     assert_equal "\n", d.instance.delimiter
   end
 
   test ' configure w/o parse section' do
     assert_raise(Fluent::ConfigError.new("<parse> section is required.")) {
-      create_driver(BASE_CONFIG)
+      create_driver(base_config)
     }
   end
 
@@ -82,10 +95,10 @@ class TcpInputTest < Test::Unit::TestCase
     payloads = data['payloads']
     expecteds = data['expecteds']
 
-    d = create_driver(BASE_CONFIG + "format #{format}")
+    d = create_driver(base_config + "format #{format}")
     d.run(expect_records: 2) do
       payloads.each do |payload|
-        create_tcp_socket('127.0.0.1', PORT) do |sock|
+        create_tcp_socket('127.0.0.1', @port) do |sock|
           sock.send(payload, 0)
         end
       end
@@ -105,9 +118,9 @@ class TcpInputTest < Test::Unit::TestCase
     payloads = data['payloads']
     expecteds = data['expecteds']
 
-    d = create_driver(BASE_CONFIG + "format #{format}")
+    d = create_driver(base_config + "format #{format}")
     d.run(expect_records: 2) do
-      create_tcp_socket('127.0.0.1', PORT) do |sock|
+      create_tcp_socket('127.0.0.1', @port) do |sock|
         payloads.each do |payload|
           sock.send(payload, 0)
         end
@@ -123,13 +136,13 @@ class TcpInputTest < Test::Unit::TestCase
   end
 
   test 'source_hostname_key' do
-    d = create_driver(BASE_CONFIG + %!
+    d = create_driver(base_config + %!
       format none
       source_hostname_key host
     !)
     hostname = nil
     d.run(expect_records: 1) do
-      create_tcp_socket('127.0.0.1', PORT) do |sock|
+      create_tcp_socket('127.0.0.1', @port) do |sock|
         sock.do_not_reverse_lookup = false
         hostname = sock.peeraddr[2]
         sock.send("test\n", 0)
@@ -144,13 +157,13 @@ class TcpInputTest < Test::Unit::TestCase
   end
 
   test 'source_address_key' do
-    d = create_driver(BASE_CONFIG + %!
+    d = create_driver(base_config + %!
       format none
       source_address_key addr
     !)
     address = nil
     d.run(expect_records: 1) do
-      create_tcp_socket('127.0.0.1', PORT) do |sock|
+      create_tcp_socket('127.0.0.1', @port) do |sock|
         address = sock.peeraddr[3]
         sock.send("test\n", 0)
       end
@@ -165,7 +178,7 @@ class TcpInputTest < Test::Unit::TestCase
 
   sub_test_case '<security>' do
     test 'accept from allowed client' do
-      d = create_driver(CONFIG + %!
+      d = create_driver(ipv4_config + %!
         <security>
           <client>
             network 127.0.0.1
@@ -173,7 +186,7 @@ class TcpInputTest < Test::Unit::TestCase
         </security>
       !)
       d.run(expect_records: 1) do
-        create_tcp_socket('127.0.0.1', PORT) do |sock|
+        create_tcp_socket('127.0.0.1', @port) do |sock|
           sock.send("hello\n", 0)
         end
       end
@@ -185,7 +198,7 @@ class TcpInputTest < Test::Unit::TestCase
     end
 
     test 'deny from disallowed client' do
-      d = create_driver(CONFIG + %!
+      d = create_driver(ipv4_config + %!
         <security>
           <client>
             network 200.0.0.0
@@ -193,7 +206,7 @@ class TcpInputTest < Test::Unit::TestCase
         </security>
       !)
       d.run(shutdown: false, expect_records: 1, timeout: 2) do
-        create_tcp_socket('127.0.0.1', PORT) do |sock|
+        create_tcp_socket('127.0.0.1', @port) do |sock|
           sock.send("hello\n", 0)
         end
       end
@@ -205,7 +218,7 @@ class TcpInputTest < Test::Unit::TestCase
 
   sub_test_case '<extract>' do
     test 'extract tag from record field' do
-      d = create_driver(BASE_CONFIG + %!
+      d = create_driver(base_config + %!
         <parse>
           @type json
         </parse>
@@ -214,7 +227,7 @@ class TcpInputTest < Test::Unit::TestCase
         </extract>
       !)
       d.run(expect_records: 1) do
-        create_tcp_socket('127.0.0.1', PORT) do |sock|
+        create_tcp_socket('127.0.0.1', @port) do |sock|
           data = {'msg' => 'hello', 'tag' => 'helper_test'}
           sock.send("#{data.to_json}\n", 0)
         end