fluentd 1.12.4 → 1.13.0

data/test/plugin/test_file_wrapper.rb

@@ -58,6 +58,15 @@ class FileWrapperTest < Test::Unit::TestCase
       assert_equal("#<Fluent::Win32Error: code: 32, The process cannot access the file because it is being used by another process. - C:\file.txt>",
                    Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, "C:\file.txt").inspect)
     end
+
+    data('0' => [false, 0],
+         '9999' => [false, 9999],
+         '10000' => [true, 10000],
+         '10001' => [true, 10001])
+    test 'wsaerr?' do |data|
+      expected, code = data
+      assert_equal(expected, Fluent::Win32Error.new(code).wsaerr?)
+    end
   end
 
   sub_test_case 'WindowsFile exceptions' do
@@ -92,7 +101,8 @@ class FileWrapperTest < Test::Unit::TestCase
         path = "#{TMP_DIR}/test_windows_file.txt"
         file1 = file2 = nil
         file1 = File.open(path, "wb")
-        assert_raise(Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, path)) do
+        win32err = Fluent::Win32Error.new(ERROR_SHARING_VIOLATION, path)
+        assert_raise(Errno::EACCES.new(win32err.message)) do
           file2 = Fluent::WindowsFile.new(path, 'r', FILE_SHARE_READ)
         ensure
           file2.close if file2

data/test/plugin/test_in_http.rb

@@ -770,6 +770,15 @@ class HttpInputTest < Test::Unit::TestCase
     end
   end
 
+  def test_get_request
+    d = create_driver(CONFIG)
+
+    d.run do
+      res = get("/cors.test", {}, {})
+      assert_equal "200", res.code
+    end
+  end
+
   def test_cors_preflight
     d = create_driver(CONFIG + 'cors_allow_origins ["*"]')
 
@@ -985,6 +994,12 @@ class HttpInputTest < Test::Unit::TestCase
     assert_equal $test_in_http_connection_object_ids[0], $test_in_http_connection_object_ids[1]
   end
 
+  def get(path, params, header = {})
+    http = Net::HTTP.new("127.0.0.1", PORT)
+    req = Net::HTTP::Get.new(path, header)
+    http.request(req)
+  end
+
   def options(path, params, header = {})
     http = Net::HTTP.new("127.0.0.1", PORT)
     req = Net::HTTP::Options.new(path, header)

data/test/plugin/test_in_tail.rb

@@ -156,6 +156,7 @@ class TailInputTest < Test::Unit::TestCase
       assert_equal 2, d.instance.rotate_wait
       assert_equal "#{TMP_DIR}/tail.pos", d.instance.pos_file
       assert_equal 1000, d.instance.read_lines_limit
+      assert_equal -1, d.instance.read_bytes_limit_per_second
       assert_equal false, d.instance.ignore_repeated_permission_error
       assert_nothing_raised do
         d.instance.have_read_capability?
@@ -195,6 +196,22 @@ class TailInputTest < Test::Unit::TestCase
       end
     end
 
+    sub_test_case "log throttling per file" do
+      test "w/o watcher timer is invalid" do
+        conf = CONFIG_ENABLE_WATCH_TIMER + config_element("ROOT", "", {"read_bytes_limit_per_second" => "8k"})
+        assert_raise(Fluent::ConfigError) do
+          create_driver(conf)
+        end
+      end
+
+      test "valid" do
+        conf = config_element("ROOT", "", {"read_bytes_limit_per_second" => "8k"})
+        assert_raise(Fluent::ConfigError) do
+          create_driver(conf)
+        end
+      end
+    end
+
     test "both enable_watch_timer and enable_stat_watcher are false" do
       assert_raise(Fluent::ConfigError) do
         create_driver(CONFIG_ENABLE_WATCH_TIMER + CONFIG_DISABLE_STAT_WATCHER + PARSE_SINGLE_LINE_CONFIG)
@@ -323,6 +340,138 @@ class TailInputTest < Test::Unit::TestCase
       assert num_events <= d.emit_count
     end
 
+    sub_test_case "log throttling per file" do
+      teardown do
+        cleanup_file("#{TMP_DIR}/tail.txt")
+      end
+
+      sub_test_case "reads_bytes_per_second w/o throttled" do
+        data("flat 8192 bytes, 2 events"        => [:flat, 100, 8192, 2, false],
+             "flat 8192 bytes, 2 events already read limit reached" => [:flat, 100, 8192, 2, true],
+             "flat 8192 bytes, 2 events w/o stat watcher" => [:flat_without_stat, 100, 8192, 2, false],
+             "flat #{8192*10} bytes, 20 events"  => [:flat, 100, (8192 * 10), 20],
+             "flat #{8192*10} bytes, 20 events w/o stat watcher"  => [:flat_without_stat, 100, (8192 * 10), 20],
+             "parse #{8192*4} bytes, 8 events"  => [:parse, 100, (8192 * 4), 8],
+             "parse #{8192*4} bytes, 8 events w/o stat watcher"  => [:parse_without_stat, 100, (8192 * 4), 8],
+             "parse #{8192*10} bytes, 20 events" => [:parse, 100, (8192 * 10), 20],
+             "parse #{8192*10} bytes, 20 events w/o stat watcher" => [:parse_without_stat, 100, (8192 * 10), 20],
+             "flat 8k bytes with unit, 2 events"        => [:flat, 100, "8k", 2],
+             "flat 8k bytes with unit, 2 events w/o stat watcher"        => [:flat_without_stat, 100, "8k", 2],
+             "flat #{8*10}k bytes with unit, 20 events"  => [:flat, 100, "#{8*10}k", 20],
+             "flat #{8*10}k bytes with unit, 20 events w/o stat watcher"  => [:flat_without_stat, 100, "#{8*10}k", 20],
+             "parse #{8*4}k bytes with unit, 8 events"  => [:parse, 100, "#{8*4}k", 8],
+             "parse #{8*4}k bytes with unit, 8 events w/o stat watcher"  => [:parse_without_stat, 100, "#{8*4}k", 8],
+             "parse #{8*10}k bytes with unit, 20 events" => [:parse, 100, "#{8*10}k", 20],
+             "parse #{8*10}k bytes with unit, 20 events w/o stat watcher" => [:parse_without_stat, 100, "#{8*10}k", 20])
+        def test_emit_with_read_bytes_limit_per_second(data)
+          config_style, limit, limit_bytes, num_events = data
+          case config_style
+          when :flat
+            config = CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes })
+          when :parse
+            config = CONFIG_READ_FROM_HEAD + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes }) + PARSE_SINGLE_LINE_CONFIG
+          when :flat_without_stat
+            config = CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + CONFIG_DISABLE_STAT_WATCHER + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes })
+          when :parse_without_stat
+            config = CONFIG_READ_FROM_HEAD + CONFIG_DISABLE_STAT_WATCHER + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes }) + PARSE_SINGLE_LINE_CONFIG
+          end
+
+          msg = 'test' * 2000 # in_tail reads 8192 bytes at once.
+          start_time = Fluent::Clock.now
+
+          d = create_driver(config)
+          d.run(expect_emits: 2) do
+            File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
+              100.times do
+                f.puts msg
+              end
+            }
+          end
+
+          assert_true(Fluent::Clock.now - start_time > 1)
+          assert_equal(num_events.times.map { {"message" => msg} },
+                       d.events.collect { |event| event[2] })
+        end
+
+        def test_read_bytes_limit_precede_read_lines_limit
+          config = CONFIG_READ_FROM_HEAD +
+                   SINGLE_LINE_CONFIG +
+                   config_element("", "", {
+                                    "read_lines_limit" => 1000,
+                                    "read_bytes_limit_per_second" => 8192
+                                  })
+          msg = 'abc'
+          start_time = Fluent::Clock.now
+          d = create_driver(config)
+          d.run(expect_emits: 2) do
+            File.open("#{TMP_DIR}/tail.txt", "ab") {|f|
+              8000.times do
+                f.puts msg
+              end
+            }
+          end
+
+          assert_true(Fluent::Clock.now - start_time > 1)
+          assert_equal(4096.times.map { {"message" => msg} },
+                       d.events.collect { |event| event[2] })
+        end
+      end
+
+      sub_test_case "reads_bytes_per_second w/ throttled already" do
+        data("flat 8192 bytes, 2 events"        => [:flat, 100, 8192, 2],
+             "flat #{8192*10} bytes, 20 events"  => [:flat, 100, (8192 * 10), 20],
+             "parse #{8192*4} bytes, 8 events"  => [:parse, 100, (8192 * 4), 8],
+             "parse #{8192*10} bytes, 20 events" => [:parse, 100, (8192 * 10), 20],
+             "flat 8k bytes with unit, 2 events"        => [:flat, 100, "8k", 2],
+             "flat #{8*10}k bytes with unit, 20 events"  => [:flat, 100, "#{8*10}k", 20],
+             "parse #{8*4}k bytes with unit, 8 events"  => [:parse, 100, "#{8*4}k", 8],
+             "parse #{8*10}k bytes with unit, 20 events" => [:parse, 100, "#{8*10}k", 20])
+        def test_emit_with_read_bytes_limit_per_second(data)
+          config_style, limit, limit_bytes, num_events = data
+          case config_style
+          when :flat
+            config = CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes })
+          when :parse
+            config = CONFIG_READ_FROM_HEAD + config_element("", "", { "read_lines_limit" => limit, "read_bytes_limit_per_second" => limit_bytes }) + PARSE_SINGLE_LINE_CONFIG
+          end
+          d = create_driver(config)
+          msg = 'test' * 2000 # in_tail reads 8192 bytes at once.
+
+          mock.proxy(d.instance).io_handler(anything, anything) do |io_handler|
+            require 'fluent/config/types'
+            limit_bytes_value = Fluent::Config.size_value(limit_bytes)
+            io_handler.instance_variable_set(:@number_bytes_read, limit_bytes_value)
+            if Fluent.linux?
+              mock.proxy(io_handler).handle_notify.at_least(5)
+            else
+              mock.proxy(io_handler).handle_notify.twice
+            end
+            io_handler
+          end
+
+          File.open("#{TMP_DIR}/tail.txt", "ab") do |f|
+            100.times do
+              f.puts msg
+            end
+          end
+
+          # We should not do shutdown here due to hard timeout.
+          d.run do
+            start_time = Fluent::Clock.now
+            while Fluent::Clock.now - start_time < 0.8 do
+              File.open("#{TMP_DIR}/tail.txt", "ab") do |f|
+                f.puts msg
+                f.flush
+              end
+              sleep 0.05
+            end
+          end
+
+          assert_equal([], d.events)
+        end
+      end
+    end
+
     data(flat: CONFIG_READ_FROM_HEAD + SINGLE_LINE_CONFIG,
          parse: CONFIG_READ_FROM_HEAD + PARSE_SINGLE_LINE_CONFIG)
     def test_emit_with_read_from_head(data)
@@ -1935,6 +2084,79 @@ class TailInputTest < Test::Unit::TestCase
       d.run(shutdown: false) {}
     end
     d.instance_shutdown
-    assert($log.out.logs.any?{|log| log.include?("stat() for #{path} failed with ENOENT. Drop tail watcher for now.\n") })
+    assert($log.out.logs.any?{|log| log.include?("stat() for #{path} failed with Errno::ENOENT. Drop tail watcher for now.\n") })
+  end
+
+  def test_EACCES_error_after_setup_watcher
+    path = "#{TMP_DIR}/noaccess/tail.txt"
+    begin
+      FileUtils.mkdir_p("#{TMP_DIR}/noaccess")
+      FileUtils.chmod(0755, "#{TMP_DIR}/noaccess")
+      FileUtils.touch(path)
+      config = config_element('', '', {
+                                'tag' => "tail",
+                                'path' => path,
+                                'format' => 'none',
+                              })
+      d = create_driver(config, false)
+      mock.proxy(d.instance).setup_watcher(anything, anything) do |tw|
+        FileUtils.chmod(0000, "#{TMP_DIR}/noaccess")
+        tw
+      end
+      assert_nothing_raised do
+        d.run(shutdown: false) {}
+      end
+      d.instance_shutdown
+      assert($log.out.logs.any?{|log| log.include?("stat() for #{path} failed with Errno::EACCES. Drop tail watcher for now.\n") })
+    end
+  ensure
+    FileUtils.chmod(0755, "#{TMP_DIR}/noaccess")
+    FileUtils.rm_rf("#{TMP_DIR}/noaccess")
+  end unless Fluent.windows?
+
+  def test_EACCES
+    path = "#{TMP_DIR}/tail.txt"
+    FileUtils.touch(path)
+    config = config_element('', '', {
+                              'format' => 'none',
+                            })
+    d = create_driver(config)
+    mock.proxy(Fluent::FileWrapper).stat(path) do |stat|
+      raise Errno::EACCES
+    end.at_least(1)
+    assert_nothing_raised do
+      d.run(shutdown: false) {}
+    end
+    d.instance_shutdown
+    assert($log.out.logs.any?{|log| log.include?("expand_paths: stat() for #{path} failed with Errno::EACCES. Skip file.\n") })
+  end
+
+  def test_shutdown_timeout
+    path = "#{TMP_DIR}/tail.txt"
+    File.open("#{TMP_DIR}/tail.txt", "wb") do |f|
+      (1024 * 1024 * 5).times do
+        f.puts "{\"test\":\"fizzbuzz\"}"
+      end
+    end
+
+    config =
+      CONFIG_READ_FROM_HEAD +
+      config_element('', '', {
+                              'format' => 'json',
+                              'skip_refresh_on_startup' => true,
+                            })
+    d = create_driver(config)
+    mock.proxy(d.instance).io_handler(anything, anything) do |io_handler|
+      io_handler.shutdown_timeout = 0.5
+      io_handler
+    end
+
+    start_time = Fluent::Clock.now
+    assert_nothing_raised do
+      d.run(expect_emits: 1)
+    end
+
+    elapsed = Fluent::Clock.now - start_time
+    assert_true(elapsed > 0.5 && elapsed < 2.5)
   end
 end

data/test/plugin/test_out_forward.rb

@@ -277,11 +277,16 @@ EOL
 </service_discovery>
     ])
 
-    assert_equal 2, d.instance.discovery_manager.services.size
-    assert_equal '127.0.0.1', d.instance.discovery_manager.services[0].host
-    assert_equal 1234, d.instance.discovery_manager.services[0].port
-    assert_equal '127.0.0.1', d.instance.discovery_manager.services[1].host
-    assert_equal 1235, d.instance.discovery_manager.services[1].port
+
+    assert_equal(
+      [
+        { host: '127.0.0.1', port: 1234 },
+        { host: '127.0.0.1', port: 1235 },
+      ],
+      d.instance.discovery_manager.services.collect do |service|
+        { host: service.host, port: service.port }
+      end
+    )
   end
 
   test 'pass username and password as empty string to HandshakeProtocol' do
@@ -1073,7 +1078,7 @@ EOL
   test 'when out_forward has @id' do
     # cancel https://github.com/fluent/fluentd/blob/077508ac817b7637307434d0c978d7cdc3d1c534/lib/fluent/plugin_id.rb#L43-L53
     # it always return true in test
-    mock.proxy(Fluent::Plugin).new_sd(:static, parent: anything) { |v|
+    mock.proxy(Fluent::Plugin).new_sd('static', parent: anything) { |v|
       stub(v).plugin_id_for_test? { false }
     }.once
 

data/test/plugin_helper/test_service_discovery.rb

@@ -17,6 +17,24 @@ class ServiceDiscoveryHelper < Test::Unit::TestCase
     end
   end
 
+  class DummyPlugin < Fluent::Plugin::TestBase
+    helpers :service_discovery
+
+    def configure(conf)
+      super
+      service_discovery_configure(:service_discovery_helper_test, static_default_service_directive: 'node')
+    end
+
+    def select_service(&block)
+      service_discovery_select_service(&block)
+    end
+
+    # Make these mehtod public
+    def discovery_manager
+      super
+    end
+  end
+
   setup do
     @sd_file_dir = File.expand_path('../plugin/data/sd_file', __dir__)
 
@@ -33,7 +51,7 @@ class ServiceDiscoveryHelper < Test::Unit::TestCase
     end
   end
 
-  test 'start discovery manager' do
+  test 'support calling #service_discovery_create_manager and #discovery_manager from plugin' do
     d = @d = Dummy.new
 
     d.service_discovery_create_manager(
@@ -55,13 +73,30 @@ class ServiceDiscoveryHelper < Test::Unit::TestCase
     assert_equal 1234, services[0].port
   end
 
-  test 'call timer_execute if dynamic configuration' do
-    d = @d = Dummy.new
+  test 'start discovery manager' do
+    d = @d = DummyPlugin.new
 
-    d.service_discovery_create_manager(
-      :service_discovery_helper_test,
-      configurations: [{ type: :file, conf: config_element('file_config', '', { 'path' => File.join(@sd_file_dir, 'config.yml') }) }],
-    )
+    services = [config_element('service', '', { 'host' => '127.0.0.1', 'port' => '1234' })]
+    d.configure(config_element('root', '', {}, [config_element('service_discovery', '', {'@type' => 'static'}, services)]))
+
+    assert_true !!d.discovery_manager
+
+    mock.proxy(d.discovery_manager).start.once
+    mock.proxy(d).timer_execute(:service_discovery_helper_test, anything).never
+
+    d.start
+    d.event_loop_wait_until_start
+
+    assert_equal 1, d.discovery_manager.services.size
+    d.select_service do |serv|
+      assert_equal "127.0.0.1", serv.host
+      assert_equal 1234, serv.port
+    end
+  end
+
+  test 'call timer_execute if dynamic configuration' do
+    d = @d = DummyPlugin.new
+    d.configure(config_element('root', '', {}, [config_element('service_discovery', '', { '@type' => 'file', 'path' => File.join(@sd_file_dir, 'config.yml' )})]))
 
     assert_true !!d.discovery_manager
     mock.proxy(d.discovery_manager).start.once
@@ -71,25 +106,22 @@ class ServiceDiscoveryHelper < Test::Unit::TestCase
   end
 
   test 'exits service discovery instances without any errors' do
-    d = @d = Dummy.new
+    d = @d = DummyPlugin.new
     mockv = flexmock('dns_resolver', getaddress: '127.0.0.1')
               .should_receive(:getresources)
               .and_return([Resolv::DNS::Resource::IN::SRV.new(1, 10, 8081, 'service1.example.com')])
               .mock
     mock(Resolv::DNS).new { mockv }
 
-    d.service_discovery_create_manager(
-      :service_discovery_helper_test2,
-      configurations: [{ type: :srv, conf: config_element('service_discovery', '', { 'service' => 'service1', 'hostname' => 'example.com' }) }],
-    )
+    d.configure(config_element('root', '', {}, [config_element('service_discovery', '', { '@type' => 'srv', 'service' => 'service1', 'hostname' => 'example.com' })]))
 
     assert_true !!d.discovery_manager
     mock.proxy(d.discovery_manager).start.once
-    mock(d).timer_execute(:service_discovery_helper_test2, anything).once
+    mock(d).timer_execute(:service_discovery_helper_test, anything).once
 
     # To avoid claring `@logs` during `terminate` step
     # https://github.com/fluent/fluentd/blob/bc78d889f93dad8c2a4e0ad1ca802546185dacba/lib/fluent/test/log.rb#L33
-    mock(d.log).reset.twice
+    mock(d.log).reset.times(3)
 
     d.start
     d.event_loop_wait_until_start
@@ -102,4 +134,32 @@ class ServiceDiscoveryHelper < Test::Unit::TestCase
 
     assert_false(d.log.out.logs.any? { |e| e.match?(/thread doesn't exit correctly/) })
   end
+
+  test 'static service discovery will be configured automatically when default service directive is specified' do
+    d = @d = DummyPlugin.new
+
+    nodes = [
+      config_element('node', '', { 'host' => '192.168.0.1', 'port' => '24224' }),
+      config_element('node', '', { 'host' => '192.168.0.2', 'port' => '24224' })
+    ]
+    d.configure(config_element('root', '', {}, nodes))
+
+    assert_true !!d.discovery_manager
+
+    mock.proxy(d.discovery_manager).start.once
+    mock.proxy(d).timer_execute(:service_discovery_helper_test, anything).never
+
+    d.start
+    d.event_loop_wait_until_start
+
+    assert_equal 2, d.discovery_manager.services.size
+    d.select_service do |serv|
+      assert_equal "192.168.0.1", serv.host
+      assert_equal 24224, serv.port
+    end
+    d.select_service do |serv|
+      assert_equal "192.168.0.2", serv.host
+      assert_equal 24224, serv.port
+    end
+  end
 end