yahns 0.0.0TP1

data/test/test_bin.rb

@@ -0,0 +1,97 @@
+# Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> and all contributors
+# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
+require_relative 'server_helper'
+class TestBin < Testcase
+  parallelize_me!
+  include ServerHelper
+  alias teardown server_helper_teardown
+
+  def setup
+    server_helper_setup
+    @cmd = %W(ruby -I lib bin/yahns)
+  end
+
+  def test_bin_daemon_noworker_inherit
+    bin_daemon(false, true)
+  end
+
+  def test_bin_daemon_worker_inherit
+    bin_daemon(true, true)
+  end
+
+  def test_bin_daemon_noworker_bind
+    bin_daemon(false, false)
+  end
+
+  def test_bin_daemon_worker_bind
+    bin_daemon(true, false)
+  end
+
+  def bin_daemon(worker, inherit)
+    @srv.close unless inherit
+    @pid = tmpfile(%w(test_bin_daemon .pid))
+    @ru = tmpfile(%w(test_bin_daemon .ru))
+    @ru.write("require 'rack/lobster'; run Rack::Lobster.new\n")
+    cfg = tmpfile(%w(test_bin_daemon_conf .rb))
+    cfg.puts "pid '#{@pid.path}'"
+    cfg.puts "stderr_path '#{@err.path}'"
+    cfg.puts "worker_processes 1" if worker
+    cfg.puts "app(:rack, '#{@ru.path}', preload: false) do"
+    cfg.puts "  listen ENV['YAHNS_TEST_LISTEN']"
+    cfg.puts "end"
+    @cmd.concat(%W(-D -c #{cfg.path}))
+    addr = IO.pipe
+    pid = fork do
+      if inherit
+        @cmd << { @srv.fileno => @srv }
+        ENV["YAHNS_FD"] = @srv.fileno.to_s
+      else
+        @srv = TCPServer.new(ENV["TEST_HOST"] || "127.0.0.1", 0)
+      end
+      host, port = @srv.addr[3], @srv.addr[1]
+      listen = ENV["YAHNS_TEST_LISTEN"] = "#{host}:#{port}"
+      addr[1].write(listen)
+      addr[1].close
+      addr[0].close
+      exec(*@cmd)
+    end
+    addr[1].close
+    listen = Timeout.timeout(10) { addr[0].read }
+    addr[0].close
+    host, port = listen.split(/:/, 2)
+    port = port.to_i
+    assert_operator port, :>, 0
+
+    unless inherit
+      # daemon_pipe guarantees socket will be usable after this:
+      Timeout.timeout(10) do # Ruby startup is slow!
+        _, status = Process.waitpid2(pid)
+        assert status.success?, status.inspect
+      end
+    end
+
+    Net::HTTP.start(host, port) do |http|
+      req = Net::HTTP::Get.new("/")
+      res = http.request(req)
+      assert_equal 200, res.code.to_i
+      assert_equal "keep-alive", res["Connection"]
+    end
+  rescue => e
+    warn "#{e.message} (#{e.class})"
+    e.backtrace.each { |l| warn "#{l}" }
+    raise
+  ensure
+    cfg.close! if cfg
+    pid = File.read(@pid.path)
+    pid = pid.to_i
+    assert_operator pid, :>, 0
+    Process.kill(:QUIT, pid)
+    if inherit
+      _, status = Timeout.timeout(10) { Process.waitpid2(pid) }
+      assert status.success?, status.inspect
+    else
+      poke_until_dead pid
+    end
+    @pid.close! if @pid
+  end
+end

data/test/test_client_expire.rb

@@ -0,0 +1,132 @@
+# Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> and all contributors
+# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
+require_relative 'server_helper'
+
+class TestClientExpire < Testcase
+  parallelize_me!
+  include ServerHelper
+  alias setup server_helper_setup
+  alias teardown server_helper_teardown
+
+  def test_client_expire_negative
+    err = @err
+    cfg = Yahns::Config.new
+    host, port = @srv.addr[3], @srv.addr[1]
+    cfg.instance_eval do
+      GTL.synchronize do
+        ru = lambda { |e| h = { "Content-Length" => "0" }; [ 200, h, [] ] }
+        app(:rack, ru) do
+          listen "#{host}:#{port}", sndbuf: 2048, rcvbuf: 2048
+        end
+        client_expire_threshold(-10)
+      end
+      logger(Logger.new(err.path))
+    end
+    srv = Yahns::Server.new(cfg)
+    pid = fork do
+      ENV["YAHNS_FD"] = @srv.fileno.to_s
+      srv.start.join
+    end
+    Net::HTTP.start(host, port) { |h|
+      res = h.get("/")
+      assert_empty res.body
+    }
+  ensure
+    quit_wait(pid)
+  end
+
+  def test_client_expire
+    nr = 32
+    err = @err
+    cfg = Yahns::Config.new
+    host, port = @srv.addr[3], @srv.addr[1]
+    cfg.instance_eval do
+      GTL.synchronize do
+        h = { "Content-Length" => "0" }
+        app(:rack, lambda { |e| [ 200, h, [] ]}) do
+          listen "#{host}:#{port}", sndbuf: 2048, rcvbuf: 2048
+          client_timeout 1
+        end
+        client_expire_threshold(32)
+      end
+      logger(Logger.new(err.path))
+    end
+    srv = Yahns::Server.new(cfg)
+    pid = fork do
+      ENV["YAHNS_FD"] = @srv.fileno.to_s
+      srv.start.join
+    end
+    f = TCPSocket.new(host, port)
+    s = TCPSocket.new(host, port)
+    req = "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n"
+    s.write(req)
+    str = Timeout.timeout(20) { s.readpartial(666) }
+    assert_match(%r{keep-alive}, str)
+    sleep 2
+    abe = tmpfile(%w(abe .err))
+    ab_res = `ab -c #{nr} -n 10000 -k http://#{host}:#{port}/ 2>#{abe.path}`
+    assert $?.success?, $?.inspect << abe.read
+    abe.close!
+    assert_match(/Complete requests:\s+10000\n/, ab_res)
+
+    [ f, s ].each do |io|
+      assert_raises(Errno::EPIPE,Errno::ECONNRESET) do
+        req.each_byte { |b| io.write(b.chr) }
+      end
+      io.close
+    end
+  rescue => e
+    Yahns::Log.exception(Logger.new($stderr), "test", e)
+    raise
+  ensure
+    quit_wait(pid)
+  end
+
+  def test_client_expire_desperate
+    skip "disabled since FD counts vary heavily in an MT process"
+    err = @err
+    cfg = Yahns::Config.new
+    host, port = @srv.addr[3], @srv.addr[1]
+    cfg.instance_eval do
+      GTL.synchronize do
+        h = { "Content-Length" => "0" }
+        app(:rack, lambda { |e| [ 200, h, [] ]}) do
+          listen "#{host}:#{port}", sndbuf: 2048, rcvbuf: 2048
+          client_timeout 1
+        end
+        client_expire_threshold 1.0
+      end
+      logger(Logger.new(err.path))
+    end
+    srv = Yahns::Server.new(cfg)
+    pid = fork do
+      Process.setrlimit :NOFILE, 512, 1024
+      ENV["YAHNS_FD"] = @srv.fileno.to_s
+      srv.start.join
+    end
+    f = TCPSocket.new(host, port)
+    s = TCPSocket.new(host, port)
+    req = "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n"
+    s.write(req)
+    str = Timeout.timeout(20) { s.readpartial(666) }
+    assert_match(%r{keep-alive}, str)
+    sleep 3
+    system "ab -c 1000 -n 10000 -k http://#{host}:#{port}/ 2>&1"
+
+    [ f, s ].each do |io|
+      assert_raises(Errno::EPIPE,Errno::ECONNRESET) do
+        req.each_byte { |b| io.write(b.chr) }
+      end
+      io.close
+    end
+    errs = File.readlines(err.path).grep(/ERROR/)
+    File.truncate(err.path, 0) # avoid error on teardown
+    re = %r{consider raising open file limits} # - accept, consider raising open file limits}
+    assert_equal errs.grep(re), errs
+  rescue => e
+    Yahns::Log.exception(Logger.new($stderr), "test", e)
+    raise
+  ensure
+    quit_wait(pid)
+  end
+end

data/test/test_config.rb

@@ -0,0 +1,56 @@
+# Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> and all contributors
+# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
+require_relative 'helper'
+require 'rack/lobster'
+
+class TestConfig < Testcase
+  parallelize_me!
+
+  def test_initialize
+    cfg = Yahns::Config.new
+    assert_instance_of Yahns::Config, cfg
+  end
+
+  def test_multi_conf_example
+    tmpdir = Dir.mktmpdir
+
+    # modify the example config file for testing
+    path = "examples/yahns_multi.conf.rb"
+    cfgs = File.read(path)
+    cfgs.gsub!(%r{/path/to/}, "#{tmpdir}/")
+    conf = File.open("#{tmpdir}/yahns_multi.conf.rb", "w")
+    conf.sync = true
+    conf.write(cfgs)
+    File.open("#{tmpdir}/another.ru", "w") do |fp|
+      fp.puts("run Rack::Lobster.new\n")
+    end
+    FileUtils.mkpath("#{tmpdir}/another")
+
+    cfg = GTL.synchronize { Yahns::Config.new(conf.path) }
+    assert_instance_of Yahns::Config, cfg
+  ensure
+    FileUtils.rm_rf(tmpdir) if tmpdir
+  end
+
+  def test_rack_basic_conf_example
+    tmpdir = Dir.mktmpdir
+
+    # modify the example config file for testing
+    path = "examples/yahns_rack_basic.conf.rb"
+    cfgs = File.read(path)
+    cfgs.gsub!(%r{/path/to/}, "#{tmpdir}/")
+    Dir.mkdir("#{tmpdir}/my_app")
+    Dir.mkdir("#{tmpdir}/my_logs")
+    Dir.mkdir("#{tmpdir}/my_pids")
+    conf = File.open("#{tmpdir}/yahns_rack_basic.conf.rb", "w")
+    conf.sync = true
+    conf.write(cfgs)
+    File.open("#{tmpdir}/my_app/config.ru", "w") do |fp|
+      fp.puts("run Rack::Lobster.new\n")
+    end
+    cfg = GTL.synchronize { Yahns::Config.new(conf.path) }
+    assert_instance_of Yahns::Config, cfg
+  ensure
+    FileUtils.rm_rf(tmpdir) if tmpdir
+  end
+end

data/test/test_fdmap.rb

@@ -0,0 +1,19 @@
+# Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> and all contributors
+# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
+require_relative 'helper'
+
+class TestFdmap < Testcase
+  def test_fdmap_negative
+    fdmap = Yahns::Fdmap.new(Logger.new($stderr), -5)
+    nr = fdmap.instance_variable_get :@client_expire_threshold
+    assert_operator nr, :>, 0
+    assert_equal nr, Process.getrlimit(:NOFILE)[0] - 5
+  end
+
+  def test_fdmap_float
+    fdmap = Yahns::Fdmap.new(Logger.new($stderr), 0.5)
+    nr = fdmap.instance_variable_get :@client_expire_threshold
+    assert_operator nr, :>, 0
+    assert_equal nr, Process.getrlimit(:NOFILE)[0]/2
+  end
+end

data/test/test_output_buffering.rb

@@ -0,0 +1,291 @@
+# Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> and all contributors
+# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
+require_relative 'server_helper'
+require 'digest/md5'
+require 'rack/file'
+
+class TestOutputBuffering < Testcase
+  parallelize_me!
+  include ServerHelper
+  alias setup server_helper_setup
+  alias teardown server_helper_teardown
+
+  GPLv3 = File.read("COPYING")
+  RAND = IO.binread("/dev/urandom", 666) * 119
+  dig = Digest::MD5.new
+  NR = 1337
+  MD5 = Thread.new do
+    NR.times { dig << RAND }
+    dig.hexdigest
+  end
+
+  class BigBody
+    def each
+      NR.times { yield RAND }
+    end
+  end
+
+  def test_output_buffer_false_curl
+    output_buffer(false, :curl)
+  end
+
+  def test_output_buffer_false_http09
+    output_buffer(false, :http09)
+  end
+
+  def test_output_buffer_true_curl
+    output_buffer(true, :curl)
+  end
+
+  def test_output_buffer_true_http09
+    output_buffer(true, :http09)
+  end
+
+  def output_buffer(btype, check_type, delay = 4)
+    err = @err
+    cfg = Yahns::Config.new
+    host, port = @srv.addr[3], @srv.addr[1]
+    len = (RAND.size * NR).to_s
+    cfg.instance_eval do
+      ru = lambda do |e|
+        [ 200, {'Content-Length'=>len}, BigBody.new ]
+      end
+      GTL.synchronize do
+        app(:rack, ru) do
+          listen "#{host}:#{port}"
+          output_buffering btype
+        end
+      end
+      logger(Logger.new(err.path))
+    end
+    srv = Yahns::Server.new(cfg)
+    pid = fork do
+      ENV["YAHNS_FD"] = @srv.fileno.to_s
+      srv.start.join
+    end
+
+    case check_type
+    when :curl
+      # curl is faster for piping gigantic wads of data than Net::HTTP
+      sh_sleep = delay ? "sleep #{delay} && " : ""
+      md5 = `curl -sSf http://#{host}:#{port}/ | (#{sh_sleep} md5sum)`
+      assert $?.success?, $?.inspect
+      (md5 =~ /([a-f0-9]{32})/i) or raise "bad md5: #{md5.inspect}"
+      md5 = $1
+      assert_equal MD5.value, md5
+    when :http09
+      # HTTP/0.9
+      c = TCPSocket.new(host, port)
+      c.write("GET /\r\n\r\n")
+      md5in = IO.pipe
+      md5out = IO.pipe
+      sleep(delay) if delay
+      md5pid = Process.spawn("md5sum", :in => md5in[0], :out => md5out[1])
+      md5in[0].close
+      md5out[1].close
+      assert_equal(NR * RAND.size, IO.copy_stream(c, md5in[1]))
+      c.shutdown
+      c.close
+      md5in[1].close
+      _, status = Timeout.timeout(10) { Process.waitpid2(md5pid) }
+      assert status.success?, status.inspect
+      md5 = md5out[0].read
+      (md5 =~ /([a-f0-9]{32})/i) or raise "bad md5: #{md5.inspect}"
+      md5 = $1
+      assert_equal MD5.value, md5
+      md5out[0].close
+    else
+      raise "TESTBUG"
+    end
+  rescue => e
+    Yahns::Log.exception(Logger.new($stderr), "test", e)
+    raise
+  ensure
+    quit_wait(pid)
+  end
+
+  class BigHeader
+    A = "A" * 65536
+    def initialize(h)
+      @h = h
+    end
+    def each
+      NR.times do |n|
+        yield("X-#{n}", A)
+      end
+      @h.each { |k,v| yield(k,v) }
+    end
+  end
+
+  def test_big_header
+    err = @err
+    cfg = Yahns::Config.new
+    host, port = @srv.addr[3], @srv.addr[1]
+    cfg.instance_eval do
+      ru = lambda do |e|
+        case e["PATH_INFO"]
+        when "/COPYING"
+          Rack::File.new(Dir.pwd).call(e)
+          gplv3 = File.open("COPYING")
+          def gplv3.each
+            raise "SHOULD NOT BE CALLED"
+          end
+          size = gplv3.stat.size
+          len = size.to_s
+          ranges = Rack::Utils.byte_ranges(e, size)
+          status = 200
+          h = { "Content-Type" => "text/plain", "Content-Length" => len }
+          if ranges && ranges.size == 1
+            status = 206
+            range = ranges[0]
+            h["Content-Range"] = "bytes #{range.begin}-#{range.end}/#{size}"
+            size = range.end - range.begin + 1
+            len.replace(size.to_s)
+          end
+          [ status , BigHeader.new(h), gplv3 ]
+        when "/"
+          h = { "Content-Type" => "text/plain", "Content-Length" => "4" }
+          [ 200, BigHeader.new(h), ["BIG\n"] ]
+        else
+          raise "WTF"
+        end
+      end
+      GTL.synchronize do
+        app(:rack, ru) do
+          listen "#{host}:#{port}"
+        end
+      end
+      logger(Logger.new(err.path))
+    end
+    srv = Yahns::Server.new(cfg)
+    pid = fork do
+      ENV["YAHNS_FD"] = @srv.fileno.to_s
+      srv.start.join
+    end
+    threads = []
+
+    # start with just a big header
+    threads << Thread.new do
+      c = TCPSocket.new(host, port)
+      c.write "GET / HTTP/1.0\r\n\r\n"
+      begin
+        sleep 1
+      end while c.nread == 0
+      nr = 0
+      last = nil
+      c.each_line do |line|
+        case line
+        when %r{\AX-} then nr += 1
+        else
+          last = line
+        end
+      end
+      assert_equal NR, nr
+      assert_equal "BIG\n", last
+      c.close
+    end
+
+    threads << Thread.new do
+      c = TCPSocket.new(host, port)
+      c.write "GET /COPYING HTTP/1.0\r\n\r\n"
+      begin
+        sleep 1
+      end while c.nread == 0
+      nr = 0
+      c.each_line do |line|
+        case line
+        when %r{\AX-} then nr += 1
+        else
+          break if line == "\r\n"
+        end
+      end
+      assert_equal NR, nr
+      assert_equal GPLv3, c.read
+      c.close
+    end
+
+    threads << Thread.new do
+      c = TCPSocket.new(host, port)
+      c.write "GET /COPYING HTTP/1.0\r\nRange: bytes=5-46\r\n\r\n"
+      begin
+        sleep 1
+      end while c.nread == 0
+      nr = 0
+      c.each_line do |line|
+        case line
+        when %r{\AX-} then nr += 1
+        else
+          break if line == "\r\n"
+        end
+      end
+      assert_equal NR, nr
+      assert_equal GPLv3[5..46], c.read
+      c.close
+    end
+    threads.each do |t|
+      assert_equal t, t.join(30)
+      assert_nil t.value
+    end
+  ensure
+    quit_wait(pid)
+  end
+
+  def test_client_timeout
+    err = @err
+    apperr = tmpfile(%w(app .err))
+    cfg = Yahns::Config.new
+    size = RAND.size * NR
+    host, port = @srv.addr[3], @srv.addr[1]
+    cfg.instance_eval do
+      ru = lambda do |e|
+        if e["PATH_INFO"] == "/bh"
+          h = { "Content-Type" => "text/plain", "Content-Length" => "4" }
+          [ 200, BigHeader.new(h), ["BIG\n"] ]
+        else
+          [ 200, {'Content-Length' => size.to_s }, BigBody.new ]
+        end
+      end
+      GTL.synchronize do
+        app(:rack, ru) do
+          listen "#{host}:#{port}"
+          output_buffering false
+          client_timeout 3
+          logger(Logger.new(apperr.path))
+        end
+      end
+      logger(Logger.new(err.path))
+    end
+    srv = Yahns::Server.new(cfg)
+    pid = fork do
+      ENV["YAHNS_FD"] = @srv.fileno.to_s
+      srv.start.join
+    end
+    threads = []
+    threads << Thread.new do
+      c = TCPSocket.new(host, port)
+      c.write("GET / HTTP/1.1\r\nHost: example.com\r\n\r\n")
+      sleep(5) # wait for timeout
+      assert_operator c.nread, :>, 0
+      c
+    end
+
+    threads << Thread.new do
+      c = TCPSocket.new(host, port)
+      c.write("GET /bh HTTP/1.1\r\nHost: example.com\r\n\r\n")
+      sleep(5) # wait for timeout
+      assert_operator c.nread, :>, 0
+      c
+    end
+    threads.each { |t| t.join(10) }
+    assert_operator size, :>, threads[0].value.read.size
+    assert_operator size, :>, threads[1].value.read.size
+    msg = File.readlines(apperr.path)
+    msg = msg.grep(/timeout on :wait_writable after 3s$/)
+    assert_equal 2, msg.size
+    threads.each { |t| t.value.close }
+  ensure
+    apperr.close! if apperr
+    quit_wait(pid)
+  end
+end if `which curl 2>/dev/null`.strip =~ /curl/ &&
+       `which md5sum 2>/dev/null`.strip =~ /md5sum/