yahns 0.0.0 → 0.0.1

data/lib/yahns/server.rb

@@ -20,6 +20,8 @@ class Yahns::Server # :nodoc:
     @pid = nil
     @worker_processes = nil
     @user = nil
+    @queues = []
+    @thr = []
   end
 
   def sqwakeup(sig)
@@ -173,19 +175,14 @@ class Yahns::Server # :nodoc:
       end
     end
 
-    @reexec_pid = fork do
-      redirects = {}
-      listeners.each do |sock|
-        sock.close_on_exec = false
-        redirects[sock.fileno] = sock
-      end
-      ENV['YAHNS_FD'] = redirects.keys.map(&:to_s).join(',')
-      Dir.chdir(@config.value(:working_directory) || Yahns::START[:cwd])
-      cmd = [ Yahns::START[0] ].concat(Yahns::START[:argv])
-      @logger.info "executing #{cmd.inspect} (in #{Dir.pwd})"
-      cmd << redirects
-      exec(*cmd)
-    end
+    opts = {}
+    @listeners.each { |sock| opts[sock.fileno] = sock }
+    env = { "YAHNS_FD" => opts.keys.map(&:to_s).join(',') }
+    opts[:chdir] = @config.value(:working_directory) || Yahns::START[:cwd]
+    cmd = [ Yahns::START[0] ].concat(Yahns::START[:argv])
+    @logger.info "spawning #{cmd.inspect} (in #{opts[:chdir]})"
+    cmd << opts
+    @reexec_pid = Process.spawn(env, *cmd)
     proc_name 'master (old)'
   end
 
@@ -267,21 +264,25 @@ class Yahns::Server # :nodoc:
 
     # initialize queues (epoll/kqueue) and associated worker threads
     queues = {}
-    @config.qeggs.each do |name, qegg|
-      queue = qegg.qc_vivify(fdmap) # worker threads run after this
-      queues[qegg] = queue
+    @config.qeggs.each do |name, qe|
+      queue = qe.vivify(fdmap)
+      qe.worker_threads.times do
+        @thr << queue.worker_thread(@logger, qe.max_events)
+      end
+      @queues << queue
+      queues[qe] = queue
     end
 
     # spin up applications (which are preload: false)
     @config.app_ctx.each { |ctx| ctx.after_fork_init }
 
-    # spin up acceptors, clients flow into worker queues after this
+    # spin up acceptor threads, clients flow into worker queues after this
     @listeners.each do |l|
       ctx = sock_opts(l)[:yahns_app_ctx]
       qegg = ctx.qegg || @config.qeggs[:default]
 
       # acceptors feed the the queues
-      l.spawn_acceptor(@logger, ctx, queues[qegg])
+      @thr << l.spawn_acceptor(@logger, ctx, queues[qegg])
     end
     fdmap
   end
@@ -293,7 +294,7 @@ class Yahns::Server # :nodoc:
   end
 
   def quit_enter(alive)
-    self.listeners = []
+    self.listeners = [] # close acceptors, we close epolls in quit_done
     exit(0) unless alive # drop connections immediately if signaled twice
     @config.config_listeners.each_value do |opts|
       ctx = opts[:yahns_app_ctx] or next
@@ -302,6 +303,13 @@ class Yahns::Server # :nodoc:
     false
   end
 
+  # drops all the the IO objects we have threads waiting on before exiting
+  def quit_finish
+    @queues.each(&:close)
+    self.listeners = [] # just in case, this is used in ensure
+    @thr.each(&:join)
+  end
+
   def sp_sig_handle(alive)
     @sev.kgio_wait_readable(alive ? nil : 0.01)
     @sev.yahns_step
@@ -332,5 +340,7 @@ class Yahns::Server # :nodoc:
       Yahns::Log.exception(@logger, "main loop", e)
     end while alive || fdmap.size > 0
     unlink_pid_safe(@pid) if @pid
+  ensure
+    quit_finish
   end
 end

data/lib/yahns/server_mp.rb

@@ -150,7 +150,7 @@ module Yahns::ServerMP # :nodoc:
 
   def fdmap_init_mp
     fdmap = fdmap_init # builds apps (if not preloading)
-    EXIT_SIGS.each { |sig| trap(sig) { sqwakeup(sig) } }
+    [:USR1, *EXIT_SIGS].each { |sig| trap(sig) { sqwakeup(sig) } }
     @config.postfork_cleanup # reduce live objects
     fdmap
   end
@@ -164,6 +164,8 @@ module Yahns::ServerMP # :nodoc:
       Yahns::Log.exception(@logger, "main worker loop", e)
     end while alive || fdmap.size > 0
     exit
+  ensure
+    quit_finish
   end
 
   def mp_sig_handle(worker, alive)

data/lib/yahns/socket_helper.rb

@@ -5,6 +5,7 @@
 module Yahns::SocketHelper # :nodoc:
   def set_server_sockopt(sock, opt)
     opt = {backlog: 1024}.merge!(opt) if opt
+    sock.close_on_exec = true
 
     TCPSocket === sock and sock.setsockopt(:IPPROTO_TCP, :TCP_NODELAY, 1)
     sock.setsockopt(:SOL_SOCKET, :SO_KEEPALIVE, 1)

data/lib/yahns/stream_input.rb

@@ -60,7 +60,7 @@ class Yahns::StreamInput # :nodoc:
   end
 
   def __rsize
-    @client.class.client_body_buffer_size
+    @client ? @client.class.client_body_buffer_size : nil
   end
 
   # :call-seq:
@@ -79,7 +79,7 @@ class Yahns::StreamInput # :nodoc:
       return rv.empty? ? nil : rv
     end
     re = /\A(.*?#{Regexp.escape(sep)})/
-    rsize = __rsize
+    rsize = __rsize or return
     begin
       @rbuf.sub!(re, '') and return $1
       return @rbuf.empty? ? nil : @rbuf.slice!(0, @rbuf.size) if eof?
@@ -124,8 +124,7 @@ class Yahns::StreamInput # :nodoc:
 
   def read_all(dst)
     dst.replace(@rbuf)
-    @client or return
-    rsize = @client.class.client_body_buffer_size
+    rsize = __rsize or return
     until eof?
       @client.kgio_read(rsize, @buf) or eof!
       filter_body(@rbuf, @buf)

data/lib/yahns/wbuf_common.rb

@@ -24,7 +24,7 @@ module Yahns::WbufCommon # :nodoc:
     @body.close if @body.respond_to?(:close)
     if @wbuf_persist.respond_to?(:call) # hijack
       @wbuf_persist.call(client)
-      :delete
+      :ignore
     else
       @wbuf_persist # true or false or Yahns::StreamFile
     end

data/test/helper.rb

@@ -1,6 +1,7 @@
 # Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> and all contributors
 # License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
 $stdout.sync = $stderr.sync = Thread.abort_on_exception = true
+$-w = true if RUBY_VERSION.to_f >= 2.0
 require 'thread'
 
 # Global Test Lock, to protect:
@@ -11,69 +12,68 @@ GTL = Mutex.new
 if ENV["COVERAGE"]
   require "coverage"
   COVMATCH = %r{/lib/yahns\b.*rb\z}
-  COVTMP = File.open("coverage.dump", IO::CREAT|IO::RDWR)
-  COVTMP.binmode
-  COVTMP.sync = true
 
   def __covmerge
     res = Coverage.result
 
-    # we own this file (at least until somebody tries to use NFS :x)
-    COVTMP.flock(File::LOCK_EX)
-
-    COVTMP.rewind
-    prev = COVTMP.read
-    prev = prev.empty? ? {} : Marshal.load(prev)
-    res.each do |filename, counts|
-      # filter out stuff that's not in our project
-      COVMATCH =~ filename or next
-
-      merge = prev[filename] || []
-      merge = merge
-      counts.each_with_index do |count, i|
-        count or next
-        merge[i] = (merge[i] || 0) + count
+    # do not create the file, Makefile does htis before any tests run
+    File.open("coverage.dump", IO::RDWR) do |covtmp|
+      covtmp.binmode
+      covtmp.sync = true
+
+      # we own this file (at least until somebody tries to use NFS :x)
+      covtmp.flock(File::LOCK_EX)
+
+      prev = covtmp.read
+      prev = prev.empty? ? {} : Marshal.load(prev)
+      res.each do |filename, counts|
+        # filter out stuff that's not in our project
+        COVMATCH =~ filename or next
+
+        merge = prev[filename] || []
+        merge = merge
+        counts.each_with_index do |count, i|
+          count or next
+          merge[i] = (merge[i] || 0) + count
+        end
+        prev[filename] = merge
       end
-      prev[filename] = merge
+      covtmp.rewind
+      covtmp.truncate(0)
+      covtmp.write(Marshal.dump(prev))
+      covtmp.flock(File::LOCK_UN)
     end
-    COVTMP.rewind
-    COVTMP.truncate(0)
-    COVTMP.write(Marshal.dump(prev))
-    COVTMP.flock(File::LOCK_UN)
   end
 
   Coverage.start
+  # we need to nest at_exit to fire after minitest runs
   at_exit { at_exit { __covmerge } }
 end
 
 gem 'minitest'
-require 'minitest/autorun'
-require "tempfile"
-
-Testcase = begin
-  Minitest::Test # minitest 5
-rescue NameError
-  Minitest::Unit::TestCase # minitest 4
+begin # favor minitest 5
+  require 'minitest'
+  Testcase = Minitest::Test
+  mtobj = Minitest
+rescue NameError, LoadError # but support minitest 4
+  require 'minitest/unit'
+  Testcase = Minitest::Unit::TestCase
+  mtobj = MiniTest::Unit.new
 end
 
-FIFOS = []
-def tmpfifo
-  tmp = Tempfile.new(%w(yahns-test .fifo))
-  path = tmp.path
-  tmp.close!
-  assert system(*%W(mkfifo #{path})), "mkfifo #{path}"
-
-  GTL.synchronize do
-    if FIFOS.empty?
-      at_exit do
-        FIFOS.each { |(pid,_path)| File.unlink(_path) if $$ == pid }
-      end
-    end
-    FIFOS << [ $$, path ]
+# Not using minitest/autorun because that doesn't guard against redundant
+# extra runs with fork.  We cannot use exit! in the tests either
+# (since users/apps hosted on yahns _should_ expect exit, not exit!).
+TSTART_PID = $$
+at_exit do
+  # skipping @@after_run stuff in minitest since we don't need it
+  case $!
+  when nil, SystemExit
+    mtobj.run(ARGV) if $$ == TSTART_PID
   end
-  path
 end
 
+require "tempfile"
 require 'tmpdir'
 class Dir
   require 'fileutils'

data/test/server_helper.rb

@@ -9,8 +9,9 @@ module ServerHelper
   def check_err(err = @err)
     err = File.open(err.path, "r") if err.respond_to?(:path)
     err.rewind
-    lines = err.readlines.delete_if { |l| l =~ /INFO/ }
-    assert lines.empty?, lines.join("\n")
+    lines = err.readlines
+    bad_lines = lines.dup.delete_if { |l| l =~ /INFO/ }
+    assert bad_lines.empty?, lines.join("\n")
     err.close! if err == @err
   end
 

data/test/test_bin.rb

@@ -47,6 +47,7 @@ class TestBin < Testcase
         ENV["YAHNS_FD"] = @srv.fileno.to_s
       else
         @srv = TCPServer.new(ENV["TEST_HOST"] || "127.0.0.1", 0)
+        @srv.close_on_exec = true # needed for 1.9.3
       end
       host, port = @srv.addr[3], @srv.addr[1]
       listen = ENV["YAHNS_TEST_LISTEN"] = "#{host}:#{port}"
@@ -94,4 +95,99 @@ class TestBin < Testcase
     end
     @pid.close! if @pid
   end
+
+  def test_usr2_preload_noworker; usr2(true, false); end
+  def test_usr2_preload_worker; usr2(true, true); end
+  def test_usr2_nopreload_worker; usr2(false, true); end
+  def test_usr2_nopreload_noworker; usr2(false, false); end
+
+  def usr2(preload, worker)
+    Dir.mktmpdir { |tmpdir| usr2_dir(tmpdir, preload, worker) }
+  end
+
+  def usr2_dir(tmpdir, preload, worker)
+    exe = "#{tmpdir}/yahns"
+
+    # need to fork here since tests are MT and the FD can leak out and go to
+    # other processes which fork (but do not exec), causing ETXTBUSY on
+    # Process.spawn
+    pid = fork do
+      ruby = "#!#{`which ruby`}"
+      File.open(exe, "w") { |y|
+        lines = File.readlines("bin/yahns")
+        lines[0] = ruby
+        y.chmod(0755)
+        y.syswrite(lines.join)
+      }
+    end
+    _, status = Process.waitpid2(pid)
+    assert status.success?, status.inspect
+
+    @pid = tmpfile(%w(test_bin_daemon .pid))
+    host, port = @srv.addr[3], @srv.addr[1]
+    @ru = tmpfile(%w(test_bin_daemon .ru))
+    @ru.puts("use Rack::ContentLength")
+    @ru.puts("use Rack::ContentType, 'text/plain'")
+    @ru.puts("run lambda { |_| [ 200, {}, [ Process.pid.to_s ] ] }")
+    cfg = tmpfile(%w(test_bin_daemon_conf .rb))
+    cfg.puts "pid '#{@pid.path}'"
+    cfg.puts "stderr_path '#{@err.path}'"
+    cfg.puts "worker_processes 1" if worker
+    cfg.puts "app(:rack, '#{@ru.path}', preload: #{preload}) do"
+    cfg.puts "  listen '#{host}:#{port}'"
+    cfg.puts "end"
+    env = {
+      "YAHNS_FD" => @srv.fileno.to_s,
+      "PATH" => "#{tmpdir}:#{ENV['PATH']}",
+      "RUBYLIB" => "#{Dir.pwd}/lib",
+    }
+    cmd = %W(#{exe} -D -c #{cfg.path})
+    cmd << { @srv => @srv, close_others: true }
+    pid = GTL.synchronize { Process.spawn(env, *cmd) }
+    res = Net::HTTP.start(host, port) { |h| h.get("/") }
+    assert_equal 200, res.code.to_i
+    orig = res.body
+    Process.kill(:USR2, pid)
+    newpid = pid
+    Timeout.timeout(10) do
+      begin
+        newpid = File.read(@pid.path)
+      rescue Errno::ENOENT
+      end while newpid.to_i == pid && sleep(0.01)
+    end
+    Process.kill(:QUIT, pid)
+    _, status = Timeout.timeout(10) { Process.waitpid2(pid) }
+    assert status.success?, status
+    res = Net::HTTP.start(host, port) { |h| h.get("/") }
+    assert_equal 200, res.code.to_i
+    second = res.body
+    refute_equal orig, second
+
+    newpid = newpid.to_i
+    assert_operator newpid, :>, 0
+    Process.kill(:HUP, newpid)
+    third = second
+    Timeout.timeout(10) do
+      begin
+        third = Net::HTTP.start(host, port) { |h| h.get("/") }.body
+      end while third == second && sleep(0.01)
+    end
+    if worker
+      Process.kill(0, newpid) # nothing should raise
+    else
+      poke_until_dead newpid
+    end
+  rescue => e
+    Yahns::Log.exception(Logger.new($stderr), "test", e)
+    raise
+  ensure
+    File.unlink(exe) if exe
+    cfg.close! if cfg
+    pid = File.read(@pid.path)
+    pid = pid.to_i
+    assert_operator pid, :>, 0
+    Process.kill(:QUIT, pid)
+    poke_until_dead pid
+    @pid.close!
+  end
 end

data/test/test_client_max_body_size.rb

@@ -0,0 +1,163 @@
+# Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> and all contributors
+# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
+require_relative 'server_helper'
+
+class TestClientMaxBodySize < Testcase
+  parallelize_me!
+  include ServerHelper
+  alias setup server_helper_setup
+  alias teardown server_helper_teardown
+  DEFMBS = 1024 * 1024
+
+  DRAINER = lambda do |e|
+    input = e["rack.input"]
+    buf = ""
+    nr = 0
+    while rv = input.read(16384, buf)
+      nr += rv.size
+    end
+    body = nr.to_s
+    h = { "Content-Length" => body.size.to_s, "Content-Type" => 'text/plain' }
+    [ 200, h, [body] ]
+  end
+
+  def identity_req(bytes, body = true)
+    body_bytes = body ? bytes : 0
+    "PUT / HTTP/1.1\r\nConnection: close\r\nHost: example.com\r\n" \
+    "Content-Length: #{bytes}\r\n\r\n#{'*' * body_bytes}"
+  end
+
+  def mkserver(cfg)
+    fork do
+      srv = Yahns::Server.new(cfg)
+      ENV["YAHNS_FD"] = @srv.fileno.to_s
+      srv.start.join
+    end
+  end
+
+  def test_0_lazy; cmbs_test_0(:lazy); end
+  def test_0_true; cmbs_test_0(true); end
+  def test_0_false; cmbs_test_0(false); end
+
+  def cmbs_test_0(btype)
+    err = @err
+    cfg = Yahns::Config.new
+    host, port = @srv.addr[3], @srv.addr[1]
+    cfg.instance_eval do
+      GTL.synchronize {
+        app(:rack, DRAINER) {
+          listen "#{host}:#{port}"
+          input_buffering btype
+          client_max_body_size 0
+        }
+      }
+      logger(Logger.new(err.path))
+    end
+    pid = mkserver(cfg)
+    default_identity_checks(host, port, 0)
+    default_chunked_checks(host, port, 0)
+  ensure
+    quit_wait(pid)
+  end
+
+  def test_cmbs_lazy; cmbs_test(:lazy); end
+  def test_cmbs_true; cmbs_test(true); end
+  def test_cmbs_false; cmbs_test(false); end
+
+  def cmbs_test(btype)
+    err = @err
+    cfg = Yahns::Config.new
+    host, port = @srv.addr[3], @srv.addr[1]
+    cfg.instance_eval do
+      GTL.synchronize {
+        app(:rack, DRAINER) {
+          listen "#{host}:#{port}"
+          input_buffering btype
+        }
+      }
+      logger(Logger.new(err.path))
+    end
+    pid = mkserver(cfg)
+    default_identity_checks(host, port)
+    default_chunked_checks(host, port)
+  ensure
+    quit_wait(pid)
+  end
+
+  def test_inf_false; big_test(false); end
+  def test_inf_true; big_test(true); end
+  def test_inf_lazy; big_test(:lazy); end
+
+  def big_test(btype)
+    err = @err
+    cfg = Yahns::Config.new
+    host, port = @srv.addr[3], @srv.addr[1]
+    cfg.instance_eval do
+      GTL.synchronize {
+        app(:rack, DRAINER) {
+          listen "#{host}:#{port}"
+          input_buffering btype
+          client_max_body_size nil
+        }
+      }
+      logger(Logger.new(err.path))
+    end
+    pid = mkserver(cfg)
+
+    bytes = 10 * 1024 * 1024
+    r = `dd if=/dev/zero bs=#{bytes} count=1 2>/dev/null | \
+         curl -sSf -HExpect: -T- http://#{host}:#{port}/`
+    assert $?.success?, $?.inspect
+    assert_equal bytes.to_s, r
+
+    r = `dd if=/dev/zero bs=#{bytes} count=1 2>/dev/null | \
+         curl -sSf -HExpect: -HContent-Length:#{bytes} -HTransfer-Encoding: \
+         -T- http://#{host}:#{port}/`
+    assert $?.success?, $?.inspect
+    assert_equal bytes.to_s, r
+  ensure
+    quit_wait(pid)
+  end
+
+  def default_chunked_checks(host, port, defmax = DEFMBS)
+    r = `curl -sSf -HExpect: -T- </dev/null http://#{host}:#{port}/`
+    assert $?.success?, $?.inspect
+    assert_equal "0", r
+
+    r = `dd if=/dev/zero bs=#{defmax} count=1 2>/dev/null | \
+         curl -sSf -HExpect: -T- http://#{host}:#{port}/`
+    assert $?.success?, $?.inspect
+    assert_equal "#{defmax}", r
+
+    r = `dd if=/dev/zero bs=#{defmax + 1} count=1 2>/dev/null | \
+         curl -sf -HExpect: -T- --write-out %{http_code} \
+         http://#{host}:#{port}/ 2>&1`
+    refute $?.success?, $?.inspect
+    assert_equal "413", r
+  end
+
+  def default_identity_checks(host, port, defmax = DEFMBS)
+    if defmax >= 666
+      c = TCPSocket.new(host, port)
+      c.write(identity_req(666))
+      assert_equal "666", c.read.split(/\r\n\r\n/)[1]
+      c.close
+    end
+
+    c = TCPSocket.new(host, port)
+    c.write(identity_req(0))
+    assert_equal "0", c.read.split(/\r\n\r\n/)[1]
+    c.close
+
+    c = TCPSocket.new(host, port)
+    c.write(identity_req(defmax))
+    assert_equal "#{defmax}", c.read.split(/\r\n\r\n/)[1]
+    c.close
+
+    toobig = defmax + 1
+    c = TCPSocket.new(host, port)
+    c.write(identity_req(toobig, false))
+    assert_match(%r{\AHTTP/1\.[01] 413 }, Timeout.timeout(10) { c.read })
+    c.close
+  end
+end