yahns 0.0.1 → 0.0.2

data/lib/yahns/log.rb

@@ -41,9 +41,9 @@ module Yahns::Log # :nodoc:
     ObjectSpace.each_object(File) { |fp| is_log?(fp) and to_reopen << fp }
 
     to_reopen.each do |fp|
-      orig_st = begin
-        fp.stat
-      rescue IOError, Errno::EBADF
+      begin
+        orig_st = fp.stat
+      rescue IOError, Errno::EBADF # race
         next
       end
 
@@ -54,8 +54,27 @@ module Yahns::Log # :nodoc:
       end
 
       begin
-        File.open(fp.path, 'a') { |tmpfp| fp.reopen(tmpfp) }
+        # stdin, stdout, stderr are special.  The following dance should
+        # guarantee there is no window where `fp' is unwritable in MRI
+        # (or any correct Ruby implementation).
+        #
+        # Fwiw, GVL has zero bearing here.  This is tricky because of
+        # the unavoidable existence of stdio FILE * pointers for
+        # std{in,out,err} in all programs which may use the standard C library
+        if fp.fileno <= 2
+          # We do not want to hit fclose(3)->dup(2) window for std{in,out,err}
+          # MRI will use freopen(3) here internally on std{in,out,err}
+          fp.reopen(fp.path, "a")
+        else
+          # We should not need this: http://bugs.ruby-lang.org/issues/9036
+          # MRI will not call call fclose(3) or freopen(3) here
+          # since there's no associated std{in,out,err} FILE * pointer
+          # This should atomically use dup3(2) (or dup2(2)) syscall
+          File.open(fp.path, "a") { |tmpfp| fp.reopen(tmpfp) }
+        end
+
         fp.sync = true
+        fp.flush # IO#sync=true may not implicitly flush
         new_st = fp.stat
 
         # this should only happen in the master:

data/lib/yahns/queue_epoll.rb

@@ -6,6 +6,7 @@ class Yahns::Queue < SleepyPenguin::Epoll::IO # :nodoc:
   attr_accessor :fdmap # Yahns::Fdmap
 
   # public
+  QEV_QUIT = Epoll::OUT # Level Trigger for QueueQuitter
   QEV_RD = Epoll::IN | Epoll::ONESHOT
   QEV_WR = Epoll::OUT | Epoll::ONESHOT
   QEV_RDWR = QEV_RD | QEV_WR
@@ -21,10 +22,21 @@ class Yahns::Queue < SleepyPenguin::Epoll::IO # :nodoc:
     epoll_ctl(Epoll::CTL_ADD, io, flags)
   end
 
+  def thr_init
+    Thread.current[:yahns_rbuf] = ""
+    Thread.current[:yahns_fdmap] = @fdmap
+  end
+
+  # use only before hijacking, once hijacked, io may be unusable to us
+  def queue_del(io)
+    @fdmap.forget(io)
+    epoll_ctl(Epoll::CTL_DEL, io, 0)
+  end
+
   # returns an array of infinitely running threads
   def worker_thread(logger, max_events)
     Thread.new do
-      Thread.current[:yahns_rbuf] = ""
+      thr_init
       begin
         epoll_wait(max_events) do |_, io| # don't care for flags for now
           case rv = io.yahns_step
@@ -35,19 +47,18 @@ class Yahns::Queue < SleepyPenguin::Epoll::IO # :nodoc:
           when :wait_readwrite
             epoll_ctl(Epoll::CTL_MOD, io, QEV_RDWR)
           when :ignore # only used by rack.hijack
-            @fdmap.decr
-          when nil
-            # this is be the ONLY place where we call IO#close on
-            # things inside the queue
-            io.close
-            @fdmap.decr
+            # we cannot call Epoll::CTL_DEL after hijacking, the hijacker
+            # may have already closed it  Likewise, io.fileno is not
+            # expected to work, so we had to erase it from fdmap before hijack
+          when nil, :close
+            # this must be the ONLY place where we call IO#close on
+            # things that got inside the queue
+            @fdmap.sync_close(io)
           else
             raise "BUG: #{io.inspect}#yahns_step returned: #{rv.inspect}"
           end
         end
       rescue => e
-        # sleep since this check is racy (and uncommon)
-        break if closed? || (sleep(0.01) && closed?)
         Yahns::Log.exception(logger, 'queue loop', e)
       end while true
     end

data/lib/yahns/queue_quitter.rb

@@ -0,0 +1,16 @@
+# Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> and all contributors
+# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
+
+require 'sleepy_penguin'
+
+# add this as a level-triggered to any thread pool stuck on epoll_wait
+# and watch it die!
+if SleepyPenguin.const_defined?(:EventFD)
+  class Yahns::QueueQuitter < Yahns::Sigevent # :nodoc:
+    def yahns_step
+      Thread.current.exit
+    end
+  end
+else
+  require_relative 'queue_quitter_pipe'
+end

data/lib/yahns/queue_quitter_pipe.rb

@@ -0,0 +1,24 @@
+# Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> and all contributors
+# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
+#
+# POSIX pipe version, see queue_quitter.rb for the (preferred) eventfd one
+class Yahns::QueueQuitter # :nodoc:
+  attr_reader :to_io
+  def initialize
+    reader, @to_io = IO.pipe
+    @to_io.close_on_exec = true
+    reader.close
+  end
+
+  def yahns_step
+    Thread.current.exit
+  end
+
+  def fileno
+    @to_io.fileno
+  end
+
+  def close
+    @to_io.close
+  end
+end

data/lib/yahns/rack.rb

@@ -18,7 +18,6 @@ class Yahns::Rack # :nodoc:
   def initialize(ru, opts = {})
     # always called after config file parsing, may be called after forking
     @app = lambda do
-      ENV["RACK_ENV"] ||= "none"
       if ru.respond_to?(:call)
         inner_app = ru.respond_to?(:to_app) ? ru.to_app : ru
       else

data/lib/yahns/rackup_handler.rb

@@ -0,0 +1,57 @@
+# Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> and all contributors
+# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
+require 'yahns'
+
+module Yahns::RackupHandler # :nodoc:
+  def self.default_host
+    environment  = ENV['RACK_ENV'] || 'development'
+    environment == 'development' ? '127.0.0.1' : '0.0.0.0'
+  end
+
+  def self.run(app, o)
+    cfg = Yahns::Config.new
+    cfg.instance_eval do
+      # we need this because "rackup -D" sends us to "/", which might be
+      # fine for most apps, but we have SIGUSR2 restarts to support
+      working_directory(Yahns::START[:cwd])
+
+      app(:rack, app) do
+        addr = o[:listen] || "#{o[:Host]||default_host}:#{o[:Port]||8080}"
+        # allow listening to multiple addresses
+        if addr =~ /,/
+          addr.split(/,/).each { |l| listen(l) }
+        else
+          listen addr
+        end
+
+        val = o[:client_timeout] and client_timeout(val)
+      end
+
+      queue do
+        wt = o[:worker_threads] and worker_threads(wt)
+      end
+
+      %w(stderr_path stdout_path).each do |x|
+        x = x.to_sym
+        val = o[x] and __send__(x, val)
+      end
+    end
+    Yahns::Server.new(cfg).start.join
+  end
+
+  def self.valid_options
+    # these should be the most common options
+    {
+      "listen=ADDRESS" => "address(es) to listen on (e.g. /tmp/sock)",
+      "worker_threads=NUM" => "number of worker threads to run",
+
+      # this affects how quickly graceful shutdown goes
+      "client_timeout=SECONDS" => "timeout for idle clients",
+
+      # I don't want these here, but rackup supports daemonize and
+      # we lose useful information when that sends stdout/stderr to /dev/null
+      "stderr_path=PATH" => "stderr destination",
+      "stdout_path=PATH" => "stdout destination",
+    }
+  end
+end

data/lib/yahns/server.rb

@@ -1,14 +1,26 @@
 # -*- encoding: binary -*-
 # Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> and all contributors
 # License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
+require_relative 'queue_quitter'
+require_relative 'tcp_server'
+require_relative 'unix_server'
+
 class Yahns::Server # :nodoc:
-  QUEUE_SIGS = [ :WINCH, :QUIT, :INT, :TERM, :USR1, :USR2, :HUP, :TTIN, :TTOU ]
+  QUEUE_SIGS = [ :WINCH, :QUIT, :INT, :TERM, :USR1, :USR2, :HUP, :TTIN, :TTOU,
+                 :CHLD ]
   attr_accessor :daemon_pipe
   attr_accessor :logger
+  attr_writer :user
+  attr_writer :before_exec
   attr_writer :worker_processes
+  attr_writer :shutdown_timeout
+  attr_writer :atfork_prepare
+  attr_writer :atfork_parent
+  attr_writer :atfork_child
   include Yahns::SocketHelper
 
   def initialize(config)
+    @shutdown_timeout = nil
     @reexec_pid = 0
     @daemon_pipe = nil # writable IO or true
     @config = config
@@ -19,9 +31,11 @@ class Yahns::Server # :nodoc:
     @listeners = []
     @pid = nil
     @worker_processes = nil
+    @before_exec = nil
+    @atfork_prepare = @atfork_parent = @atfork_child = nil
     @user = nil
     @queues = []
-    @thr = []
+    @wthr = []
   end
 
   def sqwakeup(sig)
@@ -34,28 +48,40 @@ class Yahns::Server # :nodoc:
     inherit_listeners!
     # we try inheriting listeners first, so we bind them later.
     # we don't write the pid file until we've bound listeners in case
-    # yahns was started twice by mistake.  Even though our #pid= method
-    # checks for stale/existing pid files, race conditions are still
-    # possible (and difficult/non-portable to avoid) and can be likely
-    # to clobber the pid if the second start was in quick succession
-    # after the first, so we rely on the listener binding to fail in
-    # that case.  Some tests (in and outside of this source tree) and
-    # monitoring tools may also rely on pid files existing before we
-    # attempt to connect to the listener(s)
+    # yahns was started twice by mistake.
 
     # setup signal handlers before writing pid file in case people get
     # trigger happy and send signals as soon as the pid file exists.
     QUEUE_SIGS.each { |sig| trap(sig) { sqwakeup(sig) } }
-    self.pid = @config.value(:pid) # write pid file
     bind_new_listeners!
+    self.pid = @config.value(:pid) # write pid file
     if @worker_processes
       require 'yahns/server_mp'
       extend Yahns::ServerMP
-      mp_init
+    else
+      switch_user(*@user) if @user
     end
     self
   end
 
+  def switch_user(user, group = nil)
+    # we do not protect the caller, checking Process.euid == 0 is
+    # insufficient because modern systems have fine-grained
+    # capabilities.  Let the caller handle any and all errors.
+    uid = Etc.getpwnam(user).uid
+    gid = Etc.getgrnam(group).gid if group
+    Yahns::Log.chown_all(uid, gid)
+    if gid && Process.egid != gid
+      Process.initgroups(user, gid)
+      Process::GID.change_privilege(gid)
+    end
+    Process.euid != uid and Process::UID.change_privilege(uid)
+  end
+
+  def drop_acceptors
+    @listeners.delete_if(&:ac_quit)
+  end
+
   # replaces current listener set with +listeners+.  This will
   # close the socket if it will not exist in the new listener set
   def listeners=(listeners)
@@ -70,19 +96,41 @@ class Yahns::Server # :nodoc:
     end
     set_names = listener_names(listeners)
     dead_names.concat(cur_names - set_names).uniq!
-
+    dying = []
     @listeners.delete_if do |io|
       if dead_names.include?(sock_name(io))
-        (io.close rescue nil).nil? # true
+        if io.ac_quit
+          true
+        else
+          dying << io
+          false
+        end
       else
         set_server_sockopt(io, sock_opts(io))
         false
       end
     end
 
+    dying.delete_if(&:ac_quit) while dying[0]
+
     (set_names - cur_names).each { |addr| listen(addr) }
   end
 
+  def clobber_pid(path)
+    unlink_pid_safe(@pid) if @pid
+    if path
+      fp = begin
+        tmp = "#{File.dirname(path)}/#{rand}.#$$"
+        File.open(tmp, File::RDWR|File::CREAT|File::EXCL, 0644)
+      rescue Errno::EEXIST
+        retry
+      end
+      fp.syswrite("#$$\n")
+      File.rename(fp.path, path)
+      fp.close
+    end
+  end
+
   # sets the path for the PID file of the master process
   def pid=(path)
     if path
@@ -97,18 +145,18 @@ class Yahns::Server # :nodoc:
                              "(or pid=#{path} is stale)"
       end
     end
-    unlink_pid_safe(@pid) if @pid
 
-    if path
-      fp = begin
-        tmp = "#{File.dirname(path)}/#{rand}.#$$"
-        File.open(tmp, File::RDWR|File::CREAT|File::EXCL, 0644)
-      rescue Errno::EEXIST
-        retry
+    # rename the old pid if possible
+    if @pid && path
+      begin
+        File.rename(@pid, path)
+      rescue Errno::ENOENT, Errno::EXDEV
+        # a user may have accidentally removed the original,
+        # obviously cross-FS renames don't work, either.
+        clobber_pid(path)
       end
-      fp.syswrite("#$$\n")
-      File.rename(fp.path, path)
-      fp.close
+    else
+      clobber_pid(path)
     end
     @pid = path
   end
@@ -125,17 +173,27 @@ class Yahns::Server # :nodoc:
   def listen(address)
     address = @config.expand_addr(address)
     return if String === address && listener_names.include?(address)
+    delay = 0.5
+    tries = 5
 
     begin
       io = bind_listen(address, sock_opts(address))
-      unless Kgio::TCPServer === io || Kgio::UNIXServer === io
+      unless Yahns::TCPServer === io || Yahns::UNIXServer === io
         io = server_cast(io)
       end
       @logger.info "listening on addr=#{sock_name(io)} fd=#{io.fileno}"
       @listeners << io
       io
     rescue Errno::EADDRINUSE => err
-      @logger.error "adding listener failed addr=#{address} (in use)"
+      if tries == 0
+        @logger.error "adding listener failed addr=#{address} (in use)"
+        raise err
+      end
+      tries -= 1
+      @logger.warn "retrying in #{delay} seconds " \
+                   "(#{tries < 0 ? 'infinite' : tries} tries left)"
+      sleep(delay)
+      retry
     rescue => err
       @logger.fatal "error adding listener addr=#{address}"
       raise err
@@ -175,15 +233,25 @@ class Yahns::Server # :nodoc:
       end
     end
 
-    opts = {}
-    @listeners.each { |sock| opts[sock.fileno] = sock }
-    env = { "YAHNS_FD" => opts.keys.map(&:to_s).join(',') }
-    opts[:chdir] = @config.value(:working_directory) || Yahns::START[:cwd]
-    cmd = [ Yahns::START[0] ].concat(Yahns::START[:argv])
-    @logger.info "spawning #{cmd.inspect} (in #{opts[:chdir]})"
-    cmd << opts
-    @reexec_pid = Process.spawn(env, *cmd)
-    proc_name 'master (old)'
+    # We cannot use Process.spawn here because of redirects + close-on-exec
+    # We must keep close_on_exec=true in the parent process and only set
+    # close_on_exec=false in the child.  There must be no opportunity
+    # for the user app to ever get a listen socket with close_on_exec=false
+    @reexec_pid = fork do
+      redirects = {}
+      @listeners.each do |sock|
+        sock.close_on_exec = false
+        redirects[sock.fileno] = sock
+      end
+      ENV['YAHNS_FD'] = redirects.keys.join(',')
+      redirects[:close_others] = true
+      Dir.chdir(@config.value(:working_directory) || Yahns::START[:cwd])
+      cmd = [ Yahns::START[0] ].concat(Yahns::START[:argv])
+      @logger.info "executing #{cmd.inspect} (in #{Dir.pwd})"
+      @before_exec.call(cmd) if @before_exec
+      cmd << redirects
+      exec(*cmd)
+    end
   end
 
   # unlinks a PID file at given +path+ if it contains the current PID
@@ -231,7 +299,16 @@ class Yahns::Server # :nodoc:
 
   def inherit_listeners!
     # inherit sockets from parents, they need to be plain Socket objects
-    # before they become Kgio::UNIXServer or Kgio::TCPServer
+    # before they become Yahns::UNIXServer or Yahns::TCPServer
+    #
+    # Note: we intentionally use a yahns-specific environment variable
+    # here because existing servers may use non-blocking listen sockets.
+    # yahns uses _blocking_ listen sockets exclusively.  We cannot
+    # change an existing socket to blocking mode if two servers are
+    # running (one expecting blocking, one expecting non-blocking)
+    # because that can completely break the non-blocking one.
+    # Unfortunately, there is no one-off MSG_DONTWAIT-like flag for
+    # accept4(2).
     inherited = ENV['YAHNS_FD'].to_s.split(/,/).map do |fd|
       io = Socket.for_fd(fd.to_i)
       set_server_sockopt(io, sock_opts(io))
@@ -247,7 +324,6 @@ class Yahns::Server # :nodoc:
   def bind_new_listeners!
     self.listeners = @config.config_listeners.keys
     raise ArgumentError, "no listeners" if @listeners.empty?
-    @listeners.each { |l| l.extend(Yahns::Acceptor) }
   end
 
   def proc_name(tag)
@@ -255,6 +331,15 @@ class Yahns::Server # :nodoc:
     $0 = ([ File.basename(s[0]), tag ]).concat(s[:argv]).join(' ')
   end
 
+  def qegg_vivify(qegg, fdmap)
+    queue = qegg.vivify(fdmap)
+    qegg.worker_threads.times do
+      @wthr << queue.worker_thread(@logger, qegg.max_events)
+    end
+    @queues << queue
+    queue
+  end
+
   # spins up processing threads of the server
   def fdmap_init
     thresh = @config.value(:client_expire_threshold)
@@ -262,27 +347,26 @@ class Yahns::Server # :nodoc:
     # keeps track of all connections, like ObjectSpace, but only for IOs
     fdmap = Yahns::Fdmap.new(@logger, thresh)
 
-    # initialize queues (epoll/kqueue) and associated worker threads
+    # once initialize queues (epoll/kqueue) and associated worker threads
     queues = {}
-    @config.qeggs.each do |name, qe|
-      queue = qe.vivify(fdmap)
-      qe.worker_threads.times do
-        @thr << queue.worker_thread(@logger, qe.max_events)
-      end
-      @queues << queue
-      queues[qe] = queue
-    end
 
     # spin up applications (which are preload: false)
-    @config.app_ctx.each { |ctx| ctx.after_fork_init }
+    @config.app_ctx.each(&:after_fork_init)
+
+    @shutdown_timeout ||= @config.app_ctx.map(&:client_timeout).max
 
     # spin up acceptor threads, clients flow into worker queues after this
     @listeners.each do |l|
-      ctx = sock_opts(l)[:yahns_app_ctx]
+      opts = sock_opts(l)
+      ctx = opts[:yahns_app_ctx]
+      ctx_list = opts[:yahns_app_ctx_list] ||= []
       qegg = ctx.qegg || @config.qeggs[:default]
-
+      ctx.queue = queues[qegg] ||= qegg_vivify(qegg, fdmap)
+      ctx = ctx.dup
+      ctx.__send__(:include, l.expire_mod)
+      ctx_list << ctx
       # acceptors feed the the queues
-      @thr << l.spawn_acceptor(@logger, ctx, queues[qegg])
+      l.spawn_acceptor(opts[:threads] || 1, @logger, ctx)
     end
     fdmap
   end
@@ -294,20 +378,55 @@ class Yahns::Server # :nodoc:
   end
 
   def quit_enter(alive)
-    self.listeners = [] # close acceptors, we close epolls in quit_done
-    exit(0) unless alive # drop connections immediately if signaled twice
+    if alive
+      @logger.info("gracefully exiting shutdown_timeout=#{@shutdown_timeout} s")
+    else # drop connections immediately if signaled twice
+      @logger.info("graceful exit aborted, exiting immediately")
+      # we will still call any app-defined at_exit hooks here
+      # use SIGKILL if you don't want that.
+      exit(0)
+    end
+
+    drop_acceptors # stop acceptors, we close epolls in quit_done
     @config.config_listeners.each_value do |opts|
-      ctx = opts[:yahns_app_ctx] or next
-      ctx.persistent_connections = false # Yahns::HttpContext
+      list= opts[:yahns_app_ctx_list] or next
+      # Yahns::HttpContext#persistent_connections=
+      list.each { |ctx| ctx.persistent_connections = false }
     end
     false
   end
 
   # drops all the the IO objects we have threads waiting on before exiting
+  # This just injects the QueueQuitter object which acts like a
+  # monkey wrench thrown into a perfectly good engine :)
   def quit_finish
-    @queues.each(&:close)
-    self.listeners = [] # just in case, this is used in ensure
-    @thr.each(&:join)
+    quitter = Yahns::QueueQuitter.new
+
+    # throw the monkey wrench into the worker threads
+    @queues.each { |q| q.queue_add(quitter, Yahns::Queue::QEV_QUIT) }
+
+    # watch the monkey wrench destroy all the threads!
+    @wthr.delete_if { |t| t.join(0.01) } while @wthr[0]
+
+    # cleanup, our job is done
+    @queues.each(&:close).clear
+    quitter.close
+  rescue => e
+    Yahns::Log.exception(@logger, "quit finish", e)
+  ensure
+    if (@wthr.size + @listeners.size) > 0
+      @logger.error("BUG: still active wthr=#{@wthr.size} "\
+                    "listeners=#{@listeners.size}")
+    end
+  end
+
+  def reap_reexec
+    @reexec_pid > 0 or return
+    wpid, status = Process.waitpid2(@reexec_pid, Process::WNOHANG)
+    wpid or return
+    @logger.error "reaped #{status.inspect} exec()-ed"
+    @reexec_pid = 0
+    self.pid = @pid.chomp('.oldbin') if @pid
   end
 
   def sp_sig_handle(alive)
@@ -316,19 +435,30 @@ class Yahns::Server # :nodoc:
     case sig = @sig_queue.shift
     when :QUIT, :TERM, :INT
       return quit_enter(alive)
+    when :CHLD
+      reap_reexec
     when :USR1
       usr1_reopen('')
     when :USR2
       reexec
     when :HUP
       reexec
-      return false
+      return quit_enter(alive)
     when :TTIN, :TTOU, :WINCH
       @logger.info("SIG#{sig} ignored in single-process mode")
     end
     alive
   end
 
+  def dropping(fdmap)
+    if drop_acceptors[0] || fdmap.size > 0
+      fdmap.desperate_expire_for(nil, @shutdown_timeout)
+      true
+    else
+      false
+    end
+  end
+
   # single-threaded only, this is overriden if @worker_processes is non-nil
   def join
     daemon_ready
@@ -338,7 +468,7 @@ class Yahns::Server # :nodoc:
       alive = sp_sig_handle(alive)
     rescue => e
       Yahns::Log.exception(@logger, "main loop", e)
-    end while alive || fdmap.size > 0
+    end while alive || dropping(fdmap)
     unlink_pid_safe(@pid) if @pid
   ensure
     quit_finish