yahns 0.0.0TP1

data/lib/yahns/server_mp.rb

@@ -0,0 +1,181 @@
+# -*- encoding: binary -*-
+# Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> and all contributors
+# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
+module Yahns::ServerMP # :nodoc:
+  EXIT_SIGS = [ :QUIT, :TERM, :INT ]
+
+  def mp_init
+    trap(:CHLD) { @sev.sev_signal }
+  end
+
+  # reaps all unreaped workers
+  def reap_all_workers
+    begin
+      wpid, status = Process.waitpid2(-1, Process::WNOHANG)
+      wpid or return
+      if @reexec_pid == wpid
+        @logger.error "reaped #{status.inspect} exec()-ed"
+        @reexec_pid = 0
+        self.pid = @pid.chomp('.oldbin') if @pid
+        proc_name 'master'
+      else
+        worker = @workers.delete(wpid)
+        worker_id = worker ? worker.nr : "(unknown)"
+        m = "reaped #{status.inspect} worker=#{worker_id}"
+        status.success? ? @logger.info(m) : @logger.error(m)
+      end
+    rescue Errno::ECHILD
+      return
+    end while true
+  end
+
+  def maintain_worker_count
+    (off = @workers.size - @worker_processes) == 0 and return
+    off < 0 and return spawn_missing_workers
+    @workers.each_pair do |wpid, worker|
+      worker.nr >= @worker_processes and Process.kill(:QUIT, wpid)
+    end
+  end
+
+  # delivers a signal to each worker
+  def kill_each_worker(signal)
+    @workers.each_key { |wpid| Process.kill(signal, wpid) }
+  end
+
+  # this is the first thing that runs after forking in a child
+  # gets rid of stuff the worker has no business keeping track of
+  # to free some resources and drops all sig handlers.
+  # traps for USR1, USR2, and HUP may be set in the after_fork Proc
+  # by the user.
+  def after_fork_internal(worker)
+    worker.atfork_child
+
+    # daemon_pipe may be true for non-initial workers
+    @daemon_pipe = @daemon_pipe.close if @daemon_pipe.respond_to?(:close)
+
+    srand # in case this pops up again: https://bugs.ruby-lang.org/issues/4338
+
+    # The OpenSSL PRNG is seeded with only the pid, and apps with frequently
+    # dying workers can recycle pids
+    OpenSSL::Random.seed(rand.to_s) if defined?(OpenSSL::Random)
+    # we'll re-trap EXIT_SIGS later for graceful shutdown iff we accept clients
+    EXIT_SIGS.each { |sig| trap(sig) { exit!(0) } }
+    exit!(0) if (@sig_queue & EXIT_SIGS)[0] # did we inherit sigs from parent?
+    @sig_queue = []
+
+    # ignore WINCH, TTIN, TTOU, HUP in the workers
+    (Yahns::Server::QUEUE_SIGS - EXIT_SIGS).each { |sig| trap(sig, nil) }
+    trap(:CHLD, 'DEFAULT')
+    @logger.info("worker=#{worker.nr} spawned pid=#$$")
+    proc_name "worker[#{worker.nr}]"
+    Yahns::START.clear
+    @sev.close
+    @sev = Yahns::Sigevent.new
+    worker.user(*@user) if @user
+    @user = @workers = nil
+  end
+
+  def spawn_missing_workers
+    worker_nr = -1
+    until (worker_nr += 1) == @worker_processes
+      @workers.value?(worker_nr) and next
+      worker = Yahns::Worker.new(worker_nr)
+      @logger.info("worker=#{worker_nr} spawning...")
+      if pid = fork
+        @workers[pid] = worker.atfork_parent
+      else
+        after_fork_internal(worker)
+        run_mp_worker(worker)
+      end
+    end
+  rescue => e
+    Yahns::Log.exception(@logger, "spawning worker", e)
+    exit!
+  end
+
+  # monitors children and receives signals forever
+  # (or until a termination signal is sent).  This handles signals
+  # one-at-a-time time and we'll happily drop signals in case somebody
+  # is signalling us too often.
+  def join
+    spawn_missing_workers
+    state = :respawn # :QUIT, :WINCH
+    proc_name 'master'
+    @logger.info "master process ready"
+    daemon_ready
+    begin
+      @sev.kgio_wait_readable
+      @sev.yahns_step
+      reap_all_workers
+      case @sig_queue.shift
+      when *EXIT_SIGS # graceful shutdown (twice for non graceful)
+        self.listeners = []
+        kill_each_worker(:QUIT)
+        state = :QUIT
+      when :USR1 # rotate logs
+        usr1_reopen("master ")
+        kill_each_worker(:USR1)
+      when :USR2 # exec binary, stay alive in case something went wrong
+        reexec
+      when :WINCH
+        if @daemon_pipe
+          state = :WINCH
+          @logger.info "gracefully stopping all workers"
+          kill_each_worker(:QUIT)
+          @worker_processes = 0
+        else
+          @logger.info "SIGWINCH ignored because we're not daemonized"
+        end
+      when :TTIN
+        state = :respawn unless state == :QUIT
+        @worker_processes += 1
+      when :TTOU
+        @worker_processes -= 1 if @worker_processes > 0
+      when :HUP
+        state = :respawn unless state == :QUIT
+        if @config.config_file
+          load_config!
+        else # exec binary and exit if there's no config file
+          @logger.info "config_file not present, reexecuting binary"
+          reexec
+        end
+      end while @sig_queue[0]
+      maintain_worker_count if state == :respawn
+    rescue => e
+      Yahns::Log.exception(@logger, "master loop error", e)
+    end while state != :QUIT || @workers.size > 0
+    @logger.info "master complete"
+    unlink_pid_safe(@pid) if @pid
+  end
+
+  def fdmap_init_mp
+    fdmap = fdmap_init # builds apps (if not preloading)
+    EXIT_SIGS.each { |sig| trap(sig) { sqwakeup(sig) } }
+    @config.postfork_cleanup # reduce live objects
+    fdmap
+  end
+
+  def run_mp_worker(worker)
+    fdmap = fdmap_init_mp
+    alive = true
+    begin
+      alive = mp_sig_handle(worker, alive)
+    rescue => e
+      Yahns::Log.exception(@logger, "main worker loop", e)
+    end while alive || fdmap.size > 0
+    exit
+  end
+
+  def mp_sig_handle(worker, alive)
+    # not performance critical
+    r = IO.select([worker, @sev], nil, nil, alive ? nil : 0.01) and
+      r[0].each { |io| io.yahns_step }
+    case @sig_queue.shift
+    when *EXIT_SIGS
+      return quit_enter(alive)
+    when :USR1
+      usr1_reopen("worker ")
+    end
+    alive
+  end
+end

data/lib/yahns/sigevent.rb

@@ -0,0 +1,7 @@
+# Copyright (C) 2013, Eric Wong <normalperson@yhbt.net>
+# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
+if SleepyPenguin.const_defined?(:EventFD)
+  require_relative 'sigevent_efd'
+else
+  require_relative 'sigevent_pipe'
+end

data/lib/yahns/sigevent_efd.rb

@@ -0,0 +1,18 @@
+# -*- encoding: binary -*-
+# Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> and all contributors
+# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
+class Yahns::Sigevent < SleepyPenguin::EventFD # :nodoc:
+  include Kgio::DefaultWaiters
+  def self.new
+    super(0, SleepyPenguin::EventFD::CLOEXEC)
+  end
+
+  def sev_signal
+    incr(1) # eventfd_write
+  end
+
+  def yahns_step
+    value(true) # eventfd_read, we ignore this data
+    :wait_readable
+  end
+end

data/lib/yahns/sigevent_pipe.rb

@@ -0,0 +1,29 @@
+# -*- encoding: binary -*-
+# Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> and all contributors
+# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
+class Yahns::Sigevent # :nodoc:
+  attr_reader :to_io
+  def initialize
+    @to_io, @wr = Kgio::Pipe.new
+  end
+
+  def kgio_wait_readable
+    @to_io.kgio_wait_readable
+  end
+
+  def sev_signal
+    @wr.kgio_write(".")
+  end
+
+  def yahns_step
+    # 11 byte strings -> no malloc on YARV
+    while String === @to_io.kgio_tryread(11)
+    end
+    :wait_readable
+  end
+
+  def close
+    @to_io.close
+    @wr.close
+  end
+end

data/lib/yahns/socket_helper.rb

@@ -0,0 +1,117 @@
+# -*- encoding: binary -*-
+# Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> and all contributors
+# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
+# this is only meant for Yahns::Server
+module Yahns::SocketHelper # :nodoc:
+  def set_server_sockopt(sock, opt)
+    opt = {backlog: 1024}.merge!(opt) if opt
+
+    TCPSocket === sock and sock.setsockopt(:IPPROTO_TCP, :TCP_NODELAY, 1)
+    sock.setsockopt(:SOL_SOCKET, :SO_KEEPALIVE, 1)
+
+    if opt[:rcvbuf] || opt[:sndbuf]
+      log_buffer_sizes(sock, "before: ")
+      { SO_RCVBUF: :rcvbuf, SO_SNDBUF: :sndbuf }.each do |optname,cfgname|
+        val = opt[cfgname] and sock.setsockopt(:SOL_SOCKET, optname, val)
+      end
+      log_buffer_sizes(sock, " after: ")
+    end
+    sock.listen(opt[:backlog])
+  rescue => e
+    Yahns::Log.exception(@logger, "#{sock_name(sock)} #{opt.inspect}", e)
+  end
+
+  def log_buffer_sizes(sock, pfx = '')
+    rcvbuf = sock.getsockopt(:SOL_SOCKET, :SO_RCVBUF).int
+    sndbuf = sock.getsockopt(:SOL_SOCKET, :SO_SNDBUF).int
+    @logger.info("#{pfx}#{sock_name(sock)} rcvbuf=#{rcvbuf} sndbuf=#{sndbuf}")
+  end
+
+  # creates a new server, socket. address may be a HOST:PORT or
+  # an absolute path to a UNIX socket.  address can even be a Socket
+  # object in which case it is immediately returned
+  def bind_listen(address, opt)
+    return address unless String === address
+    opt ||= {}
+
+    sock = if address[0] == ?/
+      if File.exist?(address)
+        if File.socket?(address)
+          begin
+            UNIXSocket.new(address).close
+            # fall through, try to bind(2) and fail with EADDRINUSE
+            # (or succeed from a small race condition we can't sanely avoid).
+          rescue Errno::ECONNREFUSED
+            @logger.info "unlinking existing socket=#{address}"
+            File.unlink(address)
+          end
+        else
+          raise ArgumentError,
+                "socket=#{address} specified but it is not a socket!"
+        end
+      end
+      old_umask = File.umask(opt[:umask] || 0)
+      begin
+        Kgio::UNIXServer.new(address)
+      ensure
+        File.umask(old_umask)
+      end
+    elsif /\A\[([a-fA-F0-9:]+)\]:(\d+)\z/ =~ address
+      new_ipv6_server($1, $2.to_i, opt)
+    elsif /\A(\d+\.\d+\.\d+\.\d+):(\d+)\z/ =~ address
+      Kgio::TCPServer.new($1, $2.to_i)
+    else
+      raise ArgumentError, "Don't know how to bind: #{address}"
+    end
+    set_server_sockopt(sock, opt)
+    sock
+  end
+
+  def new_ipv6_server(addr, port, opt)
+    opt.key?(:ipv6only) or return Kgio::TCPServer.new(addr, port)
+    sock = Socket.new(:AF_INET6, :SOCK_STREAM, 0)
+    sock.setsockopt(:IPPROTO_IPV6, :IPV6_V6ONLY, opt[:ipv6only] ? 1 : 0)
+    sock.setsockopt(:SOL_SOCKET, :SO_REUSEADDR, 1)
+    sock.bind(Socket.pack_sockaddr_in(port, addr))
+    sock.autoclose = false
+    Kgio::TCPServer.for_fd(sock.fileno)
+  end
+
+  # returns rfc2732-style (e.g. "[::1]:666") addresses for IPv6
+  def tcp_name(sock)
+    port, addr = Socket.unpack_sockaddr_in(sock.getsockname)
+    /:/ =~ addr ? "[#{addr}]:#{port}" : "#{addr}:#{port}"
+  end
+
+  # Returns the configuration name of a socket as a string.  sock may
+  # be a string value, in which case it is returned as-is
+  # Warning: TCP sockets may not always return the name given to it.
+  def sock_name(sock)
+    case sock
+    when String then sock
+    when UNIXServer
+      Socket.unpack_sockaddr_un(sock.getsockname)
+    when TCPServer
+      tcp_name(sock)
+    when Socket
+      begin
+        tcp_name(sock)
+      rescue ArgumentError
+        Socket.unpack_sockaddr_un(sock.getsockname)
+      end
+    else
+      raise ArgumentError, "Unhandled class #{sock.class}: #{sock.inspect}"
+    end
+  end
+
+  # casts a given Socket to be a TCPServer or UNIXServer
+  def server_cast(sock)
+    sock.autoclose = false
+    begin
+      Socket.unpack_sockaddr_in(sock.getsockname)
+      Kgio::TCPServer.for_fd(sock.fileno)
+    rescue ArgumentError
+      Kgio::UNIXServer.for_fd(sock.fileno)
+    end
+  end
+end

data/lib/yahns/stream_file.rb

@@ -0,0 +1,34 @@
+# -*- encoding: binary -*-
+# Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> et. al.
+# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
+require_relative 'wbuf_common'
+
+class Yahns::StreamFile # :nodoc:
+  include Yahns::WbufCommon
+
+  def initialize(body, persist, offset, count)
+    if body.respond_to?(:to_io)
+      @tmpio = body.to_io
+    else
+      path = body.to_path
+      if path =~ %r{\A/dev/fd/(\d+)\z}
+        @tmpio = IO.for_fd($1.to_i)
+        @tmpio.autoclose = false
+      else
+        @tmpio = File.open(path)
+      end
+    end
+    @sf_offset = offset
+    @sf_count = count || @tmpio.stat.size
+    @wbuf_persist = persist # whether or not we keep the connection alive
+    @body = body
+  end
+
+  # called by last wbuf_flush
+  def wbuf_close(client)
+    if File === @tmpio && @tmpio != @body
+      @tmpio.close
+    end
+    wbuf_close_common(client)
+  end
+end

data/lib/yahns/stream_input.rb

@@ -0,0 +1,150 @@
+# -*- encoding: binary -*-
+# Copyright (C) 2009-2013, Eric Wong <normalperson@yhbt.net> et. al.
+# License: GPLv2 or later (https://www.gnu.org/licenses/gpl-2.0.txt)
+
+# When processing uploads, Yahns may expose a StreamInput object under
+# "rack.input" of the (future) Rack (2.x) environment.
+class Yahns::StreamInput # :nodoc:
+  # Initializes a new StreamInput object.  You normally do not have to call
+  # this unless you are writing an HTTP server.
+  def initialize(client, request)
+    @chunked = request.content_length.nil?
+    @client = client
+    @parser = request
+    @buf = request.buf
+    @rbuf = ''
+    @bytes_read = 0
+    filter_body(@rbuf, @buf) unless @buf.empty?
+  end
+
+  # :call-seq:
+  #   ios.read([length [, buffer ]]) => string, buffer, or nil
+  #
+  # Reads at most length bytes from the I/O stream, or to the end of
+  # file if length is omitted or is nil. length must be a non-negative
+  # integer or nil. If the optional buffer argument is present, it
+  # must reference a String, which will receive the data.
+  #
+  # At end of file, it returns nil or '' depend on length.
+  # ios.read() and ios.read(nil) returns ''.
+  # ios.read(length [, buffer]) returns nil.
+  #
+  # If the Content-Length of the HTTP request is known (as is the common
+  # case for POST requests), then ios.read(length [, buffer]) will block
+  # until the specified length is read (or it is the last chunk).
+  # Otherwise, for uncommon "Transfer-Encoding: chunked" requests,
+  # ios.read(length [, buffer]) will return immediately if there is
+  # any data and only block when nothing is available (providing
+  # IO#readpartial semantics).
+  def read(length = nil, rv = '')
+    if length
+      if length <= @rbuf.size
+        length < 0 and raise ArgumentError, "negative length #{length} given"
+        rv.replace(@rbuf.slice!(0, length))
+      else
+        to_read = length - @rbuf.size
+        rv.replace(@rbuf.slice!(0, @rbuf.size))
+        until to_read == 0 || eof? || (rv.size > 0 && @chunked)
+          @client.kgio_read(to_read, @buf) or eof!
+          filter_body(@rbuf, @buf)
+          rv << @rbuf
+          to_read -= @rbuf.size
+        end
+        @rbuf.replace('')
+      end
+      rv = nil if rv.empty? && length != 0
+    else
+      read_all(rv)
+    end
+    rv
+  end
+
+  def __rsize
+    @client.class.client_body_buffer_size
+  end
+
+  # :call-seq:
+  #   ios.gets   => string or nil
+  #
+  # Reads the next ``line'' from the I/O stream; lines are separated
+  # by the global record separator ($/, typically "\n"). A global
+  # record separator of nil reads the entire unread contents of ios.
+  # Returns nil if called at the end of file.
+  # This takes zero arguments for strict Rack::Lint compatibility,
+  # unlike IO#gets.
+  def gets
+    sep = $/
+    if sep.nil?
+      read_all(rv = '')
+      return rv.empty? ? nil : rv
+    end
+    re = /\A(.*?#{Regexp.escape(sep)})/
+    rsize = __rsize
+    begin
+      @rbuf.sub!(re, '') and return $1
+      return @rbuf.empty? ? nil : @rbuf.slice!(0, @rbuf.size) if eof?
+      @client.kgio_read(rsize, @buf) or eof!
+      filter_body(once = '', @buf)
+      @rbuf << once
+    end while true
+  end
+
+  # :call-seq:
+  #   ios.each { |line| block }  => ios
+  #
+  # Executes the block for every ``line'' in *ios*, where lines are
+  # separated by the global record separator ($/, typically "\n").
+  def each
+    while line = gets
+      yield line
+    end
+
+    self # Rack does not specify what the return value is here
+  end
+
+  def eof?
+    if @parser.body_eof?
+      rsize = __rsize
+      while @chunked && ! @parser.parse
+        once = @client.kgio_read(rsize) or eof!
+        @buf << once
+      end
+      @client = nil
+      true
+    else
+      false
+    end
+  end
+
+  def filter_body(dst, src)
+    rv = @parser.filter_body(dst, src)
+    @bytes_read += dst.size
+    rv
+  end
+
+  def read_all(dst)
+    dst.replace(@rbuf)
+    @client or return
+    rsize = @client.class.client_body_buffer_size
+    until eof?
+      @client.kgio_read(rsize, @buf) or eof!
+      filter_body(@rbuf, @buf)
+      dst << @rbuf
+    end
+  ensure
+    @rbuf.replace('')
+  end
+
+  def eof!
+    # in case client only did a premature shutdown(SHUT_WR)
+    # we do support clients that shutdown(SHUT_WR) after the
+    # _entire_ request has been sent, and those will not have
+    # raised EOFError on us.
+    @client.shutdown if @client
+  ensure
+    raise Yahns::ClientShutdown, "bytes_read=#{@bytes_read}", []
+  end
+
+  def close # return nil
+  end
+end