yahns 0.0.1 → 0.0.2

data/lib/yahns.rb

@@ -1,5 +1,7 @@
 # Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> and all contributors
 # License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
+$stdout.sync = $stderr.sync = true
+
 require 'unicorn' # pulls in raindrops, kgio, fcntl, etc, stringio, and logger
 require 'sleepy_penguin'
 
@@ -48,6 +50,9 @@ module Yahns # :nodoc:
   # for client shutdowns/disconnects.
   class ClientShutdown < EOFError # :nodoc:
   end
+
+  class ClientTimeout < RuntimeError # :nodoc:
+  end
 end
 
 # FIXME: require lazily
@@ -56,7 +61,6 @@ require_relative 'yahns/queue_epoll'
 require_relative 'yahns/stream_input'
 require_relative 'yahns/tee_input'
 require_relative 'yahns/queue_egg'
-require_relative 'yahns/client_expire'
 require_relative 'yahns/http_response'
 require_relative 'yahns/http_client'
 require_relative 'yahns/http_context'
@@ -65,7 +69,6 @@ require_relative 'yahns/config'
 require_relative 'yahns/tmpio'
 require_relative 'yahns/worker'
 require_relative 'yahns/sigevent'
-require_relative 'yahns/daemon'
 require_relative 'yahns/socket_helper'
 require_relative 'yahns/server'
 require_relative 'yahns/fdmap'

data/lib/yahns/acceptor.rb

@@ -1,30 +1,72 @@
+# -*- encoding: binary -*-
 # Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> et. al.
 # License: GPLv3 or later (see COPYING for details)
+require_relative 'client_expire_tcpi'
+require_relative 'client_expire_generic'
 module Yahns::Acceptor # :nodoc:
-  def spawn_acceptor(logger, client_class, queue)
-    Thread.new do
-      accept_flags = Kgio::SOCK_NONBLOCK | Kgio::SOCK_CLOEXEC
-      Thread.current.abort_on_exception = true
-      qev_flags = client_class.superclass::QEV_FLAGS
+  def __ac_quit_done?
+    @thrs.delete_if do |t|
       begin
-        # We want the accept/accept4 syscall to be _blocking_
-        # so it can distribute work evenly between processes
-        if client = kgio_accept(client_class, accept_flags)
-          client.yahns_init
+        t.join(0.01)
+      rescue
+        ! t.alive?
+      end
+    end
+    return false if @thrs[0]
+    close
+    true
+  end
+
+  # just keep looping this on every acceptor until the associated thread dies
+  def ac_quit
+    return true unless defined?(@thrs)
+    @thrs.each { |t| t[:yahns_quit] = true }
+    return true if __ac_quit_done?
+
+    @thrs.each do
+      begin
+        # try to connect to kick it out of the blocking accept() syscall
+        killer = Kgio::Socket.start(getsockname)
+        killer.kgio_write("G") # first byte of "GET / HTTP/1.0\r\n\r\n"
+      ensure
+        killer.close if killer
+      end
+    end
+    false # now hope __ac_quit_done? is true next time around
+  rescue SystemCallError
+    return __ac_quit_done?
+  end
+
+  def spawn_acceptor(nr, logger, client_class)
+    @thrs = nr.times.map do
+      Thread.new do
+        queue = client_class.queue
+        t = Thread.current
+        accept_flags = Kgio::SOCK_NONBLOCK | Kgio::SOCK_CLOEXEC
+        qev_flags = client_class.superclass::QEV_FLAGS
+        begin
+          # We want the accept/accept4 syscall to be _blocking_
+          # so it can distribute work evenly between processes
+          if client = kgio_accept(client_class, accept_flags)
+            client.yahns_init
 
-          # it is not safe to touch client in this thread after this,
-          # a worker thread may grab client right away
-          queue.queue_add(client, qev_flags)
-        end
-      rescue Errno::EMFILE, Errno::ENFILE => e
-        logger.error("#{e.message}, consider raising open file limits")
-        queue.fdmap.desperate_expire_for(self, 5)
-        sleep 1 # let other threads do some work
-      rescue => e
-        # sleep since this check is racy (and uncommon)
-        break if closed? || (sleep(0.01) && closed?)
-        Yahns::Log.exception(logger, "accept loop", e)
-      end while true
+            # it is not safe to touch client in this thread after this,
+            # a worker thread may grab client right away
+            queue.queue_add(client, qev_flags)
+          end
+        rescue Errno::EMFILE, Errno::ENFILE => e
+          logger.error("#{e.message}, consider raising open file limits")
+          queue.fdmap.desperate_expire_for(nil, 5)
+          sleep 1 # let other threads do some work
+        rescue => e
+          Yahns::Log.exception(logger, "accept loop", e)
+        end until t[:yahns_quit]
+      end
     end
   end
+
+  def expire_mod
+    (Yahns::TCPServer === self && Yahns.const_defined?(:ClientExpireTCPI)) ?
+     Yahns::ClientExpireTCPI : Yahns::ClientExpireGeneric
+  end
 end

data/lib/yahns/cap_input.rb

@@ -7,8 +7,8 @@
 class Yahns::CapInput < Yahns::TmpIO # :nodoc:
   attr_writer :bytes_left
 
-  def self.new(limit)
-    rv = super()
+  def self.new(limit, tmpdir)
+    rv = super(tmpdir)
     rv.bytes_left = limit
     rv
   end

data/lib/yahns/{client_expire_portable.rb → client_expire_generic.rb}

@@ -1,38 +1,39 @@
 # Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> and all contributors
 # License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
-module Yahns::ClientExpire # :nodoc:
+module Yahns::ClientExpireGeneric # :nodoc:
   def __timestamp
     Time.now.to_f
   end
 
+  def yahns_init
+    super # Yahns::HttpClient#yahns_init
+    @last_io_at = 0
+  end
+
   def yahns_expire(timeout)
-    return 0 if closed? # still racy, but avoid the exception in most cases
+    return 0 if closed?
     if (__timestamp - @last_io_at) > timeout
       shutdown
       1
     else
       0
     end
-  rescue # the IO#closed? check is racy
+  # shutdown may race with the shutdown in http_response_done
+  rescue
     0
   end
 
-  def kgio_read(*args)
-    @last_io_at = __timestamp
-    super
-  end
-
-  def kgio_write(*args)
+  def kgio_trywrite(*args)
     @last_io_at = __timestamp
     super
   end
 
-  def kgio_trywrite(*args)
+  def kgio_tryread(*args)
     @last_io_at = __timestamp
     super
   end
 
-  def kgio_tryread(*args)
+  def trysendfile(*args)
     @last_io_at = __timestamp
     super
   end

data/lib/yahns/{client_expire.rb → client_expire_tcpi.rb}

@@ -1,5 +1,6 @@
 # Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> and all contributors
 # License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
+require 'raindrops'
 
 # included in Yahns::HttpClient
 #
@@ -7,9 +8,9 @@
 # on idle clients
 #
 # we absolutely DO NOT issue IO#close in here, only BasicSocket#shutdown
-module Yahns::ClientExpire # :nodoc:
+module Yahns::ClientExpireTCPI # :nodoc:
   def yahns_expire(timeout) # rarely called
-    return 0 if closed? # still racy, but avoid the exception in most cases
+    return 0 if closed?
 
     info = Raindrops::TCP_Info.new(self)
     return 0 if info.state != 1 # TCP_ESTABLISHED == 1
@@ -31,10 +32,10 @@ module Yahns::ClientExpire # :nodoc:
     else
       0
     end
-  # we also do not expire UNIX domain sockets
-  # (since those are the most trusted of local clients)
-  # the IO#closed? check is racy
+  # shutdown may race with the shutdown in http_response_done
   rescue
     0
   end
-end
+# FreeBSD has "struct tcp_info", too, but does not support all the fields
+# Linux does as of FreeBSD 9 (haven't checked FreeBSD 10, yet).
+end if RUBY_PLATFORM =~ /linux/

data/lib/yahns/config.rb

@@ -22,9 +22,10 @@ class Yahns::Config # :nodoc:
       return var if @block == nil
       msg = "#{var} must be called outside of #{@block.type}"
     else
-      return var if @block && ctx == @block.type
+      ctx = Array(ctx)
+      return var if @block && ctx.include?(@block.type)
       msg = @block ? "may not be used inside a #{@block.type} block" :
-                     "must be used with a #{ctx} block"
+                     "must be used with a #{ctx.join(' or ')} block"
     end
     raise ArgumentError, msg
   end
@@ -38,7 +39,7 @@ class Yahns::Config # :nodoc:
     @config_listeners = {} # name/address -> options
     @app_ctx = []
     @set = Hash.new(:unset)
-    @qeggs = {}
+    @qeggs = Hash.new { |h,k| h[k] = Yahns::QueueEgg.new }
     @app_instances = {}
 
     # set defaults:
@@ -73,10 +74,44 @@ class Yahns::Config # :nodoc:
     end
   end
 
-  def worker_processes(nr)
-    # TODO: allow zero
-    var = _check_in_block(nil, :worker_processes)
+  def shutdown_timeout(sec)
+    var = _check_in_block(nil, :shutdown_timeout)
+    @set[var] = _check_num(var, sec, 0)
+  end
+
+  def worker_processes(nr, &blk)
+    var =_check_in_block(nil, :worker_processes)
     @set[var] = _check_int(var, nr, 1)
+    if block_given?
+      @block = CfgBlock.new(var, nil)
+      instance_eval(&blk)
+      @block = nil
+    end
+  end
+
+  %w(atfork_prepare atfork_parent atfork_child).each do |fn|
+    eval(
+    %Q(def #{fn}(*args, &blk);) <<
+    %Q(  _check_in_block([:worker_processes,:app], :#{fn});) <<
+    %Q(  _add_hook(:#{fn}, block_given? ? blk : args[0]);) <<
+    %Q(end)
+    )
+  end
+
+  def before_exec(&blk)
+    var = _check_in_block(nil, :before_exec)
+    @set[var] = (block_given? ? blk : args[0])
+  end
+
+  def _add_hook(var, my_proc)
+    Proc === my_proc or
+      raise ArgumentError, "invalid type: #{var}=#{my_proc.inspect}"
+
+    # this sets:
+    # :atfork_prepare, :atfork_parent, :atfork_child
+    key = var.to_sym
+    @set[key] = [] unless @set.include?(key)
+    @set[key] << my_proc
   end
 
   # sets the +path+ for the PID file of the yahns master process
@@ -104,7 +139,7 @@ class Yahns::Config # :nodoc:
   # for error checking and cannot be undone by unsetting it in the
   # configuration file and reloading.
   def working_directory(path)
-    var = :working_directory
+    var = _check_in_block(nil, :working_directory)
     @app_ctx.empty? or
       raise ArgumentError, "#{var} must be declared before any apps"
 
@@ -127,7 +162,7 @@ class Yahns::Config # :nodoc:
   # if the Worker#user method is not called in the after_fork hooks
   # +group+ is optional and will not change if unspecified.
   def user(user, group = nil)
-    var = :user
+    var = _check_in_block(nil, :user)
     @block and raise "#{var} is not valid inside #{@block.type}"
     # raises ArgumentError on invalid user/group
     Etc.getpwnam(user)
@@ -151,12 +186,17 @@ class Yahns::Config # :nodoc:
     address = expand_addr(address)
     String === address or
       raise ArgumentError, "address=#{address.inspect} must be a string"
-    [ :umask, :backlog, :sndbuf, :rcvbuf ].each do |key|
+    [ :umask, :backlog ].each do |key|
+      # :backlog may be negative on some OSes
       value = options[key] or next
       Integer === value or
         raise ArgumentError, "#{var}: not an integer: #{key}=#{value.inspect}"
     end
-    [ :ipv6only ].each do |key|
+    [ :sndbuf, :rcvbuf, :threads ].each do |key|
+       value = options[key] and _check_int(key, value, 1)
+    end
+
+    [ :ipv6only, :reuseport ].each do |key|
       (value = options[key]).nil? and next
       [ true, false ].include?(value) or
         raise ArgumentError, "#{var}: not boolean: #{key}=#{value.inspect}"
@@ -197,16 +237,33 @@ class Yahns::Config # :nodoc:
     /:/ =~ addr ? "[#{addr}]:#{port}" : "#{addr}:#{port}"
   end
 
-  def queue(name = :default, &block)
+  def queue(*args, &block)
     var = :queue
-    qegg = @qeggs[name] ||= Yahns::QueueEgg.new
     prev_block = @block
-    _check_in_block(:app, var) if prev_block
+    if prev_block
+      _check_in_block(:app, var)
+      if block_given?
+        args.size == 0 or
+          raise ArgumentError,
+                "queues defined with a block inside app must not have names"
+        name = @block
+      else
+        name = args[0] or
+          raise ArgumentError, "queue must be given a name if no block given"
+      end
+    else
+      name = args[0] || :default
+    end
+    args.size > 1 and
+      raise ArgumentError, "queue only takes one name argument"
+    qegg = @qeggs[name]
     if block_given?
       @block = CfgBlock.new(:queue, qegg)
       instance_eval(&block)
       @block = prev_block
     end
+
+    # associate the queue if we're inside an app
     prev_block.ctx.qegg = qegg if prev_block
   end
 
@@ -226,12 +283,19 @@ class Yahns::Config # :nodoc:
     n
   end
 
+  def _check_num(var, n, min)
+    Numeric === n or raise ArgumentError, "not a number: #{var}=#{n.inspect}"
+    n >= min or raise ArgumentError, "too low (< #{min}): #{var}=#{n.inspect}"
+    n
+  end
+
   # global
   def client_expire_threshold(val)
     var = _check_in_block(nil, :client_expire_threshold)
     case val
     when Float
-      val <= 1.0 or raise ArgumentError, "#{var} must be <= 1.0 if a ratio"
+      (val > 0 && val <= 1.0) or
+        raise ArgumentError, "#{var} must be > 0 and <= 1.0 if a ratio"
     when Integer
     else
       raise ArgumentError, "#{var} must be a float or integer"
@@ -270,9 +334,7 @@ class Yahns::Config # :nodoc:
   end
 
   # boolean config directives for app
-  %w(check_client_connection
-     output_buffering
-     persistent_connections).each do |_v|
+  %w(check_client_connection persistent_connections).each do |_v|
     eval(
     %Q(def #{_v}(bool);) <<
     %Q(  _check_in_block(:app, :#{_v});) <<
@@ -281,13 +343,27 @@ class Yahns::Config # :nodoc:
     )
   end
 
+  def output_buffering(bool, opts = {})
+    var = _check_in_block(:app, :output_buffering)
+    @block.ctx.__send__("#{var}=", _check_bool(var, bool))
+    tmpdir = opts[:tmpdir] and
+      @block.ctx.output_buffer_tmpdir = _check_tmpdir(var, tmpdir)
+  end
+
+  def _check_tmpdir(var, path)
+    File.directory?(path) or
+      raise ArgumentError, "#{var} tmpdir: #{path} is not a directory"
+    File.writable?(path) or
+      raise ArgumentError, "#{var} tmpdir: #{path} is not writable"
+    path
+  end
+
   # integer config directives for app
   {
     # config name, minimum value
     client_body_buffer_size: 1,
     client_header_buffer_size: 1,
     client_max_header_size: 1,
-    client_timeout: 0,
   }.each do |_v,minval|
     eval(
     %Q(def #{_v}(val);) <<
@@ -297,18 +373,25 @@ class Yahns::Config # :nodoc:
     )
   end
 
+  def client_timeout(val)
+    var = _check_in_block(:app, :client_timeout)
+    @block.ctx.__send__("#{var}=", _check_num(var, val, 0))
+  end
+
   def client_max_body_size(val)
     var = _check_in_block(:app, :client_max_body_size)
     val = _check_int(var, val, 0) if val != nil
     @block.ctx.__send__("#{var}=", val)
   end
 
-  def input_buffering(val)
+  def input_buffering(val, opts = {})
     var = _check_in_block(:app, :input_buffering)
     ok = [ :lazy, true, false ]
     ok.include?(val) or
       raise ArgumentError, "`#{var}' must be one of: #{ok.inspect}"
     @block.ctx.__send__("#{var}=", val)
+    tmpdir = opts[:tmpdir] and
+      @block.ctx.input_buffer_tmpdir = _check_tmpdir(var, tmpdir)
   end
 
   # used to configure rack.errors destination
@@ -335,14 +418,16 @@ class Yahns::Config # :nodoc:
         @set[key] = path = "/dev/null"
       end
       File.open(path, 'a') { |fp| io.reopen(fp) } if String === path
-      io.close_on_exec = io.sync = true
+      io.sync = true
     end
 
-    [ :logger, :pid, :worker_processes ].each do |var|
+    [ :logger, :pid, :worker_processes, :user, :shutdown_timeout, :before_exec,
+      :atfork_prepare, :atfork_parent, :atfork_child
+    ].each do |var|
       val = @set[var]
       server.__send__("#{var}=", val) if val != :unset
     end
-    queue(:default) if @qeggs.empty?
+
     @app_ctx.each { |app| app.logger ||= server.logger }
   end
 end