yahns 0.0.0TP1

data/examples/logrotate.conf

@@ -0,0 +1,32 @@
+# To the extent possible under law, Eric Wong has waived all copyright and
+# related or neighboring rights to this examples
+#
+# example logrotate config file, I usually keep this in
+# /etc/logrotate.d/yahns_app on my Debian systems
+#
+# See the logrotate(8) manpage for more information:
+#    http://linux.die.net/man/8/logrotate
+
+# Modify the following glob to match the logfiles your app writes to:
+/var/log/yahns_app/*.log {
+	# this first block is mostly just personal preference, though
+	# I wish logrotate offered an "hourly" option...
+	daily
+	missingok
+	rotate 180
+	compress # must use with delaycompress below
+	dateext
+
+	# this is important if using "compress" since we need to call
+	# the "lastaction" script below before compressing:
+	delaycompress
+
+	# note the lack of the evil "copytruncate" option in this
+	# config.  yahns supports the USR1 signal and we send it
+	# as our "lastaction" action:
+	lastaction
+		# assuming your pid file is in /var/run/yahns_app/pid
+		pid=/var/run/yahns_app/pid
+		test -s $pid && kill -USR1 "$(cat $pid)"
+	endscript
+}

data/examples/yahns_multi.conf.rb

@@ -0,0 +1,89 @@
+# To the extent possible under law, Eric Wong has waived all copyright and
+# related or neighboring rights to this example.
+
+# By default, this based on the soft limit of RLIMIT_NOFILE
+#   count = Process.getrlimit(:NOFILE)[0]) * 0.5
+# yahns will start expiring idle clients once we hit it
+client_expire_threshold 0.5
+
+# each queue definition configures a thread pool and epoll_wait usage
+# The default queue is always present
+queue(:default) do
+  worker_threads 7 # this is the default value
+  max_events 1 # 1: fairest, best in all multi-threaded cases
+end
+
+# This is an example of a small queue with fewer workers and unfair scheduling.
+# It is rarely necessary or even advisable to configure multiple queues.
+queue(:small) do
+  worker_threads 2
+
+  # increase max_events only under one of the following circumstances:
+  # 1) worker_threads is 1
+  # 2) epoll_wait lock contention inside the kernel is the biggest bottleneck
+  #    (this is unlikely outside of "hello world" apps)
+  max_events 64
+end
+
+# This is an example of a Rack application configured in yahns
+# All values below are defaults
+app(:rack, "/path/to/config.ru", preload: false) do
+  listen 8080, backlog: 1024, tcp_nodelay: false
+  client_max_body_size 1024*1024
+  check_client_connection false
+  logger Logger.new($stderr)
+  client_timeout 15
+  input_buffering true
+  output_buffering true # output buffering is always lazy if enabled
+  persistent_connections true
+  errors $stderr
+  queue :default
+end
+
+# same as first, just listen on different port and small queue
+app(:rack, "/path/to/config.ru") do
+  listen "10.0.0.1:10000"
+  client_max_body_size 1024*1024*10
+  check_client_connection true
+  logger Logger.new("/path/to/another/log")
+  client_timeout 30
+  persistent_connections true
+  errors "/path/to/errors.log"
+  queue :small
+end
+
+# totally different app
+app(:rack, "/path/to/another.ru", preload: true) do
+  listen 8081, sndbuf: 1024 * 1024
+  listen "/path/to/unix.sock"
+  client_max_body_size 1024*1024*1024
+  input_buffering :lazy
+  output_buffering false
+  client_timeout 4
+  persistent_connections false
+
+  # different apps may share the same queue, but listen on different ports.
+  queue :default
+end
+
+# yet another totally different app, this app is not-thread safe but fast
+# enough for multi-process to not matter.
+# Use unicorn if you need multi-process performance on single-threaded apps
+app(:rack, "/path/to/not_thread_safe.ru") do
+  # listeners _always_ get a private thread in yahns
+  listen "/path/to/yet_another.sock"
+  listen 8082
+
+  # inline private queue definition here
+  queue do
+    worker_threads 1 # single-threaded queue
+    max_events 64 # very high max_events is perfectly fair for single thread
+  end
+
+  # single (or few)-threaded apps must use full buffering if serving
+  # untrusted/slow clients.
+  input_buffering true
+  output_buffering true
+end
+# Note: this file is used by test_config.rb, be sure to update that
+# if we update this

data/examples/yahns_rack_basic.conf.rb

@@ -0,0 +1,27 @@
+# To the extent possible under law, Eric Wong has waived all copyright and
+# related or neighboring rights to this examples
+# A typical Rack example for hosting a single Rack application with yahns
+# and only frequently-useful config values
+
+worker_processes 1
+# working_directory "/path/to/my_app"
+stdout_path "/path/to/my_logs/out.log"
+stderr_path "/path/to/my_logs/err.log"
+pid "/path/to/my_pids/yahns.pid"
+client_expire_threshold 0.5
+
+queue do
+  worker_threads 50
+end
+
+app(:rack, "config.ru", preload: false) do
+  listen 8080
+  client_max_body_size 1024 * 1024
+  input_buffering true
+  output_buffering true # this lazy by default
+  client_timeout 5
+  persistent_connections true
+end
+
+# Note: this file is used by test_config.rb, be sure to update that
+# if we update this

data/lib/yahns.rb

@@ -0,0 +1,73 @@
+# Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> and all contributors
+# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
+require 'unicorn' # pulls in raindrops, kgio, fcntl, etc, stringio, and logger
+require 'sleepy_penguin'
+
+# kill off some unicorn internals we don't need
+# we'll probably just make kcar into a server parser so we don't depend
+# on unicorn at all
+[ :ClientShutdown, :Const, :SocketHelper, :StreamInput, :TeeInput,
+  :SSLConfigurator, :Configurator, :TmpIO, :Util, :Worker, :SSLServer,
+  :HttpServer ].each { |sym| Unicorn.__send__(:remove_const, sym) }
+
+# yahns exposes no user-visible API outside of the config file
+# Internals are subject to change.
+module Yahns # :nodoc:
+  # We populate this at startup so we can figure out how to reexecute
+  # and upgrade the currently running instance of yahns
+  # Unlike unicorn, this Hash is NOT a stable/public interface.
+  #
+  # * 0 - the path to the yahns executable
+  # * :argv - a deep copy of the ARGV array the executable originally saw
+  # * :cwd - the working directory of the application, this is where
+  # you originally started yahns.
+  #
+  # To change your yahns executable to a different path without downtime,
+  # you can set the following in your yahns config file, HUP and then
+  # continue with the traditional USR2 + QUIT upgrade steps:
+  #
+  #   Yahns::START[0] = "/home/bofh/2.0.0/bin/yahns"
+  START = {
+    :argv => ARGV.map { |arg| arg.dup },
+    0 => $0.dup,
+  }
+
+  # We favor ENV['PWD'] since it is (usually) symlink aware for Capistrano
+  # and like systems
+  START[:cwd] = begin
+    a = File.stat(pwd = ENV['PWD'])
+    b = File.stat(Dir.pwd)
+    a.ino == b.ino && a.dev == b.dev ? pwd : Dir.pwd
+  rescue
+    Dir.pwd
+  end
+
+  # Raised inside TeeInput when a client closes the socket inside the
+  # application dispatch.  This is always raised with an empty backtrace
+  # since there is nothing in the application stack that is responsible
+  # for client shutdowns/disconnects.
+  class ClientShutdown < EOFError # :nodoc:
+  end
+end
+
+# FIXME: require lazily
+require_relative 'yahns/log'
+require_relative 'yahns/queue_epoll'
+require_relative 'yahns/stream_input'
+require_relative 'yahns/tee_input'
+require_relative 'yahns/queue_egg'
+require_relative 'yahns/client_expire'
+require_relative 'yahns/http_response'
+require_relative 'yahns/http_client'
+require_relative 'yahns/http_context'
+require_relative 'yahns/queue'
+require_relative 'yahns/config'
+require_relative 'yahns/tmpio'
+require_relative 'yahns/worker'
+require_relative 'yahns/sigevent'
+require_relative 'yahns/daemon'
+require_relative 'yahns/socket_helper'
+require_relative 'yahns/server'
+require_relative 'yahns/fdmap'
+require_relative 'yahns/acceptor'
+require_relative 'yahns/wbuf'

data/lib/yahns/acceptor.rb

@@ -0,0 +1,28 @@
+# Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> et. al.
+# License: GPLv3 or later (see COPYING for details)
+module Yahns::Acceptor # :nodoc:
+  def spawn_acceptor(logger, client_class, queue)
+    accept_flags = Kgio::SOCK_NONBLOCK | Kgio::SOCK_CLOEXEC
+    Thread.new do
+      Thread.current.abort_on_exception = true
+      qev_flags = client_class.superclass::QEV_FLAGS
+      begin
+        # We want the accept/accept4 syscall to be _blocking_
+        # so it can distribute work evenly between processes
+        if client = kgio_accept(client_class, accept_flags)
+          client.yahns_init
+
+          # it is not safe to touch client in this thread after this,
+          # a worker thread may grab client right away
+          queue.queue_add(client, qev_flags)
+        end
+      rescue Errno::EMFILE, Errno::ENFILE => e
+        logger.error("#{e.message}, consider raising open file limits")
+        queue.fdmap.desperate_expire_for(self, 5)
+        sleep 1 # let other threads do some work
+      rescue => e
+        Yahns::Log.exception(logger, "accept loop error", e) unless closed?
+      end until closed?
+    end
+  end
+end

data/lib/yahns/client_expire.rb

@@ -0,0 +1,40 @@
+# Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> and all contributors
+# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
+
+# included in Yahns::HttpClient
+#
+# this provides the ability to expire idle clients once we hit a soft limit
+# on idle clients
+#
+# we absolutely DO NOT issue IO#close in here, only BasicSocket#shutdown
+module Yahns::ClientExpire # :nodoc:
+  def yahns_expire(timeout) # rarely called
+    return 0 if closed? # still racy, but avoid the exception in most cases
+
+    info = Raindrops::TCP_Info.new(self)
+    return 0 if info.state != 1 # TCP_ESTABLISHED == 1
+
+    # Linux struct tcp_info timers are in milliseconds
+    timeout *= 1000
+
+    send_timedout = !!(info.last_data_sent > timeout)
+
+    # tcpi_last_data_recv is not valid unless tcpi_ato (ACK timeout) is set
+    if 0 == info.ato
+      sd = send_timedout && (info.last_ack_recv > timeout)
+    else
+      sd = send_timedout && (info.last_data_recv > timeout)
+    end
+    if sd
+      shutdown
+      1
+    else
+      0
+    end
+  # we also do not expire UNIX domain sockets
+  # (since those are the most trusted of local clients)
+  # the IO#closed? check is racy
+  rescue
+    0
+  end
+end

data/lib/yahns/client_expire_portable.rb

@@ -0,0 +1,39 @@
+# Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> and all contributors
+# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
+module Yahns::ClientExpire # :nodoc:
+  def __timestamp
+    Time.now.to_f
+  end
+
+  def yahns_expire(timeout)
+    return 0 if closed? # still racy, but avoid the exception in most cases
+    if (__timestamp - @last_io_at) > timeout
+      shutdown
+      1
+    else
+      0
+    end
+  rescue # the IO#closed? check is racy
+    0
+  end
+
+  def kgio_read(*args)
+    @last_io_at = __timestamp
+    super
+  end
+
+  def kgio_write(*args)
+    @last_io_at = __timestamp
+    super
+  end
+
+  def kgio_trywrite(*args)
+    @last_io_at = __timestamp
+    super
+  end
+
+  def kgio_tryread(*args)
+    @last_io_at = __timestamp
+    super
+  end
+end

data/lib/yahns/config.rb

@@ -0,0 +1,344 @@
+# -*- encoding: binary -*-
+# Copyright (C) 2013, Eric Wong <normalperson@yhbt.net> and all contributors
+# License: GPLv3 or later (https://www.gnu.org/licenses/gpl-3.0.txt)
+#
+# Implements a DSL for configuring a yahns server.
+# See http://yahns.yhbt.net/examples/yahns_multi.conf.rb for a full
+# example configuration file.
+class Yahns::Config # :nodoc:
+  APP_CLASS = {} # public, see yahns/rack for usage example
+  CfgBlock = Struct.new(:type, :ctx) # :nodoc:
+  attr_reader :config_file, :config_listeners, :set
+  attr_reader :qeggs, :app_ctx
+
+  def initialize(config_file = nil)
+    @config_file = config_file
+    @block = nil
+    config_reload!
+  end
+
+  def _check_in_block(ctx, var)
+    if ctx == nil
+      return var if @block == nil
+      msg = "#{var} must be called outside of #{@block.type}"
+    else
+      return var if @block && ctx == @block.type
+      msg = @block ? "may not be used inside a #{@block.type} block" :
+                     "must be used with a #{ctx} block"
+    end
+    raise ArgumentError, msg
+  end
+
+  def postfork_cleanup
+    @app_ctx = @set = @qeggs = @app_instances = @config_file = nil
+  end
+
+  def config_reload! #:nodoc:
+    # app_instance:app_ctx is a 1:N relationship
+    @config_listeners = {} # name/address -> options
+    @app_ctx = []
+    @set = Hash.new(:unset)
+    @qeggs = {}
+    @app_instances = {}
+
+    # set defaults:
+    client_expire_threshold(0.5) # default is half of the open file limit
+
+    instance_eval(File.read(@config_file), @config_file) if @config_file
+
+    # working_directory binds immediately (easier error checking that way),
+    # now ensure any paths we changed are correctly set.
+    [ :pid, :stderr_path, :stdout_path ].each do |var|
+      String === (path = @set[var]) or next
+      path = File.expand_path(path)
+      File.writable?(path) || File.writable?(File.dirname(path)) or \
+            raise ArgumentError, "directory for #{var}=#{path} not writable"
+    end
+  end
+
+  def logger(obj)
+    var = :logger
+    %w(debug info warn error fatal).each do |m|
+      obj.respond_to?(m) and next
+      raise ArgumentError, "#{var}=#{obj} does not respond to method=#{m}"
+    end
+    if @block
+      if @block.ctx.respond_to?(:logger=)
+        @block.ctx.logger = obj
+      else
+        raise ArgumentError, "#{var} not valid inside #{@block.type}"
+      end
+    else
+      @set[var] = obj
+    end
+  end
+
+  def worker_processes(nr)
+    # TODO: allow zero
+    var = _check_in_block(nil, :worker_processes)
+    @set[var] = _check_int(var, nr, 1)
+  end
+
+  # sets the +path+ for the PID file of the yahns master process
+  def pid(path)
+    _set_path(:pid, path)
+  end
+
+  def stderr_path(path)
+    _set_path(:stderr_path, path)
+  end
+
+  def stdout_path(path)
+    _set_path(:stdout_path, path)
+  end
+
+  def value(var)
+    val = @set[var]
+    val == :unset ? nil : val
+  end
+
+  # sets the working directory for yahns.  This ensures SIGUSR2 will
+  # start a new instance of yahns in this directory.  This may be
+  # a symlink, a common scenario for Capistrano users.  Unlike
+  # all other yahns configuration directives, this binds immediately
+  # for error checking and cannot be undone by unsetting it in the
+  # configuration file and reloading.
+  def working_directory(path)
+    var = :working_directory
+    @app_ctx.empty? or
+      raise ArgumentError, "#{var} must be declared before any apps"
+
+    # just let chdir raise errors
+    path = File.expand_path(path)
+    if @config_file &&
+       @config_file[0] != ?/ &&
+       ! File.readable?("#{path}/#@config_file")
+      raise ArgumentError,
+            "config_file=#@config_file would not be accessible in" \
+            " #{var}=#{path}"
+    end
+    Dir.chdir(path)
+    @set[var] = ENV["PWD"] = path
+  end
+
+  # Runs worker processes as the specified +user+ and +group+.
+  # The master process always stays running as the user who started it.
+  # This switch will occur after calling the after_fork hooks, and only
+  # if the Worker#user method is not called in the after_fork hooks
+  # +group+ is optional and will not change if unspecified.
+  def user(user, group = nil)
+    var = :user
+    @block and raise "#{var} is not valid inside #{@block.type}"
+    # raises ArgumentError on invalid user/group
+    Etc.getpwnam(user)
+    Etc.getgrnam(group) if group
+    @set[var] = [ user, group ]
+  end
+
+  def _set_path(var, path) #:nodoc:
+    _check_in_block(nil, var)
+    case path
+    when NilClass, String
+      @set[var] = path
+    else
+      raise ArgumentError
+    end
+  end
+
+  def listen(address, options = {})
+    options = options.dup
+    var = _check_in_block(:app, :listen)
+    address = expand_addr(address)
+    String === address or
+      raise ArgumentError, "address=#{address.inspect} must be a string"
+    [ :umask, :backlog, :sndbuf, :rcvbuf ].each do |key|
+      value = options[key] or next
+      Integer === value or
+        raise ArgumentError, "#{var}: not an integer: #{key}=#{value.inspect}"
+    end
+    [ :ipv6only ].each do |key|
+      (value = options[key]).nil? and next
+      [ true, false ].include?(value) or
+        raise ArgumentError, "#{var}: not boolean: #{key}=#{value.inspect}"
+    end
+
+    options[:yahns_app_ctx] = @block.ctx
+    @config_listeners.include?(address) and
+      raise ArgumentError, "listen #{address} already in use"
+    @config_listeners[address] = options
+  end
+
+  # expands "unix:path/to/foo" to a socket relative to the current path
+  # expands pathnames of sockets if relative to "~" or "~username"
+  # expands "*:port and ":port" to "0.0.0.0:port"
+  def expand_addr(address) #:nodoc:
+    return "0.0.0.0:#{address}" if Integer === address
+    return address unless String === address
+
+    case address
+    when %r{\Aunix:(.*)\z}
+      File.expand_path($1)
+    when %r{\A~}
+      File.expand_path(address)
+    when %r{\A(?:\*:)?(\d+)\z}
+      "0.0.0.0:#$1"
+    when %r{\A\[([a-fA-F0-9:]+)\]\z}, %r/\A((?:\d+\.){3}\d+)\z/
+      canonicalize_tcp($1, 80)
+    when %r{\A\[([a-fA-F0-9:]+)\]:(\d+)\z}, %r{\A(.*):(\d+)\z}
+      canonicalize_tcp($1, $2.to_i)
+    else
+      address
+    end
+  end
+
+  def canonicalize_tcp(addr, port)
+    packed = Socket.pack_sockaddr_in(port, addr)
+    port, addr = Socket.unpack_sockaddr_in(packed)
+    /:/ =~ addr ? "[#{addr}]:#{port}" : "#{addr}:#{port}"
+  end
+
+  def queue(name = :default, &block)
+    var = :queue
+    qegg = @qeggs[name] ||= Yahns::QueueEgg.new
+    prev_block = @block
+    _check_in_block(:app, var) if prev_block
+    if block_given?
+      @block = CfgBlock.new(:queue, qegg)
+      instance_eval(&block)
+      @block = prev_block
+    end
+    prev_block.ctx.qegg = qegg if prev_block
+  end
+
+  # queue parameters (Yahns::QueueEgg)
+  %w(max_events worker_threads).each do |_v|
+    eval(
+    %Q(def #{_v}(val);) <<
+    %Q(  _check_in_block(:queue, :#{_v});) <<
+    %Q(  @block.ctx.__send__("#{_v}=", _check_int(:#{_v}, val, 1));) <<
+    %Q(end)
+    )
+  end
+
+  def _check_int(var, n, min)
+    Integer === n or raise ArgumentError, "not an integer: #{var}=#{n.inspect}"
+    n >= min or raise ArgumentError, "too low (< #{min}): #{var}=#{n.inspect}"
+    n
+  end
+
+  # global
+  def client_expire_threshold(val)
+    var = _check_in_block(nil, :client_expire_threshold)
+    case val
+    when Float
+      val <= 1.0 or raise ArgumentError, "#{var} must be <= 1.0 if a ratio"
+    when Integer
+    else
+      raise ArgumentError, "#{var} must be a float or integer"
+    end
+    @set[var] = val
+  end
+
+  # type = :rack
+  def app(type, *args, &block)
+    var = _check_in_block(nil, :app)
+    file = "yahns/#{type.to_s}"
+    begin
+      require file
+    rescue LoadError => e
+      raise ArgumentError, "#{type.inspect} is not a supported app type",
+            e.backtrace
+    end
+    klass = APP_CLASS[type] or
+      raise TypeError,
+        "#{var}: #{file} did not register #{type} in #{self.class}::APP_CLASS"
+
+    # apps may have multiple configurator contexts
+    app = @app_instances[klass.instance_key(*args)] = klass.new(*args)
+    ctx = app.config_context
+    if block_given?
+      @block = CfgBlock.new(:app, ctx)
+      instance_eval(&block)
+      @block = nil
+    end
+    @app_ctx << ctx
+  end
+
+  def _check_bool(var, val)
+    return val if [ true, false ].include?(val)
+    raise ArgumentError, "#{var} must be boolean"
+  end
+
+  # boolean config directives for app
+  %w(check_client_connection
+     output_buffering
+     persistent_connections).each do |_v|
+    eval(
+    %Q(def #{_v}(bool);) <<
+    %Q(  _check_in_block(:app, :#{_v});) <<
+    %Q(  @block.ctx.__send__("#{_v}=", _check_bool(:#{_v}, bool));) <<
+    %Q(end)
+    )
+  end
+
+  # integer config directives for app
+  {
+    # config name, minimum value
+    client_body_buffer_size: 1,
+    client_max_body_size: 0,
+    client_header_buffer_size: 1,
+    client_max_header_size: 1,
+    client_timeout: 0,
+  }.each do |_v,minval|
+    eval(
+    %Q(def #{_v}(val);) <<
+    %Q(  _check_in_block(:app, :#{_v});) <<
+    %Q(  @block.ctx.__send__("#{_v}=", _check_int(:#{_v}, val, #{minval}));) <<
+    %Q(end)
+    )
+  end
+
+  def input_buffering(val)
+    var = _check_in_block(:app, :input_buffering)
+    ok = [ :lazy, true, false ]
+    ok.include?(val) or
+      raise ArgumentError, "`#{var}' must be one of: #{ok.inspect}"
+    @block.ctx.__send__("#{var}=", val)
+  end
+
+  # used to configure rack.errors destination
+  def errors(val)
+    var = _check_in_block(:app, :errors)
+    if String === val
+      # we've already bound working_directory by the time we get here
+      val = File.open(File.expand_path(val), "a")
+      val.binmode
+      val.sync = true
+    else
+      rt = %w(puts write flush).map(&:to_sym) # match Rack::Lint
+      rt.all? { |m| val.respond_to?(m) } or raise ArgumentError,
+                   "`#{var}' destination must respond to all of: #{rt.inspect}"
+    end
+    @block.ctx.__send__("#{var}=", val)
+  end
+
+  def commit!(server)
+    # redirect IOs
+    { stdout_path: $stdout, stderr_path: $stderr }.each do |key, io|
+      path = @set[key]
+      if path == :unset && server.daemon_pipe
+        @set[key] = path = "/dev/null"
+      end
+      File.open(path, 'a') { |fp| io.reopen(fp) } if String === path
+      io.sync = true
+    end
+
+    [ :logger, :pid, :worker_processes ].each do |var|
+      val = @set[var]
+      server.__send__("#{var}=", val) if val != :unset
+    end
+    queue(:default) if @qeggs.empty?
+    @qeggs.each_value { |qegg| qegg.logger ||= server.logger }
+    @app_ctx.each { |app| app.logger ||= server.logger }
+  end
+end