unicorn-lb-patch 4.3.1.11.g21b8.dirty

data/lib/unicorn/launcher.rb

@@ -0,0 +1,62 @@
+# -*- encoding: binary -*-
+
+# :enddoc:
+$stdout.sync = $stderr.sync = true
+$stdin.binmode
+$stdout.binmode
+$stderr.binmode
+
+require 'unicorn'
+
+module Unicorn::Launcher
+
+  # We don't do a lot of standard daemonization stuff:
+  #   * umask is whatever was set by the parent process at startup
+  #     and can be set in config.ru and config_file, so making it
+  #     0000 and potentially exposing sensitive log data can be bad
+  #     policy.
+  #   * don't bother to chdir("/") here since unicorn is designed to
+  #     run inside APP_ROOT.  Unicorn will also re-chdir() to
+  #     the directory it was started in when being re-executed
+  #     to pickup code changes if the original deployment directory
+  #     is a symlink or otherwise got replaced.
+  def self.daemonize!(options)
+    cfg = Unicorn::Configurator
+    $stdin.reopen("/dev/null")
+
+    # We only start a new process group if we're not being reexecuted
+    # and inheriting file descriptors from our parent
+    unless ENV['UNICORN_FD']
+      # grandparent - reads pipe, exits when master is ready
+      #  \_ parent  - exits immediately ASAP
+      #      \_ unicorn master - writes to pipe when ready
+
+      rd, wr = IO.pipe
+      grandparent = $$
+      if fork
+        wr.close # grandparent does not write
+      else
+        rd.close # unicorn master does not read
+        Process.setsid
+        exit if fork # parent dies now
+      end
+
+      if grandparent == $$
+        # this will block until HttpServer#join runs (or it dies)
+        master_pid = (rd.readpartial(16) rescue nil).to_i
+        unless master_pid > 1
+          warn "master failed to start, check stderr log for details"
+          exit!(1)
+        end
+        exit 0
+      else # unicorn master process
+        options[:ready_pipe] = wr
+      end
+    end
+    # $stderr/$stderr can/will be redirected separately in the Unicorn config
+    cfg::DEFAULTS[:stderr_path] ||= "/dev/null"
+    cfg::DEFAULTS[:stdout_path] ||= "/dev/null"
+    cfg::RACKUP[:daemonized] = true
+  end
+
+end

data/lib/unicorn/oob_gc.rb

@@ -0,0 +1,71 @@
+# -*- encoding: binary -*-
+
+# Runs GC after requests, after closing the client socket and
+# before attempting to accept more connections.
+#
+# This shouldn't hurt overall performance as long as the server cluster
+# is at <50% CPU capacity, and improves the performance of most memory
+# intensive requests.  This serves to improve _client-visible_
+# performance (possibly at the cost of overall performance).
+#
+# Increasing the number of +worker_processes+ may be necessary to
+# improve average client response times because some of your workers
+# will be busy doing GC and unable to service clients.  Think of
+# using more workers with this module as a poor man's concurrent GC.
+#
+# We'll call GC after each request is been written out to the socket, so
+# the client never sees the extra GC hit it.
+#
+# This middleware is _only_ effective for applications that use a lot
+# of memory, and will hurt simpler apps/endpoints that can process
+# multiple requests before incurring GC.
+#
+# This middleware is only designed to work with unicorn, as it harms
+# performance with keepalive-enabled servers.
+#
+# Example (in config.ru):
+#
+#     require 'unicorn/oob_gc'
+#
+#     # GC ever two requests that hit /expensive/foo or /more_expensive/foo
+#     # in your app.  By default, this will GC once every 5 requests
+#     # for all endpoints in your app
+#     use Unicorn::OobGC, 2, %r{\A/(?:expensive/foo|more_expensive/foo)}
+#
+# Feedback from users of early implementations of this module:
+# * http://comments.gmane.org/gmane.comp.lang.ruby.unicorn.general/486
+# * http://article.gmane.org/gmane.comp.lang.ruby.unicorn.general/596
+module Unicorn::OobGC
+
+  # this pretends to be Rack middleware because it used to be
+  # But we need to hook into unicorn internals so we need to close
+  # the socket before clearing the request env.
+  #
+  # +interval+ is the number of requests matching the +path+ regular
+  # expression before invoking GC.
+  def self.new(app, interval = 5, path = %r{\A/})
+    @@nr = interval
+    self.const_set :OOBGC_PATH, path
+    self.const_set :OOBGC_INTERVAL, interval
+    ObjectSpace.each_object(Unicorn::HttpServer) do |s|
+      s.extend(self)
+      self.const_set :OOBGC_ENV, s.instance_variable_get(:@request).env
+    end
+    app # pretend to be Rack middleware since it was in the past
+  end
+
+  #:stopdoc:
+  PATH_INFO = "PATH_INFO"
+  def process_client(client)
+    super(client) # Unicorn::HttpServer#process_client
+    if OOBGC_PATH =~ OOBGC_ENV[PATH_INFO] && ((@@nr -= 1) <= 0)
+      @@nr = OOBGC_INTERVAL
+      OOBGC_ENV.clear
+      disabled = GC.enable
+      GC.start
+      GC.disable if disabled
+    end
+  end
+
+  # :startdoc:
+end

data/lib/unicorn/preread_input.rb

@@ -0,0 +1,33 @@
+# -*- encoding: binary -*-
+
+module Unicorn
+# This middleware is used to ensure input is buffered to memory
+# or disk (depending on size) before the application is dispatched
+# by entirely consuming it (from TeeInput) beforehand.
+#
+# Usage (in config.ru):
+#
+#     require 'unicorn/preread_input'
+#     if defined?(Unicorn)
+#       use Unicorn::PrereadInput
+#     end
+#     run YourApp.new
+class PrereadInput
+
+  # :stopdoc:
+  def initialize(app)
+    @app = app
+  end
+
+  def call(env)
+    buf = ""
+    input = env["rack.input"]
+    if input.respond_to?(:rewind)
+      true while input.read(16384, buf)
+      input.rewind
+    end
+    @app.call(env)
+  end
+  # :startdoc:
+end
+end

data/lib/unicorn/socket_helper.rb

@@ -0,0 +1,208 @@
+# -*- encoding: binary -*-
+# :enddoc:
+require 'socket'
+
+module Unicorn
+  module SocketHelper
+    # :stopdoc:
+    include Socket::Constants
+
+    # prevents IO objects in here from being GC-ed
+    IO_PURGATORY = []
+
+    # internal interface, only used by Rainbows!/Zbatery
+    DEFAULTS = {
+      # The semantics for TCP_DEFER_ACCEPT changed in Linux 2.6.32+
+      # with commit d1b99ba41d6c5aa1ed2fc634323449dd656899e9
+      # This change shouldn't affect Unicorn users behind nginx (a
+      # value of 1 remains an optimization), but Rainbows! users may
+      # want to use a higher value on Linux 2.6.32+ to protect against
+      # denial-of-service attacks
+      :tcp_defer_accept => 1,
+
+      # FreeBSD, we need to override this to 'dataready' if we
+      # eventually get HTTPS support
+      :accept_filter => 'httpready',
+
+      # same default value as Mongrel
+      :backlog => 1024,
+
+      # favor latency over bandwidth savings
+      :tcp_nopush => false,
+      :tcp_nodelay => true,
+    }
+    #:startdoc:
+
+    # configure platform-specific options (only tested on Linux 2.6 so far)
+    case RUBY_PLATFORM
+    when /linux/
+      # from /usr/include/linux/tcp.h
+      TCP_DEFER_ACCEPT = 9 unless defined?(TCP_DEFER_ACCEPT)
+
+      # do not send out partial frames (Linux)
+      TCP_CORK = 3 unless defined?(TCP_CORK)
+    when /freebsd/
+      # do not send out partial frames (FreeBSD)
+      TCP_NOPUSH = 4 unless defined?(TCP_NOPUSH)
+
+      def accf_arg(af_name)
+        [ af_name, nil ].pack('a16a240')
+      end if defined?(SO_ACCEPTFILTER)
+    end
+
+    def set_tcp_sockopt(sock, opt)
+      # just in case, even LANs can break sometimes.  Linux sysadmins
+      # can lower net.ipv4.tcp_keepalive_* sysctl knobs to very low values.
+      sock.setsockopt(SOL_SOCKET, SO_KEEPALIVE, 1) if defined?(SO_KEEPALIVE)
+
+      if defined?(TCP_NODELAY)
+        val = opt[:tcp_nodelay]
+        val = DEFAULTS[:tcp_nodelay] if nil == val
+        sock.setsockopt(IPPROTO_TCP, TCP_NODELAY, val ? 1 : 0)
+      end
+
+      val = opt[:tcp_nopush]
+      val = DEFAULTS[:tcp_nopush] if nil == val
+      val = val ? 1 : 0
+      if defined?(TCP_CORK) # Linux
+        sock.setsockopt(IPPROTO_TCP, TCP_CORK, val)
+      elsif defined?(TCP_NOPUSH) # TCP_NOPUSH is untested (FreeBSD)
+        sock.setsockopt(IPPROTO_TCP, TCP_NOPUSH, val)
+      end
+
+      # No good reason to ever have deferred accepts off
+      # (except maybe benchmarking)
+      if defined?(TCP_DEFER_ACCEPT)
+        # this differs from nginx, since nginx doesn't allow us to
+        # configure the the timeout...
+        seconds = opt[:tcp_defer_accept]
+        seconds = DEFAULTS[:tcp_defer_accept] if [true,nil].include?(seconds)
+        seconds = 0 unless seconds # nil/false means disable this
+        sock.setsockopt(SOL_TCP, TCP_DEFER_ACCEPT, seconds)
+      elsif respond_to?(:accf_arg)
+        name = opt[:accept_filter]
+        name = DEFAULTS[:accept_filter] if nil == name
+        begin
+          sock.setsockopt(SOL_SOCKET, SO_ACCEPTFILTER, accf_arg(name))
+        rescue => e
+          logger.error("#{sock_name(sock)} " \
+                       "failed to set accept_filter=#{name} (#{e.inspect})")
+        end
+      end
+    end
+
+    def set_server_sockopt(sock, opt)
+      opt = DEFAULTS.merge(opt || {})
+
+      TCPSocket === sock and set_tcp_sockopt(sock, opt)
+
+      if opt[:rcvbuf] || opt[:sndbuf]
+        log_buffer_sizes(sock, "before: ")
+        sock.setsockopt(SOL_SOCKET, SO_RCVBUF, opt[:rcvbuf]) if opt[:rcvbuf]
+        sock.setsockopt(SOL_SOCKET, SO_SNDBUF, opt[:sndbuf]) if opt[:sndbuf]
+        log_buffer_sizes(sock, " after: ")
+      end
+      sock.listen(opt[:backlog])
+      rescue => e
+        Unicorn.log_error(logger, "#{sock_name(sock)} #{opt.inspect}", e)
+    end
+
+    def log_buffer_sizes(sock, pfx = '')
+      rcvbuf = sock.getsockopt(SOL_SOCKET, SO_RCVBUF).unpack('i')
+      sndbuf = sock.getsockopt(SOL_SOCKET, SO_SNDBUF).unpack('i')
+      logger.info "#{pfx}#{sock_name(sock)} rcvbuf=#{rcvbuf} sndbuf=#{sndbuf}"
+    end
+
+    # creates a new server, socket. address may be a HOST:PORT or
+    # an absolute path to a UNIX socket.  address can even be a Socket
+    # object in which case it is immediately returned
+    def bind_listen(address = '0.0.0.0:8080', opt = {})
+      return address unless String === address
+
+      sock = if address[0] == ?/
+        if File.exist?(address)
+          if File.socket?(address)
+            begin
+              UNIXSocket.new(address).close
+              # fall through, try to bind(2) and fail with EADDRINUSE
+              # (or succeed from a small race condition we can't sanely avoid).
+            rescue Errno::ECONNREFUSED
+              logger.info "unlinking existing socket=#{address}"
+              File.unlink(address)
+            end
+          else
+            raise ArgumentError,
+                  "socket=#{address} specified but it is not a socket!"
+          end
+        end
+        old_umask = File.umask(opt[:umask] || 0)
+        begin
+          Kgio::UNIXServer.new(address)
+        ensure
+          File.umask(old_umask)
+        end
+      elsif /\A\[([a-fA-F0-9:]+)\]:(\d+)\z/ =~ address
+        new_ipv6_server($1, $2.to_i, opt)
+      elsif /\A(\d+\.\d+\.\d+\.\d+):(\d+)\z/ =~ address
+        Kgio::TCPServer.new($1, $2.to_i)
+      else
+        raise ArgumentError, "Don't know how to bind: #{address}"
+      end
+      set_server_sockopt(sock, opt)
+      sock
+    end
+
+    def new_ipv6_server(addr, port, opt)
+      opt.key?(:ipv6only) or return Kgio::TCPServer.new(addr, port)
+      defined?(IPV6_V6ONLY) or
+        abort "Socket::IPV6_V6ONLY not defined, upgrade Ruby and/or your OS"
+      sock = Socket.new(AF_INET6, SOCK_STREAM, 0)
+      sock.setsockopt(IPPROTO_IPV6, IPV6_V6ONLY, opt[:ipv6only] ? 1 : 0)
+      sock.setsockopt(SOL_SOCKET, SO_REUSEADDR, 1)
+      sock.bind(Socket.pack_sockaddr_in(port, addr))
+      IO_PURGATORY << sock
+      Kgio::TCPServer.for_fd(sock.fileno)
+    end
+
+    # returns rfc2732-style (e.g. "[::1]:666") addresses for IPv6
+    def tcp_name(sock)
+      port, addr = Socket.unpack_sockaddr_in(sock.getsockname)
+      /:/ =~ addr ? "[#{addr}]:#{port}" : "#{addr}:#{port}"
+    end
+    module_function :tcp_name
+
+    # Returns the configuration name of a socket as a string.  sock may
+    # be a string value, in which case it is returned as-is
+    # Warning: TCP sockets may not always return the name given to it.
+    def sock_name(sock)
+      case sock
+      when String then sock
+      when UNIXServer
+        Socket.unpack_sockaddr_un(sock.getsockname)
+      when TCPServer
+        tcp_name(sock)
+      when Socket
+        begin
+          tcp_name(sock)
+        rescue ArgumentError
+          Socket.unpack_sockaddr_un(sock.getsockname)
+        end
+      else
+        raise ArgumentError, "Unhandled class #{sock.class}: #{sock.inspect}"
+      end
+    end
+
+    module_function :sock_name
+
+    # casts a given Socket to be a TCPServer or UNIXServer
+    def server_cast(sock)
+      begin
+        Socket.unpack_sockaddr_in(sock.getsockname)
+        Kgio::TCPServer.for_fd(sock.fileno)
+      rescue ArgumentError
+        Kgio::UNIXServer.for_fd(sock.fileno)
+      end
+    end
+
+  end # module SocketHelper
+end # module Unicorn

data/lib/unicorn/ssl_client.rb

@@ -0,0 +1,11 @@
+# -*- encoding: binary -*-
+# :stopdoc:
+class Unicorn::SSLClient < Kgio::SSL
+  alias write kgio_write
+  alias close kgio_close
+
+  # this is no-op for now, to be fixed in kgio-monkey if people care
+  # about SSL support...
+  def shutdown(how = nil)
+  end
+end

data/lib/unicorn/ssl_configurator.rb

@@ -0,0 +1,104 @@
+# -*- encoding: binary -*-
+# :stopdoc:
+# This module is included in Unicorn::Configurator
+# :startdoc:
+#
+module Unicorn::SSLConfigurator
+  def ssl(&block)
+    ssl_require!
+    before = @set[:listeners].dup
+    opts = @set[:ssl_opts] = {}
+    yield
+    (@set[:listeners] - before).each do |address|
+      (@set[:listener_opts][address] ||= {})[:ssl_opts] = opts
+    end
+    ensure
+      @set.delete(:ssl_opts)
+  end
+
+  def ssl_certificate(file)
+    ssl_set(:ssl_certificate, file)
+  end
+
+  def ssl_certificate_key(file)
+    ssl_set(:ssl_certificate_key, file)
+  end
+
+  def ssl_client_certificate(file)
+    ssl_set(:ssl_client_certificate, file)
+  end
+
+  def ssl_dhparam(file)
+    ssl_set(:ssl_dhparam, file)
+  end
+
+  def ssl_ciphers(openssl_cipherlist_spec)
+    ssl_set(:ssl_ciphers, openssl_cipherlist_spec)
+  end
+
+  def ssl_crl(file)
+    ssl_set(:ssl_crl, file)
+  end
+
+  def ssl_prefer_server_ciphers(bool)
+    ssl_set(:ssl_prefer_server_ciphers, check_bool(bool))
+  end
+
+  def ssl_protocols(list)
+    ssl_set(:ssl_protocols, list)
+  end
+
+  def ssl_verify_client(on_off_optional)
+    ssl_set(:ssl_verify_client, on_off_optional)
+  end
+
+  def ssl_session_timeout(seconds)
+    ssl_set(:ssl_session_timeout, seconds)
+  end
+
+  def ssl_verify_depth(depth)
+    ssl_set(:ssl_verify_depth, depth)
+  end
+
+  # Allows specifying an engine for OpenSSL to use.  We have not been
+  # able to successfully test this feature due to a lack of hardware,
+  # Reports of success or patches to mongrel-unicorn@rubyforge.org is
+  # greatly appreciated.
+  def ssl_engine(engine)
+    ssl_warn_global(:ssl_engine)
+    ssl_require!
+    OpenSSL::Engine.load
+    OpenSSL::Engine.by_id(engine)
+    @set[:ssl_engine] = engine
+  end
+
+  def ssl_compression(bool)
+    # OpenSSL uses the SSL_OP_NO_COMPRESSION flag, Flipper follows suit
+    # with :ssl_no_compression, but we negate it to avoid exposing double
+    # negatives to the user.
+    ssl_set(:ssl_no_compression, check_bool(:ssl_compression, ! bool))
+  end
+
+private
+
+  def ssl_warn_global(func) # :nodoc:
+    Hash === @set[:ssl_opts] or return
+    warn("`#{func}' affects all SSL contexts in this process, " \
+         "not just this block")
+  end
+
+  def ssl_set(key, value) # :nodoc:
+    cur = @set[:ssl_opts]
+    Hash === cur or
+             raise ArgumentError, "#{key} must be called inside an `ssl' block"
+    cur[key] = value
+  end
+
+  def ssl_require! # :nodoc:
+    require "flipper"
+    require "unicorn/ssl_client"
+    rescue LoadError
+      warn "install 'kgio-monkey' for SSL support"
+      raise
+  end
+end