unicorn 0.2.3 → 0.4.1

data/lib/unicorn/configurator.rb

@@ -8,22 +8,21 @@ module Unicorn
   #
   # Example (when used with the unicorn config file):
   #   worker_processes 4
-  #   listeners %w(0.0.0.0:9292 /tmp/my_app.sock)
+  #   listen '/tmp/my_app.sock', :backlog => 1
+  #   listen '0.0.0.0:9292'
   #   timeout 10
   #   pid "/tmp/my_app.pid"
   #   after_fork do |server,worker_nr|
   #     server.listen("127.0.0.1:#{9293 + worker_nr}") rescue nil
   #   end
   class Configurator
-    include ::Unicorn::SocketHelper
-
     # The default logger writes its output to $stderr
     DEFAULT_LOGGER = Logger.new($stderr) unless defined?(DEFAULT_LOGGER)
 
     # Default settings for Unicorn
     DEFAULTS = {
       :timeout => 60,
-      :listeners => [ Const::DEFAULT_LISTEN ],
+      :listeners => [],
       :logger => DEFAULT_LOGGER,
       :worker_processes => 1,
       :after_fork => lambda { |server, worker_nr|
@@ -43,7 +42,6 @@ module Unicorn
           server.logger.info("forked child re-executing...")
         },
       :pid => nil,
-      :backlog => 1024,
       :preload_app => false,
       :stderr_path => nil,
       :stdout_path => nil,
@@ -83,23 +81,6 @@ module Unicorn
       @set[key]
     end
 
-    # Changes the listen() syscall backlog to +nr+ for yet-to-be-created
-    # sockets.  Due to limitations of the OS, this cannot affect
-    # existing listener sockets in any way, sockets must be completely
-    # closed and rebound (inherited sockets preserve their existing
-    # backlog setting).  Some operating systems allow negative values
-    # here to specify the maximum allowable value.  See the listen(2)
-    # syscall documentation of your OS for the exact semantics of this.
-    #
-    # If you are running unicorn on multiple machines, lowering this number
-    # can help your load balancer detect when a machine is overloaded
-    # and give requests to a different machine.
-    def backlog(nr)
-      Integer === nr or raise ArgumentError,
-         "not an integer: backlog=#{nr.inspect}"
-      @set[:backlog] = nr
-    end
-
     # sets object to the +new+ Logger-like object.  The new logger-like
     # object must respond to the following methods:
     #  +debug+, +info+, +warn+, +error+, +fatal+, +close+
@@ -171,16 +152,65 @@ module Unicorn
     # sets listeners to the given +addresses+, replacing or augmenting the
     # current set.  This is for the global listener pool shared by all
     # worker processes.  For per-worker listeners, see the after_fork example
-    def listeners(addresses)
+    # This is for internal API use only, do not use it in your Unicorn
+    # config file.  Use listen instead.
+    def listeners(addresses) # :nodoc:
       Array === addresses or addresses = Array(addresses)
       addresses.map! { |addr| expand_addr(addr) }
       @set[:listeners] = addresses
     end
 
-    # adds an +address+ to the existing listener set
-    def listen(address)
+    # adds an +address+ to the existing listener set.
+    #
+    # The following options may be specified (but are generally not needed):
+    #
+    # +backlog+: this is the backlog of the listen() syscall.
+    #
+    #   Some operating systems allow negative values here to specify the
+    #   maximum allowable value.  In most cases, this number is only
+    #   recommendation and there are other OS-specific tunables and
+    #   variables that can affect this number.  See the listen(2)
+    #   syscall documentation of your OS for the exact semantics of
+    #   this.
+    #
+    #   If you are running unicorn on multiple machines, lowering this number
+    #   can help your load balancer detect when a machine is overloaded
+    #   and give requests to a different machine.
+    #
+    #   Default: 1024
+    #
+    # +rcvbuf+, +sndbuf+: maximum send and receive buffer sizes of sockets
+    #
+    #   These correspond to the SO_RCVBUF and SO_SNDBUF settings which
+    #   can be set via the setsockopt(2) syscall.  Some kernels
+    #   (e.g. Linux 2.4+) have intelligent auto-tuning mechanisms and
+    #   there is no need (and it is sometimes detrimental) to specify them.
+    #
+    #   See the socket API documentation of your operating system
+    #   to determine the exact semantics of these settings and
+    #   other operating system-specific knobs where they can be
+    #   specified.
+    #
+    #   Defaults: operating system defaults
+    #
+    # Due to limitations of the operating system, options specified here
+    # cannot affect existing listener sockets in any way, sockets must be
+    # completely closed and rebound.
+    def listen(address, opt = { :backlog => 1024 })
+      address = expand_addr(address)
+      if String === address
+        Hash === @set[:listener_opts] or
+          @set[:listener_opts] = Hash.new { |hash,key| hash[key] = {} }
+        [ :backlog, :sndbuf, :rcvbuf ].each do |key|
+          value = opt[key] or next
+          Integer === value or
+            raise ArgumentError, "not an integer: #{key}=#{value.inspect}"
+        end
+        @set[:listener_opts][address].merge!(opt)
+      end
+
       @set[:listeners] = [] unless Array === @set[:listeners]
-      @set[:listeners] << expand_addr(address)
+      @set[:listeners] << address
     end
 
     # sets the +path+ for the PID file of the unicorn master process
@@ -254,16 +284,26 @@ module Unicorn
       @set[var] = my_proc
     end
 
+    # expands "unix:path/to/foo" to a socket relative to the current path
     # expands pathnames of sockets if relative to "~" or "~username"
     # expands "*:port and ":port" to "0.0.0.0:port"
     def expand_addr(address) #:nodoc
       return address unless String === address
-      if address[0..0] == '~'
-        return File.expand_path(address)
-      elsif address =~ %r{\A\*?:(\d+)\z}
-        return "0.0.0.0:#$1"
+
+      case address
+      when %r{\Aunix:(.*)\z}
+        File.expand_path($1)
+      when %r{\A~}
+        File.expand_path(address)
+      when %r{\A\*:(\d+)\z}
+        "0.0.0.0:#$1"
+      when %r{\A(.*):(\d+)\z}
+        # canonicalize the name
+        packed = Socket.pack_sockaddr_in($2.to_i, $1)
+        Socket.unpack_sockaddr_in(packed).reverse!.join(':')
+      else
+        address
       end
-      address
     end
 
   end

data/lib/unicorn/const.rb

@@ -48,27 +48,17 @@ module Unicorn
   # the constant just refers to a string with the same contents.  Using these constants
   # gave about a 3% to 10% performance improvement over using the strings directly.
   # Symbols did not really improve things much compared to constants.
-  #
-  # While Unicorn does try to emulate the CGI/1.2 protocol, it does not use the REMOTE_IDENT,
-  # REMOTE_USER, or REMOTE_HOST parameters since those are either a security problem or 
-  # too taxing on performance.
   module Const
     DATE="Date".freeze
 
     # This is the part of the path after the SCRIPT_NAME.
     PATH_INFO="PATH_INFO".freeze
     
-    # Request body
-    HTTP_BODY="HTTP_BODY".freeze
-
-    # This is the initial part that your handler is identified as by URIClassifier.
-    SCRIPT_NAME="SCRIPT_NAME".freeze
-
-    # The original URI requested by the client.  Passed to URIClassifier to build PATH_INFO and SCRIPT_NAME.
+    # The original URI requested by the client.
     REQUEST_URI='REQUEST_URI'.freeze
     REQUEST_PATH='REQUEST_PATH'.freeze
     
-    UNICORN_VERSION="0.2.3".freeze
+    UNICORN_VERSION="0.4.1".freeze
 
     UNICORN_TMP_BASE="unicorn".freeze
 
@@ -76,14 +66,6 @@ module Unicorn
     DEFAULT_PORT = "8080".freeze    # default TCP listen port
     DEFAULT_LISTEN = "#{DEFAULT_HOST}:#{DEFAULT_PORT}".freeze
 
-    # The standard empty 404 response for bad requests.  Use Error4040Handler for custom stuff.
-    ERROR_404_RESPONSE="HTTP/1.1 404 Not Found\r\nConnection: close\r\nServer: Unicorn #{UNICORN_VERSION}\r\n\r\nNOT FOUND".freeze
-
-    CONTENT_LENGTH="CONTENT_LENGTH".freeze
-
-    # A common header for indicating the server is too busy.  Not used yet.
-    ERROR_503_RESPONSE="HTTP/1.1 503 Service Unavailable\r\n\r\nBUSY".freeze
-
     # The basic max request size we'll try to read.
     CHUNK_SIZE=(16 * 1024)
 
@@ -94,23 +76,16 @@ module Unicorn
     # Maximum request body size before it is moved out of memory and into a tempfile for reading.
     MAX_BODY=MAX_HEADER
 
+    # common errors we'll send back
+    ERROR_400_RESPONSE = "HTTP/1.1 400 Bad Request\r\n\r\n".freeze
+    ERROR_500_RESPONSE = "HTTP/1.1 500 Internal Server Error\r\n\r\n".freeze
+
     # A frozen format for this is about 15% faster
-    CONTENT_TYPE = "Content-Type".freeze
-    LAST_MODIFIED = "Last-Modified".freeze
-    ETAG = "ETag".freeze
-    REQUEST_METHOD="REQUEST_METHOD".freeze
-    GET="GET".freeze
-    HEAD="HEAD".freeze
-    # ETag is based on the apache standard of hex mtime-size-inode (inode is 0 on win32)
-    ETAG_FORMAT="\"%x-%x-%x\"".freeze
-    LINE_END="\r\n".freeze
+    CONTENT_LENGTH="CONTENT_LENGTH".freeze
     REMOTE_ADDR="REMOTE_ADDR".freeze
     HTTP_X_FORWARDED_FOR="HTTP_X_FORWARDED_FOR".freeze
-    HTTP_IF_MODIFIED_SINCE="HTTP_IF_MODIFIED_SINCE".freeze
-    HTTP_IF_NONE_MATCH="HTTP_IF_NONE_MATCH".freeze
-    REDIRECT = "HTTP/1.1 302 Found\r\nLocation: %s\r\nConnection: close\r\n\r\n".freeze
-    HOST = "HOST".freeze
-    CONNECTION = "Connection".freeze
+    QUERY_STRING="QUERY_STRING".freeze
+    RACK_INPUT="rack.input".freeze
   end
 
 end

data/lib/unicorn/http_request.rb

@@ -13,6 +13,20 @@ module Unicorn
   # 
   class HttpRequest
 
+     # default parameters we merge into the request env for Rack handlers
+     DEF_PARAMS = {
+       "rack.errors" => $stderr,
+       "rack.multiprocess" => true,
+       "rack.multithread" => false,
+       "rack.run_once" => false,
+       "rack.url_scheme" => "http",
+       "rack.version" => [0, 1],
+       "SCRIPT_NAME" => "",
+
+       # this is not in the Rack spec, but some apps may rely on it
+       "SERVER_SOFTWARE" => "Unicorn #{Const::UNICORN_VERSION}"
+     }.freeze
+
     def initialize(logger)
       @logger = logger
       @body = nil
@@ -29,68 +43,47 @@ module Unicorn
       @body = nil
     end
 
-    #
     # Does the majority of the IO processing.  It has been written in
-    # Ruby using about 7 different IO processing strategies and no
-    # matter how it's done the performance just does not improve.  It is
-    # currently carefully constructed to make sure that it gets the best
-    # possible performance, but anyone who thinks they can make it
-    # faster is more than welcome to take a crack at it.
+    # Ruby using about 8 different IO processing strategies.
+    #
+    # It is currently carefully constructed to make sure that it gets
+    # the best possible performance for the common case: GET requests
+    # that are fully complete after a single read(2)
+    #
+    # Anyone who thinks they can make it faster is more than welcome to
+    # take a crack at it.
     #
     # returns an environment hash suitable for Rack if successful
     # This does minimal exception trapping and it is up to the caller
     # to handle any socket errors (e.g. user aborted upload).
     def read(socket)
-      data = String.new(read_socket(socket))
-      nparsed = 0
-
-      # Assumption: nparsed will always be less since data will get
-      # filled with more after each parsing.  If it doesn't get more
-      # then there was a problem with the read operation on the client
-      # socket.  Effect is to stop processing when the socket can't
-      # fill the buffer for further parsing.
-      while nparsed < data.length
-        nparsed = @parser.execute(@params, data, nparsed)
-
-        if @parser.finished?
-          # From http://www.ietf.org/rfc/rfc3875:
-          # "Script authors should be aware that the REMOTE_ADDR and
-          #  REMOTE_HOST meta-variables (see sections 4.1.8 and 4.1.9)
-          #  may not identify the ultimate source of the request.  They
-          #  identify the client for the immediate request to the server;
-          #  that client may be a proxy, gateway, or other intermediary
-          #  acting on behalf of the actual source client."
-          @params[Const::REMOTE_ADDR] = socket.unicorn_peeraddr
-
-          handle_body(socket) and return rack_env # success!
-          return nil # fail
-        else
-          # Parser is not done, queue up more data to read and continue
-          # parsing
-          data << read_socket(socket)
-          if data.length >= Const::MAX_HEADER
-            raise HttpParserError.new("HEADER is longer than allowed, " \
-                                      "aborting client early.")
-          end
-        end
+      # short circuit the common case with small GET requests first
+      @parser.execute(@params, read_socket(socket)) and
+          return handle_body(socket)
+
+      data = @buffer.dup # read_socket will clobber @buffer
+
+      # Parser is not done, queue up more data to read and continue parsing
+      # an Exception thrown from the @parser will throw us out of the loop
+      loop do
+        data << read_socket(socket)
+        @parser.execute(@params, data) and return handle_body(socket)
       end
-      nil # XXX bug?
       rescue HttpParserError => e
         @logger.error "HTTP parse error, malformed request " \
                       "(#{@params[Const::HTTP_X_FORWARDED_FOR] ||
                           socket.unicorn_peeraddr}): #{e.inspect}"
         @logger.error "REQUEST DATA: #{data.inspect}\n---\n" \
                       "PARAMS: #{@params.inspect}\n---\n"
-        socket.closed? or socket.close rescue nil
-        nil
+        raise e
     end
 
     private
 
     # Handles dealing with the rest of the request
-    # returns true if successful, false if not
+    # returns a Rack environment if successful, raises an exception if not
     def handle_body(socket)
-      http_body = @params[Const::HTTP_BODY]
+      http_body = @params.delete(:http_body)
       content_length = @params[Const::CONTENT_LENGTH].to_i
       remain = content_length - http_body.length
 
@@ -108,9 +101,7 @@ module Unicorn
       # Some clients (like FF1.0) report 0 for body and then send a body.
       # This will probably truncate them but at least the request goes through
       # usually.
-      if remain > 0
-        read_body(socket, remain) or return false # fail!
-      end
+      read_body(socket, remain) if remain > 0
       @body.rewind
       @body.sysseek(0) if @body.respond_to?(:sysseek)
 
@@ -118,29 +109,37 @@ module Unicorn
       # another request, we'll truncate it.  Again, we don't do pipelining
       # or keepalive
       @body.truncate(content_length)
-      true
+      rack_env(socket)
     end
 
     # Returns an environment which is rackable:
     # http://rack.rubyforge.org/doc/files/SPEC.html
     # Based on Rack's old Mongrel handler.
-    def rack_env
+    def rack_env(socket)
+      # I'm considering enabling "unicorn.client".  It gives
+      # applications some rope to do some "interesting" things like
+      # replacing a worker with another process that has full control
+      # over the HTTP response.
+      # @params["unicorn.client"] = socket
+
+      # From http://www.ietf.org/rfc/rfc3875:
+      # "Script authors should be aware that the REMOTE_ADDR and
+      #  REMOTE_HOST meta-variables (see sections 4.1.8 and 4.1.9)
+      #  may not identify the ultimate source of the request.  They
+      #  identify the client for the immediate request to the server;
+      #  that client may be a proxy, gateway, or other intermediary
+      #  acting on behalf of the actual source client."
+      @params[Const::REMOTE_ADDR] = socket.unicorn_peeraddr
+
       # It might be a dumbass full host request header
-      @params[Const::REQUEST_PATH] ||=
-                           URI.parse(@params[Const::REQUEST_URI]).path
-      raise "No REQUEST PATH" unless @params[Const::REQUEST_PATH]
-
-      @params["QUERY_STRING"] ||= ''
-      @params.update({ "rack.version" => [0,1],
-                      "rack.input" => @body,
-                      "rack.errors" => $stderr,
-                      "rack.multithread" => false,
-                      "rack.multiprocess" => true,
-                      "rack.run_once" => false,
-                      "rack.url_scheme" => "http",
-                      Const::PATH_INFO => @params[Const::REQUEST_PATH],
-                      Const::SCRIPT_NAME => "",
-                    })
+      @params[Const::PATH_INFO] = (
+          @params[Const::REQUEST_PATH] ||=
+              URI.parse(@params[Const::REQUEST_URI]).path) or
+         raise "No REQUEST_PATH"
+
+      @params[Const::QUERY_STRING] ||= ''
+      @params[Const::RACK_INPUT] = @body
+      @params.update(DEF_PARAMS)
     end
 
     # Does the heavy lifting of properly reading the larger body requests in
@@ -152,16 +151,14 @@ module Unicorn
         # writes always write the requested amount on a POSIX filesystem
         remain -= @body.syswrite(read_socket(socket))
       end
-      true # success!
     rescue Object => e
       @logger.error "Error reading HTTP body: #{e.inspect}"
-      socket.closed? or socket.close rescue nil
 
       # Any errors means we should delete the file, including if the file
       # is dumped.  Truncate it ASAP to help avoid page flushes to disk.
       @body.truncate(0) rescue nil
       reset
-      false
+      raise e
     end
 
     # read(2) on "slow" devices like sockets can be interrupted by signals

data/lib/unicorn/http_response.rb

@@ -35,7 +35,11 @@ module Unicorn
       # the time anyways so just hope our app knows what it's doing
       headers.each do |key, value|
         next if SKIP.include?(key.downcase)
-        value.split(/\n/).each { |v| out << "#{key}: #{v}" }
+        if value =~ /\n/
+          value.split(/\n/).each { |v| out << "#{key}: #{v}" }
+        else
+          out << "#{key}: #{value}"
+        end
       end
 
       # Rack should enforce Content-Length or chunked transfer encoding,
@@ -45,6 +49,7 @@ module Unicorn
                    "Connection: close\r\n" \
                    "#{out.join("\r\n")}\r\n\r\n")
       body.each { |chunk| socket_write(socket, chunk) }
+      socket.close # uncorks the socket immediately
       ensure
         body.respond_to?(:close) and body.close rescue nil
     end

data/lib/unicorn/socket.rb

@@ -62,10 +62,17 @@ module Unicorn
       end
     end
 
+    def log_buffer_sizes(sock, pfx = '')
+      respond_to?(:logger) or return
+      rcvbuf = sock.getsockopt(SOL_SOCKET, SO_RCVBUF).unpack('i')
+      sndbuf = sock.getsockopt(SOL_SOCKET, SO_SNDBUF).unpack('i')
+      logger.info "#{pfx}#{sock_name(sock)} rcvbuf=#{rcvbuf} sndbuf=#{sndbuf}"
+    end
+
     # creates a new server, socket. address may be a HOST:PORT or
     # an absolute path to a UNIX socket.  address can even be a Socket
     # object in which case it is immediately returned
-    def bind_listen(address = '0.0.0.0:8080', backlog = 1024)
+    def bind_listen(address = '0.0.0.0:8080', opt = { :backlog => 1024 })
       return address unless String === address
 
       domain, bind_addr = if address[0..0] == "/"
@@ -95,7 +102,13 @@ module Unicorn
         sock.close rescue nil
         return nil
       end
-      sock.listen(backlog)
+      if opt[:rcvbuf] || opt[:sndbuf]
+        log_buffer_sizes(sock, "before: ")
+        sock.setsockopt(SOL_SOCKET, SO_RCVBUF, opt[:rcvbuf]) if opt[:rcvbuf]
+        sock.setsockopt(SOL_SOCKET, SO_SNDBUF, opt[:sndbuf]) if opt[:sndbuf]
+        log_buffer_sizes(sock, " after: ")
+      end
+      sock.listen(opt[:backlog] || 1024)
       set_server_sockopt(sock) if domain == AF_INET
       sock
     end