unicorn-fotopedia 0.99.1

data/lib/unicorn/http_request.rb

@@ -0,0 +1,72 @@
+# -*- encoding: binary -*-
+
+require 'stringio'
+require 'unicorn_http'
+
+module Unicorn
+  class HttpRequest
+
+    # default parameters we merge into the request env for Rack handlers
+    DEFAULTS = {
+      "rack.errors" => $stderr,
+      "rack.multiprocess" => true,
+      "rack.multithread" => false,
+      "rack.run_once" => false,
+      "rack.version" => [1, 1],
+      "SCRIPT_NAME" => "",
+
+      # this is not in the Rack spec, but some apps may rely on it
+      "SERVER_SOFTWARE" => "Unicorn #{Const::UNICORN_VERSION}"
+    }
+
+    NULL_IO = StringIO.new("")
+    LOCALHOST = '127.0.0.1'
+
+    # Being explicitly single-threaded, we have certain advantages in
+    # not having to worry about variables being clobbered :)
+    BUF = ""
+    PARSER = HttpParser.new
+    REQ = {}
+
+    # Does the majority of the IO processing.  It has been written in
+    # Ruby using about 8 different IO processing strategies.
+    #
+    # It is currently carefully constructed to make sure that it gets
+    # the best possible performance for the common case: GET requests
+    # that are fully complete after a single read(2)
+    #
+    # Anyone who thinks they can make it faster is more than welcome to
+    # take a crack at it.
+    #
+    # returns an environment hash suitable for Rack if successful
+    # This does minimal exception trapping and it is up to the caller
+    # to handle any socket errors (e.g. user aborted upload).
+    def read(socket)
+      REQ.clear
+      PARSER.reset
+
+      # From http://www.ietf.org/rfc/rfc3875:
+      # "Script authors should be aware that the REMOTE_ADDR and
+      #  REMOTE_HOST meta-variables (see sections 4.1.8 and 4.1.9)
+      #  may not identify the ultimate source of the request.  They
+      #  identify the client for the immediate request to the server;
+      #  that client may be a proxy, gateway, or other intermediary
+      #  acting on behalf of the actual source client."
+      REQ[Const::REMOTE_ADDR] =
+                    TCPSocket === socket ? socket.peeraddr.last : LOCALHOST
+
+      # short circuit the common case with small GET requests first
+      if PARSER.headers(REQ, socket.readpartial(Const::CHUNK_SIZE, BUF)).nil?
+        # Parser is not done, queue up more data to read and continue parsing
+        # an Exception thrown from the PARSER will throw us out of the loop
+        begin
+          BUF << socket.readpartial(Const::CHUNK_SIZE)
+        end while PARSER.headers(REQ, BUF).nil?
+      end
+      REQ[Const::RACK_INPUT] = 0 == PARSER.content_length ?
+                   NULL_IO : Unicorn::TeeInput.new(socket, REQ, PARSER, BUF)
+      REQ.update(DEFAULTS)
+    end
+
+  end
+end

data/lib/unicorn/http_response.rb

@@ -0,0 +1,75 @@
+# -*- encoding: binary -*-
+
+require 'time'
+
+module Unicorn
+  # Writes a Rack response to your client using the HTTP/1.1 specification.
+  # You use it by simply doing:
+  #
+  #   status, headers, body = rack_app.call(env)
+  #   HttpResponse.write(socket, [ status, headers, body ])
+  #
+  # Most header correctness (including Content-Length and Content-Type)
+  # is the job of Rack, with the exception of the "Connection: close"
+  # and "Date" headers.
+  #
+  # A design decision was made to force the client to not pipeline or
+  # keepalive requests.  HTTP/1.1 pipelining really kills the
+  # performance due to how it has to be handled and how unclear the
+  # standard is.  To fix this the HttpResponse always gives a
+  # "Connection: close" header which forces the client to close right
+  # away.  The bonus for this is that it gives a pretty nice speed boost
+  # to most clients since they can close their connection immediately.
+
+  class HttpResponse
+
+    # Every standard HTTP code mapped to the appropriate message.
+    CODES = Rack::Utils::HTTP_STATUS_CODES.inject({}) { |hash,(code,msg)|
+      hash[code] = "#{code} #{msg}"
+      hash
+    }
+
+    # Rack does not set/require a Date: header.  We always override the
+    # Connection: and Date: headers no matter what (if anything) our
+    # Rack application sent us.
+    SKIP = { 'connection' => true, 'date' => true, 'status' => true }
+
+    # writes the rack_response to socket as an HTTP response
+    def self.write(socket, rack_response, have_header = true)
+      status, headers, body = rack_response
+
+      if have_header
+        status = CODES[status.to_i] || status
+        out = []
+
+        # Don't bother enforcing duplicate supression, it's a Hash most of
+        # the time anyways so just hope our app knows what it's doing
+        headers.each do |key, value|
+          next if SKIP.include?(key.downcase)
+          if value =~ /\n/
+            # avoiding blank, key-only cookies with /\n+/
+            out.concat(value.split(/\n+/).map! { |v| "#{key}: #{v}\r\n" })
+          else
+            out << "#{key}: #{value}\r\n"
+          end
+        end
+
+        # Rack should enforce Content-Length or chunked transfer encoding,
+        # so don't worry or care about them.
+        # Date is required by HTTP/1.1 as long as our clock can be trusted.
+        # Some broken clients require a "Status" header so we accomodate them
+        socket.write("HTTP/1.1 #{status}\r\n" \
+                     "Date: #{Time.now.httpdate}\r\n" \
+                     "Status: #{status}\r\n" \
+                     "Connection: close\r\n" \
+                     "#{out.join('')}\r\n")
+      end
+
+      body.each { |chunk| socket.write(chunk) }
+      socket.close # flushes and uncorks the socket immediately
+      ensure
+        body.respond_to?(:close) and body.close
+    end
+
+  end
+end

data/lib/unicorn/launcher.rb

@@ -0,0 +1,65 @@
+# -*- encoding: binary -*-
+
+$stdout.sync = $stderr.sync = true
+$stdin.binmode
+$stdout.binmode
+$stderr.binmode
+
+require 'unicorn'
+
+class Unicorn::Launcher
+
+  # We don't do a lot of standard daemonization stuff:
+  #   * umask is whatever was set by the parent process at startup
+  #     and can be set in config.ru and config_file, so making it
+  #     0000 and potentially exposing sensitive log data can be bad
+  #     policy.
+  #   * don't bother to chdir("/") here since unicorn is designed to
+  #     run inside APP_ROOT.  Unicorn will also re-chdir() to
+  #     the directory it was started in when being re-executed
+  #     to pickup code changes if the original deployment directory
+  #     is a symlink or otherwise got replaced.
+  def self.daemonize!(options = nil)
+    $stdin.reopen("/dev/null")
+
+    # We only start a new process group if we're not being reexecuted
+    # and inheriting file descriptors from our parent
+    unless ENV['UNICORN_FD']
+      if options
+        # grandparent - reads pipe, exits when master is ready
+        #  \_ parent  - exits immediately ASAP
+        #      \_ unicorn master - writes to pipe when ready
+
+        rd, wr = IO.pipe
+        grandparent = $$
+        if fork
+          wr.close # grandparent does not write
+        else
+          rd.close # unicorn master does not read
+          Process.setsid
+          exit if fork # parent dies now
+        end
+
+        if grandparent == $$
+          # this will block until HttpServer#join runs (or it dies)
+          master_pid = (rd.readpartial(16) rescue nil).to_i
+          unless master_pid > 1
+            warn "master failed to start, check stderr log for details"
+            exit!(1)
+          end
+          exit 0
+        else # unicorn master process
+          options[:ready_pipe] = wr
+        end
+      else # backwards compat
+        exit if fork
+        Process.setsid
+        exit if fork
+      end
+      # $stderr/$stderr can/will be redirected separately in the Unicorn config
+      Unicorn::Configurator::DEFAULTS[:stderr_path] = "/dev/null"
+      Unicorn::Configurator::DEFAULTS[:stdout_path] = "/dev/null"
+    end
+  end
+
+end

data/lib/unicorn/oob_gc.rb

@@ -0,0 +1,58 @@
+# -*- encoding: binary -*-
+module Unicorn
+
+  # Run GC after every request, after closing the client socket and
+  # before attempting to accept more connections.
+  #
+  # This shouldn't hurt overall performance as long as the server cluster
+  # is at <50% CPU capacity, and improves the performance of most memory
+  # intensive requests.  This serves to improve _client-visible_
+  # performance (possibly at the cost of overall performance).
+  #
+  # We'll call GC after each request is been written out to the socket, so
+  # the client never sees the extra GC hit it.
+  #
+  # This middleware is _only_ effective for applications that use a lot
+  # of memory, and will hurt simpler apps/endpoints that can process
+  # multiple requests before incurring GC.
+  #
+  # This middleware is only designed to work with Unicorn, as it harms
+  # keepalive performance.
+  #
+  # Example (in config.ru):
+  #
+  #     require 'unicorn/oob_gc'
+  #
+  #     # GC ever two requests that hit /expensive/foo or /more_expensive/foo
+  #     # in your app.  By default, this will GC once every 5 requests
+  #     # for all endpoints in your app
+  #     use Unicorn::OobGC, 2, %r{\A/(?:expensive/foo|more_expensive/foo)}
+  class OobGC < Struct.new(:app, :interval, :path, :nr, :env, :body)
+
+    def initialize(app, interval = 5, path = %r{\A/})
+      super(app, interval, path, interval)
+    end
+
+    def call(env)
+      status, headers, self.body = app.call(self.env = env)
+      [ status, headers, self ]
+    end
+
+    def each(&block)
+      body.each(&block)
+    end
+
+    # in Unicorn, this is closed _after_ the client socket
+    def close
+      body.close if body.respond_to?(:close)
+
+      if path =~ env['PATH_INFO'] && ((self.nr -= 1) <= 0)
+        self.nr = interval
+        self.body = nil
+        env.clear
+        GC.start
+      end
+    end
+
+  end
+end

data/lib/unicorn/socket_helper.rb

@@ -0,0 +1,152 @@
+# -*- encoding: binary -*-
+
+require 'socket'
+
+module Unicorn
+  module SocketHelper
+    include Socket::Constants
+
+    # configure platform-specific options (only tested on Linux 2.6 so far)
+    case RUBY_PLATFORM
+    when /linux/
+      # from /usr/include/linux/tcp.h
+      TCP_DEFER_ACCEPT = 9 unless defined?(TCP_DEFER_ACCEPT)
+
+      # do not send out partial frames (Linux)
+      TCP_CORK = 3 unless defined?(TCP_CORK)
+    when /freebsd(([1-4]\..{1,2})|5\.[0-4])/
+      # Do nothing for httpready, just closing a bug when freebsd <= 5.4
+      TCP_NOPUSH = 4 unless defined?(TCP_NOPUSH) # :nodoc:
+    when /freebsd/
+      # do not send out partial frames (FreeBSD)
+      TCP_NOPUSH = 4 unless defined?(TCP_NOPUSH)
+
+      # Use the HTTP accept filter if available.
+      # The struct made by pack() is defined in /usr/include/sys/socket.h
+      # as accept_filter_arg
+      unless `/sbin/sysctl -nq net.inet.accf.http`.empty?
+        # set set the "httpready" accept filter in FreeBSD if available
+        # if other protocols are to be supported, this may be
+        # String#replace-d with "dataready" arguments instead
+        FILTER_ARG = ['httpready', nil].pack('a16a240')
+      end
+    end
+
+    def set_tcp_sockopt(sock, opt)
+
+      # highly portable, but off by default because we don't do keepalive
+      if defined?(TCP_NODELAY) && ! (val = opt[:tcp_nodelay]).nil?
+        sock.setsockopt(IPPROTO_TCP, TCP_NODELAY, val ? 1 : 0)
+      end
+
+      unless (val = opt[:tcp_nopush]).nil?
+        val = val ? 1 : 0
+        if defined?(TCP_CORK) # Linux
+          sock.setsockopt(IPPROTO_TCP, TCP_CORK, val)
+        elsif defined?(TCP_NOPUSH) # TCP_NOPUSH is untested (FreeBSD)
+          sock.setsockopt(IPPROTO_TCP, TCP_NOPUSH, val)
+        end
+      end
+
+      # No good reason to ever have deferred accepts off
+      if defined?(TCP_DEFER_ACCEPT)
+        sock.setsockopt(SOL_TCP, TCP_DEFER_ACCEPT, 1)
+      elsif defined?(SO_ACCEPTFILTER) && defined?(FILTER_ARG)
+        sock.setsockopt(SOL_SOCKET, SO_ACCEPTFILTER, FILTER_ARG)
+      end
+    end
+
+    def set_server_sockopt(sock, opt)
+      opt ||= {}
+
+      TCPSocket === sock and set_tcp_sockopt(sock, opt)
+
+      if opt[:rcvbuf] || opt[:sndbuf]
+        log_buffer_sizes(sock, "before: ")
+        sock.setsockopt(SOL_SOCKET, SO_RCVBUF, opt[:rcvbuf]) if opt[:rcvbuf]
+        sock.setsockopt(SOL_SOCKET, SO_SNDBUF, opt[:sndbuf]) if opt[:sndbuf]
+        log_buffer_sizes(sock, " after: ")
+      end
+      sock.listen(opt[:backlog] || 1024)
+      rescue => e
+        if respond_to?(:logger)
+          logger.error "error setting socket options: #{e.inspect}"
+          logger.error e.backtrace.join("\n")
+        end
+    end
+
+    def log_buffer_sizes(sock, pfx = '')
+      respond_to?(:logger) or return
+      rcvbuf = sock.getsockopt(SOL_SOCKET, SO_RCVBUF).unpack('i')
+      sndbuf = sock.getsockopt(SOL_SOCKET, SO_SNDBUF).unpack('i')
+      logger.info "#{pfx}#{sock_name(sock)} rcvbuf=#{rcvbuf} sndbuf=#{sndbuf}"
+    end
+
+    # creates a new server, socket. address may be a HOST:PORT or
+    # an absolute path to a UNIX socket.  address can even be a Socket
+    # object in which case it is immediately returned
+    def bind_listen(address = '0.0.0.0:8080', opt = {})
+      return address unless String === address
+
+      sock = if address[0] == ?/
+        if File.exist?(address)
+          if File.socket?(address)
+            if self.respond_to?(:logger)
+              logger.info "unlinking existing socket=#{address}"
+            end
+            File.unlink(address)
+          else
+            raise ArgumentError,
+                  "socket=#{address} specified but it is not a socket!"
+          end
+        end
+        old_umask = File.umask(opt[:umask] || 0)
+        begin
+          UNIXServer.new(address)
+        ensure
+          File.umask(old_umask)
+        end
+      elsif address =~ /^(\d+\.\d+\.\d+\.\d+):(\d+)$/
+        TCPServer.new($1, $2.to_i)
+      else
+        raise ArgumentError, "Don't know how to bind: #{address}"
+      end
+      set_server_sockopt(sock, opt)
+      sock
+    end
+
+    # Returns the configuration name of a socket as a string.  sock may
+    # be a string value, in which case it is returned as-is
+    # Warning: TCP sockets may not always return the name given to it.
+    def sock_name(sock)
+      case sock
+      when String then sock
+      when UNIXServer
+        Socket.unpack_sockaddr_un(sock.getsockname)
+      when TCPServer
+        Socket.unpack_sockaddr_in(sock.getsockname).reverse!.join(':')
+      when Socket
+        begin
+          Socket.unpack_sockaddr_in(sock.getsockname).reverse!.join(':')
+        rescue ArgumentError
+          Socket.unpack_sockaddr_un(sock.getsockname)
+        end
+      else
+        raise ArgumentError, "Unhandled class #{sock.class}: #{sock.inspect}"
+      end
+    end
+
+    module_function :sock_name
+
+    # casts a given Socket to be a TCPServer or UNIXServer
+    def server_cast(sock)
+      begin
+        Socket.unpack_sockaddr_in(sock.getsockname)
+        TCPServer.for_fd(sock.fileno)
+      rescue ArgumentError
+        UNIXServer.for_fd(sock.fileno)
+      end
+    end
+
+  end # module SocketHelper
+end # module Unicorn

data/lib/unicorn/tee_input.rb

@@ -0,0 +1,217 @@
+# -*- encoding: binary -*-
+
+module Unicorn
+
+  # acts like tee(1) on an input input to provide a input-like stream
+  # while providing rewindable semantics through a File/StringIO backing
+  # store.  On the first pass, the input is only read on demand so your
+  # Rack application can use input notification (upload progress and
+  # like).  This should fully conform to the Rack::Lint::InputWrapper
+  # specification on the public API.  This class is intended to be a
+  # strict interpretation of Rack::Lint::InputWrapper functionality and
+  # will not support any deviations from it.
+  #
+  # When processing uploads, Unicorn exposes a TeeInput object under
+  # "rack.input" of the Rack environment.
+  class TeeInput < Struct.new(:socket, :req, :parser, :buf, :len, :tmp, :buf2)
+
+    # Initializes a new TeeInput object.  You normally do not have to call
+    # this unless you are writing an HTTP server.
+    def initialize(*args)
+      super(*args)
+      self.len = parser.content_length
+      self.tmp = len && len < Const::MAX_BODY ? StringIO.new("") : Util.tmpio
+      self.buf2 = ""
+      if buf.size > 0
+        parser.filter_body(buf2, buf) and finalize_input
+        tmp.write(buf2)
+        tmp.seek(0)
+      end
+    end
+
+    # :call-seq:
+    #   ios.size  => Integer
+    #
+    # Returns the size of the input.  For requests with a Content-Length
+    # header value, this will not read data off the socket and just return
+    # the value of the Content-Length header as an Integer.
+    #
+    # For Transfer-Encoding:chunked requests, this requires consuming
+    # all of the input stream before returning since there's no other
+    # way to determine the size of the request body beforehand.
+    def size
+      len and return len
+
+      if socket
+        pos = tmp.pos
+        while tee(Const::CHUNK_SIZE, buf2)
+        end
+        tmp.seek(pos)
+      end
+
+      self.len = tmp.size
+    end
+
+    # :call-seq:
+    #   ios.read([length [, buffer ]]) => string, buffer, or nil
+    #
+    # Reads at most length bytes from the I/O stream, or to the end of
+    # file if length is omitted or is nil. length must be a non-negative
+    # integer or nil. If the optional buffer argument is present, it
+    # must reference a String, which will receive the data.
+    #
+    # At end of file, it returns nil or "" depend on length.
+    # ios.read() and ios.read(nil) returns "".
+    # ios.read(length [, buffer]) returns nil.
+    #
+    # If the Content-Length of the HTTP request is known (as is the common
+    # case for POST requests), then ios.read(length [, buffer]) will block
+    # until the specified length is read (or it is the last chunk).
+    # Otherwise, for uncommon "Transfer-Encoding: chunked" requests,
+    # ios.read(length [, buffer]) will return immediately if there is
+    # any data and only block when nothing is available (providing
+    # IO#readpartial semantics).
+    def read(*args)
+      socket or return tmp.read(*args)
+
+      length = args.shift
+      if nil == length
+        rv = tmp.read || ""
+        while tee(Const::CHUNK_SIZE, buf2)
+          rv << buf2
+        end
+        rv
+      else
+        rv = args.shift || ""
+        diff = tmp.size - tmp.pos
+        if 0 == diff
+          ensure_length(tee(length, rv), length)
+        else
+          ensure_length(tmp.read(diff > length ? length : diff, rv), length)
+        end
+      end
+    end
+
+    # :call-seq:
+    #   ios.gets   => string or nil
+    #
+    # Reads the next ``line'' from the I/O stream; lines are separated
+    # by the global record separator ($/, typically "\n"). A global
+    # record separator of nil reads the entire unread contents of ios.
+    # Returns nil if called at the end of file.
+    # This takes zero arguments for strict Rack::Lint compatibility,
+    # unlike IO#gets.
+    def gets
+      socket or return tmp.gets
+      nil == $/ and return read
+
+      orig_size = tmp.size
+      if tmp.pos == orig_size
+        tee(Const::CHUNK_SIZE, buf2) or return nil
+        tmp.seek(orig_size)
+      end
+
+      line = tmp.gets # cannot be nil here since size > pos
+      $/ == line[-$/.size, $/.size] and return line
+
+      # unlikely, if we got here, then tmp is at EOF
+      begin
+        orig_size = tmp.pos
+        tee(Const::CHUNK_SIZE, buf2) or break
+        tmp.seek(orig_size)
+        line << tmp.gets
+        $/ == line[-$/.size, $/.size] and return line
+        # tmp is at EOF again here, retry the loop
+      end while true
+
+      line
+    end
+
+    # :call-seq:
+    #   ios.each { |line| block }  => ios
+    #
+    # Executes the block for every ``line'' in *ios*, where lines are
+    # separated by the global record separator ($/, typically "\n").
+    def each(&block)
+      while line = gets
+        yield line
+      end
+
+      self # Rack does not specify what the return value is here
+    end
+
+    # :call-seq:
+    #   ios.rewind    => 0
+    #
+    # Positions the *ios* pointer to the beginning of input, returns
+    # the offset (zero) of the +ios+ pointer.  Subsequent reads will
+    # start from the beginning of the previously-buffered input.
+    def rewind
+      tmp.rewind # Rack does not specify what the return value is here
+    end
+
+  private
+
+    def client_error(e)
+      case e
+      when EOFError
+        # in case client only did a premature shutdown(SHUT_WR)
+        # we do support clients that shutdown(SHUT_WR) after the
+        # _entire_ request has been sent, and those will not have
+        # raised EOFError on us.
+        socket.close if socket
+        raise ClientShutdown, "bytes_read=#{tmp.size}", []
+      when HttpParserError
+        e.set_backtrace([])
+      end
+      raise e
+    end
+
+    # tees off a +length+ chunk of data from the input into the IO
+    # backing store as well as returning it.  +dst+ must be specified.
+    # returns nil if reading from the input returns nil
+    def tee(length, dst)
+      unless parser.body_eof?
+        if parser.filter_body(dst, socket.readpartial(length, buf)).nil?
+          tmp.write(dst)
+          tmp.seek(0, IO::SEEK_END) # workaround FreeBSD/OSX + MRI 1.8.x bug
+          return dst
+        end
+      end
+      finalize_input
+      rescue => e
+        client_error(e)
+    end
+
+    def finalize_input
+      while parser.trailers(req, buf).nil?
+        # Don't worry about raising ClientShutdown here on EOFError, tee()
+        # will catch EOFError when app is processing it, otherwise in
+        # initialize we never get any chance to enter the app so the
+        # EOFError will just get trapped by Unicorn and not the Rack app
+        buf << socket.readpartial(Const::CHUNK_SIZE)
+      end
+      self.socket = nil
+    end
+
+    # tee()s into +dst+ until it is of +length+ bytes (or until
+    # we've reached the Content-Length of the request body).
+    # Returns +dst+ (the exact object, not a duplicate)
+    # To continue supporting applications that need near-real-time
+    # streaming input bodies, this is a no-op for
+    # "Transfer-Encoding: chunked" requests.
+    def ensure_length(dst, length)
+      # len is nil for chunked bodies, so we can't ensure length for those
+      # since they could be streaming bidirectionally and we don't want to
+      # block the caller in that case.
+      return dst if dst.nil? || len.nil?
+
+      while dst.size < length && tee(length - dst.size, buf2)
+        dst << buf2
+      end
+
+      dst
+    end
+
+  end
+end