unicorn 0.8.4 → 0.9.0

data/lib/unicorn/configurator.rb

@@ -65,7 +65,7 @@ module Unicorn
     def commit!(server, options = {}) #:nodoc:
       skip = options[:skip] || []
       @set.each do |key, value|
-        (Symbol === value && value == :unset) and next
+        value == :unset and next
         skip.include?(key) and next
         setter = "#{key}="
         if server.respond_to?(setter)

data/lib/unicorn/const.rb

@@ -5,7 +5,7 @@ module Unicorn
   # gave about a 3% to 10% performance improvement over using the strings directly.
   # Symbols did not really improve things much compared to constants.
   module Const
-    UNICORN_VERSION="0.8.4".freeze
+    UNICORN_VERSION="0.9.0".freeze
 
     DEFAULT_HOST = "0.0.0.0".freeze # default TCP listen host address
     DEFAULT_PORT = "8080".freeze    # default TCP listen port
@@ -24,11 +24,15 @@ module Unicorn
     # common errors we'll send back
     ERROR_400_RESPONSE = "HTTP/1.1 400 Bad Request\r\n\r\n".freeze
     ERROR_500_RESPONSE = "HTTP/1.1 500 Internal Server Error\r\n\r\n".freeze
+    EXPECT_100_RESPONSE = "HTTP/1.1 100 Continue\r\n\r\n"
 
     # A frozen format for this is about 15% faster
+    HTTP_TRANSFER_ENCODING = 'HTTP_TRANSFER_ENCODING'.freeze
     CONTENT_LENGTH="CONTENT_LENGTH".freeze
     REMOTE_ADDR="REMOTE_ADDR".freeze
     HTTP_X_FORWARDED_FOR="HTTP_X_FORWARDED_FOR".freeze
+    HTTP_EXPECT="HTTP_EXPECT".freeze
+    HTTP_TRAILER="HTTP_TRAILER".freeze
     RACK_INPUT="rack.input".freeze
   end
 

data/lib/unicorn/http_request.rb

@@ -1,15 +1,9 @@
-require 'tempfile'
 require 'stringio'
 
 # compiled extension
 require 'unicorn/http11'
 
 module Unicorn
-  #
-  # The HttpRequest.initialize method will convert any request that is larger than
-  # Const::MAX_BODY into a Tempfile and use that as the body.  Otherwise it uses 
-  # a StringIO object.  To be safe, you should assume it works like a file.
-  # 
   class HttpRequest
 
     attr_accessor :logger
@@ -27,14 +21,13 @@ module Unicorn
       "SERVER_SOFTWARE" => "Unicorn #{Const::UNICORN_VERSION}".freeze
     }
 
-    # Optimize for the common case where there's no request body
-    # (GET/HEAD) requests.
-    NULL_IO = StringIO.new
+    NULL_IO = StringIO.new(Z)
     LOCALHOST = '127.0.0.1'.freeze
 
     # Being explicitly single-threaded, we have certain advantages in
     # not having to worry about variables being clobbered :)
     BUFFER = ' ' * Const::CHUNK_SIZE # initial size, may grow
+    BUFFER.force_encoding(Encoding::BINARY) if Z.respond_to?(:force_encoding)
     PARSER = HttpParser.new
     PARAMS = Hash.new
 
@@ -56,11 +49,6 @@ module Unicorn
     # This does minimal exception trapping and it is up to the caller
     # to handle any socket errors (e.g. user aborted upload).
     def read(socket)
-      # reset the parser
-      unless NULL_IO == (input = PARAMS[Const::RACK_INPUT]) # unlikely
-        input.close rescue nil
-        input.close! rescue nil
-      end
       PARAMS.clear
       PARSER.reset
 
@@ -98,57 +86,25 @@ module Unicorn
     private
 
     # Handles dealing with the rest of the request
-    # returns a Rack environment if successful, raises an exception if not
+    # returns a Rack environment if successful
     def handle_body(socket)
-      http_body = PARAMS.delete(:http_body)
-      content_length = PARAMS[Const::CONTENT_LENGTH].to_i
-
-      if content_length == 0 # short circuit the common case
-        PARAMS[Const::RACK_INPUT] = NULL_IO.closed? ? NULL_IO.reopen : NULL_IO
-        return PARAMS.update(DEFAULTS)
+      PARAMS[Const::RACK_INPUT] = if (body = PARAMS.delete(:http_body))
+        length = PARAMS[Const::CONTENT_LENGTH].to_i
+
+        if te = PARAMS[Const::HTTP_TRANSFER_ENCODING]
+          if /\Achunked\z/i =~ te
+            socket = ChunkedReader.new(PARAMS, socket, body)
+            length = body = nil
+          end
+        end
+
+        TeeInput.new(socket, length, body)
+      else
+        NULL_IO.closed? ? NULL_IO.reopen(Z) : NULL_IO
       end
 
-      # must read more data to complete body
-      remain = content_length - http_body.length
-
-      body = PARAMS[Const::RACK_INPUT] = (remain < Const::MAX_BODY) ?
-          StringIO.new : Tempfile.new('unicorn')
-
-      body.binmode
-      body.write(http_body)
-
-      # Some clients (like FF1.0) report 0 for body and then send a body.
-      # This will probably truncate them but at least the request goes through
-      # usually.
-      read_body(socket, remain, body) if remain > 0
-      body.rewind
-
-      # in case read_body overread because the client tried to pipeline
-      # another request, we'll truncate it.  Again, we don't do pipelining
-      # or keepalive
-      body.truncate(content_length)
       PARAMS.update(DEFAULTS)
     end
 
-    # Does the heavy lifting of properly reading the larger body
-    # requests in small chunks.  It expects PARAMS['rack.input'] to be
-    # an IO object, socket to be valid, It also expects any initial part
-    # of the body that has been read to be in the PARAMS['rack.input']
-    # already.  It will return true if successful and false if not.
-    def read_body(socket, remain, body)
-      begin
-        # write always writes the requested amount on a POSIX filesystem
-        remain -= body.write(socket.readpartial(Const::CHUNK_SIZE, BUFFER))
-      end while remain > 0
-    rescue Object => e
-      @logger.error "Error reading HTTP body: #{e.inspect}"
-
-      # Any errors means we should delete the file, including if the file
-      # is dumped.  Truncate it ASAP to help avoid page flushes to disk.
-      body.truncate(0) rescue nil
-      reset
-      raise e
-    end
-
   end
 end

data/lib/unicorn/http_response.rb

@@ -31,13 +31,12 @@ module Unicorn
     # Connection: and Date: headers no matter what (if anything) our
     # Rack application sent us.
     SKIP = { 'connection' => true, 'date' => true, 'status' => true }.freeze
-    EMPTY = ''.freeze # :nodoc
     OUT = [] # :nodoc
 
     # writes the rack_response to socket as an HTTP response
     def self.write(socket, rack_response)
       status, headers, body = rack_response
-      status = CODES[status.to_i] || status
+      status = CODES[status.to_i]
       OUT.clear
 
       # Don't bother enforcing duplicate supression, it's a Hash most of
@@ -59,7 +58,7 @@ module Unicorn
                    "Date: #{Time.now.httpdate}\r\n" \
                    "Status: #{status}\r\n" \
                    "Connection: close\r\n" \
-                   "#{OUT.join(EMPTY)}\r\n")
+                   "#{OUT.join(Z)}\r\n")
       body.each { |chunk| socket.write(chunk) }
       socket.close # flushes and uncorks the socket immediately
       ensure

data/lib/unicorn/tee_input.rb

@@ -0,0 +1,135 @@
+# Copyright (c) 2009 Eric Wong
+# You can redistribute it and/or modify it under the same terms as Ruby.
+
+require 'tempfile'
+
+# acts like tee(1) on an input input to provide a input-like stream
+# while providing rewindable semantics through a Tempfile/StringIO
+# backing store.  On the first pass, the input is only read on demand
+# so your Rack application can use input notification (upload progress
+# and like).  This should fully conform to the Rack::InputWrapper
+# specification on the public API.  This class is intended to be a
+# strict interpretation of Rack::InputWrapper functionality and will
+# not support any deviations from it.
+
+module Unicorn
+  class TeeInput
+
+    def initialize(input, size, body)
+      @tmp = Tempfile.new(nil)
+      @tmp.unlink
+      @tmp.binmode
+      @tmp.sync = true
+
+      if body
+        @tmp.write(body)
+        @tmp.seek(0)
+      end
+      @input = input
+      @size = size # nil if chunked
+    end
+
+    # returns the size of the input.  This is what the Content-Length
+    # header value should be, and how large our input is expected to be.
+    # For TE:chunked, this requires consuming all of the input stream
+    # before returning since there's no other way
+    def size
+      @size and return @size
+
+      if @input
+        buf = Z.dup
+        while tee(Const::CHUNK_SIZE, buf)
+        end
+        @tmp.rewind
+      end
+
+      @size = @tmp.stat.size
+    end
+
+    def read(*args)
+      @input or return @tmp.read(*args)
+
+      length = args.shift
+      if nil == length
+        rv = @tmp.read || Z.dup
+        tmp = Z.dup
+        while tee(Const::CHUNK_SIZE, tmp)
+          rv << tmp
+        end
+        rv
+      else
+        buf = args.shift || Z.dup
+        diff = @tmp.stat.size - @tmp.pos
+        if 0 == diff
+          tee(length, buf)
+        else
+          @tmp.read(diff > length ? length : diff, buf)
+        end
+      end
+    end
+
+    # takes zero arguments for strict Rack::Lint compatibility, unlike IO#gets
+    def gets
+      @input or return @tmp.gets
+      nil == $/ and return read
+
+      line = nil
+      if @tmp.pos < @tmp.stat.size
+        line = @tmp.gets # cannot be nil here
+        $/ == line[-$/.size, $/.size] and return line
+
+        # half the line was already read, and the rest of has not been read
+        if buf = @input.gets
+          @tmp.write(buf)
+          line << buf
+        else
+          @input = nil
+        end
+      elsif line = @input.gets
+        @tmp.write(line)
+      end
+
+      line
+    end
+
+    def each(&block)
+      while line = gets
+        yield line
+      end
+
+      self # Rack does not specify what the return value here
+    end
+
+    def rewind
+      @tmp.rewind # Rack does not specify what the return value here
+    end
+
+  private
+
+    # tees off a +length+ chunk of data from the input into the IO
+    # backing store as well as returning it.  +buf+ must be specified.
+    # returns nil if reading from the input returns nil
+    def tee(length, buf)
+      begin
+        if @size
+          left = @size - @tmp.stat.size
+          0 == left and return nil
+          if length >= left
+            @input.readpartial(left, buf) == left and @input = nil
+          elsif @input.nil?
+            return nil
+          else
+            @input.readpartial(length, buf)
+          end
+        else # ChunkedReader#readpartial just raises EOFError when done
+          @input.readpartial(length, buf)
+        end
+      rescue EOFError
+        return @input = nil
+      end
+      @tmp.write(buf)
+      buf
+    end
+
+  end
+end

data/lib/unicorn/trailer_parser.rb

@@ -0,0 +1,52 @@
+# Copyright (c) 2009 Eric Wong
+# You can redistribute it and/or modify it under the same terms as Ruby.
+require 'unicorn'
+require 'unicorn/http11'
+
+# Eventually I should integrate this into HttpParser...
+module Unicorn
+  class TrailerParser
+
+    TR_FR = 'a-z-'.freeze
+    TR_TO = 'A-Z_'.freeze
+
+    # initializes HTTP trailer parser with acceptable +trailer+
+    def initialize(http_trailer)
+      @trailers = http_trailer.split(/\s*,\s*/).inject({}) { |hash, key|
+        hash[key.tr(TR_FR, TR_TO)] = true
+        hash
+      }
+    end
+
+    # Executes our TrailerParser on +data+ and modifies +env+  This will
+    # shrink +data+ as it is being consumed.  Returns true if it has
+    # parsed all trailers, false if not.  It raises HttpParserError on
+    # parse failure or unknown headers.  It has slightly smaller limits
+    # than the C-based HTTP parser but should not be an issue in practice
+    # since Content-MD5 is probably the only legitimate use for it.
+    def execute!(env, data)
+      data.size > 0xffff and
+        raise HttpParserError, "trailer buffer too large: #{data.size} bytes"
+
+      begin
+        data.sub!(/\A([^\r]+)\r\n/, Z) or return false # need more data
+
+        key, val = $1.split(/:\s*/, 2)
+
+        key.size > 256 and
+          raise HttpParserError, "trailer key #{key.inspect} is too long"
+        val.size > 8192 and
+          raise HttpParserError, "trailer value #{val.inspect} is too long"
+
+        key.tr!(TR_FR, TR_TO)
+
+        @trailers.delete(key) or
+          raise HttpParserError, "unknown trailer: #{key.inspect}"
+        env["HTTP_#{key}"] = val
+
+        @trailers.empty? and return true
+      end while true
+    end
+
+  end
+end

data/lib/unicorn/util.rb

@@ -1,5 +1,4 @@
 require 'fcntl'
-require 'tmpdir'
 
 module Unicorn
   class Util
@@ -40,22 +39,6 @@ module Unicorn
         nr
       end
 
-      # creates and returns a new File object.  The File is unlinked
-      # immediately, switched to binary mode, and userspace output
-      # buffering is disabled
-      def tmpio
-        fp = begin
-          File.open("#{Dir::tmpdir}/#{rand}",
-                    File::RDWR|File::CREAT|File::EXCL, 0600)
-        rescue Errno::EEXIST
-          retry
-        end
-        File.unlink(fp.path)
-        fp.binmode
-        fp.sync = true
-        fp
-      end
-
     end
 
   end

data/local.mk.sample

@@ -38,7 +38,7 @@ publish_doc:
 # Create gzip variants of the same timestamp as the original so nginx
 # "gzip_static on" can serve the gzipped versions directly.
 doc_gz: suf := html js css
-doc_gz: globs := $(addprefix doc/*.,$(suf)) $(addprefix doc/*/*.,$(suf))
-doc_gz: docs := $(wildcard $(globs))
+doc_gz: docs = $(shell find doc/ -regex '^.*\.\(html\|js\|css\)$$')
 doc_gz:
-	for i in $(docs); do gzip < $$i > $$i.gz; touch -r $$i $$i.gz; done
+	for i in $(docs); do \
+	  gzip --rsyncable < $$i > $$i.gz; touch -r $$i $$i.gz; done

data/test/rails/test_rails.rb

@@ -142,18 +142,24 @@ logger Logger.new('#{COMMON_TMP.path}')
         end
       end
     end
-    resp = `curl -isSfN -Ffile=@#{tmp.path} http://#@addr:#@port/foo/xpost`
-    assert $?.success?
-    resp = resp.split(/\r?\n/)
-    grepped = resp.grep(/^sha1: (.{40})/)
-    assert_equal 1, grepped.size
-    assert_equal(sha1.hexdigest, /^sha1: (.{40})/.match(grepped.first)[1])
-
-    grepped = resp.grep(/^Content-Type:\s+(.+)/i)
-    assert_equal 1, grepped.size
-    assert_match %r{^text/plain}, grepped.first.split(/\s*:\s*/)[1]
-
-    assert_equal 1, resp.grep(/^Status:/i).size
+
+    # fixed in Rack commit 44ed4640f077504a49b7f1cabf8d6ad7a13f6441,
+    # no released version of Rails or Rack has this fix
+    if RB_V[0] >= 1 && RB_V[1] >= 9
+      warn "multipart broken with Rack 1.0.0 and Rails 2.3.2.1 under 1.9"
+    else
+      resp = `curl -isSfN -Ffile=@#{tmp.path} http://#@addr:#@port/foo/xpost`
+      assert $?.success?
+      resp = resp.split(/\r?\n/)
+      grepped = resp.grep(/^sha1: (.{40})/)
+      assert_equal 1, grepped.size
+      assert_equal(sha1.hexdigest, /^sha1: (.{40})/.match(grepped.first)[1])
+
+      grepped = resp.grep(/^Content-Type:\s+(.+)/i)
+      assert_equal 1, grepped.size
+      assert_match %r{^text/plain}, grepped.first.split(/\s*:\s*/)[1]
+      assert_equal 1, resp.grep(/^Status:/i).size
+    end
 
     # make sure we can get 403 responses, too
     uri = URI.parse("http://#@addr:#@port/foo/xpost")

data/test/test_helper.rb

@@ -262,3 +262,29 @@ def wait_for_death(pid)
   end
   raise "PID:#{pid} never died!"
 end
+
+# executes +cmd+ and chunks its STDOUT
+def chunked_spawn(stdout, *cmd)
+  fork {
+    crd, cwr = IO.pipe
+    crd.binmode
+    cwr.binmode
+    crd.sync = cwr.sync = true
+
+    pid = fork {
+      STDOUT.reopen(cwr)
+      crd.close
+      cwr.close
+      exec(*cmd)
+    }
+    cwr.close
+    begin
+      buf = crd.readpartial(16384)
+      stdout.write("#{'%x' % buf.size}\r\n#{buf}")
+    rescue EOFError
+      stdout.write("0\r\n")
+      pid, status = Process.waitpid(pid)
+      exit status.exitstatus
+    end while true
+  }
+end