unicorn 0.8.4 → 0.9.0

data/README

@@ -131,16 +131,28 @@ regarding this.
 
 == Known Issues
 
-* WONTFIX: code reloading with Sinatra 0.3.2 (and likely older
+* WONTFIX: code reloading and restarts with Sinatra 0.3.x (and likely older
   versions) apps is broken.  The workaround is to force production
-  mode to disable code reloading in your Sinatra application:
+  mode to disable code reloading as well as disabling "run" in your
+  Sinatra application:
     set :env, :production
-  Since this is no longer an issue with Sinatra 0.9.x apps and only
-  affected non-production instances, this will not be fixed on our end.
-  Also remember we're capable of replacing the running binary without
-  dropping any connections regardless of framework :)
+    set :run, false
+  Since this is no longer an issue with Sinatra 0.9.x apps, this will not be
+  fixed on our end.  Since Unicorn is itself the application launcher, the
+  at_exit handler used in old Sinatra always caused Mongrel to be launched
+  whenever a Unicorn worker was about to exit.
+
+  Also remember we're capable of replacing the running binary without dropping
+  any connections regardless of framework :)
 
 == Contact
 
-Email Eric Wong at normalperson@yhbt.net for now.
-Newsgroup and mailing list maybe coming...
+All feedback (bug reports, user/development dicussion, patches, pull
+requests) go to the mailing list.  Patches must be sent inline
+(git format-patch -M + git send-email).  No subscription is necessary
+to post on the mailing list.  No top posting.  Address replies +To:+ (or
++Cc:+) the original sender and +Cc:+ the mailing list.
+
+* email: mongrel-unicorn@rubyforge.org
+* archives: http://rubyforge.org/pipermail/mongrel-unicorn/
+* subscribe: http://rubyforge.org/mailman/listinfo/mongrel-unicorn/

data/TODO

@@ -1,17 +1,9 @@
-== 1.0.0
+* integration tests with nginx including bad client handling
 
-  * integration tests with nginx including bad client handling
+* manpages (why do so few Ruby executables come with proper manpages?)
 
-  * manpages (why do so few Ruby executables come with proper manpages?)
+* code cleanups (launchers)
 
-== 1.1.0
+* Pure Ruby HTTP parser
 
-  * Transfer-Encoding: chunked request handling.  Testcase:
-
-      curl -T- http://host:port/path < file_from_stdin
-
-  * code cleanups (launchers)
-
-  * Pure Ruby HTTP parser
-
-  * Rubinius support?
+* Rubinius support

data/examples/echo.ru

@@ -0,0 +1,32 @@
+#\-E none
+#
+# Example application that echoes read data back to the HTTP client.
+# This emulates the old echo protocol people used to run.
+#
+# An example of using this in a client would be to run:
+#   curl --no-buffer -T- http://host:port/
+#
+# Then type random stuff in your terminal to watch it get echoed back!
+
+class EchoBody
+  def initialize(input)
+    @input = input
+  end
+
+  def each(&block)
+    while buf = @input.read(4096)
+      yield buf
+    end
+    self
+  end
+
+  def close
+    @input = nil
+  end
+end
+
+use Rack::Chunked
+run lambda { |env|
+  [ 200, { 'Content-Type' => 'application/octet-stream' },
+    EchoBody.new(env['rack.input']) ]
+}

data/examples/git.ru

@@ -0,0 +1,13 @@
+#\-E none
+
+# See http://thread.gmane.org/gmane.comp.web.curl.general/10473/raw on
+# how to setup git for this.  A better version of the above patch was
+# accepted and committed on June 15, 2009, so you can pull the latest
+# curl CVS snapshot to try this out.
+require 'unicorn/app/inetd'
+
+use Rack::Lint
+use Rack::Chunked # important!
+run Unicorn::App::Inetd.new(
+ *%w(git daemon --verbose --inetd --export-all --base-path=/home/ew/unicorn)
+)

data/ext/unicorn/http11/http11.c

@@ -324,10 +324,17 @@ static void header_done(void *data, const char *at, size_t length)
   }
   rb_hash_aset(req, global_server_name, server_name);
   rb_hash_aset(req, global_server_port, server_port);
+  rb_hash_aset(req, global_server_protocol, global_server_protocol_value);
 
   /* grab the initial body and stuff it into the hash */
-  rb_hash_aset(req, sym_http_body, rb_str_new(at, length));
-  rb_hash_aset(req, global_server_protocol, global_server_protocol_value);
+  temp = rb_hash_aref(req, global_request_method);
+  if (temp != Qnil) {
+    long len = RSTRING_LEN(temp);
+    char *ptr = RSTRING_PTR(temp);
+
+    if (memcmp(ptr, "HEAD", len) && memcmp(ptr, "GET", len))
+      rb_hash_aset(req, sym_http_body, rb_str_new(at, length));
+  }
 }
 
 static void HttpParser_free(void *data) {

data/lib/unicorn.rb

@@ -1,4 +1,5 @@
 require 'fcntl'
+require 'tempfile'
 require 'unicorn/socket_helper'
 autoload :Rack, 'rack'
 
@@ -10,8 +11,15 @@ module Unicorn
   autoload :HttpRequest, 'unicorn/http_request'
   autoload :HttpResponse, 'unicorn/http_response'
   autoload :Configurator, 'unicorn/configurator'
+  autoload :TeeInput, 'unicorn/tee_input'
+  autoload :ChunkedReader, 'unicorn/chunked_reader'
+  autoload :TrailerParser, 'unicorn/trailer_parser'
   autoload :Util, 'unicorn/util'
 
+  Z = '' # the stock empty string we use everywhere...
+  Z.force_encoding(Encoding::BINARY) if Z.respond_to?(:force_encoding)
+  Z.freeze
+
   class << self
     def run(app, options = {})
       HttpServer.new(app, options).start.join
@@ -51,10 +59,11 @@ module Unicorn
       # don't rely on Dir.pwd here since it's not symlink-aware, and
       # symlink dirs are the default with Capistrano...
       :cwd => `/bin/sh -c pwd`.chomp("\n"),
-      0 => $0.dup,
+      :zero => $0.dup,
     }
 
-    class Worker < Struct.new(:nr, :tmp)
+    Worker = Struct.new(:nr, :tempfile) unless defined?(Worker)
+    class Worker
       # worker objects may be compared to just plain numbers
       def ==(other_nr)
         self.nr == other_nr
@@ -322,7 +331,7 @@ module Unicorn
             self.pid = @pid.chomp('.oldbin') if @pid
             proc_name 'master'
           else
-            worker = WORKERS.delete(pid) and worker.tmp.close rescue nil
+            worker = WORKERS.delete(pid) and worker.tempfile.close rescue nil
             logger.info "reaped #{status.inspect} " \
                         "worker=#{worker.nr rescue 'unknown'}"
           end
@@ -362,7 +371,7 @@ module Unicorn
         listener_fds = LISTENERS.map { |sock| sock.fileno }
         ENV['UNICORN_FD'] = listener_fds.join(',')
         Dir.chdir(START_CTX[:cwd])
-        cmd = [ START_CTX[0] ].concat(START_CTX[:argv])
+        cmd = [ START_CTX[:zero] ] + START_CTX[:argv]
 
         # avoid leaking FDs we don't know about, but let before_exec
         # unset FD_CLOEXEC, if anything else in the app eventually
@@ -382,16 +391,16 @@ module Unicorn
     end
 
     # forcibly terminate all workers that haven't checked in in @timeout
-    # seconds.  The timeout is implemented using an unlinked File
+    # seconds.  The timeout is implemented using an unlinked tempfile
     # shared between the parent process and each worker.  The worker
-    # runs File#chmod to modify the ctime of the File.  If the ctime
+    # runs File#chmod to modify the ctime of the tempfile.  If the ctime
     # is stale for >@timeout seconds, then we'll kill the corresponding
     # worker.
     def murder_lazy_workers
       diff = stat = nil
       WORKERS.dup.each_pair do |pid, worker|
         stat = begin
-          worker.tmp.stat
+          worker.tempfile.stat
         rescue => e
           logger.warn "worker=#{worker.nr} PID:#{pid} stat error: #{e.inspect}"
           kill_worker(:QUIT, pid)
@@ -415,7 +424,9 @@ module Unicorn
           SIG_QUEUE << :QUIT # forcibly emulate SIGQUIT
           return
         end
-        worker = Worker.new(worker_nr, Unicorn::Util.tmpio)
+        tempfile = Tempfile.new(nil) # as short as possible to save dir space
+        tempfile.unlink # don't allow other processes to find or see it
+        worker = Worker.new(worker_nr, tempfile)
         @before_fork.call(self, worker)
         pid = fork { worker_loop(worker) }
         WORKERS[pid] = worker
@@ -433,8 +444,15 @@ module Unicorn
     # once a client is accepted, it is processed in its entirety here
     # in 3 easy steps: read request, call app, write app response
     def process_client(app, client)
-      client.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC)
-      HttpResponse.write(client, app.call(REQUEST.read(client)))
+      response = app.call(env = REQUEST.read(client))
+
+      if 100 == response.first.to_i
+        client.write(Const::EXPECT_100_RESPONSE)
+        env.delete(Const::HTTP_EXPECT)
+        response = app.call(env)
+      end
+
+      HttpResponse.write(client, response)
     # if we get any error, try to write something back to the client
     # assuming we haven't closed the socket, but don't get hung up
     # if the socket is already closed or broken.  We'll always ensure
@@ -457,16 +475,16 @@ module Unicorn
     # traps for USR1, USR2, and HUP may be set in the @after_fork Proc
     # by the user.
     def init_worker_process(worker)
-      QUEUE_SIGS.each { |sig| trap(sig, nil) }
+      QUEUE_SIGS.each { |sig| trap(sig, 'IGNORE') }
       trap(:CHLD, 'DEFAULT')
       SIG_QUEUE.clear
       proc_name "worker[#{worker.nr}]"
       START_CTX.clear
       init_self_pipe!
-      WORKERS.values.each { |other| other.tmp.close rescue nil }
+      WORKERS.values.each { |other| other.tempfile.close! rescue nil }
       WORKERS.clear
       LISTENERS.each { |sock| sock.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC) }
-      worker.tmp.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC)
+      worker.tempfile.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC)
       @after_fork.call(self, worker) # can drop perms
       @timeout /= 2.0 # halve it for select()
       build_app! unless @preload_app
@@ -486,7 +504,7 @@ module Unicorn
       master_pid = Process.ppid # slightly racy, but less memory usage
       init_worker_process(worker)
       nr = 0 # this becomes negative if we need to reopen logs
-      alive = worker.tmp # tmp is our lifeline to the master process
+      alive = worker.tempfile # tempfile is our lifeline to the master process
       ready = LISTENERS
       t = ti = 0
 
@@ -552,7 +570,7 @@ module Unicorn
       begin
         Process.kill(signal, pid)
       rescue Errno::ESRCH
-        worker = WORKERS.delete(pid) and worker.tmp.close rescue nil
+        worker = WORKERS.delete(pid) and worker.tempfile.close rescue nil
       end
     end
 
@@ -613,8 +631,8 @@ module Unicorn
     end
 
     def proc_name(tag)
-      $0 = ([ File.basename(START_CTX[0]), tag
-            ]).concat(START_CTX[:argv]).join(' ')
+      $0 = ([ File.basename(START_CTX[:zero]), tag ] +
+              START_CTX[:argv]).join(' ')
     end
 
     def redirect_io(io, path)

data/lib/unicorn/app/exec_cgi.rb

@@ -42,8 +42,11 @@ module Unicorn::App
 
     # Calls the app
     def call(env)
-      out, err = Unicorn::Util.tmpio, Unicorn::Util.tmpio
+      out, err = Tempfile.new(nil), Tempfile.new(nil)
+      out.unlink
+      err.unlink
       inp = force_file_input(env)
+      out.sync = err.sync = true
       pid = fork { run_child(inp, out, err, env) }
       inp.close
       pid, status = Process.waitpid2(pid)
@@ -118,15 +121,15 @@ module Unicorn::App
     # ensures rack.input is a file handle that we can redirect stdin to
     def force_file_input(env)
       inp = env['rack.input']
-      if inp.respond_to?(:fileno) && Integer === inp.fileno
-        inp
-      elsif inp.size == 0 # inp could be a StringIO or StringIO-like object
+      if inp.size == 0 # inp could be a StringIO or StringIO-like object
         ::File.open('/dev/null', 'rb')
       else
-        tmp = Unicorn::Util.tmpio
+        tmp = Tempfile.new(nil)
+        tmp.unlink
+        tmp.binmode
+        tmp.sync = true
 
-        # Rack::Lint::InputWrapper doesn't allow sysread :(
-        buf = Unicorn::Z.dup
+        buf = Z.dup
         while inp.read(CHUNK_SIZE, buf)
           tmp.syswrite(buf)
         end

data/lib/unicorn/app/inetd.rb

@@ -0,0 +1,108 @@
+# Copyright (c) 2009 Eric Wong
+# You can redistribute it and/or modify it under the same terms as Ruby.
+
+# this class *must* be used with Rack::Chunked
+
+module Unicorn::App
+  class Inetd
+
+    class CatBody
+      def initialize(env, cmd)
+        @cmd = cmd
+        @input, @errors = env['rack.input'], env['rack.errors']
+        in_rd, in_wr = IO.pipe
+        @err_rd, err_wr = IO.pipe
+        @out_rd, out_wr = IO.pipe
+
+        @cmd_pid = fork {
+          inp, out, err = (0..2).map { |i| IO.new(i) }
+          inp.reopen(in_rd)
+          out.reopen(out_wr)
+          err.reopen(err_wr)
+          [ in_rd, in_wr, @err_rd, err_wr, @out_rd, out_wr ].each { |io|
+            io.close
+          }
+          exec(*cmd)
+        }
+        [ in_rd, err_wr, out_wr ].each { |io| io.close }
+        [ in_wr, @err_rd, @out_rd ].each { |io| io.binmode }
+        in_wr.sync = true
+
+        # Unfortunately, input here must be processed inside a seperate
+        # thread/process using blocking I/O since env['rack.input'] is not
+        # IO.select-able and attempting to make it so would trip Rack::Lint
+        @inp_pid = fork {
+          [ @err_rd, @out_rd ].each { |io| io.close }
+          buf = Unicorn::Z.dup
+
+          # this is dependent on @input.read having readpartial semantics:
+          while @input.read(16384, buf)
+            in_wr.write(buf)
+          end
+          in_wr.close
+        }
+        in_wr.close
+      end
+
+      def each(&block)
+        buf = Unicorn::Z.dup
+        begin
+          rd, = IO.select([@err_rd, @out_rd])
+          rd && rd.first or next
+
+          if rd.include?(@err_rd)
+            begin
+              @errors.write(@err_rd.read_nonblock(16384, buf))
+            rescue Errno::EINTR
+            rescue Errno::EAGAIN
+              break
+            end while true
+          end
+
+          rd.include?(@out_rd) or next
+
+          begin
+            yield @out_rd.read_nonblock(16384, buf)
+          rescue Errno::EINTR
+          rescue Errno::EAGAIN
+            break
+          end while true
+        rescue EOFError,Errno::EPIPE,Errno::EBADF,Errno::EINVAL
+          break
+        end while true
+
+        self
+      end
+
+      def close
+        @input = nil
+        [ [ @cmd.inspect, @cmd_pid ], [ 'input streamer', @inp_pid ]
+        ].each { |str, pid|
+          begin
+            pid, status = Process.waitpid2(pid)
+            status.success? or
+              @errors.write("#{str}: #{status.inspect} (PID:#{pid})\n")
+          rescue Errno::ECHILD
+            @errors.write("Failed to reap #{str} (PID:#{pid})\n")
+          end
+        }
+      end
+
+    end
+
+    def initialize(*cmd)
+      @cmd = cmd
+    end
+
+    def call(env)
+      expect = env[Unicorn::Const::HTTP_EXPECT] and
+        /\A100-continue\z/i =~ expect and
+          return [ 100, {} , [] ]
+
+      [ 200, { 'Content-Type' => 'application/octet-stream' },
+       CatBody.new(env, @cmd) ]
+    end
+
+  end
+
+end

data/lib/unicorn/chunked_reader.rb

@@ -0,0 +1,94 @@
+# Copyright (c) 2009 Eric Wong
+# You can redistribute it and/or modify it under the same terms as Ruby.
+
+require 'unicorn'
+require 'unicorn/http11'
+
+module Unicorn
+  class ChunkedReader
+
+    def initialize(env, input, buf)
+      @env, @input, @buf = env, input, buf
+      @chunk_left = 0
+      parse_chunk_header
+    end
+
+    def readpartial(max, buf = Z.dup)
+      while @input && @chunk_left <= 0 && ! parse_chunk_header
+        @buf << @input.readpartial(Const::CHUNK_SIZE, buf)
+      end
+
+      if @input
+        begin
+          @buf << @input.read_nonblock(Const::CHUNK_SIZE, buf)
+        rescue Errno::EAGAIN, Errno::EINTR
+        end
+      end
+
+      max = @chunk_left if max > @chunk_left
+      buf.replace(last_block(max) || Z)
+      @chunk_left -= buf.size
+      (0 == buf.size && @input.nil?) and raise EOFError
+      buf
+    end
+
+    def gets
+      line = nil
+      begin
+        line = readpartial(Const::CHUNK_SIZE)
+        begin
+          if line.sub!(%r{\A(.*?#{$/})}, Z)
+            @chunk_left += line.size
+            @buf = @buf ? (line << @buf) : line
+            return $1.dup
+          end
+          line << readpartial(Const::CHUNK_SIZE)
+        end while true
+      rescue EOFError
+        return line
+      end
+    end
+
+  private
+
+    def last_block(max = nil)
+      rv = @buf
+      if max && rv && max < rv.size
+        @buf = rv[max - rv.size, rv.size - max]
+        return rv[0, max]
+      end
+      @buf = Z.dup
+      rv
+    end
+
+    def parse_chunk_header
+      buf = @buf
+      # ignoring chunk-extension info for now, I haven't seen any use for it
+      # (or any users, and TE:chunked sent by clients is rare already)
+      # if there was not enough data in buffer to parse length of the chunk
+      # then just return
+      if buf.sub!(/\A(?:\r\n)?([a-fA-F0-9]{1,8})[^\r]*?\r\n/, Z)
+        @chunk_left = $1.to_i(16)
+        if 0 == @chunk_left # EOF
+          buf.sub!(/\A\r\n(?:\r\n)?/, Z) # cleanup for future requests
+          if trailer = @env[Const::HTTP_TRAILER]
+            tp = TrailerParser.new(trailer)
+            while ! tp.execute!(@env, buf)
+              buf << @input.readpartial(Const::CHUNK_SIZE)
+            end
+          end
+          @input = nil
+        end
+        return @chunk_left
+      end
+
+      buf.size > 256 and
+          raise HttpParserError,
+                "malformed chunk, chunk-length not found in buffer: " \
+                "#{buf.inspect}"
+      nil
+    end
+
+  end
+
+end