unicorn 3.7.0 → 4.0.0

data/DESIGN

@@ -76,14 +76,6 @@
   Applications that use threads continue to work if Unicorn
   is only serving LAN or localhost clients.
 
-* Timeout implementation is done via fchmod(2) in each worker
-  on a shared file descriptor to update st_ctime on the inode.
-  Master process wakeups for checking on timeouts is throttled
-  one a second to minimize the performance impact and simplify
-  the code path within the worker.  Neither futimes(2) nor
-  pwrite(2)/pread(2) are supported by base MRI, nor are they as
-  portable on UNIX systems as fchmod(2).
-
 * SIGKILL is used to terminate the timed-out workers from misbehaving apps
   as reliably as possible on a UNIX system.  The default timeout is a
   generous 60 seconds (same default as in Mongrel).

data/GIT-VERSION-GEN

@@ -1,7 +1,7 @@
 #!/bin/sh
 
 GVF=GIT-VERSION-FILE
-DEF_VER=v3.7.0.GIT
+DEF_VER=v4.0.0.GIT
 
 LF='
 '

data/TODO

@@ -2,6 +2,4 @@
 
 * improve test suite
 
-* scalability to >= 1024 worker processes for crazy NUMA systems
-
 * Rack 2.x support (when Rack 2.x exists)

data/bin/unicorn

@@ -106,6 +106,7 @@ op = OptionParser.new("", 24, '  ') do |opts|
 end
 
 app = Unicorn.builder(ARGV[0] || 'config.ru', op)
+op = nil
 
 if $DEBUG
   require 'pp'
@@ -117,4 +118,4 @@ if $DEBUG
 end
 
 Unicorn::Launcher.daemonize!(options) if rackup_opts[:daemonize]
-Unicorn.run(app, options)
+Unicorn::HttpServer.new(app, options).start.join

data/bin/unicorn_rails

@@ -186,6 +186,7 @@ def rails_builder(ru, op, daemonize)
 end
 
 app = rails_builder(ARGV[0], op, rackup_opts[:daemonize])
+op = nil
 
 if $DEBUG
   require 'pp'
@@ -205,4 +206,4 @@ if rackup_opts[:daemonize]
   options[:pid] = "tmp/pids/unicorn.pid"
   Unicorn::Launcher.daemonize!(options)
 end
-Unicorn.run(app, options)
+Unicorn::HttpServer.new(app, options).start.join

data/ext/unicorn_http/unicorn_http.rl

@@ -629,6 +629,34 @@ static VALUE HttpParser_clear(VALUE self)
   return self;
 }
 
+/**
+ * call-seq:
+ *    parser.dechunk! => parser
+ *
+ * Resets the parser to a state suitable for dechunking response bodies
+ *
+ */
+static VALUE HttpParser_dechunk_bang(VALUE self)
+{
+  struct http_parser *hp = data_get(self);
+
+  http_parser_init(hp);
+
+  /*
+   * we don't care about trailers in dechunk-only mode,
+   * but if we did we'd set UH_FL_HASTRAILER and clear hp->env
+   */
+  if (0) {
+    rb_funcall(hp->env, id_clear, 0);
+    hp->flags = UH_FL_HASTRAILER;
+  }
+
+  hp->flags |= UH_FL_HASBODY | UH_FL_INBODY | UH_FL_CHUNKED;
+  hp->cs = http_parser_en_ChunkedBody;
+
+  return self;
+}
+
 /**
  * call-seq:
  *    parser.reset => nil
@@ -856,72 +884,80 @@ static VALUE HttpParser_env(VALUE self)
 
 /**
  * call-seq:
- *    parser.filter_body(buf, data) => nil/data
+ *    parser.filter_body(dst, src) => nil/src
  *
- * Takes a String of +data+, will modify data if dechunking is done.
+ * Takes a String of +src+, will modify data if dechunking is done.
  * Returns +nil+ if there is more data left to process.  Returns
- * +data+ if body processing is complete. When returning +data+,
- * it may modify +data+ so the start of the string points to where
+ * +src+ if body processing is complete. When returning +src+,
+ * it may modify +src+ so the start of the string points to where
  * the body ended so that trailer processing can begin.
  *
  * Raises HttpParserError if there are dechunking errors.
- * Basically this is a glorified memcpy(3) that copies +data+
+ * Basically this is a glorified memcpy(3) that copies +src+
  * into +buf+ while filtering it through the dechunker.
  */
-static VALUE HttpParser_filter_body(VALUE self, VALUE buf, VALUE data)
+static VALUE HttpParser_filter_body(VALUE self, VALUE dst, VALUE src)
 {
   struct http_parser *hp = data_get(self);
-  char *dptr;
-  long dlen;
+  char *srcptr;
+  long srclen;
 
-  dptr = RSTRING_PTR(data);
-  dlen = RSTRING_LEN(data);
+  srcptr = RSTRING_PTR(src);
+  srclen = RSTRING_LEN(src);
 
-  StringValue(buf);
-  rb_str_modify(buf);
-  rb_str_resize(buf, dlen); /* we can never copy more than dlen bytes */
-  OBJ_TAINT(buf); /* keep weirdo $SAFE users happy */
+  StringValue(dst);
 
   if (HP_FL_TEST(hp, CHUNKED)) {
     if (!chunked_eof(hp)) {
+      rb_str_modify(dst);
+      rb_str_resize(dst, srclen); /* we can never copy more than srclen bytes */
+
       hp->s.dest_offset = 0;
-      hp->cont = buf;
-      hp->buf = data;
-      http_parser_execute(hp, dptr, dlen);
+      hp->cont = dst;
+      hp->buf = src;
+      http_parser_execute(hp, srcptr, srclen);
       if (hp->cs == http_parser_error)
         parser_raise(eHttpParserError, "Invalid HTTP format, parsing fails.");
 
       assert(hp->s.dest_offset <= hp->offset &&
              "destination buffer overflow");
-      advance_str(data, hp->offset);
-      rb_str_set_len(buf, hp->s.dest_offset);
+      advance_str(src, hp->offset);
+      rb_str_set_len(dst, hp->s.dest_offset);
 
-      if (RSTRING_LEN(buf) == 0 && chunked_eof(hp)) {
+      if (RSTRING_LEN(dst) == 0 && chunked_eof(hp)) {
         assert(hp->len.chunk == 0 && "chunk at EOF but more to parse");
       } else {
-        data = Qnil;
+        src = Qnil;
       }
     }
   } else {
     /* no need to enter the Ragel machine for unchunked transfers */
     assert(hp->len.content >= 0 && "negative Content-Length");
     if (hp->len.content > 0) {
-      long nr = MIN(dlen, hp->len.content);
-
-      hp->buf = data;
-      memcpy(RSTRING_PTR(buf), dptr, nr);
+      long nr = MIN(srclen, hp->len.content);
+
+      rb_str_modify(dst);
+      rb_str_resize(dst, nr);
+      /*
+       * using rb_str_replace() to avoid memcpy() doesn't help in
+       * most cases because a GC-aware programmer will pass an explicit
+       * buffer to env["rack.input"].read and reuse the buffer in a loop.
+       * This causes copy-on-write behavior to be triggered anyways
+       * when the +src+ buffer is modified (when reading off the socket).
+       */
+      hp->buf = src;
+      memcpy(RSTRING_PTR(dst), srcptr, nr);
       hp->len.content -= nr;
       if (hp->len.content == 0) {
         HP_FL_SET(hp, REQEOF);
         hp->cs = http_parser_first_final;
       }
-      advance_str(data, nr);
-      rb_str_set_len(buf, nr);
-      data = Qnil;
+      advance_str(src, nr);
+      src = Qnil;
     }
   }
   hp->offset = 0; /* for trailer parsing */
-  return data;
+  return src;
 }
 
 #define SET_GLOBAL(var,str) do { \
@@ -947,6 +983,7 @@ void Init_unicorn_http(void)
   rb_define_method(cHttpParser, "initialize", HttpParser_init, 0);
   rb_define_method(cHttpParser, "clear", HttpParser_clear, 0);
   rb_define_method(cHttpParser, "reset", HttpParser_reset, 0);
+  rb_define_method(cHttpParser, "dechunk!", HttpParser_dechunk_bang, 0);
   rb_define_method(cHttpParser, "parse", HttpParser_parse, 0);
   rb_define_method(cHttpParser, "add_parse", HttpParser_add_parse, 1);
   rb_define_method(cHttpParser, "headers", HttpParser_headers, 2);

data/lib/unicorn.rb

@@ -26,17 +26,14 @@ module Unicorn
   end
 
   # :stopdoc:
-  def self.run(app, options = {})
-    Unicorn::HttpServer.new(app, options).start.join
-  end
 
   # This returns a lambda to pass in as the app, this does not "build" the
   # app (which we defer based on the outcome of "preload_app" in the
   # Unicorn config).  The returned lambda will be called when it is
   # time to build the app.
-  def self.builder(ru, opts)
+  def self.builder(ru, op)
     # allow Configurator to parse cli switches embedded in the ru file
-    Unicorn::Configurator::RACKUP.update(:file => ru, :optparse => opts)
+    op = Unicorn::Configurator::RACKUP.merge!(:file => ru, :optparse => op)
 
     # always called after config file parsing, may be called after forking
     lambda do ||
@@ -44,7 +41,7 @@ module Unicorn
       when /\.ru$/
         raw = File.read(ru)
         raw.sub!(/^__END__\n.*/, '')
-        eval("Rack::Builder.new {(#{raw}\n)}.to_app", TOPLEVEL_BINDING, ru)
+        eval("Rack::Builder.new {(\n#{raw}\n)}.to_app", TOPLEVEL_BINDING, ru)
       else
         require ru
         Object.const_get(File.basename(ru, '.rb').capitalize)
@@ -80,6 +77,11 @@ module Unicorn
       Unicorn::SocketHelper.sock_name(io)
     end
   end
+
+  def self.log_error(logger, message, exc)
+    logger.error "#{message}: #{exc.message} (#{exc.class})"
+    exc.backtrace.each { |line| logger.error(line) }
+  end
   # :startdoc:
 end
 # :enddoc:

data/lib/unicorn/configurator.rb

@@ -264,8 +264,9 @@ class Unicorn::Configurator
   #
   #   This has no effect on UNIX sockets.
   #
-  #   Default: +false+ (Nagle's algorithm enabled) in \Unicorn,
-  #   +true+ in Rainbows!
+  #   Default: +true+ (Nagle's algorithm disabled) in \Unicorn,
+  #   +true+ in Rainbows!  This defaulted to +false+ in \Unicorn
+  #   3.x
   #
   # [:tcp_nopush => true or false]
   #
@@ -279,7 +280,8 @@ class Unicorn::Configurator
   #
   #   This has no effect on UNIX sockets.
   #
-  #   Default: +true+ in \Unicorn 3.4+, +false+ in Rainbows!
+  #   Default: +false+
+  #   This defaulted to +true+ in \Unicorn 3.4 - 3.7
   #
   # [:ipv6only => true or false]
   #

data/lib/unicorn/const.rb

@@ -8,8 +8,8 @@
 # improve things much compared to constants.
 module Unicorn::Const
 
-  # The current version of Unicorn, currently 3.7.0
-  UNICORN_VERSION = "3.7.0"
+  # The current version of Unicorn, currently 4.0.0
+  UNICORN_VERSION = "4.0.0"
 
   # default TCP listen host address (0.0.0.0, all interfaces)
   DEFAULT_HOST = "0.0.0.0"

data/lib/unicorn/http_server.rb

@@ -97,7 +97,7 @@ class Unicorn::HttpServer
     self.reexec_pid = 0
     options = options.dup
     @ready_pipe = options.delete(:ready_pipe)
-    self.init_listeners = options[:listeners] ? options[:listeners].dup : []
+    @init_listeners = options[:listeners] ? options[:listeners].dup : []
     options[:use_defaults] = true
     self.config = Unicorn::Configurator.new(options)
     self.listener_opts = {}
@@ -118,35 +118,7 @@ class Unicorn::HttpServer
 
   # Runs the thing.  Returns self so you can run join on it
   def start
-    BasicSocket.do_not_reverse_lookup = true
-
-    # inherit sockets from parents, they need to be plain Socket objects
-    # before they become Kgio::UNIXServer or Kgio::TCPServer
-    inherited = ENV['UNICORN_FD'].to_s.split(/,/).map do |fd|
-      io = Socket.for_fd(fd.to_i)
-      set_server_sockopt(io, listener_opts[sock_name(io)])
-      IO_PURGATORY << io
-      logger.info "inherited addr=#{sock_name(io)} fd=#{fd}"
-      server_cast(io)
-    end
-
-    config_listeners = config[:listeners].dup
-    LISTENERS.replace(inherited)
-
-    # we start out with generic Socket objects that get cast to either
-    # Kgio::TCPServer or Kgio::UNIXServer objects; but since the Socket
-    # objects share the same OS-level file descriptor as the higher-level
-    # *Server objects; we need to prevent Socket objects from being
-    # garbage-collected
-    config_listeners -= listener_names
-    if config_listeners.empty? && LISTENERS.empty?
-      config_listeners << Unicorn::Const::DEFAULT_LISTEN
-      init_listeners << Unicorn::Const::DEFAULT_LISTEN
-      START_CTX[:argv] << "-l#{Unicorn::Const::DEFAULT_LISTEN}"
-    end
-    config_listeners.each { |addr| listen(addr) }
-    raise ArgumentError, "no listeners" if LISTENERS.empty?
-
+    inherit_listeners!
     # this pipe is used to wake us up from select(2) in #join when signals
     # are trapped.  See trap_deferred.
     init_self_pipe!
@@ -160,7 +132,7 @@ class Unicorn::HttpServer
 
     self.master_pid = $$
     build_app! if preload_app
-    maintain_worker_count
+    spawn_missing_workers
     self
   end
 
@@ -281,8 +253,7 @@ class Unicorn::HttpServer
     logger.info "master process ready" # test_exec.rb relies on this message
     if @ready_pipe
       @ready_pipe.syswrite($$.to_s)
-      @ready_pipe.close rescue nil
-      @ready_pipe = nil
+      @ready_pipe = @ready_pipe.close rescue nil
     end
     begin
       reap_all_workers
@@ -333,10 +304,8 @@ class Unicorn::HttpServer
           reexec
         end
       end
-    rescue Errno::EINTR
     rescue => e
-      logger.error "Unhandled master loop exception #{e.inspect}."
-      logger.error e.backtrace.join("\n")
+      Unicorn.log_error(@logger, "master loop error", e)
     end while true
     stop # gracefully shutdown all workers on our way out
     logger.info "master complete"
@@ -403,7 +372,7 @@ class Unicorn::HttpServer
         self.pid = pid.chomp('.oldbin') if pid
         proc_name 'master'
       else
-        worker = WORKERS.delete(wpid) and worker.tmp.close rescue nil
+        worker = WORKERS.delete(wpid) and worker.close rescue nil
         m = "reaped #{status.inspect} worker=#{worker.nr rescue 'unknown'}"
         status.success? ? logger.info(m) : logger.error(m)
       end
@@ -460,22 +429,17 @@ class Unicorn::HttpServer
     proc_name 'master (old)'
   end
 
-  # forcibly terminate all workers that haven't checked in in timeout
-  # seconds.  The timeout is implemented using an unlinked File
-  # shared between the parent process and each worker.  The worker
-  # runs File#chmod to modify the ctime of the File.  If the ctime
-  # is stale for >timeout seconds, then we'll kill the corresponding
-  # worker.
+  # forcibly terminate all workers that haven't checked in in timeout seconds.  The timeout is implemented using an unlinked File
   def murder_lazy_workers
     t = @timeout
     next_sleep = 1
+    now = Time.now.to_i
     WORKERS.dup.each_pair do |wpid, worker|
-      stat = worker.tmp.stat
-      # skip workers that disable fchmod or have never fchmod-ed
-      stat.mode == 0100600 and next
-      diff = Time.now - stat.ctime
-      if diff <= t
-        tmp = t - diff
+      tick = worker.tick
+      0 == tick and next # skip workers that are sleeping
+      diff = now - tick
+      tmp = t - diff
+      if tmp >= 0
         next_sleep < tmp and next_sleep = tmp
         next
       end
@@ -488,7 +452,9 @@ class Unicorn::HttpServer
 
   def after_fork_internal
     @ready_pipe.close if @ready_pipe
-    self.ready_pipe = nil # XXX Rainbows! compat, change for Unicorn 4.x
+    Unicorn::Configurator::RACKUP.clear
+    @ready_pipe = @init_listeners = @config = @before_exec = @before_fork = nil
+
     srand # http://redmine.ruby-lang.org/issues/4338
 
     # The OpenSSL PRNG is seeded with only the pid, and apps with frequently
@@ -497,15 +463,22 @@ class Unicorn::HttpServer
   end
 
   def spawn_missing_workers
-    (0...worker_processes).each do |worker_nr|
-      WORKERS.values.include?(worker_nr) and next
-      worker = Worker.new(worker_nr, Unicorn::TmpIO.new)
+    worker_nr = -1
+    until (worker_nr += 1) == @worker_processes
+      WORKERS.value?(worker_nr) and next
+      worker = Worker.new(worker_nr)
       before_fork.call(self, worker)
-      WORKERS[fork {
+      if pid = fork
+        WORKERS[pid] = worker
+      else
         after_fork_internal
         worker_loop(worker)
-      }] = worker
+        exit
+      end
     end
+    rescue => e
+      @logger.error(e) rescue nil
+      exit!
   end
 
   def maintain_worker_count
@@ -531,8 +504,7 @@ class Unicorn::HttpServer
     when Unicorn::HttpParserError # try to tell the client they're bad
       Unicorn::Const::ERROR_400_RESPONSE
     else
-      logger.error "Read error: #{e.inspect}"
-      logger.error e.backtrace.join("\n")
+      Unicorn.log_error(@logger, "app error", e)
       Unicorn::Const::ERROR_500_RESPONSE
     end
     client.kgio_trywrite(msg)
@@ -568,10 +540,8 @@ class Unicorn::HttpServer
     proc_name "worker[#{worker.nr}]"
     START_CTX.clear
     init_self_pipe!
-    WORKERS.values.each { |other| other.tmp.close rescue nil }
     WORKERS.clear
     LISTENERS.each { |sock| sock.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC) }
-    worker.tmp.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC)
     after_fork.call(self, worker) # can drop perms
     worker.user(*user) if user.kind_of?(Array) && ! worker.switched
     self.timeout /= 2.0 # halve it for select()
@@ -595,35 +565,25 @@ class Unicorn::HttpServer
     ppid = master_pid
     init_worker_process(worker)
     nr = 0 # this becomes negative if we need to reopen logs
-    alive = worker.tmp # tmp is our lifeline to the master process
-    ready = LISTENERS
+    l = LISTENERS.dup
+    ready = l.dup
 
     # closing anything we IO.select on will raise EBADF
     trap(:USR1) { nr = -65536; SELF_PIPE[0].close rescue nil }
-    trap(:QUIT) { alive = nil; LISTENERS.each { |s| s.close rescue nil }.clear }
+    trap(:QUIT) { worker = nil; LISTENERS.each { |s| s.close rescue nil }.clear }
     [:TERM, :INT].each { |sig| trap(sig) { exit!(0) } } # instant shutdown
     logger.info "worker=#{worker.nr} ready"
-    m = 0
 
     begin
       nr < 0 and reopen_worker_logs(worker.nr)
       nr = 0
 
-      # we're a goner in timeout seconds anyways if alive.chmod
-      # breaks, so don't trap the exception.  Using fchmod() since
-      # futimes() is not available in base Ruby and I very strongly
-      # prefer temporary files to be unlinked for security,
-      # performance and reliability reasons, so utime is out.  No-op
-      # changes with chmod doesn't update ctime on all filesystems; so
-      # we change our counter each and every time (after process_client
-      # and before IO.select).
-      alive.chmod(m = 0 == m ? 1 : 0)
-
-      ready.each do |sock|
+      worker.tick = Time.now.to_i
+      while sock = ready.shift
         if client = sock.kgio_tryaccept
           process_client(client)
           nr += 1
-          alive.chmod(m = 0 == m ? 1 : 0)
+          worker.tick = Time.now.to_i
         end
         break if nr < 0
       end
@@ -632,23 +592,21 @@ class Unicorn::HttpServer
       # we're probably reasonably busy, so avoid calling select()
       # and do a speculative non-blocking accept() on ready listeners
       # before we sleep again in select().
-      redo unless nr == 0 # (nr < 0) => reopen logs
+      unless nr == 0 # (nr < 0) => reopen logs (unlikely)
+        ready = l.dup
+        redo
+      end
 
       ppid == Process.ppid or return
-      alive.chmod(m = 0 == m ? 1 : 0)
 
       # timeout used so we can detect parent death:
-      ret = IO.select(LISTENERS, nil, SELF_PIPE, timeout) and ready = ret[0]
-    rescue Errno::EINTR
-      ready = LISTENERS
+      worker.tick = Time.now.to_i
+      ret = IO.select(l, nil, SELF_PIPE, @timeout) and ready = ret[0]
     rescue Errno::EBADF
       nr < 0 or return
     rescue => e
-      if alive
-        logger.error "Unhandled listen loop exception #{e.inspect}."
-        logger.error e.backtrace.join("\n")
-      end
-    end while alive
+      Unicorn.log_error(@logger, "listen loop error", e) if worker
+    end while worker
   end
 
   # delivers a signal to a worker and fails gracefully if the worker
@@ -656,7 +614,7 @@ class Unicorn::HttpServer
   def kill_worker(signal, wpid)
     Process.kill(signal, wpid)
     rescue Errno::ESRCH
-      worker = WORKERS.delete(wpid) and worker.tmp.close rescue nil
+      worker = WORKERS.delete(wpid) and worker.close rescue nil
   end
 
   # delivers a signal to each worker
@@ -686,7 +644,7 @@ class Unicorn::HttpServer
   def load_config!
     loaded_app = app
     logger.info "reloading config_file=#{config.config_file}"
-    config[:listeners].replace(init_listeners)
+    config[:listeners].replace(@init_listeners)
     config.reload
     config.commit!(self)
     kill_each_worker(:QUIT)
@@ -695,8 +653,8 @@ class Unicorn::HttpServer
     build_app! if preload_app
     logger.info "done reloading config_file=#{config.config_file}"
   rescue StandardError, LoadError, SyntaxError => e
-    logger.error "error reloading config_file=#{config.config_file}: " \
-                 "#{e.class} #{e.message} #{e.backtrace}"
+    Unicorn.log_error(@logger,
+        "error reloading config_file=#{config.config_file}", e)
     self.app = loaded_app
   end
 
@@ -730,5 +688,33 @@ class Unicorn::HttpServer
     SELF_PIPE.replace(Kgio::Pipe.new)
     SELF_PIPE.each { |io| io.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC) }
   end
-end
 
+  def inherit_listeners!
+    # inherit sockets from parents, they need to be plain Socket objects
+    # before they become Kgio::UNIXServer or Kgio::TCPServer
+    inherited = ENV['UNICORN_FD'].to_s.split(/,/).map do |fd|
+      io = Socket.for_fd(fd.to_i)
+      set_server_sockopt(io, listener_opts[sock_name(io)])
+      IO_PURGATORY << io
+      logger.info "inherited addr=#{sock_name(io)} fd=#{fd}"
+      server_cast(io)
+    end
+
+    config_listeners = config[:listeners].dup
+    LISTENERS.replace(inherited)
+
+    # we start out with generic Socket objects that get cast to either
+    # Kgio::TCPServer or Kgio::UNIXServer objects; but since the Socket
+    # objects share the same OS-level file descriptor as the higher-level
+    # *Server objects; we need to prevent Socket objects from being
+    # garbage-collected
+    config_listeners -= listener_names
+    if config_listeners.empty? && LISTENERS.empty?
+      config_listeners << Unicorn::Const::DEFAULT_LISTEN
+      @init_listeners << Unicorn::Const::DEFAULT_LISTEN
+      START_CTX[:argv] << "-l#{Unicorn::Const::DEFAULT_LISTEN}"
+    end
+    config_listeners.each { |addr| listen(addr) }
+    raise ArgumentError, "no listeners" if LISTENERS.empty?
+  end
+end

data/lib/unicorn/socket_helper.rb

@@ -27,9 +27,9 @@ module Unicorn
       # same default value as Mongrel
       :backlog => 1024,
 
-      # since we don't do keepalive, we'll always flush-on-close and
-      # this saves packets for everyone.
-      :tcp_nopush => true,
+      # favor latency over bandwidth savings
+      :tcp_nopush => false,
+      :tcp_nodelay => true,
     }
     #:startdoc:
 
@@ -101,8 +101,7 @@ module Unicorn
       end
       sock.listen(opt[:backlog])
       rescue => e
-        logger.error "error setting socket options: #{e.inspect}"
-        logger.error e.backtrace.join("\n")
+        Unicorn.log_error(logger, message, e)
     end
 
     def log_buffer_sizes(sock, pfx = '')

data/lib/unicorn/worker.rb

@@ -1,4 +1,5 @@
 # -*- encoding: binary -*-
+require "raindrops"
 
 # This class and its members can be considered a stable interface
 # and will not change in a backwards-incompatible fashion between
@@ -7,13 +8,53 @@
 #
 # Some users may want to access it in the before_fork/after_fork hooks.
 # See the Unicorn::Configurator RDoc for examples.
-class Unicorn::Worker < Struct.new(:nr, :tmp, :switched)
+class Unicorn::Worker
+  # :stopdoc:
+  attr_accessor :nr, :switched
+  attr_writer :tmp
+
+  PER_DROP = Raindrops::PAGE_SIZE / Raindrops::SIZE
+  DROPS = []
+
+  def initialize(nr)
+    drop_index = nr / PER_DROP
+    @raindrop = DROPS[drop_index] ||= Raindrops.new(PER_DROP)
+    @offset = nr % PER_DROP
+    @raindrop[@offset] = 0
+    @nr = nr
+    @tmp = @switched = false
+  end
 
   # worker objects may be compared to just plain Integers
   def ==(other_nr) # :nodoc:
-    nr == other_nr
+    @nr == other_nr
+  end
+
+  # called in the worker process
+  def tick=(value) # :nodoc:
+    @raindrop[@offset] = value
+  end
+
+  # called in the master process
+  def tick # :nodoc:
+    @raindrop[@offset]
   end
 
+  # only exists for compatibility
+  def tmp # :nodoc:
+    @tmp ||= begin
+      tmp = Unicorn::TmpIO.new
+      tmp.fcntl(Fcntl::F_SETFD, Fcntl::FD_CLOEXEC)
+      tmp
+    end
+  end
+
+  def close # :nodoc:
+    @tmp.close if @tmp
+  end
+
+  # :startdoc:
+
   # In most cases, you should be using the Unicorn::Configurator#user
   # directive instead.  This method should only be used if you need
   # fine-grained control of exactly when you want to change permissions
@@ -36,12 +77,12 @@ class Unicorn::Worker < Struct.new(:nr, :tmp, :switched)
     uid = Etc.getpwnam(user).uid
     gid = Etc.getgrnam(group).gid if group
     Unicorn::Util.chown_logs(uid, gid)
-    tmp.chown(uid, gid)
+    @tmp.chown(uid, gid) if @tmp
     if gid && Process.egid != gid
       Process.initgroups(user, gid)
       Process::GID.change_privilege(gid)
     end
     Process.euid != uid and Process::UID.change_privilege(uid)
-    self.switched = true
+    @switched = true
   end
 end

data/script/isolate_for_tests

@@ -17,8 +17,9 @@ opts = {
 pid = fork do
   Isolate.now!(opts) do
     gem 'sqlite3-ruby', '1.2.5'
-    gem 'kgio', '2.3.3'
-    gem 'rack', '1.2.2'
+    gem 'raindrops', '0.6.1'
+    gem 'kgio', '2.5.0'
+    gem 'rack', '1.3.0'
   end
 end
 _, status = Process.waitpid2(pid)

data/t/broken-app.ru

@@ -0,0 +1,12 @@
+# we do not want Rack::Lint or anything to protect us
+use Rack::ContentLength
+use Rack::ContentType, "text/plain"
+map "/" do
+  run lambda { |env| [ 200, {}, [ "OK\n" ] ] }
+end
+map "/raise" do
+  run lambda { |env| raise "BAD" }
+end
+map "/nil" do
+  run lambda { |env| nil }
+end

data/t/heartbeat-timeout.ru

@@ -0,0 +1,12 @@
+use Rack::ContentLength
+headers = { 'Content-Type' => 'text/plain' }
+run lambda { |env|
+  case env['PATH_INFO']
+  when "/block-forever"
+    Process.kill(:STOP, $$)
+    sleep # in case STOP signal is not received in time
+    [ 500, headers, [ "Should never get here\n" ] ]
+  else
+    [ 200, headers, [ "#$$\n" ] ]
+  end
+}

data/t/t0001-reload-bad-config.sh

@@ -34,6 +34,7 @@ t_begin "reload signal succeeds" && {
 	done
 
 	grep 'error reloading' $r_err >/dev/null
+	> $r_err
 }
 
 t_begin "hit with curl" && {

data/t/t0004-heartbeat-timeout.sh

@@ -0,0 +1,69 @@
+#!/bin/sh
+. ./test-lib.sh
+
+t_plan 11 "heartbeat/timeout test"
+
+t_begin "setup and startup" && {
+	unicorn_setup
+	echo timeout 3 >> $unicorn_config
+	echo preload_app true >> $unicorn_config
+	unicorn -D heartbeat-timeout.ru -c $unicorn_config
+	unicorn_wait_start
+}
+
+t_begin "read worker PID" && {
+	worker_pid=$(curl -sSf http://$listen/)
+	t_info "worker_pid=$worker_pid"
+}
+
+t_begin "sleep for a bit, ensure worker PID does not change" && {
+	sleep 4
+	test $(curl -sSf http://$listen/) -eq $worker_pid
+}
+
+t_begin "block the worker process to force it to die" && {
+	rm $ok
+	t0=$(date +%s)
+	err="$(curl -sSf http://$listen/block-forever 2>&1 || > $ok)"
+	t1=$(date +%s)
+	elapsed=$(($t1 - $t0))
+	t_info "elapsed=$elapsed err=$err"
+	test x"$err" != x"Should never get here"
+	test x"$err" != x"$worker_pid"
+}
+
+t_begin "ensure worker was killed" && {
+	test -e $ok
+	test 1 -eq $(grep timeout $r_err | grep killing | wc -l)
+}
+
+t_begin "ensure timeout took at least 3 seconds" && {
+	test $elapsed -ge 3
+}
+
+t_begin "we get a fresh new worker process" && {
+	new_worker_pid=$(curl -sSf http://$listen/)
+	test $new_worker_pid -ne $worker_pid
+}
+
+t_begin "truncate the server error log" && {
+	> $r_err
+}
+
+t_begin "SIGSTOP and SIGCONT on unicorn master does not kill worker" && {
+	kill -STOP $unicorn_pid
+	sleep 4
+	kill -CONT $unicorn_pid
+	sleep 2
+	test $new_worker_pid -eq $(curl -sSf http://$listen/)
+}
+
+t_begin "stop server" && {
+	kill -QUIT $unicorn_pid
+}
+
+t_begin "check stderr" && check_stderr
+
+dbgcat r_err
+
+t_done

data/t/t0009-broken-app.sh

@@ -0,0 +1,56 @@
+#!/bin/sh
+. ./test-lib.sh
+
+t_plan 9 "graceful handling of broken apps"
+
+t_begin "setup and start" && {
+	unicorn_setup
+	unicorn -E none -D broken-app.ru -c $unicorn_config
+	unicorn_wait_start
+}
+
+t_begin "normal response is alright" && {
+	test xOK = x"$(curl -sSf http://$listen/)"
+}
+
+t_begin "app raised exception" && {
+	curl -sSf http://$listen/raise 2> $tmp || :
+	grep -F 500 $tmp
+	> $tmp
+}
+
+t_begin "app exception logged and backtrace not swallowed" && {
+	grep -F 'app error' $r_err
+	grep -A1 -F 'app error' $r_err | tail -1 | grep broken-app.ru:
+	dbgcat r_err
+	> $r_err
+}
+
+t_begin "trigger bad response" && {
+	curl -sSf http://$listen/nil 2> $tmp || :
+	grep -F 500 $tmp
+	> $tmp
+}
+
+t_begin "app exception logged" && {
+	grep -F 'app error' $r_err
+	> $r_err
+}
+
+t_begin "normal responses alright afterwards" && {
+	> $tmp
+	curl -sSf http://$listen/ >> $tmp &
+	curl -sSf http://$listen/ >> $tmp &
+	curl -sSf http://$listen/ >> $tmp &
+	curl -sSf http://$listen/ >> $tmp &
+	wait
+	test xOK = x$(sort < $tmp | uniq)
+}
+
+t_begin "teardown" && {
+	kill $unicorn_pid
+}
+
+t_begin "check stderr" && check_stderr
+
+t_done

data/t/t0020-at_exit-handler.sh

@@ -0,0 +1,49 @@
+#!/bin/sh
+. ./test-lib.sh
+
+t_plan 5 "at_exit/END handlers work as expected"
+
+t_begin "setup and startup" && {
+	unicorn_setup
+	cat >> $unicorn_config <<EOF
+at_exit { \$stdout.syswrite("#{Process.pid} BOTH\\n") }
+END { \$stdout.syswrite("#{Process.pid} END BOTH\\n") }
+after_fork do |_,_|
+  at_exit { \$stdout.syswrite("#{Process.pid} WORKER ONLY\\n") }
+  END { \$stdout.syswrite("#{Process.pid} END WORKER ONLY\\n") }
+end
+EOF
+
+	unicorn -D pid.ru -c $unicorn_config
+	unicorn_wait_start
+}
+
+t_begin "read worker PID" && {
+	worker_pid=$(curl -sSf http://$listen/)
+	t_info "worker_pid=$worker_pid"
+}
+
+t_begin "issue graceful shutdown (SIGQUIT) and wait for termination" && {
+	kill -QUIT $unicorn_pid
+
+	while kill -0 $unicorn_pid >/dev/null 2>&1
+	do
+		sleep 1
+	done
+}
+
+t_begin "check stderr" && check_stderr
+
+dbgcat r_err
+dbgcat r_out
+
+t_begin "all at_exit handlers ran" && {
+	grep "$worker_pid BOTH" $r_out
+	grep "$unicorn_pid BOTH" $r_out
+	grep "$worker_pid END BOTH" $r_out
+	grep "$unicorn_pid END BOTH" $r_out
+	grep "$worker_pid WORKER ONLY" $r_out
+	grep "$worker_pid END WORKER ONLY" $r_out
+}
+
+t_done

data/test/benchmark/stack.ru

@@ -0,0 +1,8 @@
+run(lambda { |env|
+  body = "#{caller.size}\n"
+  h = {
+    "Content-Length" => body.size.to_s,
+    "Content-Type" => "text/plain",
+  }
+  [ 200, h, [ body ] ]
+})

data/test/unit/test_droplet.rb

@@ -0,0 +1,28 @@
+require 'test/unit'
+require 'unicorn'
+
+class TestDroplet < Test::Unit::TestCase
+  def test_create_many_droplets
+    now = Time.now.to_i
+    tmp = (0..1024).map do |i|
+      droplet = Unicorn::Worker.new(i)
+      assert droplet.respond_to?(:tick)
+      assert_equal 0, droplet.tick
+      assert_equal(now, droplet.tick = now)
+      assert_equal now, droplet.tick
+      assert_equal(0, droplet.tick = 0)
+      assert_equal 0, droplet.tick
+    end
+  end
+
+  def test_shared_process
+    droplet = Unicorn::Worker.new(0)
+    _, status = Process.waitpid2(fork { droplet.tick += 1; exit!(0) })
+    assert status.success?, status.inspect
+    assert_equal 1, droplet.tick
+
+    _, status = Process.waitpid2(fork { droplet.tick += 1; exit!(0) })
+    assert status.success?, status.inspect
+    assert_equal 2, droplet.tick
+  end
+end

data/test/unit/test_http_parser.rb

@@ -740,7 +740,7 @@ class HttpParserTest < Test::Unit::TestCase
     # then that large header names are caught
     10.times do |c|
       get = "GET /#{rand_data(10,120)} HTTP/1.1\r\nX-#{rand_data(1024, 1024+(c*1024))}: Test\r\n\r\n"
-      assert_raises Unicorn::HttpParserError do
+      assert_raises(Unicorn::HttpParserError,Unicorn::RequestURITooLongError) do
         parser.buf << get
         parser.parse
         parser.clear
@@ -750,7 +750,7 @@ class HttpParserTest < Test::Unit::TestCase
     # then that large mangled field values are caught
     10.times do |c|
       get = "GET /#{rand_data(10,120)} HTTP/1.1\r\nX-Test: #{rand_data(1024, 1024+(c*1024), false)}\r\n\r\n"
-      assert_raises Unicorn::HttpParserError do
+      assert_raises(Unicorn::HttpParserError,Unicorn::RequestURITooLongError) do
         parser.buf << get
         parser.parse
         parser.clear
@@ -761,7 +761,7 @@ class HttpParserTest < Test::Unit::TestCase
     get = "GET /#{rand_data(10,120)} HTTP/1.1\r\n"
     get << "X-Test: test\r\n" * (80 * 1024)
     parser.buf << get
-    assert_raises Unicorn::HttpParserError do
+    assert_raises(Unicorn::HttpParserError,Unicorn::RequestURITooLongError) do
       parser.parse
     end
     parser.clear
@@ -769,7 +769,7 @@ class HttpParserTest < Test::Unit::TestCase
     # finally just that random garbage gets blocked all the time
     10.times do |c|
       get = "GET #{rand_data(1024, 1024+(c*1024), false)} #{rand_data(1024, 1024+(c*1024), false)}\r\n\r\n"
-      assert_raises Unicorn::HttpParserError do
+      assert_raises(Unicorn::HttpParserError,Unicorn::RequestURITooLongError) do
         parser.buf << get
         parser.parse
         parser.clear

data/test/unit/test_http_parser_ng.rb

@@ -237,6 +237,17 @@ class HttpParserNgTest < Test::Unit::TestCase
     assert @parser.keepalive?
   end
 
+  def test_chunked_empty
+    str = @parser.buf
+    req = @parser.env
+    str << "PUT / HTTP/1.1\r\ntransfer-Encoding: chunked\r\n\r\n"
+    assert_equal req, @parser.parse, "msg=#{str}"
+    assert_equal 0, str.size
+    tmp = ""
+    assert_equal str, @parser.filter_body(tmp, str << "0\r\n\r\n")
+    assert_equal "", tmp
+  end
+
   def test_two_chunks
     str = @parser.buf
     str << "PUT / HTTP/1.1\r\ntransfer-Encoding: chunked\r\n\r\n"
@@ -651,4 +662,47 @@ class HttpParserNgTest < Test::Unit::TestCase
     assert_equal expect, @parser.parse
     assert ! @parser.next?
   end
+
+  def test_chunk_only
+    tmp = ""
+    assert_equal @parser, @parser.dechunk!
+    assert_nil @parser.filter_body(tmp, "6\r\n")
+    assert_equal "", tmp
+    assert_nil @parser.filter_body(tmp, "abcdef")
+    assert_equal "abcdef", tmp
+    assert_nil @parser.filter_body(tmp, "\r\n")
+    assert_equal "", tmp
+    src = "0\r\n\r\n"
+    assert_equal src.object_id, @parser.filter_body(tmp, src).object_id
+    assert_equal "", tmp
+  end
+
+  def test_chunk_only_bad_align
+    tmp = ""
+    assert_equal @parser, @parser.dechunk!
+    assert_nil @parser.filter_body(tmp, "6\r\na")
+    assert_equal "a", tmp
+    assert_nil @parser.filter_body(tmp, "bcde")
+    assert_equal "bcde", tmp
+    assert_nil @parser.filter_body(tmp, "f\r")
+    assert_equal "f", tmp
+    src = "\n0\r\n\r\n"
+    assert_equal src.object_id, @parser.filter_body(tmp, src).object_id
+    assert_equal "", tmp
+  end
+
+  def test_chunk_only_reset_ok
+    tmp = ""
+    assert_equal @parser, @parser.dechunk!
+    src = "1\r\na\r\n0\r\n\r\n"
+    assert_nil @parser.filter_body(tmp, src)
+    assert_equal "a", tmp
+    assert_equal src.object_id, @parser.filter_body(tmp, src).object_id
+
+    assert_equal @parser, @parser.dechunk!
+    src = "0\r\n\r\n"
+    assert_equal src.object_id, @parser.filter_body(tmp, src).object_id
+    assert_equal "", tmp
+    assert_equal src, @parser.filter_body(tmp, src)
+  end
 end

data/test/unit/test_response.rb

@@ -15,7 +15,7 @@ class ResponseTest < Test::Unit::TestCase
   include Unicorn::HttpResponse
 
   def test_httpdate
-    before = Time.now.to_i
+    before = Time.now.to_i - 1
     str = httpdate
     assert_kind_of(String, str)
     middle = Time.parse(str).to_i

data/test/unit/test_server.rb

@@ -39,7 +39,7 @@ class WebServerTest < Test::Unit::TestCase
     redirect_test_io do
       wait_workers_ready("test_stderr.#$$.log", 1)
       File.truncate("test_stderr.#$$.log", 0)
-      @server.stop(true)
+      @server.stop(false)
     end
     reset_sig_handlers
   end

data/test/unit/test_signals.rb

@@ -124,7 +124,7 @@ class SignalsTest < Test::Unit::TestCase
     assert diff > 1.0, "diff was #{diff.inspect}"
     assert diff < 60.0
     ensure
-      Process.kill(:QUIT, pid) rescue nil
+      Process.kill(:TERM, pid) rescue nil
   end
 
   def test_response_write

data/test/unit/test_upload.rb

@@ -53,7 +53,7 @@ class UploadTest < Test::Unit::TestCase
   end
 
   def teardown
-    redirect_test_io { @server.stop(true) } if @server
+    redirect_test_io { @server.stop(false) } if @server
     @random.close
     reset_sig_handlers
   end

data/unicorn.gemspec

@@ -34,7 +34,8 @@ Gem::Specification.new do |s|
   # commented out.  Nevertheless, upgrading to Rails 2.3.4 or later is
   # *strongly* recommended for security reasons.
   s.add_dependency(%q<rack>)
-  s.add_dependency(%q<kgio>, '~> 2.3')
+  s.add_dependency(%q<kgio>, '~> 2.4')
+  s.add_dependency(%q<raindrops>, '~> 0.6')
 
   s.add_development_dependency('isolate', '~> 3.1')
   s.add_development_dependency('wrongdoc', '~> 1.5')

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: unicorn
 version: !ruby/object:Gem::Version 
-  hash: 27
+  hash: 63
   prerelease: 
   segments: 
-  - 3
-  - 7
+  - 4
   - 0
-  version: 3.7.0
+  - 0
+  version: 4.0.0
 platform: ruby
 authors: 
 - Unicorn hackers
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-06-09 00:00:00 Z
+date: 2011-06-27 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: rack
@@ -39,17 +39,32 @@ dependencies:
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
-        hash: 5
+        hash: 11
         segments: 
         - 2
-        - 3
-        version: "2.3"
+        - 4
+        version: "2.4"
   type: :runtime
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
-  name: isolate
+  name: raindrops
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 7
+        segments: 
+        - 0
+        - 6
+        version: "0.6"
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: isolate
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -60,11 +75,11 @@ dependencies:
         - 1
         version: "3.1"
   type: :development
-  version_requirements: *id003
+  version_requirements: *id004
 - !ruby/object:Gem::Dependency 
   name: wrongdoc
   prerelease: false
-  requirement: &id004 !ruby/object:Gem::Requirement 
+  requirement: &id005 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -75,7 +90,7 @@ dependencies:
         - 5
         version: "1.5"
   type: :development
-  version_requirements: *id004
+  version_requirements: *id005
 description: |-
   \Unicorn is an HTTP server for Rack applications designed to only serve
   fast clients on low-latency, high-bandwidth connections and take
@@ -212,7 +227,9 @@ files:
 - t/bin/sha1sum.rb
 - t/bin/unused_listen
 - t/bin/utee
+- t/broken-app.ru
 - t/env.ru
+- t/heartbeat-timeout.ru
 - t/my-tap-lib.sh
 - t/oob_gc.ru
 - t/oob_gc_path.ru
@@ -255,12 +272,14 @@ files:
 - t/t0002-config-conflict.sh
 - t/t0002-parser-error.sh
 - t/t0003-working_directory.sh
+- t/t0004-heartbeat-timeout.sh
 - t/t0004-working_directory_broken.sh
 - t/t0005-working_directory_app.rb.sh
 - t/t0006-reopen-logs.sh
 - t/t0006.ru
 - t/t0007-working_directory_no_embed_cli.sh
 - t/t0008-back_out_of_upgrade.sh
+- t/t0009-broken-app.sh
 - t/t0009-winch_ttin.sh
 - t/t0010-reap-logging.sh
 - t/t0011-active-unix-socket.sh
@@ -274,6 +293,7 @@ files:
 - t/t0017-trust-x-forwarded-true.sh
 - t/t0018-write-on-close.sh
 - t/t0019-max_header_len.sh
+- t/t0020-at_exit-handler.sh
 - t/t0100-rack-input-tests.sh
 - t/t0116-client_body_buffer_size.sh
 - t/t0116.ru
@@ -291,6 +311,7 @@ files:
 - test/aggregate.rb
 - test/benchmark/README
 - test/benchmark/dd.ru
+- test/benchmark/stack.ru
 - test/exec/README
 - test/exec/test_exec.rb
 - test/rails/app-1.2.3/.gitignore
@@ -356,6 +377,7 @@ files:
 - test/rails/test_rails.rb
 - test/test_helper.rb
 - test/unit/test_configurator.rb
+- test/unit/test_droplet.rb
 - test/unit/test_http_parser.rb
 - test/unit/test_http_parser_ng.rb
 - test/unit/test_http_parser_xftrust.rb