unicorn 5.2.0 → 5.3.0

data/ext/unicorn_http/httpdate.c

@@ -64,13 +64,13 @@ static VALUE httpdate(VALUE self)
 	return buf;
 }
 
-void init_unicorn_httpdate(void)
+void init_unicorn_httpdate(VALUE mark_ary)
 {
 	VALUE mod = rb_define_module("Unicorn");
 	mod = rb_define_module_under(mod, "HttpResponse");
 
 	buf = rb_str_new(0, buf_capa - 1);
-	rb_global_variable(&buf);
+	rb_ary_push(mark_ary, buf);
 	buf_ptr = RSTRING_PTR(buf);
 	httpdate(Qnil);
 

data/ext/unicorn_http/unicorn_http.c

@@ -15,7 +15,7 @@
 #include "global_variables.h"
 #include "c_util.h"
 
-void init_unicorn_httpdate(void);
+void init_unicorn_httpdate(VALUE mark_ary);
 
 #define UH_FL_CHUNKED  0x1
 #define UH_FL_HASBODY  0x2
@@ -4211,8 +4211,10 @@ static VALUE HttpParser_rssget(VALUE self)
 
 void Init_unicorn_http(void)
 {
+  static VALUE mark_ary;
   VALUE mUnicorn, cHttpParser;
 
+  mark_ary = rb_ary_new();
   mUnicorn = rb_define_module("Unicorn");
   cHttpParser = rb_define_class_under(mUnicorn, "HttpParser", rb_cObject);
   eHttpParserError =
@@ -4222,7 +4224,7 @@ void Init_unicorn_http(void)
   e414 = rb_define_class_under(mUnicorn, "RequestURITooLongError",
                                eHttpParserError);
 
-  init_globals();
+  init_globals(mark_ary);
   rb_define_alloc_func(cHttpParser, HttpParser_alloc);
   rb_define_method(cHttpParser, "initialize", HttpParser_init, 0);
   rb_define_method(cHttpParser, "clear", HttpParser_clear, 0);
@@ -4258,14 +4260,17 @@ void Init_unicorn_http(void)
 
   rb_define_singleton_method(cHttpParser, "max_header_len=", set_maxhdrlen, 1);
 
-  init_common_fields();
+  init_common_fields(mark_ary);
   SET_GLOBAL(g_http_host, "HOST");
   SET_GLOBAL(g_http_trailer, "TRAILER");
   SET_GLOBAL(g_http_transfer_encoding, "TRANSFER_ENCODING");
   SET_GLOBAL(g_content_length, "CONTENT_LENGTH");
   SET_GLOBAL(g_http_connection, "CONNECTION");
   id_set_backtrace = rb_intern("set_backtrace");
-  init_unicorn_httpdate();
+  init_unicorn_httpdate(mark_ary);
+
+  OBJ_FREEZE(mark_ary);
+  rb_global_variable(&mark_ary);
 
 #ifndef HAVE_RB_HASH_CLEAR
   id_clear = rb_intern("clear");

data/ext/unicorn_http/unicorn_http.rl

@@ -13,7 +13,7 @@
 #include "global_variables.h"
 #include "c_util.h"
 
-void init_unicorn_httpdate(void);
+void init_unicorn_httpdate(VALUE mark_ary);
 
 #define UH_FL_CHUNKED  0x1
 #define UH_FL_HASBODY  0x2
@@ -917,8 +917,10 @@ static VALUE HttpParser_rssget(VALUE self)
 
 void Init_unicorn_http(void)
 {
+  static VALUE mark_ary;
   VALUE mUnicorn, cHttpParser;
 
+  mark_ary = rb_ary_new();
   mUnicorn = rb_define_module("Unicorn");
   cHttpParser = rb_define_class_under(mUnicorn, "HttpParser", rb_cObject);
   eHttpParserError =
@@ -928,7 +930,7 @@ void Init_unicorn_http(void)
   e414 = rb_define_class_under(mUnicorn, "RequestURITooLongError",
                                eHttpParserError);
 
-  init_globals();
+  init_globals(mark_ary);
   rb_define_alloc_func(cHttpParser, HttpParser_alloc);
   rb_define_method(cHttpParser, "initialize", HttpParser_init, 0);
   rb_define_method(cHttpParser, "clear", HttpParser_clear, 0);
@@ -964,14 +966,17 @@ void Init_unicorn_http(void)
 
   rb_define_singleton_method(cHttpParser, "max_header_len=", set_maxhdrlen, 1);
 
-  init_common_fields();
+  init_common_fields(mark_ary);
   SET_GLOBAL(g_http_host, "HOST");
   SET_GLOBAL(g_http_trailer, "TRAILER");
   SET_GLOBAL(g_http_transfer_encoding, "TRANSFER_ENCODING");
   SET_GLOBAL(g_content_length, "CONTENT_LENGTH");
   SET_GLOBAL(g_http_connection, "CONNECTION");
   id_set_backtrace = rb_intern("set_backtrace");
-  init_unicorn_httpdate();
+  init_unicorn_httpdate(mark_ary);
+
+  OBJ_FREEZE(mark_ary);
+  rb_global_variable(&mark_ary);
 
 #ifndef HAVE_RB_HASH_CLEAR
   id_clear = rb_intern("clear");

data/lib/unicorn.rb

@@ -95,7 +95,7 @@ def self.builder(ru, op)
 
   # returns an array of strings representing TCP listen socket addresses
   # and Unix domain socket paths.  This is useful for use with
-  # Raindrops::Middleware under Linux: http://raindrops.bogomips.org/
+  # Raindrops::Middleware under Linux: https://bogomips.org/raindrops/
   def self.listener_names
     Unicorn::HttpServer::LISTENERS.map do |io|
       Unicorn::SocketHelper.sock_name(io)

data/lib/unicorn/configurator.rb

@@ -41,10 +41,22 @@ class Unicorn::Configurator
     :before_exec => lambda { |server|
         server.logger.info("forked child re-executing...")
       },
+    :after_worker_exit => lambda { |server, worker, status|
+        m = "reaped #{status.inspect} worker=#{worker.nr rescue 'unknown'}"
+        if status.success?
+          server.logger.info(m)
+        else
+          server.logger.error(m)
+        end
+      },
+    :after_worker_ready => lambda { |server, worker|
+        server.logger.info("worker=#{worker.nr} ready")
+      },
     :pid => nil,
+    :worker_exec => false,
     :preload_app => false,
     :check_client_connection => false,
-    :rewindable_input => true, # for Rack 2.x: (Rack::VERSION[0] <= 1),
+    :rewindable_input => true,
     :client_body_buffer_size => Unicorn::Const::MAX_BODY,
   }
   #:startdoc:
@@ -151,6 +163,38 @@ def after_fork(*args, &block)
     set_hook(:after_fork, block_given? ? block : args[0])
   end
 
+  # sets after_worker_exit hook to a given block.  This block will be called
+  # by the master process after a worker exits:
+  #
+  #  after_worker_exit do |server,worker,status|
+  #    # status is a Process::Status instance for the exited worker process
+  #    unless status.success?
+  #      server.logger.error("worker process failure: #{status.inspect}")
+  #    end
+  #  end
+  #
+  # after_worker_exit is only available in unicorn 5.3.0+
+  def after_worker_exit(*args, &block)
+    set_hook(:after_worker_exit, block_given? ? block : args[0], 3)
+  end
+
+  # sets after_worker_ready hook to a given block.  This block will be called
+  # by a worker process after it has been fully loaded, directly before it
+  # starts responding to requests:
+  #
+  #  after_worker_ready do |server,worker|
+  #    server.logger.info("worker #{worker.nr} ready, dropping privileges")
+  #    worker.user('username', 'groupname')
+  #  end
+  #
+  # Do not use Configurator#user if you rely on changing users in the
+  # after_worker_ready hook.
+  #
+  # after_worker_ready is only available in unicorn 5.3.0+
+  def after_worker_ready(*args, &block)
+    set_hook(:after_worker_ready, block_given? ? block : args[0])
+  end
+
   # sets before_fork got be a given Proc object.  This Proc
   # object will be called by the master process before forking
   # each worker.
@@ -200,6 +244,17 @@ def timeout(seconds)
     set[:timeout] = seconds > max ? max : seconds
   end
 
+  # Whether to exec in each worker process after forking.  This changes the
+  # memory layout of each worker process, which is a security feature designed
+  # to defeat possible address space discovery attacks.  Note that using
+  # worker_exec only makes sense if you are not preloading the application,
+  # and will result in higher memory usage.
+  #
+  # worker_exec is only available in unicorn 5.3.0+
+  def worker_exec(bool)
+    set_bool(:worker_exec, bool)
+  end
+
   # sets the current number of worker_processes to +nr+.  Each worker
   # process will serve exactly one client at a time.  You can
   # increment or decrement this value at runtime by sending SIGTTIN
@@ -466,13 +521,12 @@ def preload_app(bool)
   # Disabling rewindability can improve performance by lowering
   # I/O and memory usage for applications that accept uploads.
   # Keep in mind that the Rack 1.x spec requires
-  # \env[\"rack.input\"] to be rewindable, so this allows
-  # intentionally violating the current Rack 1.x spec.
+  # \env[\"rack.input\"] to be rewindable,
+  # but the Rack 2.x spec does not.
   #
-  # +rewindable_input+ defaults to +true+ when used with Rack 1.x for
-  # Rack conformance.  When Rack 2.x is finalized, this will most
-  # likely default to +false+ while still conforming to the newer
-  # (less demanding) spec.
+  # +rewindable_input+ defaults to +true+ for compatibility.
+  # Setting it to +false+ may be safe for applications and
+  # frameworks developed for Rack 2.x and later.
   def rewindable_input(bool)
     set_bool(:rewindable_input, bool)
   end
@@ -548,6 +602,10 @@ def working_directory(path)
   # This switch will occur after calling the after_fork hook, and only
   # if the Worker#user method is not called in the after_fork hook
   # +group+ is optional and will not change if unspecified.
+  #
+  # Do not use Configurator#user if you rely on changing users in the
+  # after_worker_ready hook.  Instead, you need to call Worker#user
+  # directly in after_worker_ready.
   def user(user, group = nil)
     # raises ArgumentError on invalid user/group
     Etc.getpwnam(user)

data/lib/unicorn/http_request.rb

@@ -2,6 +2,7 @@
 # :enddoc:
 # no stable API here
 require 'unicorn_http'
+require 'raindrops'
 
 # TODO: remove redundant names
 Unicorn.const_set(:HttpRequest, Unicorn::HttpParser)
@@ -24,12 +25,11 @@ class Unicorn::HttpParser
   NULL_IO = StringIO.new("")
 
   # :stopdoc:
-  # A frozen format for this is about 15% faster
-  # Drop these frozen strings when Ruby 2.2 becomes more prevalent,
-  # 2.2+ optimizes hash assignments when used with literal string keys
-  HTTP_RESPONSE_START = [ 'HTTP', '/1.1 ']
+  HTTP_RESPONSE_START = [ 'HTTP'.freeze, '/1.1 '.freeze ]
+  EMPTY_ARRAY = [].freeze
   @@input_class = Unicorn::TeeInput
   @@check_client_connection = false
+  @@tcpi_inspect_ok = Socket.const_defined?(:TCP_INFO)
 
   def self.input_class
     @@input_class
@@ -83,11 +83,7 @@ def read(socket)
       false until add_parse(socket.kgio_read!(16384))
     end
 
-    # detect if the socket is valid by writing a partial response:
-    if @@check_client_connection && headers?
-      self.response_start_sent = true
-      HTTP_RESPONSE_START.each { |c| socket.write(c) }
-    end
+    check_client_connection(socket) if @@check_client_connection
 
     e['rack.input'] = 0 == content_length ?
                       NULL_IO : @@input_class.new(socket, self)
@@ -108,4 +104,88 @@ def call
   def hijacked?
     env.include?('rack.hijack_io'.freeze)
   end
+
+  if Raindrops.const_defined?(:TCP_Info)
+    TCPI = Raindrops::TCP_Info.allocate
+
+    def check_client_connection(socket) # :nodoc:
+      if Unicorn::TCPClient === socket
+        # Raindrops::TCP_Info#get!, #state (reads struct tcp_info#tcpi_state)
+        raise Errno::EPIPE, "client closed connection".freeze,
+              EMPTY_ARRAY if closed_state?(TCPI.get!(socket).state)
+      else
+        write_http_header(socket)
+      end
+    end
+
+    if Raindrops.const_defined?(:TCP)
+      # raindrops 0.18.0+ supports FreeBSD + Linux using the same names
+      # Evaluate these hash lookups at load time so we can
+      # generate an opt_case_dispatch instruction
+      eval <<-EOS
+      def closed_state?(state) # :nodoc:
+        case state
+        when #{Raindrops::TCP[:ESTABLISHED]}
+          false
+        when #{Raindrops::TCP.values_at(
+              :CLOSE_WAIT, :TIME_WAIT, :CLOSE, :LAST_ACK, :CLOSING).join(',')}
+          true
+        else
+          false
+        end
+      end
+      EOS
+    else
+      # raindrops before 0.18 only supported TCP_INFO under Linux
+      def closed_state?(state) # :nodoc:
+        case state
+        when 1 # ESTABLISHED
+          false
+        when 8, 6, 7, 9, 11 # CLOSE_WAIT, TIME_WAIT, CLOSE, LAST_ACK, CLOSING
+          true
+        else
+          false
+        end
+      end
+    end
+  else
+
+    # Ruby 2.2+ can show struct tcp_info as a string Socket::Option#inspect.
+    # Not that efficient, but probably still better than doing unnecessary
+    # work after a client gives up.
+    def check_client_connection(socket) # :nodoc:
+      if Unicorn::TCPClient === socket && @@tcpi_inspect_ok
+        opt = socket.getsockopt(Socket::IPPROTO_TCP, Socket::TCP_INFO).inspect
+        if opt =~ /\bstate=(\S+)/
+          raise Errno::EPIPE, "client closed connection".freeze,
+                EMPTY_ARRAY if closed_state_str?($1)
+        else
+          @@tcpi_inspect_ok = false
+          write_http_header(socket)
+        end
+        opt.clear
+      else
+        write_http_header(socket)
+      end
+    end
+
+    def closed_state_str?(state)
+      case state
+      when 'ESTABLISHED'
+        false
+      # not a typo, ruby maps TCP_CLOSE (no 'D') to state=CLOSED (w/ 'D')
+      when 'CLOSE_WAIT', 'TIME_WAIT', 'CLOSED', 'LAST_ACK', 'CLOSING'
+        true
+      else
+        false
+      end
+    end
+  end
+
+  def write_http_header(socket) # :nodoc:
+    if headers?
+      self.response_start_sent = true
+      HTTP_RESPONSE_START.each { |c| socket.write(c) }
+    end
+  end
 end

data/lib/unicorn/http_server.rb

@@ -15,6 +15,7 @@ class Unicorn::HttpServer
                 :before_fork, :after_fork, :before_exec,
                 :listener_opts, :preload_app,
                 :orig_app, :config, :ready_pipe, :user
+  attr_writer   :after_worker_exit, :after_worker_ready, :worker_exec
 
   attr_reader :pid, :logger
   include Unicorn::SocketHelper
@@ -88,6 +89,7 @@ def initialize(app, options = {})
     @self_pipe = []
     @workers = {} # hash maps PIDs to Workers
     @sig_queue = [] # signal queue used for self-piping
+    @pid = nil
 
     # we try inheriting listeners first, so we bind them later.
     # we don't write the pid file until we've bound listeners in case
@@ -104,6 +106,14 @@ def initialize(app, options = {})
     # list of signals we care about and trap in master.
     @queue_sigs = [
       :WINCH, :QUIT, :INT, :TERM, :USR1, :USR2, :HUP, :TTIN, :TTOU ]
+
+    @worker_data = if worker_data = ENV['UNICORN_WORKER']
+      worker_data = worker_data.split(',').map!(&:to_i)
+      worker_data[1] = worker_data.slice!(1..2).map do |i|
+        Kgio::Pipe.for_fd(i)
+      end
+      worker_data
+    end
   end
 
   # Runs the thing.  Returns self so you can run join on it
@@ -112,7 +122,7 @@ def start
     # this pipe is used to wake us up from select(2) in #join when signals
     # are trapped.  See trap_deferred.
     @self_pipe.replace(Unicorn.pipe)
-    @master_pid = $$
+    @master_pid = @worker_data ? Process.ppid : $$
 
     # setup signal handlers before writing pid file in case people get
     # trigger happy and send signals as soon as the pid file exists.
@@ -395,8 +405,7 @@ def reap_all_workers
         proc_name 'master'
       else
         worker = @workers.delete(wpid) and worker.close rescue nil
-        m = "reaped #{status.inspect} worker=#{worker.nr rescue 'unknown'}"
-        status.success? ? logger.info(m) : logger.error(m)
+        @after_worker_exit.call(self, worker, status)
       end
     rescue Errno::ECHILD
       break
@@ -430,11 +439,7 @@ def reexec
     end
 
     @reexec_pid = fork do
-      listener_fds = {}
-      LISTENERS.each do |sock|
-        sock.close_on_exec = false
-        listener_fds[sock.fileno] = sock
-      end
+      listener_fds = listener_sockets
       ENV['UNICORN_FD'] = listener_fds.keys.join(',')
       Dir.chdir(START_CTX[:cwd])
       cmd = [ START_CTX[0] ].concat(START_CTX[:argv])
@@ -442,12 +447,7 @@ def reexec
       # avoid leaking FDs we don't know about, but let before_exec
       # unset FD_CLOEXEC, if anything else in the app eventually
       # relies on FD inheritence.
-      (3..1024).each do |io|
-        next if listener_fds.include?(io)
-        io = IO.for_fd(io) rescue next
-        io.autoclose = false
-        io.close_on_exec = true
-      end
+      close_sockets_on_exec(listener_fds)
 
       # exec(command, hash) works in at least 1.9.1+, but will only be
       # required in 1.9.4/2.0.0 at earliest.
@@ -459,6 +459,40 @@ def reexec
     proc_name 'master (old)'
   end
 
+  def worker_spawn(worker)
+    listener_fds = listener_sockets
+    env = {}
+    env['UNICORN_FD'] = listener_fds.keys.join(',')
+
+    listener_fds[worker.to_io.fileno] = worker.to_io
+    listener_fds[worker.master.fileno] = worker.master
+
+    worker_info = [worker.nr, worker.to_io.fileno, worker.master.fileno]
+    env['UNICORN_WORKER'] = worker_info.join(',')
+
+    close_sockets_on_exec(listener_fds)
+
+    Process.spawn(env, START_CTX[0], *START_CTX[:argv], listener_fds)
+  end
+
+  def listener_sockets
+    listener_fds = {}
+    LISTENERS.each do |sock|
+      sock.close_on_exec = false
+      listener_fds[sock.fileno] = sock
+    end
+    listener_fds
+  end
+
+  def close_sockets_on_exec(sockets)
+    (3..1024).each do |io|
+      next if sockets.include?(io)
+      io = IO.for_fd(io) rescue next
+      io.autoclose = false
+      io.close_on_exec = true
+    end
+  end
+
   # forcibly terminate all workers that haven't checked in in timeout seconds.  The timeout is implemented using an unlinked File
   def murder_lazy_workers
     next_sleep = @timeout - 1
@@ -495,19 +529,29 @@ def after_fork_internal
   end
 
   def spawn_missing_workers
+    if @worker_data
+      worker = Unicorn::Worker.new(*@worker_data)
+      after_fork_internal
+      worker_loop(worker)
+      exit
+    end
+
     worker_nr = -1
     until (worker_nr += 1) == @worker_processes
       @workers.value?(worker_nr) and next
       worker = Unicorn::Worker.new(worker_nr)
       before_fork.call(self, worker)
-      if pid = fork
-        @workers[pid] = worker
-        worker.atfork_parent
-      else
+
+      pid = @worker_exec ? worker_spawn(worker) : fork
+
+      unless pid
         after_fork_internal
         worker_loop(worker)
         exit
       end
+
+      @workers[pid] = worker
+      worker.atfork_parent
     end
     rescue => e
       @logger.error(e) rescue nil
@@ -644,7 +688,7 @@ def worker_loop(worker)
     trap(:USR1) { nr = -65536 }
 
     ready = readers.dup
-    @logger.info "worker=#{worker.nr} ready"
+    @after_worker_ready.call(self, worker)
 
     begin
       nr < 0 and reopen_worker_logs(worker.nr)