unicorn 5.2.0 → 5.3.1

data/TUNING

@@ -72,10 +72,28 @@ See Unicorn::Configurator for details on the config file format.
   have them unbuffered (File#sync = true) or they are
   record(line)-buffered in userspace before any writes.
 
-== Kernel Parameters (Linux sysctl)
+== Kernel Parameters (Linux sysctl and sysfs)
 
 WARNING: Do not change system parameters unless you know what you're doing!
 
+* Transparent hugepages (THP) improves performance in many cases,
+  but can also increase memory use when relying on a
+  copy-on-write(CoW)-friendly GC (Ruby 2.0+) with "preload_app true".
+  CoW operates at the page level, so writing to a huge page would
+  trigger a 2 MB copy (x86-64), as opposed to a 4 KB copy on a
+  regular (non-huge) page.
+
+  Consider only allowing THP to be used when it is requested via the
+  madvise(2) syscall:
+
+	echo madvise >/sys/kernel/mm/transparent_hugepage/enabled
+
+  Or disabling it system-wide, via "never".
+
+  n.b. "page" in this context only applies to the OS kernel,
+  Ruby GC implementations also use this term for the same concept
+  in a way that is agnostic to the OS.
+
 * net.core.rmem_max and net.core.wmem_max can increase the allowed
   size of :rcvbuf and :sndbuf respectively. This is mostly only useful
   for UNIX domain sockets which do not have auto-tuning buffer sizes.

data/ext/unicorn_http/common_field_optimization.h

@@ -60,7 +60,7 @@ static struct common_field common_http_fields[] = {
 #define HTTP_PREFIX_LEN (sizeof(HTTP_PREFIX) - 1)
 
 /* this function is not performance-critical, called only at load time */
-static void init_common_fields(void)
+static void init_common_fields(VALUE mark_ary)
 {
   int i;
   struct common_field *cf = common_http_fields;
@@ -77,7 +77,7 @@ static void init_common_fields(void)
       cf->value = rb_str_new(tmp, HTTP_PREFIX_LEN + cf->len);
     }
     cf->value = rb_obj_freeze(cf->value);
-    rb_global_variable(&cf->value);
+    rb_ary_push(mark_ary, cf->value);
   }
 }
 

data/ext/unicorn_http/ext_help.h

@@ -1,26 +1,6 @@
 #ifndef ext_help_h
 #define ext_help_h
 
-#ifndef RSTRING_PTR
-#define RSTRING_PTR(s) (RSTRING(s)->ptr)
-#endif /* !defined(RSTRING_PTR) */
-#ifndef RSTRING_LEN
-#define RSTRING_LEN(s) (RSTRING(s)->len)
-#endif /* !defined(RSTRING_LEN) */
-
-#ifndef HAVE_RB_STR_SET_LEN
-#  ifdef RUBINIUS
-#    error we should never get here with current Rubinius (1.x)
-#  endif
-/* this is taken from Ruby 1.8.7, 1.8.6 may not have it */
-static void rb_18_str_set_len(VALUE str, long len)
-{
-  RSTRING(str)->len = len;
-  RSTRING(str)->ptr[len] = '\0';
-}
-#  define rb_str_set_len(str,len) rb_18_str_set_len(str,len)
-#endif /* !defined(HAVE_RB_STR_SET_LEN) */
-
 /* not all Ruby implementations support frozen objects (Rubinius does not) */
 #if defined(OBJ_FROZEN)
 #  define assert_frozen(f) assert(OBJ_FROZEN(f) && "unfrozen object")

data/ext/unicorn_http/extconf.rb

@@ -4,7 +4,7 @@
 have_macro("SIZEOF_OFF_T", "ruby.h") or check_sizeof("off_t", "sys/types.h")
 have_macro("SIZEOF_SIZE_T", "ruby.h") or check_sizeof("size_t", "sys/types.h")
 have_macro("SIZEOF_LONG", "ruby.h") or check_sizeof("long", "sys/types.h")
-have_func("rb_str_set_len", "ruby.h")
+have_func("rb_str_set_len", "ruby.h") or abort 'Ruby 1.9.3+ required'
 have_func("rb_hash_clear", "ruby.h") # Ruby 2.0+
 have_func("gmtime_r", "time.h")
 

data/ext/unicorn_http/global_variables.h

@@ -56,7 +56,7 @@ NORETURN(static void parser_raise(VALUE klass, const char *));
 /** Defines global strings in the init method. */
 #define DEF_GLOBAL(N, val) do { \
   g_##N = rb_obj_freeze(rb_str_new(val, sizeof(val) - 1)); \
-  rb_global_variable(&g_##N); \
+  rb_ary_push(mark_ary, g_##N); \
 } while (0)
 
 /* Defines the maximum allowed lengths for various input elements.*/
@@ -67,7 +67,7 @@ DEF_MAX_LENGTH(FRAGMENT, 1024); /* Don't know if this length is specified somewh
 DEF_MAX_LENGTH(REQUEST_PATH, 4096); /* common PATH_MAX on modern systems */
 DEF_MAX_LENGTH(QUERY_STRING, (1024 * 10));
 
-static void init_globals(void)
+static void init_globals(VALUE mark_ary)
 {
   DEF_GLOBAL(rack_url_scheme, "rack.url_scheme");
   DEF_GLOBAL(request_method, "REQUEST_METHOD");

data/ext/unicorn_http/httpdate.c

@@ -64,13 +64,13 @@ static VALUE httpdate(VALUE self)
 	return buf;
 }
 
-void init_unicorn_httpdate(void)
+void init_unicorn_httpdate(VALUE mark_ary)
 {
 	VALUE mod = rb_define_module("Unicorn");
 	mod = rb_define_module_under(mod, "HttpResponse");
 
 	buf = rb_str_new(0, buf_capa - 1);
-	rb_global_variable(&buf);
+	rb_ary_push(mark_ary, buf);
 	buf_ptr = RSTRING_PTR(buf);
 	httpdate(Qnil);
 

data/ext/unicorn_http/unicorn_http.c

@@ -15,7 +15,7 @@
 #include "global_variables.h"
 #include "c_util.h"
 
-void init_unicorn_httpdate(void);
+void init_unicorn_httpdate(VALUE mark_ary);
 
 #define UH_FL_CHUNKED  0x1
 #define UH_FL_HASBODY  0x2
@@ -4211,8 +4211,11 @@ static VALUE HttpParser_rssget(VALUE self)
 
 void Init_unicorn_http(void)
 {
+  static VALUE mark_ary;
   VALUE mUnicorn, cHttpParser;
 
+  mark_ary = rb_ary_new();
+  rb_global_variable(&mark_ary);
   mUnicorn = rb_define_module("Unicorn");
   cHttpParser = rb_define_class_under(mUnicorn, "HttpParser", rb_cObject);
   eHttpParserError =
@@ -4222,7 +4225,7 @@ void Init_unicorn_http(void)
   e414 = rb_define_class_under(mUnicorn, "RequestURITooLongError",
                                eHttpParserError);
 
-  init_globals();
+  init_globals(mark_ary);
   rb_define_alloc_func(cHttpParser, HttpParser_alloc);
   rb_define_method(cHttpParser, "initialize", HttpParser_init, 0);
   rb_define_method(cHttpParser, "clear", HttpParser_clear, 0);
@@ -4258,14 +4261,16 @@ void Init_unicorn_http(void)
 
   rb_define_singleton_method(cHttpParser, "max_header_len=", set_maxhdrlen, 1);
 
-  init_common_fields();
+  init_common_fields(mark_ary);
   SET_GLOBAL(g_http_host, "HOST");
   SET_GLOBAL(g_http_trailer, "TRAILER");
   SET_GLOBAL(g_http_transfer_encoding, "TRANSFER_ENCODING");
   SET_GLOBAL(g_content_length, "CONTENT_LENGTH");
   SET_GLOBAL(g_http_connection, "CONNECTION");
   id_set_backtrace = rb_intern("set_backtrace");
-  init_unicorn_httpdate();
+  init_unicorn_httpdate(mark_ary);
+
+  OBJ_FREEZE(mark_ary);
 
 #ifndef HAVE_RB_HASH_CLEAR
   id_clear = rb_intern("clear");

data/ext/unicorn_http/unicorn_http.rl

@@ -13,7 +13,7 @@
 #include "global_variables.h"
 #include "c_util.h"
 
-void init_unicorn_httpdate(void);
+void init_unicorn_httpdate(VALUE mark_ary);
 
 #define UH_FL_CHUNKED  0x1
 #define UH_FL_HASBODY  0x2
@@ -917,8 +917,11 @@ static VALUE HttpParser_rssget(VALUE self)
 
 void Init_unicorn_http(void)
 {
+  static VALUE mark_ary;
   VALUE mUnicorn, cHttpParser;
 
+  mark_ary = rb_ary_new();
+  rb_global_variable(&mark_ary);
   mUnicorn = rb_define_module("Unicorn");
   cHttpParser = rb_define_class_under(mUnicorn, "HttpParser", rb_cObject);
   eHttpParserError =
@@ -928,7 +931,7 @@ void Init_unicorn_http(void)
   e414 = rb_define_class_under(mUnicorn, "RequestURITooLongError",
                                eHttpParserError);
 
-  init_globals();
+  init_globals(mark_ary);
   rb_define_alloc_func(cHttpParser, HttpParser_alloc);
   rb_define_method(cHttpParser, "initialize", HttpParser_init, 0);
   rb_define_method(cHttpParser, "clear", HttpParser_clear, 0);
@@ -964,14 +967,16 @@ void Init_unicorn_http(void)
 
   rb_define_singleton_method(cHttpParser, "max_header_len=", set_maxhdrlen, 1);
 
-  init_common_fields();
+  init_common_fields(mark_ary);
   SET_GLOBAL(g_http_host, "HOST");
   SET_GLOBAL(g_http_trailer, "TRAILER");
   SET_GLOBAL(g_http_transfer_encoding, "TRANSFER_ENCODING");
   SET_GLOBAL(g_content_length, "CONTENT_LENGTH");
   SET_GLOBAL(g_http_connection, "CONNECTION");
   id_set_backtrace = rb_intern("set_backtrace");
-  init_unicorn_httpdate();
+  init_unicorn_httpdate(mark_ary);
+
+  OBJ_FREEZE(mark_ary);
 
 #ifndef HAVE_RB_HASH_CLEAR
   id_clear = rb_intern("clear");

data/lib/unicorn.rb

@@ -95,7 +95,7 @@ def self.builder(ru, op)
 
   # returns an array of strings representing TCP listen socket addresses
   # and Unix domain socket paths.  This is useful for use with
-  # Raindrops::Middleware under Linux: http://raindrops.bogomips.org/
+  # Raindrops::Middleware under Linux: https://bogomips.org/raindrops/
   def self.listener_names
     Unicorn::HttpServer::LISTENERS.map do |io|
       Unicorn::SocketHelper.sock_name(io)

data/lib/unicorn/configurator.rb

@@ -41,10 +41,22 @@ class Unicorn::Configurator
     :before_exec => lambda { |server|
         server.logger.info("forked child re-executing...")
       },
+    :after_worker_exit => lambda { |server, worker, status|
+        m = "reaped #{status.inspect} worker=#{worker.nr rescue 'unknown'}"
+        if status.success?
+          server.logger.info(m)
+        else
+          server.logger.error(m)
+        end
+      },
+    :after_worker_ready => lambda { |server, worker|
+        server.logger.info("worker=#{worker.nr} ready")
+      },
     :pid => nil,
+    :worker_exec => false,
     :preload_app => false,
     :check_client_connection => false,
-    :rewindable_input => true, # for Rack 2.x: (Rack::VERSION[0] <= 1),
+    :rewindable_input => true,
     :client_body_buffer_size => Unicorn::Const::MAX_BODY,
   }
   #:startdoc:
@@ -151,6 +163,38 @@ def after_fork(*args, &block)
     set_hook(:after_fork, block_given? ? block : args[0])
   end
 
+  # sets after_worker_exit hook to a given block.  This block will be called
+  # by the master process after a worker exits:
+  #
+  #  after_worker_exit do |server,worker,status|
+  #    # status is a Process::Status instance for the exited worker process
+  #    unless status.success?
+  #      server.logger.error("worker process failure: #{status.inspect}")
+  #    end
+  #  end
+  #
+  # after_worker_exit is only available in unicorn 5.3.0+
+  def after_worker_exit(*args, &block)
+    set_hook(:after_worker_exit, block_given? ? block : args[0], 3)
+  end
+
+  # sets after_worker_ready hook to a given block.  This block will be called
+  # by a worker process after it has been fully loaded, directly before it
+  # starts responding to requests:
+  #
+  #  after_worker_ready do |server,worker|
+  #    server.logger.info("worker #{worker.nr} ready, dropping privileges")
+  #    worker.user('username', 'groupname')
+  #  end
+  #
+  # Do not use Configurator#user if you rely on changing users in the
+  # after_worker_ready hook.
+  #
+  # after_worker_ready is only available in unicorn 5.3.0+
+  def after_worker_ready(*args, &block)
+    set_hook(:after_worker_ready, block_given? ? block : args[0])
+  end
+
   # sets before_fork got be a given Proc object.  This Proc
   # object will be called by the master process before forking
   # each worker.
@@ -200,6 +244,17 @@ def timeout(seconds)
     set[:timeout] = seconds > max ? max : seconds
   end
 
+  # Whether to exec in each worker process after forking.  This changes the
+  # memory layout of each worker process, which is a security feature designed
+  # to defeat possible address space discovery attacks.  Note that using
+  # worker_exec only makes sense if you are not preloading the application,
+  # and will result in higher memory usage.
+  #
+  # worker_exec is only available in unicorn 5.3.0+
+  def worker_exec(bool)
+    set_bool(:worker_exec, bool)
+  end
+
   # sets the current number of worker_processes to +nr+.  Each worker
   # process will serve exactly one client at a time.  You can
   # increment or decrement this value at runtime by sending SIGTTIN
@@ -466,13 +521,12 @@ def preload_app(bool)
   # Disabling rewindability can improve performance by lowering
   # I/O and memory usage for applications that accept uploads.
   # Keep in mind that the Rack 1.x spec requires
-  # \env[\"rack.input\"] to be rewindable, so this allows
-  # intentionally violating the current Rack 1.x spec.
+  # \env[\"rack.input\"] to be rewindable,
+  # but the Rack 2.x spec does not.
   #
-  # +rewindable_input+ defaults to +true+ when used with Rack 1.x for
-  # Rack conformance.  When Rack 2.x is finalized, this will most
-  # likely default to +false+ while still conforming to the newer
-  # (less demanding) spec.
+  # +rewindable_input+ defaults to +true+ for compatibility.
+  # Setting it to +false+ may be safe for applications and
+  # frameworks developed for Rack 2.x and later.
   def rewindable_input(bool)
     set_bool(:rewindable_input, bool)
   end
@@ -548,6 +602,10 @@ def working_directory(path)
   # This switch will occur after calling the after_fork hook, and only
   # if the Worker#user method is not called in the after_fork hook
   # +group+ is optional and will not change if unspecified.
+  #
+  # Do not use Configurator#user if you rely on changing users in the
+  # after_worker_ready hook.  Instead, you need to call Worker#user
+  # directly in after_worker_ready.
   def user(user, group = nil)
     # raises ArgumentError on invalid user/group
     Etc.getpwnam(user)

data/lib/unicorn/http_request.rb

@@ -2,6 +2,7 @@
 # :enddoc:
 # no stable API here
 require 'unicorn_http'
+require 'raindrops'
 
 # TODO: remove redundant names
 Unicorn.const_set(:HttpRequest, Unicorn::HttpParser)
@@ -24,12 +25,11 @@ class Unicorn::HttpParser
   NULL_IO = StringIO.new("")
 
   # :stopdoc:
-  # A frozen format for this is about 15% faster
-  # Drop these frozen strings when Ruby 2.2 becomes more prevalent,
-  # 2.2+ optimizes hash assignments when used with literal string keys
-  HTTP_RESPONSE_START = [ 'HTTP', '/1.1 ']
+  HTTP_RESPONSE_START = [ 'HTTP'.freeze, '/1.1 '.freeze ]
+  EMPTY_ARRAY = [].freeze
   @@input_class = Unicorn::TeeInput
   @@check_client_connection = false
+  @@tcpi_inspect_ok = Socket.const_defined?(:TCP_INFO)
 
   def self.input_class
     @@input_class
@@ -83,11 +83,7 @@ def read(socket)
       false until add_parse(socket.kgio_read!(16384))
     end
 
-    # detect if the socket is valid by writing a partial response:
-    if @@check_client_connection && headers?
-      self.response_start_sent = true
-      HTTP_RESPONSE_START.each { |c| socket.write(c) }
-    end
+    check_client_connection(socket) if @@check_client_connection
 
     e['rack.input'] = 0 == content_length ?
                       NULL_IO : @@input_class.new(socket, self)
@@ -108,4 +104,88 @@ def call
   def hijacked?
     env.include?('rack.hijack_io'.freeze)
   end
+
+  if Raindrops.const_defined?(:TCP_Info)
+    TCPI = Raindrops::TCP_Info.allocate
+
+    def check_client_connection(socket) # :nodoc:
+      if Unicorn::TCPClient === socket
+        # Raindrops::TCP_Info#get!, #state (reads struct tcp_info#tcpi_state)
+        raise Errno::EPIPE, "client closed connection".freeze,
+              EMPTY_ARRAY if closed_state?(TCPI.get!(socket).state)
+      else
+        write_http_header(socket)
+      end
+    end
+
+    if Raindrops.const_defined?(:TCP)
+      # raindrops 0.18.0+ supports FreeBSD + Linux using the same names
+      # Evaluate these hash lookups at load time so we can
+      # generate an opt_case_dispatch instruction
+      eval <<-EOS
+      def closed_state?(state) # :nodoc:
+        case state
+        when #{Raindrops::TCP[:ESTABLISHED]}
+          false
+        when #{Raindrops::TCP.values_at(
+              :CLOSE_WAIT, :TIME_WAIT, :CLOSE, :LAST_ACK, :CLOSING).join(',')}
+          true
+        else
+          false
+        end
+      end
+      EOS
+    else
+      # raindrops before 0.18 only supported TCP_INFO under Linux
+      def closed_state?(state) # :nodoc:
+        case state
+        when 1 # ESTABLISHED
+          false
+        when 8, 6, 7, 9, 11 # CLOSE_WAIT, TIME_WAIT, CLOSE, LAST_ACK, CLOSING
+          true
+        else
+          false
+        end
+      end
+    end
+  else
+
+    # Ruby 2.2+ can show struct tcp_info as a string Socket::Option#inspect.
+    # Not that efficient, but probably still better than doing unnecessary
+    # work after a client gives up.
+    def check_client_connection(socket) # :nodoc:
+      if Unicorn::TCPClient === socket && @@tcpi_inspect_ok
+        opt = socket.getsockopt(Socket::IPPROTO_TCP, Socket::TCP_INFO).inspect
+        if opt =~ /\bstate=(\S+)/
+          raise Errno::EPIPE, "client closed connection".freeze,
+                EMPTY_ARRAY if closed_state_str?($1)
+        else
+          @@tcpi_inspect_ok = false
+          write_http_header(socket)
+        end
+        opt.clear
+      else
+        write_http_header(socket)
+      end
+    end
+
+    def closed_state_str?(state)
+      case state
+      when 'ESTABLISHED'
+        false
+      # not a typo, ruby maps TCP_CLOSE (no 'D') to state=CLOSED (w/ 'D')
+      when 'CLOSE_WAIT', 'TIME_WAIT', 'CLOSED', 'LAST_ACK', 'CLOSING'
+        true
+      else
+        false
+      end
+    end
+  end
+
+  def write_http_header(socket) # :nodoc:
+    if headers?
+      self.response_start_sent = true
+      HTTP_RESPONSE_START.each { |c| socket.write(c) }
+    end
+  end
 end