unicorn 5.1.0.pre1 → 6.1.0

data/ext/unicorn_http/unicorn_http.rl

@@ -12,6 +12,7 @@
 #include "common_field_optimization.h"
 #include "global_variables.h"
 #include "c_util.h"
+#include "epollexclusive.h"
 
 void init_unicorn_httpdate(void);
 
@@ -26,6 +27,7 @@ void init_unicorn_httpdate(void);
 #define UH_FL_HASHEADER 0x100
 #define UH_FL_TO_CLEAR 0x200
 #define UH_FL_RESSTART 0x400 /* for check_client_connection */
+#define UH_FL_HIJACK 0x800
 
 /* all of these flags need to be set for keepalive to be supported */
 #define UH_FL_KEEPALIVE (UH_FL_KAVERSION | UH_FL_REQEOF | UH_FL_HASHEADER)
@@ -61,19 +63,8 @@ struct http_parser {
   } len;
 };
 
-static ID id_set_backtrace;
-
-#ifdef HAVE_RB_HASH_CLEAR /* Ruby >= 2.0 */
-#  define my_hash_clear(h) (void)rb_hash_clear(h)
-#else /* !HAVE_RB_HASH_CLEAR - Ruby <= 1.9.3 */
-
-static ID id_clear;
-
-static void my_hash_clear(VALUE h)
-{
-  rb_funcall(h, id_clear, 0);
-}
-#endif /* HAVE_RB_HASH_CLEAR */
+static ID id_set_backtrace, id_is_chunked_p;
+static VALUE cHttpParser;
 
 static void finalize_header(struct http_parser *hp);
 
@@ -219,6 +210,19 @@ static void write_cont_value(struct http_parser *hp,
   rb_str_buf_cat(hp->cont, vptr, end + 1);
 }
 
+static int is_chunked(VALUE v)
+{
+  /* common case first */
+  if (STR_CSTR_CASE_EQ(v, "chunked"))
+    return 1;
+
+  /*
+   * call Ruby function in unicorn/http_request.rb to deal with unlikely
+   * comma-delimited case
+   */
+  return rb_funcall(cHttpParser, id_is_chunked_p, 1, v) != Qfalse;
+}
+
 static void write_value(struct http_parser *hp,
                         const char *buffer, const char *p)
 {
@@ -245,7 +249,9 @@ static void write_value(struct http_parser *hp,
     f = uncommon_field(field, flen);
   } else if (f == g_http_connection) {
     hp_keepalive_connection(hp, v);
-  } else if (f == g_content_length) {
+  } else if (f == g_content_length && !HP_FL_TEST(hp, CHUNKED)) {
+    if (hp->len.content)
+      parser_raise(eHttpParserError, "Content-Length already set");
     hp->len.content = parse_length(RSTRING_PTR(v), RSTRING_LEN(v));
     if (hp->len.content < 0)
       parser_raise(eHttpParserError, "invalid Content-Length");
@@ -253,9 +259,30 @@ static void write_value(struct http_parser *hp,
       HP_FL_SET(hp, HASBODY);
     hp_invalid_if_trailer(hp);
   } else if (f == g_http_transfer_encoding) {
-    if (STR_CSTR_CASE_EQ(v, "chunked")) {
+    if (is_chunked(v)) {
+      if (HP_FL_TEST(hp, CHUNKED))
+        /*
+         * RFC 7230 3.3.1:
+         * A sender MUST NOT apply chunked more than once to a message body
+         * (i.e., chunking an already chunked message is not allowed).
+         */
+        parser_raise(eHttpParserError, "Transfer-Encoding double chunked");
+
       HP_FL_SET(hp, CHUNKED);
       HP_FL_SET(hp, HASBODY);
+
+      /* RFC 7230 3.3.3, 3: favor chunked if Content-Length exists */
+      hp->len.content = 0;
+    } else if (HP_FL_TEST(hp, CHUNKED)) {
+      /*
+       * RFC 7230 3.3.3, point 3 states:
+       * If a Transfer-Encoding header field is present in a request and
+       * the chunked transfer coding is not the final encoding, the
+       * message body length cannot be determined reliably; the server
+       * MUST respond with the 400 (Bad Request) status code and then
+       * close the connection.
+       */
+      parser_raise(eHttpParserError, "invalid Transfer-Encoding");
     }
     hp_invalid_if_trailer(hp);
   } else if (f == g_http_trailer) {
@@ -486,7 +513,7 @@ static void set_url_scheme(VALUE env, VALUE *server_port)
      * and X-Forwarded-Proto handling from this parser?  We've had it
      * forever and nobody has said anything against it, either.
      * Anyways, please send comments to our public mailing list:
-     * unicorn-public@bogomips.org (no HTML mail, no subscription necessary)
+     * unicorn-public@yhbt.net (no HTML mail, no subscription necessary)
      */
     scheme = rb_hash_aref(env, g_http_x_forwarded_ssl);
     if (!NIL_P(scheme) && STR_CSTR_EQ(scheme, "on")) {
@@ -607,8 +634,12 @@ static VALUE HttpParser_clear(VALUE self)
 {
   struct http_parser *hp = data_get(self);
 
+  /* we can't safely reuse .buf and .env if hijacked */
+  if (HP_FL_TEST(hp, HIJACK))
+    return HttpParser_init(self);
+
   http_parser_init(hp);
-  my_hash_clear(hp->env);
+  rb_hash_clear(hp->env);
 
   return self;
 }
@@ -813,6 +844,15 @@ static VALUE HttpParser_env(VALUE self)
   return data_get(self)->env;
 }
 
+static VALUE HttpParser_hijacked_bang(VALUE self)
+{
+  struct http_parser *hp = data_get(self);
+
+  HP_FL_SET(hp, HIJACK);
+
+  return self;
+}
+
 /**
  * call-seq:
  *    parser.filter_body(dst, src) => nil/src
@@ -917,7 +957,7 @@ static VALUE HttpParser_rssget(VALUE self)
 
 void Init_unicorn_http(void)
 {
-  VALUE mUnicorn, cHttpParser;
+  VALUE mUnicorn;
 
   mUnicorn = rb_define_module("Unicorn");
   cHttpParser = rb_define_class_under(mUnicorn, "HttpParser", rb_cObject);
@@ -928,6 +968,7 @@ void Init_unicorn_http(void)
   e414 = rb_define_class_under(mUnicorn, "RequestURITooLongError",
                                eHttpParserError);
 
+  id_uminus = rb_intern("-@");
   init_globals();
   rb_define_alloc_func(cHttpParser, HttpParser_alloc);
   rb_define_method(cHttpParser, "initialize", HttpParser_init, 0);
@@ -944,6 +985,7 @@ void Init_unicorn_http(void)
   rb_define_method(cHttpParser, "next?", HttpParser_next, 0);
   rb_define_method(cHttpParser, "buf", HttpParser_buf, 0);
   rb_define_method(cHttpParser, "env", HttpParser_env, 0);
+  rb_define_method(cHttpParser, "hijacked!", HttpParser_hijacked_bang, 0);
   rb_define_method(cHttpParser, "response_start_sent=", HttpParser_rssset, 1);
   rb_define_method(cHttpParser, "response_start_sent", HttpParser_rssget, 0);
 
@@ -976,5 +1018,8 @@ void Init_unicorn_http(void)
 #ifndef HAVE_RB_HASH_CLEAR
   id_clear = rb_intern("clear");
 #endif
+  id_is_chunked_p = rb_intern("is_chunked?");
+
+  init_epollexclusive(mUnicorn);
 }
 #undef SET_GLOBAL

data/ext/unicorn_http/unicorn_http_common.rl

@@ -4,7 +4,7 @@
 
 #### HTTP PROTOCOL GRAMMAR
 # line endings
-  CRLF = "\r\n";
+  CRLF = ("\r\n" | "\n");
 
 # character types
   CTL = (cntrl | 127);

data/lib/unicorn/configurator.rb

@@ -3,11 +3,11 @@ require 'logger'
 
 # Implements a simple DSL for configuring a unicorn server.
 #
-# See http://unicorn.bogomips.org/examples/unicorn.conf.rb and
-# http://unicorn.bogomips.org/examples/unicorn.conf.minimal.rb
+# See https://yhbt.net/unicorn/examples/unicorn.conf.rb and
+# https://yhbt.net/unicorn/examples/unicorn.conf.minimal.rb
 # example configuration files.  An example config file for use with
 # nginx is also available at
-# http://unicorn.bogomips.org/examples/nginx.conf
+# https://yhbt.net/unicorn/examples/nginx.conf
 #
 # See the link:/TUNING.html document for more information on tuning unicorn.
 class Unicorn::Configurator
@@ -41,10 +41,23 @@ class Unicorn::Configurator
     :before_exec => lambda { |server|
         server.logger.info("forked child re-executing...")
       },
+    :after_worker_exit => lambda { |server, worker, status|
+        m = "reaped #{status.inspect} worker=#{worker.nr rescue 'unknown'}"
+        if status.success?
+          server.logger.info(m)
+        else
+          server.logger.error(m)
+        end
+      },
+    :after_worker_ready => lambda { |server, worker|
+        server.logger.info("worker=#{worker.nr} ready")
+      },
     :pid => nil,
+    :early_hints => false,
+    :worker_exec => false,
     :preload_app => false,
     :check_client_connection => false,
-    :rewindable_input => true, # for Rack 2.x: (Rack::VERSION[0] <= 1),
+    :rewindable_input => true,
     :client_body_buffer_size => Unicorn::Const::MAX_BODY,
   }
   #:startdoc:
@@ -76,6 +89,9 @@ class Unicorn::Configurator
     RACKUP[:set_listener] and
       set[:listeners] << "#{RACKUP[:host]}:#{RACKUP[:port]}"
 
+    RACKUP[:no_default_middleware] and
+      set[:default_middleware] = false
+
     # unicorn_rails creates dirs here after working_directory is bound
     after_reload.call if after_reload
 
@@ -151,6 +167,38 @@ class Unicorn::Configurator
     set_hook(:after_fork, block_given? ? block : args[0])
   end
 
+  # sets after_worker_exit hook to a given block.  This block will be called
+  # by the master process after a worker exits:
+  #
+  #  after_worker_exit do |server,worker,status|
+  #    # status is a Process::Status instance for the exited worker process
+  #    unless status.success?
+  #      server.logger.error("worker process failure: #{status.inspect}")
+  #    end
+  #  end
+  #
+  # after_worker_exit is only available in unicorn 5.3.0+
+  def after_worker_exit(*args, &block)
+    set_hook(:after_worker_exit, block_given? ? block : args[0], 3)
+  end
+
+  # sets after_worker_ready hook to a given block.  This block will be called
+  # by a worker process after it has been fully loaded, directly before it
+  # starts responding to requests:
+  #
+  #  after_worker_ready do |server,worker|
+  #    server.logger.info("worker #{worker.nr} ready, dropping privileges")
+  #    worker.user('username', 'groupname')
+  #  end
+  #
+  # Do not use Configurator#user if you rely on changing users in the
+  # after_worker_ready hook.
+  #
+  # after_worker_ready is only available in unicorn 5.3.0+
+  def after_worker_ready(*args, &block)
+    set_hook(:after_worker_ready, block_given? ? block : args[0])
+  end
+
   # sets before_fork got be a given Proc object.  This Proc
   # object will be called by the master process before forking
   # each worker.
@@ -191,7 +239,7 @@ class Unicorn::Configurator
   #      server 192.168.0.9:8080 fail_timeout=0;
   #    }
   #
-  # See http://nginx.org/en/docs/http/ngx_http_upstream_module.html
+  # See https://nginx.org/en/docs/http/ngx_http_upstream_module.html
   # for more details on nginx upstream configuration.
   def timeout(seconds)
     set_int(:timeout, seconds, 3)
@@ -200,6 +248,17 @@ class Unicorn::Configurator
     set[:timeout] = seconds > max ? max : seconds
   end
 
+  # Whether to exec in each worker process after forking.  This changes the
+  # memory layout of each worker process, which is a security feature designed
+  # to defeat possible address space discovery attacks.  Note that using
+  # worker_exec only makes sense if you are not preloading the application,
+  # and will result in higher memory usage.
+  #
+  # worker_exec is only available in unicorn 5.3.0+
+  def worker_exec(bool)
+    set_bool(:worker_exec, bool)
+  end
+
   # sets the current number of worker_processes to +nr+.  Each worker
   # process will serve exactly one client at a time.  You can
   # increment or decrement this value at runtime by sending SIGTTIN
@@ -210,6 +269,23 @@ class Unicorn::Configurator
     set_int(:worker_processes, nr, 1)
   end
 
+  # sets whether to add default middleware in the development and
+  # deployment RACK_ENVs.
+  #
+  # default_middleware is only available in unicorn 5.5.0+
+  def default_middleware(bool)
+    set_bool(:default_middleware, bool)
+  end
+
+  # sets whether to enable the proposed early hints Rack API.
+  # If enabled, Rails 5.2+ will automatically send a 103 Early Hint
+  # for all the `javascript_include_tag` and `stylesheet_link_tag`
+  # in your response. See: https://api.rubyonrails.org/v5.2/classes/ActionDispatch/Request.html#method-i-send_early_hints
+  # See also https://tools.ietf.org/html/rfc8297
+  def early_hints(bool)
+    set_bool(:early_hints, bool)
+  end
+
   # sets listeners to the given +addresses+, replacing or augmenting the
   # current set.  This is for the global listener pool shared by all
   # worker processes.  For per-worker listeners, see the after_fork example
@@ -466,13 +542,12 @@ class Unicorn::Configurator
   # Disabling rewindability can improve performance by lowering
   # I/O and memory usage for applications that accept uploads.
   # Keep in mind that the Rack 1.x spec requires
-  # \env[\"rack.input\"] to be rewindable, so this allows
-  # intentionally violating the current Rack 1.x spec.
+  # \env[\"rack.input\"] to be rewindable,
+  # but the Rack 2.x spec does not.
   #
-  # +rewindable_input+ defaults to +true+ when used with Rack 1.x for
-  # Rack conformance.  When Rack 2.x is finalized, this will most
-  # likely default to +false+ while still conforming to the newer
-  # (less demanding) spec.
+  # +rewindable_input+ defaults to +true+ for compatibility.
+  # Setting it to +false+ may be safe for applications and
+  # frameworks developed for Rack 2.x and later.
   def rewindable_input(bool)
     set_bool(:rewindable_input, bool)
   end
@@ -533,7 +608,7 @@ class Unicorn::Configurator
     # just let chdir raise errors
     path = File.expand_path(path)
     if config_file &&
-       config_file[0] != ?/ &&
+       ! config_file.start_with?('/') &&
        ! File.readable?("#{path}/#{config_file}")
       raise ArgumentError,
             "config_file=#{config_file} would not be accessible in" \
@@ -548,6 +623,10 @@ class Unicorn::Configurator
   # This switch will occur after calling the after_fork hook, and only
   # if the Worker#user method is not called in the after_fork hook
   # +group+ is optional and will not change if unspecified.
+  #
+  # Do not use Configurator#user if you rely on changing users in the
+  # after_worker_ready hook.  Instead, you need to call Worker#user
+  # directly in after_worker_ready.
   def user(user, group = nil)
     # raises ArgumentError on invalid user/group
     Etc.getpwnam(user)

data/lib/unicorn/http_request.rb

@@ -24,12 +24,11 @@ class Unicorn::HttpParser
   NULL_IO = StringIO.new("")
 
   # :stopdoc:
-  # A frozen format for this is about 15% faster
-  # Drop these frozen strings when Ruby 2.2 becomes more prevalent,
-  # 2.2+ optimizes hash assignments when used with literal string keys
-  HTTP_RESPONSE_START = [ 'HTTP', '/1.1 ']
+  HTTP_RESPONSE_START = [ 'HTTP'.freeze, '/1.1 '.freeze ]
+  EMPTY_ARRAY = [].freeze
   @@input_class = Unicorn::TeeInput
   @@check_client_connection = false
+  @@tcpi_inspect_ok = Socket.const_defined?(:TCP_INFO)
 
   def self.input_class
     @@input_class
@@ -63,10 +62,9 @@ class Unicorn::HttpParser
   # This does minimal exception trapping and it is up to the caller
   # to handle any socket errors (e.g. user aborted upload).
   def read(socket)
-    clear
     e = env
 
-    # From http://www.ietf.org/rfc/rfc3875:
+    # From https://www.ietf.org/rfc/rfc3875:
     # "Script authors should be aware that the REMOTE_ADDR and
     #  REMOTE_HOST meta-variables (see sections 4.1.8 and 4.1.9)
     #  may not identify the ultimate source of the request.  They
@@ -83,11 +81,7 @@ class Unicorn::HttpParser
       false until add_parse(socket.kgio_read!(16384))
     end
 
-    # detect if the socket is valid by writing a partial response:
-    if @@check_client_connection && headers?
-      self.response_start_sent = true
-      HTTP_RESPONSE_START.each { |c| socket.write(c) }
-    end
+    check_client_connection(socket) if @@check_client_connection
 
     e['rack.input'] = 0 == content_length ?
                       NULL_IO : @@input_class.new(socket, self)
@@ -102,10 +96,106 @@ class Unicorn::HttpParser
   # for rack.hijack, we respond to this method so no extra allocation
   # of a proc object
   def call
+    hijacked!
     env['rack.hijack_io'] = env['unicorn.socket']
   end
 
   def hijacked?
     env.include?('rack.hijack_io'.freeze)
   end
+
+  if Raindrops.const_defined?(:TCP_Info)
+    TCPI = Raindrops::TCP_Info.allocate
+
+    def check_client_connection(socket) # :nodoc:
+      if Unicorn::TCPClient === socket
+        # Raindrops::TCP_Info#get!, #state (reads struct tcp_info#tcpi_state)
+        raise Errno::EPIPE, "client closed connection".freeze,
+              EMPTY_ARRAY if closed_state?(TCPI.get!(socket).state)
+      else
+        write_http_header(socket)
+      end
+    end
+
+    if Raindrops.const_defined?(:TCP)
+      # raindrops 0.18.0+ supports FreeBSD + Linux using the same names
+      # Evaluate these hash lookups at load time so we can
+      # generate an opt_case_dispatch instruction
+      eval <<-EOS
+      def closed_state?(state) # :nodoc:
+        case state
+        when #{Raindrops::TCP[:ESTABLISHED]}
+          false
+        when #{Raindrops::TCP.values_at(
+              :CLOSE_WAIT, :TIME_WAIT, :CLOSE, :LAST_ACK, :CLOSING).join(',')}
+          true
+        else
+          false
+        end
+      end
+      EOS
+    else
+      # raindrops before 0.18 only supported TCP_INFO under Linux
+      def closed_state?(state) # :nodoc:
+        case state
+        when 1 # ESTABLISHED
+          false
+        when 8, 6, 7, 9, 11 # CLOSE_WAIT, TIME_WAIT, CLOSE, LAST_ACK, CLOSING
+          true
+        else
+          false
+        end
+      end
+    end
+  else
+
+    # Ruby 2.2+ can show struct tcp_info as a string Socket::Option#inspect.
+    # Not that efficient, but probably still better than doing unnecessary
+    # work after a client gives up.
+    def check_client_connection(socket) # :nodoc:
+      if Unicorn::TCPClient === socket && @@tcpi_inspect_ok
+        opt = socket.getsockopt(Socket::IPPROTO_TCP, Socket::TCP_INFO).inspect
+        if opt =~ /\bstate=(\S+)/
+          raise Errno::EPIPE, "client closed connection".freeze,
+                EMPTY_ARRAY if closed_state_str?($1)
+        else
+          @@tcpi_inspect_ok = false
+          write_http_header(socket)
+        end
+        opt.clear
+      else
+        write_http_header(socket)
+      end
+    end
+
+    def closed_state_str?(state)
+      case state
+      when 'ESTABLISHED'
+        false
+      # not a typo, ruby maps TCP_CLOSE (no 'D') to state=CLOSED (w/ 'D')
+      when 'CLOSE_WAIT', 'TIME_WAIT', 'CLOSED', 'LAST_ACK', 'CLOSING'
+        true
+      else
+        false
+      end
+    end
+  end
+
+  def write_http_header(socket) # :nodoc:
+    if headers?
+      self.response_start_sent = true
+      HTTP_RESPONSE_START.each { |c| socket.write(c) }
+    end
+  end
+
+  # called by ext/unicorn_http/unicorn_http.rl via rb_funcall
+  def self.is_chunked?(v) # :nodoc:
+    vals = v.split(/[ \t]*,[ \t]*/).map!(&:downcase)
+    if vals.pop == 'chunked'.freeze
+      return true unless vals.include?('chunked'.freeze)
+      raise Unicorn::HttpParserError, 'double chunked', []
+    end
+    return false unless vals.include?('chunked'.freeze)
+    raise Unicorn::HttpParserError, 'chunked not last', []
+  end
 end

data/lib/unicorn/http_response.rb

@@ -21,13 +21,13 @@ module Unicorn::HttpResponse
 
   # writes the rack_response to socket as an HTTP response
   def http_response_write(socket, status, headers, body,
-                          response_start_sent=false)
+                          req = Unicorn::HttpRequest.new)
     hijack = nil
 
     if headers
       code = status.to_i
       msg = STATUS_CODES[code]
-      start = response_start_sent ? ''.freeze : 'HTTP/1.1 '.freeze
+      start = req.response_start_sent ? ''.freeze : 'HTTP/1.1 '.freeze
       buf = "#{start}#{msg ? %Q(#{code} #{msg}) : status}\r\n" \
             "Date: #{httpdate}\r\n" \
             "Connection: close\r\n"
@@ -52,6 +52,7 @@ module Unicorn::HttpResponse
     end
 
     if hijack
+      req.hijacked!
       hijack.call(socket)
     else
       body.each { |chunk| socket.write(chunk) }