unicorn 5.4.0 → 5.6.0

data/ext/unicorn_http/unicorn_http.rl

@@ -13,7 +13,7 @@
 #include "global_variables.h"
 #include "c_util.h"
 
-void init_unicorn_httpdate(VALUE mark_ary);
+void init_unicorn_httpdate(void);
 
 #define UH_FL_CHUNKED  0x1
 #define UH_FL_HASBODY  0x2
@@ -62,7 +62,8 @@ struct http_parser {
   } len;
 };
 
-static ID id_set_backtrace;
+static ID id_set_backtrace, id_is_chunked_p;
+static VALUE cHttpParser;
 
 #ifdef HAVE_RB_HASH_CLEAR /* Ruby >= 2.0 */
 #  define my_hash_clear(h) (void)rb_hash_clear(h)
@@ -220,6 +221,19 @@ static void write_cont_value(struct http_parser *hp,
   rb_str_buf_cat(hp->cont, vptr, end + 1);
 }
 
+static int is_chunked(VALUE v)
+{
+  /* common case first */
+  if (STR_CSTR_CASE_EQ(v, "chunked"))
+    return 1;
+
+  /*
+   * call Ruby function in unicorn/http_request.rb to deal with unlikely
+   * comma-delimited case
+   */
+  return rb_funcall(cHttpParser, id_is_chunked_p, 1, v) != Qfalse;
+}
+
 static void write_value(struct http_parser *hp,
                         const char *buffer, const char *p)
 {
@@ -246,7 +260,9 @@ static void write_value(struct http_parser *hp,
     f = uncommon_field(field, flen);
   } else if (f == g_http_connection) {
     hp_keepalive_connection(hp, v);
-  } else if (f == g_content_length) {
+  } else if (f == g_content_length && !HP_FL_TEST(hp, CHUNKED)) {
+    if (hp->len.content)
+      parser_raise(eHttpParserError, "Content-Length already set");
     hp->len.content = parse_length(RSTRING_PTR(v), RSTRING_LEN(v));
     if (hp->len.content < 0)
       parser_raise(eHttpParserError, "invalid Content-Length");
@@ -254,9 +270,30 @@ static void write_value(struct http_parser *hp,
       HP_FL_SET(hp, HASBODY);
     hp_invalid_if_trailer(hp);
   } else if (f == g_http_transfer_encoding) {
-    if (STR_CSTR_CASE_EQ(v, "chunked")) {
+    if (is_chunked(v)) {
+      if (HP_FL_TEST(hp, CHUNKED))
+        /*
+         * RFC 7230 3.3.1:
+         * A sender MUST NOT apply chunked more than once to a message body
+         * (i.e., chunking an already chunked message is not allowed).
+         */
+        parser_raise(eHttpParserError, "Transfer-Encoding double chunked");
+
       HP_FL_SET(hp, CHUNKED);
       HP_FL_SET(hp, HASBODY);
+
+      /* RFC 7230 3.3.3, 3: favor chunked if Content-Length exists */
+      hp->len.content = 0;
+    } else if (HP_FL_TEST(hp, CHUNKED)) {
+      /*
+       * RFC 7230 3.3.3, point 3 states:
+       * If a Transfer-Encoding header field is present in a request and
+       * the chunked transfer coding is not the final encoding, the
+       * message body length cannot be determined reliably; the server
+       * MUST respond with the 400 (Bad Request) status code and then
+       * close the connection.
+       */
+      parser_raise(eHttpParserError, "invalid Transfer-Encoding");
     }
     hp_invalid_if_trailer(hp);
   } else if (f == g_http_trailer) {
@@ -487,7 +524,7 @@ static void set_url_scheme(VALUE env, VALUE *server_port)
      * and X-Forwarded-Proto handling from this parser?  We've had it
      * forever and nobody has said anything against it, either.
      * Anyways, please send comments to our public mailing list:
-     * unicorn-public@bogomips.org (no HTML mail, no subscription necessary)
+     * unicorn-public@yhbt.net (no HTML mail, no subscription necessary)
      */
     scheme = rb_hash_aref(env, g_http_x_forwarded_ssl);
     if (!NIL_P(scheme) && STR_CSTR_EQ(scheme, "on")) {
@@ -931,11 +968,8 @@ static VALUE HttpParser_rssget(VALUE self)
 
 void Init_unicorn_http(void)
 {
-  static VALUE mark_ary;
-  VALUE mUnicorn, cHttpParser;
+  VALUE mUnicorn;
 
-  mark_ary = rb_ary_new();
-  rb_global_variable(&mark_ary);
   mUnicorn = rb_define_module("Unicorn");
   cHttpParser = rb_define_class_under(mUnicorn, "HttpParser", rb_cObject);
   eHttpParserError =
@@ -945,7 +979,7 @@ void Init_unicorn_http(void)
   e414 = rb_define_class_under(mUnicorn, "RequestURITooLongError",
                                eHttpParserError);
 
-  init_globals(mark_ary);
+  init_globals();
   rb_define_alloc_func(cHttpParser, HttpParser_alloc);
   rb_define_method(cHttpParser, "initialize", HttpParser_init, 0);
   rb_define_method(cHttpParser, "clear", HttpParser_clear, 0);
@@ -982,19 +1016,18 @@ void Init_unicorn_http(void)
 
   rb_define_singleton_method(cHttpParser, "max_header_len=", set_maxhdrlen, 1);
 
-  init_common_fields(mark_ary);
+  init_common_fields();
   SET_GLOBAL(g_http_host, "HOST");
   SET_GLOBAL(g_http_trailer, "TRAILER");
   SET_GLOBAL(g_http_transfer_encoding, "TRANSFER_ENCODING");
   SET_GLOBAL(g_content_length, "CONTENT_LENGTH");
   SET_GLOBAL(g_http_connection, "CONNECTION");
   id_set_backtrace = rb_intern("set_backtrace");
-  init_unicorn_httpdate(mark_ary);
-
-  OBJ_FREEZE(mark_ary);
+  init_unicorn_httpdate();
 
 #ifndef HAVE_RB_HASH_CLEAR
   id_clear = rb_intern("clear");
 #endif
+  id_is_chunked_p = rb_intern("is_chunked?");
 }
 #undef SET_GLOBAL

data/lib/unicorn/configurator.rb

@@ -3,11 +3,11 @@ require 'logger'
 
 # Implements a simple DSL for configuring a unicorn server.
 #
-# See https://bogomips.org/unicorn/examples/unicorn.conf.rb and
-# https://bogomips.org/unicorn/examples/unicorn.conf.minimal.rb
+# See https://yhbt.net/unicorn/examples/unicorn.conf.rb and
+# https://yhbt.net/unicorn/examples/unicorn.conf.minimal.rb
 # example configuration files.  An example config file for use with
 # nginx is also available at
-# https://bogomips.org/unicorn/examples/nginx.conf
+# https://yhbt.net/unicorn/examples/nginx.conf
 #
 # See the link:/TUNING.html document for more information on tuning unicorn.
 class Unicorn::Configurator
@@ -53,6 +53,7 @@ class Unicorn::Configurator
         server.logger.info("worker=#{worker.nr} ready")
       },
     :pid => nil,
+    :early_hints => false,
     :worker_exec => false,
     :preload_app => false,
     :check_client_connection => false,
@@ -88,6 +89,9 @@ class Unicorn::Configurator
     RACKUP[:set_listener] and
       set[:listeners] << "#{RACKUP[:host]}:#{RACKUP[:port]}"
 
+    RACKUP[:no_default_middleware] and
+      set[:default_middleware] = false
+
     # unicorn_rails creates dirs here after working_directory is bound
     after_reload.call if after_reload
 
@@ -235,7 +239,7 @@ class Unicorn::Configurator
   #      server 192.168.0.9:8080 fail_timeout=0;
   #    }
   #
-  # See http://nginx.org/en/docs/http/ngx_http_upstream_module.html
+  # See https://nginx.org/en/docs/http/ngx_http_upstream_module.html
   # for more details on nginx upstream configuration.
   def timeout(seconds)
     set_int(:timeout, seconds, 3)
@@ -265,6 +269,23 @@ class Unicorn::Configurator
     set_int(:worker_processes, nr, 1)
   end
 
+  # sets whether to add default middleware in the development and
+  # deployment RACK_ENVs.
+  #
+  # default_middleware is only available in unicorn 5.5.0+
+  def default_middleware(bool)
+    set_bool(:default_middleware, bool)
+  end
+
+  # sets whether to enable the proposed early hints Rack API.
+  # If enabled, Rails 5.2+ will automatically send a 103 Early Hint
+  # for all the `javascript_include_tag` and `stylesheet_link_tag`
+  # in your response. See: https://api.rubyonrails.org/v5.2/classes/ActionDispatch/Request.html#method-i-send_early_hints
+  # See also https://tools.ietf.org/html/rfc8297
+  def early_hints(bool)
+    set_bool(:early_hints, bool)
+  end
+
   # sets listeners to the given +addresses+, replacing or augmenting the
   # current set.  This is for the global listener pool shared by all
   # worker processes.  For per-worker listeners, see the after_fork example

data/lib/unicorn/http_request.rb

@@ -2,7 +2,6 @@
 # :enddoc:
 # no stable API here
 require 'unicorn_http'
-require 'raindrops'
 
 # TODO: remove redundant names
 Unicorn.const_set(:HttpRequest, Unicorn::HttpParser)
@@ -66,7 +65,7 @@ class Unicorn::HttpParser
     clear
     e = env
 
-    # From http://www.ietf.org/rfc/rfc3875:
+    # From https://www.ietf.org/rfc/rfc3875:
     # "Script authors should be aware that the REMOTE_ADDR and
     #  REMOTE_HOST meta-variables (see sections 4.1.8 and 4.1.9)
     #  may not identify the ultimate source of the request.  They
@@ -189,4 +188,15 @@ class Unicorn::HttpParser
       HTTP_RESPONSE_START.each { |c| socket.write(c) }
     end
   end
+
+  # called by ext/unicorn_http/unicorn_http.rl via rb_funcall
+  def self.is_chunked?(v) # :nodoc:
+    vals = v.split(/[ \t]*,[ \t]*/).map!(&:downcase)
+    if vals.pop == 'chunked'.freeze
+      return true unless vals.include?('chunked'.freeze)
+      raise Unicorn::HttpParserError, 'double chunked', []
+    end
+    return false unless vals.include?('chunked'.freeze)
+    raise Unicorn::HttpParserError, 'chunked not last', []
+  end
 end

data/lib/unicorn/http_server.rb

@@ -6,7 +6,7 @@
 # forked worker children.
 #
 # Users do not need to know the internals of this class, but reading the
-# {source}[https://bogomips.org/unicorn.git/tree/lib/unicorn/http_server.rb]
+# {source}[https://yhbt.net/unicorn.git/tree/lib/unicorn/http_server.rb]
 # is education for programmers wishing to learn how unicorn works.
 # See Unicorn::Configurator for information on how to configure unicorn.
 class Unicorn::HttpServer
@@ -14,7 +14,8 @@ class Unicorn::HttpServer
   attr_accessor :app, :timeout, :worker_processes,
                 :before_fork, :after_fork, :before_exec,
                 :listener_opts, :preload_app,
-                :orig_app, :config, :ready_pipe, :user
+                :orig_app, :config, :ready_pipe, :user,
+                :default_middleware, :early_hints
   attr_writer   :after_worker_exit, :after_worker_ready, :worker_exec
 
   attr_reader :pid, :logger
@@ -70,6 +71,7 @@ class Unicorn::HttpServer
     @app = app
     @request = Unicorn::HttpRequest.new
     @reexec_pid = 0
+    @default_middleware = true
     options = options.dup
     @ready_pipe = options.delete(:ready_pipe)
     @init_listeners = options[:listeners] ? options[:listeners].dup : []
@@ -82,7 +84,7 @@ class Unicorn::HttpServer
     # * The master process never closes or reinitializes this once
     # initialized.  Signal handlers in the master process will write to
     # it to wake up the master from IO.select in exactly the same manner
-    # djb describes in http://cr.yp.to/docs/selfpipe.html
+    # djb describes in https://cr.yp.to/docs/selfpipe.html
     #
     # * The workers immediately close the pipe they inherit.  See the
     # Unicorn::Worker class for the pipe workers use.
@@ -380,7 +382,7 @@ class Unicorn::HttpServer
 
   # wait for a signal hander to wake us up and then consume the pipe
   def master_sleep(sec)
-    @self_pipe[0].kgio_wait_readable(sec) or return
+    @self_pipe[0].wait(sec) or return
     # 11 bytes is the maximum string length which can be embedded within
     # the Ruby itself and not require a separate malloc (on 32-bit MRI 1.9+).
     # Most reads are only one byte here and uncommon, so it's not worth a
@@ -520,9 +522,6 @@ class Unicorn::HttpServer
     Unicorn::Configurator::RACKUP.clear
     @ready_pipe = @init_listeners = @before_exec = @before_fork = nil
 
-    # http://blade.nagaokaut.ac.jp/cgi-bin/scat.rb/ruby/ruby-core/36450
-    srand # remove in unicorn 6
-
     # The OpenSSL PRNG is seeded with only the pid, and apps with frequently
     # dying workers can recycle pids
     OpenSSL::Random.seed(rand.to_s) if defined?(OpenSSL::Random)
@@ -553,9 +552,9 @@ class Unicorn::HttpServer
       @workers[pid] = worker
       worker.atfork_parent
     end
-    rescue => e
-      @logger.error(e) rescue nil
-      exit!
+  rescue => e
+    @logger.error(e) rescue nil
+    exit!
   end
 
   def maintain_worker_count
@@ -586,7 +585,26 @@ class Unicorn::HttpServer
       client.kgio_trywrite(err_response(code, @request.response_start_sent))
     end
     client.close
-    rescue
+  rescue
+  end
+
+  def e103_response_write(client, headers)
+    response = if @request.response_start_sent
+      "103 Early Hints\r\n"
+    else
+      "HTTP/1.1 103 Early Hints\r\n"
+    end
+
+    headers.each_pair do |k, vs|
+      next if !vs || vs.empty?
+      values = vs.to_s.split("\n".freeze)
+      values.each do |v|
+        response << "#{k}: #{v}\r\n"
+      end
+    end
+    response << "\r\n".freeze
+    response << "HTTP/1.1 ".freeze if @request.response_start_sent
+    client.write(response)
   end
 
   def e100_response_write(client, env)
@@ -603,7 +621,15 @@ class Unicorn::HttpServer
   # once a client is accepted, it is processed in its entirety here
   # in 3 easy steps: read request, call app, write app response
   def process_client(client)
-    status, headers, body = @app.call(env = @request.read(client))
+    env = @request.read(client)
+
+    if early_hints
+      env["rack.early_hints"] = lambda do |headers|
+        e103_response_write(client, headers)
+      end
+    end
+
+    status, headers, body = @app.call(env)
 
     begin
       return if @request.hijacked?
@@ -669,9 +695,9 @@ class Unicorn::HttpServer
     logger.info "worker=#{worker_nr} reopening logs..."
     Unicorn::Util.reopen_logs
     logger.info "worker=#{worker_nr} done reopening logs"
-    rescue => e
-      logger.error(e) rescue nil
-      exit!(77) # EX_NOPERM in sysexits.h
+  rescue => e
+    logger.error(e) rescue nil
+    exit!(77) # EX_NOPERM in sysexits.h
   end
 
   # runs inside each forked worker, this sits around and waits
@@ -687,6 +713,7 @@ class Unicorn::HttpServer
     trap(:USR1) { nr = -65536 }
 
     ready = readers.dup
+    nr_listeners = readers.size
     @after_worker_ready.call(self, worker)
 
     begin
@@ -709,7 +736,7 @@ class Unicorn::HttpServer
       # we're probably reasonably busy, so avoid calling select()
       # and do a speculative non-blocking accept() on ready listeners
       # before we sleep again in select().
-      unless nr == 0
+      if nr == nr_listeners
         tmp = ready.dup
         redo
       end
@@ -757,11 +784,11 @@ class Unicorn::HttpServer
     wpid <= 0 and return
     Process.kill(0, wpid)
     wpid
-    rescue Errno::EPERM
-      logger.info "pid=#{path} possibly stale, got EPERM signalling PID:#{wpid}"
-      nil
-    rescue Errno::ESRCH, Errno::ENOENT
-      # don't unlink stale pid files, racy without non-portable locking...
+  rescue Errno::EPERM
+    logger.info "pid=#{path} possibly stale, got EPERM signalling PID:#{wpid}"
+    nil
+  rescue Errno::ESRCH, Errno::ENOENT
+    # don't unlink stale pid files, racy without non-portable locking...
   end
 
   def load_config!
@@ -787,12 +814,12 @@ class Unicorn::HttpServer
   end
 
   def build_app!
-    if app.respond_to?(:arity) && app.arity == 0
+    if app.respond_to?(:arity) && (app.arity == 0 || app.arity == 2)
       if defined?(Gem) && Gem.respond_to?(:refresh)
         logger.info "Refreshing Gem list"
         Gem.refresh
       end
-      self.app = app.call
+      self.app = app.arity == 0 ? app.call : app.call(nil, self)
     end
   end
 

data/lib/unicorn/launcher.rb

@@ -31,7 +31,7 @@ module Unicorn::Launcher
       #  \_ parent  - exits immediately ASAP
       #      \_ unicorn master - writes to pipe when ready
 
-      rd, wr = IO.pipe
+      rd, wr = Unicorn.pipe
       grandparent = $$
       if fork
         wr.close # grandparent does not write

data/lib/unicorn/oob_gc.rb

@@ -43,8 +43,8 @@
 #     use Unicorn::OobGC, 2, %r{\A/(?:expensive/foo|more_expensive/foo)}
 #
 # Feedback from users of early implementations of this module:
-# * https://bogomips.org/unicorn-public/0BFC98E9-072B-47EE-9A70-05478C20141B@lukemelia.com/
-# * https://bogomips.org/unicorn-public/AANLkTilUbgdyDv9W1bi-s_W6kq9sOhWfmuYkKLoKGOLj@mail.gmail.com/
+# * https://yhbt.net/unicorn-public/0BFC98E9-072B-47EE-9A70-05478C20141B@lukemelia.com/
+# * https://yhbt.net/unicorn-public/AANLkTilUbgdyDv9W1bi-s_W6kq9sOhWfmuYkKLoKGOLj@mail.gmail.com/
 
 module Unicorn::OobGC
 

data/lib/unicorn/socket_helper.rb

@@ -83,6 +83,7 @@ module Unicorn
         rescue => e
           logger.error("#{sock_name(sock)} " \
                        "failed to set accept_filter=#{name} (#{e.inspect})")
+          logger.error("perhaps accf_http(9) needs to be loaded".freeze)
         end if arg != got
       end
     end
@@ -100,8 +101,8 @@ module Unicorn
         log_buffer_sizes(sock, " after: ")
       end
       sock.listen(opt[:backlog])
-      rescue => e
-        Unicorn.log_error(logger, "#{sock_name(sock)} #{opt.inspect}", e)
+    rescue => e
+      Unicorn.log_error(logger, "#{sock_name(sock)} #{opt.inspect}", e)
     end
 
     def log_buffer_sizes(sock, pfx = '')

data/lib/unicorn/tmpio.rb

@@ -11,12 +11,18 @@ class Unicorn::TmpIO < File
   # immediately, switched to binary mode, and userspace output
   # buffering is disabled
   def self.new
+    path = nil
+
+    # workaround File#path being tainted:
+    # https://bugs.ruby-lang.org/issues/14485
     fp = begin
-      super("#{Dir::tmpdir}/#{rand}", RDWR|CREAT|EXCL, 0600)
+      path = "#{Dir::tmpdir}/#{rand}"
+      super(path, RDWR|CREAT|EXCL, 0600)
     rescue Errno::EEXIST
       retry
     end
-    unlink(fp.path)
+
+    unlink(path)
     fp.binmode
     fp.sync = true
     fp

data/lib/unicorn/util.rb

@@ -11,8 +11,8 @@ module Unicorn::Util # :nodoc:
       fp.stat.file? &&
       fp.sync &&
       (fp.fcntl(Fcntl::F_GETFL) & append_flags) == append_flags
-    rescue IOError, Errno::EBADF
-      false
+  rescue IOError, Errno::EBADF
+    false
   end
 
   def self.chown_logs(uid, gid)
@@ -64,7 +64,7 @@ module Unicorn::Util # :nodoc:
           fp.reopen(fp.path, "a")
         else
           # We should not need this workaround, Ruby can be fixed:
-          #    http://bugs.ruby-lang.org/issues/9036
+          #    https://bugs.ruby-lang.org/issues/9036
           # MRI will not call call fclose(3) or freopen(3) here
           # since there's no associated std{in,out,err} FILE * pointer
           # This should atomically use dup3(2) (or dup2(2)) syscall

data/lib/unicorn/version.rb

@@ -1 +1 @@
-Unicorn::Const::UNICORN_VERSION = '5.4.0'
+Unicorn::Const::UNICORN_VERSION = '5.6.0'

data/lib/unicorn/worker.rb

@@ -122,6 +122,11 @@ class Unicorn::Worker
   # the +after_fork+ hook after any privileged functions need to be
   # run (e.g. to set per-worker CPU affinity, niceness, etc)
   #
+  # +group+ can be specified as a string, or as an array of two
+  # strings.  If an array of two strings is given, the first string
+  # is used as the primary group of the process, and the second is
+  # used as the group of the log files.
+  #
   # Any and all errors raised within this method will be propagated
   # directly back to the caller (usually the +after_fork+ hook.
   # These errors commonly include ArgumentError for specifying an
@@ -134,8 +139,17 @@ class Unicorn::Worker
     # insufficient because modern systems have fine-grained
     # capabilities.  Let the caller handle any and all errors.
     uid = Etc.getpwnam(user).uid
-    gid = Etc.getgrnam(group).gid if group
-    Unicorn::Util.chown_logs(uid, gid)
+
+    if group
+      if group.is_a?(Array)
+        group, log_group = group
+        log_gid = Etc.getgrnam(log_group).gid
+      end
+      gid = Etc.getgrnam(group).gid
+      log_gid ||= gid
+    end
+
+    Unicorn::Util.chown_logs(uid, log_gid)
     if gid && Process.egid != gid
       Process.initgroups(user, gid)
       Process::GID.change_privilege(gid)

data/lib/unicorn.rb

@@ -2,6 +2,8 @@
 require 'etc'
 require 'stringio'
 require 'kgio'
+require 'raindrops'
+require 'io/wait'
 
 begin
   require 'rack'
@@ -43,12 +45,8 @@ module Unicorn
       abort "rack and Rack::Builder must be available for processing #{ru}"
     end
 
-    # Op is going to get cleared before the returned lambda is called, so
-    # save this value so that it's still there when we need it:
-    no_default_middleware = op[:no_default_middleware]
-
     # always called after config file parsing, may be called after forking
-    lambda do ||
+    lambda do |_, server|
       inner_app = case ru
       when /\.ru$/
         raw = File.read(ru)
@@ -64,7 +62,7 @@ module Unicorn
         pp({ :inner_app => inner_app })
       end
 
-      return inner_app if no_default_middleware
+      return inner_app unless server.default_middleware
 
       middleware = { # order matters
         ContentLength: nil,
@@ -98,7 +96,7 @@ module Unicorn
 
   # returns an array of strings representing TCP listen socket addresses
   # and Unix domain socket paths.  This is useful for use with
-  # Raindrops::Middleware under Linux: https://bogomips.org/raindrops/
+  # Raindrops::Middleware under Linux: https://yhbt.net/raindrops/
   def self.listener_names
     Unicorn::HttpServer::LISTENERS.map do |io|
       Unicorn::SocketHelper.sock_name(io)
@@ -112,9 +110,25 @@ module Unicorn
     exc.backtrace.each { |line| logger.error(line) }
   end
 
-  # remove this when we only support Ruby >= 2.0
+  F_SETPIPE_SZ = 1031 if RUBY_PLATFORM =~ /linux/
+
   def self.pipe # :nodoc:
-    Kgio::Pipe.new.each { |io| io.close_on_exec = true }
+    Kgio::Pipe.new.each do |io|
+      io.close_on_exec = true  # remove this when we only support Ruby >= 2.0
+
+      # shrink pipes to minimize impact on /proc/sys/fs/pipe-user-pages-soft
+      # limits.
+      if defined?(F_SETPIPE_SZ)
+        begin
+          io.fcntl(F_SETPIPE_SZ, Raindrops::PAGE_SIZE)
+        rescue Errno::EINVAL
+          # old kernel
+        rescue Errno::EPERM
+          # resizes fail if Linux is close to the pipe limit for the user
+          # or if the user does not have permissions to resize
+        end
+      end
+    end
   end
   # :startdoc:
 end