unicorn 5.5.2 → 6.1.0

data/ext/unicorn_http/unicorn_http.rl

@@ -12,6 +12,7 @@
 #include "common_field_optimization.h"
 #include "global_variables.h"
 #include "c_util.h"
+#include "epollexclusive.h"
 
 void init_unicorn_httpdate(void);
 
@@ -62,19 +63,8 @@ struct http_parser {
   } len;
 };
 
-static ID id_set_backtrace;
-
-#ifdef HAVE_RB_HASH_CLEAR /* Ruby >= 2.0 */
-#  define my_hash_clear(h) (void)rb_hash_clear(h)
-#else /* !HAVE_RB_HASH_CLEAR - Ruby <= 1.9.3 */
-
-static ID id_clear;
-
-static void my_hash_clear(VALUE h)
-{
-  rb_funcall(h, id_clear, 0);
-}
-#endif /* HAVE_RB_HASH_CLEAR */
+static ID id_set_backtrace, id_is_chunked_p;
+static VALUE cHttpParser;
 
 static void finalize_header(struct http_parser *hp);
 
@@ -220,6 +210,19 @@ static void write_cont_value(struct http_parser *hp,
   rb_str_buf_cat(hp->cont, vptr, end + 1);
 }
 
+static int is_chunked(VALUE v)
+{
+  /* common case first */
+  if (STR_CSTR_CASE_EQ(v, "chunked"))
+    return 1;
+
+  /*
+   * call Ruby function in unicorn/http_request.rb to deal with unlikely
+   * comma-delimited case
+   */
+  return rb_funcall(cHttpParser, id_is_chunked_p, 1, v) != Qfalse;
+}
+
 static void write_value(struct http_parser *hp,
                         const char *buffer, const char *p)
 {
@@ -246,7 +249,9 @@ static void write_value(struct http_parser *hp,
     f = uncommon_field(field, flen);
   } else if (f == g_http_connection) {
     hp_keepalive_connection(hp, v);
-  } else if (f == g_content_length) {
+  } else if (f == g_content_length && !HP_FL_TEST(hp, CHUNKED)) {
+    if (hp->len.content)
+      parser_raise(eHttpParserError, "Content-Length already set");
     hp->len.content = parse_length(RSTRING_PTR(v), RSTRING_LEN(v));
     if (hp->len.content < 0)
       parser_raise(eHttpParserError, "invalid Content-Length");
@@ -254,9 +259,30 @@ static void write_value(struct http_parser *hp,
       HP_FL_SET(hp, HASBODY);
     hp_invalid_if_trailer(hp);
   } else if (f == g_http_transfer_encoding) {
-    if (STR_CSTR_CASE_EQ(v, "chunked")) {
+    if (is_chunked(v)) {
+      if (HP_FL_TEST(hp, CHUNKED))
+        /*
+         * RFC 7230 3.3.1:
+         * A sender MUST NOT apply chunked more than once to a message body
+         * (i.e., chunking an already chunked message is not allowed).
+         */
+        parser_raise(eHttpParserError, "Transfer-Encoding double chunked");
+
       HP_FL_SET(hp, CHUNKED);
       HP_FL_SET(hp, HASBODY);
+
+      /* RFC 7230 3.3.3, 3: favor chunked if Content-Length exists */
+      hp->len.content = 0;
+    } else if (HP_FL_TEST(hp, CHUNKED)) {
+      /*
+       * RFC 7230 3.3.3, point 3 states:
+       * If a Transfer-Encoding header field is present in a request and
+       * the chunked transfer coding is not the final encoding, the
+       * message body length cannot be determined reliably; the server
+       * MUST respond with the 400 (Bad Request) status code and then
+       * close the connection.
+       */
+      parser_raise(eHttpParserError, "invalid Transfer-Encoding");
     }
     hp_invalid_if_trailer(hp);
   } else if (f == g_http_trailer) {
@@ -487,7 +513,7 @@ static void set_url_scheme(VALUE env, VALUE *server_port)
      * and X-Forwarded-Proto handling from this parser?  We've had it
      * forever and nobody has said anything against it, either.
      * Anyways, please send comments to our public mailing list:
-     * unicorn-public@bogomips.org (no HTML mail, no subscription necessary)
+     * unicorn-public@yhbt.net (no HTML mail, no subscription necessary)
      */
     scheme = rb_hash_aref(env, g_http_x_forwarded_ssl);
     if (!NIL_P(scheme) && STR_CSTR_EQ(scheme, "on")) {
@@ -613,7 +639,7 @@ static VALUE HttpParser_clear(VALUE self)
     return HttpParser_init(self);
 
   http_parser_init(hp);
-  my_hash_clear(hp->env);
+  rb_hash_clear(hp->env);
 
   return self;
 }
@@ -931,7 +957,7 @@ static VALUE HttpParser_rssget(VALUE self)
 
 void Init_unicorn_http(void)
 {
-  VALUE mUnicorn, cHttpParser;
+  VALUE mUnicorn;
 
   mUnicorn = rb_define_module("Unicorn");
   cHttpParser = rb_define_class_under(mUnicorn, "HttpParser", rb_cObject);
@@ -942,6 +968,7 @@ void Init_unicorn_http(void)
   e414 = rb_define_class_under(mUnicorn, "RequestURITooLongError",
                                eHttpParserError);
 
+  id_uminus = rb_intern("-@");
   init_globals();
   rb_define_alloc_func(cHttpParser, HttpParser_alloc);
   rb_define_method(cHttpParser, "initialize", HttpParser_init, 0);
@@ -991,5 +1018,8 @@ void Init_unicorn_http(void)
 #ifndef HAVE_RB_HASH_CLEAR
   id_clear = rb_intern("clear");
 #endif
+  id_is_chunked_p = rb_intern("is_chunked?");
+
+  init_epollexclusive(mUnicorn);
 }
 #undef SET_GLOBAL

data/lib/unicorn/configurator.rb

@@ -3,11 +3,11 @@ require 'logger'
 
 # Implements a simple DSL for configuring a unicorn server.
 #
-# See https://bogomips.org/unicorn/examples/unicorn.conf.rb and
-# https://bogomips.org/unicorn/examples/unicorn.conf.minimal.rb
+# See https://yhbt.net/unicorn/examples/unicorn.conf.rb and
+# https://yhbt.net/unicorn/examples/unicorn.conf.minimal.rb
 # example configuration files.  An example config file for use with
 # nginx is also available at
-# https://bogomips.org/unicorn/examples/nginx.conf
+# https://yhbt.net/unicorn/examples/nginx.conf
 #
 # See the link:/TUNING.html document for more information on tuning unicorn.
 class Unicorn::Configurator
@@ -53,6 +53,7 @@ class Unicorn::Configurator
         server.logger.info("worker=#{worker.nr} ready")
       },
     :pid => nil,
+    :early_hints => false,
     :worker_exec => false,
     :preload_app => false,
     :check_client_connection => false,
@@ -276,6 +277,15 @@ class Unicorn::Configurator
     set_bool(:default_middleware, bool)
   end
 
+  # sets whether to enable the proposed early hints Rack API.
+  # If enabled, Rails 5.2+ will automatically send a 103 Early Hint
+  # for all the `javascript_include_tag` and `stylesheet_link_tag`
+  # in your response. See: https://api.rubyonrails.org/v5.2/classes/ActionDispatch/Request.html#method-i-send_early_hints
+  # See also https://tools.ietf.org/html/rfc8297
+  def early_hints(bool)
+    set_bool(:early_hints, bool)
+  end
+
   # sets listeners to the given +addresses+, replacing or augmenting the
   # current set.  This is for the global listener pool shared by all
   # worker processes.  For per-worker listeners, see the after_fork example

data/lib/unicorn/http_request.rb

@@ -62,7 +62,6 @@ class Unicorn::HttpParser
   # This does minimal exception trapping and it is up to the caller
   # to handle any socket errors (e.g. user aborted upload).
   def read(socket)
-    clear
     e = env
 
     # From https://www.ietf.org/rfc/rfc3875:
@@ -188,4 +187,15 @@ class Unicorn::HttpParser
       HTTP_RESPONSE_START.each { |c| socket.write(c) }
     end
   end
+
+  # called by ext/unicorn_http/unicorn_http.rl via rb_funcall
+  def self.is_chunked?(v) # :nodoc:
+    vals = v.split(/[ \t]*,[ \t]*/).map!(&:downcase)
+    if vals.pop == 'chunked'.freeze
+      return true unless vals.include?('chunked'.freeze)
+      raise Unicorn::HttpParserError, 'double chunked', []
+    end
+    return false unless vals.include?('chunked'.freeze)
+    raise Unicorn::HttpParserError, 'chunked not last', []
+  end
 end

data/lib/unicorn/http_server.rb

@@ -6,7 +6,7 @@
 # forked worker children.
 #
 # Users do not need to know the internals of this class, but reading the
-# {source}[https://bogomips.org/unicorn.git/tree/lib/unicorn/http_server.rb]
+# {source}[https://yhbt.net/unicorn.git/tree/lib/unicorn/http_server.rb]
 # is education for programmers wishing to learn how unicorn works.
 # See Unicorn::Configurator for information on how to configure unicorn.
 class Unicorn::HttpServer
@@ -15,7 +15,7 @@ class Unicorn::HttpServer
                 :before_fork, :after_fork, :before_exec,
                 :listener_opts, :preload_app,
                 :orig_app, :config, :ready_pipe, :user,
-                :default_middleware
+                :default_middleware, :early_hints
   attr_writer   :after_worker_exit, :after_worker_ready, :worker_exec
 
   attr_reader :pid, :logger
@@ -69,7 +69,6 @@ class Unicorn::HttpServer
   # incoming requests on the socket.
   def initialize(app, options = {})
     @app = app
-    @request = Unicorn::HttpRequest.new
     @reexec_pid = 0
     @default_middleware = true
     options = options.dup
@@ -588,10 +587,29 @@ class Unicorn::HttpServer
   rescue
   end
 
+  def e103_response_write(client, headers)
+    response = if @request.response_start_sent
+      "103 Early Hints\r\n"
+    else
+      "HTTP/1.1 103 Early Hints\r\n"
+    end
+
+    headers.each_pair do |k, vs|
+      next if !vs || vs.empty?
+      values = vs.to_s.split("\n".freeze)
+      values.each do |v|
+        response << "#{k}: #{v}\r\n"
+      end
+    end
+    response << "\r\n".freeze
+    response << "HTTP/1.1 ".freeze if @request.response_start_sent
+    client.write(response)
+  end
+
   def e100_response_write(client, env)
     # We use String#freeze to avoid allocations under Ruby 2.1+
     # Not many users hit this code path, so it's better to reduce the
-    # constant table sizes even for 1.9.3-2.0 users who'll hit extra
+    # constant table sizes even for Ruby 2.0 users who'll hit extra
     # allocations here.
     client.write(@request.response_start_sent ?
                  "100 Continue\r\n\r\nHTTP/1.1 ".freeze :
@@ -602,7 +620,18 @@ class Unicorn::HttpServer
   # once a client is accepted, it is processed in its entirety here
   # in 3 easy steps: read request, call app, write app response
   def process_client(client)
-    status, headers, body = @app.call(env = @request.read(client))
+    @request = Unicorn::HttpRequest.new
+    env = @request.read(client)
+
+    if early_hints
+      env["rack.early_hints"] = lambda do |headers|
+        e103_response_write(client, headers)
+      end
+    end
+
+    env["rack.after_reply"] = []
+
+    status, headers, body = @app.call(env)
 
     begin
       return if @request.hijacked?
@@ -624,6 +653,8 @@ class Unicorn::HttpServer
     end
   rescue => e
     handle_error(client, e)
+  ensure
+    env["rack.after_reply"].each(&:call) if env
   end
 
   def nuke_listeners!(readers)
@@ -654,7 +685,6 @@ class Unicorn::HttpServer
     LISTENERS.each { |sock| sock.close_on_exec = true }
 
     worker.user(*user) if user.kind_of?(Array) && ! worker.switched
-    self.timeout /= 2.0 # halve it for select()
     @config = nil
     build_app! unless preload_app
     @after_fork = @listener_opts = @orig_app = nil
@@ -668,58 +698,55 @@ class Unicorn::HttpServer
     logger.info "worker=#{worker_nr} reopening logs..."
     Unicorn::Util.reopen_logs
     logger.info "worker=#{worker_nr} done reopening logs"
+    false
   rescue => e
     logger.error(e) rescue nil
     exit!(77) # EX_NOPERM in sysexits.h
   end
 
+  def prep_readers(readers)
+    wtr = Unicorn::Waiter.prep_readers(readers)
+    @timeout *= 500 # to milliseconds for epoll, but halved
+    wtr
+  rescue
+    require_relative 'select_waiter'
+    @timeout /= 2.0 # halved for IO.select
+    Unicorn::SelectWaiter.new
+  end
+
   # runs inside each forked worker, this sits around and waits
   # for connections and doesn't die until the parent dies (or is
   # given a INT, QUIT, or TERM signal)
   def worker_loop(worker)
-    ppid = @master_pid
     readers = init_worker_process(worker)
-    nr = 0 # this becomes negative if we need to reopen logs
+    waiter = prep_readers(readers)
+    reopen = false
 
     # this only works immediately if the master sent us the signal
     # (which is the normal case)
-    trap(:USR1) { nr = -65536 }
+    trap(:USR1) { reopen = true }
 
     ready = readers.dup
     @after_worker_ready.call(self, worker)
 
     begin
-      nr < 0 and reopen_worker_logs(worker.nr)
-      nr = 0
+      reopen = reopen_worker_logs(worker.nr) if reopen
       worker.tick = time_now.to_i
-      tmp = ready.dup
-      while sock = tmp.shift
+      while sock = ready.shift
         # Unicorn::Worker#kgio_tryaccept is not like accept(2) at all,
         # but that will return false
         if client = sock.kgio_tryaccept
           process_client(client)
-          nr += 1
           worker.tick = time_now.to_i
         end
-        break if nr < 0
+        break if reopen
       end
 
-      # make the following bet: if we accepted clients this round,
-      # we're probably reasonably busy, so avoid calling select()
-      # and do a speculative non-blocking accept() on ready listeners
-      # before we sleep again in select().
-      unless nr == 0
-        tmp = ready.dup
-        redo
-      end
-
-      ppid == Process.ppid or return
-
-      # timeout used so we can detect parent death:
+      # timeout so we can .tick and keep parent from SIGKILL-ing us
       worker.tick = time_now.to_i
-      ret = IO.select(readers, nil, nil, @timeout) and ready = ret[0]
+      waiter.get_readers(ready, readers, @timeout)
     rescue => e
-      redo if nr < 0 && readers[0]
+      redo if reopen && readers[0]
       Unicorn.log_error(@logger, "listen loop error", e) if readers[0]
     end while readers[0]
   end

data/lib/unicorn/oob_gc.rb

@@ -43,8 +43,8 @@
 #     use Unicorn::OobGC, 2, %r{\A/(?:expensive/foo|more_expensive/foo)}
 #
 # Feedback from users of early implementations of this module:
-# * https://bogomips.org/unicorn-public/0BFC98E9-072B-47EE-9A70-05478C20141B@lukemelia.com/
-# * https://bogomips.org/unicorn-public/AANLkTilUbgdyDv9W1bi-s_W6kq9sOhWfmuYkKLoKGOLj@mail.gmail.com/
+# * https://yhbt.net/unicorn-public/0BFC98E9-072B-47EE-9A70-05478C20141B@lukemelia.com/
+# * https://yhbt.net/unicorn-public/AANLkTilUbgdyDv9W1bi-s_W6kq9sOhWfmuYkKLoKGOLj@mail.gmail.com/
 
 module Unicorn::OobGC
 
@@ -60,7 +60,6 @@ module Unicorn::OobGC
     self.const_set :OOBGC_INTERVAL, interval
     ObjectSpace.each_object(Unicorn::HttpServer) do |s|
       s.extend(self)
-      self.const_set :OOBGC_ENV, s.instance_variable_get(:@request).env
     end
     app # pretend to be Rack middleware since it was in the past
   end
@@ -68,9 +67,10 @@ module Unicorn::OobGC
   #:stopdoc:
   def process_client(client)
     super(client) # Unicorn::HttpServer#process_client
-    if OOBGC_PATH =~ OOBGC_ENV['PATH_INFO'] && ((@@nr -= 1) <= 0)
+    env = instance_variable_get(:@request).env
+    if OOBGC_PATH =~ env['PATH_INFO'] && ((@@nr -= 1) <= 0)
       @@nr = OOBGC_INTERVAL
-      OOBGC_ENV.clear
+      env.clear
       disabled = GC.enable
       GC.start
       GC.disable if disabled

data/lib/unicorn/select_waiter.rb

@@ -0,0 +1,6 @@
+# fallback for non-Linux and Linux <4.5 systems w/o EPOLLEXCLUSIVE
+class Unicorn::SelectWaiter # :nodoc:
+  def get_readers(ready, readers, timeout) # :nodoc:
+    ret = IO.select(readers, nil, nil, timeout) and ready.replace(ret[0])
+  end
+end

data/lib/unicorn/version.rb

@@ -1 +1 @@
-Unicorn::Const::UNICORN_VERSION = '5.5.2'
+Unicorn::Const::UNICORN_VERSION = '6.1.0'

data/lib/unicorn.rb

@@ -96,7 +96,7 @@ module Unicorn
 
   # returns an array of strings representing TCP listen socket addresses
   # and Unix domain socket paths.  This is useful for use with
-  # Raindrops::Middleware under Linux: https://bogomips.org/raindrops/
+  # Raindrops::Middleware under Linux: https://yhbt.net/raindrops/
   def self.listener_names
     Unicorn::HttpServer::LISTENERS.map do |io|
       Unicorn::SocketHelper.sock_name(io)
@@ -114,8 +114,6 @@ module Unicorn
 
   def self.pipe # :nodoc:
     Kgio::Pipe.new.each do |io|
-      io.close_on_exec = true  # remove this when we only support Ruby >= 2.0
-
       # shrink pipes to minimize impact on /proc/sys/fs/pipe-user-pages-soft
       # limits.
       if defined?(F_SETPIPE_SZ)

data/man/man1/unicorn.1

@@ -154,7 +154,7 @@ TTIN \- increment the number of worker processes by one
 .IP \[bu] 2
 TTOU \- decrement the number of worker processes by one
 .PP
-See the SIGNALS (https://bogomips.org/unicorn/SIGNALS.html) document for
+See the SIGNALS (https://yhbt.net/unicorn/SIGNALS.html) document for
 full description of all signals used by Unicorn.
 .SH RACK ENVIRONMENT
 .PP
@@ -204,11 +204,11 @@ the unicorn config file.
 \f[I]Rack::Builder\f[] ri/RDoc
 .IP \[bu] 2
 \f[I]Unicorn::Configurator\f[] ri/RDoc
-.UR https://bogomips.org/unicorn/Unicorn/Configurator.html
+.UR https://yhbt.net/unicorn/Unicorn/Configurator.html
 .UE
 .IP \[bu] 2
 unicorn RDoc
-.UR https://bogomips.org/unicorn/
+.UR https://yhbt.net/unicorn/
 .UE
 .IP \[bu] 2
 Rack RDoc
@@ -219,4 +219,4 @@ Rackup HowTo
 .UR https://github.com/rack/rack/wiki/(tutorial)-rackup-howto
 .UE
 .SH AUTHORS
-The Unicorn Community <unicorn-public@bogomips.org>.
+The Unicorn Community <unicorn-public@yhbt.net>.

data/man/man1/unicorn_rails.1

@@ -180,7 +180,7 @@ TTIN \- increment the number of worker processes by one
 .IP \[bu] 2
 TTOU \- decrement the number of worker processes by one
 .PP
-See the SIGNALS (https://bogomips.org/unicorn/SIGNALS.html) document for
+See the SIGNALS (https://yhbt.net/unicorn/SIGNALS.html) document for
 full description of all signals used by Unicorn.
 .SH SEE ALSO
 .IP \[bu] 2
@@ -189,11 +189,11 @@ unicorn(1)
 \f[I]Rack::Builder\f[] ri/RDoc
 .IP \[bu] 2
 \f[I]Unicorn::Configurator\f[] ri/RDoc
-.UR https://bogomips.org/unicorn/Unicorn/Configurator.html
+.UR https://yhbt.net/unicorn/Unicorn/Configurator.html
 .UE
 .IP \[bu] 2
 unicorn RDoc
-.UR https://bogomips.org/unicorn/
+.UR https://yhbt.net/unicorn/
 .UE
 .IP \[bu] 2
 Rack RDoc
@@ -204,4 +204,4 @@ Rackup HowTo
 .UR https://github.com/rack/rack/wiki/(tutorial)-rackup-howto
 .UE
 .SH AUTHORS
-The Unicorn Community <unicorn-public@bogomips.org>.
+The Unicorn Community <unicorn-public@yhbt.net>.

data/t/GNUmakefile

@@ -1,74 +1,5 @@
-# we can run tests in parallel with GNU make
+# there used to be more, here, but we stopped relying on recursive make
 all::
+	$(MAKE) -C .. test-integration
 
-pid := $(shell echo $$PPID)
-
-RUBY = ruby
-RAKE = rake
--include ../local.mk
-ifeq ($(RUBY_VERSION),)
-  RUBY_VERSION := $(shell $(RUBY) -e 'puts RUBY_VERSION')
-endif
-
-ifeq ($(RUBY_VERSION),)
-  $(error unable to detect RUBY_VERSION)
-endif
-
-RUBY_ENGINE := $(shell $(RUBY) -e 'puts((RUBY_ENGINE rescue "ruby"))')
-export RUBY_ENGINE
-
-MYLIBS := $(RUBYLIB)
-
-T = $(wildcard t[0-9][0-9][0-9][0-9]-*.sh)
-
-all:: $(T)
-
-# can't rely on "set -o pipefail" since we don't require bash or ksh93 :<
-t_pfx = trash/$@-$(RUBY_ENGINE)-$(RUBY_VERSION)
-TEST_OPTS =
-# TRACER = strace -f -o $(t_pfx).strace -s 100000
-# TRACER = /usr/bin/time -o $(t_pfx).time
-
-ifdef V
-  ifeq ($(V),2)
-    TEST_OPTS += --trace
-  else
-    TEST_OPTS += --verbose
-  endif
-endif
-
-random_blob:
-	dd if=/dev/urandom bs=1M count=30 of=$@.$(pid)
-	mv $@.$(pid) $@
-
-$(T): random_blob
-
-dependencies := socat curl
-deps := $(addprefix .dep+,$(dependencies))
-$(deps): dep_bin = $(lastword $(subst +, ,$@))
-$(deps):
-	@which $(dep_bin) > $@.$(pid) 2>/dev/null || :
-	@test -s $@.$(pid) || \
-	  { echo >&2 "E '$(dep_bin)' not found in PATH=$(PATH)"; exit 1; }
-	@mv $@.$(pid) $@
-dep: $(deps)
-
-test_prefix := $(CURDIR)/../test/$(RUBY_ENGINE)-$(RUBY_VERSION)
-$(test_prefix)/.stamp:
-	$(MAKE) -C .. test-install
-
-$(T): export RUBY := $(RUBY)
-$(T): export RAKE := $(RAKE)
-$(T): export PATH := $(test_prefix)/bin:$(PATH)
-$(T): export RUBYLIB := $(test_prefix)/lib:$(MYLIBS)
-$(T): dep $(test_prefix)/.stamp trash/.gitignore
-	$(TRACER) $(SHELL) $(SH_TEST_OPTS) $@ $(TEST_OPTS)
-
-trash/.gitignore:
-	mkdir -p $(@D)
-	echo '*' > $@
-
-clean:
-	$(RM) -r trash/*
-
-.PHONY: $(T) clean
+.PHONY: all

data/t/README

@@ -10,7 +10,7 @@ comfortable writing integration tests with.
 
 == Requirements
 
-* {Ruby 1.9.3+}[https://www.ruby-lang.org/en/] (duh!)
+* {Ruby 2.0.0+}[https://www.ruby-lang.org/en/] (duh!)
 * {GNU make}[https://www.gnu.org/software/make/]
 * {socat}[http://www.dest-unreach.org/socat/]
 * {curl}[https://curl.haxx.se/]

data/t/test-lib.sh

@@ -94,7 +94,8 @@ check_stderr () {
 	set +u
 	_r_err=${1-${r_err}}
 	set -u
-	if grep -v $T $_r_err | grep -i Error
+	if grep -v $T $_r_err | grep -i Error | \
+		grep -v NameError.*Unicorn::Waiter
 	then
 		die "Errors found in $_r_err"
 	elif grep SIGKILL $_r_err