unicorn 5.5.3 → 5.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: def232de6614b9d1e5394c6a8ebde552a8c5d063ddc04e946478d7dbc3220526
-  data.tar.gz: f1dc8153f477d7c81b9be38c0e6b184863d8eb5d6d35779f637f9259e76a1272
+  metadata.gz: d3166eb1eac760d39cac35a6fc46332d6a6a2ee75bdf594118fddb3c6a959f6d
+  data.tar.gz: 3e18599fef201efdd79c67eb126aab35787c196054313a727d6bb184f830587e
 SHA512:
-  metadata.gz: 330cd4ce75af35523f2e6bbdee24794f42222d1c14ffabeccc37bfb5d8cd5e4774d940e6528e6fa72b2890c07acc02c4f917458459718794c1709d6cf0aa0493
-  data.tar.gz: ecbc2df84176fa9a99522b06dc127fd5d19e78d782f8f027f8f74f25dca841295299ef310698fb7c42f2cee7bb9b3531dbd2331c328dd8e1d358666df6b2a706
+  metadata.gz: 356f0c951cee2e69de8e49169f2c9560e6fead9fe759bd0af5a51097888a23794eb17175ff5393f07d1fadec5678215b494b688cdeeadb8a95c37201f3808a92
+  data.tar.gz: ffa1f8781231ac559157310d2d235340522c6c1cccf69be3d87d432659d2a2a7a39d744a84306c1f4292a3f6b36141cc1bb80497f1a9047902ec0d592f77871b

data/GIT-VERSION-GEN

@@ -1,5 +1,5 @@
 #!/usr/bin/env ruby
-DEF_VER = "v5.5.3"
+DEF_VER = "v5.8.0"
 CONSTANT = "Unicorn::Const::UNICORN_VERSION"
 RVF = "lib/unicorn/version.rb"
 GVF = "GIT-VERSION-FILE"

data/GNUmakefile

@@ -10,6 +10,7 @@ RAGEL = ragel
 RSYNC = rsync
 OLDDOC = olddoc
 RDOC = rdoc
+INSTALL = install
 
 GIT-VERSION-FILE: .FORCE-GIT-VERSION-FILE
 	@./GIT-VERSION-GEN
@@ -25,7 +26,38 @@ endif
 
 RUBY_ENGINE := $(shell $(RUBY) -e 'puts((RUBY_ENGINE rescue "ruby"))')
 
-MYLIBS = $(RUBYLIB)
+# we should never package more than one ext to avoid DSO proliferation:
+# https://udrepper.livejournal.com/8790.html
+ext := $(firstword $(wildcard ext/*))
+
+ragel: $(ext)/unicorn_http.c
+
+rl_files := $(wildcard $(ext)/*.rl)
+ragel: $(ext)/unicorn_http.c
+$(ext)/unicorn_http.c: $(rl_files)
+	cd $(@D) && $(RAGEL) unicorn_http.rl -C $(RLFLAGS) -o $(@F)
+ext_pfx := test/$(RUBY_ENGINE)-$(RUBY_VERSION)
+tmp_bin := $(ext_pfx)/bin
+ext_h := $(wildcard $(ext)/*/*.h $(ext)/*.h)
+ext_src := $(sort $(wildcard $(ext)/*.c) $(ext_h) $(ext)/unicorn_http.c)
+ext_pfx_src := $(addprefix $(ext_pfx)/,$(ext_src))
+ext_dir := $(ext_pfx)/$(ext)
+$(ext)/extconf.rb: $(wildcard $(ext)/*.h)
+	@>>$@
+$(ext_dir) $(tmp_bin) man/man1 doc/man1 pkg t/trash:
+	@mkdir -p $@
+$(ext_pfx)/$(ext)/%: $(ext)/% | $(ext_dir)
+	$(INSTALL) -m 644 $< $@
+$(ext_pfx)/$(ext)/Makefile: $(ext)/extconf.rb $(ext_h) | $(ext_dir)
+	$(RM) -f $(@D)/*.o
+	cd $(@D) && $(RUBY) $(CURDIR)/$(ext)/extconf.rb $(EXTCONF_ARGS)
+ext_sfx := _ext.$(DLEXT)
+ext_dl := $(ext_pfx)/$(ext)/$(notdir $(ext)_ext.$(DLEXT))
+$(ext_dl): $(ext_src) $(ext_pfx_src) $(ext_pfx)/$(ext)/Makefile
+	$(MAKE) -C $(@D)
+lib := $(CURDIR)/lib:$(CURDIR)/$(ext_pfx)/$(ext)
+http build: $(ext_dl)
+$(ext_pfx)/$(ext)/unicorn_http.c: ext/unicorn_http/unicorn_http.c
 
 # dunno how to implement this as concisely in Ruby, and hell, I love awk
 awk_slow := awk '/def test_/{print FILENAME"--"$$2".n"}' 2>/dev/null
@@ -37,44 +69,21 @@ T := $(filter-out $(slow_tests), $(wildcard test/*/test*.rb))
 T_n := $(shell $(awk_slow) $(slow_tests))
 T_log := $(subst .rb,$(log_suffix),$(T))
 T_n_log := $(subst .n,$(log_suffix),$(T_n))
-test_prefix = $(CURDIR)/test/$(RUBY_ENGINE)-$(RUBY_VERSION)
 
-ext := ext/unicorn_http
-c_files := $(ext)/unicorn_http.c $(ext)/httpdate.c $(wildcard $(ext)/*.h)
-rl_files := $(wildcard $(ext)/*.rl)
 base_bins := unicorn unicorn_rails
 bins := $(addprefix bin/, $(base_bins))
 man1_rdoc := $(addsuffix _1, $(base_bins))
 man1_bins := $(addsuffix .1, $(base_bins))
 man1_paths := $(addprefix man/man1/, $(man1_bins))
-rb_files := $(bins) $(shell find lib ext -type f -name '*.rb')
-inst_deps := $(c_files) $(rb_files) GNUmakefile test/test_helper.rb
-
-ragel: $(ext)/unicorn_http.c
-$(ext)/unicorn_http.c: $(rl_files)
-	cd $(@D) && $(RAGEL) unicorn_http.rl -C $(RLFLAGS) -o $(@F)
-$(ext)/Makefile: $(ext)/extconf.rb $(c_files)
-	cd $(@D) && $(RUBY) extconf.rb
-$(ext)/unicorn_http.$(DLEXT): $(ext)/Makefile
-	$(MAKE) -C $(@D)
-http: $(ext)/unicorn_http.$(DLEXT)
+tmp_bins = $(addprefix $(tmp_bin)/, unicorn unicorn_rails)
+pid := $(shell echo $$PPID)
 
-# only used for tests
-http-install: $(ext)/unicorn_http.$(DLEXT)
-	install -m644 $< lib/
+$(tmp_bin)/%: bin/% | $(tmp_bin)
+	$(INSTALL) -m 755 $< $@.$(pid)
+	$(MRI) -i -p -e '$$_.gsub!(%r{^#!.*$$},"#!$(ruby_bin)")' $@.$(pid)
+	mv $@.$(pid) $@
 
-test-install: $(test_prefix)/.stamp
-$(test_prefix)/.stamp: $(inst_deps)
-	mkdir -p $(test_prefix)/.ccache
-	tar cf - $(inst_deps) GIT-VERSION-GEN | \
-	  (cd $(test_prefix) && tar xf -)
-	$(MAKE) -C $(test_prefix) clean
-	$(MAKE) -C $(test_prefix) http-install shebang RUBY="$(RUBY)"
-	> $@
-
-# this is only intended to be run within $(test_prefix)
-shebang: $(bins)
-	$(MRI) -i -p -e '$$_.gsub!(%r{^#!.*$$},"#!$(ruby_bin)")' $^
+bins: $(tmp_bins)
 
 t_log := $(T_log) $(T_n_log)
 test: $(T) $(T_n)
@@ -83,15 +92,54 @@ test: $(T) $(T_n)
 
 test-exec: $(wildcard test/exec/test_*.rb)
 test-unit: $(wildcard test/unit/test_*.rb)
-$(slow_tests): $(test_prefix)/.stamp
+$(slow_tests): $(ext_dl)
 	@$(MAKE) $(shell $(awk_slow) $@)
 
 # ensure we can require just the HTTP parser without the rest of unicorn
-test-require: $(ext)/unicorn_http.$(DLEXT)
-	$(RUBY) --disable-gems -I$(ext) -runicorn_http -e Unicorn
+test-require: $(ext_dl)
+	$(RUBY) --disable-gems -I$(ext_pfx)/$(ext) -runicorn_http -e Unicorn
+
+test_prereq := $(tmp_bins) $(ext_dl)
+
+SH_TEST_OPTS =
+ifdef V
+  ifeq ($(V),2)
+    SH_TEST_OPTS += --trace
+  else
+    SH_TEST_OPTS += --verbose
+  endif
+endif
 
-test-integration: $(test_prefix)/.stamp
-	$(MAKE) -C t
+# do we trust Ruby behavior to be stable? some tests are
+# (mostly) POSIX sh (not bash or ksh93, so no "set -o pipefail"
+# TRACER = strace -f -o $(t_pfx).strace -s 100000
+# TRACER = /usr/bin/time -o $(t_pfx).time
+t_pfx = trash/$@-$(RUBY_ENGINE)-$(RUBY_VERSION)
+T_sh = $(wildcard t/t[0-9][0-9][0-9][0-9]-*.sh)
+$(T_sh): export RUBY := $(RUBY)
+$(T_sh): export PATH := $(CURDIR)/$(tmp_bin):$(PATH)
+$(T_sh): export RUBYLIB := $(lib):$(RUBYLIB)
+$(T_sh): dep $(test_prereq) t/random_blob t/trash/.gitignore
+	cd t && $(TRACER) $(SHELL) $(SH_TEST_OPTS) $(@F) $(TEST_OPTS)
+
+t/trash/.gitignore : | t/trash
+	echo '*' >$@
+
+dependencies := socat curl
+deps := $(addprefix t/.dep+,$(dependencies))
+$(deps): dep_bin = $(lastword $(subst +, ,$@))
+$(deps):
+	@which $(dep_bin) > $@.$(pid) 2>/dev/null || :
+	@test -s $@.$(pid) || \
+	  { echo >&2 "E '$(dep_bin)' not found in PATH=$(PATH)"; exit 1; }
+	@mv $@.$(pid) $@
+dep: $(deps)
+
+t/random_blob:
+	dd if=/dev/urandom bs=1M count=30 of=$@.$(pid)
+	mv $@.$(pid) $@
+
+test-integration: $(T_sh)
 
 check: test-require test test-integration
 test-all: check
@@ -122,16 +170,16 @@ run_test = $(quiet_pre) \
 
 %.n: arg = $(subst .n,,$(subst --, -n ,$@))
 %.n: t = $(subst .n,$(log_suffix),$@)
-%.n: export PATH := $(test_prefix)/bin:$(PATH)
-%.n: export RUBYLIB := $(test_prefix)/lib:$(MYLIBS)
-%.n: $(test_prefix)/.stamp
+%.n: export PATH := $(CURDIR)/$(tmp_bin):$(PATH)
+%.n: export RUBYLIB := $(lib):$(RUBYLIB)
+%.n: $(test_prereq)
 	$(run_test)
 
 $(T): arg = $@
 $(T): t = $(subst .rb,$(log_suffix),$@)
-$(T): export PATH := $(test_prefix)/bin:$(PATH)
-$(T): export RUBYLIB := $(test_prefix)/lib:$(MYLIBS)
-$(T): $(test_prefix)/.stamp
+$(T): export PATH := $(CURDIR)/$(tmp_bin):$(PATH)
+$(T): export RUBYLIB := $(lib):$(RUBYLIB)
+$(T): $(test_prereq)
 	$(run_test)
 
 install: $(bins) $(ext)/unicorn_http.c
@@ -152,18 +200,16 @@ clean:
 	-$(MAKE) -C $(ext) clean
 	$(RM) $(ext)/Makefile
 	$(RM) $(setup_rb_files) $(t_log)
-	$(RM) -r $(test_prefix) man
-	$(RM) $(man1) $(html1)
+	$(RM) -r $(ext_pfx) man t/trash
+	$(RM) $(html1)
 
 man1 := $(addprefix Documentation/, unicorn.1 unicorn_rails.1)
 html1 := $(addsuffix .html, $(man1))
-man :
-	mkdir -p man/man1
-	install -m 644 $(man1) man/man1
+man : $(man1) | man/man1
+	$(INSTALL) -m 644 $(man1) man/man1
 
-html : $(html1)
-	mkdir -p doc/man1
-	install -m 644 $(html1) doc/man1
+html : $(html1) | doc/man1
+	$(INSTALL) -m 644 $(html1) doc/man1
 
 %.1.html: %.1
 	$(OLDDOC) man2html -o $@ ./$<
@@ -187,10 +233,10 @@ doc: .document $(ext)/unicorn_http.c man html .olddoc.yml $(PLACEHOLDERS)
 	$(OLDDOC) prepare
 	$(RDOC) -f dark216
 	$(OLDDOC) merge
-	install -m644 COPYING doc/COPYING
-	install -m644 NEWS.atom.xml doc/NEWS.atom.xml
-	install -m644 $(shell LC_ALL=C grep '^[A-Z]' .document) doc/
-	install -m644 $(man1_paths) doc/
+	$(INSTALL) -m 644 COPYING doc/COPYING
+	$(INSTALL) -m 644 NEWS.atom.xml doc/NEWS.atom.xml
+	$(INSTALL) -m 644 $(shell LC_ALL=C grep '^[A-Z]' .document) doc/
+	$(INSTALL) -m 644 $(man1_paths) doc/
 	tar cf - $$(git ls-files examples/) | (cd doc && tar xf -)
 
 # publishes docs to https://yhbt.net/unicorn/
@@ -199,7 +245,8 @@ publish_doc:
 	$(MAKE) doc
 	$(MAKE) doc_gz
 	chmod 644 $$(find doc -type f)
-	$(RSYNC) -av doc/ yhbt.net:/srv/yhbt/unicorn/
+	$(RSYNC) -av doc/ yhbt.net:/srv/yhbt/unicorn/ \
+		--exclude index.html* --exclude created.rid*
 	git ls-files | xargs touch
 
 # Create gzip variants of the same timestamp as the original so nginx
@@ -231,9 +278,8 @@ gem: $(pkggem)
 install-gem: $(pkggem)
 	gem install --local $(CURDIR)/$<
 
-$(pkggem): .manifest fix-perms
+$(pkggem): .manifest fix-perms | pkg
 	gem build $(rfpackage).gemspec
-	mkdir -p pkg
 	mv $(@F) $@
 
 $(pkgtgz): distdir = $(basename $@)
@@ -264,5 +310,4 @@ check-warnings:
 	  do $(RUBY) --disable-gems -d -W2 -c \
 	  $$i; done) | grep -v '^Syntax OK$$' || :
 
-.PHONY: .FORCE-GIT-VERSION-FILE doc $(T) $(slow_tests) man
-.PHONY: test-install
+.PHONY: .FORCE-GIT-VERSION-FILE doc $(T) $(slow_tests) man $(T_sh) clean

data/ISSUES

@@ -76,15 +76,18 @@ document distributed with git) on guidelines for patch submission.
 * public: mailto:unicorn-public@yhbt.net
 * nntp://news.gmane.io/gmane.comp.lang.ruby.unicorn.general
 * nntp://news.public-inbox.org/inbox.comp.lang.ruby.unicorn
+* imaps://news.public-inbox.org/inbox.comp.lang.ruby.unicorn.0
 * https://yhbt.net/unicorn-public/
 * http://ou63pmih66umazou.onion/unicorn-public/
 
 Mailing list subscription is optional, so Cc: all participants.
 
-You can follow along via NNTP (read-only):
+You can follow along via NNTP or IMAP (read-only):
 
 	nntp://news.public-inbox.org/inbox.comp.lang.ruby.unicorn
 	nntp://news.gmane.io/gmane.comp.lang.ruby.unicorn.general
+	imaps://news.public-inbox.org/inbox.comp.lang.ruby.unicorn.0
+	imap://ou63pmih66umazou.onion/inbox.comp.lang.ruby.unicorn.0
 
 Or Atom feeds:
 

data/README

@@ -136,10 +136,17 @@ requests) go to the mailing list/newsgroup.  See the ISSUES document for
 information on the {mailing list}[mailto:unicorn-public@yhbt.net].
 
 The mailing list is archived at https://yhbt.net/unicorn-public/
+
 Read-only NNTP access is available at:
 nntp://news.public-inbox.org/inbox.comp.lang.ruby.unicorn and
 nntp://news.gmane.io/gmane.comp.lang.ruby.unicorn.general
 
+Read-only IMAP access is also avaialble at:
+imaps://news.public-inbox.org/inbox.comp.lang.ruby.unicorn.0 and
+imap://ou63pmih66umazou.onion/inbox.comp.lang.ruby.unicorn.0
+AUTH=ANONYMOUS mechanism is supported, as is any username+password
+combination.
+
 For the latest on unicorn releases, you may also finger us at
 unicorn@yhbt.net or check our NEWS page (and subscribe to our Atom
 feed).

data/ext/unicorn_http/extconf.rb

@@ -1,6 +1,11 @@
 # -*- encoding: binary -*-
 require 'mkmf'
 
+unless RUBY_VERSION < '3.1'
+  warn "Unicorn was only tested against MRI up to 3.0.\n" \
+       "It might not properly work with #{RUBY_VERSION}"
+end
+
 have_macro("SIZEOF_OFF_T", "ruby.h") or check_sizeof("off_t", "sys/types.h")
 have_macro("SIZEOF_SIZE_T", "ruby.h") or check_sizeof("size_t", "sys/types.h")
 have_macro("SIZEOF_LONG", "ruby.h") or check_sizeof("long", "sys/types.h")

data/ext/unicorn_http/unicorn_http.rl

@@ -62,7 +62,8 @@ struct http_parser {
   } len;
 };
 
-static ID id_set_backtrace;
+static ID id_set_backtrace, id_is_chunked_p;
+static VALUE cHttpParser;
 
 #ifdef HAVE_RB_HASH_CLEAR /* Ruby >= 2.0 */
 #  define my_hash_clear(h) (void)rb_hash_clear(h)
@@ -220,6 +221,19 @@ static void write_cont_value(struct http_parser *hp,
   rb_str_buf_cat(hp->cont, vptr, end + 1);
 }
 
+static int is_chunked(VALUE v)
+{
+  /* common case first */
+  if (STR_CSTR_CASE_EQ(v, "chunked"))
+    return 1;
+
+  /*
+   * call Ruby function in unicorn/http_request.rb to deal with unlikely
+   * comma-delimited case
+   */
+  return rb_funcall(cHttpParser, id_is_chunked_p, 1, v) != Qfalse;
+}
+
 static void write_value(struct http_parser *hp,
                         const char *buffer, const char *p)
 {
@@ -246,7 +260,9 @@ static void write_value(struct http_parser *hp,
     f = uncommon_field(field, flen);
   } else if (f == g_http_connection) {
     hp_keepalive_connection(hp, v);
-  } else if (f == g_content_length) {
+  } else if (f == g_content_length && !HP_FL_TEST(hp, CHUNKED)) {
+    if (hp->len.content)
+      parser_raise(eHttpParserError, "Content-Length already set");
     hp->len.content = parse_length(RSTRING_PTR(v), RSTRING_LEN(v));
     if (hp->len.content < 0)
       parser_raise(eHttpParserError, "invalid Content-Length");
@@ -254,9 +270,30 @@ static void write_value(struct http_parser *hp,
       HP_FL_SET(hp, HASBODY);
     hp_invalid_if_trailer(hp);
   } else if (f == g_http_transfer_encoding) {
-    if (STR_CSTR_CASE_EQ(v, "chunked")) {
+    if (is_chunked(v)) {
+      if (HP_FL_TEST(hp, CHUNKED))
+        /*
+         * RFC 7230 3.3.1:
+         * A sender MUST NOT apply chunked more than once to a message body
+         * (i.e., chunking an already chunked message is not allowed).
+         */
+        parser_raise(eHttpParserError, "Transfer-Encoding double chunked");
+
       HP_FL_SET(hp, CHUNKED);
       HP_FL_SET(hp, HASBODY);
+
+      /* RFC 7230 3.3.3, 3: favor chunked if Content-Length exists */
+      hp->len.content = 0;
+    } else if (HP_FL_TEST(hp, CHUNKED)) {
+      /*
+       * RFC 7230 3.3.3, point 3 states:
+       * If a Transfer-Encoding header field is present in a request and
+       * the chunked transfer coding is not the final encoding, the
+       * message body length cannot be determined reliably; the server
+       * MUST respond with the 400 (Bad Request) status code and then
+       * close the connection.
+       */
+      parser_raise(eHttpParserError, "invalid Transfer-Encoding");
     }
     hp_invalid_if_trailer(hp);
   } else if (f == g_http_trailer) {
@@ -931,7 +968,7 @@ static VALUE HttpParser_rssget(VALUE self)
 
 void Init_unicorn_http(void)
 {
-  VALUE mUnicorn, cHttpParser;
+  VALUE mUnicorn;
 
   mUnicorn = rb_define_module("Unicorn");
   cHttpParser = rb_define_class_under(mUnicorn, "HttpParser", rb_cObject);
@@ -991,5 +1028,6 @@ void Init_unicorn_http(void)
 #ifndef HAVE_RB_HASH_CLEAR
   id_clear = rb_intern("clear");
 #endif
+  id_is_chunked_p = rb_intern("is_chunked?");
 }
 #undef SET_GLOBAL

data/lib/unicorn/configurator.rb

@@ -53,6 +53,7 @@ class Unicorn::Configurator
         server.logger.info("worker=#{worker.nr} ready")
       },
     :pid => nil,
+    :early_hints => false,
     :worker_exec => false,
     :preload_app => false,
     :check_client_connection => false,
@@ -276,6 +277,15 @@ def default_middleware(bool)
     set_bool(:default_middleware, bool)
   end
 
+  # sets whether to enable the proposed early hints Rack API.
+  # If enabled, Rails 5.2+ will automatically send a 103 Early Hint
+  # for all the `javascript_include_tag` and `stylesheet_link_tag`
+  # in your response. See: https://api.rubyonrails.org/v5.2/classes/ActionDispatch/Request.html#method-i-send_early_hints
+  # See also https://tools.ietf.org/html/rfc8297
+  def early_hints(bool)
+    set_bool(:early_hints, bool)
+  end
+
   # sets listeners to the given +addresses+, replacing or augmenting the
   # current set.  This is for the global listener pool shared by all
   # worker processes.  For per-worker listeners, see the after_fork example

data/lib/unicorn/http_request.rb

@@ -188,4 +188,15 @@ def write_http_header(socket) # :nodoc:
       HTTP_RESPONSE_START.each { |c| socket.write(c) }
     end
   end
+
+  # called by ext/unicorn_http/unicorn_http.rl via rb_funcall
+  def self.is_chunked?(v) # :nodoc:
+    vals = v.split(/[ \t]*,[ \t]*/).map!(&:downcase)
+    if vals.pop == 'chunked'.freeze
+      return true unless vals.include?('chunked'.freeze)
+      raise Unicorn::HttpParserError, 'double chunked', []
+    end
+    return false unless vals.include?('chunked'.freeze)
+    raise Unicorn::HttpParserError, 'chunked not last', []
+  end
 end

data/lib/unicorn/http_server.rb

@@ -15,7 +15,7 @@ class Unicorn::HttpServer
                 :before_fork, :after_fork, :before_exec,
                 :listener_opts, :preload_app,
                 :orig_app, :config, :ready_pipe, :user,
-                :default_middleware
+                :default_middleware, :early_hints
   attr_writer   :after_worker_exit, :after_worker_ready, :worker_exec
 
   attr_reader :pid, :logger
@@ -588,6 +588,25 @@ def handle_error(client, e)
   rescue
   end
 
+  def e103_response_write(client, headers)
+    response = if @request.response_start_sent
+      "103 Early Hints\r\n"
+    else
+      "HTTP/1.1 103 Early Hints\r\n"
+    end
+
+    headers.each_pair do |k, vs|
+      next if !vs || vs.empty?
+      values = vs.to_s.split("\n".freeze)
+      values.each do |v|
+        response << "#{k}: #{v}\r\n"
+      end
+    end
+    response << "\r\n".freeze
+    response << "HTTP/1.1 ".freeze if @request.response_start_sent
+    client.write(response)
+  end
+
   def e100_response_write(client, env)
     # We use String#freeze to avoid allocations under Ruby 2.1+
     # Not many users hit this code path, so it's better to reduce the
@@ -602,7 +621,17 @@ def e100_response_write(client, env)
   # once a client is accepted, it is processed in its entirety here
   # in 3 easy steps: read request, call app, write app response
   def process_client(client)
-    status, headers, body = @app.call(env = @request.read(client))
+    env = @request.read(client)
+
+    if early_hints
+      env["rack.early_hints"] = lambda do |headers|
+        e103_response_write(client, headers)
+      end
+    end
+
+    env["rack.after_reply"] = []
+
+    status, headers, body = @app.call(env)
 
     begin
       return if @request.hijacked?
@@ -624,6 +653,8 @@ def process_client(client)
     end
   rescue => e
     handle_error(client, e)
+  ensure
+    env["rack.after_reply"].each(&:call) if env
   end
 
   def nuke_listeners!(readers)
@@ -686,6 +717,7 @@ def worker_loop(worker)
     trap(:USR1) { nr = -65536 }
 
     ready = readers.dup
+    nr_listeners = readers.size
     @after_worker_ready.call(self, worker)
 
     begin
@@ -708,7 +740,7 @@ def worker_loop(worker)
       # we're probably reasonably busy, so avoid calling select()
       # and do a speculative non-blocking accept() on ready listeners
       # before we sleep again in select().
-      unless nr == 0
+      if nr == nr_listeners
         tmp = ready.dup
         redo
       end

data/t/GNUmakefile

@@ -1,74 +1,5 @@
-# we can run tests in parallel with GNU make
+# there used to be more, here, but we stopped relying on recursive make
 all::
+	$(MAKE) -C .. test-integration
 
-pid := $(shell echo $$PPID)
-
-RUBY = ruby
-RAKE = rake
--include ../local.mk
-ifeq ($(RUBY_VERSION),)
-  RUBY_VERSION := $(shell $(RUBY) -e 'puts RUBY_VERSION')
-endif
-
-ifeq ($(RUBY_VERSION),)
-  $(error unable to detect RUBY_VERSION)
-endif
-
-RUBY_ENGINE := $(shell $(RUBY) -e 'puts((RUBY_ENGINE rescue "ruby"))')
-export RUBY_ENGINE
-
-MYLIBS := $(RUBYLIB)
-
-T = $(wildcard t[0-9][0-9][0-9][0-9]-*.sh)
-
-all:: $(T)
-
-# can't rely on "set -o pipefail" since we don't require bash or ksh93 :<
-t_pfx = trash/$@-$(RUBY_ENGINE)-$(RUBY_VERSION)
-TEST_OPTS =
-# TRACER = strace -f -o $(t_pfx).strace -s 100000
-# TRACER = /usr/bin/time -o $(t_pfx).time
-
-ifdef V
-  ifeq ($(V),2)
-    TEST_OPTS += --trace
-  else
-    TEST_OPTS += --verbose
-  endif
-endif
-
-random_blob:
-	dd if=/dev/urandom bs=1M count=30 of=$@.$(pid)
-	mv $@.$(pid) $@
-
-$(T): random_blob
-
-dependencies := socat curl
-deps := $(addprefix .dep+,$(dependencies))
-$(deps): dep_bin = $(lastword $(subst +, ,$@))
-$(deps):
-	@which $(dep_bin) > $@.$(pid) 2>/dev/null || :
-	@test -s $@.$(pid) || \
-	  { echo >&2 "E '$(dep_bin)' not found in PATH=$(PATH)"; exit 1; }
-	@mv $@.$(pid) $@
-dep: $(deps)
-
-test_prefix := $(CURDIR)/../test/$(RUBY_ENGINE)-$(RUBY_VERSION)
-$(test_prefix)/.stamp:
-	$(MAKE) -C .. test-install
-
-$(T): export RUBY := $(RUBY)
-$(T): export RAKE := $(RAKE)
-$(T): export PATH := $(test_prefix)/bin:$(PATH)
-$(T): export RUBYLIB := $(test_prefix)/lib:$(MYLIBS)
-$(T): dep $(test_prefix)/.stamp trash/.gitignore
-	$(TRACER) $(SHELL) $(SH_TEST_OPTS) $@ $(TEST_OPTS)
-
-trash/.gitignore:
-	mkdir -p $(@D)
-	echo '*' > $@
-
-clean:
-	$(RM) -r trash/*
-
-.PHONY: $(T) clean
+.PHONY: all

data/test/test_helper.rb

@@ -34,16 +34,33 @@
   Debugger.start
 end
 
+unless RUBY_VERSION < '3.1'
+  warn "Unicorn was only tested against MRI up to 3.0.\n" \
+       "It might not properly work with #{RUBY_VERSION}"
+end
+
 def redirect_test_io
   orig_err = STDERR.dup
   orig_out = STDOUT.dup
-  STDERR.reopen("test_stderr.#{$$}.log", "a")
-  STDOUT.reopen("test_stdout.#{$$}.log", "a")
+  new_out = File.open("test_stdout.#$$.log", "a")
+  new_err = File.open("test_stderr.#$$.log", "a")
+  new_out.sync = new_err.sync = true
+
+  if tail = ENV['TAIL'] # "tail -F" if GNU, "tail -f" otherwise
+    require 'shellwords'
+    cmd = tail.shellsplit
+    cmd << new_out.path
+    cmd << new_err.path
+    pid = Process.spawn(*cmd, { 1 => 2, :pgroup => true })
+    sleep 0.1 # wait for tail(1) to startup
+  end
+  STDERR.reopen(new_err)
+  STDOUT.reopen(new_out)
   STDERR.sync = STDOUT.sync = true
 
   at_exit do
-    File.unlink("test_stderr.#{$$}.log") rescue nil
-    File.unlink("test_stdout.#{$$}.log") rescue nil
+    File.unlink(new_out.path) rescue nil
+    File.unlink(new_err.path) rescue nil
   end
 
   begin
@@ -51,6 +68,7 @@ def redirect_test_io
   ensure
     STDERR.reopen(orig_err)
     STDOUT.reopen(orig_out)
+    Process.kill(:TERM, pid) if pid
   end
 end
 

data/test/unit/test_http_parser_ng.rb

@@ -11,6 +11,20 @@ def setup
     @parser = HttpParser.new
   end
 
+  # RFC 7230 allows gzip/deflate/compress Transfer-Encoding,
+  # but "chunked" must be last if used
+  def test_is_chunked
+    [ 'chunked,chunked', 'chunked,gzip', 'chunked,gzip,chunked' ].each do |x|
+      assert_raise(HttpParserError) { HttpParser.is_chunked?(x) }
+    end
+    [ 'gzip, chunked', 'gzip,chunked', 'gzip ,chunked' ].each do |x|
+      assert HttpParser.is_chunked?(x)
+    end
+    [ 'gzip', 'xhunked', 'xchunked' ].each do |x|
+      assert !HttpParser.is_chunked?(x)
+    end
+  end
+
   def test_parser_max_len
     assert_raises(RangeError) do
       HttpParser.max_header_len = 0xffffffff + 1
@@ -566,6 +580,73 @@ def test_invalid_content_length
     end
   end
 
+  def test_duplicate_content_length
+    str = "PUT / HTTP/1.1\r\n" \
+          "Content-Length: 1\r\n" \
+          "Content-Length: 9\r\n" \
+          "\r\n"
+    assert_raises(HttpParserError) { @parser.headers({}, str) }
+  end
+
+  def test_chunked_overrides_content_length
+    order = [ 'Transfer-Encoding: chunked', 'Content-Length: 666' ]
+    %w(a b).each do |x|
+      str = "PUT /#{x} HTTP/1.1\r\n" \
+            "#{order.join("\r\n")}" \
+            "\r\n\r\na\r\nhelloworld\r\n0\r\n\r\n"
+      order.reverse!
+      env = @parser.headers({}, str)
+      assert_nil @parser.content_length
+      assert_equal 'chunked', env['HTTP_TRANSFER_ENCODING']
+      assert_equal '666', env['CONTENT_LENGTH'],
+        'Content-Length logged so the app can log a possible client bug/attack'
+      @parser.filter_body(dst = '', str)
+      assert_equal 'helloworld', dst
+      @parser.parse # handle the non-existent trailer
+      assert @parser.next?
+    end
+  end
+
+  def test_chunked_order_good
+    str = "PUT /x HTTP/1.1\r\n" \
+          "Transfer-Encoding: gzip\r\n" \
+          "Transfer-Encoding: chunked\r\n" \
+          "\r\n"
+    env = @parser.headers({}, str)
+    assert_equal 'gzip,chunked', env['HTTP_TRANSFER_ENCODING']
+    assert_nil @parser.content_length
+
+    @parser.clear
+    str = "PUT /x HTTP/1.1\r\n" \
+          "Transfer-Encoding: gzip, chunked\r\n" \
+          "\r\n"
+    env = @parser.headers({}, str)
+    assert_equal 'gzip, chunked', env['HTTP_TRANSFER_ENCODING']
+    assert_nil @parser.content_length
+  end
+
+  def test_chunked_order_bad
+    str = "PUT /x HTTP/1.1\r\n" \
+          "Transfer-Encoding: chunked\r\n" \
+          "Transfer-Encoding: gzip\r\n" \
+          "\r\n"
+    assert_raise(HttpParserError) { @parser.headers({}, str) }
+  end
+
+  def test_double_chunked
+    str = "PUT /x HTTP/1.1\r\n" \
+          "Transfer-Encoding: chunked\r\n" \
+          "Transfer-Encoding: chunked\r\n" \
+          "\r\n"
+    assert_raise(HttpParserError) { @parser.headers({}, str) }
+
+    @parser.clear
+    str = "PUT /x HTTP/1.1\r\n" \
+          "Transfer-Encoding: chunked,chunked\r\n" \
+          "\r\n"
+    assert_raise(HttpParserError) { @parser.headers({}, str) }
+  end
+
   def test_backtrace_is_empty
     begin
       @parser.headers({}, "AAADFSFDSFD\r\n\r\n")

data/test/unit/test_server.rb

@@ -23,6 +23,34 @@ def call(env)
   end
 end
 
+class TestEarlyHintsHandler
+  def call(env)
+    while env['rack.input'].read(4096)
+    end
+    env['rack.early_hints'].call(
+      "Link" => "</style.css>; rel=preload; as=style\n</script.js>; rel=preload"
+    )
+    [200, { 'Content-Type' => 'text/plain' }, ['hello!\n']]
+  end
+end
+
+class TestRackAfterReply
+  def initialize
+    @called = false
+  end
+
+  def call(env)
+    while env['rack.input'].read(4096)
+    end
+
+    env["rack.after_reply"] << -> { @called = true }
+
+    [200, { 'Content-Type' => 'text/plain' }, ["after_reply_called: #{@called}"]]
+  rescue Unicorn::ClientShutdown, Unicorn::HttpParserError => e
+    $stderr.syswrite("#{e.class}: #{e.message} #{e.backtrace.empty?}\n")
+    raise e
+  end
+end
 
 class WebServerTest < Test::Unit::TestCase
 
@@ -84,6 +112,52 @@ def test_preload_app_config
     tmp.close!
   end
 
+  def test_early_hints
+    teardown
+    redirect_test_io do
+      @server = HttpServer.new(TestEarlyHintsHandler.new,
+                               :listeners => [ "127.0.0.1:#@port"],
+                               :early_hints => true)
+      @server.start
+    end
+
+    sock = TCPSocket.new('127.0.0.1', @port)
+    sock.syswrite("GET / HTTP/1.0\r\n\r\n")
+
+    responses = sock.read(4096)
+    assert_match %r{\AHTTP/1.[01] 103\b}, responses
+    assert_match %r{^Link: </style\.css>}, responses
+    assert_match %r{^Link: </script\.js>}, responses
+
+    assert_match %r{^HTTP/1.[01] 200\b}, responses
+  end
+
+  def test_after_reply
+    teardown
+
+    redirect_test_io do
+      @server = HttpServer.new(TestRackAfterReply.new,
+                               :listeners => [ "127.0.0.1:#@port"])
+      @server.start
+    end
+
+    sock = TCPSocket.new('127.0.0.1', @port)
+    sock.syswrite("GET / HTTP/1.0\r\n\r\n")
+
+    responses = sock.read(4096)
+    assert_match %r{\AHTTP/1.[01] 200\b}, responses
+    assert_match %r{^after_reply_called: false}, responses
+
+    sock = TCPSocket.new('127.0.0.1', @port)
+    sock.syswrite("GET / HTTP/1.0\r\n\r\n")
+
+    responses = sock.read(4096)
+    assert_match %r{\AHTTP/1.[01] 200\b}, responses
+    assert_match %r{^after_reply_called: true}, responses
+
+    sock.close
+  end
+
   def test_broken_app
     teardown
     app = lambda { |env| raise RuntimeError, "hello" }

data/unicorn.gemspec

@@ -11,7 +11,7 @@
 
 Gem::Specification.new do |s|
   s.name = %q{unicorn}
-  s.version = (ENV['VERSION'] || '5.5.3').dup
+  s.version = (ENV['VERSION'] || '5.8.0').dup
   s.authors = ['unicorn hackers']
   s.summary = 'Rack HTTP server for fast clients and Unix'
   s.description = File.read('README').split("\n\n")[1]
@@ -25,10 +25,11 @@
   s.homepage = 'https://yhbt.net/unicorn/'
   s.test_files = test_files
 
-  # technically we need ">= 1.9.3", too, but avoid the array here since
-  # old rubygems versions (1.8.23.2 at least) do not support multiple
-  # version requirements here.
-  s.required_ruby_version = '< 3.0'
+  # 1.9.3 is the minumum supported version. We don't specify
+  # a maximum version to make it easier to test pre-releases,
+  # but we do warn users if they install unicorn on an untested
+  # version in extconf.rb
+  s.required_ruby_version = ">= 1.9.3"
 
   # We do not have a hard dependency on rack, it's possible to load
   # things which respond to #call.  HTTP status lines in responses

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: unicorn
 version: !ruby/object:Gem::Version
-  version: 5.5.3
+  version: 5.8.0
 platform: ruby
 authors:
 - unicorn hackers
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-31 00:00:00.000000000 Z
+date: 2020-12-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack
@@ -282,9 +282,9 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - "<"
+  - - ">="
     - !ruby/object:Gem::Version
-      version: '3.0'
+      version: 1.9.3
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="