unicorn 5.5.1 → 5.6.0

data/lib/unicorn/http_request.rb

@@ -188,4 +188,15 @@ def write_http_header(socket) # :nodoc:
       HTTP_RESPONSE_START.each { |c| socket.write(c) }
     end
   end
+
+  # called by ext/unicorn_http/unicorn_http.rl via rb_funcall
+  def self.is_chunked?(v) # :nodoc:
+    vals = v.split(/[ \t]*,[ \t]*/).map!(&:downcase)
+    if vals.pop == 'chunked'.freeze
+      return true unless vals.include?('chunked'.freeze)
+      raise Unicorn::HttpParserError, 'double chunked', []
+    end
+    return false unless vals.include?('chunked'.freeze)
+    raise Unicorn::HttpParserError, 'chunked not last', []
+  end
 end

data/lib/unicorn/http_server.rb

@@ -6,7 +6,7 @@
 # forked worker children.
 #
 # Users do not need to know the internals of this class, but reading the
-# {source}[https://bogomips.org/unicorn.git/tree/lib/unicorn/http_server.rb]
+# {source}[https://yhbt.net/unicorn.git/tree/lib/unicorn/http_server.rb]
 # is education for programmers wishing to learn how unicorn works.
 # See Unicorn::Configurator for information on how to configure unicorn.
 class Unicorn::HttpServer
@@ -15,7 +15,7 @@ class Unicorn::HttpServer
                 :before_fork, :after_fork, :before_exec,
                 :listener_opts, :preload_app,
                 :orig_app, :config, :ready_pipe, :user,
-                :default_middleware
+                :default_middleware, :early_hints
   attr_writer   :after_worker_exit, :after_worker_ready, :worker_exec
 
   attr_reader :pid, :logger
@@ -588,6 +588,25 @@ def handle_error(client, e)
   rescue
   end
 
+  def e103_response_write(client, headers)
+    response = if @request.response_start_sent
+      "103 Early Hints\r\n"
+    else
+      "HTTP/1.1 103 Early Hints\r\n"
+    end
+
+    headers.each_pair do |k, vs|
+      next if !vs || vs.empty?
+      values = vs.to_s.split("\n".freeze)
+      values.each do |v|
+        response << "#{k}: #{v}\r\n"
+      end
+    end
+    response << "\r\n".freeze
+    response << "HTTP/1.1 ".freeze if @request.response_start_sent
+    client.write(response)
+  end
+
   def e100_response_write(client, env)
     # We use String#freeze to avoid allocations under Ruby 2.1+
     # Not many users hit this code path, so it's better to reduce the
@@ -602,7 +621,15 @@ def e100_response_write(client, env)
   # once a client is accepted, it is processed in its entirety here
   # in 3 easy steps: read request, call app, write app response
   def process_client(client)
-    status, headers, body = @app.call(env = @request.read(client))
+    env = @request.read(client)
+
+    if early_hints
+      env["rack.early_hints"] = lambda do |headers|
+        e103_response_write(client, headers)
+      end
+    end
+
+    status, headers, body = @app.call(env)
 
     begin
       return if @request.hijacked?
@@ -686,6 +713,7 @@ def worker_loop(worker)
     trap(:USR1) { nr = -65536 }
 
     ready = readers.dup
+    nr_listeners = readers.size
     @after_worker_ready.call(self, worker)
 
     begin
@@ -708,7 +736,7 @@ def worker_loop(worker)
       # we're probably reasonably busy, so avoid calling select()
       # and do a speculative non-blocking accept() on ready listeners
       # before we sleep again in select().
-      unless nr == 0
+      if nr == nr_listeners
         tmp = ready.dup
         redo
       end

data/lib/unicorn/oob_gc.rb

@@ -43,8 +43,8 @@
 #     use Unicorn::OobGC, 2, %r{\A/(?:expensive/foo|more_expensive/foo)}
 #
 # Feedback from users of early implementations of this module:
-# * https://bogomips.org/unicorn-public/0BFC98E9-072B-47EE-9A70-05478C20141B@lukemelia.com/
-# * https://bogomips.org/unicorn-public/AANLkTilUbgdyDv9W1bi-s_W6kq9sOhWfmuYkKLoKGOLj@mail.gmail.com/
+# * https://yhbt.net/unicorn-public/0BFC98E9-072B-47EE-9A70-05478C20141B@lukemelia.com/
+# * https://yhbt.net/unicorn-public/AANLkTilUbgdyDv9W1bi-s_W6kq9sOhWfmuYkKLoKGOLj@mail.gmail.com/
 
 module Unicorn::OobGC
 

data/lib/unicorn/tmpio.rb

@@ -11,12 +11,18 @@ class Unicorn::TmpIO < File
   # immediately, switched to binary mode, and userspace output
   # buffering is disabled
   def self.new
+    path = nil
+
+    # workaround File#path being tainted:
+    # https://bugs.ruby-lang.org/issues/14485
     fp = begin
-      super("#{Dir::tmpdir}/#{rand}", RDWR|CREAT|EXCL, 0600)
+      path = "#{Dir::tmpdir}/#{rand}"
+      super(path, RDWR|CREAT|EXCL, 0600)
     rescue Errno::EEXIST
       retry
     end
-    unlink(fp.path)
+
+    unlink(path)
     fp.binmode
     fp.sync = true
     fp

data/test/benchmark/README

@@ -42,9 +42,19 @@ The benchmark client is usually httperf.
 Another gentle reminder: performance with slow networks/clients
 is NOT our problem.  That is the job of nginx (or similar).
 
+== ddstream.ru
+
+Standalone Rack app intended to show how BAD we are at slow clients.
+See usage in comments.
+
+== readinput.ru
+
+Standalone Rack app intended to show how bad we are with slow uploaders.
+See usage in comments.
+
 == Contributors
 
-This directory is maintained independently in the "benchmark" branch
-based against v0.1.0.  Only changes to this directory (test/benchmarks)
-are committed to this branch although the master branch may merge this
-branch occassionaly.
+This directory is intended to remain stable.  Do not make changes
+to benchmarking code which can change performance and invalidate
+results across revisions.  Instead, write new benchmarks and update
+coments/documentation as necessary.

data/test/benchmark/ddstream.ru

@@ -0,0 +1,50 @@
+# This app is intended to test large HTTP responses with or without
+# a fully-buffering reverse proxy such as nginx. Without a fully-buffering
+# reverse proxy, unicorn will be unresponsive when client count exceeds
+# worker_processes.
+#
+# To demonstrate how bad unicorn is at slowly reading clients:
+#
+#   # in one terminal, start unicorn with one worker:
+#   unicorn -E none -l 127.0.0.1:8080 test/benchmark/ddstream.ru
+#
+#   # in a different terminal, start more slow curl processes than
+#   # unicorn workers and watch time outputs
+#   curl --limit-rate 8K --trace-time -vsN http://127.0.0.1:8080/ >/dev/null &
+#   curl --limit-rate 8K --trace-time -vsN http://127.0.0.1:8080/ >/dev/null &
+#   wait
+#
+# The last client won't see a response until the first one is done reading
+#
+# nginx note: do not change the default "proxy_buffering" behavior.
+# Setting "proxy_buffering off" prevents nginx from protecting unicorn.
+
+# totally standalone rack app to stream a giant response
+class BigResponse
+  def initialize(bs, count)
+    @buf = "#{bs.to_s(16)}\r\n#{' ' * bs}\r\n"
+    @count = count
+    @res = [ 200,
+      { 'Transfer-Encoding' => -'chunked', 'Content-Type' => 'text/plain' },
+      self
+    ]
+  end
+
+  # rack response body iterator
+  def each
+    (1..@count).each { yield @buf }
+    yield -"0\r\n\r\n"
+  end
+
+  # rack app entry endpoint
+  def call(_env)
+    @res
+  end
+end
+
+# default to a giant (128M) response because kernel socket buffers
+# can be ridiculously large on some systems
+bs = ENV['bs'] ? ENV['bs'].to_i : 65536
+count = ENV['count'] ? ENV['count'].to_i : 2048
+warn "serving response with bs=#{bs} count=#{count} (#{bs*count} bytes)"
+run BigResponse.new(bs, count)

data/test/benchmark/readinput.ru

@@ -0,0 +1,40 @@
+# This app is intended to test large HTTP requests with or without
+# a fully-buffering reverse proxy such as nginx. Without a fully-buffering
+# reverse proxy, unicorn will be unresponsive when client count exceeds
+# worker_processes.
+
+DOC = <<DOC
+To demonstrate how bad unicorn is at slowly uploading clients:
+
+  # in one terminal, start unicorn with one worker:
+  unicorn -E none -l 127.0.0.1:8080 test/benchmark/readinput.ru
+
+  # in a different terminal, upload 45M from multiple curl processes:
+  dd if=/dev/zero bs=45M count=1 | curl -T- -HExpect: --limit-rate 1M \
+     --trace-time -v http://127.0.0.1:8080/ &
+  dd if=/dev/zero bs=45M count=1 | curl -T- -HExpect: --limit-rate 1M \
+     --trace-time -v http://127.0.0.1:8080/ &
+  wait
+
+# The last client won't see a response until the first one is done uploading
+# You also won't be able to make GET requests to view this documentation
+# while clients are uploading.  You can also view the stderr debug output
+# of unicorn (see logging code in #{__FILE__}).
+DOC
+
+run(lambda do |env|
+  input = env['rack.input']
+  buf = ''.b
+
+  # default logger contains timestamps, rely on that so users can
+  # see what the server is doing
+  l = env['rack.logger']
+
+  l.debug('BEGIN reading input ...') if l
+  :nop while input.read(16384, buf)
+  l.debug('DONE reading input ...') if l
+
+  buf.clear
+  [ 200, [ %W(Content-Length #{DOC.size}), %w(Content-Type text/plain) ],
+    [ DOC ] ]
+end)

data/test/benchmark/uconnect.perl

@@ -0,0 +1,66 @@
+#!/usr/bin/perl -w
+# Benchmark script to spawn some processes and hammer a local unicorn
+# to test accept loop performance.  This only does Unix sockets.
+# There's plenty of TCP benchmarking tools out there, and TCP port reuse
+# has predictability problems since unicorn can't do persistent connections.
+# Written in Perl for the same reason: predictability.
+# Ruby GC is not as predictable as Perl refcounting.
+use strict;
+use Socket qw(AF_UNIX SOCK_STREAM sockaddr_un);
+use POSIX qw(:sys_wait_h);
+use Getopt::Std;
+# -c / -n switches stolen from ab(1)
+my $usage = "$0 [-c CONCURRENCY] [-n NUM_REQUESTS] SOCKET_PATH\n";
+our $opt_c = 2;
+our $opt_n = 1000;
+getopts('c:n:') or die $usage;
+my $unix_path = shift or die $usage;
+use constant REQ => "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n";
+use constant REQ_LEN => length(REQ);
+use constant BUFSIZ => 8192;
+$^F = 99; # don't waste syscall time with FD_CLOEXEC
+
+my %workers; # pid => worker num
+die "-n $opt_n not evenly divisible by -c $opt_c\n" if $opt_n % $opt_c;
+my $n_per_worker = $opt_n / $opt_c;
+my $addr = sockaddr_un($unix_path);
+
+for my $num (1..$opt_c) {
+	defined(my $pid = fork) or die "fork failed: $!\n";
+	if ($pid) {
+		$workers{$pid} = $num;
+	} else {
+		work($n_per_worker);
+	}
+}
+
+reap_worker(0) while scalar keys %workers;
+exit;
+
+sub work {
+	my ($n) = @_;
+	my ($buf, $x);
+	for (1..$n) {
+		socket(S, AF_UNIX, SOCK_STREAM, 0) or die "socket: $!";
+		connect(S, $addr) or die "connect: $!";
+		defined($x = syswrite(S, REQ)) or die "write: $!";
+		$x == REQ_LEN or die "short write: $x != ".REQ_LEN."\n";
+		do {
+			$x = sysread(S, $buf, BUFSIZ);
+			unless (defined $x) {
+				next if $!{EINTR};
+				die "sysread: $!\n";
+			}
+		} until ($x == 0);
+	}
+	exit 0;
+}
+
+sub reap_worker {
+	my ($flags) = @_;
+	my $pid = waitpid(-1, $flags);
+	return if !defined $pid || $pid <= 0;
+	my $p = delete $workers{$pid} || '(unknown)';
+	warn("$pid [$p] exited with $?\n") if $?;
+	$p;
+}

data/test/exec/test_exec.rb

@@ -45,8 +45,9 @@ def call(env)
 
   COMMON_TMP = Tempfile.new('unicorn_tmp') unless defined?(COMMON_TMP)
 
+  HEAVY_WORKERS = 2
   HEAVY_CFG = <<-EOS
-worker_processes 4
+worker_processes #{HEAVY_WORKERS}
 timeout 30
 logger Logger.new('#{COMMON_TMP.path}')
 before_fork do |server, worker|
@@ -606,6 +607,7 @@ def test_unicorn_config_listen_augments_cli
   def test_weird_config_settings
     File.open("config.ru", "wb") { |fp| fp.syswrite(HI) }
     ucfg = Tempfile.new('unicorn_test_config')
+    proc_total = HEAVY_WORKERS + 1 # + 1 for master
     ucfg.syswrite(HEAVY_CFG)
     pid = xfork do
       redirect_test_io do
@@ -616,9 +618,9 @@ def test_weird_config_settings
     results = retry_hit(["http://#{@addr}:#{@port}/"])
     assert_equal String, results[0].class
     wait_master_ready(COMMON_TMP.path)
-    wait_workers_ready(COMMON_TMP.path, 4)
+    wait_workers_ready(COMMON_TMP.path, HEAVY_WORKERS)
     bf = File.readlines(COMMON_TMP.path).grep(/\bbefore_fork: worker=/)
-    assert_equal 4, bf.size
+    assert_equal HEAVY_WORKERS, bf.size
     rotate = Tempfile.new('unicorn_rotate')
 
     File.rename(COMMON_TMP.path, rotate.path)
@@ -630,20 +632,20 @@ def test_weird_config_settings
     tries = DEFAULT_TRIES
     log = File.readlines(rotate.path)
     while (tries -= 1) > 0 &&
-          log.grep(/reopening logs\.\.\./).size < 5
+          log.grep(/reopening logs\.\.\./).size < proc_total
       sleep DEFAULT_RES
       log = File.readlines(rotate.path)
     end
-    assert_equal 5, log.grep(/reopening logs\.\.\./).size
+    assert_equal proc_total, log.grep(/reopening logs\.\.\./).size
     assert_equal 0, log.grep(/done reopening logs/).size
 
     tries = DEFAULT_TRIES
     log = File.readlines(COMMON_TMP.path)
-    while (tries -= 1) > 0 && log.grep(/done reopening logs/).size < 5
+    while (tries -= 1) > 0 && log.grep(/done reopening logs/).size < proc_total
       sleep DEFAULT_RES
       log = File.readlines(COMMON_TMP.path)
     end
-    assert_equal 5, log.grep(/done reopening logs/).size
+    assert_equal proc_total, log.grep(/done reopening logs/).size
     assert_equal 0, log.grep(/reopening logs\.\.\./).size
 
     Process.kill(:QUIT, pid)

data/test/test_helper.rb

@@ -265,32 +265,6 @@ def wait_for_death(pid)
   raise "PID:#{pid} never died!"
 end
 
-# executes +cmd+ and chunks its STDOUT
-def chunked_spawn(stdout, *cmd)
-  fork {
-    crd, cwr = IO.pipe
-    crd.binmode
-    cwr.binmode
-    crd.sync = cwr.sync = true
-
-    pid = fork {
-      STDOUT.reopen(cwr)
-      crd.close
-      cwr.close
-      exec(*cmd)
-    }
-    cwr.close
-    begin
-      buf = crd.readpartial(16384)
-      stdout.write("#{'%x' % buf.size}\r\n#{buf}")
-    rescue EOFError
-      stdout.write("0\r\n")
-      pid, status = Process.waitpid(pid)
-      exit status.exitstatus
-    end while true
-  }
-end
-
 def reset_sig_handlers
   %w(WINCH QUIT INT TERM USR1 USR2 HUP TTIN TTOU CHLD).each do |sig|
     trap(sig, "DEFAULT")

data/test/unit/test_http_parser_ng.rb

@@ -11,6 +11,20 @@ def setup
     @parser = HttpParser.new
   end
 
+  # RFC 7230 allows gzip/deflate/compress Transfer-Encoding,
+  # but "chunked" must be last if used
+  def test_is_chunked
+    [ 'chunked,chunked', 'chunked,gzip', 'chunked,gzip,chunked' ].each do |x|
+      assert_raise(HttpParserError) { HttpParser.is_chunked?(x) }
+    end
+    [ 'gzip, chunked', 'gzip,chunked', 'gzip ,chunked' ].each do |x|
+      assert HttpParser.is_chunked?(x)
+    end
+    [ 'gzip', 'xhunked', 'xchunked' ].each do |x|
+      assert !HttpParser.is_chunked?(x)
+    end
+  end
+
   def test_parser_max_len
     assert_raises(RangeError) do
       HttpParser.max_header_len = 0xffffffff + 1
@@ -566,6 +580,73 @@ def test_invalid_content_length
     end
   end
 
+  def test_duplicate_content_length
+    str = "PUT / HTTP/1.1\r\n" \
+          "Content-Length: 1\r\n" \
+          "Content-Length: 9\r\n" \
+          "\r\n"
+    assert_raises(HttpParserError) { @parser.headers({}, str) }
+  end
+
+  def test_chunked_overrides_content_length
+    order = [ 'Transfer-Encoding: chunked', 'Content-Length: 666' ]
+    %w(a b).each do |x|
+      str = "PUT /#{x} HTTP/1.1\r\n" \
+            "#{order.join("\r\n")}" \
+            "\r\n\r\na\r\nhelloworld\r\n0\r\n\r\n"
+      order.reverse!
+      env = @parser.headers({}, str)
+      assert_nil @parser.content_length
+      assert_equal 'chunked', env['HTTP_TRANSFER_ENCODING']
+      assert_equal '666', env['CONTENT_LENGTH'],
+        'Content-Length logged so the app can log a possible client bug/attack'
+      @parser.filter_body(dst = '', str)
+      assert_equal 'helloworld', dst
+      @parser.parse # handle the non-existent trailer
+      assert @parser.next?
+    end
+  end
+
+  def test_chunked_order_good
+    str = "PUT /x HTTP/1.1\r\n" \
+          "Transfer-Encoding: gzip\r\n" \
+          "Transfer-Encoding: chunked\r\n" \
+          "\r\n"
+    env = @parser.headers({}, str)
+    assert_equal 'gzip,chunked', env['HTTP_TRANSFER_ENCODING']
+    assert_nil @parser.content_length
+
+    @parser.clear
+    str = "PUT /x HTTP/1.1\r\n" \
+          "Transfer-Encoding: gzip, chunked\r\n" \
+          "\r\n"
+    env = @parser.headers({}, str)
+    assert_equal 'gzip, chunked', env['HTTP_TRANSFER_ENCODING']
+    assert_nil @parser.content_length
+  end
+
+  def test_chunked_order_bad
+    str = "PUT /x HTTP/1.1\r\n" \
+          "Transfer-Encoding: chunked\r\n" \
+          "Transfer-Encoding: gzip\r\n" \
+          "\r\n"
+    assert_raise(HttpParserError) { @parser.headers({}, str) }
+  end
+
+  def test_double_chunked
+    str = "PUT /x HTTP/1.1\r\n" \
+          "Transfer-Encoding: chunked\r\n" \
+          "Transfer-Encoding: chunked\r\n" \
+          "\r\n"
+    assert_raise(HttpParserError) { @parser.headers({}, str) }
+
+    @parser.clear
+    str = "PUT /x HTTP/1.1\r\n" \
+          "Transfer-Encoding: chunked,chunked\r\n" \
+          "\r\n"
+    assert_raise(HttpParserError) { @parser.headers({}, str) }
+  end
+
   def test_backtrace_is_empty
     begin
       @parser.headers({}, "AAADFSFDSFD\r\n\r\n")