unicorn 5.4.1 → 6.1.0

data/t/t0301-no-default-middleware-ignored-in-config.sh

@@ -0,0 +1,25 @@
+#!/bin/sh
+. ./test-lib.sh
+t_plan 3 "-N / --no-default-middleware option not supported in config.ru"
+
+t_begin "setup and start" && {
+	unicorn_setup
+	RACK_ENV=development unicorn -D -c $unicorn_config t0301.ru
+	unicorn_wait_start
+}
+
+t_begin "check switches parsed as expected and -N ignored for Rack::Lint" && {
+	debug=false
+	lint=
+	eval "$(curl -sf http://$listen/vars)"
+	test x"$debug" = xtrue
+	test x"$lint" != x
+	test -f "$lint"
+}
+
+t_begin "killing succeeds" && {
+	kill $unicorn_pid
+	check_stderr
+}
+
+t_done

data/t/t0301.ru

@@ -0,0 +1,13 @@
+#\-N --debug
+run(lambda do |env|
+  case env['PATH_INFO']
+  when '/vars'
+    b = "debug=#{$DEBUG.inspect}\n" \
+        "lint=#{caller.grep(%r{rack/lint\.rb})[0].split(':')[0]}\n"
+  end
+  h = {
+    'Content-Length' => b.size.to_s,
+    'Content-Type' => 'text/plain',
+  }
+  [ 200, h, [ b ] ]
+end)

data/t/test-lib.sh

@@ -94,7 +94,8 @@ check_stderr () {
 	set +u
 	_r_err=${1-${r_err}}
 	set -u
-	if grep -v $T $_r_err | grep -i Error
+	if grep -v $T $_r_err | grep -i Error | \
+		grep -v NameError.*Unicorn::Waiter
 	then
 		die "Errors found in $_r_err"
 	elif grep SIGKILL $_r_err

data/test/benchmark/README

@@ -42,9 +42,19 @@ The benchmark client is usually httperf.
 Another gentle reminder: performance with slow networks/clients
 is NOT our problem.  That is the job of nginx (or similar).
 
+== ddstream.ru
+
+Standalone Rack app intended to show how BAD we are at slow clients.
+See usage in comments.
+
+== readinput.ru
+
+Standalone Rack app intended to show how bad we are with slow uploaders.
+See usage in comments.
+
 == Contributors
 
-This directory is maintained independently in the "benchmark" branch
-based against v0.1.0.  Only changes to this directory (test/benchmarks)
-are committed to this branch although the master branch may merge this
-branch occassionaly.
+This directory is intended to remain stable.  Do not make changes
+to benchmarking code which can change performance and invalidate
+results across revisions.  Instead, write new benchmarks and update
+coments/documentation as necessary.

data/test/benchmark/ddstream.ru

@@ -0,0 +1,50 @@
+# This app is intended to test large HTTP responses with or without
+# a fully-buffering reverse proxy such as nginx. Without a fully-buffering
+# reverse proxy, unicorn will be unresponsive when client count exceeds
+# worker_processes.
+#
+# To demonstrate how bad unicorn is at slowly reading clients:
+#
+#   # in one terminal, start unicorn with one worker:
+#   unicorn -E none -l 127.0.0.1:8080 test/benchmark/ddstream.ru
+#
+#   # in a different terminal, start more slow curl processes than
+#   # unicorn workers and watch time outputs
+#   curl --limit-rate 8K --trace-time -vsN http://127.0.0.1:8080/ >/dev/null &
+#   curl --limit-rate 8K --trace-time -vsN http://127.0.0.1:8080/ >/dev/null &
+#   wait
+#
+# The last client won't see a response until the first one is done reading
+#
+# nginx note: do not change the default "proxy_buffering" behavior.
+# Setting "proxy_buffering off" prevents nginx from protecting unicorn.
+
+# totally standalone rack app to stream a giant response
+class BigResponse
+  def initialize(bs, count)
+    @buf = "#{bs.to_s(16)}\r\n#{' ' * bs}\r\n"
+    @count = count
+    @res = [ 200,
+      { 'Transfer-Encoding' => -'chunked', 'Content-Type' => 'text/plain' },
+      self
+    ]
+  end
+
+  # rack response body iterator
+  def each
+    (1..@count).each { yield @buf }
+    yield -"0\r\n\r\n"
+  end
+
+  # rack app entry endpoint
+  def call(_env)
+    @res
+  end
+end
+
+# default to a giant (128M) response because kernel socket buffers
+# can be ridiculously large on some systems
+bs = ENV['bs'] ? ENV['bs'].to_i : 65536
+count = ENV['count'] ? ENV['count'].to_i : 2048
+warn "serving response with bs=#{bs} count=#{count} (#{bs*count} bytes)"
+run BigResponse.new(bs, count)

data/test/benchmark/readinput.ru

@@ -0,0 +1,40 @@
+# This app is intended to test large HTTP requests with or without
+# a fully-buffering reverse proxy such as nginx. Without a fully-buffering
+# reverse proxy, unicorn will be unresponsive when client count exceeds
+# worker_processes.
+
+DOC = <<DOC
+To demonstrate how bad unicorn is at slowly uploading clients:
+
+  # in one terminal, start unicorn with one worker:
+  unicorn -E none -l 127.0.0.1:8080 test/benchmark/readinput.ru
+
+  # in a different terminal, upload 45M from multiple curl processes:
+  dd if=/dev/zero bs=45M count=1 | curl -T- -HExpect: --limit-rate 1M \
+     --trace-time -v http://127.0.0.1:8080/ &
+  dd if=/dev/zero bs=45M count=1 | curl -T- -HExpect: --limit-rate 1M \
+     --trace-time -v http://127.0.0.1:8080/ &
+  wait
+
+# The last client won't see a response until the first one is done uploading
+# You also won't be able to make GET requests to view this documentation
+# while clients are uploading.  You can also view the stderr debug output
+# of unicorn (see logging code in #{__FILE__}).
+DOC
+
+run(lambda do |env|
+  input = env['rack.input']
+  buf = ''.b
+
+  # default logger contains timestamps, rely on that so users can
+  # see what the server is doing
+  l = env['rack.logger']
+
+  l.debug('BEGIN reading input ...') if l
+  :nop while input.read(16384, buf)
+  l.debug('DONE reading input ...') if l
+
+  buf.clear
+  [ 200, [ %W(Content-Length #{DOC.size}), %w(Content-Type text/plain) ],
+    [ DOC ] ]
+end)

data/test/benchmark/uconnect.perl

@@ -0,0 +1,66 @@
+#!/usr/bin/perl -w
+# Benchmark script to spawn some processes and hammer a local unicorn
+# to test accept loop performance.  This only does Unix sockets.
+# There's plenty of TCP benchmarking tools out there, and TCP port reuse
+# has predictability problems since unicorn can't do persistent connections.
+# Written in Perl for the same reason: predictability.
+# Ruby GC is not as predictable as Perl refcounting.
+use strict;
+use Socket qw(AF_UNIX SOCK_STREAM sockaddr_un);
+use POSIX qw(:sys_wait_h);
+use Getopt::Std;
+# -c / -n switches stolen from ab(1)
+my $usage = "$0 [-c CONCURRENCY] [-n NUM_REQUESTS] SOCKET_PATH\n";
+our $opt_c = 2;
+our $opt_n = 1000;
+getopts('c:n:') or die $usage;
+my $unix_path = shift or die $usage;
+use constant REQ => "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n";
+use constant REQ_LEN => length(REQ);
+use constant BUFSIZ => 8192;
+$^F = 99; # don't waste syscall time with FD_CLOEXEC
+
+my %workers; # pid => worker num
+die "-n $opt_n not evenly divisible by -c $opt_c\n" if $opt_n % $opt_c;
+my $n_per_worker = $opt_n / $opt_c;
+my $addr = sockaddr_un($unix_path);
+
+for my $num (1..$opt_c) {
+	defined(my $pid = fork) or die "fork failed: $!\n";
+	if ($pid) {
+		$workers{$pid} = $num;
+	} else {
+		work($n_per_worker);
+	}
+}
+
+reap_worker(0) while scalar keys %workers;
+exit;
+
+sub work {
+	my ($n) = @_;
+	my ($buf, $x);
+	for (1..$n) {
+		socket(S, AF_UNIX, SOCK_STREAM, 0) or die "socket: $!";
+		connect(S, $addr) or die "connect: $!";
+		defined($x = syswrite(S, REQ)) or die "write: $!";
+		$x == REQ_LEN or die "short write: $x != ".REQ_LEN."\n";
+		do {
+			$x = sysread(S, $buf, BUFSIZ);
+			unless (defined $x) {
+				next if $!{EINTR};
+				die "sysread: $!\n";
+			}
+		} until ($x == 0);
+	}
+	exit 0;
+}
+
+sub reap_worker {
+	my ($flags) = @_;
+	my $pid = waitpid(-1, $flags);
+	return if !defined $pid || $pid <= 0;
+	my $p = delete $workers{$pid} || '(unknown)';
+	warn("$pid [$p] exited with $?\n") if $?;
+	$p;
+}

data/test/exec/test_exec.rb

@@ -45,8 +45,9 @@ end
 
   COMMON_TMP = Tempfile.new('unicorn_tmp') unless defined?(COMMON_TMP)
 
+  HEAVY_WORKERS = 2
   HEAVY_CFG = <<-EOS
-worker_processes 4
+worker_processes #{HEAVY_WORKERS}
 timeout 30
 logger Logger.new('#{COMMON_TMP.path}')
 before_fork do |server, worker|
@@ -573,7 +574,7 @@ EOF
     assert_equal String, results[0].class
     worker_pid = results[0].to_i
     assert_not_equal pid, worker_pid
-    s = UNIXSocket.new(tmp.path)
+    s = unix_socket(tmp.path)
     s.syswrite("GET / HTTP/1.0\r\n\r\n")
     results = ''
     loop { results << s.sysread(4096) } rescue nil
@@ -606,6 +607,7 @@ EOF
   def test_weird_config_settings
     File.open("config.ru", "wb") { |fp| fp.syswrite(HI) }
     ucfg = Tempfile.new('unicorn_test_config')
+    proc_total = HEAVY_WORKERS + 1 # + 1 for master
     ucfg.syswrite(HEAVY_CFG)
     pid = xfork do
       redirect_test_io do
@@ -616,9 +618,9 @@ EOF
     results = retry_hit(["http://#{@addr}:#{@port}/"])
     assert_equal String, results[0].class
     wait_master_ready(COMMON_TMP.path)
-    wait_workers_ready(COMMON_TMP.path, 4)
+    wait_workers_ready(COMMON_TMP.path, HEAVY_WORKERS)
     bf = File.readlines(COMMON_TMP.path).grep(/\bbefore_fork: worker=/)
-    assert_equal 4, bf.size
+    assert_equal HEAVY_WORKERS, bf.size
     rotate = Tempfile.new('unicorn_rotate')
 
     File.rename(COMMON_TMP.path, rotate.path)
@@ -630,20 +632,20 @@ EOF
     tries = DEFAULT_TRIES
     log = File.readlines(rotate.path)
     while (tries -= 1) > 0 &&
-          log.grep(/reopening logs\.\.\./).size < 5
+          log.grep(/reopening logs\.\.\./).size < proc_total
       sleep DEFAULT_RES
       log = File.readlines(rotate.path)
     end
-    assert_equal 5, log.grep(/reopening logs\.\.\./).size
+    assert_equal proc_total, log.grep(/reopening logs\.\.\./).size
     assert_equal 0, log.grep(/done reopening logs/).size
 
     tries = DEFAULT_TRIES
     log = File.readlines(COMMON_TMP.path)
-    while (tries -= 1) > 0 && log.grep(/done reopening logs/).size < 5
+    while (tries -= 1) > 0 && log.grep(/done reopening logs/).size < proc_total
       sleep DEFAULT_RES
       log = File.readlines(COMMON_TMP.path)
     end
-    assert_equal 5, log.grep(/done reopening logs/).size
+    assert_equal proc_total, log.grep(/done reopening logs/).size
     assert_equal 0, log.grep(/reopening logs\.\.\./).size
 
     Process.kill(:QUIT, pid)
@@ -730,7 +732,7 @@ EOF
     wait_for_file(sock_path)
     assert File.socket?(sock_path)
 
-    sock = UNIXSocket.new(sock_path)
+    sock = unix_socket(sock_path)
     sock.syswrite("GET / HTTP/1.0\r\n\r\n")
     results = sock.sysread(4096)
 
@@ -740,7 +742,7 @@ EOF
     wait_for_file(sock_path)
     assert File.socket?(sock_path)
 
-    sock = UNIXSocket.new(sock_path)
+    sock = unix_socket(sock_path)
     sock.syswrite("GET / HTTP/1.0\r\n\r\n")
     results = sock.sysread(4096)
 
@@ -775,7 +777,7 @@ EOF
     assert_equal pid, File.read(pid_file).to_i
     assert File.socket?(sock_path), "socket created"
 
-    sock = UNIXSocket.new(sock_path)
+    sock = unix_socket(sock_path)
     sock.syswrite("GET / HTTP/1.0\r\n\r\n")
     results = sock.sysread(4096)
 
@@ -801,7 +803,7 @@ EOF
     wait_for_file(new_sock_path)
     assert File.socket?(new_sock_path), "socket exists"
     @sockets.each do |path|
-      sock = UNIXSocket.new(path)
+      sock = unix_socket(path)
       sock.syswrite("GET / HTTP/1.0\r\n\r\n")
       results = sock.sysread(4096)
       assert_equal String, results.class

data/test/test_helper.rb

@@ -28,22 +28,43 @@ require 'tempfile'
 require 'fileutils'
 require 'logger'
 require 'unicorn'
+require 'io/nonblock'
 
 if ENV['DEBUG']
   require 'ruby-debug'
   Debugger.start
 end
 
+unless RUBY_VERSION < '3.1'
+  warn "Unicorn was only tested against MRI up to 3.0.\n" \
+       "It might not properly work with #{RUBY_VERSION}"
+end
+
 def redirect_test_io
   orig_err = STDERR.dup
   orig_out = STDOUT.dup
-  STDERR.reopen("test_stderr.#{$$}.log", "a")
-  STDOUT.reopen("test_stdout.#{$$}.log", "a")
+  rdr_pid = $$
+  new_out = File.open("test_stdout.#$$.log", "a")
+  new_err = File.open("test_stderr.#$$.log", "a")
+  new_out.sync = new_err.sync = true
+
+  if tail = ENV['TAIL'] # "tail -F" if GNU, "tail -f" otherwise
+    require 'shellwords'
+    cmd = tail.shellsplit
+    cmd << new_out.path
+    cmd << new_err.path
+    pid = Process.spawn(*cmd, { 1 => 2, :pgroup => true })
+    sleep 0.1 # wait for tail(1) to startup
+  end
+  STDERR.reopen(new_err)
+  STDOUT.reopen(new_out)
   STDERR.sync = STDOUT.sync = true
 
   at_exit do
-    File.unlink("test_stderr.#{$$}.log") rescue nil
-    File.unlink("test_stdout.#{$$}.log") rescue nil
+    if rdr_pid == $$
+      File.unlink(new_out.path) rescue nil
+      File.unlink(new_err.path) rescue nil
+    end
   end
 
   begin
@@ -51,6 +72,7 @@ def redirect_test_io
   ensure
     STDERR.reopen(orig_err)
     STDOUT.reopen(orig_out)
+    Process.kill(:TERM, pid) if pid
   end
 end
 
@@ -265,34 +287,20 @@ def wait_for_death(pid)
   raise "PID:#{pid} never died!"
 end
 
-# executes +cmd+ and chunks its STDOUT
-def chunked_spawn(stdout, *cmd)
-  fork {
-    crd, cwr = IO.pipe
-    crd.binmode
-    cwr.binmode
-    crd.sync = cwr.sync = true
-
-    pid = fork {
-      STDOUT.reopen(cwr)
-      crd.close
-      cwr.close
-      exec(*cmd)
-    }
-    cwr.close
-    begin
-      buf = crd.readpartial(16384)
-      stdout.write("#{'%x' % buf.size}\r\n#{buf}")
-    rescue EOFError
-      stdout.write("0\r\n")
-      pid, status = Process.waitpid(pid)
-      exit status.exitstatus
-    end while true
-  }
-end
-
 def reset_sig_handlers
   %w(WINCH QUIT INT TERM USR1 USR2 HUP TTIN TTOU CHLD).each do |sig|
     trap(sig, "DEFAULT")
   end
 end
+
+def tcp_socket(*args)
+  sock = TCPSocket.new(*args)
+  sock.nonblock = false
+  sock
+end
+
+def unix_socket(*args)
+  sock = UNIXSocket.new(*args)
+  sock.nonblock = false
+  sock
+end

data/test/unit/test_ccc.rb

@@ -3,6 +3,7 @@ require 'unicorn'
 require 'io/wait'
 require 'tempfile'
 require 'test/unit'
+require './test/test_helper'
 
 class TestCccTCPI < Test::Unit::TestCase
   def test_ccc_tcpi
@@ -42,7 +43,7 @@ class TestCccTCPI < Test::Unit::TestCase
     wr.close
 
     # make sure the server is running, at least
-    client = TCPSocket.new(host, port)
+    client = tcp_socket(host, port)
     client.write("GET / HTTP/1.1\r\nHost: example.com\r\n\r\n")
     assert client.wait(10), 'never got response from server'
     res = client.read
@@ -51,13 +52,13 @@ class TestCccTCPI < Test::Unit::TestCase
     client.close
 
     # start a slow request...
-    sleeper = TCPSocket.new(host, port)
+    sleeper = tcp_socket(host, port)
     sleeper.write("GET /sleep HTTP/1.1\r\nHost: example.com\r\n\r\n")
 
     # and a bunch of aborted ones
     nr = 100
     nr.times do |i|
-      client = TCPSocket.new(host, port)
+      client = tcp_socket(host, port)
       client.write("GET /collections/#{rand(10000)} HTTP/1.1\r\n" \
                    "Host: example.com\r\n\r\n")
       client.close

data/test/unit/test_http_parser.rb

@@ -865,4 +865,20 @@ class HttpParserTest < Test::Unit::TestCase
   rescue LoadError
     # not all Ruby implementations have objspace
   end
+
+  def test_dedupe
+    parser = HttpParser.new
+    # n.b. String#freeze optimization doesn't work under modern test-unit
+    exp = -'HTTP_HOST'
+    get = "GET / HTTP/1.1\r\nHost: example.com\r\nHavpbea-fhpxf: true\r\n\r\n"
+    assert parser.add_parse(get)
+    key = parser.env.keys.detect { |k| k == exp }
+    assert_same exp, key
+
+    if RUBY_VERSION.to_r >= 2.6 # 2.6.0-rc1+
+      exp = -'HTTP_HAVPBEA_FHPXF'
+      key = parser.env.keys.detect { |k| k == exp }
+      assert_same exp, key
+    end
+  end if RUBY_VERSION.to_r >= 2.5 && RUBY_ENGINE == 'ruby'
 end

data/test/unit/test_http_parser_ng.rb

@@ -11,6 +11,20 @@ class HttpParserNgTest < Test::Unit::TestCase
     @parser = HttpParser.new
   end
 
+  # RFC 7230 allows gzip/deflate/compress Transfer-Encoding,
+  # but "chunked" must be last if used
+  def test_is_chunked
+    [ 'chunked,chunked', 'chunked,gzip', 'chunked,gzip,chunked' ].each do |x|
+      assert_raise(HttpParserError) { HttpParser.is_chunked?(x) }
+    end
+    [ 'gzip, chunked', 'gzip,chunked', 'gzip ,chunked' ].each do |x|
+      assert HttpParser.is_chunked?(x)
+    end
+    [ 'gzip', 'xhunked', 'xchunked' ].each do |x|
+      assert !HttpParser.is_chunked?(x)
+    end
+  end
+
   def test_parser_max_len
     assert_raises(RangeError) do
       HttpParser.max_header_len = 0xffffffff + 1
@@ -566,6 +580,73 @@ class HttpParserNgTest < Test::Unit::TestCase
     end
   end
 
+  def test_duplicate_content_length
+    str = "PUT / HTTP/1.1\r\n" \
+          "Content-Length: 1\r\n" \
+          "Content-Length: 9\r\n" \
+          "\r\n"
+    assert_raises(HttpParserError) { @parser.headers({}, str) }
+  end
+
+  def test_chunked_overrides_content_length
+    order = [ 'Transfer-Encoding: chunked', 'Content-Length: 666' ]
+    %w(a b).each do |x|
+      str = "PUT /#{x} HTTP/1.1\r\n" \
+            "#{order.join("\r\n")}" \
+            "\r\n\r\na\r\nhelloworld\r\n0\r\n\r\n"
+      order.reverse!
+      env = @parser.headers({}, str)
+      assert_nil @parser.content_length
+      assert_equal 'chunked', env['HTTP_TRANSFER_ENCODING']
+      assert_equal '666', env['CONTENT_LENGTH'],
+        'Content-Length logged so the app can log a possible client bug/attack'
+      @parser.filter_body(dst = '', str)
+      assert_equal 'helloworld', dst
+      @parser.parse # handle the non-existent trailer
+      assert @parser.next?
+    end
+  end
+
+  def test_chunked_order_good
+    str = "PUT /x HTTP/1.1\r\n" \
+          "Transfer-Encoding: gzip\r\n" \
+          "Transfer-Encoding: chunked\r\n" \
+          "\r\n"
+    env = @parser.headers({}, str)
+    assert_equal 'gzip,chunked', env['HTTP_TRANSFER_ENCODING']
+    assert_nil @parser.content_length
+
+    @parser.clear
+    str = "PUT /x HTTP/1.1\r\n" \
+          "Transfer-Encoding: gzip, chunked\r\n" \
+          "\r\n"
+    env = @parser.headers({}, str)
+    assert_equal 'gzip, chunked', env['HTTP_TRANSFER_ENCODING']
+    assert_nil @parser.content_length
+  end
+
+  def test_chunked_order_bad
+    str = "PUT /x HTTP/1.1\r\n" \
+          "Transfer-Encoding: chunked\r\n" \
+          "Transfer-Encoding: gzip\r\n" \
+          "\r\n"
+    assert_raise(HttpParserError) { @parser.headers({}, str) }
+  end
+
+  def test_double_chunked
+    str = "PUT /x HTTP/1.1\r\n" \
+          "Transfer-Encoding: chunked\r\n" \
+          "Transfer-Encoding: chunked\r\n" \
+          "\r\n"
+    assert_raise(HttpParserError) { @parser.headers({}, str) }
+
+    @parser.clear
+    str = "PUT /x HTTP/1.1\r\n" \
+          "Transfer-Encoding: chunked,chunked\r\n" \
+          "\r\n"
+    assert_raise(HttpParserError) { @parser.headers({}, str) }
+  end
+
   def test_backtrace_is_empty
     begin
       @parser.headers({}, "AAADFSFDSFD\r\n\r\n")