unicorn 4.8.0 → 6.1.0

data/test/benchmark/readinput.ru

@@ -0,0 +1,40 @@
+# This app is intended to test large HTTP requests with or without
+# a fully-buffering reverse proxy such as nginx. Without a fully-buffering
+# reverse proxy, unicorn will be unresponsive when client count exceeds
+# worker_processes.
+
+DOC = <<DOC
+To demonstrate how bad unicorn is at slowly uploading clients:
+
+  # in one terminal, start unicorn with one worker:
+  unicorn -E none -l 127.0.0.1:8080 test/benchmark/readinput.ru
+
+  # in a different terminal, upload 45M from multiple curl processes:
+  dd if=/dev/zero bs=45M count=1 | curl -T- -HExpect: --limit-rate 1M \
+     --trace-time -v http://127.0.0.1:8080/ &
+  dd if=/dev/zero bs=45M count=1 | curl -T- -HExpect: --limit-rate 1M \
+     --trace-time -v http://127.0.0.1:8080/ &
+  wait
+
+# The last client won't see a response until the first one is done uploading
+# You also won't be able to make GET requests to view this documentation
+# while clients are uploading.  You can also view the stderr debug output
+# of unicorn (see logging code in #{__FILE__}).
+DOC
+
+run(lambda do |env|
+  input = env['rack.input']
+  buf = ''.b
+
+  # default logger contains timestamps, rely on that so users can
+  # see what the server is doing
+  l = env['rack.logger']
+
+  l.debug('BEGIN reading input ...') if l
+  :nop while input.read(16384, buf)
+  l.debug('DONE reading input ...') if l
+
+  buf.clear
+  [ 200, [ %W(Content-Length #{DOC.size}), %w(Content-Type text/plain) ],
+    [ DOC ] ]
+end)

data/test/benchmark/uconnect.perl

@@ -0,0 +1,66 @@
+#!/usr/bin/perl -w
+# Benchmark script to spawn some processes and hammer a local unicorn
+# to test accept loop performance.  This only does Unix sockets.
+# There's plenty of TCP benchmarking tools out there, and TCP port reuse
+# has predictability problems since unicorn can't do persistent connections.
+# Written in Perl for the same reason: predictability.
+# Ruby GC is not as predictable as Perl refcounting.
+use strict;
+use Socket qw(AF_UNIX SOCK_STREAM sockaddr_un);
+use POSIX qw(:sys_wait_h);
+use Getopt::Std;
+# -c / -n switches stolen from ab(1)
+my $usage = "$0 [-c CONCURRENCY] [-n NUM_REQUESTS] SOCKET_PATH\n";
+our $opt_c = 2;
+our $opt_n = 1000;
+getopts('c:n:') or die $usage;
+my $unix_path = shift or die $usage;
+use constant REQ => "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n";
+use constant REQ_LEN => length(REQ);
+use constant BUFSIZ => 8192;
+$^F = 99; # don't waste syscall time with FD_CLOEXEC
+
+my %workers; # pid => worker num
+die "-n $opt_n not evenly divisible by -c $opt_c\n" if $opt_n % $opt_c;
+my $n_per_worker = $opt_n / $opt_c;
+my $addr = sockaddr_un($unix_path);
+
+for my $num (1..$opt_c) {
+	defined(my $pid = fork) or die "fork failed: $!\n";
+	if ($pid) {
+		$workers{$pid} = $num;
+	} else {
+		work($n_per_worker);
+	}
+}
+
+reap_worker(0) while scalar keys %workers;
+exit;
+
+sub work {
+	my ($n) = @_;
+	my ($buf, $x);
+	for (1..$n) {
+		socket(S, AF_UNIX, SOCK_STREAM, 0) or die "socket: $!";
+		connect(S, $addr) or die "connect: $!";
+		defined($x = syswrite(S, REQ)) or die "write: $!";
+		$x == REQ_LEN or die "short write: $x != ".REQ_LEN."\n";
+		do {
+			$x = sysread(S, $buf, BUFSIZ);
+			unless (defined $x) {
+				next if $!{EINTR};
+				die "sysread: $!\n";
+			}
+		} until ($x == 0);
+	}
+	exit 0;
+}
+
+sub reap_worker {
+	my ($flags) = @_;
+	my $pid = waitpid(-1, $flags);
+	return if !defined $pid || $pid <= 0;
+	my $p = delete $workers{$pid} || '(unknown)';
+	warn("$pid [$p] exited with $?\n") if $?;
+	$p;
+}

data/test/exec/test_exec.rb

@@ -2,7 +2,7 @@
 
 # Copyright (c) 2009 Eric Wong
 FLOCK_PATH = File.expand_path(__FILE__)
-require 'test/test_helper'
+require './test/test_helper'
 
 do_test = true
 $unicorn_bin = ENV['UNICORN_TEST_BIN'] || "unicorn"
@@ -45,8 +45,9 @@ end
 
   COMMON_TMP = Tempfile.new('unicorn_tmp') unless defined?(COMMON_TMP)
 
+  HEAVY_WORKERS = 2
   HEAVY_CFG = <<-EOS
-worker_processes 4
+worker_processes #{HEAVY_WORKERS}
 timeout 30
 logger Logger.new('#{COMMON_TMP.path}')
 before_fork do |server, worker|
@@ -96,6 +97,59 @@ run lambda { |env|
     end
   end
 
+  def test_sd_listen_fds_emulation
+    # [ruby-core:69895] [Bug #11336] fixed by r51576
+    return if RUBY_VERSION.to_f < 2.3
+
+    File.open("config.ru", "wb") { |fp| fp.write(HI) }
+    sock = TCPServer.new(@addr, @port)
+
+    [ %W(-l #@addr:#@port), nil ].each do |l|
+      sock.setsockopt(:SOL_SOCKET, :SO_KEEPALIVE, 0)
+
+      pid = xfork do
+        redirect_test_io do
+          # pretend to be systemd
+          ENV['LISTEN_PID'] = "#$$"
+          ENV['LISTEN_FDS'] = '1'
+
+          # 3 = SD_LISTEN_FDS_START
+          args = [ $unicorn_bin ]
+          args.concat(l) if l
+          args << { 3 => sock }
+          exec(*args)
+        end
+      end
+      res = hit(["http://#@addr:#@port/"])
+      assert_equal [ "HI\n" ], res
+      assert_shutdown(pid)
+      assert sock.getsockopt(:SOL_SOCKET, :SO_KEEPALIVE).bool,
+                  'unicorn should always set SO_KEEPALIVE on inherited sockets'
+    end
+  ensure
+    sock.close if sock
+  end
+
+  def test_inherit_listener_unspecified
+    File.open("config.ru", "wb") { |fp| fp.write(HI) }
+    sock = TCPServer.new(@addr, @port)
+    sock.setsockopt(:SOL_SOCKET, :SO_KEEPALIVE, 0)
+
+    pid = xfork do
+      redirect_test_io do
+        ENV['UNICORN_FD'] = sock.fileno.to_s
+        exec($unicorn_bin, sock.fileno => sock.fileno)
+      end
+    end
+    res = hit(["http://#@addr:#@port/"])
+    assert_equal [ "HI\n" ], res
+    assert_shutdown(pid)
+    assert sock.getsockopt(:SOL_SOCKET, :SO_KEEPALIVE).bool,
+                'unicorn should always set SO_KEEPALIVE on inherited sockets'
+  ensure
+    sock.close if sock
+  end
+
   def test_working_directory_rel_path_config_file
     other = Tempfile.new('unicorn.wd')
     File.unlink(other.path)
@@ -140,8 +194,8 @@ EOF
     assert_equal other.path, results.first
 
     Process.kill(:QUIT, pid)
-    ensure
-      FileUtils.rmtree(other.path)
+  ensure
+    FileUtils.rmtree(other.path)
   end
 
   def test_working_directory
@@ -176,8 +230,8 @@ EOF
     assert_equal other.path, results.first
 
     Process.kill(:QUIT, pid)
-    ensure
-      FileUtils.rmtree(other.path)
+  ensure
+    FileUtils.rmtree(other.path)
   end
 
   def test_working_directory_controls_relative_paths
@@ -218,11 +272,10 @@ EOF
     wait_master_ready("#{other.path}/stderr_log_here")
 
     Process.kill(:QUIT, pid)
-    ensure
-      FileUtils.rmtree(other.path)
+  ensure
+    FileUtils.rmtree(other.path)
   end
 
-
   def test_exit_signals
     %w(INT TERM QUIT).each do |sig|
       File.open("config.ru", "wb") { |fp| fp.syswrite(HI) }
@@ -521,7 +574,7 @@ EOF
     assert_equal String, results[0].class
     worker_pid = results[0].to_i
     assert_not_equal pid, worker_pid
-    s = UNIXSocket.new(tmp.path)
+    s = unix_socket(tmp.path)
     s.syswrite("GET / HTTP/1.0\r\n\r\n")
     results = ''
     loop { results << s.sysread(4096) } rescue nil
@@ -554,6 +607,7 @@ EOF
   def test_weird_config_settings
     File.open("config.ru", "wb") { |fp| fp.syswrite(HI) }
     ucfg = Tempfile.new('unicorn_test_config')
+    proc_total = HEAVY_WORKERS + 1 # + 1 for master
     ucfg.syswrite(HEAVY_CFG)
     pid = xfork do
       redirect_test_io do
@@ -564,9 +618,9 @@ EOF
     results = retry_hit(["http://#{@addr}:#{@port}/"])
     assert_equal String, results[0].class
     wait_master_ready(COMMON_TMP.path)
-    wait_workers_ready(COMMON_TMP.path, 4)
+    wait_workers_ready(COMMON_TMP.path, HEAVY_WORKERS)
     bf = File.readlines(COMMON_TMP.path).grep(/\bbefore_fork: worker=/)
-    assert_equal 4, bf.size
+    assert_equal HEAVY_WORKERS, bf.size
     rotate = Tempfile.new('unicorn_rotate')
 
     File.rename(COMMON_TMP.path, rotate.path)
@@ -578,20 +632,20 @@ EOF
     tries = DEFAULT_TRIES
     log = File.readlines(rotate.path)
     while (tries -= 1) > 0 &&
-          log.grep(/reopening logs\.\.\./).size < 5
+          log.grep(/reopening logs\.\.\./).size < proc_total
       sleep DEFAULT_RES
       log = File.readlines(rotate.path)
     end
-    assert_equal 5, log.grep(/reopening logs\.\.\./).size
+    assert_equal proc_total, log.grep(/reopening logs\.\.\./).size
     assert_equal 0, log.grep(/done reopening logs/).size
 
     tries = DEFAULT_TRIES
     log = File.readlines(COMMON_TMP.path)
-    while (tries -= 1) > 0 && log.grep(/done reopening logs/).size < 5
+    while (tries -= 1) > 0 && log.grep(/done reopening logs/).size < proc_total
       sleep DEFAULT_RES
       log = File.readlines(COMMON_TMP.path)
     end
-    assert_equal 5, log.grep(/done reopening logs/).size
+    assert_equal proc_total, log.grep(/done reopening logs/).size
     assert_equal 0, log.grep(/reopening logs\.\.\./).size
 
     Process.kill(:QUIT, pid)
@@ -678,7 +732,7 @@ EOF
     wait_for_file(sock_path)
     assert File.socket?(sock_path)
 
-    sock = UNIXSocket.new(sock_path)
+    sock = unix_socket(sock_path)
     sock.syswrite("GET / HTTP/1.0\r\n\r\n")
     results = sock.sysread(4096)
 
@@ -688,7 +742,7 @@ EOF
     wait_for_file(sock_path)
     assert File.socket?(sock_path)
 
-    sock = UNIXSocket.new(sock_path)
+    sock = unix_socket(sock_path)
     sock.syswrite("GET / HTTP/1.0\r\n\r\n")
     results = sock.sysread(4096)
 
@@ -723,7 +777,7 @@ EOF
     assert_equal pid, File.read(pid_file).to_i
     assert File.socket?(sock_path), "socket created"
 
-    sock = UNIXSocket.new(sock_path)
+    sock = unix_socket(sock_path)
     sock.syswrite("GET / HTTP/1.0\r\n\r\n")
     results = sock.sysread(4096)
 
@@ -749,7 +803,7 @@ EOF
     wait_for_file(new_sock_path)
     assert File.socket?(new_sock_path), "socket exists"
     @sockets.each do |path|
-      sock = UNIXSocket.new(path)
+      sock = unix_socket(path)
       sock.syswrite("GET / HTTP/1.0\r\n\r\n")
       results = sock.sysread(4096)
       assert_equal String, results.class

data/test/test_helper.rb

@@ -1,10 +1,10 @@
 # -*- encoding: binary -*-
 
-# Copyright (c) 2005 Zed A. Shaw 
+# Copyright (c) 2005 Zed A. Shaw
 # You can redistribute it and/or modify it under the same terms as Ruby 1.8 or
 # the GPLv2+ (GPLv3+ preferred)
 #
-# Additional work donated by contributors.  See http://mongrel.rubyforge.org/attributions.html 
+# Additional work donated by contributors.  See git history
 # for more information.
 
 STDIN.sync = STDOUT.sync = STDERR.sync = true # buffering makes debugging hard
@@ -28,22 +28,43 @@ require 'tempfile'
 require 'fileutils'
 require 'logger'
 require 'unicorn'
+require 'io/nonblock'
 
 if ENV['DEBUG']
   require 'ruby-debug'
   Debugger.start
 end
 
+unless RUBY_VERSION < '3.1'
+  warn "Unicorn was only tested against MRI up to 3.0.\n" \
+       "It might not properly work with #{RUBY_VERSION}"
+end
+
 def redirect_test_io
   orig_err = STDERR.dup
   orig_out = STDOUT.dup
-  STDERR.reopen("test_stderr.#{$$}.log", "a")
-  STDOUT.reopen("test_stdout.#{$$}.log", "a")
+  rdr_pid = $$
+  new_out = File.open("test_stdout.#$$.log", "a")
+  new_err = File.open("test_stderr.#$$.log", "a")
+  new_out.sync = new_err.sync = true
+
+  if tail = ENV['TAIL'] # "tail -F" if GNU, "tail -f" otherwise
+    require 'shellwords'
+    cmd = tail.shellsplit
+    cmd << new_out.path
+    cmd << new_err.path
+    pid = Process.spawn(*cmd, { 1 => 2, :pgroup => true })
+    sleep 0.1 # wait for tail(1) to startup
+  end
+  STDERR.reopen(new_err)
+  STDOUT.reopen(new_out)
   STDERR.sync = STDOUT.sync = true
 
   at_exit do
-    File.unlink("test_stderr.#{$$}.log") rescue nil
-    File.unlink("test_stdout.#{$$}.log") rescue nil
+    if rdr_pid == $$
+      File.unlink(new_out.path) rescue nil
+      File.unlink(new_err.path) rescue nil
+    end
   end
 
   begin
@@ -51,6 +72,7 @@ def redirect_test_io
   ensure
     STDERR.reopen(orig_err)
     STDOUT.reopen(orig_out)
+    Process.kill(:TERM, pid) if pid
   end
 end
 
@@ -265,33 +287,20 @@ def wait_for_death(pid)
   raise "PID:#{pid} never died!"
 end
 
-# executes +cmd+ and chunks its STDOUT
-def chunked_spawn(stdout, *cmd)
-  fork {
-    crd, cwr = IO.pipe
-    crd.binmode
-    cwr.binmode
-    crd.sync = cwr.sync = true
-
-    pid = fork {
-      STDOUT.reopen(cwr)
-      crd.close
-      cwr.close
-      exec(*cmd)
-    }
-    cwr.close
-    begin
-      buf = crd.readpartial(16384)
-      stdout.write("#{'%x' % buf.size}\r\n#{buf}")
-    rescue EOFError
-      stdout.write("0\r\n")
-      pid, status = Process.waitpid(pid)
-      exit status.exitstatus
-    end while true
-  }
+def reset_sig_handlers
+  %w(WINCH QUIT INT TERM USR1 USR2 HUP TTIN TTOU CHLD).each do |sig|
+    trap(sig, "DEFAULT")
+  end
 end
 
-def reset_sig_handlers
-  sigs = %w(CHLD).concat(Unicorn::HttpServer::QUEUE_SIGS)
-  sigs.each { |sig| trap(sig, "DEFAULT") }
+def tcp_socket(*args)
+  sock = TCPSocket.new(*args)
+  sock.nonblock = false
+  sock
+end
+
+def unix_socket(*args)
+  sock = UNIXSocket.new(*args)
+  sock.nonblock = false
+  sock
 end

data/test/unit/test_ccc.rb

@@ -0,0 +1,91 @@
+require 'socket'
+require 'unicorn'
+require 'io/wait'
+require 'tempfile'
+require 'test/unit'
+require './test/test_helper'
+
+class TestCccTCPI < Test::Unit::TestCase
+  def test_ccc_tcpi
+    start_pid = $$
+    host = '127.0.0.1'
+    srv = TCPServer.new(host, 0)
+    port = srv.addr[1]
+    err = Tempfile.new('unicorn_ccc')
+    rd, wr = IO.pipe
+    sleep_pipe = IO.pipe
+    pid = fork do
+      sleep_pipe[1].close
+      reqs = 0
+      rd.close
+      worker_pid = nil
+      app = lambda do |env|
+        worker_pid ||= begin
+          at_exit { wr.write(reqs.to_s) if worker_pid == $$ }
+          $$
+        end
+        reqs += 1
+
+        # will wake up when writer closes
+        sleep_pipe[0].read if env['PATH_INFO'] == '/sleep'
+
+        [ 200, [ %w(Content-Length 0),  %w(Content-Type text/plain) ], [] ]
+      end
+      ENV['UNICORN_FD'] = srv.fileno.to_s
+      opts = {
+        listeners: [ "#{host}:#{port}" ],
+        stderr_path: err.path,
+        check_client_connection: true,
+      }
+      uni = Unicorn::HttpServer.new(app, opts)
+      uni.start.join
+    end
+    wr.close
+
+    # make sure the server is running, at least
+    client = tcp_socket(host, port)
+    client.write("GET / HTTP/1.1\r\nHost: example.com\r\n\r\n")
+    assert client.wait(10), 'never got response from server'
+    res = client.read
+    assert_match %r{\AHTTP/1\.1 200}, res, 'got part of first response'
+    assert_match %r{\r\n\r\n\z}, res, 'got end of response, server is ready'
+    client.close
+
+    # start a slow request...
+    sleeper = tcp_socket(host, port)
+    sleeper.write("GET /sleep HTTP/1.1\r\nHost: example.com\r\n\r\n")
+
+    # and a bunch of aborted ones
+    nr = 100
+    nr.times do |i|
+      client = tcp_socket(host, port)
+      client.write("GET /collections/#{rand(10000)} HTTP/1.1\r\n" \
+                   "Host: example.com\r\n\r\n")
+      client.close
+    end
+    sleep_pipe[1].close # wake up the reader in the worker
+    res = sleeper.read
+    assert_match %r{\AHTTP/1\.1 200}, res, 'got part of first sleeper response'
+    assert_match %r{\r\n\r\n\z}, res, 'got end of sleeper response'
+    sleeper.close
+    kpid = pid
+    pid = nil
+    Process.kill(:QUIT, kpid)
+    _, status = Process.waitpid2(kpid)
+    assert status.success?
+    reqs = rd.read.to_i
+    warn "server got #{reqs} requests with #{nr} CCC aborted\n" if $DEBUG
+    assert_operator reqs, :<, nr
+    assert_operator reqs, :>=, 2, 'first 2 requests got through, at least'
+  ensure
+    return if start_pid != $$
+    srv.close if srv
+    if pid
+      Process.kill(:QUIT, pid)
+      _, status = Process.waitpid2(pid)
+      assert status.success?
+    end
+    err.close! if err
+    rd.close if rd
+  end
+end

data/test/unit/test_droplet.rb

@@ -4,7 +4,7 @@ require 'unicorn'
 class TestDroplet < Test::Unit::TestCase
   def test_create_many_droplets
     now = Time.now.to_i
-    tmp = (0..1024).map do |i|
+    (0..1024).each do |i|
       droplet = Unicorn::Worker.new(i)
       assert droplet.respond_to?(:tick)
       assert_equal 0, droplet.tick

data/test/unit/test_http_parser.rb

@@ -1,13 +1,13 @@
 # -*- encoding: binary -*-
 
-# Copyright (c) 2005 Zed A. Shaw 
+# Copyright (c) 2005 Zed A. Shaw
 # You can redistribute it and/or modify it under the same terms as Ruby 1.8 or
 # the GPLv2+ (GPLv3+ preferred)
 #
-# Additional work donated by contributors.  See http://mongrel.rubyforge.org/attributions.html
+# Additional work donated by contributors.  See git history
 # for more information.
 
-require 'test/test_helper'
+require './test/test_helper'
 
 include Unicorn
 
@@ -230,6 +230,24 @@ class HttpParserTest < Test::Unit::TestCase
     assert_equal expect, req['HTTP_X_SSL_BULLSHIT']
   end
 
+  def test_multiline_header_0d0a
+    parser = HttpParser.new
+    parser.buf << "GET / HTTP/1.0\r\n" \
+      "X-Multiline-Header: foo bar\r\n\tcha cha\r\n\tzha zha\r\n\r\n"
+    req = parser.env
+    assert_equal req, parser.parse
+    assert_equal 'foo bar cha cha zha zha', req['HTTP_X_MULTILINE_HEADER']
+  end
+
+  def test_multiline_header_0a
+    parser = HttpParser.new
+    parser.buf << "GET / HTTP/1.0\n" \
+      "X-Multiline-Header: foo bar\n\tcha cha\n\tzha zha\n\n"
+    req = parser.env
+    assert_equal req, parser.parse
+    assert_equal 'foo bar cha cha zha zha', req['HTTP_X_MULTILINE_HEADER']
+  end
+
   def test_continuation_eats_leading_spaces
     parser = HttpParser.new
     header = "GET / HTTP/1.1\r\n" \
@@ -833,22 +851,34 @@ class HttpParserTest < Test::Unit::TestCase
     assert_equal '', parser.env['HTTP_HOST']
   end
 
-  # so we don't  care about the portability of this test
-  # if it doesn't leak on Linux, it won't leak anywhere else
-  # unless your C compiler or platform is otherwise broken
-  LINUX_PROC_PID_STATUS = "/proc/self/status"
-  def test_memory_leak
-    match_rss = /^VmRSS:\s+(\d+)/
-    if File.read(LINUX_PROC_PID_STATUS) =~ match_rss
-      before = $1.to_i
-      1000000.times { Unicorn::HttpParser.new }
-      File.read(LINUX_PROC_PID_STATUS) =~ match_rss
-      after = $1.to_i
-      diff = after - before
-      assert(diff < 10000, "memory grew more than 10M: #{diff}")
+  def test_memsize
+    require 'objspace'
+    if ObjectSpace.respond_to?(:memsize_of)
+      n = ObjectSpace.memsize_of(Unicorn::HttpParser.new)
+      assert_kind_of Integer, n
+      # need to update this when 128-bit machines come out
+      # n.b. actual struct size on 64-bit is 56 bytes + 40 bytes for RVALUE
+      # Ruby <= 2.2 objspace did not count the 40-byte RVALUE, 2.3 does.
+      assert_operator n, :<=, 96
+      assert_operator n, :>, 0
     end
-  end if RUBY_PLATFORM =~ /linux/ &&
-         File.readable?(LINUX_PROC_PID_STATUS) &&
-         !defined?(RUBY_ENGINE)
+  rescue LoadError
+    # not all Ruby implementations have objspace
+  end
 
+  def test_dedupe
+    parser = HttpParser.new
+    # n.b. String#freeze optimization doesn't work under modern test-unit
+    exp = -'HTTP_HOST'
+    get = "GET / HTTP/1.1\r\nHost: example.com\r\nHavpbea-fhpxf: true\r\n\r\n"
+    assert parser.add_parse(get)
+    key = parser.env.keys.detect { |k| k == exp }
+    assert_same exp, key
+
+    if RUBY_VERSION.to_r >= 2.6 # 2.6.0-rc1+
+      exp = -'HTTP_HAVPBEA_FHPXF'
+      key = parser.env.keys.detect { |k| k == exp }
+      assert_same exp, key
+    end
+  end if RUBY_VERSION.to_r >= 2.5 && RUBY_ENGINE == 'ruby'
 end