unicorn 4.9.0 → 6.0.0

data/test/test_helper.rb

@@ -28,22 +28,43 @@
 require 'fileutils'
 require 'logger'
 require 'unicorn'
+require 'io/nonblock'
 
 if ENV['DEBUG']
   require 'ruby-debug'
   Debugger.start
 end
 
+unless RUBY_VERSION < '3.1'
+  warn "Unicorn was only tested against MRI up to 3.0.\n" \
+       "It might not properly work with #{RUBY_VERSION}"
+end
+
 def redirect_test_io
   orig_err = STDERR.dup
   orig_out = STDOUT.dup
-  STDERR.reopen("test_stderr.#{$$}.log", "a")
-  STDOUT.reopen("test_stdout.#{$$}.log", "a")
+  rdr_pid = $$
+  new_out = File.open("test_stdout.#$$.log", "a")
+  new_err = File.open("test_stderr.#$$.log", "a")
+  new_out.sync = new_err.sync = true
+
+  if tail = ENV['TAIL'] # "tail -F" if GNU, "tail -f" otherwise
+    require 'shellwords'
+    cmd = tail.shellsplit
+    cmd << new_out.path
+    cmd << new_err.path
+    pid = Process.spawn(*cmd, { 1 => 2, :pgroup => true })
+    sleep 0.1 # wait for tail(1) to startup
+  end
+  STDERR.reopen(new_err)
+  STDOUT.reopen(new_out)
   STDERR.sync = STDOUT.sync = true
 
   at_exit do
-    File.unlink("test_stderr.#{$$}.log") rescue nil
-    File.unlink("test_stdout.#{$$}.log") rescue nil
+    if rdr_pid == $$
+      File.unlink(new_out.path) rescue nil
+      File.unlink(new_err.path) rescue nil
+    end
   end
 
   begin
@@ -51,6 +72,7 @@ def redirect_test_io
   ensure
     STDERR.reopen(orig_err)
     STDOUT.reopen(orig_out)
+    Process.kill(:TERM, pid) if pid
   end
 end
 
@@ -265,33 +287,20 @@ def wait_for_death(pid)
   raise "PID:#{pid} never died!"
 end
 
-# executes +cmd+ and chunks its STDOUT
-def chunked_spawn(stdout, *cmd)
-  fork {
-    crd, cwr = IO.pipe
-    crd.binmode
-    cwr.binmode
-    crd.sync = cwr.sync = true
-
-    pid = fork {
-      STDOUT.reopen(cwr)
-      crd.close
-      cwr.close
-      exec(*cmd)
-    }
-    cwr.close
-    begin
-      buf = crd.readpartial(16384)
-      stdout.write("#{'%x' % buf.size}\r\n#{buf}")
-    rescue EOFError
-      stdout.write("0\r\n")
-      pid, status = Process.waitpid(pid)
-      exit status.exitstatus
-    end while true
-  }
+def reset_sig_handlers
+  %w(WINCH QUIT INT TERM USR1 USR2 HUP TTIN TTOU CHLD).each do |sig|
+    trap(sig, "DEFAULT")
+  end
 end
 
-def reset_sig_handlers
-  sigs = %w(CHLD).concat(Unicorn::HttpServer::QUEUE_SIGS)
-  sigs.each { |sig| trap(sig, "DEFAULT") }
+def tcp_socket(*args)
+  sock = TCPSocket.new(*args)
+  sock.nonblock = false
+  sock
+end
+
+def unix_socket(*args)
+  sock = UNIXSocket.new(*args)
+  sock.nonblock = false
+  sock
 end

data/test/unit/test_ccc.rb

@@ -0,0 +1,91 @@
+require 'socket'
+require 'unicorn'
+require 'io/wait'
+require 'tempfile'
+require 'test/unit'
+require './test/test_helper'
+
+class TestCccTCPI < Test::Unit::TestCase
+  def test_ccc_tcpi
+    start_pid = $$
+    host = '127.0.0.1'
+    srv = TCPServer.new(host, 0)
+    port = srv.addr[1]
+    err = Tempfile.new('unicorn_ccc')
+    rd, wr = IO.pipe
+    sleep_pipe = IO.pipe
+    pid = fork do
+      sleep_pipe[1].close
+      reqs = 0
+      rd.close
+      worker_pid = nil
+      app = lambda do |env|
+        worker_pid ||= begin
+          at_exit { wr.write(reqs.to_s) if worker_pid == $$ }
+          $$
+        end
+        reqs += 1
+
+        # will wake up when writer closes
+        sleep_pipe[0].read if env['PATH_INFO'] == '/sleep'
+
+        [ 200, [ %w(Content-Length 0),  %w(Content-Type text/plain) ], [] ]
+      end
+      ENV['UNICORN_FD'] = srv.fileno.to_s
+      opts = {
+        listeners: [ "#{host}:#{port}" ],
+        stderr_path: err.path,
+        check_client_connection: true,
+      }
+      uni = Unicorn::HttpServer.new(app, opts)
+      uni.start.join
+    end
+    wr.close
+
+    # make sure the server is running, at least
+    client = tcp_socket(host, port)
+    client.write("GET / HTTP/1.1\r\nHost: example.com\r\n\r\n")
+    assert client.wait(10), 'never got response from server'
+    res = client.read
+    assert_match %r{\AHTTP/1\.1 200}, res, 'got part of first response'
+    assert_match %r{\r\n\r\n\z}, res, 'got end of response, server is ready'
+    client.close
+
+    # start a slow request...
+    sleeper = tcp_socket(host, port)
+    sleeper.write("GET /sleep HTTP/1.1\r\nHost: example.com\r\n\r\n")
+
+    # and a bunch of aborted ones
+    nr = 100
+    nr.times do |i|
+      client = tcp_socket(host, port)
+      client.write("GET /collections/#{rand(10000)} HTTP/1.1\r\n" \
+                   "Host: example.com\r\n\r\n")
+      client.close
+    end
+    sleep_pipe[1].close # wake up the reader in the worker
+    res = sleeper.read
+    assert_match %r{\AHTTP/1\.1 200}, res, 'got part of first sleeper response'
+    assert_match %r{\r\n\r\n\z}, res, 'got end of sleeper response'
+    sleeper.close
+    kpid = pid
+    pid = nil
+    Process.kill(:QUIT, kpid)
+    _, status = Process.waitpid2(kpid)
+    assert status.success?
+    reqs = rd.read.to_i
+    warn "server got #{reqs} requests with #{nr} CCC aborted\n" if $DEBUG
+    assert_operator reqs, :<, nr
+    assert_operator reqs, :>=, 2, 'first 2 requests got through, at least'
+  ensure
+    return if start_pid != $$
+    srv.close if srv
+    if pid
+      Process.kill(:QUIT, pid)
+      _, status = Process.waitpid2(pid)
+      assert status.success?
+    end
+    err.close! if err
+    rd.close if rd
+  end
+end

data/test/unit/test_droplet.rb

@@ -4,7 +4,7 @@
 class TestDroplet < Test::Unit::TestCase
   def test_create_many_droplets
     now = Time.now.to_i
-    tmp = (0..1024).map do |i|
+    (0..1024).each do |i|
       droplet = Unicorn::Worker.new(i)
       assert droplet.respond_to?(:tick)
       assert_equal 0, droplet.tick

data/test/unit/test_http_parser.rb

@@ -230,6 +230,24 @@ def test_nasty_pound_header
     assert_equal expect, req['HTTP_X_SSL_BULLSHIT']
   end
 
+  def test_multiline_header_0d0a
+    parser = HttpParser.new
+    parser.buf << "GET / HTTP/1.0\r\n" \
+      "X-Multiline-Header: foo bar\r\n\tcha cha\r\n\tzha zha\r\n\r\n"
+    req = parser.env
+    assert_equal req, parser.parse
+    assert_equal 'foo bar cha cha zha zha', req['HTTP_X_MULTILINE_HEADER']
+  end
+
+  def test_multiline_header_0a
+    parser = HttpParser.new
+    parser.buf << "GET / HTTP/1.0\n" \
+      "X-Multiline-Header: foo bar\n\tcha cha\n\tzha zha\n\n"
+    req = parser.env
+    assert_equal req, parser.parse
+    assert_equal 'foo bar cha cha zha zha', req['HTTP_X_MULTILINE_HEADER']
+  end
+
   def test_continuation_eats_leading_spaces
     parser = HttpParser.new
     header = "GET / HTTP/1.1\r\n" \
@@ -833,22 +851,34 @@ def test_empty_header
     assert_equal '', parser.env['HTTP_HOST']
   end
 
-  # so we don't  care about the portability of this test
-  # if it doesn't leak on Linux, it won't leak anywhere else
-  # unless your C compiler or platform is otherwise broken
-  LINUX_PROC_PID_STATUS = "/proc/self/status"
-  def test_memory_leak
-    match_rss = /^VmRSS:\s+(\d+)/
-    if File.read(LINUX_PROC_PID_STATUS) =~ match_rss
-      before = $1.to_i
-      1000000.times { Unicorn::HttpParser.new }
-      File.read(LINUX_PROC_PID_STATUS) =~ match_rss
-      after = $1.to_i
-      diff = after - before
-      assert(diff < 10000, "memory grew more than 10M: #{diff}")
+  def test_memsize
+    require 'objspace'
+    if ObjectSpace.respond_to?(:memsize_of)
+      n = ObjectSpace.memsize_of(Unicorn::HttpParser.new)
+      assert_kind_of Integer, n
+      # need to update this when 128-bit machines come out
+      # n.b. actual struct size on 64-bit is 56 bytes + 40 bytes for RVALUE
+      # Ruby <= 2.2 objspace did not count the 40-byte RVALUE, 2.3 does.
+      assert_operator n, :<=, 96
+      assert_operator n, :>, 0
     end
-  end if RUBY_PLATFORM =~ /linux/ &&
-         File.readable?(LINUX_PROC_PID_STATUS) &&
-         !defined?(RUBY_ENGINE)
+  rescue LoadError
+    # not all Ruby implementations have objspace
+  end
 
+  def test_dedupe
+    parser = HttpParser.new
+    # n.b. String#freeze optimization doesn't work under modern test-unit
+    exp = -'HTTP_HOST'
+    get = "GET / HTTP/1.1\r\nHost: example.com\r\nHavpbea-fhpxf: true\r\n\r\n"
+    assert parser.add_parse(get)
+    key = parser.env.keys.detect { |k| k == exp }
+    assert_same exp, key
+
+    if RUBY_VERSION.to_r >= 2.6 # 2.6.0-rc1+
+      exp = -'HTTP_HAVPBEA_FHPXF'
+      key = parser.env.keys.detect { |k| k == exp }
+      assert_same exp, key
+    end
+  end if RUBY_VERSION.to_r >= 2.5 && RUBY_ENGINE == 'ruby'
 end

data/test/unit/test_http_parser_ng.rb

@@ -8,10 +8,29 @@
 class HttpParserNgTest < Test::Unit::TestCase
 
   def setup
-    HttpParser.keepalive_requests = HttpParser::KEEPALIVE_REQUESTS_DEFAULT
     @parser = HttpParser.new
   end
 
+  # RFC 7230 allows gzip/deflate/compress Transfer-Encoding,
+  # but "chunked" must be last if used
+  def test_is_chunked
+    [ 'chunked,chunked', 'chunked,gzip', 'chunked,gzip,chunked' ].each do |x|
+      assert_raise(HttpParserError) { HttpParser.is_chunked?(x) }
+    end
+    [ 'gzip, chunked', 'gzip,chunked', 'gzip ,chunked' ].each do |x|
+      assert HttpParser.is_chunked?(x)
+    end
+    [ 'gzip', 'xhunked', 'xchunked' ].each do |x|
+      assert !HttpParser.is_chunked?(x)
+    end
+  end
+
+  def test_parser_max_len
+    assert_raises(RangeError) do
+      HttpParser.max_header_len = 0xffffffff + 1
+    end
+  end
+
   def test_next_clear
     r = "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n"
     @parser.buf << r
@@ -29,23 +48,15 @@ def test_next_clear
     assert_equal false, @parser.response_start_sent
   end
 
-  def test_keepalive_requests_default_constant
-    assert_kind_of Integer, HttpParser::KEEPALIVE_REQUESTS_DEFAULT
-    assert HttpParser::KEEPALIVE_REQUESTS_DEFAULT >= 0
-  end
-
-  def test_keepalive_requests_setting
-    HttpParser.keepalive_requests = 0
-    assert_equal 0, HttpParser.keepalive_requests
-    HttpParser.keepalive_requests = nil
-    assert HttpParser.keepalive_requests >= 0xffffffff
-    HttpParser.keepalive_requests = 1
-    assert_equal 1, HttpParser.keepalive_requests
-    HttpParser.keepalive_requests = 666
-    assert_equal 666, HttpParser.keepalive_requests
-
-    assert_raises(TypeError) { HttpParser.keepalive_requests = "666" }
-    assert_raises(TypeError) { HttpParser.keepalive_requests = [] }
+  def test_response_start_sent
+    assert_equal false, @parser.response_start_sent, "default is false"
+    @parser.response_start_sent = true
+    assert_equal true, @parser.response_start_sent
+    @parser.response_start_sent = false
+    assert_equal false, @parser.response_start_sent
+    @parser.response_start_sent = true
+    @parser.clear
+    assert_equal false, @parser.response_start_sent
   end
 
   def test_connection_TE
@@ -71,41 +82,11 @@ def test_keepalive_requests_with_next?
       "REQUEST_METHOD" => "GET",
       "QUERY_STRING" => ""
     }.freeze
-    HttpParser::KEEPALIVE_REQUESTS_DEFAULT.times do |nr|
+    100.times do |nr|
       @parser.buf << req
       assert_equal expect, @parser.parse
       assert @parser.next?
     end
-    @parser.buf << req
-    assert_equal expect, @parser.parse
-    assert ! @parser.next?
-  end
-
-  def test_fewer_keepalive_requests_with_next?
-    HttpParser.keepalive_requests = 5
-    @parser = HttpParser.new
-    req = "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n".freeze
-    expect = {
-      "SERVER_NAME" => "example.com",
-      "HTTP_HOST" => "example.com",
-      "rack.url_scheme" => "http",
-      "REQUEST_PATH" => "/",
-      "SERVER_PROTOCOL" => "HTTP/1.1",
-      "PATH_INFO" => "/",
-      "HTTP_VERSION" => "HTTP/1.1",
-      "REQUEST_URI" => "/",
-      "SERVER_PORT" => "80",
-      "REQUEST_METHOD" => "GET",
-      "QUERY_STRING" => ""
-    }.freeze
-    5.times do |nr|
-      @parser.buf << req
-      assert_equal expect, @parser.parse
-      assert @parser.next?
-    end
-    @parser.buf << req
-    assert_equal expect, @parser.parse
-    assert ! @parser.next?
   end
 
   def test_default_keepalive_is_off
@@ -599,6 +580,73 @@ def test_invalid_content_length
     end
   end
 
+  def test_duplicate_content_length
+    str = "PUT / HTTP/1.1\r\n" \
+          "Content-Length: 1\r\n" \
+          "Content-Length: 9\r\n" \
+          "\r\n"
+    assert_raises(HttpParserError) { @parser.headers({}, str) }
+  end
+
+  def test_chunked_overrides_content_length
+    order = [ 'Transfer-Encoding: chunked', 'Content-Length: 666' ]
+    %w(a b).each do |x|
+      str = "PUT /#{x} HTTP/1.1\r\n" \
+            "#{order.join("\r\n")}" \
+            "\r\n\r\na\r\nhelloworld\r\n0\r\n\r\n"
+      order.reverse!
+      env = @parser.headers({}, str)
+      assert_nil @parser.content_length
+      assert_equal 'chunked', env['HTTP_TRANSFER_ENCODING']
+      assert_equal '666', env['CONTENT_LENGTH'],
+        'Content-Length logged so the app can log a possible client bug/attack'
+      @parser.filter_body(dst = '', str)
+      assert_equal 'helloworld', dst
+      @parser.parse # handle the non-existent trailer
+      assert @parser.next?
+    end
+  end
+
+  def test_chunked_order_good
+    str = "PUT /x HTTP/1.1\r\n" \
+          "Transfer-Encoding: gzip\r\n" \
+          "Transfer-Encoding: chunked\r\n" \
+          "\r\n"
+    env = @parser.headers({}, str)
+    assert_equal 'gzip,chunked', env['HTTP_TRANSFER_ENCODING']
+    assert_nil @parser.content_length
+
+    @parser.clear
+    str = "PUT /x HTTP/1.1\r\n" \
+          "Transfer-Encoding: gzip, chunked\r\n" \
+          "\r\n"
+    env = @parser.headers({}, str)
+    assert_equal 'gzip, chunked', env['HTTP_TRANSFER_ENCODING']
+    assert_nil @parser.content_length
+  end
+
+  def test_chunked_order_bad
+    str = "PUT /x HTTP/1.1\r\n" \
+          "Transfer-Encoding: chunked\r\n" \
+          "Transfer-Encoding: gzip\r\n" \
+          "\r\n"
+    assert_raise(HttpParserError) { @parser.headers({}, str) }
+  end
+
+  def test_double_chunked
+    str = "PUT /x HTTP/1.1\r\n" \
+          "Transfer-Encoding: chunked\r\n" \
+          "Transfer-Encoding: chunked\r\n" \
+          "\r\n"
+    assert_raise(HttpParserError) { @parser.headers({}, str) }
+
+    @parser.clear
+    str = "PUT /x HTTP/1.1\r\n" \
+          "Transfer-Encoding: chunked,chunked\r\n" \
+          "\r\n"
+    assert_raise(HttpParserError) { @parser.headers({}, str) }
+  end
+
   def test_backtrace_is_empty
     begin
       @parser.headers({}, "AAADFSFDSFD\r\n\r\n")
@@ -663,69 +711,4 @@ def test_pipelined_requests
     assert_equal expect, env2
     assert_equal "", @parser.buf
   end
-
-  def test_keepalive_requests_disabled
-    req = "GET / HTTP/1.1\r\nHost: example.com\r\n\r\n".freeze
-    expect = {
-      "SERVER_NAME" => "example.com",
-      "HTTP_HOST" => "example.com",
-      "rack.url_scheme" => "http",
-      "REQUEST_PATH" => "/",
-      "SERVER_PROTOCOL" => "HTTP/1.1",
-      "PATH_INFO" => "/",
-      "HTTP_VERSION" => "HTTP/1.1",
-      "REQUEST_URI" => "/",
-      "SERVER_PORT" => "80",
-      "REQUEST_METHOD" => "GET",
-      "QUERY_STRING" => ""
-    }.freeze
-    HttpParser.keepalive_requests = 0
-    @parser = HttpParser.new
-    @parser.buf << req
-    assert_equal expect, @parser.parse
-    assert ! @parser.next?
-  end
-
-  def test_chunk_only
-    tmp = ""
-    assert_equal @parser, @parser.dechunk!
-    assert_nil @parser.filter_body(tmp, "6\r\n")
-    assert_equal "", tmp
-    assert_nil @parser.filter_body(tmp, "abcdef")
-    assert_equal "abcdef", tmp
-    assert_nil @parser.filter_body(tmp, "\r\n")
-    assert_equal "", tmp
-    src = "0\r\n\r\n"
-    assert_equal src.object_id, @parser.filter_body(tmp, src).object_id
-    assert_equal "", tmp
-  end
-
-  def test_chunk_only_bad_align
-    tmp = ""
-    assert_equal @parser, @parser.dechunk!
-    assert_nil @parser.filter_body(tmp, "6\r\na")
-    assert_equal "a", tmp
-    assert_nil @parser.filter_body(tmp, "bcde")
-    assert_equal "bcde", tmp
-    assert_nil @parser.filter_body(tmp, "f\r")
-    assert_equal "f", tmp
-    src = "\n0\r\n\r\n"
-    assert_equal src.object_id, @parser.filter_body(tmp, src).object_id
-    assert_equal "", tmp
-  end
-
-  def test_chunk_only_reset_ok
-    tmp = ""
-    assert_equal @parser, @parser.dechunk!
-    src = "1\r\na\r\n0\r\n\r\n"
-    assert_nil @parser.filter_body(tmp, src)
-    assert_equal "a", tmp
-    assert_equal src.object_id, @parser.filter_body(tmp, src).object_id
-
-    assert_equal @parser, @parser.dechunk!
-    src = "0\r\n\r\n"
-    assert_equal src.object_id, @parser.filter_body(tmp, src).object_id
-    assert_equal "", tmp
-    assert_equal src, @parser.filter_body(tmp, src)
-  end
 end