unicorn-shopify 4.8.2.5.23

data/test/unit/test_tee_input.rb

@@ -0,0 +1,304 @@
+# -*- encoding: binary -*-
+
+require 'test/unit'
+require 'digest/sha1'
+require 'unicorn'
+
+class TeeInput < Unicorn::TeeInput
+  attr_accessor :tmp, :len
+end
+
+class TestTeeInput < Test::Unit::TestCase
+
+  def setup
+    @rs = $/
+    @rd, @wr = Kgio::UNIXSocket.pair
+    @rd.sync = @wr.sync = true
+    @start_pid = $$
+  end
+
+  def teardown
+    return if $$ != @start_pid
+    $/ = @rs
+    @rd.close rescue nil
+    @wr.close rescue nil
+    begin
+      Process.wait
+    rescue Errno::ECHILD
+      break
+    end while true
+  end
+
+  def check_tempfiles
+    tmp = @parser.env["rack.tempfiles"]
+    assert_instance_of Array, tmp
+    assert_operator tmp.size, :>=, 1
+    assert_instance_of Unicorn::TmpIO, tmp[0]
+  end
+
+  def test_gets_long
+    r = init_request("hello", 5 + (4096 * 4 * 3) + "#$/foo#$/".size)
+    ti = TeeInput.new(@rd, r)
+    status = line = nil
+    pid = fork {
+      @rd.close
+      3.times { @wr.write("ffff" * 4096) }
+      @wr.write "#$/foo#$/"
+      @wr.close
+    }
+    @wr.close
+    line = ti.gets
+    assert_equal(4096 * 4 * 3 + 5 + $/.size, line.size)
+    assert_equal("hello" << ("ffff" * 4096 * 3) << "#$/", line)
+    line = ti.gets
+    assert_equal "foo#$/", line
+    assert_nil ti.gets
+    pid, status = Process.waitpid2(pid)
+    assert status.success?
+  end
+
+  def test_gets_short
+    r = init_request("hello", 5 + "#$/foo".size)
+    ti = TeeInput.new(@rd, r)
+    status = line = nil
+    pid = fork {
+      @rd.close
+      @wr.write "#$/foo"
+      @wr.close
+    }
+    @wr.close
+    line = ti.gets
+    assert_equal("hello#$/", line)
+    line = ti.gets
+    assert_equal "foo", line
+    assert_nil ti.gets
+    pid, status = Process.waitpid2(pid)
+    assert status.success?
+  end
+
+  def test_small_body
+    r = init_request('hello')
+    ti = TeeInput.new(@rd, r)
+    assert_equal 0, @parser.content_length
+    assert @parser.body_eof?
+    assert_equal StringIO, ti.tmp.class
+    assert_equal 0, ti.tmp.pos
+    assert_equal 5, ti.size
+    assert_equal 'hello', ti.read
+    assert_equal '', ti.read
+    assert_nil ti.read(4096)
+    assert_equal 5, ti.size
+  end
+
+  def test_read_with_buffer
+    r = init_request('hello')
+    ti = TeeInput.new(@rd, r)
+    buf = ''
+    rv = ti.read(4, buf)
+    assert_equal 'hell', rv
+    assert_equal 'hell', buf
+    assert_equal rv.object_id, buf.object_id
+    assert_equal 'o', ti.read
+    assert_equal nil, ti.read(5, buf)
+    assert_equal 0, ti.rewind
+    assert_equal 'hello', ti.read(5, buf)
+    assert_equal 'hello', buf
+  end
+
+  def test_big_body
+    r = init_request('.' * Unicorn::Const::MAX_BODY << 'a')
+    ti = TeeInput.new(@rd, r)
+    assert_equal 0, @parser.content_length
+    assert @parser.body_eof?
+    assert_kind_of File, ti.tmp
+    assert_equal 0, ti.tmp.pos
+    assert_equal Unicorn::Const::MAX_BODY + 1, ti.size
+    check_tempfiles
+  end
+
+  def test_read_in_full_if_content_length
+    a, b = 300, 3
+    r = init_request('.' * b, 300)
+    assert_equal 300, @parser.content_length
+    ti = TeeInput.new(@rd, r)
+    pid = fork {
+      @wr.write('.' * 197)
+      sleep 1 # still a *potential* race here that would make the test moot...
+      @wr.write('.' * 100)
+    }
+    assert_equal a, ti.read(a).size
+    _, status = Process.waitpid2(pid)
+    assert status.success?
+    @wr.close
+  end
+
+  def test_big_body_multi
+    r = init_request('.', Unicorn::Const::MAX_BODY + 1)
+    ti = TeeInput.new(@rd, r)
+    assert_equal Unicorn::Const::MAX_BODY, @parser.content_length
+    assert ! @parser.body_eof?
+    assert_kind_of File, ti.tmp
+    assert_equal 0, ti.tmp.pos
+    assert_equal Unicorn::Const::MAX_BODY + 1, ti.size
+    nr = Unicorn::Const::MAX_BODY / 4
+    pid = fork {
+      @rd.close
+      nr.times { @wr.write('....') }
+      @wr.close
+    }
+    @wr.close
+    assert_equal '.', ti.read(1)
+    assert_equal Unicorn::Const::MAX_BODY + 1, ti.size
+    nr.times { |x|
+      assert_equal '....', ti.read(4), "nr=#{x}"
+      assert_equal Unicorn::Const::MAX_BODY + 1, ti.size
+    }
+    assert_nil ti.read(1)
+    pid, status = Process.waitpid2(pid)
+    assert status.success?
+    check_tempfiles
+  end
+
+  def test_chunked
+    @parser = Unicorn::HttpParser.new
+    @parser.buf << "POST / HTTP/1.1\r\n" \
+                   "Host: localhost\r\n" \
+                   "Transfer-Encoding: chunked\r\n" \
+                   "\r\n"
+    assert @parser.parse
+    assert_equal "", @parser.buf
+
+    pid = fork {
+      @rd.close
+      5.times { @wr.write("5\r\nabcde\r\n") }
+      @wr.write("0\r\n\r\n")
+    }
+    @wr.close
+    ti = TeeInput.new(@rd, @parser)
+    assert_nil @parser.content_length
+    assert_nil ti.len
+    assert ! @parser.body_eof?
+    assert_equal 25, ti.size
+    assert @parser.body_eof?
+    assert_equal 25, ti.len
+    assert_equal 0, ti.tmp.pos
+    ti.rewind
+    assert_equal 0, ti.tmp.pos
+    assert_equal 'abcdeabcdeabcdeabcde', ti.read(20)
+    assert_equal 20, ti.tmp.pos
+    ti.rewind
+    assert_equal 0, ti.tmp.pos
+    assert_kind_of File, ti.tmp
+    status = nil
+    pid, status = Process.waitpid2(pid)
+    assert status.success?
+    check_tempfiles
+  end
+
+  def test_chunked_ping_pong
+    @parser = Unicorn::HttpParser.new
+    buf = @parser.buf
+    buf << "POST / HTTP/1.1\r\n" \
+           "Host: localhost\r\n" \
+           "Transfer-Encoding: chunked\r\n" \
+           "\r\n"
+    assert @parser.parse
+    assert_equal "", buf
+    chunks = %w(aa bbb cccc dddd eeee)
+    rd, wr = IO.pipe
+
+    pid = fork {
+      chunks.each do |chunk|
+        rd.read(1) == "." and
+          @wr.write("#{'%x' % [ chunk.size]}\r\n#{chunk}\r\n")
+      end
+      @wr.write("0\r\n\r\n")
+    }
+    ti = TeeInput.new(@rd, @parser)
+    assert_nil @parser.content_length
+    assert_nil ti.len
+    assert ! @parser.body_eof?
+    chunks.each do |chunk|
+      wr.write('.')
+      assert_equal chunk, ti.read(16384)
+    end
+    _, status = Process.waitpid2(pid)
+    assert status.success?
+  end
+
+  def test_chunked_with_trailer
+    @parser = Unicorn::HttpParser.new
+    buf = @parser.buf
+    buf << "POST / HTTP/1.1\r\n" \
+           "Host: localhost\r\n" \
+           "Trailer: Hello\r\n" \
+           "Transfer-Encoding: chunked\r\n" \
+           "\r\n"
+    assert @parser.parse
+    assert_equal "", buf
+
+    pid = fork {
+      @rd.close
+      5.times { @wr.write("5\r\nabcde\r\n") }
+      @wr.write("0\r\n")
+      @wr.write("Hello: World\r\n\r\n")
+    }
+    @wr.close
+    ti = TeeInput.new(@rd, @parser)
+    assert_nil @parser.content_length
+    assert_nil ti.len
+    assert ! @parser.body_eof?
+    assert_equal 25, ti.size
+    assert_equal "World", @parser.env['HTTP_HELLO']
+    pid, status = Process.waitpid2(pid)
+    assert status.success?
+  end
+
+  def test_chunked_and_size_slow
+    @parser = Unicorn::HttpParser.new
+    buf = @parser.buf
+    buf << "POST / HTTP/1.1\r\n" \
+           "Host: localhost\r\n" \
+           "Trailer: Hello\r\n" \
+           "Transfer-Encoding: chunked\r\n" \
+           "\r\n"
+    assert @parser.parse
+    assert_equal "", buf
+
+    @wr.write("9\r\nabcde")
+    ti = TeeInput.new(@rd, @parser)
+    assert_nil @parser.content_length
+    assert_equal "abcde", ti.read(9)
+    assert ! @parser.body_eof?
+    @wr.write("fghi\r\n0\r\nHello: World\r\n\r\n")
+    assert_equal 9, ti.size
+    assert_equal "fghi", ti.read(9)
+    assert_equal nil, ti.read(9)
+    assert_equal "World", @parser.env['HTTP_HELLO']
+  end
+
+  def test_gets_read_mix
+    r = init_request("hello\nasdfasdf")
+    ti = Unicorn::TeeInput.new(@rd, r)
+    assert_equal "hello\n", ti.gets
+    assert_equal "asdfasdf", ti.read(9)
+    assert_nil ti.read(9)
+  end
+
+private
+
+  def init_request(body, size = nil)
+    @parser = Unicorn::HttpParser.new
+    body = body.to_s.freeze
+    buf = @parser.buf
+    buf << "POST / HTTP/1.1\r\n" \
+           "Host: localhost\r\n" \
+           "Content-Length: #{size || body.size}\r\n" \
+           "\r\n#{body}"
+    assert @parser.parse
+    assert_equal body, buf
+    @buf = buf
+    @parser
+  end
+
+end

data/test/unit/test_upload.rb

@@ -0,0 +1,306 @@
+# -*- encoding: binary -*-
+
+# Copyright (c) 2009 Eric Wong
+require './test/test_helper'
+require 'digest/md5'
+
+include Unicorn
+
+class UploadTest < Test::Unit::TestCase
+
+  def setup
+    @addr = ENV['UNICORN_TEST_ADDR'] || '127.0.0.1'
+    @port = unused_port
+    @hdr = {'Content-Type' => 'text/plain', 'Content-Length' => '0'}
+    @bs = 4096
+    @count = 256
+    @server = nil
+
+    # we want random binary data to test 1.9 encoding-aware IO craziness
+    @random = File.open('/dev/urandom','rb')
+    @sha1 = Digest::SHA1.new
+    @sha1_app = lambda do |env|
+      input = env['rack.input']
+      resp = {}
+
+      @sha1.reset
+      while buf = input.read(@bs)
+        @sha1.update(buf)
+      end
+      resp[:sha1] = @sha1.hexdigest
+
+      # rewind and read again
+      input.rewind
+      @sha1.reset
+      while buf = input.read(@bs)
+        @sha1.update(buf)
+      end
+
+      if resp[:sha1] == @sha1.hexdigest
+        resp[:sysread_read_byte_match] = true
+      end
+
+      if expect_size = env['HTTP_X_EXPECT_SIZE']
+        if expect_size.to_i == input.size
+          resp[:expect_size_match] = true
+        end
+      end
+      resp[:size] = input.size
+      resp[:content_md5] = env['HTTP_CONTENT_MD5']
+
+      [ 200, @hdr.merge({'X-Resp' => resp.inspect}), [] ]
+    end
+  end
+
+  def teardown
+    redirect_test_io { @server.stop(false) } if @server
+    @random.close
+    reset_sig_handlers
+  end
+
+  def test_put
+    start_server(@sha1_app)
+    sock = TCPSocket.new(@addr, @port)
+    sock.syswrite("PUT / HTTP/1.0\r\nContent-Length: #{length}\r\n\r\n")
+    @count.times do |i|
+      buf = @random.sysread(@bs)
+      @sha1.update(buf)
+      sock.syswrite(buf)
+    end
+    read = sock.read.split(/\r\n/)
+    assert_equal "HTTP/1.1 200 OK", read[0]
+    resp = eval(read.grep(/^X-Resp: /).first.sub!(/X-Resp: /, ''))
+    assert_equal length, resp[:size]
+    assert_equal @sha1.hexdigest, resp[:sha1]
+  end
+
+  def test_put_content_md5
+    md5 = Digest::MD5.new
+    start_server(@sha1_app)
+    sock = TCPSocket.new(@addr, @port)
+    sock.syswrite("PUT / HTTP/1.0\r\nTransfer-Encoding: chunked\r\n" \
+                  "Trailer: Content-MD5\r\n\r\n")
+    @count.times do |i|
+      buf = @random.sysread(@bs)
+      @sha1.update(buf)
+      md5.update(buf)
+      sock.syswrite("#{'%x' % buf.size}\r\n")
+      sock.syswrite(buf << "\r\n")
+    end
+    sock.syswrite("0\r\n")
+
+    content_md5 = [ md5.digest! ].pack('m').strip.freeze
+    sock.syswrite("Content-MD5: #{content_md5}\r\n\r\n")
+    read = sock.read.split(/\r\n/)
+    assert_equal "HTTP/1.1 200 OK", read[0]
+    resp = eval(read.grep(/^X-Resp: /).first.sub!(/X-Resp: /, ''))
+    assert_equal length, resp[:size]
+    assert_equal @sha1.hexdigest, resp[:sha1]
+    assert_equal content_md5, resp[:content_md5]
+  end
+
+  def test_put_trickle_small
+    @count, @bs = 2, 128
+    start_server(@sha1_app)
+    assert_equal 256, length
+    sock = TCPSocket.new(@addr, @port)
+    hdr = "PUT / HTTP/1.0\r\nContent-Length: #{length}\r\n\r\n"
+    @count.times do
+      buf = @random.sysread(@bs)
+      @sha1.update(buf)
+      hdr << buf
+      sock.syswrite(hdr)
+      hdr = ''
+      sleep 0.6
+    end
+    read = sock.read.split(/\r\n/)
+    assert_equal "HTTP/1.1 200 OK", read[0]
+    resp = eval(read.grep(/^X-Resp: /).first.sub!(/X-Resp: /, ''))
+    assert_equal length, resp[:size]
+    assert_equal @sha1.hexdigest, resp[:sha1]
+  end
+
+  def test_put_keepalive_truncates_small_overwrite
+    start_server(@sha1_app)
+    sock = TCPSocket.new(@addr, @port)
+    to_upload = length + 1
+    sock.syswrite("PUT / HTTP/1.0\r\nContent-Length: #{to_upload}\r\n\r\n")
+    @count.times do
+      buf = @random.sysread(@bs)
+      @sha1.update(buf)
+      sock.syswrite(buf)
+    end
+    sock.syswrite('12345') # write 4 bytes more than we expected
+    @sha1.update('1')
+
+    buf = sock.readpartial(4096)
+    while buf !~ /\r\n\r\n/
+      buf << sock.readpartial(4096)
+    end
+    read = buf.split(/\r\n/)
+    assert_equal "HTTP/1.1 200 OK", read[0]
+    resp = eval(read.grep(/^X-Resp: /).first.sub!(/X-Resp: /, ''))
+    assert_equal to_upload, resp[:size]
+    assert_equal @sha1.hexdigest, resp[:sha1]
+  end
+
+  def test_put_excessive_overwrite_closed
+    tmp = Tempfile.new('overwrite_check')
+    tmp.sync = true
+    start_server(lambda { |env|
+      nr = 0
+      while buf = env['rack.input'].read(65536)
+        nr += buf.size
+      end
+      tmp.write(nr.to_s)
+      [ 200, @hdr, [] ]
+    })
+    sock = TCPSocket.new(@addr, @port)
+    buf = ' ' * @bs
+    sock.syswrite("PUT / HTTP/1.0\r\nContent-Length: #{length}\r\n\r\n")
+
+    @count.times { sock.syswrite(buf) }
+    assert_raise(Errno::ECONNRESET, Errno::EPIPE) do
+      ::Unicorn::Const::CHUNK_SIZE.times { sock.syswrite(buf) }
+    end
+    sock.gets
+    tmp.rewind
+    assert_equal length, tmp.read.to_i
+  end
+
+  # Despite reading numerous articles and inspecting the 1.9.1-p0 C
+  # source, Eric Wong will never trust that we're always handling
+  # encoding-aware IO objects correctly.  Thus this test uses shell
+  # utilities that should always operate on files/sockets on a
+  # byte-level.
+  def test_uncomfortable_with_onenine_encodings
+    # POSIX doesn't require all of these to be present on a system
+    which('curl') or return
+    which('sha1sum') or return
+    which('dd') or return
+
+    start_server(@sha1_app)
+
+    tmp = Tempfile.new('dd_dest')
+    assert(system("dd", "if=#{@random.path}", "of=#{tmp.path}",
+                        "bs=#{@bs}", "count=#{@count}"),
+           "dd #@random to #{tmp}")
+    sha1_re = %r!\b([a-f0-9]{40})\b!
+    sha1_out = `sha1sum #{tmp.path}`
+    assert $?.success?, 'sha1sum ran OK'
+
+    assert_match(sha1_re, sha1_out)
+    sha1 = sha1_re.match(sha1_out)[1]
+    resp = `curl -isSfN -T#{tmp.path} http://#@addr:#@port/`
+    assert $?.success?, 'curl ran OK'
+    assert_match(%r!\b#{sha1}\b!, resp)
+    assert_match(/sysread_read_byte_match/, resp)
+
+    # small StringIO path
+    assert(system("dd", "if=#{@random.path}", "of=#{tmp.path}",
+                        "bs=1024", "count=1"),
+           "dd #@random to #{tmp}")
+    sha1_re = %r!\b([a-f0-9]{40})\b!
+    sha1_out = `sha1sum #{tmp.path}`
+    assert $?.success?, 'sha1sum ran OK'
+
+    assert_match(sha1_re, sha1_out)
+    sha1 = sha1_re.match(sha1_out)[1]
+    resp = `curl -isSfN -T#{tmp.path} http://#@addr:#@port/`
+    assert $?.success?, 'curl ran OK'
+    assert_match(%r!\b#{sha1}\b!, resp)
+    assert_match(/sysread_read_byte_match/, resp)
+  end
+
+  def test_chunked_upload_via_curl
+    # POSIX doesn't require all of these to be present on a system
+    which('curl') or return
+    which('sha1sum') or return
+    which('dd') or return
+
+    start_server(@sha1_app)
+
+    tmp = Tempfile.new('dd_dest')
+    assert(system("dd", "if=#{@random.path}", "of=#{tmp.path}",
+                        "bs=#{@bs}", "count=#{@count}"),
+           "dd #@random to #{tmp}")
+    sha1_re = %r!\b([a-f0-9]{40})\b!
+    sha1_out = `sha1sum #{tmp.path}`
+    assert $?.success?, 'sha1sum ran OK'
+
+    assert_match(sha1_re, sha1_out)
+    sha1 = sha1_re.match(sha1_out)[1]
+    cmd = "curl -H 'X-Expect-Size: #{tmp.size}' --tcp-nodelay \
+           -isSf --no-buffer -T- " \
+          "http://#@addr:#@port/"
+    resp = Tempfile.new('resp')
+    resp.sync = true
+
+    rd, wr = IO.pipe
+    wr.sync = rd.sync = true
+    pid = fork {
+      STDIN.reopen(rd)
+      rd.close
+      wr.close
+      STDOUT.reopen(resp)
+      exec cmd
+    }
+    rd.close
+
+    tmp.rewind
+    @count.times { |i|
+      wr.write(tmp.read(@bs))
+      sleep(rand / 10) if 0 == i % 8
+    }
+    wr.close
+    pid, status = Process.waitpid2(pid)
+
+    resp.rewind
+    resp = resp.read
+    assert status.success?, 'curl ran OK'
+    assert_match(%r!\b#{sha1}\b!, resp)
+    assert_match(/sysread_read_byte_match/, resp)
+    assert_match(/expect_size_match/, resp)
+  end
+
+  def test_curl_chunked_small
+    # POSIX doesn't require all of these to be present on a system
+    which('curl') or return
+    which('sha1sum') or return
+    which('dd') or return
+
+    start_server(@sha1_app)
+
+    tmp = Tempfile.new('dd_dest')
+    # small StringIO path
+    assert(system("dd", "if=#{@random.path}", "of=#{tmp.path}",
+                        "bs=1024", "count=1"),
+           "dd #@random to #{tmp}")
+    sha1_re = %r!\b([a-f0-9]{40})\b!
+    sha1_out = `sha1sum #{tmp.path}`
+    assert $?.success?, 'sha1sum ran OK'
+
+    assert_match(sha1_re, sha1_out)
+    sha1 = sha1_re.match(sha1_out)[1]
+    resp = `curl -H 'X-Expect-Size: #{tmp.size}' --tcp-nodelay \
+            -isSf --no-buffer -T- http://#@addr:#@port/ < #{tmp.path}`
+    assert $?.success?, 'curl ran OK'
+    assert_match(%r!\b#{sha1}\b!, resp)
+    assert_match(/sysread_read_byte_match/, resp)
+    assert_match(/expect_size_match/, resp)
+  end
+
+  private
+
+  def length
+    @bs * @count
+  end
+
+  def start_server(app)
+    redirect_test_io do
+      @server = HttpServer.new(app, :listeners => [ "#{@addr}:#{@port}" ] )
+      @server.start
+    end
+  end
+
+end