http_spew 0.1.0

data/test/content-md5.ru

@@ -0,0 +1,21 @@
+# SHA1 checksum generator
+bs = ENV['bs'] ? ENV['bs'].to_i : 4096
+require 'digest/md5'
+use Rack::ContentLength
+app = lambda do |env|
+  digest = Digest::MD5.new
+  input = env['rack.input']
+  if buf = input.read(bs)
+    begin
+      digest.update(buf)
+    end while input.read(bs, buf)
+  end
+
+  expect = env['HTTP_CONTENT_MD5']
+  readed = [ digest.digest ].pack('m').strip
+  body = "expect=#{expect}\nreaded=#{readed}\n"
+  status = expect == readed ? 200 : 500
+
+  [ status, {'Content-Type' => 'text/plain'}, [ body ] ]
+end
+run app

data/test/helper.rb

@@ -0,0 +1,41 @@
+# -*- encoding: binary -*-
+$stderr.sync = $stdout.sync = true
+require "test/unit"
+require "digest/sha1"
+require "stringio"
+require "tmpdir"
+require "socket"
+require "rack"
+require "tempfile"
+$-w = true
+require "http_spew"
+
+def start_server(config, worker_processes = 4)
+  addr = ENV["TEST_HOST"] || "127.0.0.1"
+  sock = TCPServer.new(addr, 0)
+  port = sock.addr[1]
+  fifo = Tempfile.new("fifo")
+  fifo_path = fifo.path
+  fifo.close!
+  system("mkfifo", fifo_path) or abort "mkfifo: #$?"
+  cfg = Tempfile.new("unicorn.config")
+  cfg.puts "worker_processes #{worker_processes}"
+  cfg.puts "preload_app true"
+  cfg.puts <<EOF
+after_fork do |s,w|
+  w.nr == (s.worker_processes - 1) and File.open("#{fifo_path}", "w").close
+end
+EOF
+  cfg.flush
+  pid = fork do
+    ENV["UNICORN_FD"] = sock.fileno.to_s
+    exec "unicorn", "-l#{addr}:#{port}", "-c#{cfg.path}", config
+  end
+  File.open(fifo_path).close
+  File.unlink fifo_path
+  [ addr, port, pid ]
+end
+
+def rand_data(nr)
+  File.open("/dev/urandom", "rb") { |fp| fp.read(nr) }
+end

data/test/mirror.ru

@@ -0,0 +1,22 @@
+# SHA1 checksum generator
+bs = ENV['bs'] ? ENV['bs'].to_i : 16384
+require 'digest/sha1'
+require 'unicorn/preread_input'
+use Unicorn::PrereadInput
+class InputWrap < Struct.new(:input)
+  def each
+    buf = ""
+    while buf = input.read(0x4000, buf)
+      yield buf
+    end
+  end
+end
+
+app = lambda do |env|
+  headers = {
+    "Content-Type" => "application/octet-stream",
+    "Content-Length" => env["CONTENT_LENGTH"].to_s,
+  }
+  [ 200, headers, InputWrap.new(env["rack.input"]) ]
+end
+run app

data/test/sha1.ru

@@ -0,0 +1,18 @@
+# SHA1 checksum generator
+bs = ENV['bs'] ? ENV['bs'].to_i : 16384
+require 'digest/sha1'
+use Rack::ContentLength
+app = lambda do |env|
+  /\A100-continue\z/i =~ env['HTTP_EXPECT'] and
+    return [ 100, {}, [] ]
+  digest = Digest::SHA1.new
+  input = env['rack.input']
+  if buf = input.read(bs)
+    begin
+      digest.update(buf)
+    end while input.read(bs, buf)
+  end
+
+  [ 200, {'Content-Type' => 'text/plain'}, [ digest.hexdigest << "\n" ] ]
+end
+run app

data/test/test_content_md5.rb

@@ -0,0 +1,85 @@
+# -*- encoding: binary -*-
+require "./test/helper"
+
+class TestContentMD5 < Test::Unit::TestCase
+  def setup
+    ENV["RACK_ENV"] = "deployment" # quiet Rack 1.2.1 bug
+    @addr, @port, @srv = start_server("./test/content-md5.ru", 1)
+    @sockaddr = Socket.pack_sockaddr_in(@port, @addr)
+    @env = {
+      "REQUEST_METHOD" => "PUT",
+      "REQUEST_URI" => "/",
+      "HTTP_HOST" => "example.com",
+    }
+    @tmpfiles = []
+  end
+
+  def teardown
+    Process.kill(:QUIT, @srv)
+    Process.waitpid2(@srv)
+    @tmpfiles.each { |tmp| tmp.closed? or tmp.close! }
+  end
+
+  def test_upload_with_md5
+    str = rand_data(123) * (8 * 1021 * 13)
+    expect = [Digest::MD5.digest(str)].pack("m").strip!
+    expect = "expect=#{expect}\nreaded=#{expect}\n"
+    @env["CONTENT_LENGTH"] = str.size.to_s
+    @env["rack.input"] = StringIO.new(str)
+    input = HTTP_Spew::ContentMD5.input(@env)
+    assert_nil @env["CONTENT_LENGTH"]
+    assert_equal "chunked", @env["HTTP_TRANSFER_ENCODING"]
+    req = HTTP_Spew::Request.new(@env, input, @sockaddr)
+    rv = HTTP_Spew.wait 1, [req], 666000
+    assert_equal 200, rv[0].response[0].to_i
+    body = ""
+    req.each { |buf| body << buf }
+    assert_equal body, expect
+  end
+
+  def test_upload_with_corrupt_md5
+    str = rand_data(123) * (8 * 1021 * 13)
+    expect = [Digest::MD5.digest(str)].pack("m").strip!
+    @env["HTTP_CONTENT_MD5"] = expect
+    str = rand_data(123) * (8 * 1021 * 13)
+    @env["CONTENT_LENGTH"] = str.size.to_s
+    @env["rack.input"] = StringIO.new(str)
+    input = HTTP_Spew::ContentMD5.input(@env)
+    assert_nil @env["CONTENT_LENGTH"]
+    assert_equal "chunked", @env["HTTP_TRANSFER_ENCODING"]
+    req = HTTP_Spew::Request.new(@env, input, @sockaddr)
+    rv = HTTP_Spew.wait 1, [req], 3600_000
+    assert_kind_of HTTP_Spew::ContentMD5::MismatchError, rv[0].error
+  end
+
+  def test_upload_with_corrupt_length
+    str = rand_data(123) * (8 * 1021 * 13)
+    expect = [Digest::MD5.digest(str)].pack("m").strip!
+    @env["HTTP_CONTENT_MD5"] = expect
+    str = rand_data(123) * (8 * 1021 * 13)
+    @env["CONTENT_LENGTH"] = (str.size + 1).to_s
+    @env["rack.input"] = StringIO.new(str)
+    input = HTTP_Spew::ContentMD5.input(@env)
+    assert_nil @env["CONTENT_LENGTH"]
+    assert_equal "chunked", @env["HTTP_TRANSFER_ENCODING"]
+    req = HTTP_Spew::Request.new(@env, input, @sockaddr)
+    rv = HTTP_Spew.wait 1, [req], 3600_000
+    assert_kind_of HTTP_Spew::ContentMD5::LengthError, rv[0].error
+  end
+
+  def test_upload_with_valid_md5
+    str = rand_data(123) * (8 * 1021 * 13)
+    expect = [Digest::MD5.digest(str)].pack("m").strip!
+    @env["HTTP_CONTENT_MD5"] = expect
+    @env["CONTENT_LENGTH"] = str.size.to_s
+    @env["rack.input"] = StringIO.new(str)
+    input = HTTP_Spew::ContentMD5.input(@env)
+    assert_nil @env["CONTENT_LENGTH"]
+    assert_equal "chunked", @env["HTTP_TRANSFER_ENCODING"]
+    req = HTTP_Spew::Request.new(@env, input, @sockaddr)
+    rv = HTTP_Spew.wait 1, [req], 3600_000
+    assert_equal 200, rv[0].response[0].to_i
+    body = ""
+    req.each { |buf| body << buf }
+  end
+end

data/test/test_hit_n_run.rb

@@ -0,0 +1,47 @@
+require "./test/helper"
+
+class TestHitNRun < Test::Unit::TestCase
+  def setup
+    @addr, @port, @srv = start_server("./test/sha1.ru", 1)
+    @sockaddr = Socket.pack_sockaddr_in(@port, @addr)
+    @env = {
+      "REQUEST_METHOD" => "PUT",
+      "REQUEST_URI" => "/",
+      "HTTP_HOST" => "example.com",
+    }
+    @tmpfiles = []
+  end
+
+  def teardown
+    Process.kill(:QUIT, @srv)
+    Process.waitpid2(@srv)
+    @tmpfiles.each { |tmp| tmp.closed? or tmp.close! }
+  end
+
+  def test_request_with_existing_socket
+    sock = Kgio::Socket.new(@sockaddr)
+    req = HTTP_Spew::HitNRun.new(@env, nil, sock)
+    assert_equal sock, req.to_io
+    assert_nothing_raised { req.close }
+    assert sock.closed?
+  end
+
+  def test_request_single
+    req = HTTP_Spew::HitNRun.new(@env, nil, @sockaddr)
+    sym = req.resume
+    if sym == :wait_writable
+      set = Kgio.poll({req => sym}, 100)
+      assert_equal [ req ], set.keys
+      sym = req.resume
+    end
+    assert_equal HTTP_Spew::HitNRun::RESPONSE.object_id, sym.object_id
+  end
+
+  def test_request_loop
+    req = HTTP_Spew::HitNRun.new(@env, nil, @sockaddr)
+    until Array === (rv = req.resume)
+      Kgio.poll(req => rv)
+    end
+    assert_equal HTTP_Spew::HitNRun::RESPONSE.object_id, rv.object_id
+  end
+end

data/test/test_input_spray.rb

@@ -0,0 +1,101 @@
+# -*- encoding: binary -*-
+require "./test/helper"
+
+class TestInputSpray < Test::Unit::TestCase
+  BUF = (rand_data(128) * 1024 * 8 * 4).freeze
+  EXPECT = Digest::SHA1.hexdigest(BUF).freeze
+
+  def setup
+    io = StringIO.new(BUF)
+    @env = {
+      "rack.input" => io,
+      "CONTENT_LENGTH" => io.size.to_s
+    }
+  end
+
+  def test_spray_ok
+    sprayer = HTTP_Spew::InputSpray.new(@env, 2)
+    readers = sprayer.readers
+    assert_equal 2, readers.size
+
+    sha1 = Hash[readers.map { |rd| [ rd, Digest::SHA1.new ] } ]
+
+    readers.delete_if do |rd|
+      if buf = rd.read(0x10000)
+        sha1[rd].update buf
+        false
+      else
+        rd.close.nil?
+      end
+    end until readers.empty?
+
+    sha1.each_value { |dig| assert_equal EXPECT, dig.hexdigest }
+    sprayer.instance_variable_get(:@writers).each { |x| assert x.closed? }
+  end
+
+  def test_spray_one_reader_dies
+    sprayer = HTTP_Spew::InputSpray.new(@env, 3)
+    readers = sprayer.readers
+    assert_equal 3, readers.size
+    sha1 = Hash[readers.map { |rd| [ rd, Digest::SHA1.new ] } ]
+    count = Hash[readers.map { |rd| [ rd, 0 ] } ]
+    to_die = readers[0]
+    readers.delete_if do |rd|
+      if buf = rd.read(0x100)
+        sha1[rd].update buf
+        n = count[rd] += buf.size
+        if rd == to_die && n >= 0x20000
+          rd.close.nil?
+        else
+          false
+        end
+      else
+        rd.close.nil?
+      end
+    end until readers.empty?
+
+    dead_sha1 = sha1.delete to_die
+    assert EXPECT != dead_sha1.hexdigest
+    sha1.each_value { |dig| assert_equal EXPECT, dig.hexdigest }
+    sprayer.instance_variable_get(:@writers).each { |x| assert x.closed? }
+  end
+
+  def test_spray_stream
+    rd, wr = IO.pipe
+    assert @env.delete("CONTENT_LENGTH")
+    io = @env.delete("rack.input")
+    buf = ""
+    thr = Thread.new do
+      while buf = io.read(128, buf)
+        wr.write buf
+      end
+      wr.close
+      :ok
+    end
+    @env["rack.input"] = rd
+    sprayer = HTTP_Spew::InputSpray.new(@env, 2)
+    readers = sprayer.readers
+    assert_equal 2, readers.size
+
+    sha1 = Hash[readers.map { |rd| [ rd, Digest::SHA1.new ] } ]
+
+    readers.delete_if do |rd|
+      assert ! rd.respond_to?(:size)
+      if buf = rd.read(0x10000)
+        sha1[rd].update buf
+        false
+      else
+        rd.close.nil?
+      end
+    end until readers.empty?
+
+    sha1.each_value { |dig| assert_equal EXPECT, dig.hexdigest }
+    sprayer.instance_variable_get(:@writers).each { |x| assert x.closed? }
+    thr.join
+    assert_equal :ok, thr.value
+    assert ! rd.closed?
+    assert wr.closed?
+    ensure
+      rd.close
+  end
+end

data/test/test_mirror.rb

@@ -0,0 +1,60 @@
+# -*- encoding: binary -*-
+require "./test/helper"
+
+class TestMirror < Test::Unit::TestCase
+  def setup
+    @addr, @port, @srv = start_server("./test/mirror.ru")
+    @sockaddr = Socket.pack_sockaddr_in(@port, @addr)
+    @env = {
+      "REQUEST_METHOD" => "PUT",
+      "REQUEST_URI" => "/",
+      "HTTP_HOST" => "example.com",
+    }
+  end
+
+  def teardown
+    Process.kill(:QUIT, @srv)
+    Process.waitpid2(@srv)
+  end
+
+  def test_mirror_small
+    str = rand_data(128)
+    expect = [ str ]
+    req = []
+    req << HTTP_Spew::Request.new(@env, StringIO.new(str), @sockaddr)
+    req << HTTP_Spew::Request.new(@env, StringIO.new(str), @sockaddr)
+    req << HTTP_Spew::Request.new(@env, StringIO.new(str), @sockaddr)
+    rv = HTTP_Spew.wait(3, req, 666000)
+    rv.each do |req|
+      assert_nil req.error
+      response = req.response
+      headers = Rack::Utils::HeaderHash.new(response[1])
+      assert_equal 128, headers["Content-Length"].to_i
+      assert_equal 200, response[0].to_i
+      tmp = []
+      response[2].each { |x| tmp << x.dup }
+      assert_equal expect, tmp
+    end
+  end
+
+  # no bidirectional input support
+  def test_mirror_big
+    str = rand_data(128) * (8 * 1024 * 8)
+    expect = str
+    req = []
+    req << HTTP_Spew::Request.new(@env, StringIO.new(str), @sockaddr)
+    req << HTTP_Spew::Request.new(@env, StringIO.new(str), @sockaddr)
+    req << HTTP_Spew::Request.new(@env, StringIO.new(str), @sockaddr)
+    rv = HTTP_Spew.wait(3, req, 6000)
+    rv.each do |req|
+      assert_nil req.error
+      response = req.response
+      headers = Rack::Utils::HeaderHash.new(response[1])
+      assert_equal str.size, headers["Content-Length"].to_i
+      assert_equal 200, response[0].to_i
+      tmp = ""
+      response[2].each { |x| tmp << x }
+      assert_equal expect, tmp
+    end
+  end
+end

data/test/test_request.rb

@@ -0,0 +1,54 @@
+require "./test/helper"
+
+class TestRequest < Test::Unit::TestCase
+  def setup
+    @addr, @port, @srv = start_server("./test/sha1.ru", 1)
+    @sockaddr = Socket.pack_sockaddr_in(@port, @addr)
+    @env = {
+      "REQUEST_METHOD" => "PUT",
+      "REQUEST_URI" => "/",
+      "HTTP_HOST" => "example.com",
+    }
+    @tmpfiles = []
+  end
+
+  def teardown
+    Process.kill(:QUIT, @srv)
+    Process.waitpid2(@srv)
+    @tmpfiles.each { |tmp| tmp.closed? or tmp.close! }
+  end
+
+  def test_request_with_existing_socket
+    sock = Kgio::Socket.new(@sockaddr)
+    req = HTTP_Spew::Request.new(@env, nil, sock)
+    assert_equal sock, req.to_io
+    assert_nothing_raised { req.close }
+    assert sock.closed?
+  end
+
+  def test_request_single
+    req = HTTP_Spew::Request.new(@env, nil, @sockaddr)
+    sym = req.resume
+    assert_kind_of(Symbol, sym)
+    if sym == :wait_writable
+      set = Kgio.poll({req => sym}, 100)
+      assert_equal [ req ], set.keys
+      sym = req.resume
+    end
+    assert_equal :wait_readable, sym
+    set = Kgio.poll({req => sym}, 100)
+    assert_equal [ req ], set.keys
+    rv = req.resume
+    assert_equal req, rv[2]
+  end
+
+  def test_request_loop
+    req = HTTP_Spew::Request.new(@env, nil, @sockaddr)
+    until Array === (rv = req.resume)
+      Kgio.poll(req => rv)
+    end
+    assert_kind_of Array, rv
+    assert_equal 3, rv.size
+    assert_equal req, rv[2]
+  end
+end

data/test/test_upload.rb

@@ -0,0 +1,127 @@
+# -*- encoding: binary -*-
+require "./test/helper"
+
+class TestUpload < Test::Unit::TestCase
+  def setup
+    @addr, @port, @srv = start_server("./test/sha1.ru")
+    @sockaddr = Socket.pack_sockaddr_in(@port, @addr)
+    @env = {
+      "REQUEST_METHOD" => "PUT",
+      "REQUEST_URI" => "/",
+      "HTTP_HOST" => "example.com",
+    }
+    @tmpfiles = []
+  end
+
+  def teardown
+    Process.kill(:QUIT, @srv)
+    Process.waitpid2(@srv)
+    @tmpfiles.each { |tmp| tmp.closed? or tmp.close! }
+  end
+
+  def test_spew_upload_empty
+    req = []
+    req << HTTP_Spew::Request.new(@env, nil, @sockaddr)
+    req << HTTP_Spew::Request.new(@env, nil, @sockaddr)
+    req << HTTP_Spew::Request.new(@env, nil, @sockaddr)
+    rv = HTTP_Spew.wait(3, req, 666000)
+    assert_equal 3, rv.size
+    rv.each do |req|
+      assert_nil req.error
+      response = req.response
+      assert_equal 200, response[0].to_i
+      tmp = []
+      response[2].each { |x| tmp << x.dup }
+      assert_equal [ "da39a3ee5e6b4b0d3255bfef95601890afd80709\n" ], tmp
+    end
+  end
+
+  def tmp_blob(str)
+    tmp = Tempfile.new "blob"
+    tmp.write str
+    tmp.flush
+    assert_equal str.size, tmp.size
+    tmp.rewind
+    @tmpfiles << tmp
+    tmp
+  end
+
+  def test_spew_upload_big
+    str = rand_data(128) * (8 * 1024 * 4)
+    expect = [ Digest::SHA1.hexdigest(str) << "\n" ]
+    req = []
+    req << HTTP_Spew::Request.new(@env, tmp_blob(str), @sockaddr)
+    req << HTTP_Spew::Request.new(@env, tmp_blob(str), @sockaddr)
+    req << HTTP_Spew::Request.new(@env, tmp_blob(str), @sockaddr)
+    rv = HTTP_Spew.wait(3, req, 666000)
+    assert_equal 3, rv.size
+    rv.each do |req|
+      assert_nil req.error
+      response = req.response
+      assert_equal 200, response[0].to_i
+      tmp = []
+      response[2].each { |x| tmp << x.dup }
+      assert_equal expect, tmp
+    end
+  end
+
+  def test_spew_upload_small
+    str = rand_data(128)
+    expect = [ Digest::SHA1.hexdigest(str) << "\n" ]
+    req = []
+    req << HTTP_Spew::Request.new(@env, StringIO.new(str), @sockaddr)
+    req << HTTP_Spew::Request.new(@env, StringIO.new(str), @sockaddr)
+    req << HTTP_Spew::Request.new(@env, StringIO.new(str), @sockaddr)
+    rv = HTTP_Spew.wait(3, req, 666000)
+    assert_equal 3, rv.size
+    rv.each do |req|
+      assert_nil req.error
+      response = req.response
+      assert_equal 200, response[0].to_i
+      tmp = []
+      response[2].each { |x| tmp << x.dup }
+      assert_equal expect, tmp
+    end
+  end
+
+  def test_spew_upload_small_two_of_three
+    str = rand_data(128)
+    expect = [ Digest::SHA1.hexdigest(str) << "\n" ]
+    req = []
+    req << HTTP_Spew::Request.new(@env, StringIO.new(str), @sockaddr)
+    req << HTTP_Spew::Request.new(@env, StringIO.new(str), @sockaddr)
+    req << HTTP_Spew::Request.new(@env, StringIO.new(str), @sockaddr)
+    rv = HTTP_Spew.wait(2, req, 666000)
+    assert_equal 3, rv.size
+    rv[0, 2].each do |req|
+      assert_nil req.error
+      response = req.response
+      assert_equal 200, response[0].to_i
+      tmp = []
+      response[2].each { |x| tmp << x.dup }
+      assert_equal expect, tmp
+      assert_nothing_raised { response[2].close }
+      assert req.to_io.closed?
+    end
+    failed = rv[2]
+    assert_kind_of(HTTP_Spew::ConnectionReset, failed.error)
+    assert failed.to_io.closed?
+  end
+
+  def test_spew_upload_nonblock
+    str = rand_data(128)
+    req = []
+    req << HTTP_Spew::Request.new(@env, StringIO.new(str), @sockaddr)
+    req << HTTP_Spew::Request.new(@env, StringIO.new(str), @sockaddr)
+    req << HTTP_Spew::Request.new(@env, StringIO.new(str), @sockaddr)
+    before = req.dup
+    rv = HTTP_Spew.wait_nonblock!(3, req)
+    assert_nil rv
+    while rv.nil? do
+      rv = HTTP_Spew.wait_nonblock!(3, req)
+    end
+    assert_nil rv.uniq!
+    assert rv.size > 0
+    rv.each { |req| assert before.include?(req) }
+  end
+end