http_spew 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +3 -0
- data/.manifest +6 -0
- data/ChangeLog +303 -0
- data/GIT-VERSION-FILE +1 -1
- data/GIT-VERSION-GEN +1 -1
- data/GNUmakefile +11 -0
- data/Gemfile +5 -0
- data/LATEST +5 -2
- data/NEWS +7 -0
- data/http_spew.gemspec +2 -2
- data/lib/http_spew/chunky_pipe.rb +18 -4
- data/lib/http_spew/class_methods.rb +125 -0
- data/lib/http_spew/content_md5.rb +36 -26
- data/lib/http_spew/headers.rb +11 -0
- data/lib/http_spew/hit_n_run.rb +2 -0
- data/lib/http_spew/input_spray.rb +19 -27
- data/lib/http_spew/request.rb +65 -11
- data/lib/http_spew/version.rb +2 -0
- data/lib/http_spew.rb +13 -83
- data/pkg.mk +4 -0
- data/test/content-md5.ru +3 -1
- data/test/helper.rb +4 -1
- data/test/response_code.ru +20 -0
- data/test/test_content_md5.rb +14 -13
- data/test/test_input_spray.rb +12 -13
- data/test/test_input_spray_with_md5.rb +107 -0
- data/test/test_mirror.rb +7 -7
- data/test/test_request.rb +2 -2
- data/test/test_unexpected_response.rb +47 -0
- data/test/test_upload.rb +21 -19
- metadata +20 -13
data/lib/http_spew/hit_n_run.rb
CHANGED
@@ -3,33 +3,25 @@
|
|
3
3
|
# Use this to wrap and replace your input object for spraying to multiple
|
4
4
|
# servers.
|
5
5
|
class HTTP_Spew::InputSpray
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
class SizedPipe < HTTP_Spew::ChunkyPipe
|
11
|
-
attr_accessor :size
|
12
|
-
end
|
13
|
-
|
14
|
-
def initialize(env, nr)
|
15
|
-
@input = env["rack.input"]
|
16
|
-
size = @input.respond_to?(:size) ? @input.size : env["CONTENT_LENGTH"]
|
17
|
-
size = size ? size.to_i : nil
|
18
|
-
klass = size ? SizedPipe : HTTP_Spew::ChunkyPipe
|
19
|
-
@readers, @writers = [], []
|
6
|
+
def initialize(env, nr, input = env["rack.input"])
|
7
|
+
@input = input
|
8
|
+
@pipes = {}.compare_by_identity
|
20
9
|
nr.times do
|
21
|
-
r, w =
|
22
|
-
r
|
23
|
-
@readers << r
|
24
|
-
@writers << w
|
10
|
+
r, w = HTTP_Spew::ChunkyPipe.new
|
11
|
+
@pipes[r] = w
|
25
12
|
end
|
26
|
-
|
13
|
+
start_write_driver
|
14
|
+
end
|
15
|
+
|
16
|
+
def readers
|
17
|
+
@pipes.keys
|
27
18
|
end
|
28
19
|
|
29
|
-
def write_fail?(wr, buf)
|
30
|
-
wr.
|
20
|
+
def write_fail?(rd, wr, buf)
|
21
|
+
wr.write(buf)
|
31
22
|
false
|
32
|
-
rescue
|
23
|
+
rescue => e
|
24
|
+
rd.error = e
|
33
25
|
wr.close
|
34
26
|
true
|
35
27
|
end
|
@@ -39,14 +31,14 @@ class HTTP_Spew::InputSpray
|
|
39
31
|
Thread.new do
|
40
32
|
begin
|
41
33
|
buf = ""
|
42
|
-
while
|
43
|
-
@
|
44
|
-
raise NoWritersError, "all writers have died"
|
34
|
+
while @input.read(0x4000, buf)
|
35
|
+
@pipes.delete_if { |rd, wr| write_fail?(rd, wr, buf) }.empty? and
|
36
|
+
raise HTTP_Spew::NoWritersError, "all writers have died", []
|
45
37
|
end
|
46
38
|
rescue => e
|
47
|
-
@
|
39
|
+
@pipes.each { |rd, _| rd.error = e }
|
48
40
|
ensure
|
49
|
-
@
|
41
|
+
@pipes.each { |_, wr| wr.close unless wr.closed? }
|
50
42
|
end
|
51
43
|
end
|
52
44
|
end
|
data/lib/http_spew/request.rb
CHANGED
@@ -1,20 +1,39 @@
|
|
1
1
|
# -*- encoding: binary -*-
|
2
|
+
require "io/wait"
|
3
|
+
|
4
|
+
# This is the base class actually capable of making a normal HTTP request
|
2
5
|
class HTTP_Spew::Request
|
6
|
+
|
7
|
+
# May be called by some Rack servers such as Rainbows! to bypass
|
8
|
+
# +to_path+ calls and avoid path lookups.
|
3
9
|
attr_reader :to_io
|
10
|
+
|
11
|
+
# Stores any exception that was raised in another thread (e.g.
|
12
|
+
# ContentMD5 or InputSpray write drivers).
|
4
13
|
attr_reader :error
|
14
|
+
|
15
|
+
# Stores the Rack response (a 3-element Array) on success
|
5
16
|
attr_reader :response
|
6
|
-
class RequestError < HTTP_Spew::Error
|
7
|
-
end
|
8
17
|
|
9
18
|
include HTTP_Spew::Headers
|
10
19
|
|
11
|
-
|
20
|
+
# Creates a new Request based on a Rack +env+ and +input+ object and
|
21
|
+
# prepares it for writing to +sock+. +input+ supercedes env["rack.input"]
|
22
|
+
# since it may be an alternate IO object (such as one filtered through
|
23
|
+
# HTTP_Spew::ContentMD5.
|
24
|
+
#
|
25
|
+
# +sock+ may be the String representing an address created with
|
26
|
+
# +Socket.pack_sockaddr_un+ or +Socket.pack_sockaddr_in+, or it
|
27
|
+
# may be an actual IO object with Kgio::SocketMethods mixed in
|
28
|
+
# (e.g. Kgio::Socket)
|
29
|
+
def initialize(env, input, sock, allow = nil)
|
12
30
|
@to_io = Kgio::SocketMethods === sock ? sock : Kgio::Socket.start(sock)
|
13
31
|
if Hash === env
|
14
32
|
@buf, @input = env_to_headers(env, input)
|
15
33
|
else
|
16
34
|
@buf, @input = env, input
|
17
35
|
end
|
36
|
+
@allow = allow
|
18
37
|
end
|
19
38
|
|
20
39
|
# returns a 3-element Rack response array on completion
|
@@ -35,43 +54,78 @@ class HTTP_Spew::Request
|
|
35
54
|
end
|
36
55
|
end
|
37
56
|
|
57
|
+
# returns a 3-element Rack response array on successful completion
|
58
|
+
# returns an Exception if one was raised
|
59
|
+
def run(timeout)
|
60
|
+
t0 = Time.now
|
61
|
+
buf, @buf = @buf, nil # make inspect nicer
|
62
|
+
@to_io.write(buf)
|
63
|
+
if @input
|
64
|
+
@to_io.write(buf) while @input.read(0x4000, buf)
|
65
|
+
end
|
66
|
+
timeout -= (Time.now - t0)
|
67
|
+
while :wait_readable == (rv = read_response) && timeout >= 0.0
|
68
|
+
t0 = Time.now
|
69
|
+
@to_io.wait(timeout) if timeout > 0.0
|
70
|
+
timeout -= (Time.now - t0)
|
71
|
+
end
|
72
|
+
rv
|
73
|
+
rescue => e
|
74
|
+
@input.respond_to?(:close) and @input.close rescue nil
|
75
|
+
self.error = e
|
76
|
+
end
|
77
|
+
|
78
|
+
# returns a 3-element Rack response array on completion
|
79
|
+
# returns :wait_readable or :wait_writable if busy
|
80
|
+
# Users do not need to call this directly, +resume+ will return the result
|
81
|
+
# of this.
|
38
82
|
def read_response
|
39
|
-
buf = @to_io.kgio_trypeek(0x4000) or
|
83
|
+
buf = @to_io.kgio_trypeek(0x4000) or
|
84
|
+
raise HttpSpew::EOF, "upstream server closed connection", []
|
40
85
|
String === buf or return buf
|
41
86
|
|
42
87
|
# Kcar::Parser#headers shortens +buf+ for us
|
43
88
|
hdr_len = buf.size
|
44
89
|
r = Kcar::Parser.new.headers({}, buf) or too_big!
|
90
|
+
if @allow && ! @allow.include?(r[0].to_i)
|
91
|
+
raise HTTP_Spew::UnexpectedResponse,
|
92
|
+
"#{r[0].to_i} not in #{@allow.inspect}", []
|
93
|
+
end
|
45
94
|
|
46
95
|
# discard the header data from the socket buffer
|
47
96
|
(hdr_len -= buf.size) > 0 and @to_io.kgio_read(hdr_len, buf)
|
48
|
-
|
49
|
-
@response = r
|
97
|
+
@response = r << self
|
50
98
|
end
|
51
99
|
|
100
|
+
# Used by some Rack-compatible servers to optimize transfers
|
101
|
+
# by using IO.copy_stream
|
52
102
|
def to_path
|
53
103
|
"/dev/fd/#{@to_io.fileno}"
|
54
104
|
end
|
55
105
|
|
56
|
-
def too_big!
|
57
|
-
raise RequestError.new(self), "response headers too large", []
|
106
|
+
def too_big! # :nodoc:
|
107
|
+
raise HTTP_Spew::RequestError.new(self), "response headers too large", []
|
58
108
|
end
|
59
109
|
|
110
|
+
# Called by Rack servers to write the response to a client
|
60
111
|
def each
|
61
112
|
buf = ""
|
62
|
-
while
|
113
|
+
while @to_io.kgio_read(0x4000, buf)
|
63
114
|
yield buf
|
64
115
|
end
|
65
116
|
end
|
66
117
|
|
118
|
+
# Used internally by various HTTP_Spew elements to report errors
|
119
|
+
# across different Threads and Fibers
|
67
120
|
def error=(exception)
|
121
|
+
@input.respond_to?(:error=) and @input.error = exception
|
68
122
|
close
|
69
123
|
@error = exception
|
70
124
|
end
|
71
125
|
|
72
|
-
#
|
126
|
+
# Called by Rack servers after writing a response to a client
|
73
127
|
def close
|
74
128
|
@to_io.close
|
75
|
-
|
129
|
+
@input = nil
|
76
130
|
end
|
77
131
|
end
|
data/lib/http_spew.rb
CHANGED
@@ -11,87 +11,17 @@ module HTTP_Spew
|
|
11
11
|
class Error < RuntimeError; end
|
12
12
|
class TimeoutError < Error; end
|
13
13
|
class ConnectionReset < Error; end
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
end
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
ready
|
28
|
-
end
|
29
|
-
|
30
|
-
# Returns an array of requests that are complete, including those
|
31
|
-
# that have errored out. Incomplete requests remain in +requests+
|
32
|
-
# If +need+ is fullfilled, it closes all incomplete requests and
|
33
|
-
# returns all requests.
|
34
|
-
def self.wait_nonblock!(need, requests)
|
35
|
-
ready, failed = [], []
|
36
|
-
requests.delete_if do |req|
|
37
|
-
begin
|
38
|
-
case req.resume
|
39
|
-
when Symbol # :wait_writable, :wait_readable
|
40
|
-
false
|
41
|
-
else
|
42
|
-
(ready << req).size == need and
|
43
|
-
return done_early(ready, failed, requests)
|
44
|
-
true
|
45
|
-
end
|
46
|
-
rescue => e
|
47
|
-
req.error = e
|
48
|
-
failed << req
|
49
|
-
end
|
50
|
-
end
|
51
|
-
ready.concat(failed).empty? ? nil : ready
|
52
|
-
end
|
53
|
-
|
54
|
-
# Returns an array of requests that are complete, including those
|
55
|
-
# that have errored out.
|
56
|
-
# If +need+ is fullfilled, it closes all incomplete requests.
|
57
|
-
def self.wait(need, requests, timeout)
|
58
|
-
ready, failed = [], []
|
59
|
-
pollset = {}
|
60
|
-
begin
|
61
|
-
requests.each do |req|
|
62
|
-
begin
|
63
|
-
case rv = req.resume
|
64
|
-
when Symbol # :wait_writable, :wait_readable
|
65
|
-
pollset[req] = rv
|
66
|
-
else
|
67
|
-
(ready << req).size == need and
|
68
|
-
return done_early(ready, failed, requests)
|
69
|
-
pollset.delete(req)
|
70
|
-
end
|
71
|
-
rescue => e
|
72
|
-
req.error = e
|
73
|
-
failed << req
|
74
|
-
pollset.delete(req)
|
75
|
-
end
|
76
|
-
end
|
77
|
-
break if pollset.empty?
|
78
|
-
|
79
|
-
t0 = Time.now
|
80
|
-
busy = pollset.keys
|
81
|
-
rv = Kgio.poll(pollset, timeout.to_i) or break
|
82
|
-
timeout -= (Time.now - t0) * 1000
|
83
|
-
rescue Errno::EINTR
|
84
|
-
timeout -= (Time.now - t0) * 1000
|
85
|
-
retry
|
86
|
-
end while timeout > 0.0 && requests = rv.keys.concat(busy).uniq!
|
87
|
-
|
88
|
-
ready.concat(failed)
|
89
|
-
unless requests.empty?
|
90
|
-
ready.concat(error_all(requests, TimeoutError.new("request timed out")))
|
91
|
-
ready.uniq!
|
92
|
-
end
|
93
|
-
ready
|
94
|
-
end
|
14
|
+
class RequestError < Error; end
|
15
|
+
class UnexpectedResponse < RequestError; end
|
16
|
+
class ChecksumError < HTTP_Spew::Error; end
|
17
|
+
class LengthError < HTTP_Spew::Error; end
|
18
|
+
class NoWritersError < HTTP_Spew::Error; end
|
19
|
+
class EOF < EOFError; end
|
20
|
+
|
21
|
+
require "http_spew/version"
|
22
|
+
require "http_spew/headers"
|
23
|
+
require "http_spew/request"
|
24
|
+
require "http_spew/class_methods"
|
25
|
+
|
26
|
+
extend HTTP_Spew::ClassMethods
|
95
27
|
end
|
96
|
-
require "http_spew/headers"
|
97
|
-
require "http_spew/request"
|
data/pkg.mk
CHANGED
@@ -167,5 +167,9 @@ doc_gz: docs = $(shell find doc -type f ! -regex '^.*\.\(gif\|jpg\|png\|gz\)$$')
|
|
167
167
|
doc_gz:
|
168
168
|
for i in $(docs); do \
|
169
169
|
gzip --rsyncable -9 < $$i > $$i.gz; touch -r $$i $$i.gz; done
|
170
|
+
check-warnings:
|
171
|
+
@(for i in $$(git ls-files '*.rb'|grep -v '^setup\.rb$$'); \
|
172
|
+
do $(RUBY) -d -W2 -c $$i; done) | grep -v '^Syntax OK$$' || :
|
170
173
|
|
171
174
|
.PHONY: all .FORCE-GIT-VERSION-FILE doc test $(test_units) manifest
|
175
|
+
.PHONY: check-warnings
|
data/test/content-md5.ru
CHANGED
@@ -2,7 +2,9 @@
|
|
2
2
|
bs = ENV['bs'] ? ENV['bs'].to_i : 4096
|
3
3
|
require 'digest/md5'
|
4
4
|
use Rack::ContentLength
|
5
|
+
use Rack::ContentType, "text/plain"
|
5
6
|
app = lambda do |env|
|
7
|
+
return exit!(5) if env["HTTP_X_FAIL"] == "true"
|
6
8
|
digest = Digest::MD5.new
|
7
9
|
input = env['rack.input']
|
8
10
|
if buf = input.read(bs)
|
@@ -16,6 +18,6 @@ app = lambda do |env|
|
|
16
18
|
body = "expect=#{expect}\nreaded=#{readed}\n"
|
17
19
|
status = expect == readed ? 200 : 500
|
18
20
|
|
19
|
-
[ status, {
|
21
|
+
[ status, {}, [ body ] ]
|
20
22
|
end
|
21
23
|
run app
|
data/test/helper.rb
CHANGED
@@ -1,5 +1,6 @@
|
|
1
1
|
# -*- encoding: binary -*-
|
2
2
|
$stderr.sync = $stdout.sync = true
|
3
|
+
Thread.abort_on_exception = true
|
3
4
|
require "test/unit"
|
4
5
|
require "digest/sha1"
|
5
6
|
require "stringio"
|
@@ -10,7 +11,8 @@ require "tempfile"
|
|
10
11
|
$-w = true
|
11
12
|
require "http_spew"
|
12
13
|
|
13
|
-
def start_server(config, worker_processes = 4)
|
14
|
+
def start_server(config, worker_processes = 4, rewindable_input = false)
|
15
|
+
ENV["RACK_ENV"] = "deployment"
|
14
16
|
addr = ENV["TEST_HOST"] || "127.0.0.1"
|
15
17
|
sock = TCPServer.new(addr, 0)
|
16
18
|
port = sock.addr[1]
|
@@ -21,6 +23,7 @@ def start_server(config, worker_processes = 4)
|
|
21
23
|
cfg = Tempfile.new("unicorn.config")
|
22
24
|
cfg.puts "worker_processes #{worker_processes}"
|
23
25
|
cfg.puts "preload_app true"
|
26
|
+
cfg.puts "rewindable_input #{rewindable_input}"
|
24
27
|
cfg.puts <<EOF
|
25
28
|
after_fork do |s,w|
|
26
29
|
w.nr == (s.worker_processes - 1) and File.open("#{fifo_path}", "w").close
|
@@ -0,0 +1,20 @@
|
|
1
|
+
# SHA1 checksum generator
|
2
|
+
bs = ENV['bs'] ? ENV['bs'].to_i : 16384
|
3
|
+
require 'digest/sha1'
|
4
|
+
use Rack::ContentLength
|
5
|
+
app = lambda do |env|
|
6
|
+
/\A100-continue\z/i =~ env['HTTP_EXPECT'] and
|
7
|
+
return [ 100, {}, [] ]
|
8
|
+
digest = Digest::SHA1.new
|
9
|
+
input = env['rack.input']
|
10
|
+
if buf = input.read(bs)
|
11
|
+
begin
|
12
|
+
digest.update(buf)
|
13
|
+
end while input.read(bs, buf)
|
14
|
+
end
|
15
|
+
code = env['HTTP_X_RESPONSE_CODE']
|
16
|
+
code = code.nil? ? 200 : code.to_i
|
17
|
+
|
18
|
+
[ code, {'Content-Type' => 'text/plain'}, [ digest.hexdigest << "\n" ] ]
|
19
|
+
end
|
20
|
+
run app
|
data/test/test_content_md5.rb
CHANGED
@@ -3,7 +3,6 @@ require "./test/helper"
|
|
3
3
|
|
4
4
|
class TestContentMD5 < Test::Unit::TestCase
|
5
5
|
def setup
|
6
|
-
ENV["RACK_ENV"] = "deployment" # quiet Rack 1.2.1 bug
|
7
6
|
@addr, @port, @srv = start_server("./test/content-md5.ru", 1)
|
8
7
|
@sockaddr = Socket.pack_sockaddr_in(@port, @addr)
|
9
8
|
@env = {
|
@@ -22,19 +21,21 @@ class TestContentMD5 < Test::Unit::TestCase
|
|
22
21
|
|
23
22
|
def test_upload_with_md5
|
24
23
|
str = rand_data(123) * (8 * 1021 * 13)
|
25
|
-
|
26
|
-
expect = "expect=#{
|
24
|
+
expect_md5 = [Digest::MD5.digest(str)].pack("m0")
|
25
|
+
expect = "expect=#{expect_md5}\nreaded=#{expect_md5}\n"
|
27
26
|
@env["CONTENT_LENGTH"] = str.size.to_s
|
28
27
|
@env["rack.input"] = StringIO.new(str)
|
29
|
-
input = HTTP_Spew::ContentMD5.
|
28
|
+
input = HTTP_Spew::ContentMD5.new(@env)
|
30
29
|
assert_nil @env["CONTENT_LENGTH"]
|
31
30
|
assert_equal "chunked", @env["HTTP_TRANSFER_ENCODING"]
|
32
31
|
req = HTTP_Spew::Request.new(@env, input, @sockaddr)
|
33
|
-
rv = HTTP_Spew.wait 1, [req],
|
32
|
+
rv = HTTP_Spew.wait 1, [req], 666
|
34
33
|
assert_equal 200, rv[0].response[0].to_i
|
35
34
|
body = ""
|
36
35
|
req.each { |buf| body << buf }
|
37
36
|
assert_equal body, expect
|
37
|
+
assert_equal expect_md5, input.content_md5
|
38
|
+
assert_equal 123 * 8 * 1021 * 13, input.bytes_digested
|
38
39
|
end
|
39
40
|
|
40
41
|
def test_upload_with_corrupt_md5
|
@@ -44,12 +45,12 @@ class TestContentMD5 < Test::Unit::TestCase
|
|
44
45
|
str = rand_data(123) * (8 * 1021 * 13)
|
45
46
|
@env["CONTENT_LENGTH"] = str.size.to_s
|
46
47
|
@env["rack.input"] = StringIO.new(str)
|
47
|
-
input = HTTP_Spew::ContentMD5.
|
48
|
+
input = HTTP_Spew::ContentMD5.new(@env)
|
48
49
|
assert_nil @env["CONTENT_LENGTH"]
|
49
50
|
assert_equal "chunked", @env["HTTP_TRANSFER_ENCODING"]
|
50
51
|
req = HTTP_Spew::Request.new(@env, input, @sockaddr)
|
51
|
-
rv = HTTP_Spew.wait 1, [req],
|
52
|
-
assert_kind_of HTTP_Spew::
|
52
|
+
rv = HTTP_Spew.wait 1, [req], 3600
|
53
|
+
assert_kind_of HTTP_Spew::ChecksumError, rv[0].error
|
53
54
|
end
|
54
55
|
|
55
56
|
def test_upload_with_corrupt_length
|
@@ -59,12 +60,12 @@ class TestContentMD5 < Test::Unit::TestCase
|
|
59
60
|
str = rand_data(123) * (8 * 1021 * 13)
|
60
61
|
@env["CONTENT_LENGTH"] = (str.size + 1).to_s
|
61
62
|
@env["rack.input"] = StringIO.new(str)
|
62
|
-
input = HTTP_Spew::ContentMD5.
|
63
|
+
input = HTTP_Spew::ContentMD5.new(@env)
|
63
64
|
assert_nil @env["CONTENT_LENGTH"]
|
64
65
|
assert_equal "chunked", @env["HTTP_TRANSFER_ENCODING"]
|
65
66
|
req = HTTP_Spew::Request.new(@env, input, @sockaddr)
|
66
|
-
rv = HTTP_Spew.wait 1, [req],
|
67
|
-
assert_kind_of HTTP_Spew::
|
67
|
+
rv = HTTP_Spew.wait 1, [req], 3600
|
68
|
+
assert_kind_of HTTP_Spew::LengthError, rv[0].error
|
68
69
|
end
|
69
70
|
|
70
71
|
def test_upload_with_valid_md5
|
@@ -73,11 +74,11 @@ class TestContentMD5 < Test::Unit::TestCase
|
|
73
74
|
@env["HTTP_CONTENT_MD5"] = expect
|
74
75
|
@env["CONTENT_LENGTH"] = str.size.to_s
|
75
76
|
@env["rack.input"] = StringIO.new(str)
|
76
|
-
input = HTTP_Spew::ContentMD5.
|
77
|
+
input = HTTP_Spew::ContentMD5.new(@env)
|
77
78
|
assert_nil @env["CONTENT_LENGTH"]
|
78
79
|
assert_equal "chunked", @env["HTTP_TRANSFER_ENCODING"]
|
79
80
|
req = HTTP_Spew::Request.new(@env, input, @sockaddr)
|
80
|
-
rv = HTTP_Spew.wait 1, [req],
|
81
|
+
rv = HTTP_Spew.wait 1, [req], 3600
|
81
82
|
assert_equal 200, rv[0].response[0].to_i
|
82
83
|
body = ""
|
83
84
|
req.each { |buf| body << buf }
|
data/test/test_input_spray.rb
CHANGED
@@ -25,12 +25,12 @@ class TestInputSpray < Test::Unit::TestCase
|
|
25
25
|
sha1[rd].update buf
|
26
26
|
false
|
27
27
|
else
|
28
|
-
|
28
|
+
true
|
29
29
|
end
|
30
30
|
end until readers.empty?
|
31
31
|
|
32
32
|
sha1.each_value { |dig| assert_equal EXPECT, dig.hexdigest }
|
33
|
-
sprayer.instance_variable_get(:@
|
33
|
+
sprayer.instance_variable_get(:@pipes).each_value { |x| assert x.closed? }
|
34
34
|
end
|
35
35
|
|
36
36
|
def test_spray_one_reader_dies
|
@@ -50,22 +50,21 @@ class TestInputSpray < Test::Unit::TestCase
|
|
50
50
|
false
|
51
51
|
end
|
52
52
|
else
|
53
|
-
|
53
|
+
true
|
54
54
|
end
|
55
55
|
end until readers.empty?
|
56
56
|
|
57
57
|
dead_sha1 = sha1.delete to_die
|
58
58
|
assert EXPECT != dead_sha1.hexdigest
|
59
59
|
sha1.each_value { |dig| assert_equal EXPECT, dig.hexdigest }
|
60
|
-
sprayer.instance_variable_get(:@
|
60
|
+
sprayer.instance_variable_get(:@pipes).each_value { |x| assert x.closed? }
|
61
61
|
end
|
62
62
|
|
63
63
|
def test_spray_stream
|
64
64
|
rd, wr = IO.pipe
|
65
65
|
assert @env.delete("CONTENT_LENGTH")
|
66
66
|
io = @env.delete("rack.input")
|
67
|
-
|
68
|
-
thr = Thread.new do
|
67
|
+
thr = Thread.new("") do |buf|
|
69
68
|
while buf = io.read(128, buf)
|
70
69
|
wr.write buf
|
71
70
|
end
|
@@ -77,20 +76,20 @@ class TestInputSpray < Test::Unit::TestCase
|
|
77
76
|
readers = sprayer.readers
|
78
77
|
assert_equal 2, readers.size
|
79
78
|
|
80
|
-
sha1 = Hash[readers.map { |
|
79
|
+
sha1 = Hash[readers.map { |x| [ x, Digest::SHA1.new ] } ]
|
81
80
|
|
82
|
-
readers.delete_if do |
|
83
|
-
assert !
|
84
|
-
if buf =
|
85
|
-
sha1[
|
81
|
+
readers.delete_if do |x|
|
82
|
+
assert ! x.respond_to?(:size)
|
83
|
+
if buf = x.read(0x10000)
|
84
|
+
sha1[x].update buf
|
86
85
|
false
|
87
86
|
else
|
88
|
-
|
87
|
+
true
|
89
88
|
end
|
90
89
|
end until readers.empty?
|
91
90
|
|
92
91
|
sha1.each_value { |dig| assert_equal EXPECT, dig.hexdigest }
|
93
|
-
sprayer.instance_variable_get(:@
|
92
|
+
sprayer.instance_variable_get(:@pipes).each_value { |x| assert x.closed? }
|
94
93
|
thr.join
|
95
94
|
assert_equal :ok, thr.value
|
96
95
|
assert ! rd.closed?
|
@@ -0,0 +1,107 @@
|
|
1
|
+
require "./test/helper"
|
2
|
+
|
3
|
+
class TestInputSprayWithMD5 < Test::Unit::TestCase
|
4
|
+
def setup
|
5
|
+
@nr = 4
|
6
|
+
@addr, @port, @srv = start_server("./test/content-md5.ru", @nr)
|
7
|
+
@sockaddr = Socket.pack_sockaddr_in(@port, @addr)
|
8
|
+
@env = {
|
9
|
+
"REQUEST_METHOD" => "PUT",
|
10
|
+
"REQUEST_URI" => "/",
|
11
|
+
"HTTP_HOST" => "example.com",
|
12
|
+
}
|
13
|
+
@tmpfiles = []
|
14
|
+
end
|
15
|
+
|
16
|
+
def teardown
|
17
|
+
Process.kill(:QUIT, @srv)
|
18
|
+
Process.waitpid2(@srv)
|
19
|
+
@tmpfiles.each { |tmp| tmp.closed? or tmp.close! }
|
20
|
+
end
|
21
|
+
|
22
|
+
def test_upload_with_valid_md5_sprayed
|
23
|
+
str = rand_data(123) * (8 * 1021 * 13)
|
24
|
+
expect_md5 = [Digest::MD5.digest(str)].pack("m0")
|
25
|
+
@env["HTTP_CONTENT_MD5"] = expect_md5
|
26
|
+
expect = "expect=#{expect_md5}\nreaded=#{expect_md5}\n"
|
27
|
+
@env["CONTENT_LENGTH"] = str.size.to_s
|
28
|
+
@env["rack.input"] = StringIO.new(str)
|
29
|
+
input = HTTP_Spew::ContentMD5.new(@env)
|
30
|
+
sprayer = HTTP_Spew::InputSpray.new(@env, @nr, input)
|
31
|
+
assert_nil @env["CONTENT_LENGTH"]
|
32
|
+
assert_equal "chunked", @env["HTTP_TRANSFER_ENCODING"]
|
33
|
+
reqs = sprayer.readers.map do |md5_input|
|
34
|
+
HTTP_Spew::Request.new(@env, md5_input, @sockaddr)
|
35
|
+
end
|
36
|
+
assert_equal @nr, reqs.size
|
37
|
+
rv = HTTP_Spew.wait_mt reqs.size, reqs, 3600
|
38
|
+
assert_equal @nr, rv.size
|
39
|
+
rv.each do |resp|
|
40
|
+
assert_equal 200, resp.response[0].to_i
|
41
|
+
body = ""
|
42
|
+
resp.response[2].each { |buf| body << buf }
|
43
|
+
assert_equal expect, body
|
44
|
+
end
|
45
|
+
assert_equal expect_md5, input.content_md5
|
46
|
+
assert_equal 123 * 8 * 1021 * 13, input.bytes_digested
|
47
|
+
end
|
48
|
+
|
49
|
+
def test_upload_with_invalid_md5_sprayed
|
50
|
+
str = rand_data(123) * (8 * 1021 * 13)
|
51
|
+
expect = [Digest::MD5.digest(str + "HI")].pack("m0")
|
52
|
+
@env["HTTP_CONTENT_MD5"] = expect
|
53
|
+
@env["CONTENT_LENGTH"] = str.size.to_s
|
54
|
+
@env["rack.input"] = StringIO.new(str)
|
55
|
+
input = HTTP_Spew::ContentMD5.new(@env)
|
56
|
+
sprayer = HTTP_Spew::InputSpray.new(@env, @nr, input)
|
57
|
+
assert_nil @env["CONTENT_LENGTH"]
|
58
|
+
assert_equal "chunked", @env["HTTP_TRANSFER_ENCODING"]
|
59
|
+
reqs = sprayer.readers.map do |md5_input|
|
60
|
+
HTTP_Spew::Request.new(@env, md5_input, @sockaddr)
|
61
|
+
end
|
62
|
+
assert_equal @nr, reqs.size
|
63
|
+
t0 = Time.now
|
64
|
+
rv = HTTP_Spew.wait_mt reqs.size, reqs, 3600
|
65
|
+
elapsed = Time.now - t0
|
66
|
+
assert(elapsed <= 30, "took too long: #{elapsed}s")
|
67
|
+
assert_equal @nr, rv.size
|
68
|
+
rv.each { |r|
|
69
|
+
assert_kind_of HTTP_Spew::ChecksumError, r.error
|
70
|
+
}
|
71
|
+
assert_nil input.content_md5
|
72
|
+
assert_nil input.bytes_digested
|
73
|
+
end
|
74
|
+
|
75
|
+
def test_upload_with_valid_md5_sprayed_one_failure
|
76
|
+
str = rand_data(123) * (8 * 1021 * 13)
|
77
|
+
expect_md5 = [Digest::MD5.digest(str)].pack("m0")
|
78
|
+
@env["HTTP_CONTENT_MD5"] = expect_md5
|
79
|
+
expect = "expect=#{expect_md5}\nreaded=#{expect_md5}\n"
|
80
|
+
@env["CONTENT_LENGTH"] = str.size.to_s
|
81
|
+
@env["rack.input"] = StringIO.new(str)
|
82
|
+
input = HTTP_Spew::ContentMD5.new(@env)
|
83
|
+
sprayer = HTTP_Spew::InputSpray.new(@env, @nr, input)
|
84
|
+
assert_nil @env["CONTENT_LENGTH"]
|
85
|
+
assert_equal "chunked", @env["HTTP_TRANSFER_ENCODING"]
|
86
|
+
reqs = []
|
87
|
+
sprayer.readers.each_with_index do |md5_input,i|
|
88
|
+
env = @env.dup
|
89
|
+
env["HTTP_X_FAIL"] = "true" if i == 0
|
90
|
+
reqs << HTTP_Spew::Request.new(env, md5_input, @sockaddr)
|
91
|
+
end
|
92
|
+
assert_equal @nr, reqs.size
|
93
|
+
rv = HTTP_Spew.wait_mt reqs.size, reqs, 3600
|
94
|
+
assert_equal @nr, rv.size
|
95
|
+
rv.each { |resp| assert reqs.include?(resp) }
|
96
|
+
assert reqs[0].error
|
97
|
+
(1..3).each do |n|
|
98
|
+
resp = reqs[n]
|
99
|
+
assert_equal 200, resp.response[0].to_i
|
100
|
+
body = ""
|
101
|
+
resp.response[2].each { |buf| body << buf }
|
102
|
+
assert_equal expect, body
|
103
|
+
end
|
104
|
+
assert_equal expect_md5, input.content_md5
|
105
|
+
assert_equal 123 * 8 * 1021 * 13, input.bytes_digested
|
106
|
+
end
|
107
|
+
end
|
data/test/test_mirror.rb
CHANGED
@@ -3,7 +3,7 @@ require "./test/helper"
|
|
3
3
|
|
4
4
|
class TestMirror < Test::Unit::TestCase
|
5
5
|
def setup
|
6
|
-
@addr, @port, @srv = start_server("./test/mirror.ru")
|
6
|
+
@addr, @port, @srv = start_server("./test/mirror.ru", 4 , true)
|
7
7
|
@sockaddr = Socket.pack_sockaddr_in(@port, @addr)
|
8
8
|
@env = {
|
9
9
|
"REQUEST_METHOD" => "PUT",
|
@@ -25,9 +25,9 @@ class TestMirror < Test::Unit::TestCase
|
|
25
25
|
req << HTTP_Spew::Request.new(@env, StringIO.new(str), @sockaddr)
|
26
26
|
req << HTTP_Spew::Request.new(@env, StringIO.new(str), @sockaddr)
|
27
27
|
rv = HTTP_Spew.wait(3, req, 666000)
|
28
|
-
rv.each do |
|
29
|
-
assert_nil
|
30
|
-
response =
|
28
|
+
rv.each do |r|
|
29
|
+
assert_nil r.error
|
30
|
+
response = r.response
|
31
31
|
headers = Rack::Utils::HeaderHash.new(response[1])
|
32
32
|
assert_equal 128, headers["Content-Length"].to_i
|
33
33
|
assert_equal 200, response[0].to_i
|
@@ -46,9 +46,9 @@ class TestMirror < Test::Unit::TestCase
|
|
46
46
|
req << HTTP_Spew::Request.new(@env, StringIO.new(str), @sockaddr)
|
47
47
|
req << HTTP_Spew::Request.new(@env, StringIO.new(str), @sockaddr)
|
48
48
|
rv = HTTP_Spew.wait(3, req, 6000)
|
49
|
-
rv.each do |
|
50
|
-
assert_nil
|
51
|
-
response =
|
49
|
+
rv.each do |r|
|
50
|
+
assert_nil r.error
|
51
|
+
response = r.response
|
52
52
|
headers = Rack::Utils::HeaderHash.new(response[1])
|
53
53
|
assert_equal str.size, headers["Content-Length"].to_i
|
54
54
|
assert_equal 200, response[0].to_i
|