glebtv-httpclient 3.1.1 → 3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.coveralls.yml +1 -0
- data/.gitignore +8 -0
- data/.ruby-gemset +1 -0
- data/.ruby-version +1 -0
- data/.travis.yml +6 -0
- data/CHANGELOG.rdoc +653 -0
- data/Gemfile +12 -0
- data/Gemfile.lock +284 -0
- data/README.md +7 -2
- data/Rakefile +22 -0
- data/bin/httpclient +1 -1
- data/dist_key/cacerts.pem +1808 -0
- data/dist_key/cert.pem +24 -0
- data/dist_key/gen_dist_cert.rb +29 -0
- data/httpclient.gemspec +33 -0
- data/lib/httpclient.rb +59 -15
- data/lib/httpclient/lru_cache.rb +171 -0
- data/lib/httpclient/lru_threadsafe.rb +29 -0
- data/lib/httpclient/session.rb +52 -13
- data/lib/httpclient/ssl_config.rb +4 -3
- data/lib/httpclient/version.rb +1 -1
- data/sample/ssl/trust_certs/.keep_me +0 -0
- data/spec/http_message_spec.rb +124 -0
- data/spec/httpclient_spec.rb +322 -0
- data/spec/keepalive_spec.rb +129 -0
- data/spec/spec_helper.rb +40 -0
- data/spec/support/1024x768.gif +0 -0
- data/spec/support/1x1.png +0 -0
- data/spec/support/base_server.rb +36 -0
- data/spec/support/file.txt +1 -0
- data/spec/support/ht_helpers.rb +10 -0
- data/spec/support/main_server.rb +155 -0
- data/spec/support/proxy_server.rb +14 -0
- data/spec/support/test_servlet.rb +73 -0
- data/stress-test/Gemfile +4 -0
- data/stress-test/Gemfile.lock +16 -0
- data/stress-test/client.rb +72 -0
- data/stress-test/config.ru +4 -0
- data/stress-test/unicorn.conf +2 -0
- data/test.rb +19 -0
- data/test/helper.rb +4 -3
- data/test/test_httpclient.rb +19 -677
- metadata +226 -38
- data/lib/httpclient/timeout.rb +0 -140
- data/test/runner.rb +0 -2
@@ -0,0 +1,129 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
require 'spec_helper'
|
3
|
+
|
4
|
+
def create_keepalive_disconnected_thread(idx, sock)
|
5
|
+
Thread.new {
|
6
|
+
# return "12345" for the first connection
|
7
|
+
sock.gets
|
8
|
+
sock.gets
|
9
|
+
sock.write("HTTP/1.1 200 OK\r\n")
|
10
|
+
sock.write("Content-Length: 5\r\n")
|
11
|
+
sock.write("\r\n")
|
12
|
+
sock.write("12345")
|
13
|
+
# for the next connection, close while reading the request for emulating
|
14
|
+
# KeepAliveDisconnected
|
15
|
+
sock.gets
|
16
|
+
sock.close
|
17
|
+
}
|
18
|
+
end
|
19
|
+
|
20
|
+
def create_keepalive_thread(count, sock)
|
21
|
+
Thread.new {
|
22
|
+
Thread.abort_on_exception = true
|
23
|
+
count.times do
|
24
|
+
req = sock.gets
|
25
|
+
while line = sock.gets
|
26
|
+
break if line.chomp.empty?
|
27
|
+
end
|
28
|
+
case req
|
29
|
+
when /chunked/
|
30
|
+
sock.write("HTTP/1.1 200 OK\r\n")
|
31
|
+
sock.write("Transfer-Encoding: chunked\r\n")
|
32
|
+
sock.write("\r\n")
|
33
|
+
sock.write("1a\r\n")
|
34
|
+
sock.write("abcdefghijklmnopqrstuvwxyz\r\n")
|
35
|
+
sock.write("10\r\n")
|
36
|
+
sock.write("1234567890abcdef\r\n")
|
37
|
+
sock.write("0\r\n")
|
38
|
+
sock.write("\r\n")
|
39
|
+
else
|
40
|
+
sock.write("HTTP/1.1 200 OK\r\n")
|
41
|
+
sock.write("Content-Length: 5\r\n")
|
42
|
+
sock.write("\r\n")
|
43
|
+
sock.write("12345")
|
44
|
+
end
|
45
|
+
end
|
46
|
+
sock.close
|
47
|
+
}
|
48
|
+
end
|
49
|
+
|
50
|
+
describe 'KeepAlive' do
|
51
|
+
it 'disconnected' do
|
52
|
+
client = HTTPClient.new
|
53
|
+
server = TCPServer.open('127.0.0.1', 0)
|
54
|
+
server.listen(30) # set enough backlogs
|
55
|
+
endpoint = "http://127.0.0.1:#{server.addr[1]}/"
|
56
|
+
Thread.new {
|
57
|
+
Thread.abort_on_exception = true
|
58
|
+
# emulate 10 keep-alive connections
|
59
|
+
10.times do |idx|
|
60
|
+
sock = server.accept
|
61
|
+
create_keepalive_disconnected_thread(idx, sock)
|
62
|
+
end
|
63
|
+
# return "23456" for the request which gets KeepAliveDisconnected
|
64
|
+
5.times do
|
65
|
+
sock = server.accept
|
66
|
+
sock.gets
|
67
|
+
sock.gets
|
68
|
+
sock.write("HTTP/1.1 200 OK\r\n")
|
69
|
+
sock.write("\r\n")
|
70
|
+
sock.write("23456")
|
71
|
+
sock.close
|
72
|
+
end
|
73
|
+
# return "34567" for the rest requests
|
74
|
+
while true
|
75
|
+
sock = server.accept
|
76
|
+
sock.gets
|
77
|
+
sock.gets
|
78
|
+
sock.write("HTTP/1.1 200 OK\r\n")
|
79
|
+
sock.write("Connection: close\r\n")
|
80
|
+
sock.write("Content-Length: 5\r\n")
|
81
|
+
sock.write("\r\n")
|
82
|
+
sock.write("34567")
|
83
|
+
sock.close
|
84
|
+
end
|
85
|
+
}
|
86
|
+
# allocate 10 keep-alive connections
|
87
|
+
(0...10).to_a.map {
|
88
|
+
Thread.new {
|
89
|
+
Thread.abort_on_exception = true
|
90
|
+
client.get(endpoint).content.should eq "12345"
|
91
|
+
}
|
92
|
+
}.each { |th| th.join }
|
93
|
+
# send 5 requests, which should get KeepAliveDesconnected.
|
94
|
+
# doing these requests, rest keep-alive connections are invalidated.
|
95
|
+
(0...5).to_a.map {
|
96
|
+
Thread.new {
|
97
|
+
Thread.abort_on_exception = true
|
98
|
+
client.get(endpoint).content.should eq "23456"
|
99
|
+
}
|
100
|
+
}.each { |th| th.join }
|
101
|
+
# rest requests won't get KeepAliveDisconnected; how can I check this?
|
102
|
+
(0...10).to_a.map {
|
103
|
+
Thread.new {
|
104
|
+
Thread.abort_on_exception = true
|
105
|
+
client.get(endpoint).content.should eq "34567"
|
106
|
+
}
|
107
|
+
}.each { |th| th.join }
|
108
|
+
end
|
109
|
+
|
110
|
+
it 'works' do
|
111
|
+
client = HTTPClient.new
|
112
|
+
server = TCPServer.open('localhost', 0)
|
113
|
+
server_thread = Thread.new {
|
114
|
+
Thread.abort_on_exception = true
|
115
|
+
sock = server.accept
|
116
|
+
create_keepalive_thread(10, sock)
|
117
|
+
}
|
118
|
+
url = "http://localhost:#{server.addr[1]}/"
|
119
|
+
# content-length
|
120
|
+
5.times do
|
121
|
+
client.get(url).body.should eq '12345'
|
122
|
+
end
|
123
|
+
# chunked
|
124
|
+
5.times do
|
125
|
+
client.get(url + 'chunked').body.should eq 'abcdefghijklmnopqrstuvwxyz1234567890abcdef'
|
126
|
+
end
|
127
|
+
server_thread.join
|
128
|
+
end
|
129
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,40 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
|
3
|
+
require 'coveralls'
|
4
|
+
Coveralls.wear!
|
5
|
+
|
6
|
+
$LOAD_PATH.unshift(File.dirname(__FILE__))
|
7
|
+
$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
|
8
|
+
|
9
|
+
require "rubygems"
|
10
|
+
require "rspec"
|
11
|
+
require 'digest/md5'
|
12
|
+
require 'uri'
|
13
|
+
require 'logger'
|
14
|
+
require 'stringio'
|
15
|
+
require 'webrick'
|
16
|
+
require 'webrick/httpproxy.rb'
|
17
|
+
require 'webrick/httputils'
|
18
|
+
require 'tempfile'
|
19
|
+
require 'zlib'
|
20
|
+
require 'httpclient'
|
21
|
+
|
22
|
+
require File.join(File.dirname(__FILE__), "support", "base_server.rb")
|
23
|
+
require File.join(File.dirname(__FILE__), "support", "test_servlet.rb")
|
24
|
+
|
25
|
+
SUPPORT = File.join(File.dirname(__FILE__), "support")
|
26
|
+
Dir["#{SUPPORT}/*.rb"].each { |f| require f }
|
27
|
+
|
28
|
+
GZIP_CONTENT = "\x1f\x8b\x08\x00\x1a\x96\xe0\x4c\x00\x03\xcb\x48\xcd\xc9\xc9\x07\x00\x86\xa6\x10\x36\x05\x00\x00\x00"
|
29
|
+
DEFLATE_CONTENT = "\x78\x9c\xcb\x48\xcd\xc9\xc9\x07\x00\x06\x2c\x02\x15"
|
30
|
+
GZIP_CONTENT.force_encoding('BINARY') if GZIP_CONTENT.respond_to?(:force_encoding)
|
31
|
+
DEFLATE_CONTENT.force_encoding('BINARY') if DEFLATE_CONTENT.respond_to?(:force_encoding)
|
32
|
+
|
33
|
+
LARGE_STR = '1234567890' * 100_000
|
34
|
+
|
35
|
+
RSpec.configure do |config|
|
36
|
+
config.before(:all) do
|
37
|
+
@srv = MainServer.new
|
38
|
+
@proxy = ProxyServer.new
|
39
|
+
end
|
40
|
+
end
|
Binary file
|
Binary file
|
@@ -0,0 +1,36 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
|
3
|
+
class BaseServer
|
4
|
+
attr_accessor :server, :port, :logger
|
5
|
+
|
6
|
+
def u(str = '')
|
7
|
+
"http://localhost:#{@port}/#{str}"
|
8
|
+
end
|
9
|
+
|
10
|
+
def set_logger
|
11
|
+
@io = StringIO.new
|
12
|
+
@logger = Logger.new(@proxyio)
|
13
|
+
@logger.level = Logger::Severity::DEBUG
|
14
|
+
end
|
15
|
+
|
16
|
+
def start
|
17
|
+
@port = @server.config[:Port]
|
18
|
+
@thread = start_server_thread(@server)
|
19
|
+
end
|
20
|
+
|
21
|
+
def start_server_thread(server)
|
22
|
+
t = Thread.new {
|
23
|
+
Thread.current.abort_on_exception = true
|
24
|
+
server.start
|
25
|
+
}
|
26
|
+
while server.status != :Running
|
27
|
+
Thread.pass
|
28
|
+
unless t.alive?
|
29
|
+
t.join
|
30
|
+
raise
|
31
|
+
end
|
32
|
+
end
|
33
|
+
t
|
34
|
+
end
|
35
|
+
|
36
|
+
end
|
@@ -0,0 +1 @@
|
|
1
|
+
a file for testing uploads
|
@@ -0,0 +1,155 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
|
3
|
+
class MainServer < BaseServer
|
4
|
+
def initialize
|
5
|
+
set_logger
|
6
|
+
@server = WEBrick::HTTPServer.new(
|
7
|
+
:BindAddress => "localhost",
|
8
|
+
:Logger => @logger,
|
9
|
+
:Port => 0,
|
10
|
+
:AccessLog => [],
|
11
|
+
:DocumentRoot => File.dirname(File.expand_path(__FILE__))
|
12
|
+
)
|
13
|
+
[
|
14
|
+
:hello, :sleep, :servlet_redirect, :servlet_temporary_redirect, :servlet_see_other,
|
15
|
+
:redirect1, :redirect2, :redirect3,
|
16
|
+
:redirect_self, :relative_redirect, :redirect_see_other, :chunked,
|
17
|
+
:largebody, :status, :compressed, :compressed_large, :charset, :continue,
|
18
|
+
:servlet_redirect_413, :servlet_413
|
19
|
+
].each do |sym|
|
20
|
+
@server.mount(
|
21
|
+
"/#{sym}",
|
22
|
+
WEBrick::HTTPServlet::ProcHandler.new(method("do_#{sym}").to_proc)
|
23
|
+
)
|
24
|
+
end
|
25
|
+
@server.mount('/servlet', TestServlet.new(@server))
|
26
|
+
start
|
27
|
+
end
|
28
|
+
|
29
|
+
def escape_noproxy
|
30
|
+
backup = HTTPClient::NO_PROXY_HOSTS.dup
|
31
|
+
HTTPClient::NO_PROXY_HOSTS.clear
|
32
|
+
yield
|
33
|
+
ensure
|
34
|
+
HTTPClient::NO_PROXY_HOSTS.replace(backup)
|
35
|
+
end
|
36
|
+
|
37
|
+
def do_hello(req, res)
|
38
|
+
res['content-type'] = 'text/html'
|
39
|
+
res.body = "hello"
|
40
|
+
end
|
41
|
+
|
42
|
+
def do_sleep(req, res)
|
43
|
+
sec = req.query['sec'].to_i
|
44
|
+
sleep sec
|
45
|
+
res['content-type'] = 'text/html'
|
46
|
+
res.body = "hello"
|
47
|
+
end
|
48
|
+
|
49
|
+
def do_servlet_redirect(req, res)
|
50
|
+
res.set_redirect(WEBrick::HTTPStatus::Found, u("servlet"))
|
51
|
+
end
|
52
|
+
|
53
|
+
def do_servlet_redirect_413(req, res)
|
54
|
+
res.set_redirect(WEBrick::HTTPStatus::Found, u("servlet_413"))
|
55
|
+
end
|
56
|
+
|
57
|
+
def do_servlet_413(req, res)
|
58
|
+
res.body = req.body.to_s
|
59
|
+
end
|
60
|
+
|
61
|
+
def do_servlet_temporary_redirect(req, res)
|
62
|
+
res.set_redirect(WEBrick::HTTPStatus::TemporaryRedirect, u("servlet"))
|
63
|
+
end
|
64
|
+
|
65
|
+
def do_servlet_see_other(req, res)
|
66
|
+
res.set_redirect(WEBrick::HTTPStatus::SeeOther, u("servlet"))
|
67
|
+
end
|
68
|
+
|
69
|
+
def do_redirect1(req, res)
|
70
|
+
res.set_redirect(WEBrick::HTTPStatus::MovedPermanently, u("hello"))
|
71
|
+
end
|
72
|
+
|
73
|
+
def do_redirect2(req, res)
|
74
|
+
res.set_redirect(WEBrick::HTTPStatus::TemporaryRedirect, u("redirect3"))
|
75
|
+
end
|
76
|
+
|
77
|
+
def do_redirect3(req, res)
|
78
|
+
res.set_redirect(WEBrick::HTTPStatus::Found, u("hello"))
|
79
|
+
end
|
80
|
+
|
81
|
+
def do_redirect_self(req, res)
|
82
|
+
res.set_redirect(WEBrick::HTTPStatus::Found, u("redirect_self"))
|
83
|
+
end
|
84
|
+
|
85
|
+
def do_relative_redirect(req, res)
|
86
|
+
res.set_redirect(WEBrick::HTTPStatus::Found, "hello")
|
87
|
+
end
|
88
|
+
|
89
|
+
def do_redirect_see_other(req, res)
|
90
|
+
if req.request_method == 'POST'
|
91
|
+
res.set_redirect(WEBrick::HTTPStatus::SeeOther, u("redirect_see_other")) # self
|
92
|
+
else
|
93
|
+
res.body = 'hello'
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
def do_chunked(req, res)
|
98
|
+
res.chunked = true
|
99
|
+
res['content-type'] = 'text/plain; charset=UTF-8'
|
100
|
+
piper, pipew = IO.pipe
|
101
|
+
res.body = piper
|
102
|
+
pipew << req.query['msg']
|
103
|
+
pipew.close
|
104
|
+
end
|
105
|
+
|
106
|
+
def do_largebody(req, res)
|
107
|
+
res['content-type'] = 'text/html'
|
108
|
+
res.body = "a" * 1_000_000
|
109
|
+
end
|
110
|
+
|
111
|
+
def gzip(string)
|
112
|
+
wio = StringIO.new("w")
|
113
|
+
w_gz = Zlib::GzipWriter.new(wio)
|
114
|
+
w_gz.write(string)
|
115
|
+
w_gz.close
|
116
|
+
compressed = wio.string
|
117
|
+
end
|
118
|
+
|
119
|
+
def do_compressed(req, res)
|
120
|
+
res['content-type'] = 'application/octet-stream'
|
121
|
+
if req.query['enc'] == 'gzip'
|
122
|
+
res['content-encoding'] = 'gzip'
|
123
|
+
res.body = GZIP_CONTENT
|
124
|
+
elsif req.query['enc'] == 'deflate'
|
125
|
+
res['content-encoding'] = 'deflate'
|
126
|
+
res.body = DEFLATE_CONTENT
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
def do_compressed_large(req, res)
|
131
|
+
res['content-type'] = 'application/octet-stream'
|
132
|
+
str = '1234567890' * 100_000
|
133
|
+
if req.query['enc'] == 'gzip'
|
134
|
+
res['content-encoding'] = 'gzip'
|
135
|
+
res.body = gzip(str)
|
136
|
+
elsif req.query['enc'] == 'deflate'
|
137
|
+
res['content-encoding'] = 'deflate'
|
138
|
+
res.body = Zlib::Deflate.deflate(str)
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
def do_charset(req, res)
|
143
|
+
res.body = 'あいうえお'.encode("euc-jp")
|
144
|
+
res['Content-Type'] = 'text/plain; charset=euc-jp'
|
145
|
+
end
|
146
|
+
|
147
|
+
def do_status(req, res)
|
148
|
+
res.status = req.query['status'].to_i
|
149
|
+
end
|
150
|
+
|
151
|
+
def do_continue(req, res)
|
152
|
+
req.continue
|
153
|
+
res.body = 'done!'
|
154
|
+
end
|
155
|
+
end
|
@@ -0,0 +1,73 @@
|
|
1
|
+
class TestServlet < WEBrick::HTTPServlet::AbstractServlet
|
2
|
+
def get_instance(*arg)
|
3
|
+
self
|
4
|
+
end
|
5
|
+
|
6
|
+
def do_HEAD(req, res)
|
7
|
+
res["x-head"] = 'head' # use this for test purpose only.
|
8
|
+
res["x-query"] = query_response(req)
|
9
|
+
end
|
10
|
+
|
11
|
+
def do_GET(req, res)
|
12
|
+
res.body = 'get'
|
13
|
+
res["x-query"] = query_response(req)
|
14
|
+
end
|
15
|
+
|
16
|
+
def do_POST(req, res)
|
17
|
+
res["content-type"] = "text/plain" # iso-8859-1, not US-ASCII
|
18
|
+
res.body = 'post,' + req.body.to_s
|
19
|
+
res["x-query"] = body_response(req)
|
20
|
+
end
|
21
|
+
|
22
|
+
def do_PUT(req, res)
|
23
|
+
res["x-query"] = body_response(req)
|
24
|
+
param = WEBrick::HTTPUtils.parse_query(req.body) || {}
|
25
|
+
res["x-size"] = (param['txt'] || '').size
|
26
|
+
res.body = param['txt'] || 'put'
|
27
|
+
end
|
28
|
+
|
29
|
+
def do_DELETE(req, res)
|
30
|
+
res.body = 'delete'
|
31
|
+
end
|
32
|
+
|
33
|
+
def do_OPTIONS(req, res)
|
34
|
+
# check RFC for legal response.
|
35
|
+
res.body = 'options'
|
36
|
+
end
|
37
|
+
|
38
|
+
def do_PROPFIND(req, res)
|
39
|
+
res.body = 'propfind'
|
40
|
+
end
|
41
|
+
|
42
|
+
def do_PROPPATCH(req, res)
|
43
|
+
res.body = 'proppatch'
|
44
|
+
res["x-query"] = body_response(req)
|
45
|
+
end
|
46
|
+
|
47
|
+
def do_TRACE(req, res)
|
48
|
+
# client SHOULD reflect the message received back to the client as the
|
49
|
+
# entity-body of a 200 (OK) response. [RFC2616]
|
50
|
+
res.body = 'trace'
|
51
|
+
res["x-query"] = query_response(req)
|
52
|
+
end
|
53
|
+
|
54
|
+
private
|
55
|
+
|
56
|
+
def query_response(req)
|
57
|
+
query_escape(WEBrick::HTTPUtils.parse_query(req.query_string))
|
58
|
+
end
|
59
|
+
|
60
|
+
def body_response(req)
|
61
|
+
query_escape(WEBrick::HTTPUtils.parse_query(req.body))
|
62
|
+
end
|
63
|
+
|
64
|
+
def query_escape(query)
|
65
|
+
escaped = []
|
66
|
+
query.sort_by { |k, v| k }.collect do |k, v|
|
67
|
+
v.to_ary.each do |ve|
|
68
|
+
escaped << CGI.escape(k) + '=' + CGI.escape(ve)
|
69
|
+
end
|
70
|
+
end
|
71
|
+
escaped.join('&')
|
72
|
+
end
|
73
|
+
end
|