glebtv-httpclient 3.1.1 → 3.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. checksums.yaml +4 -4
  2. data/.coveralls.yml +1 -0
  3. data/.gitignore +8 -0
  4. data/.ruby-gemset +1 -0
  5. data/.ruby-version +1 -0
  6. data/.travis.yml +6 -0
  7. data/CHANGELOG.rdoc +653 -0
  8. data/Gemfile +12 -0
  9. data/Gemfile.lock +284 -0
  10. data/README.md +7 -2
  11. data/Rakefile +22 -0
  12. data/bin/httpclient +1 -1
  13. data/dist_key/cacerts.pem +1808 -0
  14. data/dist_key/cert.pem +24 -0
  15. data/dist_key/gen_dist_cert.rb +29 -0
  16. data/httpclient.gemspec +33 -0
  17. data/lib/httpclient.rb +59 -15
  18. data/lib/httpclient/lru_cache.rb +171 -0
  19. data/lib/httpclient/lru_threadsafe.rb +29 -0
  20. data/lib/httpclient/session.rb +52 -13
  21. data/lib/httpclient/ssl_config.rb +4 -3
  22. data/lib/httpclient/version.rb +1 -1
  23. data/sample/ssl/trust_certs/.keep_me +0 -0
  24. data/spec/http_message_spec.rb +124 -0
  25. data/spec/httpclient_spec.rb +322 -0
  26. data/spec/keepalive_spec.rb +129 -0
  27. data/spec/spec_helper.rb +40 -0
  28. data/spec/support/1024x768.gif +0 -0
  29. data/spec/support/1x1.png +0 -0
  30. data/spec/support/base_server.rb +36 -0
  31. data/spec/support/file.txt +1 -0
  32. data/spec/support/ht_helpers.rb +10 -0
  33. data/spec/support/main_server.rb +155 -0
  34. data/spec/support/proxy_server.rb +14 -0
  35. data/spec/support/test_servlet.rb +73 -0
  36. data/stress-test/Gemfile +4 -0
  37. data/stress-test/Gemfile.lock +16 -0
  38. data/stress-test/client.rb +72 -0
  39. data/stress-test/config.ru +4 -0
  40. data/stress-test/unicorn.conf +2 -0
  41. data/test.rb +19 -0
  42. data/test/helper.rb +4 -3
  43. data/test/test_httpclient.rb +19 -677
  44. metadata +226 -38
  45. data/lib/httpclient/timeout.rb +0 -140
  46. data/test/runner.rb +0 -2
@@ -0,0 +1,129 @@
1
+ # coding: utf-8
2
+ require 'spec_helper'
3
+
4
+ def create_keepalive_disconnected_thread(idx, sock)
5
+ Thread.new {
6
+ # return "12345" for the first connection
7
+ sock.gets
8
+ sock.gets
9
+ sock.write("HTTP/1.1 200 OK\r\n")
10
+ sock.write("Content-Length: 5\r\n")
11
+ sock.write("\r\n")
12
+ sock.write("12345")
13
+ # for the next connection, close while reading the request for emulating
14
+ # KeepAliveDisconnected
15
+ sock.gets
16
+ sock.close
17
+ }
18
+ end
19
+
20
+ def create_keepalive_thread(count, sock)
21
+ Thread.new {
22
+ Thread.abort_on_exception = true
23
+ count.times do
24
+ req = sock.gets
25
+ while line = sock.gets
26
+ break if line.chomp.empty?
27
+ end
28
+ case req
29
+ when /chunked/
30
+ sock.write("HTTP/1.1 200 OK\r\n")
31
+ sock.write("Transfer-Encoding: chunked\r\n")
32
+ sock.write("\r\n")
33
+ sock.write("1a\r\n")
34
+ sock.write("abcdefghijklmnopqrstuvwxyz\r\n")
35
+ sock.write("10\r\n")
36
+ sock.write("1234567890abcdef\r\n")
37
+ sock.write("0\r\n")
38
+ sock.write("\r\n")
39
+ else
40
+ sock.write("HTTP/1.1 200 OK\r\n")
41
+ sock.write("Content-Length: 5\r\n")
42
+ sock.write("\r\n")
43
+ sock.write("12345")
44
+ end
45
+ end
46
+ sock.close
47
+ }
48
+ end
49
+
50
+ describe 'KeepAlive' do
51
+ it 'disconnected' do
52
+ client = HTTPClient.new
53
+ server = TCPServer.open('127.0.0.1', 0)
54
+ server.listen(30) # set enough backlogs
55
+ endpoint = "http://127.0.0.1:#{server.addr[1]}/"
56
+ Thread.new {
57
+ Thread.abort_on_exception = true
58
+ # emulate 10 keep-alive connections
59
+ 10.times do |idx|
60
+ sock = server.accept
61
+ create_keepalive_disconnected_thread(idx, sock)
62
+ end
63
+ # return "23456" for the request which gets KeepAliveDisconnected
64
+ 5.times do
65
+ sock = server.accept
66
+ sock.gets
67
+ sock.gets
68
+ sock.write("HTTP/1.1 200 OK\r\n")
69
+ sock.write("\r\n")
70
+ sock.write("23456")
71
+ sock.close
72
+ end
73
+ # return "34567" for the rest requests
74
+ while true
75
+ sock = server.accept
76
+ sock.gets
77
+ sock.gets
78
+ sock.write("HTTP/1.1 200 OK\r\n")
79
+ sock.write("Connection: close\r\n")
80
+ sock.write("Content-Length: 5\r\n")
81
+ sock.write("\r\n")
82
+ sock.write("34567")
83
+ sock.close
84
+ end
85
+ }
86
+ # allocate 10 keep-alive connections
87
+ (0...10).to_a.map {
88
+ Thread.new {
89
+ Thread.abort_on_exception = true
90
+ client.get(endpoint).content.should eq "12345"
91
+ }
92
+ }.each { |th| th.join }
93
+ # send 5 requests, which should get KeepAliveDesconnected.
94
+ # doing these requests, rest keep-alive connections are invalidated.
95
+ (0...5).to_a.map {
96
+ Thread.new {
97
+ Thread.abort_on_exception = true
98
+ client.get(endpoint).content.should eq "23456"
99
+ }
100
+ }.each { |th| th.join }
101
+ # rest requests won't get KeepAliveDisconnected; how can I check this?
102
+ (0...10).to_a.map {
103
+ Thread.new {
104
+ Thread.abort_on_exception = true
105
+ client.get(endpoint).content.should eq "34567"
106
+ }
107
+ }.each { |th| th.join }
108
+ end
109
+
110
+ it 'works' do
111
+ client = HTTPClient.new
112
+ server = TCPServer.open('localhost', 0)
113
+ server_thread = Thread.new {
114
+ Thread.abort_on_exception = true
115
+ sock = server.accept
116
+ create_keepalive_thread(10, sock)
117
+ }
118
+ url = "http://localhost:#{server.addr[1]}/"
119
+ # content-length
120
+ 5.times do
121
+ client.get(url).body.should eq '12345'
122
+ end
123
+ # chunked
124
+ 5.times do
125
+ client.get(url + 'chunked').body.should eq 'abcdefghijklmnopqrstuvwxyz1234567890abcdef'
126
+ end
127
+ server_thread.join
128
+ end
129
+ end
@@ -0,0 +1,40 @@
1
+ # coding: utf-8
2
+
3
+ require 'coveralls'
4
+ Coveralls.wear!
5
+
6
+ $LOAD_PATH.unshift(File.dirname(__FILE__))
7
+ $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
8
+
9
+ require "rubygems"
10
+ require "rspec"
11
+ require 'digest/md5'
12
+ require 'uri'
13
+ require 'logger'
14
+ require 'stringio'
15
+ require 'webrick'
16
+ require 'webrick/httpproxy.rb'
17
+ require 'webrick/httputils'
18
+ require 'tempfile'
19
+ require 'zlib'
20
+ require 'httpclient'
21
+
22
+ require File.join(File.dirname(__FILE__), "support", "base_server.rb")
23
+ require File.join(File.dirname(__FILE__), "support", "test_servlet.rb")
24
+
25
+ SUPPORT = File.join(File.dirname(__FILE__), "support")
26
+ Dir["#{SUPPORT}/*.rb"].each { |f| require f }
27
+
28
+ GZIP_CONTENT = "\x1f\x8b\x08\x00\x1a\x96\xe0\x4c\x00\x03\xcb\x48\xcd\xc9\xc9\x07\x00\x86\xa6\x10\x36\x05\x00\x00\x00"
29
+ DEFLATE_CONTENT = "\x78\x9c\xcb\x48\xcd\xc9\xc9\x07\x00\x06\x2c\x02\x15"
30
+ GZIP_CONTENT.force_encoding('BINARY') if GZIP_CONTENT.respond_to?(:force_encoding)
31
+ DEFLATE_CONTENT.force_encoding('BINARY') if DEFLATE_CONTENT.respond_to?(:force_encoding)
32
+
33
+ LARGE_STR = '1234567890' * 100_000
34
+
35
+ RSpec.configure do |config|
36
+ config.before(:all) do
37
+ @srv = MainServer.new
38
+ @proxy = ProxyServer.new
39
+ end
40
+ end
Binary file
Binary file
@@ -0,0 +1,36 @@
1
+ # coding: utf-8
2
+
3
+ class BaseServer
4
+ attr_accessor :server, :port, :logger
5
+
6
+ def u(str = '')
7
+ "http://localhost:#{@port}/#{str}"
8
+ end
9
+
10
+ def set_logger
11
+ @io = StringIO.new
12
+ @logger = Logger.new(@proxyio)
13
+ @logger.level = Logger::Severity::DEBUG
14
+ end
15
+
16
+ def start
17
+ @port = @server.config[:Port]
18
+ @thread = start_server_thread(@server)
19
+ end
20
+
21
+ def start_server_thread(server)
22
+ t = Thread.new {
23
+ Thread.current.abort_on_exception = true
24
+ server.start
25
+ }
26
+ while server.status != :Running
27
+ Thread.pass
28
+ unless t.alive?
29
+ t.join
30
+ raise
31
+ end
32
+ end
33
+ t
34
+ end
35
+
36
+ end
@@ -0,0 +1 @@
1
+ a file for testing uploads
@@ -0,0 +1,10 @@
1
+ module HtHelpers
2
+ def params(str)
3
+ HTTP::Message.parse(str).inject({}) { |r, (k, v)| r[k] = v.first; r }
4
+ end
5
+ end
6
+
7
+ RSpec.configure do |config|
8
+ config.extend HtHelpers
9
+ config.include HTTPClient::Util
10
+ end
@@ -0,0 +1,155 @@
1
+ # coding: utf-8
2
+
3
+ class MainServer < BaseServer
4
+ def initialize
5
+ set_logger
6
+ @server = WEBrick::HTTPServer.new(
7
+ :BindAddress => "localhost",
8
+ :Logger => @logger,
9
+ :Port => 0,
10
+ :AccessLog => [],
11
+ :DocumentRoot => File.dirname(File.expand_path(__FILE__))
12
+ )
13
+ [
14
+ :hello, :sleep, :servlet_redirect, :servlet_temporary_redirect, :servlet_see_other,
15
+ :redirect1, :redirect2, :redirect3,
16
+ :redirect_self, :relative_redirect, :redirect_see_other, :chunked,
17
+ :largebody, :status, :compressed, :compressed_large, :charset, :continue,
18
+ :servlet_redirect_413, :servlet_413
19
+ ].each do |sym|
20
+ @server.mount(
21
+ "/#{sym}",
22
+ WEBrick::HTTPServlet::ProcHandler.new(method("do_#{sym}").to_proc)
23
+ )
24
+ end
25
+ @server.mount('/servlet', TestServlet.new(@server))
26
+ start
27
+ end
28
+
29
+ def escape_noproxy
30
+ backup = HTTPClient::NO_PROXY_HOSTS.dup
31
+ HTTPClient::NO_PROXY_HOSTS.clear
32
+ yield
33
+ ensure
34
+ HTTPClient::NO_PROXY_HOSTS.replace(backup)
35
+ end
36
+
37
+ def do_hello(req, res)
38
+ res['content-type'] = 'text/html'
39
+ res.body = "hello"
40
+ end
41
+
42
+ def do_sleep(req, res)
43
+ sec = req.query['sec'].to_i
44
+ sleep sec
45
+ res['content-type'] = 'text/html'
46
+ res.body = "hello"
47
+ end
48
+
49
+ def do_servlet_redirect(req, res)
50
+ res.set_redirect(WEBrick::HTTPStatus::Found, u("servlet"))
51
+ end
52
+
53
+ def do_servlet_redirect_413(req, res)
54
+ res.set_redirect(WEBrick::HTTPStatus::Found, u("servlet_413"))
55
+ end
56
+
57
+ def do_servlet_413(req, res)
58
+ res.body = req.body.to_s
59
+ end
60
+
61
+ def do_servlet_temporary_redirect(req, res)
62
+ res.set_redirect(WEBrick::HTTPStatus::TemporaryRedirect, u("servlet"))
63
+ end
64
+
65
+ def do_servlet_see_other(req, res)
66
+ res.set_redirect(WEBrick::HTTPStatus::SeeOther, u("servlet"))
67
+ end
68
+
69
+ def do_redirect1(req, res)
70
+ res.set_redirect(WEBrick::HTTPStatus::MovedPermanently, u("hello"))
71
+ end
72
+
73
+ def do_redirect2(req, res)
74
+ res.set_redirect(WEBrick::HTTPStatus::TemporaryRedirect, u("redirect3"))
75
+ end
76
+
77
+ def do_redirect3(req, res)
78
+ res.set_redirect(WEBrick::HTTPStatus::Found, u("hello"))
79
+ end
80
+
81
+ def do_redirect_self(req, res)
82
+ res.set_redirect(WEBrick::HTTPStatus::Found, u("redirect_self"))
83
+ end
84
+
85
+ def do_relative_redirect(req, res)
86
+ res.set_redirect(WEBrick::HTTPStatus::Found, "hello")
87
+ end
88
+
89
+ def do_redirect_see_other(req, res)
90
+ if req.request_method == 'POST'
91
+ res.set_redirect(WEBrick::HTTPStatus::SeeOther, u("redirect_see_other")) # self
92
+ else
93
+ res.body = 'hello'
94
+ end
95
+ end
96
+
97
+ def do_chunked(req, res)
98
+ res.chunked = true
99
+ res['content-type'] = 'text/plain; charset=UTF-8'
100
+ piper, pipew = IO.pipe
101
+ res.body = piper
102
+ pipew << req.query['msg']
103
+ pipew.close
104
+ end
105
+
106
+ def do_largebody(req, res)
107
+ res['content-type'] = 'text/html'
108
+ res.body = "a" * 1_000_000
109
+ end
110
+
111
+ def gzip(string)
112
+ wio = StringIO.new("w")
113
+ w_gz = Zlib::GzipWriter.new(wio)
114
+ w_gz.write(string)
115
+ w_gz.close
116
+ compressed = wio.string
117
+ end
118
+
119
+ def do_compressed(req, res)
120
+ res['content-type'] = 'application/octet-stream'
121
+ if req.query['enc'] == 'gzip'
122
+ res['content-encoding'] = 'gzip'
123
+ res.body = GZIP_CONTENT
124
+ elsif req.query['enc'] == 'deflate'
125
+ res['content-encoding'] = 'deflate'
126
+ res.body = DEFLATE_CONTENT
127
+ end
128
+ end
129
+
130
+ def do_compressed_large(req, res)
131
+ res['content-type'] = 'application/octet-stream'
132
+ str = '1234567890' * 100_000
133
+ if req.query['enc'] == 'gzip'
134
+ res['content-encoding'] = 'gzip'
135
+ res.body = gzip(str)
136
+ elsif req.query['enc'] == 'deflate'
137
+ res['content-encoding'] = 'deflate'
138
+ res.body = Zlib::Deflate.deflate(str)
139
+ end
140
+ end
141
+
142
+ def do_charset(req, res)
143
+ res.body = 'あいうえお'.encode("euc-jp")
144
+ res['Content-Type'] = 'text/plain; charset=euc-jp'
145
+ end
146
+
147
+ def do_status(req, res)
148
+ res.status = req.query['status'].to_i
149
+ end
150
+
151
+ def do_continue(req, res)
152
+ req.continue
153
+ res.body = 'done!'
154
+ end
155
+ end
@@ -0,0 +1,14 @@
1
+ # coding: utf-8
2
+
3
+ class ProxyServer < BaseServer
4
+ def initialize
5
+ set_logger
6
+ @server = WEBrick::HTTPProxyServer.new(
7
+ :BindAddress => "localhost",
8
+ :Logger => @logger,
9
+ :Port => 0,
10
+ :AccessLog => []
11
+ )
12
+ start
13
+ end
14
+ end
@@ -0,0 +1,73 @@
1
+ class TestServlet < WEBrick::HTTPServlet::AbstractServlet
2
+ def get_instance(*arg)
3
+ self
4
+ end
5
+
6
+ def do_HEAD(req, res)
7
+ res["x-head"] = 'head' # use this for test purpose only.
8
+ res["x-query"] = query_response(req)
9
+ end
10
+
11
+ def do_GET(req, res)
12
+ res.body = 'get'
13
+ res["x-query"] = query_response(req)
14
+ end
15
+
16
+ def do_POST(req, res)
17
+ res["content-type"] = "text/plain" # iso-8859-1, not US-ASCII
18
+ res.body = 'post,' + req.body.to_s
19
+ res["x-query"] = body_response(req)
20
+ end
21
+
22
+ def do_PUT(req, res)
23
+ res["x-query"] = body_response(req)
24
+ param = WEBrick::HTTPUtils.parse_query(req.body) || {}
25
+ res["x-size"] = (param['txt'] || '').size
26
+ res.body = param['txt'] || 'put'
27
+ end
28
+
29
+ def do_DELETE(req, res)
30
+ res.body = 'delete'
31
+ end
32
+
33
+ def do_OPTIONS(req, res)
34
+ # check RFC for legal response.
35
+ res.body = 'options'
36
+ end
37
+
38
+ def do_PROPFIND(req, res)
39
+ res.body = 'propfind'
40
+ end
41
+
42
+ def do_PROPPATCH(req, res)
43
+ res.body = 'proppatch'
44
+ res["x-query"] = body_response(req)
45
+ end
46
+
47
+ def do_TRACE(req, res)
48
+ # client SHOULD reflect the message received back to the client as the
49
+ # entity-body of a 200 (OK) response. [RFC2616]
50
+ res.body = 'trace'
51
+ res["x-query"] = query_response(req)
52
+ end
53
+
54
+ private
55
+
56
+ def query_response(req)
57
+ query_escape(WEBrick::HTTPUtils.parse_query(req.query_string))
58
+ end
59
+
60
+ def body_response(req)
61
+ query_escape(WEBrick::HTTPUtils.parse_query(req.body))
62
+ end
63
+
64
+ def query_escape(query)
65
+ escaped = []
66
+ query.sort_by { |k, v| k }.collect do |k, v|
67
+ v.to_ary.each do |ve|
68
+ escaped << CGI.escape(k) + '=' + CGI.escape(ve)
69
+ end
70
+ end
71
+ escaped.join('&')
72
+ end
73
+ end