httpclient-jgraichen 2.3.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/README.txt +759 -0
- data/bin/httpclient +65 -0
- data/lib/hexdump.rb +50 -0
- data/lib/http-access2.rb +55 -0
- data/lib/http-access2/cookie.rb +1 -0
- data/lib/http-access2/http.rb +1 -0
- data/lib/httpclient.rb +1156 -0
- data/lib/httpclient/auth.rb +899 -0
- data/lib/httpclient/cacert.p7s +1912 -0
- data/lib/httpclient/connection.rb +88 -0
- data/lib/httpclient/cookie.rb +438 -0
- data/lib/httpclient/http.rb +1046 -0
- data/lib/httpclient/include_client.rb +83 -0
- data/lib/httpclient/session.rb +1028 -0
- data/lib/httpclient/ssl_config.rb +405 -0
- data/lib/httpclient/timeout.rb +140 -0
- data/lib/httpclient/util.rb +178 -0
- data/lib/httpclient/version.rb +3 -0
- data/lib/oauthclient.rb +110 -0
- data/sample/async.rb +8 -0
- data/sample/auth.rb +11 -0
- data/sample/cookie.rb +18 -0
- data/sample/dav.rb +103 -0
- data/sample/howto.rb +49 -0
- data/sample/oauth_buzz.rb +57 -0
- data/sample/oauth_friendfeed.rb +59 -0
- data/sample/oauth_twitter.rb +61 -0
- data/sample/ssl/0cert.pem +22 -0
- data/sample/ssl/0key.pem +30 -0
- data/sample/ssl/1000cert.pem +19 -0
- data/sample/ssl/1000key.pem +18 -0
- data/sample/ssl/htdocs/index.html +10 -0
- data/sample/ssl/ssl_client.rb +22 -0
- data/sample/ssl/webrick_httpsd.rb +29 -0
- data/sample/stream.rb +21 -0
- data/sample/thread.rb +27 -0
- data/sample/wcat.rb +21 -0
- data/test/ca-chain.cert +44 -0
- data/test/ca.cert +23 -0
- data/test/client.cert +19 -0
- data/test/client.key +15 -0
- data/test/helper.rb +129 -0
- data/test/htdigest +1 -0
- data/test/htpasswd +2 -0
- data/test/runner.rb +2 -0
- data/test/server.cert +19 -0
- data/test/server.key +15 -0
- data/test/sslsvr.rb +65 -0
- data/test/subca.cert +21 -0
- data/test/test_auth.rb +348 -0
- data/test/test_cookie.rb +412 -0
- data/test/test_hexdump.rb +14 -0
- data/test/test_http-access2.rb +507 -0
- data/test/test_httpclient.rb +1783 -0
- data/test/test_include_client.rb +52 -0
- data/test/test_ssl.rb +235 -0
- metadata +100 -0
@@ -0,0 +1,83 @@
|
|
1
|
+
# It is useful to re-use a HTTPClient instance for multiple requests, to
|
2
|
+
# re-use HTTP 1.1 persistent connections.
|
3
|
+
#
|
4
|
+
# To do that, you sometimes want to store an HTTPClient instance in a global/
|
5
|
+
# class variable location, so it can be accessed and re-used.
|
6
|
+
#
|
7
|
+
# This mix-in makes it easy to create class-level access to one or more
|
8
|
+
# HTTPClient instances. The HTTPClient instances are lazily initialized
|
9
|
+
# on first use (to, for instance, avoid interfering with WebMock/VCR),
|
10
|
+
# and are initialized in a thread-safe manner. Note that a
|
11
|
+
# HTTPClient, once initialized, is safe for use in multiple threads.
|
12
|
+
#
|
13
|
+
# Note that you `extend` HTTPClient::IncludeClient, not `include.
|
14
|
+
#
|
15
|
+
# require 'httpclient/include_client'
|
16
|
+
# class Widget
|
17
|
+
# extend HTTPClient::IncludeClient
|
18
|
+
#
|
19
|
+
# include_http_client
|
20
|
+
# # and/or, specify more stuff
|
21
|
+
# include_http_client('http://myproxy:8080', :method_name => :my_client) do |client|
|
22
|
+
# # any init you want
|
23
|
+
# client.set_cookie_store nil
|
24
|
+
# client.
|
25
|
+
# end
|
26
|
+
# end
|
27
|
+
#
|
28
|
+
# That creates two HTTPClient instances available at the class level.
|
29
|
+
# The first will be available from Widget.http_client (default method
|
30
|
+
# name for `include_http_client`), with default initialization.
|
31
|
+
#
|
32
|
+
# The second will be available at Widget.my_client, with the init arguments
|
33
|
+
# provided, further initialized by the block provided.
|
34
|
+
#
|
35
|
+
# In addition to a class-level method, for convenience instance-level methods
|
36
|
+
# are also provided. Widget.http_client is identical to Widget.new.http_client
|
37
|
+
#
|
38
|
+
#
|
39
|
+
class HTTPClient
|
40
|
+
module IncludeClient
|
41
|
+
|
42
|
+
|
43
|
+
def include_http_client(*args, &block)
|
44
|
+
# We're going to dynamically define a class
|
45
|
+
# to hold our state, namespaced, as well as possibly dynamic
|
46
|
+
# name of cover method.
|
47
|
+
method_name = (args.last.delete(:method_name) if args.last.kind_of? Hash) || :http_client
|
48
|
+
args.pop if args.last == {} # if last arg was named methods now empty, remove it.
|
49
|
+
|
50
|
+
# By the amazingness of closures, we can create these things
|
51
|
+
# in local vars here and use em in our method, we don't even
|
52
|
+
# need iVars for state.
|
53
|
+
client_instance = nil
|
54
|
+
client_mutex = Mutex.new
|
55
|
+
client_args = args
|
56
|
+
client_block = block
|
57
|
+
|
58
|
+
# to define a _class method_ on the specific class that's currently
|
59
|
+
# `self`, we have to use this bit of metaprogramming, sorry.
|
60
|
+
(class << self; self ; end).instance_eval do
|
61
|
+
define_method(method_name) do
|
62
|
+
# implementation copied from ruby stdlib singleton
|
63
|
+
# to create this global obj thread-safely.
|
64
|
+
return client_instance if client_instance
|
65
|
+
client_mutex.synchronize do
|
66
|
+
return client_instance if client_instance
|
67
|
+
# init HTTPClient with specified args/block
|
68
|
+
client_instance = HTTPClient.new(*client_args)
|
69
|
+
client_block.call(client_instance) if client_block
|
70
|
+
end
|
71
|
+
return client_instance
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
# And for convenience, an _instance method_ on the class that just
|
76
|
+
# delegates to the class method.
|
77
|
+
define_method(method_name) do
|
78
|
+
self.class.send(method_name)
|
79
|
+
end
|
80
|
+
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
@@ -0,0 +1,1028 @@
|
|
1
|
+
# HTTPClient - HTTP client library.
|
2
|
+
# Copyright (C) 2000-2009 NAKAMURA, Hiroshi <nahi@ruby-lang.org>.
|
3
|
+
#
|
4
|
+
# This program is copyrighted free software by NAKAMURA, Hiroshi. You can
|
5
|
+
# redistribute it and/or modify it under the same terms of Ruby's license;
|
6
|
+
# either the dual license version in 2003, or any later version.
|
7
|
+
#
|
8
|
+
# httpclient/session.rb is based on http-access.rb in http-access/0.0.4. Some
|
9
|
+
# part of it is copyrighted by Maebashi-san who made and published
|
10
|
+
# http-access/0.0.4. http-access/0.0.4 did not include license notice but when
|
11
|
+
# I asked Maebashi-san he agreed that I can redistribute it under the same terms
|
12
|
+
# of Ruby. Many thanks to Maebashi-san.
|
13
|
+
|
14
|
+
|
15
|
+
require 'socket'
|
16
|
+
require 'thread'
|
17
|
+
require 'stringio'
|
18
|
+
require 'zlib'
|
19
|
+
|
20
|
+
require 'httpclient/timeout'
|
21
|
+
require 'httpclient/ssl_config'
|
22
|
+
require 'httpclient/http'
|
23
|
+
|
24
|
+
|
25
|
+
class HTTPClient
|
26
|
+
|
27
|
+
|
28
|
+
# Represents a Site: protocol scheme, host String and port Number.
|
29
|
+
class Site
|
30
|
+
# Protocol scheme.
|
31
|
+
attr_accessor :scheme
|
32
|
+
# Host String.
|
33
|
+
attr_accessor :host
|
34
|
+
alias hostname host
|
35
|
+
# Port number.
|
36
|
+
attr_accessor :port
|
37
|
+
|
38
|
+
# Creates a new Site based on the given URI.
|
39
|
+
def initialize(uri = nil)
|
40
|
+
if uri
|
41
|
+
@scheme = uri.scheme
|
42
|
+
@host = uri.hostname
|
43
|
+
@port = uri.port.to_i
|
44
|
+
else
|
45
|
+
@scheme = 'tcp'
|
46
|
+
@host = '0.0.0.0'
|
47
|
+
@port = 0
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
# Returns address String.
|
52
|
+
def addr
|
53
|
+
"#{@scheme}://#{@host}:#{@port.to_s}"
|
54
|
+
end
|
55
|
+
|
56
|
+
# Returns true is scheme, host and port are '=='
|
57
|
+
def ==(rhs)
|
58
|
+
(@scheme == rhs.scheme) and (@host == rhs.host) and (@port == rhs.port)
|
59
|
+
end
|
60
|
+
|
61
|
+
# Same as ==.
|
62
|
+
def eql?(rhs)
|
63
|
+
self == rhs
|
64
|
+
end
|
65
|
+
|
66
|
+
def hash # :nodoc:
|
67
|
+
[@scheme, @host, @port].hash
|
68
|
+
end
|
69
|
+
|
70
|
+
def to_s # :nodoc:
|
71
|
+
addr
|
72
|
+
end
|
73
|
+
|
74
|
+
# Returns true if scheme, host and port of the given URI matches with this.
|
75
|
+
def match(uri)
|
76
|
+
(@scheme == uri.scheme) and (@host == uri.host) and (@port == uri.port.to_i)
|
77
|
+
end
|
78
|
+
|
79
|
+
def inspect # :nodoc:
|
80
|
+
sprintf("#<%s:0x%x %s>", self.class.name, __id__, addr)
|
81
|
+
end
|
82
|
+
|
83
|
+
EMPTY = Site.new.freeze
|
84
|
+
end
|
85
|
+
|
86
|
+
|
87
|
+
# Manages sessions for a HTTPClient instance.
|
88
|
+
class SessionManager
|
89
|
+
# Name of this client. Used for 'User-Agent' header in HTTP request.
|
90
|
+
attr_accessor :agent_name
|
91
|
+
# Owner of this client. Used for 'From' header in HTTP request.
|
92
|
+
attr_accessor :from
|
93
|
+
|
94
|
+
# Requested protocol version
|
95
|
+
attr_accessor :protocol_version
|
96
|
+
# Chunk size for chunked request
|
97
|
+
attr_accessor :chunk_size
|
98
|
+
# Device for dumping log for debugging
|
99
|
+
attr_accessor :debug_dev
|
100
|
+
# Boolean value for Socket#sync
|
101
|
+
attr_accessor :socket_sync
|
102
|
+
|
103
|
+
attr_accessor :connect_timeout
|
104
|
+
# Maximum retry count. 0 for infinite.
|
105
|
+
attr_accessor :connect_retry
|
106
|
+
attr_accessor :send_timeout
|
107
|
+
attr_accessor :receive_timeout
|
108
|
+
attr_accessor :keep_alive_timeout
|
109
|
+
attr_accessor :read_block_size
|
110
|
+
attr_accessor :protocol_retry_count
|
111
|
+
|
112
|
+
# Local address to bind local side of the socket to
|
113
|
+
attr_accessor :socket_local
|
114
|
+
|
115
|
+
attr_accessor :ssl_config
|
116
|
+
|
117
|
+
attr_reader :test_loopback_http_response
|
118
|
+
|
119
|
+
attr_accessor :transparent_gzip_decompression
|
120
|
+
|
121
|
+
def initialize(client)
|
122
|
+
@client = client
|
123
|
+
@proxy = client.proxy
|
124
|
+
|
125
|
+
@agent_name = nil
|
126
|
+
@from = nil
|
127
|
+
|
128
|
+
@protocol_version = nil
|
129
|
+
@debug_dev = client.debug_dev
|
130
|
+
@socket_sync = true
|
131
|
+
@chunk_size = ::HTTP::Message::Body::DEFAULT_CHUNK_SIZE
|
132
|
+
|
133
|
+
@connect_timeout = 60
|
134
|
+
@connect_retry = 1
|
135
|
+
@send_timeout = 120
|
136
|
+
@receive_timeout = 60 # For each read_block_size bytes
|
137
|
+
@keep_alive_timeout = 15 # '15' is from Apache 2 default
|
138
|
+
@read_block_size = 1024 * 16 # follows net/http change in 1.8.7
|
139
|
+
@protocol_retry_count = 5
|
140
|
+
|
141
|
+
@ssl_config = nil
|
142
|
+
@test_loopback_http_response = []
|
143
|
+
|
144
|
+
@transparent_gzip_decompression = false
|
145
|
+
@socket_local = Site.new
|
146
|
+
|
147
|
+
@sess_pool = {}
|
148
|
+
@sess_pool_mutex = Mutex.new
|
149
|
+
@sess_pool_last_checked = Time.now
|
150
|
+
end
|
151
|
+
|
152
|
+
def proxy=(proxy)
|
153
|
+
if proxy.nil?
|
154
|
+
@proxy = nil
|
155
|
+
else
|
156
|
+
@proxy = Site.new(proxy)
|
157
|
+
end
|
158
|
+
end
|
159
|
+
|
160
|
+
def query(req, via_proxy)
|
161
|
+
req.http_body.chunk_size = @chunk_size
|
162
|
+
sess = open(req.header.request_uri, via_proxy)
|
163
|
+
begin
|
164
|
+
sess.query(req)
|
165
|
+
rescue
|
166
|
+
sess.close
|
167
|
+
raise
|
168
|
+
end
|
169
|
+
sess
|
170
|
+
end
|
171
|
+
|
172
|
+
def reset(uri)
|
173
|
+
site = Site.new(uri)
|
174
|
+
close(site)
|
175
|
+
end
|
176
|
+
|
177
|
+
def reset_all
|
178
|
+
close_all
|
179
|
+
end
|
180
|
+
|
181
|
+
# assert: sess.last_used must not be nil
|
182
|
+
def keep(sess)
|
183
|
+
add_cached_session(sess)
|
184
|
+
end
|
185
|
+
|
186
|
+
def invalidate(site)
|
187
|
+
@sess_pool_mutex.synchronize do
|
188
|
+
if pool = @sess_pool[site]
|
189
|
+
pool.each do |sess|
|
190
|
+
sess.invalidate
|
191
|
+
end
|
192
|
+
end
|
193
|
+
end
|
194
|
+
end
|
195
|
+
|
196
|
+
private
|
197
|
+
|
198
|
+
def open(uri, via_proxy = false)
|
199
|
+
site = Site.new(uri)
|
200
|
+
sess = nil
|
201
|
+
if cached = get_cached_session(site)
|
202
|
+
sess = cached
|
203
|
+
else
|
204
|
+
sess = Session.new(@client, site, @agent_name, @from)
|
205
|
+
sess.proxy = via_proxy ? @proxy : nil
|
206
|
+
sess.socket_sync = @socket_sync
|
207
|
+
sess.requested_version = @protocol_version if @protocol_version
|
208
|
+
sess.connect_timeout = @connect_timeout
|
209
|
+
sess.connect_retry = @connect_retry
|
210
|
+
sess.send_timeout = @send_timeout
|
211
|
+
sess.receive_timeout = @receive_timeout
|
212
|
+
sess.read_block_size = @read_block_size
|
213
|
+
sess.protocol_retry_count = @protocol_retry_count
|
214
|
+
sess.ssl_config = @ssl_config
|
215
|
+
sess.debug_dev = @debug_dev
|
216
|
+
sess.socket_local = @socket_local
|
217
|
+
sess.test_loopback_http_response = @test_loopback_http_response
|
218
|
+
sess.transparent_gzip_decompression = @transparent_gzip_decompression
|
219
|
+
end
|
220
|
+
sess
|
221
|
+
end
|
222
|
+
|
223
|
+
def close_all
|
224
|
+
@sess_pool_mutex.synchronize do
|
225
|
+
@sess_pool.each do |site, pool|
|
226
|
+
pool.each do |sess|
|
227
|
+
sess.close
|
228
|
+
end
|
229
|
+
end
|
230
|
+
end
|
231
|
+
@sess_pool.clear
|
232
|
+
end
|
233
|
+
|
234
|
+
# This method might not work as you expected...
|
235
|
+
def close(dest)
|
236
|
+
if cached = get_cached_session(Site.new(dest))
|
237
|
+
cached.close
|
238
|
+
true
|
239
|
+
else
|
240
|
+
false
|
241
|
+
end
|
242
|
+
end
|
243
|
+
|
244
|
+
def get_cached_session(site)
|
245
|
+
@sess_pool_mutex.synchronize do
|
246
|
+
now = Time.now
|
247
|
+
if now > @sess_pool_last_checked + @keep_alive_timeout
|
248
|
+
scrub_cached_session(now)
|
249
|
+
@sess_pool_last_checked = now
|
250
|
+
end
|
251
|
+
if pool = @sess_pool[site]
|
252
|
+
pool.each_with_index do |sess, idx|
|
253
|
+
if valid_session?(sess, now)
|
254
|
+
return pool.slice!(idx)
|
255
|
+
end
|
256
|
+
end
|
257
|
+
end
|
258
|
+
end
|
259
|
+
nil
|
260
|
+
end
|
261
|
+
|
262
|
+
def scrub_cached_session(now)
|
263
|
+
@sess_pool.each do |site, pool|
|
264
|
+
pool.replace(pool.select { |sess|
|
265
|
+
if valid_session?(sess, now)
|
266
|
+
true
|
267
|
+
else
|
268
|
+
sess.close # close & remove from the pool
|
269
|
+
false
|
270
|
+
end
|
271
|
+
})
|
272
|
+
end
|
273
|
+
end
|
274
|
+
|
275
|
+
def valid_session?(sess, now)
|
276
|
+
!sess.invalidated? and (now <= sess.last_used + @keep_alive_timeout)
|
277
|
+
end
|
278
|
+
|
279
|
+
def add_cached_session(sess)
|
280
|
+
@sess_pool_mutex.synchronize do
|
281
|
+
(@sess_pool[sess.dest] ||= []).unshift(sess)
|
282
|
+
end
|
283
|
+
end
|
284
|
+
end
|
285
|
+
|
286
|
+
|
287
|
+
# Wraps up OpenSSL::SSL::SSLSocket and offers debugging features.
|
288
|
+
class SSLSocketWrap
|
289
|
+
def initialize(socket, context, debug_dev = nil)
|
290
|
+
unless SSLEnabled
|
291
|
+
raise ConfigurationError.new('Ruby/OpenSSL module is required')
|
292
|
+
end
|
293
|
+
@context = context
|
294
|
+
@socket = socket
|
295
|
+
@ssl_socket = create_openssl_socket(@socket)
|
296
|
+
@debug_dev = debug_dev
|
297
|
+
end
|
298
|
+
|
299
|
+
def ssl_connect(hostname = nil)
|
300
|
+
if hostname && @ssl_socket.respond_to?(:hostname=)
|
301
|
+
@ssl_socket.hostname = hostname
|
302
|
+
end
|
303
|
+
@ssl_socket.connect
|
304
|
+
end
|
305
|
+
|
306
|
+
def post_connection_check(host)
|
307
|
+
verify_mode = @context.verify_mode || OpenSSL::SSL::VERIFY_NONE
|
308
|
+
if verify_mode == OpenSSL::SSL::VERIFY_NONE
|
309
|
+
return
|
310
|
+
elsif @ssl_socket.peer_cert.nil? and
|
311
|
+
check_mask(verify_mode, OpenSSL::SSL::VERIFY_FAIL_IF_NO_PEER_CERT)
|
312
|
+
raise OpenSSL::SSL::SSLError.new('no peer cert')
|
313
|
+
end
|
314
|
+
hostname = host.host
|
315
|
+
if @ssl_socket.respond_to?(:post_connection_check) and RUBY_VERSION > "1.8.4"
|
316
|
+
@ssl_socket.post_connection_check(hostname)
|
317
|
+
else
|
318
|
+
@context.post_connection_check(@ssl_socket.peer_cert, hostname)
|
319
|
+
end
|
320
|
+
end
|
321
|
+
|
322
|
+
def ssl_version
|
323
|
+
@ssl_socket.ssl_version if @ssl_socket.respond_to?(:ssl_version)
|
324
|
+
end
|
325
|
+
|
326
|
+
def ssl_cipher
|
327
|
+
@ssl_socket.cipher
|
328
|
+
end
|
329
|
+
|
330
|
+
def ssl_state
|
331
|
+
@ssl_socket.state
|
332
|
+
end
|
333
|
+
|
334
|
+
def peer_cert
|
335
|
+
@ssl_socket.peer_cert
|
336
|
+
end
|
337
|
+
|
338
|
+
def close
|
339
|
+
@ssl_socket.close
|
340
|
+
@socket.close
|
341
|
+
end
|
342
|
+
|
343
|
+
def closed?
|
344
|
+
@socket.closed?
|
345
|
+
end
|
346
|
+
|
347
|
+
def eof?
|
348
|
+
@ssl_socket.eof?
|
349
|
+
end
|
350
|
+
|
351
|
+
def gets(*args)
|
352
|
+
str = @ssl_socket.gets(*args)
|
353
|
+
debug(str)
|
354
|
+
str
|
355
|
+
end
|
356
|
+
|
357
|
+
def read(*args)
|
358
|
+
str = @ssl_socket.read(*args)
|
359
|
+
debug(str)
|
360
|
+
str
|
361
|
+
end
|
362
|
+
|
363
|
+
def readpartial(*args)
|
364
|
+
str = @ssl_socket.readpartial(*args)
|
365
|
+
debug(str)
|
366
|
+
str
|
367
|
+
end
|
368
|
+
|
369
|
+
def <<(str)
|
370
|
+
rv = @ssl_socket.write(str)
|
371
|
+
debug(str)
|
372
|
+
rv
|
373
|
+
end
|
374
|
+
|
375
|
+
def flush
|
376
|
+
@ssl_socket.flush
|
377
|
+
end
|
378
|
+
|
379
|
+
def sync
|
380
|
+
@ssl_socket.sync
|
381
|
+
end
|
382
|
+
|
383
|
+
def sync=(sync)
|
384
|
+
@ssl_socket.sync = sync
|
385
|
+
end
|
386
|
+
|
387
|
+
private
|
388
|
+
|
389
|
+
def check_mask(value, mask)
|
390
|
+
value & mask == mask
|
391
|
+
end
|
392
|
+
|
393
|
+
def create_openssl_socket(socket)
|
394
|
+
ssl_socket = nil
|
395
|
+
if OpenSSL::SSL.const_defined?("SSLContext")
|
396
|
+
ctx = OpenSSL::SSL::SSLContext.new
|
397
|
+
@context.set_context(ctx)
|
398
|
+
ssl_socket = OpenSSL::SSL::SSLSocket.new(socket, ctx)
|
399
|
+
else
|
400
|
+
ssl_socket = OpenSSL::SSL::SSLSocket.new(socket)
|
401
|
+
@context.set_context(ssl_socket)
|
402
|
+
end
|
403
|
+
ssl_socket
|
404
|
+
end
|
405
|
+
|
406
|
+
def debug(str)
|
407
|
+
@debug_dev << str if @debug_dev && str
|
408
|
+
end
|
409
|
+
end
|
410
|
+
|
411
|
+
|
412
|
+
# Wraps up a Socket for method interception.
|
413
|
+
module SocketWrap
|
414
|
+
def initialize(socket, *args)
|
415
|
+
super(*args)
|
416
|
+
@socket = socket
|
417
|
+
end
|
418
|
+
|
419
|
+
def close
|
420
|
+
@socket.close
|
421
|
+
end
|
422
|
+
|
423
|
+
def closed?
|
424
|
+
@socket.closed?
|
425
|
+
end
|
426
|
+
|
427
|
+
def eof?
|
428
|
+
@socket.eof?
|
429
|
+
end
|
430
|
+
|
431
|
+
def gets(*args)
|
432
|
+
@socket.gets(*args)
|
433
|
+
end
|
434
|
+
|
435
|
+
def read(*args)
|
436
|
+
@socket.read(*args)
|
437
|
+
end
|
438
|
+
|
439
|
+
def readpartial(*args)
|
440
|
+
# StringIO doesn't support :readpartial
|
441
|
+
if @socket.respond_to?(:readpartial)
|
442
|
+
@socket.readpartial(*args)
|
443
|
+
else
|
444
|
+
@socket.read(*args)
|
445
|
+
end
|
446
|
+
end
|
447
|
+
|
448
|
+
def <<(str)
|
449
|
+
@socket << str
|
450
|
+
end
|
451
|
+
|
452
|
+
def flush
|
453
|
+
@socket.flush
|
454
|
+
end
|
455
|
+
|
456
|
+
def sync
|
457
|
+
@socket.sync
|
458
|
+
end
|
459
|
+
|
460
|
+
def sync=(sync)
|
461
|
+
@socket.sync = sync
|
462
|
+
end
|
463
|
+
end
|
464
|
+
|
465
|
+
|
466
|
+
# Module for intercepting Socket methods and dumps in/out to given debugging
|
467
|
+
# device. debug_dev must respond to <<.
|
468
|
+
module DebugSocket
|
469
|
+
extend SocketWrap
|
470
|
+
|
471
|
+
def debug_dev=(debug_dev)
|
472
|
+
@debug_dev = debug_dev
|
473
|
+
end
|
474
|
+
|
475
|
+
def close
|
476
|
+
super
|
477
|
+
debug("! CONNECTION CLOSED\n")
|
478
|
+
end
|
479
|
+
|
480
|
+
def gets(*args)
|
481
|
+
str = super
|
482
|
+
debug(str)
|
483
|
+
str
|
484
|
+
end
|
485
|
+
|
486
|
+
def read(*args)
|
487
|
+
str = super
|
488
|
+
debug(str)
|
489
|
+
str
|
490
|
+
end
|
491
|
+
|
492
|
+
def readpartial(*args)
|
493
|
+
str = super
|
494
|
+
debug(str)
|
495
|
+
str
|
496
|
+
end
|
497
|
+
|
498
|
+
def <<(str)
|
499
|
+
super
|
500
|
+
debug(str)
|
501
|
+
end
|
502
|
+
|
503
|
+
private
|
504
|
+
|
505
|
+
def debug(str)
|
506
|
+
if str && @debug_dev
|
507
|
+
if str.index("\0")
|
508
|
+
require 'hexdump'
|
509
|
+
str.force_encoding('BINARY') if str.respond_to?(:force_encoding)
|
510
|
+
@debug_dev << HexDump.encode(str).join("\n")
|
511
|
+
else
|
512
|
+
@debug_dev << str
|
513
|
+
end
|
514
|
+
end
|
515
|
+
end
|
516
|
+
end
|
517
|
+
|
518
|
+
|
519
|
+
# Dummy Socket for emulating loopback test.
|
520
|
+
class LoopBackSocket
|
521
|
+
include SocketWrap
|
522
|
+
|
523
|
+
def initialize(host, port, response)
|
524
|
+
super(response.is_a?(StringIO) ? response : StringIO.new(response))
|
525
|
+
@host = host
|
526
|
+
@port = port
|
527
|
+
end
|
528
|
+
|
529
|
+
def <<(str)
|
530
|
+
# ignored
|
531
|
+
end
|
532
|
+
end
|
533
|
+
|
534
|
+
|
535
|
+
# Manages a HTTP session with a Site.
|
536
|
+
class Session
|
537
|
+
include HTTPClient::Timeout
|
538
|
+
include Util
|
539
|
+
|
540
|
+
# Destination site
|
541
|
+
attr_reader :dest
|
542
|
+
# Proxy site
|
543
|
+
attr_accessor :proxy
|
544
|
+
# Boolean value for Socket#sync
|
545
|
+
attr_accessor :socket_sync
|
546
|
+
# Requested protocol version
|
547
|
+
attr_accessor :requested_version
|
548
|
+
# Device for dumping log for debugging
|
549
|
+
attr_accessor :debug_dev
|
550
|
+
|
551
|
+
attr_accessor :connect_timeout
|
552
|
+
attr_accessor :connect_retry
|
553
|
+
attr_accessor :send_timeout
|
554
|
+
attr_accessor :receive_timeout
|
555
|
+
attr_accessor :read_block_size
|
556
|
+
attr_accessor :protocol_retry_count
|
557
|
+
|
558
|
+
attr_accessor :socket_local
|
559
|
+
|
560
|
+
attr_accessor :ssl_config
|
561
|
+
attr_reader :ssl_peer_cert
|
562
|
+
attr_accessor :test_loopback_http_response
|
563
|
+
|
564
|
+
attr_accessor :transparent_gzip_decompression
|
565
|
+
attr_reader :last_used
|
566
|
+
|
567
|
+
def initialize(client, dest, agent_name, from)
|
568
|
+
@client = client
|
569
|
+
@dest = dest
|
570
|
+
@invalidated = false
|
571
|
+
@proxy = nil
|
572
|
+
@socket_sync = true
|
573
|
+
@requested_version = nil
|
574
|
+
|
575
|
+
@debug_dev = nil
|
576
|
+
|
577
|
+
@connect_timeout = nil
|
578
|
+
@connect_retry = 1
|
579
|
+
@send_timeout = nil
|
580
|
+
@receive_timeout = nil
|
581
|
+
@read_block_size = nil
|
582
|
+
@protocol_retry_count = 5
|
583
|
+
|
584
|
+
@ssl_config = nil
|
585
|
+
@ssl_peer_cert = nil
|
586
|
+
|
587
|
+
@test_loopback_http_response = nil
|
588
|
+
@socket_local = Site::EMPTY
|
589
|
+
|
590
|
+
@agent_name = agent_name
|
591
|
+
@from = from
|
592
|
+
@state = :INIT
|
593
|
+
|
594
|
+
@requests = []
|
595
|
+
|
596
|
+
@status = nil
|
597
|
+
@reason = nil
|
598
|
+
@headers = []
|
599
|
+
|
600
|
+
@socket = nil
|
601
|
+
@readbuf = nil
|
602
|
+
|
603
|
+
@transparent_gzip_decompression = false
|
604
|
+
@last_used = nil
|
605
|
+
end
|
606
|
+
|
607
|
+
# Send a request to the server
|
608
|
+
def query(req)
|
609
|
+
connect if @state == :INIT
|
610
|
+
# Use absolute URI (not absolute path) iif via proxy AND not HTTPS.
|
611
|
+
req.header.request_absolute_uri = !@proxy.nil? && !https?(@dest)
|
612
|
+
begin
|
613
|
+
timeout(@send_timeout, SendTimeoutError) do
|
614
|
+
set_header(req)
|
615
|
+
req.dump(@socket)
|
616
|
+
# flush the IO stream as IO::sync mode is false
|
617
|
+
@socket.flush unless @socket_sync
|
618
|
+
end
|
619
|
+
rescue Errno::ECONNABORTED, Errno::ECONNRESET, Errno::EPIPE, IOError
|
620
|
+
# JRuby can raise IOError instead of ECONNRESET for now
|
621
|
+
close
|
622
|
+
raise KeepAliveDisconnected.new(self)
|
623
|
+
rescue HTTPClient::TimeoutError
|
624
|
+
close
|
625
|
+
raise
|
626
|
+
rescue
|
627
|
+
close
|
628
|
+
if SSLEnabled and $!.is_a?(OpenSSL::SSL::SSLError)
|
629
|
+
raise KeepAliveDisconnected.new(self)
|
630
|
+
else
|
631
|
+
raise
|
632
|
+
end
|
633
|
+
end
|
634
|
+
|
635
|
+
@state = :META if @state == :WAIT
|
636
|
+
@next_connection = nil
|
637
|
+
@requests.push(req)
|
638
|
+
@last_used = Time.now
|
639
|
+
end
|
640
|
+
|
641
|
+
def close
|
642
|
+
if !@socket.nil? and !@socket.closed?
|
643
|
+
# @socket.flush may block when it the socket is already closed by
|
644
|
+
# foreign host and the client runs under MT-condition.
|
645
|
+
@socket.close
|
646
|
+
end
|
647
|
+
@state = :INIT
|
648
|
+
end
|
649
|
+
|
650
|
+
def closed?
|
651
|
+
@state == :INIT
|
652
|
+
end
|
653
|
+
|
654
|
+
def invalidate
|
655
|
+
@invalidated = true
|
656
|
+
end
|
657
|
+
|
658
|
+
def invalidated?
|
659
|
+
@invalidated
|
660
|
+
end
|
661
|
+
|
662
|
+
def get_header
|
663
|
+
begin
|
664
|
+
if @state != :META
|
665
|
+
raise RuntimeError.new("get_status must be called at the beginning of a session")
|
666
|
+
end
|
667
|
+
read_header
|
668
|
+
rescue
|
669
|
+
close
|
670
|
+
raise
|
671
|
+
end
|
672
|
+
[@version, @status, @reason, @headers]
|
673
|
+
end
|
674
|
+
|
675
|
+
def eof?
|
676
|
+
if !@content_length.nil?
|
677
|
+
@content_length == 0
|
678
|
+
else
|
679
|
+
@socket.closed? or @socket.eof?
|
680
|
+
end
|
681
|
+
end
|
682
|
+
|
683
|
+
def get_body(&block)
|
684
|
+
begin
|
685
|
+
read_header if @state == :META
|
686
|
+
return nil if @state != :DATA
|
687
|
+
if @gzipped and @transparent_gzip_decompression
|
688
|
+
# zlib itself has a functionality to decompress gzip stream.
|
689
|
+
# - zlib 1.2.5 Manual
|
690
|
+
# http://www.zlib.net/manual.html#Advanced
|
691
|
+
# > windowBits can also be greater than 15 for optional gzip decoding. Add 32 to
|
692
|
+
# > windowBits to enable zlib and gzip decoding with automatic header detection,
|
693
|
+
# > or add 16 to decode only the gzip format
|
694
|
+
inflate_stream = Zlib::Inflate.new(Zlib::MAX_WBITS + 32)
|
695
|
+
original_block = block
|
696
|
+
block = Proc.new { |buf|
|
697
|
+
original_block.call(inflate_stream.inflate(buf))
|
698
|
+
}
|
699
|
+
end
|
700
|
+
if @chunked
|
701
|
+
read_body_chunked(&block)
|
702
|
+
elsif @content_length
|
703
|
+
read_body_length(&block)
|
704
|
+
else
|
705
|
+
read_body_rest(&block)
|
706
|
+
end
|
707
|
+
rescue
|
708
|
+
close
|
709
|
+
raise
|
710
|
+
end
|
711
|
+
if eof?
|
712
|
+
if @next_connection
|
713
|
+
@state = :WAIT
|
714
|
+
else
|
715
|
+
close
|
716
|
+
end
|
717
|
+
end
|
718
|
+
nil
|
719
|
+
end
|
720
|
+
|
721
|
+
private
|
722
|
+
|
723
|
+
def set_header(req)
|
724
|
+
if @requested_version
|
725
|
+
if /^(?:HTTP\/|)(\d+.\d+)$/ =~ @requested_version
|
726
|
+
req.http_version = $1
|
727
|
+
end
|
728
|
+
end
|
729
|
+
if @agent_name && req.header.get('User-Agent').empty?
|
730
|
+
req.header.set('User-Agent', "#{@agent_name} #{LIB_NAME}")
|
731
|
+
end
|
732
|
+
if @from && req.header.get('From').empty?
|
733
|
+
req.header.set('From', @from)
|
734
|
+
end
|
735
|
+
if req.header.get('Accept').empty?
|
736
|
+
req.header.set('Accept', '*/*')
|
737
|
+
end
|
738
|
+
if @transparent_gzip_decompression
|
739
|
+
req.header.set('Accept-Encoding', 'gzip,deflate')
|
740
|
+
end
|
741
|
+
if req.header.get('Date').empty?
|
742
|
+
req.header.set_date_header
|
743
|
+
end
|
744
|
+
end
|
745
|
+
|
746
|
+
# Connect to the server
|
747
|
+
def connect
|
748
|
+
site = @proxy || @dest
|
749
|
+
retry_number = 0
|
750
|
+
begin
|
751
|
+
timeout(@connect_timeout, ConnectTimeoutError) do
|
752
|
+
@socket = create_socket(site)
|
753
|
+
if https?(@dest)
|
754
|
+
if @socket.is_a?(LoopBackSocket)
|
755
|
+
connect_ssl_proxy(@socket, urify(@dest.to_s)) if @proxy
|
756
|
+
else
|
757
|
+
@socket = create_ssl_socket(@socket)
|
758
|
+
connect_ssl_proxy(@socket, urify(@dest.to_s)) if @proxy
|
759
|
+
begin
|
760
|
+
@socket.ssl_connect(@dest.host)
|
761
|
+
ensure
|
762
|
+
if $DEBUG
|
763
|
+
warn("Protocol version: #{@socket.ssl_version}")
|
764
|
+
warn("Cipher: #{@socket.ssl_cipher.inspect}")
|
765
|
+
warn("State: #{@socket.ssl_state}")
|
766
|
+
end
|
767
|
+
end
|
768
|
+
@socket.post_connection_check(@dest)
|
769
|
+
@ssl_peer_cert = @socket.peer_cert
|
770
|
+
end
|
771
|
+
end
|
772
|
+
# Use Ruby internal buffering instead of passing data immediately
|
773
|
+
# to the underlying layer
|
774
|
+
# => we need to to call explicitly flush on the socket
|
775
|
+
@socket.sync = @socket_sync
|
776
|
+
end
|
777
|
+
rescue RetryableResponse
|
778
|
+
retry_number += 1
|
779
|
+
if retry_number < @protocol_retry_count
|
780
|
+
retry
|
781
|
+
end
|
782
|
+
raise BadResponseError.new("connect to the server failed with status #{@status} #{@reason}")
|
783
|
+
rescue TimeoutError
|
784
|
+
if @connect_retry == 0
|
785
|
+
retry
|
786
|
+
else
|
787
|
+
retry_number += 1
|
788
|
+
retry if retry_number < @connect_retry
|
789
|
+
end
|
790
|
+
close
|
791
|
+
raise
|
792
|
+
end
|
793
|
+
@state = :WAIT
|
794
|
+
end
|
795
|
+
|
796
|
+
def create_socket(site)
|
797
|
+
socket = nil
|
798
|
+
begin
|
799
|
+
@debug_dev << "! CONNECT TO #{site.host}:#{site.port}\n" if @debug_dev
|
800
|
+
clean_host = site.host.delete("[]")
|
801
|
+
clean_local = @socket_local.host.delete("[]")
|
802
|
+
if str = @test_loopback_http_response.shift
|
803
|
+
socket = LoopBackSocket.new(clean_host, site.port, str)
|
804
|
+
elsif @socket_local == Site::EMPTY
|
805
|
+
socket = TCPSocket.new(clean_host, site.port)
|
806
|
+
else
|
807
|
+
socket = TCPSocket.new(clean_host, site.port, clean_local, @socket_local.port)
|
808
|
+
end
|
809
|
+
if @debug_dev
|
810
|
+
@debug_dev << "! CONNECTION ESTABLISHED\n"
|
811
|
+
socket.extend(DebugSocket)
|
812
|
+
socket.debug_dev = @debug_dev
|
813
|
+
end
|
814
|
+
rescue SystemCallError => e
|
815
|
+
e.message << " (#{site})"
|
816
|
+
raise
|
817
|
+
rescue SocketError => e
|
818
|
+
e.message << " (#{site})"
|
819
|
+
raise
|
820
|
+
end
|
821
|
+
socket
|
822
|
+
end
|
823
|
+
|
824
|
+
# wrap socket with OpenSSL.
|
825
|
+
def create_ssl_socket(raw_socket)
|
826
|
+
SSLSocketWrap.new(raw_socket, @ssl_config, @debug_dev)
|
827
|
+
end
|
828
|
+
|
829
|
+
def connect_ssl_proxy(socket, uri)
|
830
|
+
req = HTTP::Message.new_connect_request(uri)
|
831
|
+
@client.request_filter.each do |filter|
|
832
|
+
filter.filter_request(req)
|
833
|
+
end
|
834
|
+
set_header(req)
|
835
|
+
req.dump(@socket)
|
836
|
+
@socket.flush unless @socket_sync
|
837
|
+
res = HTTP::Message.new_response('')
|
838
|
+
parse_header
|
839
|
+
res.http_version, res.status, res.reason = @version, @status, @reason
|
840
|
+
@headers.each do |key, value|
|
841
|
+
res.header.set(key.to_s, value)
|
842
|
+
end
|
843
|
+
commands = @client.request_filter.collect { |filter|
|
844
|
+
filter.filter_response(req, res)
|
845
|
+
}
|
846
|
+
if commands.find { |command| command == :retry }
|
847
|
+
raise RetryableResponse.new
|
848
|
+
end
|
849
|
+
unless @status == 200
|
850
|
+
raise BadResponseError.new("connect to ssl proxy failed with status #{@status} #{@reason}", res)
|
851
|
+
end
|
852
|
+
end
|
853
|
+
|
854
|
+
# Read status block.
|
855
|
+
def read_header
|
856
|
+
@content_length = nil
|
857
|
+
@chunked = false
|
858
|
+
@gzipped = false
|
859
|
+
@chunk_length = 0
|
860
|
+
parse_header
|
861
|
+
# Header of the request has been parsed.
|
862
|
+
@state = :DATA
|
863
|
+
req = @requests.shift
|
864
|
+
if req.header.request_method == 'HEAD' or no_message_body?(@status)
|
865
|
+
@content_length = 0
|
866
|
+
if @next_connection
|
867
|
+
@state = :WAIT
|
868
|
+
else
|
869
|
+
close
|
870
|
+
end
|
871
|
+
end
|
872
|
+
@next_connection = false if !@content_length and !@chunked
|
873
|
+
end
|
874
|
+
|
875
|
+
StatusParseRegexp = %r(\AHTTP/(\d+\.\d+)\s+(\d\d\d)\s*([^\r\n]+)?\r?\n\z)
|
876
|
+
def parse_header
|
877
|
+
timeout(@receive_timeout, ReceiveTimeoutError) do
|
878
|
+
initial_line = nil
|
879
|
+
begin
|
880
|
+
begin
|
881
|
+
initial_line = @socket.gets("\n")
|
882
|
+
if initial_line.nil?
|
883
|
+
close
|
884
|
+
raise KeepAliveDisconnected.new(self)
|
885
|
+
end
|
886
|
+
rescue Errno::ECONNABORTED, Errno::ECONNRESET, Errno::EPIPE, IOError
|
887
|
+
# JRuby can raise IOError instead of ECONNRESET for now
|
888
|
+
close
|
889
|
+
raise KeepAliveDisconnected.new(self)
|
890
|
+
end
|
891
|
+
if StatusParseRegexp !~ initial_line
|
892
|
+
@version = '0.9'
|
893
|
+
@status = nil
|
894
|
+
@reason = nil
|
895
|
+
@next_connection = false
|
896
|
+
@content_length = nil
|
897
|
+
@readbuf = initial_line
|
898
|
+
break
|
899
|
+
end
|
900
|
+
@version, @status, @reason = $1, $2.to_i, $3
|
901
|
+
@next_connection = HTTP::Message.keep_alive_enabled?(@version)
|
902
|
+
@headers = []
|
903
|
+
while true
|
904
|
+
line = @socket.gets("\n")
|
905
|
+
unless line
|
906
|
+
raise BadResponseError.new('unexpected EOF')
|
907
|
+
end
|
908
|
+
line.chomp!
|
909
|
+
break if line.empty?
|
910
|
+
if line[0] == ?\ or line[0] == ?\t
|
911
|
+
last = @headers.last[1]
|
912
|
+
last << ' ' unless last.empty?
|
913
|
+
last << line.strip
|
914
|
+
else
|
915
|
+
key, value = line.strip.split(/\s*:\s*/, 2)
|
916
|
+
parse_keepalive_header(key, value)
|
917
|
+
@headers << [key, value]
|
918
|
+
end
|
919
|
+
end
|
920
|
+
end while (@version == '1.1' && @status == 100)
|
921
|
+
end
|
922
|
+
end
|
923
|
+
|
924
|
+
def no_message_body?(status)
|
925
|
+
!status.nil? && # HTTP/0.9
|
926
|
+
((status >= 100 && status < 200) || status == 204 || status == 304)
|
927
|
+
end
|
928
|
+
|
929
|
+
def parse_keepalive_header(key, value)
|
930
|
+
key = key.downcase
|
931
|
+
if key == 'content-length'
|
932
|
+
@content_length = value.to_i
|
933
|
+
elsif key == 'content-encoding' and ( value.downcase == 'gzip' or
|
934
|
+
value.downcase == 'x-gzip' or value.downcase == 'deflate' )
|
935
|
+
@gzipped = true
|
936
|
+
elsif key == 'transfer-encoding' and value.downcase == 'chunked'
|
937
|
+
@chunked = true
|
938
|
+
@chunk_length = 0
|
939
|
+
@content_length = nil
|
940
|
+
elsif key == 'connection' or key == 'proxy-connection'
|
941
|
+
if value.downcase == 'keep-alive'
|
942
|
+
@next_connection = true
|
943
|
+
else
|
944
|
+
@next_connection = false
|
945
|
+
end
|
946
|
+
end
|
947
|
+
end
|
948
|
+
|
949
|
+
def read_body_length(&block)
|
950
|
+
return nil if @content_length == 0
|
951
|
+
while true
|
952
|
+
buf = empty_bin_str
|
953
|
+
maxbytes = @read_block_size
|
954
|
+
maxbytes = @content_length if maxbytes > @content_length && @content_length > 0
|
955
|
+
timeout(@receive_timeout, ReceiveTimeoutError) do
|
956
|
+
begin
|
957
|
+
@socket.readpartial(maxbytes, buf)
|
958
|
+
rescue EOFError
|
959
|
+
close
|
960
|
+
buf = nil
|
961
|
+
end
|
962
|
+
end
|
963
|
+
if buf && buf.bytesize > 0
|
964
|
+
@content_length -= buf.bytesize
|
965
|
+
yield buf
|
966
|
+
else
|
967
|
+
@content_length = 0
|
968
|
+
end
|
969
|
+
return if @content_length == 0
|
970
|
+
end
|
971
|
+
end
|
972
|
+
|
973
|
+
RS = "\r\n"
|
974
|
+
def read_body_chunked(&block)
|
975
|
+
buf = empty_bin_str
|
976
|
+
while true
|
977
|
+
len = @socket.gets(RS)
|
978
|
+
if len.nil? # EOF
|
979
|
+
close
|
980
|
+
return
|
981
|
+
end
|
982
|
+
@chunk_length = len.hex
|
983
|
+
if @chunk_length == 0
|
984
|
+
@content_length = 0
|
985
|
+
@socket.gets(RS)
|
986
|
+
return
|
987
|
+
end
|
988
|
+
timeout(@receive_timeout, ReceiveTimeoutError) do
|
989
|
+
@socket.read(@chunk_length, buf)
|
990
|
+
@socket.read(2)
|
991
|
+
end
|
992
|
+
unless buf.empty?
|
993
|
+
yield buf
|
994
|
+
end
|
995
|
+
end
|
996
|
+
end
|
997
|
+
|
998
|
+
def read_body_rest
|
999
|
+
if @readbuf and @readbuf.bytesize > 0
|
1000
|
+
yield @readbuf
|
1001
|
+
@readbuf = nil
|
1002
|
+
end
|
1003
|
+
while true
|
1004
|
+
buf = empty_bin_str
|
1005
|
+
timeout(@receive_timeout, ReceiveTimeoutError) do
|
1006
|
+
begin
|
1007
|
+
@socket.readpartial(@read_block_size, buf)
|
1008
|
+
rescue EOFError
|
1009
|
+
buf = nil
|
1010
|
+
end
|
1011
|
+
end
|
1012
|
+
if buf && buf.bytesize > 0
|
1013
|
+
yield buf
|
1014
|
+
else
|
1015
|
+
return
|
1016
|
+
end
|
1017
|
+
end
|
1018
|
+
end
|
1019
|
+
|
1020
|
+
def empty_bin_str
|
1021
|
+
str = ''
|
1022
|
+
str.force_encoding('BINARY') if str.respond_to?(:force_encoding)
|
1023
|
+
str
|
1024
|
+
end
|
1025
|
+
end
|
1026
|
+
|
1027
|
+
|
1028
|
+
end
|