em-http-request 1.0.0.beta.3 → 1.0.0.beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of em-http-request might be problematic. Click here for more details.
- data/.gemtest +0 -0
- data/Gemfile +12 -1
- data/README.md +2 -2
- data/Rakefile +2 -1
- data/benchmarks/clients.rb +156 -0
- data/benchmarks/em-excon.rb +87 -0
- data/benchmarks/em-profile.gif +0 -0
- data/benchmarks/em-profile.txt +65 -0
- data/benchmarks/server.rb +48 -0
- data/em-http-request.gemspec +2 -1
- data/examples/.gitignore +1 -0
- data/examples/fibered-http.rb +10 -6
- data/examples/oauth-tweet.rb +22 -37
- data/lib/em-http.rb +3 -2
- data/lib/em-http/client.rb +52 -37
- data/lib/em-http/decoders.rb +1 -1
- data/lib/em-http/http_client_options.rb +56 -0
- data/lib/em-http/http_connection.rb +97 -41
- data/lib/em-http/http_connection_options.rb +23 -0
- data/lib/em-http/http_encoding.rb +1 -1
- data/lib/em-http/middleware/cookie_jar.rb +38 -0
- data/lib/em-http/middleware/json_response.rb +15 -0
- data/lib/em-http/middleware/oauth.rb +21 -0
- data/lib/em-http/multi.rb +18 -18
- data/lib/em-http/request.rb +7 -29
- data/lib/em-http/version.rb +1 -1
- data/spec/client_spec.rb +39 -0
- data/spec/external_spec.rb +17 -1
- data/spec/helper.rb +12 -0
- data/spec/middleware_spec.rb +85 -7
- data/spec/multi_spec.rb +77 -22
- data/spec/pipelining_spec.rb +1 -1
- data/spec/redirect_spec.rb +86 -0
- data/spec/socksify_proxy_spec.rb +14 -14
- data/spec/stallion.rb +33 -2
- metadata +28 -6
- data/lib/em-http/http_options.rb +0 -53
@@ -0,0 +1,23 @@
|
|
1
|
+
class HttpConnectionOptions
|
2
|
+
attr_reader :host, :port, :tls, :proxy
|
3
|
+
attr_reader :connect_timeout, :inactivity_timeout
|
4
|
+
|
5
|
+
def initialize(uri, options)
|
6
|
+
@connect_timeout = options[:connect_timeout] || 5 # default connection setup timeout
|
7
|
+
@inactivity_timeout = options[:inactivity_timeout] ||= 10 # default connection inactivity (post-setup) timeout
|
8
|
+
|
9
|
+
@tls = options[:tls] || options[:ssl] || {}
|
10
|
+
@proxy = options[:proxy]
|
11
|
+
|
12
|
+
uri = uri.kind_of?(Addressable::URI) ? uri : Addressable::URI::parse(uri.to_s)
|
13
|
+
uri.port = (uri.scheme == "https" ? (uri.port || 443) : (uri.port || 80))
|
14
|
+
|
15
|
+
if proxy = options[:proxy]
|
16
|
+
@host = proxy[:host]
|
17
|
+
@port = proxy[:port]
|
18
|
+
else
|
19
|
+
@host = uri.host
|
20
|
+
@port = uri.port
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -63,7 +63,7 @@ module EventMachine
|
|
63
63
|
query.to_s
|
64
64
|
end
|
65
65
|
|
66
|
-
if !uri.query.to_s.empty?
|
66
|
+
if uri && !uri.query.to_s.empty?
|
67
67
|
encoded_query = [encoded_query, uri.query].reject {|part| part.empty?}.join("&")
|
68
68
|
end
|
69
69
|
encoded_query.to_s.empty? ? uri.path : "#{uri.path}?#{encoded_query}"
|
@@ -0,0 +1,38 @@
|
|
1
|
+
require 'cookiejar'
|
2
|
+
|
3
|
+
module EventMachine
|
4
|
+
module Middleware
|
5
|
+
class CookieJar
|
6
|
+
class << self
|
7
|
+
attr_reader :cookiejar
|
8
|
+
|
9
|
+
def cookiejar=(val)
|
10
|
+
@cookiejar = val
|
11
|
+
end
|
12
|
+
|
13
|
+
def set_cookie(url, cookie)
|
14
|
+
@cookiejar.set_cookie url, cookie
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def request(c, h, r)
|
19
|
+
raise ArgumentError, "You may not set cookies outside of the cookie jar" if h.delete('cookie')
|
20
|
+
cookies = CookieJar::cookiejar.get_cookie_header(c.last_effective_url)
|
21
|
+
h['cookie'] = cookies unless cookies.empty?
|
22
|
+
[h, r]
|
23
|
+
end
|
24
|
+
|
25
|
+
def response(r)
|
26
|
+
cookies = r.response_header.cookie
|
27
|
+
if cookies
|
28
|
+
[cookies].flatten.each { |c|
|
29
|
+
EventMachine::Middleware::CookieJar.cookiejar.set_cookie r.last_effective_url, c
|
30
|
+
}
|
31
|
+
end
|
32
|
+
r.response
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
EventMachine::Middleware::CookieJar.cookiejar = CookieJar::Jar.new
|
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'oauth'
|
2
|
+
require 'oauth/client/em_http'
|
3
|
+
|
4
|
+
module EventMachine
|
5
|
+
module Middleware
|
6
|
+
|
7
|
+
class OAuth
|
8
|
+
def initialize(opts = {})
|
9
|
+
@consumer = ::OAuth::Consumer.new(opts[:consumer_key], opts[:consumer_secret])
|
10
|
+
@access_token = ::OAuth::AccessToken.new(@consumer, opts[:access_token], opts[:access_token_secret])
|
11
|
+
end
|
12
|
+
|
13
|
+
def request(client, head, body)
|
14
|
+
@consumer.sign!(client, @access_token)
|
15
|
+
head.merge!(client.options[:head])
|
16
|
+
|
17
|
+
[head,body]
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
data/lib/em-http/multi.rb
CHANGED
@@ -8,17 +8,17 @@ module EventMachine
|
|
8
8
|
#
|
9
9
|
# EventMachine.run {
|
10
10
|
#
|
11
|
-
#
|
11
|
+
# multi = EventMachine::MultiRequest.new
|
12
12
|
#
|
13
|
-
#
|
14
|
-
#
|
15
|
-
#
|
13
|
+
# # add multiple requests to the multi-handler
|
14
|
+
# multi.add(:a, EventMachine::HttpRequest.new('http://www.google.com/').get)
|
15
|
+
# multi.add(:b, EventMachine::HttpRequest.new('http://www.yahoo.com/').get)
|
16
16
|
#
|
17
|
-
# multi.callback
|
18
|
-
#
|
19
|
-
#
|
17
|
+
# multi.callback {
|
18
|
+
# p multi.responses[:callback]
|
19
|
+
# p multi.responses[:errback]
|
20
20
|
#
|
21
|
-
#
|
21
|
+
# EventMachine.stop
|
22
22
|
# }
|
23
23
|
# }
|
24
24
|
#
|
@@ -28,27 +28,27 @@ module EventMachine
|
|
28
28
|
|
29
29
|
attr_reader :requests, :responses
|
30
30
|
|
31
|
-
def initialize
|
31
|
+
def initialize
|
32
32
|
@requests = []
|
33
|
-
@responses = {:
|
34
|
-
|
35
|
-
conns.each {|conn| add(conn)}
|
36
|
-
callback(&block) if block_given?
|
33
|
+
@responses = {:callback => {}, :errback => {}}
|
37
34
|
end
|
38
35
|
|
39
|
-
def add(conn)
|
36
|
+
def add(name, conn)
|
40
37
|
@requests.push(conn)
|
41
38
|
|
42
|
-
conn.callback { @responses[:
|
43
|
-
conn.errback { @responses[:
|
39
|
+
conn.callback { @responses[:callback][name] = conn; check_progress }
|
40
|
+
conn.errback { @responses[:errback][name] = conn; check_progress }
|
41
|
+
end
|
42
|
+
|
43
|
+
def finished?
|
44
|
+
(@responses[:callback].size + @responses[:errback].size) == @requests.size
|
44
45
|
end
|
45
46
|
|
46
47
|
protected
|
47
48
|
|
48
49
|
# invoke callback if all requests have completed
|
49
50
|
def check_progress
|
50
|
-
succeed(self) if
|
51
|
-
@responses[:failed].size) == @requests.size
|
51
|
+
succeed(self) if finished?
|
52
52
|
end
|
53
53
|
|
54
54
|
end
|
data/lib/em-http/request.rb
CHANGED
@@ -3,38 +3,16 @@ module EventMachine
|
|
3
3
|
@middleware = []
|
4
4
|
|
5
5
|
def self.new(uri, options={})
|
6
|
-
|
7
|
-
req = HttpOptions.new(uri, options)
|
6
|
+
connopt = HttpConnectionOptions.new(uri, options)
|
8
7
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
c.comm_inactivity_timeout = req.options[:inactivity_timeout]
|
14
|
-
end
|
15
|
-
|
16
|
-
rescue EventMachine::ConnectionError => e
|
17
|
-
#
|
18
|
-
# Currently, this can only fire on initial connection setup
|
19
|
-
# since #connect is a synchronous method. Hence, rescue the
|
20
|
-
# exception, and return a failed deferred which will immediately
|
21
|
-
# fail any client request.
|
22
|
-
#
|
23
|
-
# Once there is async-DNS, then we'll iterate over the outstanding
|
24
|
-
# client requests and fail them in order.
|
25
|
-
#
|
26
|
-
# Net outcome: failed connection will invoke the same ConnectionError
|
27
|
-
# message on the connection deferred, and on the client deferred.
|
28
|
-
#
|
29
|
-
conn = EventMachine::FailedConnection.new(req)
|
30
|
-
conn.error = e.message
|
31
|
-
conn.fail
|
32
|
-
conn
|
33
|
-
end
|
8
|
+
c = HttpConnection.new
|
9
|
+
c.connopts = connopt
|
10
|
+
c.uri = uri
|
11
|
+
c
|
34
12
|
end
|
35
13
|
|
36
|
-
def self.use(klass)
|
37
|
-
@middleware << klass
|
14
|
+
def self.use(klass, *args, &block)
|
15
|
+
@middleware << klass.new(*args, &block)
|
38
16
|
end
|
39
17
|
|
40
18
|
def self.middleware
|
data/lib/em-http/version.rb
CHANGED
data/spec/client_spec.rb
CHANGED
@@ -228,6 +228,25 @@ describe EventMachine::HttpRequest do
|
|
228
228
|
}
|
229
229
|
end
|
230
230
|
|
231
|
+
it "should return peer's IP address" do
|
232
|
+
EventMachine.run {
|
233
|
+
|
234
|
+
conn = EventMachine::HttpRequest.new('http://127.0.0.1:8090/')
|
235
|
+
conn.peer.should be_nil
|
236
|
+
|
237
|
+
http = conn.get
|
238
|
+
http.peer.should be_nil
|
239
|
+
|
240
|
+
http.errback { failed(http) }
|
241
|
+
http.callback {
|
242
|
+
conn.peer.should == '127.0.0.1'
|
243
|
+
http.peer.should == '127.0.0.1'
|
244
|
+
|
245
|
+
EventMachine.stop
|
246
|
+
}
|
247
|
+
}
|
248
|
+
end
|
249
|
+
|
231
250
|
it "should remove all newlines from long basic auth header" do
|
232
251
|
EventMachine.run {
|
233
252
|
auth = {'authorization' => ['aaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'zzzzzzzzzzzzzzzzzzzzzzzzzzzzzz']}
|
@@ -305,6 +324,26 @@ describe EventMachine::HttpRequest do
|
|
305
324
|
}
|
306
325
|
end
|
307
326
|
|
327
|
+
it "should not decode the response when configured so" do
|
328
|
+
EventMachine.run {
|
329
|
+
|
330
|
+
http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/gzip').get :head => {
|
331
|
+
"accept-encoding" => "gzip, compressed"
|
332
|
+
}, :decoding => false
|
333
|
+
|
334
|
+
http.errback { failed(http) }
|
335
|
+
http.callback {
|
336
|
+
http.response_header.status.should == 200
|
337
|
+
http.response_header["CONTENT_ENCODING"].should == "gzip"
|
338
|
+
|
339
|
+
raw = http.response
|
340
|
+
Zlib::GzipReader.new(StringIO.new(raw)).read.should == "compressed"
|
341
|
+
|
342
|
+
EventMachine.stop
|
343
|
+
}
|
344
|
+
}
|
345
|
+
end
|
346
|
+
|
308
347
|
it "should timeout after 0.1 seconds of inactivity" do
|
309
348
|
EventMachine.run {
|
310
349
|
t = Time.now.to_i
|
data/spec/external_spec.rb
CHANGED
@@ -79,6 +79,22 @@ requires_connection do
|
|
79
79
|
}
|
80
80
|
end
|
81
81
|
|
82
|
+
it "should detect deflate encoding" do
|
83
|
+
EventMachine.run {
|
84
|
+
|
85
|
+
options = {:head => {"accept-encoding" => "deflate"}, :redirects => 5}
|
86
|
+
http = EventMachine::HttpRequest.new('http://www.msn.com').get options
|
87
|
+
|
88
|
+
http.errback { failed(http) }
|
89
|
+
http.callback {
|
90
|
+
http.response_header.status.should == 200
|
91
|
+
http.response_header["CONTENT_ENCODING"].should == "deflate"
|
92
|
+
|
93
|
+
EventMachine.stop
|
94
|
+
}
|
95
|
+
}
|
96
|
+
end
|
97
|
+
|
82
98
|
context "keepalive" do
|
83
99
|
it "should default to non-keepalive" do
|
84
100
|
EventMachine.run {
|
@@ -88,7 +104,7 @@ requires_connection do
|
|
88
104
|
http.errback { fail }
|
89
105
|
start = Time.now.to_i
|
90
106
|
http.callback {
|
91
|
-
(
|
107
|
+
(Time.now.to_i - start).should be_within(2).of(0)
|
92
108
|
EventMachine.stop
|
93
109
|
}
|
94
110
|
}
|
data/spec/helper.rb
CHANGED
@@ -14,4 +14,16 @@ end
|
|
14
14
|
|
15
15
|
def requires_connection(&blk)
|
16
16
|
blk.call if system('ping -t1 -c1 google.com &> /dev/null')
|
17
|
+
end
|
18
|
+
|
19
|
+
def requires_port(port, &blk)
|
20
|
+
port_open = true
|
21
|
+
begin
|
22
|
+
s = TCPSocket.new('localhost', port)
|
23
|
+
s.close()
|
24
|
+
rescue
|
25
|
+
port_open = false
|
26
|
+
end
|
27
|
+
|
28
|
+
blk.call if port_open
|
17
29
|
end
|
data/spec/middleware_spec.rb
CHANGED
@@ -1,11 +1,12 @@
|
|
1
1
|
require 'helper'
|
2
|
+
require 'em-http/middleware/cookie_jar'
|
2
3
|
|
3
4
|
describe EventMachine::HttpRequest do
|
4
5
|
|
5
|
-
|
6
|
+
class EmptyMiddleware; end
|
6
7
|
|
7
8
|
class GlobalMiddleware
|
8
|
-
def
|
9
|
+
def response(resp)
|
9
10
|
resp.response_header['X-Global'] = 'middleware'
|
10
11
|
end
|
11
12
|
end
|
@@ -22,15 +23,44 @@ describe EventMachine::HttpRequest do
|
|
22
23
|
}
|
23
24
|
end
|
24
25
|
|
26
|
+
context "configuration" do
|
27
|
+
class ConfigurableMiddleware
|
28
|
+
def initialize(conf, &block)
|
29
|
+
@conf = conf
|
30
|
+
@block = block
|
31
|
+
end
|
32
|
+
|
33
|
+
def response(resp)
|
34
|
+
resp.response_header['X-Conf'] = @conf
|
35
|
+
resp.response_header['X-Block'] = @block.call
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
it "should accept middleware initialization parameters" do
|
40
|
+
EventMachine.run {
|
41
|
+
conn = EM::HttpRequest.new('http://127.0.0.1:8090')
|
42
|
+
conn.use ConfigurableMiddleware, 'conf-value' do
|
43
|
+
'block-value'
|
44
|
+
end
|
45
|
+
|
46
|
+
req = conn.get
|
47
|
+
req.callback {
|
48
|
+
req.response_header['X-Conf'].should match('conf-value')
|
49
|
+
req.response_header['X-Block'].should match('block-value')
|
50
|
+
EM.stop
|
51
|
+
}
|
52
|
+
}
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
25
56
|
context "request" do
|
26
57
|
class ResponseMiddleware
|
27
|
-
def
|
58
|
+
def response(resp)
|
28
59
|
resp.response_header['X-Header'] = 'middleware'
|
29
60
|
resp.response = 'Hello, Middleware!'
|
30
61
|
end
|
31
62
|
end
|
32
63
|
|
33
|
-
|
34
64
|
it "should execute response middleware before user callbacks" do
|
35
65
|
EventMachine.run {
|
36
66
|
conn = EM::HttpRequest.new('http://127.0.0.1:8090')
|
@@ -62,7 +92,7 @@ describe EventMachine::HttpRequest do
|
|
62
92
|
|
63
93
|
context "request" do
|
64
94
|
class RequestMiddleware
|
65
|
-
def
|
95
|
+
def request(client, head, body)
|
66
96
|
head['X-Middleware'] = 'middleware' # insert new header
|
67
97
|
body += ' modified' # modify post body
|
68
98
|
|
@@ -87,11 +117,11 @@ describe EventMachine::HttpRequest do
|
|
87
117
|
|
88
118
|
context "jsonify" do
|
89
119
|
class JSONify
|
90
|
-
def
|
120
|
+
def request(client, head, body)
|
91
121
|
[head, Yajl::Encoder.encode(body)]
|
92
122
|
end
|
93
123
|
|
94
|
-
def
|
124
|
+
def response(resp)
|
95
125
|
resp.response = Yajl::Parser.parse(resp.response)
|
96
126
|
end
|
97
127
|
end
|
@@ -111,4 +141,52 @@ describe EventMachine::HttpRequest do
|
|
111
141
|
end
|
112
142
|
end
|
113
143
|
|
144
|
+
context "CookieJar" do
|
145
|
+
it "should use the cookie jar as opposed to any other method when in use" do
|
146
|
+
lambda {
|
147
|
+
EventMachine.run {
|
148
|
+
conn = EventMachine::HttpRequest.new('http://127.0.0.1:8090/')
|
149
|
+
middleware = EventMachine::Middleware::CookieJar
|
150
|
+
conn.use middleware
|
151
|
+
middleware.set_cookie('http://127.0.0.1:8090/', 'id=1')
|
152
|
+
req = conn.get :head => {'cookie' => 'id=2;'}
|
153
|
+
req.callback { failed(req) }
|
154
|
+
req.errback { failed(req) }
|
155
|
+
}
|
156
|
+
}.should raise_error(ArgumentError)
|
157
|
+
end
|
158
|
+
|
159
|
+
it "should send cookies" do
|
160
|
+
EventMachine.run {
|
161
|
+
uri = 'http://127.0.0.1:8090/cookie_parrot'
|
162
|
+
conn = EventMachine::HttpRequest.new(uri)
|
163
|
+
middleware = EventMachine::Middleware::CookieJar
|
164
|
+
conn.use middleware
|
165
|
+
middleware.set_cookie(uri, 'id=1')
|
166
|
+
req = conn.get
|
167
|
+
req.callback {
|
168
|
+
req.response_header.cookie.should == 'id=1'
|
169
|
+
EventMachine.stop
|
170
|
+
}
|
171
|
+
}
|
172
|
+
end
|
173
|
+
|
174
|
+
it "should store cookies and send them" do
|
175
|
+
EventMachine.run {
|
176
|
+
uri = 'http://127.0.0.1:8090/set_cookie'
|
177
|
+
conn = EventMachine::HttpRequest.new(uri)
|
178
|
+
middleware = EventMachine::Middleware::CookieJar
|
179
|
+
conn.use middleware
|
180
|
+
req = conn.get
|
181
|
+
req.callback {
|
182
|
+
req.response_header.cookie[0..3].should == 'id=1'
|
183
|
+
cookies = middleware.cookiejar.get_cookies(uri)
|
184
|
+
cookies.length.should == 1
|
185
|
+
cookies[0].to_s.should == "id=1"
|
186
|
+
EventMachine.stop
|
187
|
+
}
|
188
|
+
}
|
189
|
+
end
|
190
|
+
end
|
191
|
+
|
114
192
|
end
|