z-http-request 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. checksums.yaml +7 -0
  2. data/.gemtest +0 -0
  3. data/.gitignore +10 -0
  4. data/.rspec +3 -0
  5. data/.ruby-version +1 -0
  6. data/.travis.yml +8 -0
  7. data/Gemfile +17 -0
  8. data/README.md +38 -0
  9. data/Rakefile +3 -0
  10. data/benchmarks/clients.rb +170 -0
  11. data/benchmarks/em-excon.rb +87 -0
  12. data/benchmarks/em-profile.gif +0 -0
  13. data/benchmarks/em-profile.txt +65 -0
  14. data/benchmarks/server.rb +48 -0
  15. data/examples/.gitignore +1 -0
  16. data/examples/digest_auth/client.rb +25 -0
  17. data/examples/digest_auth/server.rb +28 -0
  18. data/examples/fetch.rb +30 -0
  19. data/examples/fibered-http.rb +51 -0
  20. data/examples/multi.rb +25 -0
  21. data/examples/oauth-tweet.rb +35 -0
  22. data/examples/socks5.rb +23 -0
  23. data/lib/z-http/client.rb +318 -0
  24. data/lib/z-http/core_ext/bytesize.rb +6 -0
  25. data/lib/z-http/decoders.rb +254 -0
  26. data/lib/z-http/http_client_options.rb +51 -0
  27. data/lib/z-http/http_connection.rb +214 -0
  28. data/lib/z-http/http_connection_options.rb +44 -0
  29. data/lib/z-http/http_encoding.rb +142 -0
  30. data/lib/z-http/http_header.rb +83 -0
  31. data/lib/z-http/http_status_codes.rb +57 -0
  32. data/lib/z-http/middleware/digest_auth.rb +112 -0
  33. data/lib/z-http/middleware/json_response.rb +15 -0
  34. data/lib/z-http/middleware/oauth.rb +40 -0
  35. data/lib/z-http/middleware/oauth2.rb +28 -0
  36. data/lib/z-http/multi.rb +57 -0
  37. data/lib/z-http/request.rb +23 -0
  38. data/lib/z-http/version.rb +5 -0
  39. data/lib/z-http-request.rb +1 -0
  40. data/lib/z-http.rb +18 -0
  41. data/spec/client_spec.rb +892 -0
  42. data/spec/digest_auth_spec.rb +48 -0
  43. data/spec/dns_spec.rb +44 -0
  44. data/spec/encoding_spec.rb +49 -0
  45. data/spec/external_spec.rb +150 -0
  46. data/spec/fixtures/google.ca +16 -0
  47. data/spec/fixtures/gzip-sample.gz +0 -0
  48. data/spec/gzip_spec.rb +68 -0
  49. data/spec/helper.rb +30 -0
  50. data/spec/middleware_spec.rb +143 -0
  51. data/spec/multi_spec.rb +104 -0
  52. data/spec/pipelining_spec.rb +66 -0
  53. data/spec/redirect_spec.rb +321 -0
  54. data/spec/socksify_proxy_spec.rb +60 -0
  55. data/spec/spec_helper.rb +6 -0
  56. data/spec/ssl_spec.rb +20 -0
  57. data/spec/stallion.rb +296 -0
  58. data/spec/stub_server.rb +42 -0
  59. data/z-http-request.gemspec +33 -0
  60. metadata +248 -0
@@ -0,0 +1,66 @@
1
+ require 'helper'
2
+
3
+ requires_connection do
4
+
5
+ describe ZMachine::HttpRequest do
6
+
7
+ it "should perform successful pipelined GETs" do
8
+ ZMachine.run do
9
+
10
+ # Mongrel doesn't support pipelined requests - bah!
11
+ conn = ZMachine::HttpRequest.new('http://www.igvita.com/')
12
+
13
+ pipe1 = conn.get :keepalive => true
14
+ pipe2 = conn.get :path => '/archives/', :keepalive => true
15
+
16
+ processed = 0
17
+ stop = proc { ZMachine.stop if processed == 2}
18
+
19
+ pipe1.errback { failed(conn) }
20
+ pipe1.callback {
21
+ processed += 1
22
+ pipe1.response_header.status.should == 200
23
+ stop.call
24
+ }
25
+
26
+ pipe2.errback { failed(conn) }
27
+ pipe2.callback {
28
+ processed += 1
29
+ pipe2.response_header.status.should == 200
30
+ pipe2.response.should match(/html/i)
31
+ stop.call
32
+ }
33
+
34
+ end
35
+ end
36
+
37
+ it "should perform successful pipelined HEAD requests" do
38
+ ZMachine.run do
39
+ conn = ZMachine::HttpRequest.new('http://www.igvita.com/')
40
+
41
+ pipe1 = conn.head :keepalive => true
42
+ pipe2 = conn.head :path => '/archives/', :keepalive => true
43
+
44
+ processed = 0
45
+ stop = proc { ZMachine.stop if processed == 2}
46
+
47
+ pipe1.errback { failed(conn) }
48
+ pipe1.callback {
49
+ processed += 1
50
+ pipe1.response_header.status.should == 200
51
+ stop.call
52
+ }
53
+
54
+ pipe2.errback { failed(conn) }
55
+ pipe2.callback {
56
+ processed += 1
57
+ pipe2.response_header.status.should == 200
58
+ stop.call
59
+ }
60
+
61
+ end
62
+
63
+ end
64
+ end
65
+
66
+ end
@@ -0,0 +1,321 @@
1
+ require 'helper'
2
+
3
+ class RedirectMiddleware
4
+ attr_reader :call_count
5
+
6
+ def initialize
7
+ @call_count = 0
8
+ end
9
+
10
+ def request(c, h, r)
11
+ @call_count += 1
12
+ [h.merge({'EM-Middleware' => @call_count.to_s}), r]
13
+ end
14
+ end
15
+
16
+ class PickyRedirectMiddleware < RedirectMiddleware
17
+ def response(r)
18
+ if r.redirect? && r.response_header['LOCATION'][-1].chr == '3'
19
+ # set redirects to 0 to avoid further processing
20
+ r.req.redirects = 0
21
+ end
22
+ end
23
+ end
24
+
25
+ describe ZMachine::HttpRequest do
26
+
27
+ it "should follow location redirects" do
28
+ ZMachine.run {
29
+ http = ZMachine::HttpRequest.new('http://127.0.0.1:8090/redirect').get :redirects => 1
30
+ http.errback { failed(http) }
31
+ http.callback {
32
+ http.response_header.status.should == 200
33
+ http.response_header["CONTENT_ENCODING"].should == "gzip"
34
+ http.response.should == "compressed"
35
+ http.last_effective_url.to_s.should == 'http://127.0.0.1:8090/gzip'
36
+ http.redirects.should == 1
37
+
38
+ ZMachine.stop
39
+ }
40
+ }
41
+ end
42
+
43
+ it "should not follow redirects on created" do
44
+ ZMachine.run {
45
+ http = ZMachine::HttpRequest.new('http://127.0.0.1:8090/redirect/created').get :redirects => 1
46
+ http.errback { failed(http) }
47
+ http.callback {
48
+ http.response_header.status.should == 201
49
+ http.response.should match(/Hello/)
50
+ ZMachine.stop
51
+ }
52
+ }
53
+ end
54
+
55
+ it "should not forward cookies across domains with http redirect" do
56
+
57
+ expires = (Date.today + 2).strftime('%a, %d %b %Y %T GMT')
58
+ response =<<-HTTP.gsub(/^ +/, '')
59
+ HTTP/1.1 301 MOVED PERMANENTLY
60
+ Location: http://localhost:8071/
61
+ Set-Cookie: foo=bar; expires=#{expires}; path=/; HttpOnly
62
+
63
+ HTTP
64
+
65
+ ZMachine.run do
66
+ @stub = StubServer.new(:host => '127.0.0.1', :port => 8070, :response => response)
67
+ @echo = StubServer.new(:host => 'localhost', :port => 8071, :echo => true)
68
+
69
+ http = ZMachine::HttpRequest.new('http://127.0.0.1:8070/').get :redirects => 1
70
+
71
+ http.errback { failed(http) }
72
+ http.callback do
73
+ http.response.should_not match(/Cookie/)
74
+ @stub.stop
75
+ @echo.stop
76
+ ZMachine.stop
77
+ end
78
+ end
79
+ end
80
+
81
+ it "should forward valid cookies across domains with http redirect" do
82
+
83
+ expires = (Date.today + 2).strftime('%a, %d %b %Y %T GMT')
84
+ response =<<-HTTP.gsub(/^ +/, '')
85
+ HTTP/1.1 301 MOVED PERMANENTLY
86
+ Location: http://127.0.0.1:8071/
87
+ Set-Cookie: foo=bar; expires=#{expires}; path=/; HttpOnly
88
+
89
+ HTTP
90
+
91
+ ZMachine.run do
92
+ @stub = StubServer.new(:host => '127.0.0.1', :port => 8070, :response => response)
93
+ @echo = StubServer.new(:host => '127.0.0.1', :port => 8071, :echo => true)
94
+
95
+ http = ZMachine::HttpRequest.new('http://127.0.0.1:8070/').get :redirects => 1
96
+
97
+ http.errback { failed(http) }
98
+ http.callback do
99
+ http.response.should match(/Cookie/)
100
+ @stub.stop
101
+ @echo.stop
102
+ ZMachine.stop
103
+ end
104
+ end
105
+ end
106
+
107
+
108
+ it "should normalize path and forward valid cookies across domains" do
109
+
110
+ expires = (Date.today + 2).strftime('%a, %d %b %Y %T GMT')
111
+ response =<<-HTTP.gsub(/^ +/, '')
112
+ HTTP/1.1 301 MOVED PERMANENTLY
113
+ Location: http://127.0.0.1:8071?omg=ponies
114
+ Set-Cookie: foo=bar; expires=#{expires}; path=/; HttpOnly
115
+
116
+ HTTP
117
+
118
+ ZMachine.run do
119
+ @stub = StubServer.new(:host => '127.0.0.1', :port => 8070, :response => response)
120
+ @echo = StubServer.new(:host => '127.0.0.1', :port => 8071, :echo => true)
121
+
122
+ http = ZMachine::HttpRequest.new('http://127.0.0.1:8070/').get :redirects => 1
123
+
124
+ http.errback { failed(http) }
125
+ http.callback do
126
+ http.response.should match(/Cookie/)
127
+ @stub.stop
128
+ @echo.stop
129
+ ZMachine.stop
130
+ end
131
+ end
132
+ end
133
+
134
+ it "should redirect with missing content-length" do
135
+ ZMachine.run {
136
+ response = "HTTP/1.0 301 MOVED PERMANENTLY\r\nlocation: http://127.0.0.1:8090/redirect\r\n\r\n"
137
+ @stub = StubServer.new(:host => '127.0.0.1', :port => 8070, :response => response)
138
+
139
+ http = ZMachine::HttpRequest.new('http://127.0.0.1:8070/').get :redirects => 3
140
+ http.errback { failed(http) }
141
+
142
+ http.callback {
143
+ http.response_header.status.should == 200
144
+ http.response_header["CONTENT_ENCODING"].should == "gzip"
145
+ http.response.should == "compressed"
146
+ http.last_effective_url.to_s.should == 'http://127.0.0.1:8090/gzip'
147
+ http.redirects.should == 3
148
+
149
+ @stub.stop
150
+ ZMachine.stop
151
+ }
152
+ }
153
+ end
154
+
155
+ it "should follow redirects on HEAD method" do
156
+ ZMachine.run {
157
+ http = ZMachine::HttpRequest.new('http://127.0.0.1:8090/redirect/head').head :redirects => 1
158
+ http.errback { failed(http) }
159
+ http.callback {
160
+ http.response_header.status.should == 200
161
+ http.last_effective_url.to_s.should == 'http://127.0.0.1:8090/'
162
+ ZMachine.stop
163
+ }
164
+ }
165
+ end
166
+
167
+ it "should report last_effective_url" do
168
+ ZMachine.run {
169
+ http = ZMachine::HttpRequest.new('http://127.0.0.1:8090/').get
170
+ http.errback { failed(http) }
171
+ http.callback {
172
+ http.response_header.status.should == 200
173
+ http.last_effective_url.to_s.should == 'http://127.0.0.1:8090/'
174
+
175
+ ZMachine.stop
176
+ }
177
+ }
178
+ end
179
+
180
+ it "should default to 0 redirects" do
181
+ ZMachine.run {
182
+ http = ZMachine::HttpRequest.new('http://127.0.0.1:8090/redirect').get
183
+ http.errback { failed(http) }
184
+ http.callback {
185
+ http.response_header.status.should == 301
186
+ http.last_effective_url.to_s.should == 'http://127.0.0.1:8090/redirect'
187
+ http.redirects.should == 0
188
+
189
+ ZMachine.stop
190
+ }
191
+ }
192
+ end
193
+
194
+ it "should not invoke redirect logic on failed(http) connections" do
195
+ ZMachine.run {
196
+ http = ZMachine::HttpRequest.new('http://127.0.0.1:8070/', :connect_timeout => 0.1).get :redirects => 5
197
+ http.callback { failed(http) }
198
+ http.errback {
199
+ http.redirects.should == 0
200
+ ZMachine.stop
201
+ }
202
+ }
203
+ end
204
+
205
+ it "should normalize redirect urls" do
206
+ ZMachine.run {
207
+ http = ZMachine::HttpRequest.new('http://127.0.0.1:8090/redirect/bad').get :redirects => 1
208
+ http.errback { failed(http) }
209
+ http.callback {
210
+ http.last_effective_url.to_s.should match('http://127.0.0.1:8090/')
211
+ http.response.should match('Hello, World!')
212
+ ZMachine.stop
213
+ }
214
+ }
215
+ end
216
+
217
+ it "should fail gracefully on a missing host in absolute Location header" do
218
+ ZMachine.run {
219
+ http = ZMachine::HttpRequest.new('http://127.0.0.1:8090/redirect/nohost').get :redirects => 1
220
+ http.callback { failed(http) }
221
+ http.errback {
222
+ http.error.should == 'Location header format error'
223
+ ZMachine.stop
224
+ }
225
+ }
226
+ end
227
+
228
+ it "should apply timeout settings on redirects" do
229
+ ZMachine.run {
230
+ t = Time.now.to_i
231
+ ZMachine.heartbeat_interval = 0.1
232
+
233
+ conn = ZMachine::HttpRequest.new('http://127.0.0.1:8090/redirect/timeout', :inactivity_timeout => 0.1)
234
+ http = conn.get :redirects => 1
235
+ http.callback { failed(http) }
236
+ http.errback {
237
+ (Time.now.to_i - t).should <= 1
238
+ ZMachine.stop
239
+ }
240
+ }
241
+ end
242
+
243
+ it "should capture and pass cookies on redirect and pass_cookies by default" do
244
+ ZMachine.run {
245
+ http = ZMachine::HttpRequest.new('http://127.0.0.1:8090/redirect/multiple-with-cookie').get :redirects => 2, :head => {'cookie' => 'id=2;'}
246
+ http.errback { failed(http) }
247
+ http.callback {
248
+ http.response_header.status.should == 200
249
+ http.response_header["CONTENT_ENCODING"].should == "gzip"
250
+ http.response.should == "compressed"
251
+ http.last_effective_url.to_s.should == 'http://127.0.0.1:8090/gzip'
252
+ http.redirects.should == 2
253
+ http.cookies.should include("id=2;")
254
+ http.cookies.should include("another_id=1")
255
+
256
+ ZMachine.stop
257
+ }
258
+ }
259
+ end
260
+
261
+ it "should capture and not pass cookies on redirect if passing is disabled via pass_cookies" do
262
+ ZMachine.run {
263
+ http = ZMachine::HttpRequest.new('http://127.0.0.1:8090/redirect/multiple-with-cookie').get :redirects => 2, :pass_cookies => false, :head => {'cookie' => 'id=2;'}
264
+ http.errback { failed(http) }
265
+ http.callback {
266
+ http.response_header.status.should == 200
267
+ http.response_header["CONTENT_ENCODING"].should == "gzip"
268
+ http.response.should == "compressed"
269
+ http.last_effective_url.to_s.should == 'http://127.0.0.1:8090/gzip'
270
+ http.redirects.should == 2
271
+ http.cookies.should include("id=2;")
272
+ http.cookies.should_not include("another_id=1; expires=Sat, 09 Aug 2031 17:53:39 GMT; path=/;")
273
+
274
+ ZMachine.stop
275
+ }
276
+ }
277
+ end
278
+
279
+ it "should follow location redirects with path" do
280
+ ZMachine.run {
281
+ http = ZMachine::HttpRequest.new('http://127.0.0.1:8090/redirect').get :path => '/redirect', :redirects => 1
282
+ http.errback { failed(http) }
283
+ http.callback {
284
+ http.last_effective_url.to_s.should == 'http://127.0.0.1:8090/gzip'
285
+ http.response_header.status.should == 200
286
+ http.redirects.should == 1
287
+
288
+ ZMachine.stop
289
+ }
290
+ }
291
+ end
292
+
293
+ it "should call middleware each time it redirects" do
294
+ ZMachine.run {
295
+ conn = ZMachine::HttpRequest.new('http://127.0.0.1:8090/redirect/middleware_redirects_1')
296
+ conn.use RedirectMiddleware
297
+ http = conn.get :redirects => 3
298
+ http.errback { failed(http) }
299
+ http.callback {
300
+ http.response_header.status.should == 200
301
+ http.response_header['EM_MIDDLEWARE'].to_i.should == 3
302
+ ZMachine.stop
303
+ }
304
+ }
305
+ end
306
+
307
+ it "should call middleware which may reject a redirection" do
308
+ ZMachine.run {
309
+ conn = ZMachine::HttpRequest.new('http://127.0.0.1:8090/redirect/middleware_redirects_1')
310
+ conn.use PickyRedirectMiddleware
311
+ http = conn.get :redirects => 3
312
+ http.errback { failed(http) }
313
+ http.callback {
314
+ http.response_header.status.should == 301
315
+ http.last_effective_url.to_s.should == 'http://127.0.0.1:8090/redirect/middleware_redirects_2'
316
+ ZMachine.stop
317
+ }
318
+ }
319
+ end
320
+
321
+ end
@@ -0,0 +1,60 @@
1
+ require 'helper'
2
+
3
+ requires_connection do
4
+
5
+ requires_port(8080) do
6
+ describe ZMachine::HttpRequest do
7
+
8
+ # ssh -D 8080 igvita
9
+ let(:proxy) { {:proxy => { :host => '127.0.0.1', :port => 8080, :type => :socks5 }} }
10
+
11
+ it "should use SOCKS5 proxy" do
12
+ ZMachine.run {
13
+ http = ZMachine::HttpRequest.new('http://jsonip.com/', proxy).get
14
+
15
+ http.errback { failed(http) }
16
+ http.callback {
17
+ http.response_header.status.should == 200
18
+ http.response.should match('173.230.151.99')
19
+ ZMachine.stop
20
+ }
21
+ }
22
+ end
23
+ end
24
+ end
25
+
26
+ requires_port(8081) do
27
+ describe ZMachine::HttpRequest do
28
+
29
+ # brew install tinyproxy
30
+ let(:http_proxy) { {:proxy => { :host => '127.0.0.1', :port => 8081 }} }
31
+
32
+ it "should use HTTP proxy by default" do
33
+ ZMachine.run {
34
+ http = ZMachine::HttpRequest.new('http://jsonip.com/', http_proxy).get
35
+
36
+ http.errback { failed(http) }
37
+ http.callback {
38
+ http.response_header.status.should == 200
39
+ http.response.should match(/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/)
40
+ ZMachine.stop
41
+ }
42
+ }
43
+ end
44
+
45
+ it "should auto CONNECT via HTTP proxy for HTTPS requests" do
46
+ ZMachine.run {
47
+ http = ZMachine::HttpRequest.new('https://ipjson.herokuapp.com/', http_proxy).get
48
+
49
+ http.errback { failed(http) }
50
+ http.callback {
51
+ http.response_header.status.should == 200
52
+ http.response.should match(/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/)
53
+ ZMachine.stop
54
+ }
55
+ }
56
+ end
57
+ end
58
+ end
59
+
60
+ end
@@ -0,0 +1,6 @@
1
+ require 'liquid/boot'
2
+ require 'zmachine'
3
+
4
+ #ZMachine.logger = $log
5
+ #ZMachine.debug = true
6
+ #$log.level = :debug
data/spec/ssl_spec.rb ADDED
@@ -0,0 +1,20 @@
1
+ require 'helper'
2
+
3
+ requires_connection do
4
+
5
+ describe ZMachine::HttpRequest do
6
+
7
+ it "should initiate SSL/TLS on HTTPS connections" do
8
+ ZMachine.run {
9
+ http = ZMachine::HttpRequest.new('https://mail.google.com:443/mail/').get
10
+
11
+ http.errback { failed(http) }
12
+ http.callback {
13
+ http.response_header.status.should == 302
14
+ ZMachine.stop
15
+ }
16
+ }
17
+ end
18
+ end
19
+
20
+ end