ably-em-http-request 1.1.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (65) hide show
  1. checksums.yaml +7 -0
  2. data/.gemtest +0 -0
  3. data/.github/workflows/ci.yml +22 -0
  4. data/.gitignore +9 -0
  5. data/.rspec +0 -0
  6. data/Changelog.md +78 -0
  7. data/Gemfile +14 -0
  8. data/LICENSE +21 -0
  9. data/README.md +66 -0
  10. data/Rakefile +10 -0
  11. data/ably-em-http-request.gemspec +33 -0
  12. data/benchmarks/clients.rb +170 -0
  13. data/benchmarks/em-excon.rb +87 -0
  14. data/benchmarks/em-profile.gif +0 -0
  15. data/benchmarks/em-profile.txt +65 -0
  16. data/benchmarks/server.rb +48 -0
  17. data/examples/.gitignore +1 -0
  18. data/examples/digest_auth/client.rb +25 -0
  19. data/examples/digest_auth/server.rb +28 -0
  20. data/examples/fetch.rb +30 -0
  21. data/examples/fibered-http.rb +51 -0
  22. data/examples/multi.rb +25 -0
  23. data/examples/oauth-tweet.rb +35 -0
  24. data/examples/socks5.rb +23 -0
  25. data/lib/em/io_streamer.rb +51 -0
  26. data/lib/em-http/client.rb +343 -0
  27. data/lib/em-http/core_ext/bytesize.rb +6 -0
  28. data/lib/em-http/decoders.rb +252 -0
  29. data/lib/em-http/http_client_options.rb +51 -0
  30. data/lib/em-http/http_connection.rb +408 -0
  31. data/lib/em-http/http_connection_options.rb +72 -0
  32. data/lib/em-http/http_encoding.rb +151 -0
  33. data/lib/em-http/http_header.rb +85 -0
  34. data/lib/em-http/http_status_codes.rb +59 -0
  35. data/lib/em-http/middleware/digest_auth.rb +114 -0
  36. data/lib/em-http/middleware/json_response.rb +17 -0
  37. data/lib/em-http/middleware/oauth.rb +42 -0
  38. data/lib/em-http/middleware/oauth2.rb +30 -0
  39. data/lib/em-http/multi.rb +59 -0
  40. data/lib/em-http/request.rb +25 -0
  41. data/lib/em-http/version.rb +7 -0
  42. data/lib/em-http-request.rb +1 -0
  43. data/lib/em-http.rb +20 -0
  44. data/spec/client_fiber_spec.rb +23 -0
  45. data/spec/client_spec.rb +1000 -0
  46. data/spec/digest_auth_spec.rb +48 -0
  47. data/spec/dns_spec.rb +41 -0
  48. data/spec/encoding_spec.rb +49 -0
  49. data/spec/external_spec.rb +146 -0
  50. data/spec/fixtures/google.ca +16 -0
  51. data/spec/fixtures/gzip-sample.gz +0 -0
  52. data/spec/gzip_spec.rb +91 -0
  53. data/spec/helper.rb +27 -0
  54. data/spec/http_proxy_spec.rb +268 -0
  55. data/spec/middleware/oauth2_spec.rb +15 -0
  56. data/spec/middleware_spec.rb +143 -0
  57. data/spec/multi_spec.rb +104 -0
  58. data/spec/pipelining_spec.rb +62 -0
  59. data/spec/redirect_spec.rb +430 -0
  60. data/spec/socksify_proxy_spec.rb +56 -0
  61. data/spec/spec_helper.rb +25 -0
  62. data/spec/ssl_spec.rb +67 -0
  63. data/spec/stallion.rb +334 -0
  64. data/spec/stub_server.rb +45 -0
  65. metadata +269 -0
@@ -0,0 +1,28 @@
1
+ require 'webrick'
2
+
3
+ include WEBrick
4
+
5
+ config = { :Realm => 'DigestAuth_REALM' }
6
+
7
+ htdigest = WEBrick::HTTPAuth::Htdigest.new 'my_password_file'
8
+ htdigest.set_passwd config[:Realm], 'digest_username', 'digest_password'
9
+ htdigest.flush
10
+
11
+ config[:UserDB] = htdigest
12
+
13
+ digest_auth = WEBrick::HTTPAuth::DigestAuth.new config
14
+
15
+ class TestServlet < HTTPServlet::AbstractServlet
16
+ def do_GET(req, res)
17
+ @options[0][:authenticator].authenticate req, res
18
+ res.body = "You are authenticated to see the super secret data\n"
19
+ end
20
+ end
21
+
22
+ s = HTTPServer.new(:Port => 3000)
23
+ s.mount('/', TestServlet, {:authenticator => digest_auth})
24
+ trap("INT") do
25
+ File.delete('my_password_file')
26
+ s.shutdown
27
+ end
28
+ s.start
data/examples/fetch.rb ADDED
@@ -0,0 +1,30 @@
1
+ require 'rubygems'
2
+ require 'eventmachine'
3
+ require '../lib/em-http'
4
+
5
+ urls = ARGV
6
+ if urls.size < 1
7
+ puts "Usage: #{$0} <url> <url> <...>"
8
+ exit
9
+ end
10
+
11
+ pending = urls.size
12
+
13
+ EM.run do
14
+ urls.each do |url|
15
+ http = EM::AblyHttpRequest::HttpRequest.new(url).get
16
+ http.callback {
17
+ puts "#{url}\n#{http.response_header.status} - #{http.response.length} bytes\n"
18
+ puts http.response
19
+
20
+ pending -= 1
21
+ EM.stop if pending < 1
22
+ }
23
+ http.errback {
24
+ puts "#{url}\n" + http.error
25
+
26
+ pending -= 1
27
+ EM.stop if pending < 1
28
+ }
29
+ end
30
+ end
@@ -0,0 +1,51 @@
1
+ $: << 'lib' << '../lib'
2
+
3
+ require 'eventmachine'
4
+ require 'em-http'
5
+ require 'fiber'
6
+
7
+ # Using Fibers in Ruby 1.9 to simulate blocking IO / IO scheduling
8
+ # while using the async EventMachine API's
9
+
10
+ def async_fetch(url)
11
+ f = Fiber.current
12
+ http = EventMachine::AblyHttpRequest::HttpRequest.new(url, :connect_timeout => 10, :inactivity_timeout => 20).get
13
+
14
+ http.callback { f.resume(http) }
15
+ http.errback { f.resume(http) }
16
+
17
+ Fiber.yield
18
+
19
+ if http.error
20
+ p [:HTTP_ERROR, http.error]
21
+ end
22
+
23
+ http
24
+ end
25
+
26
+ EventMachine.run do
27
+ Fiber.new{
28
+
29
+ puts "Setting up HTTP request #1"
30
+ data = async_fetch('http://0.0.0.0/')
31
+ puts "Fetched page #1: #{data.response_header.status}"
32
+
33
+ puts "Setting up HTTP request #2"
34
+ data = async_fetch('http://www.yahoo.com/')
35
+ puts "Fetched page #2: #{data.response_header.status}"
36
+
37
+ puts "Setting up HTTP request #3"
38
+ data = async_fetch('http://non-existing.domain/')
39
+ puts "Fetched page #3: #{data.response_header.status}"
40
+
41
+ EventMachine.stop
42
+ }.resume
43
+ end
44
+
45
+ puts "Done"
46
+
47
+ # Setting up HTTP request #1
48
+ # Fetched page #1: 302
49
+ # Setting up HTTP request #2
50
+ # Fetched page #2: 200
51
+ # Done
data/examples/multi.rb ADDED
@@ -0,0 +1,25 @@
1
+ $: << '../lib' << 'lib'
2
+
3
+ require 'eventmachine'
4
+ require 'em-http'
5
+
6
+ EventMachine.run {
7
+ multi = EventMachine::AblyHttpRequest::MultiRequest.new
8
+
9
+ reqs = [
10
+ 'http://google.com/',
11
+ 'http://google.ca:81/'
12
+ ]
13
+
14
+ reqs.each_with_index do |url, idx|
15
+ http = EventMachine::AblyHttpRequest::HttpRequest.new(url, :connect_timeout => 1)
16
+ req = http.get
17
+ multi.add idx, req
18
+ end
19
+
20
+ multi.callback do
21
+ p multi.responses[:callback].size
22
+ p multi.responses[:errback].size
23
+ EventMachine.stop
24
+ end
25
+ }
@@ -0,0 +1,35 @@
1
+ $: << 'lib' << '../lib'
2
+
3
+ require 'em-http'
4
+ require 'em-http/middleware/oauth'
5
+ require 'em-http/middleware/json_response'
6
+
7
+ require 'pp'
8
+
9
+ OAuthConfig = {
10
+ :consumer_key => '',
11
+ :consumer_secret => '',
12
+ :access_token => '',
13
+ :access_token_secret => ''
14
+ }
15
+
16
+ EM.run do
17
+ # automatically parse the JSON response into a Ruby object
18
+ EventMachine::AblyHttpRequest::HttpRequest.use EventMachine::AblyHttpRequest::Middleware::JSONResponse
19
+
20
+ # sign the request with OAuth credentials
21
+ conn = EventMachine::AblyHttpRequest::HttpRequest.new('http://api.twitter.com/1/statuses/home_timeline.json')
22
+ conn.use EventMachine::AblyHttpRequest::Middleware::OAuth, OAuthConfig
23
+
24
+ http = conn.get
25
+ http.callback do
26
+ pp http.response
27
+ EM.stop
28
+ end
29
+
30
+ http.errback do
31
+ puts "Failed retrieving user stream."
32
+ pp http.response
33
+ EM.stop
34
+ end
35
+ end
@@ -0,0 +1,23 @@
1
+ require 'rubygems'
2
+ require 'eventmachine'
3
+ require '../lib/em-http'
4
+
5
+ EM.run do
6
+ # Establish a SOCKS5 tunnel via SSH
7
+ # ssh -D 8000 some_remote_machine
8
+
9
+ connection_options = {:proxy => {:host => '127.0.0.1', :port => 8000, :type => :socks5}}
10
+ http = EM::AblyHttpRequest::HttpRequest.new('http://igvita.com/', connection_options).get :redirects => 2
11
+
12
+ http.callback {
13
+ puts "#{http.response_header.status} - #{http.response.length} bytes\n"
14
+ puts http.response
15
+ EM.stop
16
+ }
17
+
18
+ http.errback {
19
+ puts "Error: " + http.error
20
+ puts http.inspect
21
+ EM.stop
22
+ }
23
+ end
@@ -0,0 +1,51 @@
1
+ require 'em/streamer'
2
+
3
+ # similar to EventMachine::FileStreamer, but for any IO object
4
+ module EventMachine
5
+ module AblyHttpRequest
6
+ class IOStreamer
7
+ include Deferrable
8
+ CHUNK_SIZE = 16384
9
+
10
+ # @param [EventMachine::Connection] connection
11
+ # @param [IO] io Data source
12
+ # @param [Integer] Data size
13
+ #
14
+ # @option opts [Boolean] :http_chunks (false) Use HTTP 1.1 style chunked-encoding semantics.
15
+ def initialize(connection, io, opts = {})
16
+ @connection = connection
17
+ @io = io
18
+ @http_chunks = opts[:http_chunks]
19
+
20
+ @buff = String.new
21
+ @io.binmode if @io.respond_to?(:binmode)
22
+ stream_one_chunk
23
+ end
24
+
25
+ private
26
+
27
+ # Used internally to stream one chunk at a time over multiple reactor ticks
28
+ # @private
29
+ def stream_one_chunk
30
+ loop do
31
+ if @io.eof?
32
+ @connection.send_data "0\r\n\r\n" if @http_chunks
33
+ succeed
34
+ break
35
+ end
36
+
37
+ if @connection.respond_to?(:get_outbound_data_size) && (@connection.get_outbound_data_size > FileStreamer::BackpressureLevel)
38
+ EventMachine::next_tick { stream_one_chunk }
39
+ break
40
+ end
41
+
42
+ if @io.read(CHUNK_SIZE, @buff)
43
+ @connection.send_data("#{@buff.length.to_s(16)}\r\n") if @http_chunks
44
+ @connection.send_data(@buff)
45
+ @connection.send_data("\r\n") if @http_chunks
46
+ end
47
+ end
48
+ end
49
+ end
50
+ end
51
+ end
@@ -0,0 +1,343 @@
1
+ require 'cookiejar'
2
+
3
+ module EventMachine
4
+ module AblyHttpRequest
5
+
6
+
7
+ class HttpClient
8
+ include Deferrable
9
+ include HttpEncoding
10
+ include HttpStatus
11
+
12
+ TRANSFER_ENCODING="TRANSFER_ENCODING"
13
+ CONTENT_ENCODING="CONTENT_ENCODING"
14
+ CONTENT_LENGTH="CONTENT_LENGTH"
15
+ CONTENT_TYPE="CONTENT_TYPE"
16
+ LAST_MODIFIED="LAST_MODIFIED"
17
+ KEEP_ALIVE="CONNECTION"
18
+ SET_COOKIE="SET_COOKIE"
19
+ LOCATION="LOCATION"
20
+ HOST="HOST"
21
+ ETAG="ETAG"
22
+
23
+ CRLF="\r\n"
24
+
25
+ attr_accessor :state, :response, :conn
26
+ attr_reader :response_header, :error, :content_charset, :req, :cookies
27
+
28
+ def initialize(conn, options)
29
+ @conn = conn
30
+ @req = options
31
+
32
+ @stream = nil
33
+ @headers = nil
34
+ @cookies = []
35
+ @cookiejar = CookieJar.new
36
+
37
+ reset!
38
+ end
39
+
40
+ def reset!
41
+ @response_header = HttpResponseHeader.new
42
+ @state = :response_header
43
+
44
+ @response = ''
45
+ @error = nil
46
+ @content_decoder = nil
47
+ @content_charset = nil
48
+ end
49
+
50
+ def last_effective_url; @req.uri; end
51
+ def redirects; @req.followed; end
52
+ def peer; @conn.peer; end
53
+
54
+ def connection_completed
55
+ @state = :response_header
56
+
57
+ head, body = build_request, @req.body
58
+ @conn.middleware.each do |m|
59
+ head, body = m.request(self, head, body) if m.respond_to?(:request)
60
+ end
61
+
62
+ send_request(head, body)
63
+ end
64
+
65
+ def on_request_complete
66
+ begin
67
+ @content_decoder.finalize! if @content_decoder
68
+ rescue HttpDecoders::DecoderError
69
+ on_error "Content-decoder error"
70
+ end
71
+
72
+ unbind
73
+ end
74
+
75
+ def continue?
76
+ @response_header.status == 100 && (@req.method == 'POST' || @req.method == 'PUT')
77
+ end
78
+
79
+ def finished?
80
+ @state == :finished || (@state == :body && @response_header.content_length.nil?)
81
+ end
82
+
83
+ def redirect?
84
+ @response_header.redirection? && @req.follow_redirect?
85
+ end
86
+
87
+ def unbind(reason = nil)
88
+ if finished?
89
+ if redirect?
90
+
91
+ begin
92
+ @conn.middleware.each do |m|
93
+ m.response(self) if m.respond_to?(:response)
94
+ end
95
+
96
+ # one of the injected middlewares could have changed
97
+ # our redirect settings, check if we still want to
98
+ # follow the location header
99
+ if redirect?
100
+ @req.followed += 1
101
+
102
+ @cookies.clear
103
+ @cookies = @cookiejar.get(@response_header.location).map(&:to_s) if @req.pass_cookies
104
+
105
+ @conn.redirect(self, @response_header.location)
106
+ else
107
+ succeed(self)
108
+ end
109
+
110
+ rescue => e
111
+ on_error(e.message)
112
+ end
113
+ else
114
+ succeed(self)
115
+ end
116
+
117
+ else
118
+ on_error(reason || 'connection closed by server')
119
+ end
120
+ end
121
+
122
+ def on_error(msg = nil)
123
+ @error = msg
124
+ fail(self)
125
+ end
126
+ alias :close :on_error
127
+
128
+ def stream(&blk); @stream = blk; end
129
+ def headers(&blk); @headers = blk; end
130
+
131
+ def normalize_body(body)
132
+ body.is_a?(Hash) ? form_encode_body(body) : body
133
+ end
134
+
135
+ def build_request
136
+ head = @req.headers ? munge_header_keys(@req.headers) : {}
137
+
138
+ if @conn.connopts.http_proxy?
139
+ proxy = @conn.connopts.proxy
140
+ head['proxy-authorization'] = proxy[:authorization] if proxy[:authorization]
141
+ end
142
+
143
+ # Set the cookie header if provided
144
+ if cookie = head['cookie']
145
+ @cookies << encode_cookie(cookie) if cookie
146
+ end
147
+ head['cookie'] = @cookies.compact.uniq.join("; ").squeeze(";") unless @cookies.empty?
148
+
149
+ # Set connection close unless keepalive
150
+ if !@req.keepalive
151
+ head['connection'] = 'close'
152
+ end
153
+
154
+ # Set the Host header if it hasn't been specified already
155
+ head['host'] ||= encode_host
156
+
157
+ # Set the User-Agent if it hasn't been specified
158
+ if !head.key?('user-agent')
159
+ head['user-agent'] = 'EventMachine HttpClient'
160
+ elsif head['user-agent'].nil?
161
+ head.delete('user-agent')
162
+ end
163
+
164
+ # Set the Accept-Encoding header if none is provided
165
+ if !head.key?('accept-encoding') && req.compressed
166
+ head['accept-encoding'] = 'gzip, compressed'
167
+ end
168
+
169
+ # Set the auth from the URI if given
170
+ head['Authorization'] = @req.uri.userinfo.split(/:/, 2) if @req.uri.userinfo
171
+
172
+ head
173
+ end
174
+
175
+ def send_request(head, body)
176
+ body = normalize_body(body)
177
+ file = @req.file
178
+ query = @req.query
179
+
180
+ # Set the Content-Length if file is given
181
+ head['content-length'] = File.size(file) if file
182
+
183
+ # Set the Content-Length if body is given,
184
+ # or we're doing an empty post or put
185
+ if body
186
+ head['content-length'] ||= body.respond_to?(:bytesize) ? body.bytesize : body.size
187
+ elsif @req.method == 'POST' or @req.method == 'PUT'
188
+ # wont happen if body is set and we already set content-length above
189
+ head['content-length'] ||= 0
190
+ end
191
+
192
+ # Set content-type header if missing and body is a Ruby hash
193
+ if !head['content-type'] and @req.body.is_a? Hash
194
+ head['content-type'] = 'application/x-www-form-urlencoded'
195
+ end
196
+
197
+ request_header ||= encode_request(@req.method, @req.uri, query, @conn.connopts)
198
+ request_header << encode_headers(head)
199
+ request_header << CRLF
200
+ @conn.send_data request_header
201
+
202
+ @req_body = body || (@req.file && Pathname.new(@req.file))
203
+ send_request_body unless @req.headers['expect'] == '100-continue'
204
+ end
205
+
206
+ def on_body_data(data)
207
+ if @content_decoder
208
+ begin
209
+ @content_decoder << data
210
+ rescue HttpDecoders::DecoderError
211
+ on_error "Content-decoder error"
212
+ end
213
+ else
214
+ on_decoded_body_data(data)
215
+ end
216
+ end
217
+
218
+ def on_decoded_body_data(data)
219
+ data.force_encoding @content_charset if @content_charset
220
+ if @stream
221
+ @stream.call(data)
222
+ else
223
+ @response << data
224
+ end
225
+ end
226
+
227
+ def request_body_pending?
228
+ !!@req_body
229
+ end
230
+
231
+ def send_request_body
232
+ return if @req_body.nil?
233
+
234
+ if @req_body.is_a?(String)
235
+ @conn.send_data @req_body
236
+
237
+ elsif @req_body.is_a?(Pathname)
238
+ @conn.stream_file_data @req_body.to_path, http_chunks: false
239
+
240
+ elsif @req_body.respond_to?(:read) && @req_body.respond_to?(:eof?) # IO or IO-like object
241
+ @conn.stream_data @req_body
242
+
243
+ else
244
+ raise "Don't know how to send request body: #{@req_body.inspect}"
245
+ end
246
+ @req_body = nil
247
+ end
248
+
249
+ def parse_response_header(header, version, status)
250
+ @response_header.raw = header
251
+ header.each do |key, val|
252
+ @response_header[key.upcase.gsub('-','_')] = val
253
+ end
254
+
255
+ @response_header.http_version = version.join('.')
256
+ @response_header.http_status = status
257
+ @response_header.http_reason = CODE[status] || 'unknown'
258
+
259
+ # invoke headers callback after full parse
260
+ # if one is specified by the user
261
+ @headers.call(@response_header) if @headers
262
+
263
+ unless @response_header.http_status and @response_header.http_reason
264
+ @state = :invalid
265
+ on_error "no HTTP response"
266
+ return
267
+ end
268
+
269
+ # add set-cookie's to cookie list
270
+ if @response_header.cookie && @req.pass_cookies
271
+ [@response_header.cookie].flatten.each {|cookie| @cookiejar.set(cookie, @req.uri)}
272
+ end
273
+
274
+ # correct location header - some servers will incorrectly give a relative URI
275
+ if @response_header.location
276
+ begin
277
+ location = Addressable::URI.parse(@response_header.location)
278
+ location.path = "/" if location.path.empty?
279
+
280
+ if location.relative?
281
+ location = @req.uri.join(location)
282
+ else
283
+ # if redirect is to an absolute url, check for correct URI structure
284
+ raise if location.host.nil?
285
+ end
286
+
287
+ @response_header[LOCATION] = location.to_s
288
+
289
+ rescue
290
+ on_error "Location header format error"
291
+ return
292
+ end
293
+ end
294
+
295
+ # Fire callbacks immediately after recieving header requests
296
+ # if the request method is HEAD. In case of a redirect, terminate
297
+ # current connection and reinitialize the process.
298
+ if @req.method == "HEAD"
299
+ @state = :finished
300
+ return
301
+ end
302
+
303
+ if @response_header.chunked_encoding?
304
+ @state = :chunk_header
305
+ elsif @response_header.content_length
306
+ @state = :body
307
+ else
308
+ @state = :body
309
+ end
310
+
311
+ if @req.decoding && decoder_class = HttpDecoders.decoder_for_encoding(response_header[CONTENT_ENCODING])
312
+ begin
313
+ @content_decoder = decoder_class.new do |s| on_decoded_body_data(s) end
314
+ rescue HttpDecoders::DecoderError
315
+ on_error "Content-decoder error"
316
+ end
317
+ end
318
+
319
+ # handle malformed header - Content-Type repetitions.
320
+ content_type = [response_header[CONTENT_TYPE]].flatten.first
321
+
322
+ if String.method_defined?(:force_encoding) && /;\s*charset=\s*(.+?)\s*(;|$)/.match(content_type)
323
+ @content_charset = Encoding.find($1.gsub(/^\"|\"$/, '')) rescue Encoding.default_external
324
+ end
325
+ end
326
+
327
+ class CookieJar
328
+ def initialize
329
+ @jar = ::CookieJar::Jar.new
330
+ end
331
+
332
+ def set string, uri
333
+ @jar.set_cookie(uri, string) rescue nil # drop invalid cookies
334
+ end
335
+
336
+ def get uri
337
+ uri = URI.parse(uri) rescue nil
338
+ uri ? @jar.get_cookies(uri) : []
339
+ end
340
+ end # CookieJar
341
+ end
342
+ end
343
+ end
@@ -0,0 +1,6 @@
1
+ # bytesize was introduced in 1.8.7+
2
+ if RUBY_VERSION <= "1.8.6"
3
+ class String
4
+ def bytesize; self.size; end
5
+ end
6
+ end