astro-em-http-request 0.2.3 → 0.2.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -3,8 +3,10 @@
3
3
  EventMachine based HTTP Request interface. Supports streaming response processing, uses Ragel HTTP parser.
4
4
  - Simple interface for single & parallel requests via deferred callbacks
5
5
  - Automatic gzip & deflate decoding
6
- - Basic-Auth support
7
- - Custom timeouts
6
+ - Basic-Auth & OAuth support
7
+ - Custom timeouts
8
+ - Proxy support (with SSL Tunneling)
9
+ - Bi-directional communication with web-socket services
8
10
 
9
11
  Screencast / Demo of using EM-HTTP-Request:
10
12
  - http://everburning.com/news/eventmachine-screencast-em-http-request/
@@ -35,6 +37,7 @@ Screencast / Demo of using EM-HTTP-Request:
35
37
  }
36
38
 
37
39
  == Multi request example
40
+ Fire and wait for multiple requess to complete via the MultiRequest interface.
38
41
 
39
42
  EventMachine.run {
40
43
  multi = EventMachine::MultiRequest.new
@@ -52,6 +55,7 @@ Screencast / Demo of using EM-HTTP-Request:
52
55
  }
53
56
 
54
57
  == Basic-Auth example
58
+ Full basic author support. For OAuth, check examples/oauth-tweet.rb file.
55
59
 
56
60
  EventMachine.run {
57
61
  http = EventMachine::HttpRequest.new('http://www.website.com/').get :head => {'authorization' => ['user', 'pass']}
@@ -59,13 +63,12 @@ Screencast / Demo of using EM-HTTP-Request:
59
63
  http.errback { failed }
60
64
  http.callback {
61
65
  p http.response_header
62
-
63
66
  EventMachine.stop
64
67
  }
65
68
  }
66
69
 
67
- == POST example
68
70
 
71
+ == POST example
69
72
  EventMachine.run {
70
73
  http1 = EventMachine::HttpRequest.new('http://www.website.com/').post :body => {"key1" => 1, "key2" => [2,3]}
71
74
  http2 = EventMachine::HttpRequest.new('http://www.website.com/').post :body => "some data"
@@ -74,9 +77,44 @@ Screencast / Demo of using EM-HTTP-Request:
74
77
  }
75
78
 
76
79
  == Streaming body processing
80
+ Allows you to consume an HTTP stream of content in real-time. Each time a new piece of conent is pushed
81
+ to the client, it is passed to the stream callback for you to operate on.
82
+
77
83
  EventMachine.run {
78
84
  http = EventMachine::HttpRequest.new('http://www.website.com/').get
79
85
  http.stream { |chunk| print chunk }
80
86
 
81
87
  # ...
82
88
  }
89
+
90
+ == Proxy example
91
+ Full transparent proxy support with support for SSL tunneling.
92
+
93
+ EventMachine.run {
94
+ http = EventMachine::HttpRequest.new('http://www.website.com/').get :proxy => {
95
+ :host => 'www.myproxy.com',
96
+ :port => 8080,
97
+ :authorization => ['username', 'password'] # authorization is optional
98
+ }
99
+
100
+ == WebSocket example
101
+ Bi-directional communication with WebSockets: simply pass in a ws:// resource and the client will
102
+ negotiate the connection upgrade for you. On successfull handshake the callback is invoked, and
103
+ any incoming messages will be passed to the stream callback. The client can also send data to the
104
+ server at will by calling the "send" method!
105
+ - http://www.igvita.com/2009/12/22/ruby-websockets-tcp-for-the-browser/
106
+
107
+ EventMachine.run {
108
+ http = EventMachine::HttpRequest.new("ws://yourservice.com/websocket").get :timeout => 0
109
+
110
+ http.errback { puts "oops" }
111
+ http.callback {
112
+ puts "WebSocket connected!"
113
+ http.send("Hello client")
114
+ }
115
+
116
+ http.stream { |msg|
117
+ puts "Recieved: #{msg}"
118
+ http.send "Pong: #{msg}"
119
+ }
120
+ }
data/Rakefile CHANGED
@@ -35,7 +35,7 @@ task :ragel do
35
35
  end
36
36
 
37
37
  task :spec do
38
- sh 'spec -c test/test_*.rb'
38
+ sh 'spec spec/*_spec.rb'
39
39
  end
40
40
 
41
41
  def make(makedir)
@@ -91,11 +91,12 @@ begin
91
91
  gemspec.description = gemspec.summary
92
92
  gemspec.email = "ilya@igvita.com"
93
93
  gemspec.homepage = "http://github.com/igrigorik/em-http-request"
94
- gemspec.authors = ["Ilya Grigorik", "Stephan Maka"]
94
+ gemspec.authors = ["Ilya Grigorik", "Stephan Maka", "Julien Genestoux"]
95
95
  gemspec.extensions = ["ext/buffer/extconf.rb" , "ext/http11_client/extconf.rb"]
96
96
  gemspec.add_dependency('eventmachine', '>= 0.12.2')
97
97
  gemspec.add_dependency('addressable', '>= 2.0.0')
98
98
  gemspec.rubyforge_project = "em-http-request"
99
+ gemspec.files = FileList[`git ls-files`.split]
99
100
  end
100
101
 
101
102
  Jeweler::GemcutterTasks.new
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.2.3
1
+ 0.2.6
@@ -0,0 +1,50 @@
1
+ # Courtesy of Darcy Laycock:
2
+ # http://gist.github.com/265261
3
+ #
4
+
5
+ require 'rubygems'
6
+
7
+ require 'em-http'
8
+ require 'oauth'
9
+
10
+ # At a minimum, require 'oauth/request_proxy/em_http_request'
11
+ # for this example, we'll use Net::HTTP like support.
12
+ require 'oauth/client/em_http'
13
+
14
+ # You need two things: an oauth consumer and an access token.
15
+ # You need to generate an access token, I suggest looking elsewhere how to do that or wait for a full tutorial.
16
+ # For a consumer key / consumer secret, signup for an app at:
17
+ # http://twitter.com/apps/new
18
+
19
+ # Edit in your details.
20
+ CONSUMER_KEY = ""
21
+ CONSUMER_SECRET = ""
22
+ ACCESS_TOKEN = ""
23
+ ACCESS_TOKEN_SECRET = ""
24
+
25
+ def twitter_oauth_consumer
26
+ @twitter_oauth_consumer ||= OAuth::Consumer.new(CONSUMER_KEY, CONSUMER_SECRET, :site => "http://twitter.com")
27
+ end
28
+
29
+ def twitter_oauth_access_token
30
+ @twitter_oauth_access_token ||= OAuth::AccessToken.new(twitter_oauth_consumer, ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
31
+ end
32
+
33
+ EM.run do
34
+
35
+ request = EventMachine::HttpRequest.new('http://twitter.com/statuses/update.json')
36
+ http = request.post(:body => {'status' => 'Hello Twitter from em-http-request with OAuth'}, :head => {"Content-Type" => "application/x-www-form-urlencoded"}) do |client|
37
+ twitter_oauth_consumer.sign!(client, twitter_oauth_access_token)
38
+ end
39
+
40
+ http.callback do
41
+ puts "Response: #{http.response} (Code: #{http.response_header.status})"
42
+ EM.stop_event_loop
43
+ end
44
+
45
+ http.errback do
46
+ puts "Failed to post"
47
+ EM.stop_event_loop
48
+ end
49
+
50
+ end
@@ -21,12 +21,27 @@ static VALUE eHttpClientParserError;
21
21
  #define id_chunk_size rb_intern("@http_chunk_size")
22
22
  #define id_last_chunk rb_intern("@last_chunk")
23
23
 
24
+ #ifndef RHASH_TBL
25
+ /* rb_hash_lookup() is only in Ruby 1.8.7 */
26
+ static VALUE rb_hash_lookup(VALUE hash, VALUE key)
27
+ {
28
+ VALUE val;
29
+
30
+ if (!st_lookup(RHASH(hash)->tbl, key, &val)) {
31
+ return Qnil; /* without Hash#default */
32
+ }
33
+
34
+ return val;
35
+ }
36
+ #endif
37
+
24
38
  void client_http_field(void *data, const char *field, size_t flen, const char *value, size_t vlen)
25
39
  {
26
40
  char *ch, *end;
27
41
  VALUE req = (VALUE)data;
28
42
  VALUE v = Qnil;
29
43
  VALUE f = Qnil;
44
+ VALUE el = Qnil;
30
45
 
31
46
  v = rb_str_new(value, vlen);
32
47
  f = rb_str_new(field, flen);
@@ -41,7 +56,18 @@ void client_http_field(void *data, const char *field, size_t flen, const char *v
41
56
  }
42
57
  }
43
58
 
44
- rb_hash_aset(req, f, v);
59
+ el = rb_hash_lookup(req, f);
60
+ switch(TYPE(el)) {
61
+ case T_ARRAY:
62
+ rb_ary_push(el, v);
63
+ break;
64
+ case T_STRING:
65
+ rb_hash_aset(req, f, rb_ary_new3(2, el, v));
66
+ break;
67
+ default:
68
+ rb_hash_aset(req, f, v);
69
+ break;
70
+ }
45
71
  }
46
72
 
47
73
  void client_reason_phrase(void *data, const char *at, size_t length)
@@ -3,10 +3,8 @@
3
3
  # You can redistribute this under the terms of the Ruby license
4
4
  # See file LICENSE for details
5
5
  #++
6
-
7
- require 'rubygems'
8
- require 'eventmachine'
9
-
6
+
7
+ require 'eventmachine'
10
8
 
11
9
  require File.dirname(__FILE__) + '/http11_client'
12
10
  require File.dirname(__FILE__) + '/em_buffer'
@@ -15,4 +13,4 @@ require File.dirname(__FILE__) + '/em-http/core_ext/hash'
15
13
  require File.dirname(__FILE__) + '/em-http/client'
16
14
  require File.dirname(__FILE__) + '/em-http/multi'
17
15
  require File.dirname(__FILE__) + '/em-http/request'
18
- require File.dirname(__FILE__) + '/em-http/decoders'
16
+ require File.dirname(__FILE__) + '/em-http/decoders'
@@ -39,7 +39,7 @@ module EventMachine
39
39
  # Length of content as an integer, or nil if chunked/unspecified
40
40
  def content_length
41
41
  @content_length ||= ((s = self[HttpClient::CONTENT_LENGTH]) &&
42
- (s =~ /^(\d+)$/)) ? $1.to_i : nil
42
+ (s =~ /^(\d+)$/)) ? $1.to_i : nil
43
43
  end
44
44
 
45
45
  # Cookie header from the server
@@ -80,7 +80,6 @@ module EventMachine
80
80
  module HttpEncoding
81
81
  HTTP_REQUEST_HEADER="%s %s HTTP/1.1\r\n"
82
82
  FIELD_ENCODING = "%s: %s\r\n"
83
- BASIC_AUTH_ENCODING = "%s: Basic %s\r\n"
84
83
 
85
84
  # Escapes a URI.
86
85
  def escape(s)
@@ -108,18 +107,18 @@ module EventMachine
108
107
  @uri.host + (@uri.port != 80 ? ":#{@uri.port}" : "")
109
108
  end
110
109
 
111
- def encode_request(method, path, query)
112
- HTTP_REQUEST_HEADER % [method.to_s.upcase, encode_query(path, query)]
110
+ def encode_request(method, path, query, uri_query)
111
+ HTTP_REQUEST_HEADER % [method.to_s.upcase, encode_query(path, query, uri_query)]
113
112
  end
114
113
 
115
- def encode_query(path, query)
114
+ def encode_query(path, query, uri_query)
116
115
  encoded_query = if query.kind_of?(Hash)
117
116
  query.map { |k, v| encode_param(k, v) }.join('&')
118
117
  else
119
118
  query.to_s
120
119
  end
121
- if !@uri.query.to_s.empty?
122
- encoded_query = [encoded_query, @uri.query].reject {|part| part.empty?}.join("&")
120
+ if !uri_query.to_s.empty?
121
+ encoded_query = [encoded_query, uri_query].reject {|part| part.empty?}.join("&")
123
122
  end
124
123
  return path if encoded_query.to_s.empty?
125
124
  "#{path}?#{encoded_query}"
@@ -141,18 +140,25 @@ module EventMachine
141
140
  end
142
141
 
143
142
  # Encode basic auth in an HTTP header
144
- def encode_basic_auth(k,v)
145
- BASIC_AUTH_ENCODING % [k, Base64.encode64(v.join(":")).chomp]
143
+ # In: Array ([user, pass]) - for basic auth
144
+ # String - custom auth string (OAuth, etc)
145
+ def encode_auth(k,v)
146
+ if v.is_a? Array
147
+ FIELD_ENCODING % [k, ["Basic", Base64.encode64(v.join(":")).chomp].join(" ")]
148
+ else
149
+ encode_field(k,v)
150
+ end
146
151
  end
147
152
 
148
153
  def encode_headers(head)
149
154
  head.inject('') do |result, (key, value)|
150
155
  # Munge keys from foo-bar-baz to Foo-Bar-Baz
151
- key = key.split('-').map { |k| k.capitalize }.join('-')
152
- unless key == "Authorization"
153
- result << encode_field(key, value)
156
+ key = key.split('-').map { |k| k.to_s.capitalize }.join('-')
157
+ result << case key
158
+ when 'Authorization', 'Proxy-authorization'
159
+ encode_auth(key, value)
154
160
  else
155
- result << encode_basic_auth(key, value)
161
+ encode_field(key, value)
156
162
  end
157
163
  end
158
164
  end
@@ -185,31 +191,43 @@ module EventMachine
185
191
  def post_init
186
192
  @parser = HttpClientParser.new
187
193
  @data = EventMachine::Buffer.new
188
- @response_header = HttpResponseHeader.new
189
194
  @chunk_header = HttpChunkHeader.new
190
-
191
- @state = :response_header
195
+ @response_header = HttpResponseHeader.new
192
196
  @parser_nbytes = 0
193
197
  @response = ''
194
198
  @errors = ''
195
199
  @content_decoder = nil
196
200
  @stream = nil
201
+ @state = :response_header
202
+ @bytes_received = 0
203
+ @options = {}
197
204
  end
198
205
 
199
206
  # start HTTP request once we establish connection to host
200
- def connection_completed
201
- if @options[:max_connection_duration]
202
- EM.add_timer(@options[:max_connection_duration]) {
203
- @aborted = true
204
- on_error("Max Connection Duration Exceeded (#{@options[:max_connection_duration]}s.)")
205
- }
206
- end
207
- ssl = @options[:tls] || @options[:ssl] || {}
208
- @bytes_received = 0
209
- start_tls(ssl) if @uri.scheme == "https" #or @uri.port == 443 # A user might not want https even on port 443.
210
-
211
- send_request_header
212
- send_request_body
207
+ def connection_completed
208
+ # if connecting to proxy, then first negotiate the connection
209
+ # to intermediate server and wait for 200 response
210
+ if @options[:proxy] and @state == :response_header
211
+ @state = :response_proxy
212
+ send_request_header
213
+
214
+ # if connecting via proxy, then state will be :proxy_connected,
215
+ # indicating successful tunnel. from here, initiate normal http
216
+ # exchange
217
+ else
218
+ if @options[:max_connection_duration]
219
+ EM.add_timer(@options[:max_connection_duration]) {
220
+ @aborted = true
221
+ on_error("Max Connection Duration Exceeded (#{@options[:max_connection_duration]}s.)")
222
+ }
223
+ end
224
+ @state = :response_header
225
+
226
+ ssl = @options[:tls] || @options[:ssl] || {}
227
+ start_tls(ssl) if @uri.scheme == "https" #or @uri.port == 443 # A user might not want https even on port 443.
228
+ send_request_header
229
+ send_request_body
230
+ end
213
231
  end
214
232
 
215
233
  # request is done, invoke the callback
@@ -236,43 +254,87 @@ module EventMachine
236
254
  @stream = blk
237
255
  end
238
256
 
257
+ # raw data push from the client (WebSocket) should
258
+ # only be invoked after handshake, otherwise it will
259
+ # inject data into the header exchange
260
+ #
261
+ # frames need to start with 0x00-0x7f byte and end with
262
+ # an 0xFF byte. Per spec, we can also set the first
263
+ # byte to a value betweent 0x80 and 0xFF, followed by
264
+ # a leading length indicator
265
+ def send(data)
266
+ if @state == :websocket
267
+ send_data("\x00#{data}\xff")
268
+ end
269
+ end
270
+
239
271
  def normalize_body
240
- if @options[:body].is_a? Hash
241
- @options[:body].to_params
242
- else
243
- @options[:body]
272
+ @normalized_body ||= begin
273
+ if @options[:body].is_a? Hash
274
+ @options[:body].to_params
275
+ else
276
+ @options[:body]
277
+ end
278
+ end
279
+ end
280
+
281
+ def normalize_uri
282
+ @normalized_uri ||= begin
283
+ uri = @uri.dup
284
+ encoded_query = encode_query(@uri.path, @options[:query], @uri.query)
285
+ path, query = encoded_query.split("?", 2)
286
+ uri.query = query unless encoded_query.empty?
287
+ uri.path = path
288
+ uri
244
289
  end
245
290
  end
246
291
 
292
+ def websocket?; @uri.scheme == 'ws'; end
293
+
247
294
  def send_request_header
248
295
  query = @options[:query]
249
296
  head = @options[:head] ? munge_header_keys(@options[:head]) : {}
250
297
  body = normalize_body
298
+ request_header = nil
299
+
300
+ if @state == :response_proxy
301
+ proxy = @options[:proxy]
302
+
303
+ # initialize headers to establish the HTTP tunnel
304
+ head = proxy[:head] ? munge_header_keys(proxy[:head]) : {}
305
+ head['proxy-authorization'] = proxy[:authorization] if proxy[:authorization]
306
+ request_header = HTTP_REQUEST_HEADER % ['CONNECT', "#{@uri.host}:#{@uri.port}"]
307
+
308
+ elsif websocket?
309
+ head['upgrade'] = 'WebSocket'
310
+ head['connection'] = 'Upgrade'
311
+ head['origin'] = @options[:origin] || @uri.host
312
+
313
+ else
314
+ # Set the Content-Length if body is given
315
+ head['content-length'] = body.length if body
316
+
317
+ # Set the cookie header if provided
318
+ if cookie = head.delete('cookie')
319
+ head['cookie'] = encode_cookie(cookie)
320
+ end
321
+
322
+ # Set content-type header if missing and body is a Ruby hash
323
+ if not head['content-type'] and options[:body].is_a? Hash
324
+ head['content-type'] = "application/x-www-form-urlencoded"
325
+ end
326
+ end
251
327
 
252
328
  # Set the Host header if it hasn't been specified already
253
329
  head['host'] ||= encode_host
254
330
 
255
- # Set the Content-Length if body is given
256
- head['content-length'] = body.length if body
257
-
258
331
  # Set the User-Agent if it hasn't been specified
259
332
  head['user-agent'] ||= "EventMachine HttpClient"
260
333
 
261
- # Set the cookie header if provided
262
- if cookie = head.delete('cookie')
263
- head['cookie'] = encode_cookie(cookie)
264
- end
265
-
266
- # Set content-type header if missing and body is a Ruby hash
267
- if not head['content-type'] and options[:body].is_a? Hash
268
- head['content-type'] = "application/x-www-form-urlencoded"
269
- end
270
-
271
- # Build the request
272
- request_header = encode_request(@method, @uri.path, query)
334
+ # Build the request headers
335
+ request_header ||= encode_request(@method, @uri.path, query, @uri.query)
273
336
  request_header << encode_headers(head)
274
337
  request_header << CRLF
275
-
276
338
  send_data request_header
277
339
  end
278
340
 
@@ -333,6 +395,8 @@ module EventMachine
333
395
 
334
396
  def dispatch
335
397
  while case @state
398
+ when :response_proxy
399
+ parse_response_proxy
336
400
  when :response_header
337
401
  parse_response_header
338
402
  when :chunk_header
@@ -345,6 +409,8 @@ module EventMachine
345
409
  process_response_footer
346
410
  when :body
347
411
  process_body
412
+ when :websocket
413
+ process_websocket
348
414
  when :finished, :invalid
349
415
  break
350
416
  else raise RuntimeError, "invalid state: #{@state}"
@@ -370,6 +436,28 @@ module EventMachine
370
436
 
371
437
  true
372
438
  end
439
+
440
+ # TODO: refactor with parse_response_header
441
+ def parse_response_proxy
442
+ return false unless parse_header(@response_header)
443
+
444
+ unless @response_header.http_status and @response_header.http_reason
445
+ @state = :invalid
446
+ on_error "no HTTP response"
447
+ return false
448
+ end
449
+
450
+ # when a successfull tunnel is established, the proxy responds with a
451
+ # 200 response code. from here, the tunnel is transparent.
452
+ if @response_header.http_status.to_i == 200
453
+ @response_header = HttpResponseHeader.new
454
+ connection_completed
455
+ else
456
+ @state = :invalid
457
+ on_error "proxy not accessible"
458
+ return false
459
+ end
460
+ end
373
461
 
374
462
  def parse_response_header
375
463
  return false unless parse_header(@response_header)
@@ -394,17 +482,25 @@ module EventMachine
394
482
  end
395
483
  end
396
484
 
397
- # shortcircuit on HEAD requests
485
+ # shortcircuit on HEAD requests
398
486
  if @method == "HEAD"
399
487
  @state = :finished
400
488
  on_request_complete
401
489
  end
402
490
 
403
- if @response_header.chunked_encoding?
491
+ if websocket?
492
+ if @response_header.status == 101
493
+ @state = :websocket
494
+ succeed
495
+ else
496
+ fail "websocket handshake failed"
497
+ end
498
+
499
+ elsif @response_header.chunked_encoding?
404
500
  @state = :chunk_header
405
501
  elsif @response_header.content_length
406
502
  @state = :body
407
- @bytes_remaining = @response_header.content_length
503
+ @bytes_remaining = @response_header.content_length
408
504
  else
409
505
  @state = :body
410
506
  @bytes_remaining = nil
@@ -519,7 +615,23 @@ module EventMachine
519
615
  false
520
616
  end
521
617
 
522
-
618
+ def process_websocket
619
+ return false if @data.empty?
620
+
621
+ # slice the message out of the buffer and pass in
622
+ # for processing, and buffer data otherwise
623
+ buffer = @data.read
624
+ while msg = buffer.slice!(/\000([^\377]*)\377/)
625
+ msg.gsub!(/^\x00|\xff$/, '')
626
+ @stream.call(msg)
627
+ end
628
+
629
+ # store remainder if message boundary has not yet
630
+ # been recieved
631
+ @data << buffer if not buffer.empty?
632
+
633
+ false
634
+ end
523
635
  end
524
636
 
525
637
  end