julien51-em-http-request 0.1.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,48 @@
1
+ /**
2
+ * Copyright (c) 2005 Zed A. Shaw
3
+ * You can redistribute it and/or modify it under the same terms as Ruby.
4
+ */
5
+
6
+ #ifndef http11_parser_h
7
+ #define http11_parser_h
8
+
9
+ #include <sys/types.h>
10
+
11
+ #if defined(_WIN32)
12
+ #include <stddef.h>
13
+ #endif
14
+
15
+ typedef void (*element_cb)(void *data, const char *at, size_t length);
16
+ typedef void (*field_cb)(void *data, const char *field, size_t flen, const char *value, size_t vlen);
17
+
18
+ typedef struct httpclient_parser {
19
+ int cs;
20
+ size_t body_start;
21
+ int content_len;
22
+ size_t nread;
23
+ size_t mark;
24
+ size_t field_start;
25
+ size_t field_len;
26
+
27
+ void *data;
28
+
29
+ field_cb http_field;
30
+ element_cb reason_phrase;
31
+ element_cb status_code;
32
+ element_cb chunk_size;
33
+ element_cb http_version;
34
+ element_cb header_done;
35
+ element_cb last_chunk;
36
+
37
+
38
+ } httpclient_parser;
39
+
40
+ int httpclient_parser_init(httpclient_parser *parser);
41
+ int httpclient_parser_finish(httpclient_parser *parser);
42
+ size_t httpclient_parser_execute(httpclient_parser *parser, const char *data, size_t len, size_t off);
43
+ int httpclient_parser_has_error(httpclient_parser *parser);
44
+ int httpclient_parser_is_finished(httpclient_parser *parser);
45
+
46
+ #define httpclient_parser_nread(parser) (parser)->nread
47
+
48
+ #endif
@@ -0,0 +1,173 @@
1
+ /**
2
+ * Copyright (c) 2005 Zed A. Shaw
3
+ * You can redistribute it and/or modify it under the same terms as Ruby.
4
+ */
5
+
6
+ #include "http11_parser.h"
7
+ #include <stdio.h>
8
+ #include <assert.h>
9
+ #include <stdlib.h>
10
+ #include <ctype.h>
11
+ #include <string.h>
12
+
13
+ #define LEN(AT, FPC) (FPC - buffer - parser->AT)
14
+ #define MARK(M,FPC) (parser->M = (FPC) - buffer)
15
+ #define PTR_TO(F) (buffer + parser->F)
16
+ #define L(M) fprintf(stderr, "" # M "\n");
17
+
18
+
19
+ /** machine **/
20
+ %%{
21
+ machine httpclient_parser;
22
+
23
+ action mark {MARK(mark, fpc); }
24
+
25
+ action start_field { MARK(field_start, fpc); }
26
+
27
+ action write_field {
28
+ parser->field_len = LEN(field_start, fpc);
29
+ }
30
+
31
+ action start_value { MARK(mark, fpc); }
32
+
33
+ action write_value {
34
+ parser->http_field(parser->data, PTR_TO(field_start), parser->field_len, PTR_TO(mark), LEN(mark, fpc));
35
+ }
36
+
37
+ action reason_phrase {
38
+ parser->reason_phrase(parser->data, PTR_TO(mark), LEN(mark, fpc));
39
+ }
40
+
41
+ action status_code {
42
+ parser->status_code(parser->data, PTR_TO(mark), LEN(mark, fpc));
43
+ }
44
+
45
+ action http_version {
46
+ parser->http_version(parser->data, PTR_TO(mark), LEN(mark, fpc));
47
+ }
48
+
49
+ action chunk_size {
50
+ parser->chunk_size(parser->data, PTR_TO(mark), LEN(mark, fpc));
51
+ }
52
+
53
+ action last_chunk {
54
+ parser->last_chunk(parser->data, NULL, 0);
55
+ }
56
+
57
+ action done {
58
+ parser->body_start = fpc - buffer + 1;
59
+ if(parser->header_done != NULL)
60
+ parser->header_done(parser->data, fpc + 1, pe - fpc - 1);
61
+ fbreak;
62
+ }
63
+
64
+ # line endings
65
+ CRLF = "\r\n";
66
+
67
+ # character types
68
+ CTL = (cntrl | 127);
69
+ tspecials = ("(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\\" | "\"" | "/" | "[" | "]" | "?" | "=" | "{" | "}" | " " | "\t");
70
+
71
+ # elements
72
+ token = (ascii -- (CTL | tspecials));
73
+
74
+ Reason_Phrase = (any -- CRLF)* >mark %reason_phrase;
75
+ Status_Code = digit{3} >mark %status_code;
76
+ http_number = (digit+ "." digit+) ;
77
+ HTTP_Version = ("HTTP/" http_number) >mark %http_version ;
78
+ Status_Line = HTTP_Version " " Status_Code " "? Reason_Phrase :> CRLF;
79
+
80
+ field_name = token+ >start_field %write_field;
81
+ field_value = any* >start_value %write_value;
82
+ message_header = field_name ":" " "* field_value :> CRLF;
83
+
84
+ Response = Status_Line (message_header)* (CRLF @done);
85
+
86
+ chunk_ext_val = token+;
87
+ chunk_ext_name = token+;
88
+ chunk_extension = (";" chunk_ext_name >start_field %write_field %start_value ("=" chunk_ext_val >start_value)? %write_value )*;
89
+ last_chunk = "0"? chunk_extension :> (CRLF @last_chunk @done);
90
+ chunk_size = xdigit+;
91
+ chunk = chunk_size >mark %chunk_size chunk_extension space* :> (CRLF @done);
92
+ Chunked_Header = (chunk | last_chunk);
93
+
94
+ main := Response | Chunked_Header;
95
+ }%%
96
+
97
+ /** Data **/
98
+ %% write data;
99
+
100
+ int httpclient_parser_init(httpclient_parser *parser) {
101
+ int cs = 0;
102
+ %% write init;
103
+ parser->cs = cs;
104
+ parser->body_start = 0;
105
+ parser->content_len = 0;
106
+ parser->mark = 0;
107
+ parser->nread = 0;
108
+ parser->field_len = 0;
109
+ parser->field_start = 0;
110
+
111
+ return(1);
112
+ }
113
+
114
+
115
+ /** exec **/
116
+ size_t httpclient_parser_execute(httpclient_parser *parser, const char *buffer, size_t len, size_t off) {
117
+ const char *p, *pe;
118
+ int cs = parser->cs;
119
+
120
+ assert(off <= len && "offset past end of buffer");
121
+
122
+ p = buffer+off;
123
+ pe = buffer+len;
124
+
125
+ assert(*pe == '\0' && "pointer does not end on NUL");
126
+ assert(pe - p == len - off && "pointers aren't same distance");
127
+
128
+
129
+ %% write exec;
130
+
131
+ parser->cs = cs;
132
+ parser->nread += p - (buffer + off);
133
+
134
+ assert(p <= pe && "buffer overflow after parsing execute");
135
+ assert(parser->nread <= len && "nread longer than length");
136
+ assert(parser->body_start <= len && "body starts after buffer end");
137
+ assert(parser->mark < len && "mark is after buffer end");
138
+ assert(parser->field_len <= len && "field has length longer than whole buffer");
139
+ assert(parser->field_start < len && "field starts after buffer end");
140
+
141
+ if(parser->body_start) {
142
+ /* final \r\n combo encountered so stop right here */
143
+ %%write eof;
144
+ parser->nread++;
145
+ }
146
+
147
+ return(parser->nread);
148
+ }
149
+
150
+ int httpclient_parser_finish(httpclient_parser *parser)
151
+ {
152
+ int cs = parser->cs;
153
+
154
+ %%write eof;
155
+
156
+ parser->cs = cs;
157
+
158
+ if (httpclient_parser_has_error(parser) ) {
159
+ return -1;
160
+ } else if (httpclient_parser_is_finished(parser) ) {
161
+ return 1;
162
+ } else {
163
+ return 0;
164
+ }
165
+ }
166
+
167
+ int httpclient_parser_has_error(httpclient_parser *parser) {
168
+ return parser->cs == httpclient_parser_error;
169
+ }
170
+
171
+ int httpclient_parser_is_finished(httpclient_parser *parser) {
172
+ return parser->cs == httpclient_parser_first_final;
173
+ }
data/lib/em-http.rb ADDED
@@ -0,0 +1,18 @@
1
+ #--
2
+ # Copyright (C)2008 Ilya Grigorik
3
+ # You can redistribute this under the terms of the Ruby license
4
+ # See file LICENSE for details
5
+ #++
6
+
7
+ require 'rubygems'
8
+ require 'eventmachine'
9
+
10
+
11
+ require File.dirname(__FILE__) + '/http11_client'
12
+ require File.dirname(__FILE__) + '/em_buffer'
13
+
14
+ require File.dirname(__FILE__) + '/em-http/core_ext/hash'
15
+ require File.dirname(__FILE__) + '/em-http/client'
16
+ require File.dirname(__FILE__) + '/em-http/multi'
17
+ require File.dirname(__FILE__) + '/em-http/request'
18
+ require File.dirname(__FILE__) + '/em-http/decoders'
@@ -0,0 +1,462 @@
1
+ # #--
2
+ # Copyright (C)2008 Ilya Grigorik
3
+ #
4
+ # Includes portion originally Copyright (C)2007 Tony Arcieri
5
+ # Includes portion originally Copyright (C)2005 Zed Shaw
6
+ # You can redistribute this under the terms of the Ruby
7
+ # license See file LICENSE for details
8
+ # #--
9
+
10
+ module EventMachine
11
+
12
+ # A simple hash is returned for each request made by HttpClient with the
13
+ # headers that were given by the server for that request.
14
+ class HttpResponseHeader < Hash
15
+ # The reason returned in the http response ("OK","File not found",etc.)
16
+ attr_accessor :http_reason
17
+
18
+ # The HTTP version returned.
19
+ attr_accessor :http_version
20
+
21
+ # The status code (as a string!)
22
+ attr_accessor :http_status
23
+
24
+ # HTTP response status as an integer
25
+ def status
26
+ Integer(http_status) rescue nil
27
+ end
28
+
29
+ # Length of content as an integer, or nil if chunked/unspecified
30
+ def content_length
31
+ Integer(self[HttpClient::CONTENT_LENGTH]) rescue nil
32
+ end
33
+
34
+ # Cookie header from the server
35
+ def cookie
36
+ self[HttpClient::SET_COOKIE]
37
+ end
38
+
39
+ # Is the transfer encoding chunked?
40
+ def chunked_encoding?
41
+ /chunked/i === self[HttpClient::TRANSFER_ENCODING]
42
+ end
43
+
44
+ def keep_alive?
45
+ /keep-alive/i === self[HttpClient::KEEP_ALIVE]
46
+ end
47
+
48
+ def compressed?
49
+ /gzip|compressed|deflate/i === self[HttpClient::CONTENT_ENCODING]
50
+ end
51
+ end
52
+
53
+ class HttpChunkHeader < Hash
54
+ # When parsing chunked encodings this is set
55
+ attr_accessor :http_chunk_size
56
+
57
+ # Size of the chunk as an integer
58
+ def chunk_size
59
+ return @chunk_size unless @chunk_size.nil?
60
+ @chunk_size = @http_chunk_size ? @http_chunk_size.to_i(base=16) : 0
61
+ end
62
+ end
63
+
64
+ # Methods for building HTTP requests
65
+ module HttpEncoding
66
+ HTTP_REQUEST_HEADER="%s %s HTTP/1.1\r\n"
67
+ FIELD_ENCODING = "%s: %s\r\n"
68
+ BASIC_AUTH_ENCODING = "%s: Basic %s\r\n"
69
+
70
+ # Escapes a URI.
71
+ def escape(s)
72
+ s.to_s.gsub(/([^ a-zA-Z0-9_.-]+)/n) {
73
+ '%'+$1.unpack('H2'*$1.size).join('%').upcase
74
+ }.tr(' ', '+')
75
+ end
76
+
77
+ # Unescapes a URI escaped string.
78
+ def unescape(s)
79
+ s.tr('+', ' ').gsub(/((?:%[0-9a-fA-F]{2})+)/n){
80
+ [$1.delete('%')].pack('H*')
81
+ }
82
+ end
83
+
84
+ # Map all header keys to a downcased string version
85
+ def munge_header_keys(head)
86
+ head.inject({}) { |h, (k, v)| h[k.to_s.downcase] = v; h }
87
+ end
88
+
89
+ # HTTP is kind of retarded that you have to specify a Host header, but if
90
+ # you include port 80 then further redirects will tack on the :80 which is
91
+ # annoying.
92
+ def encode_host
93
+ @uri.host + (@uri.port.to_i != 80 ? ":#{@uri.port}" : "")
94
+ end
95
+
96
+ def encode_request(method, path, query)
97
+ HTTP_REQUEST_HEADER % [method.to_s.upcase, encode_query(path, query)]
98
+ end
99
+
100
+ def encode_query(path, query)
101
+ return path unless query
102
+ if query.kind_of? String
103
+ return "#{path}?#{query}"
104
+ else
105
+ return path + "?" + query.map { |k, v| encode_param(k, v) }.join('&')
106
+ end
107
+ end
108
+
109
+ # URL encodes query parameters:
110
+ # single k=v, or a URL encoded array, if v is an array of values
111
+ def encode_param(k, v)
112
+ if v.is_a?(Array)
113
+ v.map { |e| escape(k) + "[]=" + escape(e) }.join("&")
114
+ else
115
+ escape(k) + "=" + escape(v)
116
+ end
117
+ end
118
+
119
+ # Encode a field in an HTTP header
120
+ def encode_field(k, v)
121
+ FIELD_ENCODING % [k, v]
122
+ end
123
+
124
+ # Encode basic auth in an HTTP header
125
+ def encode_basic_auth(k,v)
126
+ BASIC_AUTH_ENCODING % [k, Base64.encode64(v.join(":")).chomp]
127
+ end
128
+
129
+ def encode_headers(head)
130
+ head.inject('') do |result, (key, value)|
131
+ # Munge keys from foo-bar-baz to Foo-Bar-Baz
132
+ key = key.split('-').map { |k| k.capitalize }.join('-')
133
+ unless key == "Authorization"
134
+ result << encode_field(key, value)
135
+ else
136
+ result << encode_basic_auth(key, value)
137
+ end
138
+ end
139
+ end
140
+
141
+ def encode_cookie(cookie)
142
+ if cookie.is_a? Hash
143
+ cookie.inject('') { |result, (k, v)| result << encode_param(k, v) + ";" }
144
+ else
145
+ cookie
146
+ end
147
+ end
148
+ end
149
+
150
+ class HttpClient < Connection
151
+ include EventMachine::Deferrable
152
+ include HttpEncoding
153
+
154
+ TRANSFER_ENCODING="TRANSFER_ENCODING"
155
+ CONTENT_ENCODING="CONTENT_ENCODING"
156
+ CONTENT_LENGTH="CONTENT_LENGTH"
157
+ KEEP_ALIVE="CONNECTION"
158
+ SET_COOKIE="SET_COOKIE"
159
+ LOCATION="LOCATION"
160
+ HOST="HOST"
161
+ CRLF="\r\n"
162
+
163
+ attr_accessor :method, :options, :uri
164
+ attr_reader :response, :response_header, :errors
165
+
166
+ def post_init
167
+ @parser = HttpClientParser.new
168
+ @data = EventMachine::Buffer.new
169
+ @response_header = HttpResponseHeader.new
170
+ @chunk_header = HttpChunkHeader.new
171
+
172
+ @state = :response_header
173
+ @parser_nbytes = 0
174
+ @response = ''
175
+ @inflate = []
176
+ @errors = ''
177
+ @content_decoder = nil
178
+ @stream = nil
179
+ end
180
+
181
+ # start HTTP request once we establish connection to host
182
+ def connection_completed
183
+ ssl = @options[:tls] || @options[:ssl] || {}
184
+ start_tls(ssl) if @uri.scheme == "https" or @uri.port == 443
185
+
186
+ send_request_header
187
+ send_request_body
188
+ end
189
+
190
+ # request is done, invoke the callback
191
+ def on_request_complete
192
+ begin
193
+ @content_decoder.finalize! if @content_decoder
194
+ rescue HttpDecoders::DecoderError
195
+ on_error "Content-decoder error"
196
+ end
197
+ unbind
198
+ end
199
+
200
+ # request failed, invoke errback
201
+ def on_error(msg)
202
+ @errors = msg
203
+ unbind
204
+ end
205
+
206
+ # assign a stream processing block
207
+ def stream(&blk)
208
+ @stream = blk
209
+ end
210
+
211
+ def normalize_body
212
+ if @options[:body].is_a? Hash
213
+ @options[:body].to_params
214
+ else
215
+ @options[:body]
216
+ end
217
+ end
218
+
219
+ def send_request_header
220
+ query = @options[:query]
221
+ head = @options[:head] ? munge_header_keys(@options[:head]) : {}
222
+ body = normalize_body
223
+
224
+ # Set the Host header if it hasn't been specified already
225
+ head['host'] ||= encode_host
226
+
227
+ # Set the Content-Length if body is given
228
+ head['content-length'] = body.length if body
229
+
230
+ # Set the User-Agent if it hasn't been specified
231
+ head['user-agent'] ||= "EventMachine HttpClient"
232
+
233
+ # Set auto-inflate flags
234
+ if head['accept-encoding']
235
+ @inflate = head['accept-encoding'].split(',').map {|t| t.strip}
236
+ end
237
+
238
+ # Set the cookie header if provided
239
+ if cookie = head.delete('cookie')
240
+ head['cookie'] = encode_cookie(cookie)
241
+ end
242
+
243
+ # Build the request
244
+ request_header = encode_request(@method, @uri.path, query)
245
+ request_header << encode_headers(head)
246
+ request_header << CRLF
247
+
248
+ send_data request_header
249
+ end
250
+
251
+ def send_request_body
252
+ return unless @options[:body]
253
+ body = normalize_body
254
+ send_data body
255
+ end
256
+
257
+ def receive_data(data)
258
+ @data << data
259
+ dispatch
260
+ end
261
+
262
+ # Called when part of the body has been read
263
+ def on_body_data(data)
264
+ if @content_decoder
265
+ begin
266
+ @content_decoder << data
267
+ rescue HttpDecoders::DecoderError
268
+ on_error "Content-decoder error"
269
+ end
270
+ else
271
+ on_decoded_body_data(data)
272
+ end
273
+ end
274
+
275
+ def on_decoded_body_data(data)
276
+ if @stream
277
+ @stream.call(data)
278
+ else
279
+ @response << data
280
+ end
281
+ end
282
+
283
+ def unbind
284
+ (@state == :finished) ? succeed(self) : fail
285
+ close_connection
286
+ end
287
+
288
+ #
289
+ # Response processing
290
+ #
291
+
292
+ def dispatch
293
+ while case @state
294
+ when :response_header
295
+ parse_response_header
296
+ when :chunk_header
297
+ parse_chunk_header
298
+ when :chunk_body
299
+ process_chunk_body
300
+ when :chunk_footer
301
+ process_chunk_footer
302
+ when :response_footer
303
+ process_response_footer
304
+ when :body
305
+ process_body
306
+ when :finished, :invalid
307
+ break
308
+ else raise RuntimeError, "invalid state: #{@state}"
309
+ end
310
+ end
311
+ end
312
+
313
+ def parse_header(header)
314
+ return false if @data.empty?
315
+
316
+ begin
317
+ @parser_nbytes = @parser.execute(header, @data.to_str, @parser_nbytes)
318
+ rescue EventMachine::HttpClientParserError
319
+ @state = :invalid
320
+ on_error "invalid HTTP format, parsing fails"
321
+ end
322
+
323
+ return false unless @parser.finished?
324
+
325
+ # Clear parsed data from the buffer
326
+ @data.read(@parser_nbytes)
327
+ @parser.reset
328
+ @parser_nbytes = 0
329
+
330
+ true
331
+ end
332
+
333
+ def parse_response_header
334
+ return false unless parse_header(@response_header)
335
+
336
+ unless @response_header.http_status and @response_header.http_reason
337
+ @state = :invalid
338
+ on_error "no HTTP response"
339
+ return false
340
+ end
341
+
342
+ if @response_header.chunked_encoding?
343
+ @state = :chunk_header
344
+ else
345
+ @state = :body
346
+ @bytes_remaining = @response_header.content_length
347
+ end
348
+
349
+ if @inflate.include?(response_header[CONTENT_ENCODING]) &&
350
+ decoder_class = HttpDecoders.decoder_for_encoding(response_header[CONTENT_ENCODING])
351
+ begin
352
+ @content_decoder = decoder_class.new do |s| on_decoded_body_data(s) end
353
+ rescue HttpDecoders::DecoderError
354
+ on_error "Content-decoder error"
355
+ end
356
+ end
357
+
358
+ true
359
+ end
360
+
361
+ def parse_chunk_header
362
+ return false unless parse_header(@chunk_header)
363
+
364
+ @bytes_remaining = @chunk_header.chunk_size
365
+ @chunk_header = HttpChunkHeader.new
366
+
367
+ @state = @bytes_remaining > 0 ? :chunk_body : :response_footer
368
+ true
369
+ end
370
+
371
+ def process_chunk_body
372
+ if @data.size < @bytes_remaining
373
+ @bytes_remaining -= @data.size
374
+ on_body_data @data.read
375
+ return false
376
+ end
377
+
378
+ on_body_data @data.read(@bytes_remaining)
379
+ @bytes_remaining = 0
380
+
381
+ @state = :chunk_footer
382
+ true
383
+ end
384
+
385
+ def process_chunk_footer
386
+ return false if @data.size < 2
387
+
388
+ if @data.read(2) == CRLF
389
+ @state = :chunk_header
390
+ else
391
+ @state = :invalid
392
+ on_error "non-CRLF chunk footer"
393
+ end
394
+
395
+ true
396
+ end
397
+
398
+ def process_response_footer
399
+ return false if @data.size < 2
400
+
401
+ if @data.read(2) == CRLF
402
+ if @data.empty?
403
+ @state = :finished
404
+ on_request_complete
405
+ else
406
+ @state = :invalid
407
+ on_error "garbage at end of chunked response"
408
+ end
409
+ else
410
+ @state = :invalid
411
+ on_error "non-CRLF response footer"
412
+ end
413
+
414
+ false
415
+ end
416
+
417
+ def process_body
418
+ if @bytes_remaining.nil?
419
+ on_body_data @data.read
420
+ return false
421
+ end
422
+
423
+ if @bytes_remaining.zero?
424
+ @state = :finished
425
+ on_request_complete
426
+ return false
427
+ end
428
+
429
+ if @data.size < @bytes_remaining
430
+ @bytes_remaining -= @data.size
431
+ on_body_data @data.read
432
+ return false
433
+ end
434
+
435
+ on_body_data @data.read(@bytes_remaining)
436
+ @bytes_remaining = 0
437
+
438
+ # If Keep-Alive is enabled, the server may be pushing more data to us
439
+ # after the first request is complete. Hence, finish first request, and
440
+ # reset state.
441
+ if @response_header.keep_alive?
442
+ @data.clear # hard reset, TODO: add support for keep-alive connections!
443
+ @state = :finished
444
+ on_request_complete
445
+
446
+ else
447
+ if @data.empty?
448
+ @state = :finished
449
+ on_request_complete
450
+ else
451
+ @state = :invalid
452
+ on_error "garbage at end of body"
453
+ end
454
+ end
455
+
456
+ false
457
+ end
458
+
459
+
460
+ end
461
+
462
+ end