dwaite-em-http-request 0.1.9
Sign up to get free protection for your applications and to get access to all the features.
- data/.autotest +1 -0
- data/.gitignore +4 -0
- data/LICENSE +58 -0
- data/README.rdoc +71 -0
- data/Rakefile +84 -0
- data/em-http-request.gemspec +42 -0
- data/ext/buffer/em_buffer.c +630 -0
- data/ext/buffer/extconf.rb +53 -0
- data/ext/http11_client/ext_help.h +14 -0
- data/ext/http11_client/extconf.rb +6 -0
- data/ext/http11_client/http11_client.c +314 -0
- data/ext/http11_client/http11_parser.c +403 -0
- data/ext/http11_client/http11_parser.h +48 -0
- data/ext/http11_client/http11_parser.rl +173 -0
- data/lib/em-http.rb +18 -0
- data/lib/em-http/client.rb +547 -0
- data/lib/em-http/core_ext/hash.rb +53 -0
- data/lib/em-http/decoders.rb +119 -0
- data/lib/em-http/multi.rb +51 -0
- data/lib/em-http/request.rb +72 -0
- data/test/helper.rb +5 -0
- data/test/stallion.rb +144 -0
- data/test/test_hash.rb +19 -0
- data/test/test_multi.rb +29 -0
- data/test/test_request.rb +358 -0
- metadata +88 -0
@@ -0,0 +1,48 @@
|
|
1
|
+
/**
|
2
|
+
* Copyright (c) 2005 Zed A. Shaw
|
3
|
+
* You can redistribute it and/or modify it under the same terms as Ruby.
|
4
|
+
*/
|
5
|
+
|
6
|
+
#ifndef http11_parser_h
|
7
|
+
#define http11_parser_h
|
8
|
+
|
9
|
+
#include <sys/types.h>
|
10
|
+
|
11
|
+
#if defined(_WIN32)
|
12
|
+
#include <stddef.h>
|
13
|
+
#endif
|
14
|
+
|
15
|
+
typedef void (*element_cb)(void *data, const char *at, size_t length);
|
16
|
+
typedef void (*field_cb)(void *data, const char *field, size_t flen, const char *value, size_t vlen);
|
17
|
+
|
18
|
+
typedef struct httpclient_parser {
|
19
|
+
int cs;
|
20
|
+
size_t body_start;
|
21
|
+
int content_len;
|
22
|
+
size_t nread;
|
23
|
+
size_t mark;
|
24
|
+
size_t field_start;
|
25
|
+
size_t field_len;
|
26
|
+
|
27
|
+
void *data;
|
28
|
+
|
29
|
+
field_cb http_field;
|
30
|
+
element_cb reason_phrase;
|
31
|
+
element_cb status_code;
|
32
|
+
element_cb chunk_size;
|
33
|
+
element_cb http_version;
|
34
|
+
element_cb header_done;
|
35
|
+
element_cb last_chunk;
|
36
|
+
|
37
|
+
|
38
|
+
} httpclient_parser;
|
39
|
+
|
40
|
+
int httpclient_parser_init(httpclient_parser *parser);
|
41
|
+
int httpclient_parser_finish(httpclient_parser *parser);
|
42
|
+
size_t httpclient_parser_execute(httpclient_parser *parser, const char *data, size_t len, size_t off);
|
43
|
+
int httpclient_parser_has_error(httpclient_parser *parser);
|
44
|
+
int httpclient_parser_is_finished(httpclient_parser *parser);
|
45
|
+
|
46
|
+
#define httpclient_parser_nread(parser) (parser)->nread
|
47
|
+
|
48
|
+
#endif
|
@@ -0,0 +1,173 @@
|
|
1
|
+
/**
|
2
|
+
* Copyright (c) 2005 Zed A. Shaw
|
3
|
+
* You can redistribute it and/or modify it under the same terms as Ruby.
|
4
|
+
*/
|
5
|
+
|
6
|
+
#include "http11_parser.h"
|
7
|
+
#include <stdio.h>
|
8
|
+
#include <assert.h>
|
9
|
+
#include <stdlib.h>
|
10
|
+
#include <ctype.h>
|
11
|
+
#include <string.h>
|
12
|
+
|
13
|
+
#define LEN(AT, FPC) (FPC - buffer - parser->AT)
|
14
|
+
#define MARK(M,FPC) (parser->M = (FPC) - buffer)
|
15
|
+
#define PTR_TO(F) (buffer + parser->F)
|
16
|
+
#define L(M) fprintf(stderr, "" # M "\n");
|
17
|
+
|
18
|
+
|
19
|
+
/** machine **/
|
20
|
+
%%{
|
21
|
+
machine httpclient_parser;
|
22
|
+
|
23
|
+
action mark {MARK(mark, fpc); }
|
24
|
+
|
25
|
+
action start_field { MARK(field_start, fpc); }
|
26
|
+
|
27
|
+
action write_field {
|
28
|
+
parser->field_len = LEN(field_start, fpc);
|
29
|
+
}
|
30
|
+
|
31
|
+
action start_value { MARK(mark, fpc); }
|
32
|
+
|
33
|
+
action write_value {
|
34
|
+
parser->http_field(parser->data, PTR_TO(field_start), parser->field_len, PTR_TO(mark), LEN(mark, fpc));
|
35
|
+
}
|
36
|
+
|
37
|
+
action reason_phrase {
|
38
|
+
parser->reason_phrase(parser->data, PTR_TO(mark), LEN(mark, fpc));
|
39
|
+
}
|
40
|
+
|
41
|
+
action status_code {
|
42
|
+
parser->status_code(parser->data, PTR_TO(mark), LEN(mark, fpc));
|
43
|
+
}
|
44
|
+
|
45
|
+
action http_version {
|
46
|
+
parser->http_version(parser->data, PTR_TO(mark), LEN(mark, fpc));
|
47
|
+
}
|
48
|
+
|
49
|
+
action chunk_size {
|
50
|
+
parser->chunk_size(parser->data, PTR_TO(mark), LEN(mark, fpc));
|
51
|
+
}
|
52
|
+
|
53
|
+
action last_chunk {
|
54
|
+
parser->last_chunk(parser->data, NULL, 0);
|
55
|
+
}
|
56
|
+
|
57
|
+
action done {
|
58
|
+
parser->body_start = fpc - buffer + 1;
|
59
|
+
if(parser->header_done != NULL)
|
60
|
+
parser->header_done(parser->data, fpc + 1, pe - fpc - 1);
|
61
|
+
fbreak;
|
62
|
+
}
|
63
|
+
|
64
|
+
# line endings
|
65
|
+
CRLF = "\r\n";
|
66
|
+
|
67
|
+
# character types
|
68
|
+
CTL = (cntrl | 127);
|
69
|
+
tspecials = ("(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\\" | "\"" | "/" | "[" | "]" | "?" | "=" | "{" | "}" | " " | "\t");
|
70
|
+
|
71
|
+
# elements
|
72
|
+
token = (ascii -- (CTL | tspecials));
|
73
|
+
|
74
|
+
Reason_Phrase = (any -- CRLF)* >mark %reason_phrase;
|
75
|
+
Status_Code = digit{3} >mark %status_code;
|
76
|
+
http_number = (digit+ "." digit+) ;
|
77
|
+
HTTP_Version = ("HTTP/" http_number) >mark %http_version ;
|
78
|
+
Status_Line = HTTP_Version " " Status_Code " "? Reason_Phrase :> CRLF;
|
79
|
+
|
80
|
+
field_name = token+ >start_field %write_field;
|
81
|
+
field_value = any* >start_value %write_value;
|
82
|
+
message_header = field_name ":" " "* field_value :> CRLF;
|
83
|
+
|
84
|
+
Response = Status_Line (message_header)* (CRLF @done);
|
85
|
+
|
86
|
+
chunk_ext_val = token+;
|
87
|
+
chunk_ext_name = token+;
|
88
|
+
chunk_extension = (";" chunk_ext_name >start_field %write_field %start_value ("=" chunk_ext_val >start_value)? %write_value )*;
|
89
|
+
last_chunk = "0"? chunk_extension :> (CRLF @last_chunk @done);
|
90
|
+
chunk_size = xdigit+;
|
91
|
+
chunk = chunk_size >mark %chunk_size chunk_extension space* :> (CRLF @done);
|
92
|
+
Chunked_Header = (chunk | last_chunk);
|
93
|
+
|
94
|
+
main := Response | Chunked_Header;
|
95
|
+
}%%
|
96
|
+
|
97
|
+
/** Data **/
|
98
|
+
%% write data;
|
99
|
+
|
100
|
+
int httpclient_parser_init(httpclient_parser *parser) {
|
101
|
+
int cs = 0;
|
102
|
+
%% write init;
|
103
|
+
parser->cs = cs;
|
104
|
+
parser->body_start = 0;
|
105
|
+
parser->content_len = 0;
|
106
|
+
parser->mark = 0;
|
107
|
+
parser->nread = 0;
|
108
|
+
parser->field_len = 0;
|
109
|
+
parser->field_start = 0;
|
110
|
+
|
111
|
+
return(1);
|
112
|
+
}
|
113
|
+
|
114
|
+
|
115
|
+
/** exec **/
|
116
|
+
size_t httpclient_parser_execute(httpclient_parser *parser, const char *buffer, size_t len, size_t off) {
|
117
|
+
const char *p, *pe;
|
118
|
+
int cs = parser->cs;
|
119
|
+
|
120
|
+
assert(off <= len && "offset past end of buffer");
|
121
|
+
|
122
|
+
p = buffer+off;
|
123
|
+
pe = buffer+len;
|
124
|
+
|
125
|
+
assert(*pe == '\0' && "pointer does not end on NUL");
|
126
|
+
assert(pe - p == len - off && "pointers aren't same distance");
|
127
|
+
|
128
|
+
|
129
|
+
%% write exec;
|
130
|
+
|
131
|
+
parser->cs = cs;
|
132
|
+
parser->nread += p - (buffer + off);
|
133
|
+
|
134
|
+
assert(p <= pe && "buffer overflow after parsing execute");
|
135
|
+
assert(parser->nread <= len && "nread longer than length");
|
136
|
+
assert(parser->body_start <= len && "body starts after buffer end");
|
137
|
+
assert(parser->mark < len && "mark is after buffer end");
|
138
|
+
assert(parser->field_len <= len && "field has length longer than whole buffer");
|
139
|
+
assert(parser->field_start < len && "field starts after buffer end");
|
140
|
+
|
141
|
+
if(parser->body_start) {
|
142
|
+
/* final \r\n combo encountered so stop right here */
|
143
|
+
%%write eof;
|
144
|
+
parser->nread++;
|
145
|
+
}
|
146
|
+
|
147
|
+
return(parser->nread);
|
148
|
+
}
|
149
|
+
|
150
|
+
int httpclient_parser_finish(httpclient_parser *parser)
|
151
|
+
{
|
152
|
+
int cs = parser->cs;
|
153
|
+
|
154
|
+
%%write eof;
|
155
|
+
|
156
|
+
parser->cs = cs;
|
157
|
+
|
158
|
+
if (httpclient_parser_has_error(parser) ) {
|
159
|
+
return -1;
|
160
|
+
} else if (httpclient_parser_is_finished(parser) ) {
|
161
|
+
return 1;
|
162
|
+
} else {
|
163
|
+
return 0;
|
164
|
+
}
|
165
|
+
}
|
166
|
+
|
167
|
+
int httpclient_parser_has_error(httpclient_parser *parser) {
|
168
|
+
return parser->cs == httpclient_parser_error;
|
169
|
+
}
|
170
|
+
|
171
|
+
int httpclient_parser_is_finished(httpclient_parser *parser) {
|
172
|
+
return parser->cs == httpclient_parser_first_final;
|
173
|
+
}
|
data/lib/em-http.rb
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
#--
|
2
|
+
# Copyright (C)2008 Ilya Grigorik
|
3
|
+
# You can redistribute this under the terms of the Ruby license
|
4
|
+
# See file LICENSE for details
|
5
|
+
#++
|
6
|
+
|
7
|
+
require 'rubygems'
|
8
|
+
require 'eventmachine'
|
9
|
+
|
10
|
+
|
11
|
+
require File.dirname(__FILE__) + '/http11_client'
|
12
|
+
require File.dirname(__FILE__) + '/em_buffer'
|
13
|
+
|
14
|
+
require File.dirname(__FILE__) + '/em-http/core_ext/hash'
|
15
|
+
require File.dirname(__FILE__) + '/em-http/client'
|
16
|
+
require File.dirname(__FILE__) + '/em-http/multi'
|
17
|
+
require File.dirname(__FILE__) + '/em-http/request'
|
18
|
+
require File.dirname(__FILE__) + '/em-http/decoders'
|
@@ -0,0 +1,547 @@
|
|
1
|
+
# #--
|
2
|
+
# Copyright (C)2008 Ilya Grigorik
|
3
|
+
#
|
4
|
+
# Includes portion originally Copyright (C)2007 Tony Arcieri
|
5
|
+
# Includes portion originally Copyright (C)2005 Zed Shaw
|
6
|
+
# You can redistribute this under the terms of the Ruby
|
7
|
+
# license See file LICENSE for details
|
8
|
+
# #--
|
9
|
+
|
10
|
+
module EventMachine
|
11
|
+
|
12
|
+
# A simple hash is returned for each request made by HttpClient with the
|
13
|
+
# headers that were given by the server for that request.
|
14
|
+
class HttpResponseHeader < Hash
|
15
|
+
# The reason returned in the http response ("OK","File not found",etc.)
|
16
|
+
attr_accessor :http_reason
|
17
|
+
|
18
|
+
# The HTTP version returned.
|
19
|
+
attr_accessor :http_version
|
20
|
+
|
21
|
+
# The status code (as a string!)
|
22
|
+
attr_accessor :http_status
|
23
|
+
|
24
|
+
# E-Tag
|
25
|
+
def etag
|
26
|
+
self["ETag"]
|
27
|
+
end
|
28
|
+
|
29
|
+
def last_modified
|
30
|
+
time = self["Last-Modified"]
|
31
|
+
Time.parse(time) if time
|
32
|
+
end
|
33
|
+
|
34
|
+
# HTTP response status as an integer
|
35
|
+
def status
|
36
|
+
Integer(http_status) rescue nil
|
37
|
+
end
|
38
|
+
|
39
|
+
# Length of content as an integer, or nil if chunked/unspecified
|
40
|
+
def content_length
|
41
|
+
Integer(self[HttpClient::CONTENT_LENGTH]) rescue nil
|
42
|
+
end
|
43
|
+
|
44
|
+
# Cookie header from the server
|
45
|
+
def cookie
|
46
|
+
self[HttpClient::SET_COOKIE]
|
47
|
+
end
|
48
|
+
|
49
|
+
# Is the transfer encoding chunked?
|
50
|
+
def chunked_encoding?
|
51
|
+
/chunked/i === self[HttpClient::TRANSFER_ENCODING]
|
52
|
+
end
|
53
|
+
|
54
|
+
def keep_alive?
|
55
|
+
/keep-alive/i === self[HttpClient::KEEP_ALIVE]
|
56
|
+
end
|
57
|
+
|
58
|
+
def compressed?
|
59
|
+
/gzip|compressed|deflate/i === self[HttpClient::CONTENT_ENCODING]
|
60
|
+
end
|
61
|
+
|
62
|
+
def location
|
63
|
+
self[HttpClient::LOCATION]
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
class HttpChunkHeader < Hash
|
68
|
+
# When parsing chunked encodings this is set
|
69
|
+
attr_accessor :http_chunk_size
|
70
|
+
|
71
|
+
# Size of the chunk as an integer
|
72
|
+
def chunk_size
|
73
|
+
return @chunk_size unless @chunk_size.nil?
|
74
|
+
@chunk_size = @http_chunk_size ? @http_chunk_size.to_i(base=16) : 0
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
# Methods for building HTTP requests
|
79
|
+
module HttpEncoding
|
80
|
+
HTTP_REQUEST_HEADER="%s %s HTTP/1.1\r\n"
|
81
|
+
FIELD_ENCODING = "%s: %s\r\n"
|
82
|
+
BASIC_AUTH_ENCODING = "%s: Basic %s\r\n"
|
83
|
+
|
84
|
+
# Escapes a URI.
|
85
|
+
def escape(s)
|
86
|
+
s.to_s.gsub(/([^ a-zA-Z0-9_.-]+)/n) {
|
87
|
+
'%'+$1.unpack('H2'*$1.size).join('%').upcase
|
88
|
+
}.tr(' ', '+')
|
89
|
+
end
|
90
|
+
|
91
|
+
# Unescapes a URI escaped string.
|
92
|
+
def unescape(s)
|
93
|
+
s.tr('+', ' ').gsub(/((?:%[0-9a-fA-F]{2})+)/n){
|
94
|
+
[$1.delete('%')].pack('H*')
|
95
|
+
}
|
96
|
+
end
|
97
|
+
|
98
|
+
# Map all header keys to a downcased string version
|
99
|
+
def munge_header_keys(head)
|
100
|
+
head.inject({}) { |h, (k, v)| h[k.to_s.downcase] = v; h }
|
101
|
+
end
|
102
|
+
|
103
|
+
# HTTP is kind of retarded that you have to specify a Host header, but if
|
104
|
+
# you include port 80 then further redirects will tack on the :80 which is
|
105
|
+
# annoying.
|
106
|
+
def encode_host
|
107
|
+
@uri.host + (@uri.port != @uri.default_port ? ":#{@uri.port}" : "")
|
108
|
+
end
|
109
|
+
|
110
|
+
def encode_request(method, path, query)
|
111
|
+
HTTP_REQUEST_HEADER % [method.to_s.upcase, encode_query(path, query)]
|
112
|
+
end
|
113
|
+
|
114
|
+
def encode_query(path, query)
|
115
|
+
encoded_query = if query.kind_of?(Hash)
|
116
|
+
query.map { |k, v| encode_param(k, v) }.join('&')
|
117
|
+
else
|
118
|
+
query.to_s
|
119
|
+
end
|
120
|
+
if !@uri.query.to_s.empty?
|
121
|
+
encoded_query = [encoded_query, @uri.query].reject {|part| part.empty?}.join("&")
|
122
|
+
end
|
123
|
+
return path if encoded_query.to_s.empty?
|
124
|
+
"#{path}?#{encoded_query}"
|
125
|
+
end
|
126
|
+
|
127
|
+
# URL encodes query parameters:
|
128
|
+
# single k=v, or a URL encoded array, if v is an array of values
|
129
|
+
def encode_param(k, v)
|
130
|
+
if v.is_a?(Array)
|
131
|
+
v.map { |e| escape(k) + "[]=" + escape(e) }.join("&")
|
132
|
+
else
|
133
|
+
escape(k) + "=" + escape(v)
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
# Encode a field in an HTTP header
|
138
|
+
def encode_field(k, v)
|
139
|
+
FIELD_ENCODING % [k, v]
|
140
|
+
end
|
141
|
+
|
142
|
+
# Encode basic auth in an HTTP header
|
143
|
+
def encode_basic_auth(k,v)
|
144
|
+
BASIC_AUTH_ENCODING % [k, Base64.encode64(v.join(":")).chomp]
|
145
|
+
end
|
146
|
+
|
147
|
+
def encode_headers(head)
|
148
|
+
head.inject('') do |result, (key, value)|
|
149
|
+
# Munge keys from foo-bar-baz to Foo-Bar-Baz
|
150
|
+
key = key.split('-').map { |k| k.capitalize }.join('-')
|
151
|
+
unless key == "Authorization"
|
152
|
+
result << encode_field(key, value)
|
153
|
+
else
|
154
|
+
result << encode_basic_auth(key, value)
|
155
|
+
end
|
156
|
+
end
|
157
|
+
end
|
158
|
+
|
159
|
+
def encode_cookie(cookie)
|
160
|
+
if cookie.is_a? Hash
|
161
|
+
cookie.inject('') { |result, (k, v)| result << encode_param(k, v) + ";" }
|
162
|
+
else
|
163
|
+
cookie
|
164
|
+
end
|
165
|
+
end
|
166
|
+
def encode_body body
|
167
|
+
params = ''
|
168
|
+
stack = []
|
169
|
+
body.each do |k, v|
|
170
|
+
if v.is_a? Hash
|
171
|
+
stack << [k, v]
|
172
|
+
elsif v.is_a? Array
|
173
|
+
stack << [k, Hash.from_array v]
|
174
|
+
else
|
175
|
+
params << encode_param(k, v) << '&'
|
176
|
+
end
|
177
|
+
end
|
178
|
+
stack.each do |parent, hash|
|
179
|
+
hash.each do |k, v|
|
180
|
+
if v.is_a? Hash
|
181
|
+
stack << ["#{parent}[#{k}]", v]
|
182
|
+
else
|
183
|
+
params << encode_param("#{parent}[#{k}]", v) << '&'
|
184
|
+
end
|
185
|
+
end
|
186
|
+
end
|
187
|
+
params.chop! # trailing &
|
188
|
+
params
|
189
|
+
end
|
190
|
+
end
|
191
|
+
|
192
|
+
class HttpClient < Connection
|
193
|
+
include EventMachine::Deferrable
|
194
|
+
include HttpEncoding
|
195
|
+
|
196
|
+
TRANSFER_ENCODING="TRANSFER_ENCODING"
|
197
|
+
CONTENT_ENCODING="CONTENT_ENCODING"
|
198
|
+
CONTENT_LENGTH="CONTENT_LENGTH"
|
199
|
+
KEEP_ALIVE="CONNECTION"
|
200
|
+
SET_COOKIE="SET_COOKIE"
|
201
|
+
LOCATION="LOCATION"
|
202
|
+
HOST="HOST"
|
203
|
+
CRLF="\r\n"
|
204
|
+
|
205
|
+
attr_accessor :method, :options, :uri
|
206
|
+
attr_reader :response, :response_header, :errors
|
207
|
+
|
208
|
+
def post_init
|
209
|
+
@parser = HttpClientParser.new
|
210
|
+
@data = EventMachine::Buffer.new
|
211
|
+
@response_header = HttpResponseHeader.new
|
212
|
+
@chunk_header = HttpChunkHeader.new
|
213
|
+
|
214
|
+
@state = :response_header
|
215
|
+
@parser_nbytes = 0
|
216
|
+
@response = ''
|
217
|
+
@inflate = []
|
218
|
+
@errors = ''
|
219
|
+
@content_decoder = nil
|
220
|
+
@stream = nil
|
221
|
+
end
|
222
|
+
|
223
|
+
# start HTTP request once we establish connection to host
|
224
|
+
def connection_completed
|
225
|
+
ssl = @options[:tls] || @options[:ssl] || {}
|
226
|
+
start_tls(ssl) if @uri.scheme == "https" or @uri.port == 443
|
227
|
+
|
228
|
+
send_request_header
|
229
|
+
send_request_body
|
230
|
+
end
|
231
|
+
|
232
|
+
# request is done, invoke the callback
|
233
|
+
def on_request_complete
|
234
|
+
begin
|
235
|
+
@content_decoder.finalize! if @content_decoder
|
236
|
+
rescue HttpDecoders::DecoderError
|
237
|
+
on_error "Content-decoder error"
|
238
|
+
end
|
239
|
+
unbind
|
240
|
+
end
|
241
|
+
|
242
|
+
# request failed, invoke errback
|
243
|
+
def on_error(msg, dns_error = false)
|
244
|
+
@errors = msg
|
245
|
+
|
246
|
+
# no connection signature on DNS failures
|
247
|
+
# fail the connection directly
|
248
|
+
dns_error == true ? fail : unbind
|
249
|
+
end
|
250
|
+
|
251
|
+
# assign a stream processing block
|
252
|
+
def stream(&blk)
|
253
|
+
@stream = blk
|
254
|
+
end
|
255
|
+
|
256
|
+
def normalize_body
|
257
|
+
if @options[:body].is_a? Hash
|
258
|
+
encode_body @options[:body]
|
259
|
+
else
|
260
|
+
@options[:body]
|
261
|
+
end
|
262
|
+
end
|
263
|
+
|
264
|
+
def send_request_header
|
265
|
+
query = @options[:query]
|
266
|
+
head = @options[:head] ? munge_header_keys(@options[:head]) : {}
|
267
|
+
@options[:head] = head
|
268
|
+
body = normalize_body
|
269
|
+
|
270
|
+
# Set the Host header if it hasn't been specified already
|
271
|
+
head['host'] ||= encode_host
|
272
|
+
|
273
|
+
# Set the Content-Length if body is given
|
274
|
+
head['content-length'] = body.length if body
|
275
|
+
|
276
|
+
# Set the User-Agent if it hasn't been specified
|
277
|
+
head['user-agent'] ||= "EventMachine HttpClient"
|
278
|
+
|
279
|
+
# Set auto-inflate flags
|
280
|
+
if head['accept-encoding']
|
281
|
+
@inflate = head['accept-encoding'].split(',').map {|t| t.strip}
|
282
|
+
end
|
283
|
+
|
284
|
+
# Set the cookie header if provided
|
285
|
+
if cookie = head.delete('cookie')
|
286
|
+
head['cookie'] = encode_cookie(cookie)
|
287
|
+
end
|
288
|
+
|
289
|
+
# Build the request
|
290
|
+
request_header = encode_request(@method, @uri.path, query)
|
291
|
+
request_header << encode_headers(head)
|
292
|
+
request_header << CRLF
|
293
|
+
|
294
|
+
send_data request_header
|
295
|
+
if @options[:debug]
|
296
|
+
puts request_header
|
297
|
+
end
|
298
|
+
end
|
299
|
+
|
300
|
+
def send_request_body
|
301
|
+
return unless @options[:body]
|
302
|
+
body = normalize_body
|
303
|
+
send_data body
|
304
|
+
if @options[:debug]
|
305
|
+
puts body
|
306
|
+
end
|
307
|
+
end
|
308
|
+
|
309
|
+
def receive_data(data)
|
310
|
+
@data << data
|
311
|
+
dispatch
|
312
|
+
end
|
313
|
+
|
314
|
+
# Called when part of the body has been read
|
315
|
+
def on_body_data(data)
|
316
|
+
if @content_decoder
|
317
|
+
begin
|
318
|
+
@content_decoder << data
|
319
|
+
rescue HttpDecoders::DecoderError
|
320
|
+
on_error "Content-decoder error"
|
321
|
+
end
|
322
|
+
else
|
323
|
+
on_decoded_body_data(data)
|
324
|
+
end
|
325
|
+
end
|
326
|
+
|
327
|
+
def on_decoded_body_data(data)
|
328
|
+
if @stream
|
329
|
+
@stream.call(data)
|
330
|
+
else
|
331
|
+
@response << data
|
332
|
+
end
|
333
|
+
end
|
334
|
+
|
335
|
+
def unbind
|
336
|
+
if @state == :finished
|
337
|
+
succeed(self)
|
338
|
+
else
|
339
|
+
fail(self)
|
340
|
+
end
|
341
|
+
close_connection
|
342
|
+
end
|
343
|
+
|
344
|
+
#
|
345
|
+
# Response processing
|
346
|
+
#
|
347
|
+
|
348
|
+
def dispatch
|
349
|
+
while case @state
|
350
|
+
when :response_header
|
351
|
+
parse_response_header
|
352
|
+
when :chunk_header
|
353
|
+
parse_chunk_header
|
354
|
+
when :chunk_body
|
355
|
+
process_chunk_body
|
356
|
+
when :chunk_footer
|
357
|
+
process_chunk_footer
|
358
|
+
when :response_footer
|
359
|
+
process_response_footer
|
360
|
+
when :body
|
361
|
+
process_body
|
362
|
+
when :finished, :invalid
|
363
|
+
break
|
364
|
+
else raise RuntimeError, "invalid state: #{@state}"
|
365
|
+
end
|
366
|
+
end
|
367
|
+
end
|
368
|
+
|
369
|
+
def parse_header(header)
|
370
|
+
return false if @data.empty?
|
371
|
+
|
372
|
+
begin
|
373
|
+
@parser_nbytes = @parser.execute(header, @data.to_str, @parser_nbytes)
|
374
|
+
rescue EventMachine::HttpClientParserError
|
375
|
+
@state = :invalid
|
376
|
+
on_error "invalid HTTP format, parsing fails"
|
377
|
+
end
|
378
|
+
|
379
|
+
return false unless @parser.finished?
|
380
|
+
|
381
|
+
# Clear parsed data from the buffer
|
382
|
+
@data.read(@parser_nbytes)
|
383
|
+
@parser.reset
|
384
|
+
@parser_nbytes = 0
|
385
|
+
|
386
|
+
true
|
387
|
+
end
|
388
|
+
|
389
|
+
def parse_response_header
|
390
|
+
return false unless parse_header(@response_header)
|
391
|
+
|
392
|
+
unless @response_header.http_status and @response_header.http_reason
|
393
|
+
@state = :invalid
|
394
|
+
on_error "no HTTP response"
|
395
|
+
return false
|
396
|
+
end
|
397
|
+
|
398
|
+
if @options[:debug]
|
399
|
+
puts "Response headers: #{@response_header.inspect}"
|
400
|
+
end
|
401
|
+
|
402
|
+
# correct location header - some servers will incorrectly give a relative URI
|
403
|
+
if @response_header.location
|
404
|
+
begin
|
405
|
+
location = URI.parse @response_header.location
|
406
|
+
if location.relative?
|
407
|
+
location = (@uri.merge location).to_s
|
408
|
+
@response_header[LOCATION] = location
|
409
|
+
end
|
410
|
+
rescue
|
411
|
+
on_error "Location header format error"
|
412
|
+
return false
|
413
|
+
end
|
414
|
+
end
|
415
|
+
|
416
|
+
# shortcircuit on HEAD requests
|
417
|
+
if @method == "HEAD"
|
418
|
+
@state = :finished
|
419
|
+
on_request_complete
|
420
|
+
end
|
421
|
+
|
422
|
+
if @response_header.chunked_encoding?
|
423
|
+
@state = :chunk_header
|
424
|
+
else
|
425
|
+
if @response_header.content_length > 0
|
426
|
+
@state = :body
|
427
|
+
@bytes_remaining = @response_header.content_length
|
428
|
+
else
|
429
|
+
@state = :finished
|
430
|
+
on_request_complete
|
431
|
+
end
|
432
|
+
end
|
433
|
+
|
434
|
+
if @inflate.include?(response_header[CONTENT_ENCODING]) &&
|
435
|
+
decoder_class = HttpDecoders.decoder_for_encoding(response_header[CONTENT_ENCODING])
|
436
|
+
begin
|
437
|
+
@content_decoder = decoder_class.new do |s| on_decoded_body_data(s) end
|
438
|
+
rescue HttpDecoders::DecoderError
|
439
|
+
on_error "Content-decoder error"
|
440
|
+
end
|
441
|
+
end
|
442
|
+
|
443
|
+
true
|
444
|
+
end
|
445
|
+
|
446
|
+
def parse_chunk_header
|
447
|
+
return false unless parse_header(@chunk_header)
|
448
|
+
|
449
|
+
@bytes_remaining = @chunk_header.chunk_size
|
450
|
+
@chunk_header = HttpChunkHeader.new
|
451
|
+
|
452
|
+
@state = @bytes_remaining > 0 ? :chunk_body : :response_footer
|
453
|
+
true
|
454
|
+
end
|
455
|
+
|
456
|
+
def process_chunk_body
|
457
|
+
if @data.size < @bytes_remaining
|
458
|
+
@bytes_remaining -= @data.size
|
459
|
+
on_body_data @data.read
|
460
|
+
return false
|
461
|
+
end
|
462
|
+
|
463
|
+
on_body_data @data.read(@bytes_remaining)
|
464
|
+
@bytes_remaining = 0
|
465
|
+
|
466
|
+
@state = :chunk_footer
|
467
|
+
true
|
468
|
+
end
|
469
|
+
|
470
|
+
def process_chunk_footer
|
471
|
+
return false if @data.size < 2
|
472
|
+
|
473
|
+
if @data.read(2) == CRLF
|
474
|
+
@state = :chunk_header
|
475
|
+
else
|
476
|
+
@state = :invalid
|
477
|
+
on_error "non-CRLF chunk footer"
|
478
|
+
end
|
479
|
+
|
480
|
+
true
|
481
|
+
end
|
482
|
+
|
483
|
+
def process_response_footer
|
484
|
+
return false if @data.size < 2
|
485
|
+
|
486
|
+
if @data.read(2) == CRLF
|
487
|
+
if @data.empty?
|
488
|
+
@state = :finished
|
489
|
+
on_request_complete
|
490
|
+
else
|
491
|
+
@state = :invalid
|
492
|
+
on_error "garbage at end of chunked response"
|
493
|
+
end
|
494
|
+
else
|
495
|
+
@state = :invalid
|
496
|
+
on_error "non-CRLF response footer"
|
497
|
+
end
|
498
|
+
|
499
|
+
false
|
500
|
+
end
|
501
|
+
|
502
|
+
def process_body
|
503
|
+
if @bytes_remaining.nil?
|
504
|
+
on_body_data @data.read
|
505
|
+
return false
|
506
|
+
end
|
507
|
+
|
508
|
+
if @bytes_remaining.zero?
|
509
|
+
@state = :finished
|
510
|
+
on_request_complete
|
511
|
+
return false
|
512
|
+
end
|
513
|
+
|
514
|
+
if @data.size < @bytes_remaining
|
515
|
+
@bytes_remaining -= @data.size
|
516
|
+
on_body_data @data.read
|
517
|
+
return false
|
518
|
+
end
|
519
|
+
|
520
|
+
on_body_data @data.read(@bytes_remaining)
|
521
|
+
@bytes_remaining = 0
|
522
|
+
|
523
|
+
# If Keep-Alive is enabled, the server may be pushing more data to us
|
524
|
+
# after the first request is complete. Hence, finish first request, and
|
525
|
+
# reset state.
|
526
|
+
if @response_header.keep_alive?
|
527
|
+
@data.clear # hard reset, TODO: add support for keep-alive connections!
|
528
|
+
@state = :finished
|
529
|
+
on_request_complete
|
530
|
+
|
531
|
+
else
|
532
|
+
if @data.empty?
|
533
|
+
@state = :finished
|
534
|
+
on_request_complete
|
535
|
+
else
|
536
|
+
@state = :invalid
|
537
|
+
on_error "garbage at end of body"
|
538
|
+
end
|
539
|
+
end
|
540
|
+
|
541
|
+
false
|
542
|
+
end
|
543
|
+
|
544
|
+
|
545
|
+
end
|
546
|
+
|
547
|
+
end
|