http2 0.0.23 → 0.0.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/Gemfile +4 -1
- data/Gemfile.lock +45 -5
- data/README.md +50 -0
- data/VERSION +1 -1
- data/http2.gemspec +12 -6
- data/include/post_multipart_helper.rb +77 -0
- data/include/response_reader.rb +251 -0
- data/lib/http2.rb +182 -449
- data/shippable.yml +7 -0
- data/spec/http2_spec.rb +20 -20
- data/spec/spec_helper.rb +4 -1
- metadata +41 -37
- data/README.rdoc +0 -45
data/lib/http2.rb
CHANGED
@@ -1,10 +1,15 @@
|
|
1
|
+
require "socket"
|
2
|
+
require "uri"
|
3
|
+
require "monitor" unless ::Kernel.const_defined?(:Monitor)
|
4
|
+
require "string-cases"
|
5
|
+
|
1
6
|
#This class tries to emulate a browser in Ruby without any visual stuff. Remember cookies, keep sessions alive, reset connections according to keep-alive rules and more.
|
2
7
|
#===Examples
|
3
8
|
# Http2.new(:host => "www.somedomain.com", :port => 80, :ssl => false, :debug => false) do |http|
|
4
9
|
# res = http.get("index.rhtml?show=some_page")
|
5
10
|
# html = res.body
|
6
11
|
# print html
|
7
|
-
#
|
12
|
+
#
|
8
13
|
# res = res.post("index.rhtml?choice=login", {"username" => "John Doe", "password" => 123})
|
9
14
|
# print res.body
|
10
15
|
# print "#{res.headers}"
|
@@ -12,10 +17,10 @@
|
|
12
17
|
class Http2
|
13
18
|
#Autoloader for subclasses.
|
14
19
|
def self.const_missing(name)
|
15
|
-
require "#{File.dirname(__FILE__)}/../include/#{name
|
20
|
+
require "#{File.dirname(__FILE__)}/../include/#{::StringCases.camel_to_snake(name)}.rb"
|
16
21
|
return Http2.const_get(name)
|
17
22
|
end
|
18
|
-
|
23
|
+
|
19
24
|
#Converts a URL to "is.gd"-short-URL.
|
20
25
|
def self.isgdlink(url)
|
21
26
|
Http2.new(:host => "is.gd") do |http|
|
@@ -23,55 +28,17 @@ class Http2
|
|
23
28
|
return resp.body
|
24
29
|
end
|
25
30
|
end
|
26
|
-
|
27
|
-
attr_reader :cookies, :args, :resp
|
28
|
-
|
31
|
+
|
32
|
+
attr_reader :cookies, :args, :resp, :raise_errors, :nl
|
33
|
+
|
29
34
|
VALID_ARGUMENTS_INITIALIZE = [:host, :port, :ssl, :nl, :user_agent, :raise_errors, :follow_redirects, :debug, :encoding_gzip, :autostate, :basic_auth, :extra_headers, :proxy]
|
30
35
|
def initialize(args = {})
|
31
|
-
args =
|
32
|
-
|
33
|
-
|
34
|
-
args.each do |key, val|
|
35
|
-
raise "Invalid key: '#{key}'." if !VALID_ARGUMENTS_INITIALIZE.include?(key)
|
36
|
-
end
|
37
|
-
|
38
|
-
@args = args
|
36
|
+
@args = parse_init_args(args)
|
37
|
+
set_default_values
|
39
38
|
@cookies = {}
|
40
|
-
@debug = @args[:debug]
|
41
|
-
@autostate_values = {} if @args[:autostate]
|
42
|
-
|
43
|
-
require "monitor" unless ::Kernel.const_defined?(:Monitor)
|
44
39
|
@mutex = Monitor.new
|
45
|
-
|
46
|
-
if !@args[:port]
|
47
|
-
if @args[:ssl]
|
48
|
-
@args[:port] = 443
|
49
|
-
else
|
50
|
-
@args[:port] = 80
|
51
|
-
end
|
52
|
-
end
|
53
|
-
|
54
|
-
if @args[:nl]
|
55
|
-
@nl = @args[:nl]
|
56
|
-
else
|
57
|
-
@nl = "\r\n"
|
58
|
-
end
|
59
|
-
|
60
|
-
if @args[:user_agent]
|
61
|
-
@uagent = @args[:user_agent]
|
62
|
-
else
|
63
|
-
@uagent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)"
|
64
|
-
end
|
65
|
-
|
66
|
-
if !@args.key?(:raise_errors) || @args[:raise_errors]
|
67
|
-
@raise_errors = true
|
68
|
-
else
|
69
|
-
@raise_errors = false
|
70
|
-
end
|
71
|
-
|
72
|
-
raise "No host was given." if !@args[:host]
|
73
40
|
self.reconnect
|
74
|
-
|
41
|
+
|
75
42
|
if block_given?
|
76
43
|
begin
|
77
44
|
yield(self)
|
@@ -80,27 +47,27 @@ class Http2
|
|
80
47
|
end
|
81
48
|
end
|
82
49
|
end
|
83
|
-
|
50
|
+
|
84
51
|
#Closes current connection if any, changes the arguments on the object and reconnects keeping all cookies and other stuff intact.
|
85
52
|
def change(args)
|
86
53
|
self.close
|
87
54
|
@args.merge!(args)
|
88
55
|
self.reconnect
|
89
56
|
end
|
90
|
-
|
57
|
+
|
91
58
|
#Closes the current connection if any.
|
92
59
|
def close
|
93
60
|
@sock.close if @sock and !@sock.closed?
|
94
61
|
@sock_ssl.close if @sock_ssl and !@sock_ssl.closed?
|
95
62
|
@sock_plain.close if @sock_plain and !@sock_plain.closed?
|
96
63
|
end
|
97
|
-
|
64
|
+
|
98
65
|
#Returns boolean based on the if the object is connected and the socket is working.
|
99
66
|
#===Examples
|
100
67
|
# puts "Socket is working." if http.socket_working?
|
101
68
|
def socket_working?
|
102
69
|
return false if !@sock or @sock.closed?
|
103
|
-
|
70
|
+
|
104
71
|
if @keepalive_timeout and @request_last
|
105
72
|
between = Time.now.to_i - @request_last.to_i
|
106
73
|
if between >= @keepalive_timeout
|
@@ -108,10 +75,10 @@ class Http2
|
|
108
75
|
return false
|
109
76
|
end
|
110
77
|
end
|
111
|
-
|
78
|
+
|
112
79
|
return true
|
113
80
|
end
|
114
|
-
|
81
|
+
|
115
82
|
#Destroys the object unsetting all variables and closing all sockets.
|
116
83
|
#===Examples
|
117
84
|
# http.destroy
|
@@ -123,72 +90,38 @@ class Http2
|
|
123
90
|
@uagent = nil
|
124
91
|
@keepalive_timeout = nil
|
125
92
|
@request_last = nil
|
126
|
-
|
93
|
+
|
127
94
|
@sock.close if @sock and !@sock.closed?
|
128
95
|
@sock = nil
|
129
|
-
|
96
|
+
|
130
97
|
@sock_plain.close if @sock_plain and !@sock_plain.closed?
|
131
98
|
@sock_plain = nil
|
132
|
-
|
99
|
+
|
133
100
|
@sock_ssl.close if @sock_ssl and !@sock_ssl.closed?
|
134
101
|
@sock_ssl = nil
|
135
102
|
end
|
136
|
-
|
103
|
+
|
137
104
|
#Reconnects to the host.
|
138
105
|
def reconnect
|
139
|
-
require "socket"
|
140
106
|
puts "Http2: Reconnect." if @debug
|
141
|
-
|
142
|
-
#Reset variables.
|
143
|
-
@keepalive_max = nil
|
144
|
-
@keepalive_timeout = nil
|
145
|
-
@connection = nil
|
146
|
-
@contenttype = nil
|
147
|
-
@charset = nil
|
148
|
-
|
107
|
+
|
149
108
|
#Open connection.
|
150
109
|
if @args[:proxy] && @args[:ssl]
|
151
|
-
|
152
|
-
@sock_plain = TCPSocket.new(@args[:proxy][:host], @args[:proxy][:port])
|
153
|
-
|
154
|
-
@sock_plain.write("CONNECT #{@args[:host]}:#{@args[:port]} HTTP/1.0#{@nl}")
|
155
|
-
@sock_plain.write("User-Agent: #{@uagent}#{@nl}")
|
156
|
-
|
157
|
-
if @args[:proxy][:user] and @args[:proxy][:passwd]
|
158
|
-
credential = ["#{@args[:proxy][:user]}:#{@args[:proxy][:passwd]}"].pack("m")
|
159
|
-
credential.delete!("\r\n")
|
160
|
-
@sock_plain.write("Proxy-Authorization: Basic #{credential}#{@nl}")
|
161
|
-
end
|
162
|
-
|
163
|
-
@sock_plain.write(@nl)
|
164
|
-
|
165
|
-
res = @sock_plain.gets
|
166
|
-
raise res if res.to_s.downcase != "http/1.0 200 connection established#{@nl}"
|
110
|
+
connect_proxy_ssl
|
167
111
|
elsif @args[:proxy]
|
168
|
-
|
169
|
-
@sock_plain = TCPSocket.new(@args[:proxy][:host], @args[:proxy][:port].to_i)
|
112
|
+
connect_proxy
|
170
113
|
else
|
171
114
|
print "Http2: Opening socket connection to '#{@args[:host]}:#{@args[:port]}'.\n" if @debug
|
172
115
|
@sock_plain = TCPSocket.new(@args[:host], @args[:port].to_i)
|
173
116
|
end
|
174
|
-
|
117
|
+
|
175
118
|
if @args[:ssl]
|
176
|
-
|
177
|
-
require "openssl" unless ::Kernel.const_defined?(:OpenSSL)
|
178
|
-
|
179
|
-
ssl_context = OpenSSL::SSL::SSLContext.new
|
180
|
-
#ssl_context.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
181
|
-
|
182
|
-
@sock_ssl = OpenSSL::SSL::SSLSocket.new(@sock_plain, ssl_context)
|
183
|
-
@sock_ssl.sync_close = true
|
184
|
-
@sock_ssl.connect
|
185
|
-
|
186
|
-
@sock = @sock_ssl
|
119
|
+
apply_ssl
|
187
120
|
else
|
188
121
|
@sock = @sock_plain
|
189
122
|
end
|
190
123
|
end
|
191
|
-
|
124
|
+
|
192
125
|
#Forces various stuff into arguments-hash like URL from original arguments and enables single-string-shortcuts and more.
|
193
126
|
def parse_args(*args)
|
194
127
|
if args.length == 1 and args.first.is_a?(String)
|
@@ -200,46 +133,46 @@ class Http2
|
|
200
133
|
else
|
201
134
|
raise "Invalid arguments: '#{args.class.name}'."
|
202
135
|
end
|
203
|
-
|
136
|
+
|
204
137
|
if !args.key?(:url) or !args[:url]
|
205
138
|
raise "No URL given: '#{args[:url]}'."
|
206
139
|
elsif args[:url].to_s.split("\n").length > 1
|
207
140
|
raise "Multiple lines given in URL: '#{args[:url]}'."
|
208
141
|
end
|
209
|
-
|
142
|
+
|
210
143
|
return args
|
211
144
|
end
|
212
|
-
|
145
|
+
|
213
146
|
#Returns a result-object based on the arguments.
|
214
147
|
#===Examples
|
215
148
|
# res = http.get("somepage.html")
|
216
149
|
# print res.body #=> <String>-object containing the HTML gotten.
|
217
150
|
def get(args)
|
218
151
|
args = self.parse_args(args)
|
219
|
-
|
152
|
+
|
220
153
|
if args.key?(:method) && args[:method]
|
221
154
|
method = args[:method].to_s.upcase
|
222
155
|
else
|
223
156
|
method = "GET"
|
224
157
|
end
|
225
|
-
|
158
|
+
|
226
159
|
header_str = "#{method} /#{args[:url]} HTTP/1.1#{@nl}"
|
227
160
|
header_str << self.header_str(self.default_headers(args), args)
|
228
161
|
header_str << @nl
|
229
|
-
|
162
|
+
|
230
163
|
@mutex.synchronize do
|
231
164
|
print "Http2: Writing headers.\n" if @debug
|
232
165
|
print "Header str: #{header_str}\n" if @debug
|
233
166
|
self.write(header_str)
|
234
|
-
|
167
|
+
|
235
168
|
print "Http2: Reading response.\n" if @debug
|
236
169
|
resp = self.read_response(args)
|
237
|
-
|
170
|
+
|
238
171
|
print "Http2: Done with get request.\n" if @debug
|
239
172
|
return resp
|
240
173
|
end
|
241
174
|
end
|
242
|
-
|
175
|
+
|
243
176
|
# Proxies the request to another method but forces the method to be "DELETE".
|
244
177
|
def delete(args)
|
245
178
|
if args[:json]
|
@@ -248,14 +181,14 @@ class Http2
|
|
248
181
|
return self.get(args.merge(:method => :delete))
|
249
182
|
end
|
250
183
|
end
|
251
|
-
|
184
|
+
|
252
185
|
#Tries to write a string to the socket. If it fails it reconnects and tries again.
|
253
186
|
def write(str)
|
254
187
|
#Reset variables.
|
255
188
|
@length = nil
|
256
189
|
@encoding = nil
|
257
190
|
self.reconnect if !self.socket_working?
|
258
|
-
|
191
|
+
|
259
192
|
begin
|
260
193
|
raise Errno::EPIPE, "The socket is closed." if !@sock or @sock.closed?
|
261
194
|
self.sock_write(str)
|
@@ -263,59 +196,59 @@ class Http2
|
|
263
196
|
self.reconnect
|
264
197
|
self.sock_write(str)
|
265
198
|
end
|
266
|
-
|
199
|
+
|
267
200
|
@request_last = Time.now
|
268
201
|
end
|
269
|
-
|
202
|
+
|
270
203
|
#Returns the default headers for a request.
|
271
204
|
#===Examples
|
272
205
|
# headers_hash = http.default_headers
|
273
206
|
# print "#{headers_hash}"
|
274
207
|
def default_headers(args = {})
|
275
208
|
return args[:default_headers] if args[:default_headers]
|
276
|
-
|
209
|
+
|
277
210
|
headers = {
|
278
211
|
"Connection" => "Keep-Alive",
|
279
212
|
"User-Agent" => @uagent
|
280
213
|
}
|
281
|
-
|
214
|
+
|
282
215
|
#Possible to give custom host-argument.
|
283
216
|
_args = args[:host] ? args : @args
|
284
217
|
headers["Host"] = _args[:host]
|
285
218
|
headers["Host"] += ":#{_args[:port]}" unless _args[:port] && [80,443].include?(_args[:port].to_i)
|
286
|
-
|
219
|
+
|
287
220
|
if !@args.key?(:encoding_gzip) or @args[:encoding_gzip]
|
288
221
|
headers["Accept-Encoding"] = "gzip"
|
289
222
|
end
|
290
|
-
|
223
|
+
|
291
224
|
if @args[:basic_auth]
|
292
225
|
require "base64" unless ::Kernel.const_defined?(:Base64)
|
293
226
|
headers["Authorization"] = "Basic #{Base64.encode64("#{@args[:basic_auth][:user]}:#{@args[:basic_auth][:passwd]}").strip}"
|
294
227
|
end
|
295
|
-
|
228
|
+
|
296
229
|
if @args[:extra_headers]
|
297
230
|
headers.merge!(@args[:extra_headers])
|
298
231
|
end
|
299
|
-
|
232
|
+
|
300
233
|
if args[:headers]
|
301
234
|
headers.merge!(args[:headers])
|
302
235
|
end
|
303
|
-
|
236
|
+
|
304
237
|
return headers
|
305
238
|
end
|
306
|
-
|
239
|
+
|
307
240
|
#This is used to convert a hash to valid post-data recursivly.
|
308
241
|
def self.post_convert_data(pdata, args = nil)
|
309
242
|
praw = ""
|
310
|
-
|
243
|
+
|
311
244
|
if pdata.is_a?(Hash)
|
312
245
|
pdata.each do |key, val|
|
313
246
|
praw << "&" if praw != ""
|
314
|
-
|
247
|
+
|
315
248
|
if args and args[:orig_key]
|
316
249
|
key = "#{args[:orig_key]}[#{key}]"
|
317
250
|
end
|
318
|
-
|
251
|
+
|
319
252
|
if val.is_a?(Hash) or val.is_a?(Array)
|
320
253
|
praw << self.post_convert_data(val, {:orig_key => key})
|
321
254
|
else
|
@@ -326,28 +259,28 @@ class Http2
|
|
326
259
|
count = 0
|
327
260
|
pdata.each do |val|
|
328
261
|
praw << "&" if praw != ""
|
329
|
-
|
262
|
+
|
330
263
|
if args and args[:orig_key]
|
331
264
|
key = "#{args[:orig_key]}[#{count}]"
|
332
265
|
else
|
333
266
|
key = count
|
334
267
|
end
|
335
|
-
|
268
|
+
|
336
269
|
if val.is_a?(Hash) or val.is_a?(Array)
|
337
270
|
praw << self.post_convert_data(val, {:orig_key => key})
|
338
271
|
else
|
339
272
|
praw << "#{Http2::Utils.urlenc(key)}=#{Http2::Utils.urlenc(Http2.post_convert_data(val))}"
|
340
273
|
end
|
341
|
-
|
274
|
+
|
342
275
|
count += 1
|
343
276
|
end
|
344
277
|
else
|
345
278
|
return pdata.to_s
|
346
279
|
end
|
347
|
-
|
280
|
+
|
348
281
|
return praw
|
349
282
|
end
|
350
|
-
|
283
|
+
|
351
284
|
VALID_ARGUMENTS_POST = [:post, :url, :default_headers, :headers, :json, :method, :cookies, :on_content, :content_type]
|
352
285
|
#Posts to a certain page.
|
353
286
|
#===Examples
|
@@ -356,15 +289,15 @@ class Http2
|
|
356
289
|
args.each do |key, val|
|
357
290
|
raise "Invalid key: '#{key}'." unless VALID_ARGUMENTS_POST.include?(key)
|
358
291
|
end
|
359
|
-
|
292
|
+
|
360
293
|
args = self.parse_args(args)
|
361
|
-
|
294
|
+
|
362
295
|
if args.key?(:method) && args[:method]
|
363
296
|
method = args[:method].to_s.upcase
|
364
297
|
else
|
365
298
|
method = "POST"
|
366
299
|
end
|
367
|
-
|
300
|
+
|
368
301
|
if args[:json]
|
369
302
|
require "json" unless ::Kernel.const_defined?(:JSON)
|
370
303
|
praw = args[:json].to_json
|
@@ -378,134 +311,80 @@ class Http2
|
|
378
311
|
end
|
379
312
|
|
380
313
|
content_type = args[:content_type] || content_type || "application/x-www-form-urlencoded"
|
381
|
-
|
314
|
+
|
382
315
|
@mutex.synchronize do
|
383
316
|
puts "Http2: Doing post." if @debug
|
384
|
-
|
317
|
+
|
385
318
|
header_str = "#{method} /#{args[:url]} HTTP/1.1#{@nl}"
|
386
319
|
header_str << self.header_str({"Content-Length" => praw.bytesize, "Content-Type" => content_type}.merge(self.default_headers(args)), args)
|
387
320
|
header_str << @nl
|
388
321
|
header_str << praw
|
389
|
-
|
322
|
+
|
390
323
|
puts "Http2: Header str: #{header_str}" if @debug
|
391
|
-
|
324
|
+
|
392
325
|
self.write(header_str)
|
393
326
|
return self.read_response(args)
|
394
327
|
end
|
395
328
|
end
|
396
|
-
|
329
|
+
|
397
330
|
#Posts to a certain page using the multipart-method.
|
398
331
|
#===Examples
|
399
332
|
# res = http.post_multipart("upload.php", {"normal_value" => 123, "file" => Tempfile.new(?)})
|
400
333
|
def post_multipart(*args)
|
401
334
|
args = self.parse_args(*args)
|
402
|
-
|
335
|
+
|
403
336
|
phash = args[:post].clone
|
404
337
|
autostate_set_on_post_hash(phash) if @args[:autostate]
|
405
|
-
|
406
|
-
|
407
|
-
|
408
|
-
|
338
|
+
|
339
|
+
post_multipart_helper = ::Http2::PostMultipartHelper.new(self)
|
340
|
+
|
409
341
|
#Use tempfile to store contents to avoid eating memory if posting something really big.
|
410
|
-
|
411
|
-
|
412
|
-
Tempfile.open("http2_post_multipart_tmp_#{boundary}") do |praw|
|
413
|
-
phash.each do |key, val|
|
414
|
-
praw << "--#{boundary}#{@nl}"
|
415
|
-
|
416
|
-
if val.class.name.to_s == "Tempfile" and val.respond_to?(:original_filename)
|
417
|
-
praw << "Content-Disposition: form-data; name=\"#{key}\"; filename=\"#{val.original_filename}\";#{@nl}"
|
418
|
-
praw << "Content-Length: #{val.to_s.bytesize}#{@nl}"
|
419
|
-
elsif val.is_a?(Hash) and val[:filename]
|
420
|
-
praw << "Content-Disposition: form-data; name=\"#{key}\"; filename=\"#{val[:filename]}\";#{@nl}"
|
421
|
-
|
422
|
-
if val[:content]
|
423
|
-
praw << "Content-Length: #{val[:content].to_s.bytesize}#{@nl}"
|
424
|
-
elsif val[:fpath]
|
425
|
-
praw << "Content-Length: #{File.size(val[:fpath])}#{@nl}"
|
426
|
-
else
|
427
|
-
raise "Could not figure out where to get content from."
|
428
|
-
end
|
429
|
-
else
|
430
|
-
praw << "Content-Disposition: form-data; name=\"#{key}\";#{@nl}"
|
431
|
-
praw << "Content-Length: #{val.to_s.bytesize}#{@nl}"
|
432
|
-
end
|
433
|
-
|
434
|
-
praw << "Content-Type: text/plain#{@nl}"
|
435
|
-
praw << @nl
|
436
|
-
|
437
|
-
if val.class.name.to_s == "StringIO"
|
438
|
-
praw << val.read
|
439
|
-
elsif val.is_a?(Hash) and val[:content]
|
440
|
-
praw << val[:content].to_s
|
441
|
-
elsif val.is_a?(Hash) and val[:fpath]
|
442
|
-
File.open(val[:fpath], "r") do |fp|
|
443
|
-
begin
|
444
|
-
while data = fp.sysread(4096)
|
445
|
-
praw << data
|
446
|
-
end
|
447
|
-
rescue EOFError
|
448
|
-
#ignore.
|
449
|
-
end
|
450
|
-
end
|
451
|
-
else
|
452
|
-
praw << val.to_s
|
453
|
-
end
|
454
|
-
|
455
|
-
praw << @nl
|
456
|
-
end
|
457
|
-
|
458
|
-
praw << "--#{boundary}--"
|
459
|
-
|
460
|
-
|
342
|
+
post_multipart_helper.generate_raw(phash) do |helper, praw|
|
461
343
|
#Generate header-string containing 'praw'-variable.
|
462
344
|
header_str = "POST /#{args[:url]} HTTP/1.1#{@nl}"
|
463
|
-
header_str <<
|
464
|
-
"Content-Type" => "multipart/form-data; boundary=#{boundary}",
|
345
|
+
header_str << header_str(default_headers(args).merge(
|
346
|
+
"Content-Type" => "multipart/form-data; boundary=#{helper.boundary}",
|
465
347
|
"Content-Length" => praw.size
|
466
348
|
), args)
|
467
349
|
header_str << @nl
|
468
|
-
|
469
|
-
|
470
|
-
#Debug.
|
350
|
+
|
471
351
|
print "Http2: Headerstr: #{header_str}\n" if @debug
|
472
|
-
|
473
|
-
|
352
|
+
|
474
353
|
#Write and return.
|
475
354
|
@mutex.synchronize do
|
476
|
-
|
477
|
-
|
355
|
+
write(header_str)
|
356
|
+
|
478
357
|
praw.rewind
|
479
|
-
praw.
|
480
|
-
|
358
|
+
praw.each_line do |data|
|
359
|
+
sock_write(data)
|
481
360
|
end
|
482
|
-
|
483
|
-
return
|
361
|
+
|
362
|
+
return read_response(args)
|
484
363
|
end
|
485
364
|
end
|
486
365
|
end
|
487
|
-
|
366
|
+
|
488
367
|
def sock_write(str)
|
489
368
|
str = str.to_s
|
490
369
|
return nil if str.empty?
|
491
370
|
count = @sock.write(str)
|
492
371
|
raise "Couldnt write to socket: '#{count}', '#{str}'." if count <= 0
|
493
372
|
end
|
494
|
-
|
373
|
+
|
495
374
|
def sock_puts(str)
|
496
375
|
self.sock_write("#{str}#{@nl}")
|
497
376
|
end
|
498
|
-
|
377
|
+
|
499
378
|
#Returns a header-string which normally would be used for a request in the given state.
|
500
379
|
def header_str(headers_hash, args = {})
|
501
380
|
if @cookies.length > 0 and (!args.key?(:cookies) or args[:cookies])
|
502
381
|
cstr = ""
|
503
|
-
|
382
|
+
|
504
383
|
first = true
|
505
384
|
@cookies.each do |cookie_name, cookie_data|
|
506
385
|
cstr << "; " if !first
|
507
386
|
first = false if first
|
508
|
-
|
387
|
+
|
509
388
|
if cookie_data.is_a?(Hash)
|
510
389
|
name = cookie_data["name"]
|
511
390
|
value = cookie_data["value"]
|
@@ -513,281 +392,135 @@ class Http2
|
|
513
392
|
name = cookie_name
|
514
393
|
value = cookie_data
|
515
394
|
end
|
516
|
-
|
395
|
+
|
517
396
|
raise "Unexpected lines: #{value.lines.to_a.length}." if value.lines.to_a.length != 1
|
518
397
|
cstr << "#{Http2::Utils.urlenc(name)}=#{Http2::Utils.urlenc(value)}"
|
519
398
|
end
|
520
|
-
|
399
|
+
|
521
400
|
headers_hash["Cookie"] = cstr
|
522
401
|
end
|
523
|
-
|
402
|
+
|
524
403
|
headers_str = ""
|
525
404
|
headers_hash.each do |key, val|
|
526
405
|
headers_str << "#{key}: #{val}#{@nl}"
|
527
406
|
end
|
528
|
-
|
407
|
+
|
529
408
|
return headers_str
|
530
409
|
end
|
531
|
-
|
410
|
+
|
532
411
|
def on_content_call(args, str)
|
533
412
|
args[:on_content].call(str) if args.key?(:on_content)
|
534
413
|
end
|
535
|
-
|
414
|
+
|
536
415
|
#Reads the response after posting headers and data.
|
537
416
|
#===Examples
|
538
417
|
# res = http.read_response
|
539
418
|
def read_response(args = {})
|
540
|
-
|
541
|
-
|
542
|
-
|
543
|
-
|
544
|
-
|
545
|
-
|
546
|
-
begin
|
547
|
-
if @length and @length > 0 and @mode == "body"
|
548
|
-
line = @sock.read(@length)
|
549
|
-
raise "Expected to get #{@length} of bytes but got #{line.bytesize}" if @length != line.bytesize
|
550
|
-
else
|
551
|
-
line = @sock.gets
|
552
|
-
end
|
553
|
-
|
554
|
-
if line
|
555
|
-
rec_count += line.length
|
556
|
-
elsif !line and rec_count <= 0
|
557
|
-
@sock = nil
|
558
|
-
raise Errno::ECONNABORTED, "Server closed the connection before being able to read anything (KeepAliveMax: '#{@keepalive_max}', Connection: '#{@connection}', PID: '#{Process.pid}')."
|
559
|
-
end
|
560
|
-
|
561
|
-
puts "<#{@mode}>: '#{line}'" if @debug
|
562
|
-
rescue Errno::ECONNRESET => e
|
563
|
-
if rec_count > 0
|
564
|
-
print "Http2: The connection was reset while reading - breaking gently...\n" if @debug
|
565
|
-
@sock = nil
|
566
|
-
break
|
567
|
-
else
|
568
|
-
raise Errno::ECONNABORTED, "Server closed the connection before being able to read anything (KeepAliveMax: '#{@keepalive_max}', Connection: '#{@connection}', PID: '#{Process.pid}')."
|
569
|
-
end
|
570
|
-
end
|
571
|
-
|
572
|
-
break if line.to_s == ""
|
573
|
-
|
574
|
-
if @mode == "headers" and (line == "\n" || line == "\r\n")
|
575
|
-
puts "Http2: Changing mode to body!" if @debug
|
576
|
-
raise "No headers was given at all? Possibly corrupt state after last request?" if @resp.headers.empty?
|
577
|
-
break if @length == 0
|
578
|
-
@mode = "body"
|
579
|
-
self.on_content_call(args, @nl)
|
580
|
-
next
|
581
|
-
end
|
582
|
-
|
583
|
-
if @mode == "headers"
|
584
|
-
self.parse_header(line, args)
|
585
|
-
elsif @mode == "body"
|
586
|
-
stat = self.parse_body(line, args)
|
587
|
-
break if stat == "break"
|
588
|
-
next if stat == "next"
|
589
|
-
end
|
590
|
-
end
|
591
|
-
|
592
|
-
|
593
|
-
#Release variables.
|
594
|
-
resp = @resp
|
595
|
-
@resp = nil
|
596
|
-
@mode = nil
|
597
|
-
|
598
|
-
|
599
|
-
#Check if we should reconnect based on keep-alive-max.
|
600
|
-
if @keepalive_max == 1 or @connection == "close"
|
601
|
-
@sock.close if !@sock.closed?
|
602
|
-
@sock = nil
|
603
|
-
end
|
604
|
-
|
605
|
-
|
606
|
-
|
607
|
-
# Validate that the response is as it should be.
|
608
|
-
puts "Http2: Validating response." if @debug
|
609
|
-
resp.validate!
|
610
|
-
|
611
|
-
|
612
|
-
#Check if the content is gzip-encoded - if so: decode it!
|
613
|
-
if @encoding == "gzip"
|
614
|
-
require "zlib"
|
615
|
-
require "stringio"
|
616
|
-
io = StringIO.new(resp.args[:body])
|
617
|
-
gz = Zlib::GzipReader.new(io)
|
618
|
-
untrusted_str = gz.read
|
619
|
-
|
620
|
-
begin
|
621
|
-
valid_string = ic.encode("UTF-8")
|
622
|
-
rescue
|
623
|
-
valid_string = untrusted_str.force_encoding("UTF-8").encode("UTF-8", :invalid => :replace, :replace => "").encode("UTF-8")
|
624
|
-
end
|
625
|
-
|
626
|
-
resp.args[:body] = valid_string
|
627
|
-
end
|
628
|
-
|
629
|
-
|
630
|
-
|
631
|
-
|
632
|
-
raise "No status-code was received from the server. Headers: '#{resp.headers}' Body: '#{resp.args[:body]}'." if !resp.args[:code]
|
633
|
-
|
634
|
-
if (resp.args[:code].to_s == "302" || resp.args[:code].to_s == "307") and resp.header?("location") and (!@args.key?(:follow_redirects) or @args[:follow_redirects])
|
635
|
-
require "uri"
|
636
|
-
uri = URI.parse(resp.header("location"))
|
637
|
-
url = uri.path
|
638
|
-
url << "?#{uri.query}" if uri.query.to_s.length > 0
|
639
|
-
|
640
|
-
args = {:host => uri.host}
|
641
|
-
args[:ssl] = true if uri.scheme == "https"
|
642
|
-
args[:port] = uri.port if uri.port
|
643
|
-
|
644
|
-
puts "Http2: Redirecting from location-header to '#{url}'." if @debug
|
645
|
-
|
646
|
-
if !args[:host] or args[:host] == @args[:host]
|
647
|
-
return self.get(url)
|
648
|
-
else
|
649
|
-
http = Http2.new(args)
|
650
|
-
return http.get(url)
|
651
|
-
end
|
652
|
-
elsif @raise_errors && resp.args[:code].to_i == 500
|
653
|
-
err = Http2::Errors::Internalserver.new(resp.body)
|
654
|
-
err.response = resp
|
655
|
-
raise err
|
656
|
-
elsif @raise_errors && resp.args[:code].to_i == 403
|
657
|
-
err = Http2::Errors::Noaccess.new(resp.body)
|
658
|
-
err.response = resp
|
659
|
-
raise err
|
660
|
-
elsif @raise_errors && resp.args[:code].to_i == 400
|
661
|
-
err = Http2::Errors::Badrequest.new(resp.body)
|
662
|
-
err.response = resp
|
663
|
-
raise err
|
664
|
-
elsif @raise_errors && resp.args[:code].to_i == 404
|
665
|
-
err = Http2::Errors::Notfound.new(resp.body)
|
666
|
-
err.response = resp
|
667
|
-
raise err
|
668
|
-
else
|
669
|
-
autostate_register(resp) if @args[:autostate]
|
670
|
-
|
671
|
-
return resp
|
672
|
-
end
|
673
|
-
end
|
674
|
-
|
675
|
-
#Parse a header-line and saves it on the object.
|
676
|
-
#===Examples
|
677
|
-
# http.parse_header("Content-Type: text/html\r\n")
|
678
|
-
def parse_header(line, args = {})
|
679
|
-
if match = line.match(/^(.+?):\s*(.+)#{@nl}$/)
|
680
|
-
key = match[1].to_s.downcase
|
681
|
-
|
682
|
-
if key == "set-cookie"
|
683
|
-
Http2::Utils.parse_set_cookies(match[2]).each do |cookie_data|
|
684
|
-
@cookies[cookie_data["name"]] = cookie_data
|
685
|
-
end
|
686
|
-
elsif key == "keep-alive"
|
687
|
-
if ka_max = match[2].to_s.match(/max=(\d+)/)
|
688
|
-
@keepalive_max = ka_max[1].to_i
|
689
|
-
print "Http2: Keepalive-max set to: '#{@keepalive_max}'.\n" if @debug
|
690
|
-
end
|
691
|
-
|
692
|
-
if ka_timeout = match[2].to_s.match(/timeout=(\d+)/)
|
693
|
-
@keepalive_timeout = ka_timeout[1].to_i
|
694
|
-
print "Http2: Keepalive-timeout set to: '#{@keepalive_timeout}'.\n" if @debug
|
695
|
-
end
|
696
|
-
elsif key == "connection"
|
697
|
-
@connection = match[2].to_s.downcase
|
698
|
-
elsif key == "content-encoding"
|
699
|
-
@encoding = match[2].to_s.downcase
|
700
|
-
elsif key == "content-length"
|
701
|
-
@length = match[2].to_i
|
702
|
-
elsif key == "content-type"
|
703
|
-
ctype = match[2].to_s
|
704
|
-
if match_charset = ctype.match(/\s*;\s*charset=(.+)/i)
|
705
|
-
@charset = match_charset[1].downcase
|
706
|
-
@resp.args[:charset] = @charset
|
707
|
-
ctype.gsub!(match_charset[0], "")
|
708
|
-
end
|
709
|
-
|
710
|
-
@ctype = ctype
|
711
|
-
@resp.args[:contenttype] = @ctype
|
712
|
-
elsif key == "transfer-encoding"
|
713
|
-
@transfer_encoding = match[2].to_s.downcase.strip
|
714
|
-
end
|
715
|
-
|
716
|
-
puts "Http2: Parsed header: #{match[1]}: #{match[2]}" if @debug
|
717
|
-
@resp.headers[key] = [] unless @resp.headers.key?(key)
|
718
|
-
@resp.headers[key] << match[2]
|
719
|
-
|
720
|
-
if key != "transfer-encoding" and key != "content-length" and key != "connection" and key != "keep-alive"
|
721
|
-
self.on_content_call(args, line)
|
722
|
-
end
|
723
|
-
elsif match = line.match(/^HTTP\/([\d\.]+)\s+(\d+)\s+(.+)$/)
|
724
|
-
@resp.args[:code] = match[2]
|
725
|
-
@resp.args[:http_version] = match[1]
|
726
|
-
|
727
|
-
self.on_content_call(args, line)
|
728
|
-
else
|
729
|
-
raise "Could not understand header string: '#{line}'.\n\n#{@sock.read(409600)}"
|
730
|
-
end
|
731
|
-
end
|
732
|
-
|
733
|
-
#Parses the body based on given headers and saves it to the result-object.
|
734
|
-
# http.parse_body(str)
|
735
|
-
def parse_body(line, args)
|
736
|
-
if @resp.args[:http_version] = "1.1"
|
737
|
-
return "break" if @length == 0
|
738
|
-
|
739
|
-
if @transfer_encoding == "chunked"
|
740
|
-
len = line.strip.hex
|
741
|
-
|
742
|
-
if len > 0
|
743
|
-
read = @sock.read(len)
|
744
|
-
return "break" if read == "" or (read == "\n" || read == "\r\n")
|
745
|
-
@resp.args[:body] << read
|
746
|
-
self.on_content_call(args, read)
|
747
|
-
end
|
748
|
-
|
749
|
-
nl = @sock.gets
|
750
|
-
if len == 0
|
751
|
-
if nl == "\n" || nl == "\r\n"
|
752
|
-
return "break"
|
753
|
-
else
|
754
|
-
raise "Dont know what to do :'-("
|
755
|
-
end
|
756
|
-
end
|
757
|
-
|
758
|
-
raise "Should have read newline but didnt: '#{nl}'." if nl != @nl
|
759
|
-
else
|
760
|
-
puts "Http2: Adding #{line.to_s.bytesize} to the body." if @debug
|
761
|
-
@resp.args[:body] << line.to_s
|
762
|
-
self.on_content_call(args, line)
|
763
|
-
return "break" if @resp.header?("content-length") && @resp.args[:body].length >= @resp.header("content-length").to_i
|
764
|
-
end
|
765
|
-
else
|
766
|
-
raise "Dont know how to read HTTP version: '#{@resp.args[:http_version]}'."
|
767
|
-
end
|
419
|
+
Http2::ResponseReader.new(
|
420
|
+
http2: self,
|
421
|
+
sock: @sock,
|
422
|
+
args: args,
|
423
|
+
debug: @debug
|
424
|
+
).response
|
768
425
|
end
|
769
|
-
|
770
|
-
|
771
|
-
|
426
|
+
|
427
|
+
private
|
428
|
+
|
772
429
|
#Registers the states from a result.
|
773
430
|
def autostate_register(res)
|
774
431
|
puts "Http2: Running autostate-register on result." if @debug
|
775
432
|
@autostate_values.clear
|
776
|
-
|
433
|
+
|
777
434
|
res.body.to_s.scan(/<input type="hidden" name="__(EVENTTARGET|EVENTARGUMENT|VIEWSTATE|LASTFOCUS)" id="(.*?)" value="(.*?)" \/>/) do |match|
|
778
435
|
name = "__#{match[0]}"
|
779
436
|
id = match[1]
|
780
437
|
value = match[2]
|
781
|
-
|
438
|
+
|
782
439
|
puts "Http2: Registered autostate-value with name '#{name}' and value '#{value}'." if @debug
|
783
440
|
@autostate_values[name] = Http2::Utils.urldec(value)
|
784
441
|
end
|
785
|
-
|
442
|
+
|
786
443
|
raise "No states could be found." if @autostate_values.empty?
|
787
444
|
end
|
788
|
-
|
445
|
+
|
789
446
|
#Sets the states on the given post-hash.
|
790
447
|
def autostate_set_on_post_hash(phash)
|
791
448
|
phash.merge!(@autostate_values)
|
792
449
|
end
|
450
|
+
|
451
|
+
def parse_init_args(args)
|
452
|
+
args = {:host => args} if args.is_a?(String)
|
453
|
+
raise "Arguments wasnt a hash." unless args.is_a?(Hash)
|
454
|
+
|
455
|
+
args.each do |key, val|
|
456
|
+
raise "Invalid key: '#{key}'." unless VALID_ARGUMENTS_INITIALIZE.include?(key)
|
457
|
+
end
|
458
|
+
|
459
|
+
raise "No host was given." unless args[:host]
|
460
|
+
return args
|
461
|
+
end
|
462
|
+
|
463
|
+
def set_default_values
|
464
|
+
@debug = @args[:debug]
|
465
|
+
@autostate_values = {} if @args[:autostate]
|
466
|
+
@nl = @args[:nl] || "\r\n"
|
467
|
+
|
468
|
+
if !@args[:port]
|
469
|
+
if @args[:ssl]
|
470
|
+
@args[:port] = 443
|
471
|
+
else
|
472
|
+
@args[:port] = 80
|
473
|
+
end
|
474
|
+
end
|
475
|
+
|
476
|
+
if @args[:user_agent]
|
477
|
+
@uagent = @args[:user_agent]
|
478
|
+
else
|
479
|
+
@uagent = "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)"
|
480
|
+
end
|
481
|
+
|
482
|
+
if !@args.key?(:raise_errors) || @args[:raise_errors]
|
483
|
+
@raise_errors = true
|
484
|
+
else
|
485
|
+
@raise_errors = false
|
486
|
+
end
|
487
|
+
end
|
488
|
+
|
489
|
+
def connect_proxy_ssl
|
490
|
+
print "Http2: Initializing proxy stuff.\n" if @debug
|
491
|
+
@sock_plain = TCPSocket.new(@args[:proxy][:host], @args[:proxy][:port])
|
492
|
+
|
493
|
+
@sock_plain.write("CONNECT #{@args[:host]}:#{@args[:port]} HTTP/1.0#{@nl}")
|
494
|
+
@sock_plain.write("User-Agent: #{@uagent}#{@nl}")
|
495
|
+
|
496
|
+
if @args[:proxy][:user] and @args[:proxy][:passwd]
|
497
|
+
credential = ["#{@args[:proxy][:user]}:#{@args[:proxy][:passwd]}"].pack("m")
|
498
|
+
credential.delete!("\r\n")
|
499
|
+
@sock_plain.write("Proxy-Authorization: Basic #{credential}#{@nl}")
|
500
|
+
end
|
501
|
+
|
502
|
+
@sock_plain.write(@nl)
|
503
|
+
|
504
|
+
res = @sock_plain.gets
|
505
|
+
raise res if res.to_s.downcase != "http/1.0 200 connection established#{@nl}"
|
506
|
+
end
|
507
|
+
|
508
|
+
def connect_proxy
|
509
|
+
puts "Http2: Opening socket connection to '#{@args[:host]}:#{@args[:port]}' through proxy '#{@args[:proxy][:host]}:#{@args[:proxy][:port]}'." if @debug
|
510
|
+
@sock_plain = TCPSocket.new(@args[:proxy][:host], @args[:proxy][:port].to_i)
|
511
|
+
end
|
512
|
+
|
513
|
+
def apply_ssl
|
514
|
+
puts "Http2: Initializing SSL." if @debug
|
515
|
+
require "openssl" unless ::Kernel.const_defined?(:OpenSSL)
|
516
|
+
|
517
|
+
ssl_context = OpenSSL::SSL::SSLContext.new
|
518
|
+
#ssl_context.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
519
|
+
|
520
|
+
@sock_ssl = OpenSSL::SSL::SSLSocket.new(@sock_plain, ssl_context)
|
521
|
+
@sock_ssl.sync_close = true
|
522
|
+
@sock_ssl.connect
|
523
|
+
|
524
|
+
@sock = @sock_ssl
|
525
|
+
end
|
793
526
|
end
|