em-http-request 1.1.5 → 1.1.7
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +5 -5
- data/README.md +1 -1
- data/em-http-request.gemspec +1 -1
- data/lib/em/io_streamer.rb +49 -0
- data/lib/em-http/client.rb +25 -6
- data/lib/em-http/http_client_options.rb +1 -1
- data/lib/em-http/http_connection.rb +77 -5
- data/lib/em-http/http_connection_options.rb +29 -4
- data/lib/em-http/version.rb +1 -1
- data/lib/em-http.rb +1 -0
- data/spec/client_spec.rb +55 -0
- data/spec/dns_spec.rb +2 -2
- data/spec/http_proxy_spec.rb +237 -76
- data/spec/spec_helper.rb +18 -1
- data/spec/ssl_spec.rb +52 -1
- data/spec/stallion.rb +1 -1
- metadata +11 -12
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: c7562e4d20c35c54a9319250e665a883c810546cb657d8f897591e113999ed3a
|
4
|
+
data.tar.gz: 643bf26ea7bfa2a85d6e4257a475295c16eca044ff0d04341537793f07d5bd04
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9d9d1f441081a034f29447cb99b26c73ef7767a989c31df007e1ecfc6ea8db1f4cbe00fb26c0f81b59108f53b54dfbccd51ea9140fd5286efaca6095b45943ad
|
7
|
+
data.tar.gz: 690dc944373085313c41c484edd25366036a3da46855b6a153aab65c19aed961d94d202de07cb3f5f8406585bb5c3a66998913debcba4d02796fb1c8f04be7c1
|
data/README.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# EM-HTTP-Request
|
2
2
|
|
3
|
-
[![Gem Version](https://badge.fury.io/rb/em-http-request.
|
3
|
+
[![Gem Version](https://badge.fury.io/rb/em-http-request.svg)](http://rubygems.org/gems/em-http-request) [![Build Status](https://travis-ci.org/igrigorik/em-http-request.svg)](https://travis-ci.org/igrigorik/em-http-request)
|
4
4
|
|
5
5
|
Async (EventMachine) HTTP client, with support for:
|
6
6
|
|
data/em-http-request.gemspec
CHANGED
@@ -23,7 +23,7 @@ Gem::Specification.new do |s|
|
|
23
23
|
|
24
24
|
s.add_development_dependency 'mongrel', '~> 1.2.0.pre2'
|
25
25
|
s.add_development_dependency 'multi_json'
|
26
|
-
s.add_development_dependency 'rack'
|
26
|
+
s.add_development_dependency 'rack', '< 2.0'
|
27
27
|
s.add_development_dependency 'rake'
|
28
28
|
s.add_development_dependency 'rspec'
|
29
29
|
|
@@ -0,0 +1,49 @@
|
|
1
|
+
require 'em/streamer'
|
2
|
+
|
3
|
+
# similar to EventMachine::FileStreamer, but for any IO object
|
4
|
+
module EventMachine
|
5
|
+
class IOStreamer
|
6
|
+
include Deferrable
|
7
|
+
CHUNK_SIZE = 16384
|
8
|
+
|
9
|
+
# @param [EventMachine::Connection] connection
|
10
|
+
# @param [IO] io Data source
|
11
|
+
# @param [Integer] Data size
|
12
|
+
#
|
13
|
+
# @option opts [Boolean] :http_chunks (false) Use HTTP 1.1 style chunked-encoding semantics.
|
14
|
+
def initialize(connection, io, opts = {})
|
15
|
+
@connection = connection
|
16
|
+
@io = io
|
17
|
+
@http_chunks = opts[:http_chunks]
|
18
|
+
|
19
|
+
@buff = String.new
|
20
|
+
@io.binmode if @io.respond_to?(:binmode)
|
21
|
+
stream_one_chunk
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
# Used internally to stream one chunk at a time over multiple reactor ticks
|
27
|
+
# @private
|
28
|
+
def stream_one_chunk
|
29
|
+
loop do
|
30
|
+
if @io.eof?
|
31
|
+
@connection.send_data "0\r\n\r\n" if @http_chunks
|
32
|
+
succeed
|
33
|
+
break
|
34
|
+
end
|
35
|
+
|
36
|
+
if @connection.respond_to?(:get_outbound_data_size) && (@connection.get_outbound_data_size > FileStreamer::BackpressureLevel)
|
37
|
+
EventMachine::next_tick { stream_one_chunk }
|
38
|
+
break
|
39
|
+
end
|
40
|
+
|
41
|
+
if @io.read(CHUNK_SIZE, @buff)
|
42
|
+
@connection.send_data("#{@buff.length.to_s(16)}\r\n") if @http_chunks
|
43
|
+
@connection.send_data(@buff)
|
44
|
+
@connection.send_data("\r\n") if @http_chunks
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
data/lib/em-http/client.rb
CHANGED
@@ -182,7 +182,7 @@ module EventMachine
|
|
182
182
|
# Set the Content-Length if body is given,
|
183
183
|
# or we're doing an empty post or put
|
184
184
|
if body
|
185
|
-
head['content-length']
|
185
|
+
head['content-length'] ||= body.respond_to?(:bytesize) ? body.bytesize : body.size
|
186
186
|
elsif @req.method == 'POST' or @req.method == 'PUT'
|
187
187
|
# wont happen if body is set and we already set content-length above
|
188
188
|
head['content-length'] ||= 0
|
@@ -198,11 +198,8 @@ module EventMachine
|
|
198
198
|
request_header << CRLF
|
199
199
|
@conn.send_data request_header
|
200
200
|
|
201
|
-
|
202
|
-
|
203
|
-
elsif @req.file
|
204
|
-
@conn.stream_file_data @req.file, :http_chunks => false
|
205
|
-
end
|
201
|
+
@req_body = body || (@req.file && Pathname.new(@req.file))
|
202
|
+
send_request_body unless @req.headers['expect'] == '100-continue'
|
206
203
|
end
|
207
204
|
|
208
205
|
def on_body_data(data)
|
@@ -226,6 +223,28 @@ module EventMachine
|
|
226
223
|
end
|
227
224
|
end
|
228
225
|
|
226
|
+
def request_body_pending?
|
227
|
+
!!@req_body
|
228
|
+
end
|
229
|
+
|
230
|
+
def send_request_body
|
231
|
+
return if @req_body.nil?
|
232
|
+
|
233
|
+
if @req_body.is_a?(String)
|
234
|
+
@conn.send_data @req_body
|
235
|
+
|
236
|
+
elsif @req_body.is_a?(Pathname)
|
237
|
+
@conn.stream_file_data @req_body.to_path, http_chunks: false
|
238
|
+
|
239
|
+
elsif @req_body.respond_to?(:read) && @req_body.respond_to?(:eof?) # IO or IO-like object
|
240
|
+
@conn.stream_data @req_body
|
241
|
+
|
242
|
+
else
|
243
|
+
raise "Don't know how to send request body: #{@req_body.inspect}"
|
244
|
+
end
|
245
|
+
@req_body = nil
|
246
|
+
end
|
247
|
+
|
229
248
|
def parse_response_header(header, version, status)
|
230
249
|
@response_header.raw = header
|
231
250
|
header.each do |key, val|
|
@@ -1,3 +1,5 @@
|
|
1
|
+
require 'em/io_streamer'
|
2
|
+
|
1
3
|
module EventMachine
|
2
4
|
|
3
5
|
module HTTPMethods
|
@@ -20,7 +22,11 @@ module EventMachine
|
|
20
22
|
end
|
21
23
|
|
22
24
|
def receive_data(data)
|
23
|
-
|
25
|
+
begin
|
26
|
+
@parent.receive_data data
|
27
|
+
rescue EventMachine::Connectify::CONNECTError => e
|
28
|
+
@parent.close(e.message)
|
29
|
+
end
|
24
30
|
end
|
25
31
|
|
26
32
|
def connection_completed
|
@@ -30,6 +36,62 @@ module EventMachine
|
|
30
36
|
def unbind(reason=nil)
|
31
37
|
@parent.unbind(reason)
|
32
38
|
end
|
39
|
+
|
40
|
+
# TLS verification support, original implementation by Mislav Marohnić
|
41
|
+
# https://github.com/lostisland/faraday/blob/63cf47c95b573539f047c729bd9ad67560bc83ff/lib/faraday/adapter/em_http_ssl_patch.rb
|
42
|
+
def ssl_verify_peer(cert_string)
|
43
|
+
cert = nil
|
44
|
+
begin
|
45
|
+
cert = OpenSSL::X509::Certificate.new(cert_string)
|
46
|
+
rescue OpenSSL::X509::CertificateError
|
47
|
+
return false
|
48
|
+
end
|
49
|
+
|
50
|
+
@last_seen_cert = cert
|
51
|
+
|
52
|
+
if certificate_store.verify(@last_seen_cert)
|
53
|
+
begin
|
54
|
+
certificate_store.add_cert(@last_seen_cert)
|
55
|
+
rescue OpenSSL::X509::StoreError => e
|
56
|
+
raise e unless e.message == 'cert already in hash table'
|
57
|
+
end
|
58
|
+
true
|
59
|
+
else
|
60
|
+
raise OpenSSL::SSL::SSLError.new(%(unable to verify the server certificate for "#{host}"))
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
def ssl_handshake_completed
|
65
|
+
unless verify_peer?
|
66
|
+
warn "[WARNING; em-http-request] TLS hostname validation is disabled (use 'tls: {verify_peer: true}'), see" +
|
67
|
+
" CVE-2020-13482 and https://github.com/igrigorik/em-http-request/issues/339 for details" unless parent.connopts.tls.has_key?(:verify_peer)
|
68
|
+
return true
|
69
|
+
end
|
70
|
+
|
71
|
+
unless OpenSSL::SSL.verify_certificate_identity(@last_seen_cert, host)
|
72
|
+
raise OpenSSL::SSL::SSLError.new(%(host "#{host}" does not match the server certificate))
|
73
|
+
else
|
74
|
+
true
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
def verify_peer?
|
79
|
+
parent.connopts.tls[:verify_peer]
|
80
|
+
end
|
81
|
+
|
82
|
+
def host
|
83
|
+
parent.connopts.host
|
84
|
+
end
|
85
|
+
|
86
|
+
def certificate_store
|
87
|
+
@certificate_store ||= begin
|
88
|
+
store = OpenSSL::X509::Store.new
|
89
|
+
store.set_default_paths
|
90
|
+
ca_file = parent.connopts.tls[:cert_chain_file]
|
91
|
+
store.add_file(ca_file) if ca_file
|
92
|
+
store
|
93
|
+
end
|
94
|
+
end
|
33
95
|
end
|
34
96
|
|
35
97
|
class HttpConnection
|
@@ -115,8 +177,13 @@ module EventMachine
|
|
115
177
|
@p.header_value_type = :mixed
|
116
178
|
@p.on_headers_complete = proc do |h|
|
117
179
|
if client
|
118
|
-
|
119
|
-
|
180
|
+
if @p.status_code == 100
|
181
|
+
client.send_request_body
|
182
|
+
@p.reset!
|
183
|
+
else
|
184
|
+
client.parse_response_header(h, @p.http_version, @p.status_code)
|
185
|
+
:reset if client.req.no_body?
|
186
|
+
end
|
120
187
|
else
|
121
188
|
# if we receive unexpected data without a pending client request
|
122
189
|
# reset the parser to avoid firing any further callbacks and close
|
@@ -160,9 +227,9 @@ module EventMachine
|
|
160
227
|
@peer = @conn.get_peername
|
161
228
|
|
162
229
|
if @connopts.socks_proxy?
|
163
|
-
socksify(client.req.uri.
|
230
|
+
socksify(client.req.uri.hostname, client.req.uri.inferred_port, *@connopts.proxy[:authorization]) { start }
|
164
231
|
elsif @connopts.connect_proxy?
|
165
|
-
connectify(client.req.uri.
|
232
|
+
connectify(client.req.uri.hostname, client.req.uri.inferred_port, *@connopts.proxy[:authorization]) { start }
|
166
233
|
else
|
167
234
|
start
|
168
235
|
end
|
@@ -185,6 +252,7 @@ module EventMachine
|
|
185
252
|
conn = HttpConnection.new
|
186
253
|
client.conn = conn
|
187
254
|
conn.connopts = @connopts
|
255
|
+
conn.connopts.https = new_location.scheme == "https"
|
188
256
|
conn.uri = client.req.uri
|
189
257
|
conn.activate_connection(client)
|
190
258
|
|
@@ -240,6 +308,10 @@ module EventMachine
|
|
240
308
|
@conn.stream_file_data filename, args
|
241
309
|
end
|
242
310
|
|
311
|
+
def stream_data(io, opts = {})
|
312
|
+
EventMachine::IOStreamer.new(self, io, opts)
|
313
|
+
end
|
314
|
+
|
243
315
|
private
|
244
316
|
|
245
317
|
def client
|
@@ -1,13 +1,13 @@
|
|
1
1
|
class HttpConnectionOptions
|
2
2
|
attr_reader :host, :port, :tls, :proxy, :bind, :bind_port
|
3
3
|
attr_reader :connect_timeout, :inactivity_timeout
|
4
|
+
attr_writer :https
|
4
5
|
|
5
6
|
def initialize(uri, options)
|
6
7
|
@connect_timeout = options[:connect_timeout] || 5 # default connection setup timeout
|
7
8
|
@inactivity_timeout = options[:inactivity_timeout] ||= 10 # default connection inactivity (post-setup) timeout
|
8
9
|
|
9
10
|
@tls = options[:tls] || options[:ssl] || {}
|
10
|
-
@proxy = options[:proxy]
|
11
11
|
|
12
12
|
if bind = options[:bind]
|
13
13
|
@bind = bind[:host] || '0.0.0.0'
|
@@ -20,13 +20,15 @@ class HttpConnectionOptions
|
|
20
20
|
uri = uri.kind_of?(Addressable::URI) ? uri : Addressable::URI::parse(uri.to_s)
|
21
21
|
@https = uri.scheme == "https"
|
22
22
|
uri.port ||= (@https ? 443 : 80)
|
23
|
-
@tls[:sni_hostname] = uri.
|
23
|
+
@tls[:sni_hostname] = uri.hostname
|
24
24
|
|
25
|
-
|
25
|
+
@proxy = options[:proxy] || proxy_from_env
|
26
|
+
|
27
|
+
if proxy
|
26
28
|
@host = proxy[:host]
|
27
29
|
@port = proxy[:port]
|
28
30
|
else
|
29
|
-
@host = uri.
|
31
|
+
@host = uri.hostname
|
30
32
|
@port = uri.port
|
31
33
|
end
|
32
34
|
end
|
@@ -42,4 +44,27 @@ class HttpConnectionOptions
|
|
42
44
|
def socks_proxy?
|
43
45
|
@proxy && (@proxy[:type] == :socks5)
|
44
46
|
end
|
47
|
+
|
48
|
+
def proxy_from_env
|
49
|
+
# TODO: Add support for $http_no_proxy or $no_proxy ?
|
50
|
+
proxy_str = if @https
|
51
|
+
ENV['HTTPS_PROXY'] || ENV['https_proxy']
|
52
|
+
else
|
53
|
+
ENV['HTTP_PROXY'] || ENV['http_proxy']
|
54
|
+
|
55
|
+
# Fall-back to $ALL_PROXY if none of the above env-vars have values
|
56
|
+
end || ENV['ALL_PROXY']
|
57
|
+
|
58
|
+
# Addressable::URI::parse will return `nil` if given `nil` and an empty URL for an empty string
|
59
|
+
# so, let's short-circuit that:
|
60
|
+
return if !proxy_str || proxy_str.empty?
|
61
|
+
|
62
|
+
proxy_env_uri = Addressable::URI::parse(proxy_str)
|
63
|
+
{ :host => proxy_env_uri.host, :port => proxy_env_uri.port, :type => :http }
|
64
|
+
|
65
|
+
rescue Addressable::URI::InvalidURIError
|
66
|
+
# An invalid env-var shouldn't crash the config step, IMHO.
|
67
|
+
# We should somehow log / warn about this, perhaps...
|
68
|
+
return
|
69
|
+
end
|
45
70
|
end
|
data/lib/em-http/version.rb
CHANGED
data/lib/em-http.rb
CHANGED
data/spec/client_spec.rb
CHANGED
@@ -256,6 +256,19 @@ describe EventMachine::HttpRequest do
|
|
256
256
|
}
|
257
257
|
end
|
258
258
|
|
259
|
+
xit "should support expect-continue header" do
|
260
|
+
EventMachine.run {
|
261
|
+
http = EventMachine::HttpRequest.new('http://127.0.0.1:8090').post :body => "data", :head => { 'expect' => '100-continue' }
|
262
|
+
|
263
|
+
http.errback { failed(http) }
|
264
|
+
http.callback {
|
265
|
+
http.response_header.status.should == 200
|
266
|
+
http.response.should == "data"
|
267
|
+
EventMachine.stop
|
268
|
+
}
|
269
|
+
}
|
270
|
+
end
|
271
|
+
|
259
272
|
it "should perform successful GET with custom header" do
|
260
273
|
EventMachine.run {
|
261
274
|
http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/').get :head => {'if-none-match' => 'evar!'}
|
@@ -789,6 +802,28 @@ describe EventMachine::HttpRequest do
|
|
789
802
|
}
|
790
803
|
end
|
791
804
|
|
805
|
+
it "streams POST request from disk via Pathname" do
|
806
|
+
EventMachine.run {
|
807
|
+
http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/').post :body => Pathname.new('spec/fixtures/google.ca')
|
808
|
+
http.errback { failed(http) }
|
809
|
+
http.callback {
|
810
|
+
http.response.should match('google')
|
811
|
+
EventMachine.stop
|
812
|
+
}
|
813
|
+
}
|
814
|
+
end
|
815
|
+
|
816
|
+
it "streams POST request from IO object" do
|
817
|
+
EventMachine.run {
|
818
|
+
http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/').post :body => StringIO.new(File.read('spec/fixtures/google.ca'))
|
819
|
+
http.errback { failed(http) }
|
820
|
+
http.callback {
|
821
|
+
http.response.should match('google')
|
822
|
+
EventMachine.stop
|
823
|
+
}
|
824
|
+
}
|
825
|
+
end
|
826
|
+
|
792
827
|
it "should reconnect if connection was closed between requests" do
|
793
828
|
EventMachine.run {
|
794
829
|
conn = EM::HttpRequest.new('http://127.0.0.1:8090/')
|
@@ -942,4 +977,24 @@ describe EventMachine::HttpRequest do
|
|
942
977
|
}
|
943
978
|
end
|
944
979
|
end
|
980
|
+
|
981
|
+
context "IPv6" do
|
982
|
+
it "should perform successful GET" do
|
983
|
+
EventMachine.run {
|
984
|
+
@s = StubServer.new({
|
985
|
+
response: "HTTP/1.1 200 OK\r\n\r\nHello IPv6",
|
986
|
+
port: 8091,
|
987
|
+
host: '::1',
|
988
|
+
})
|
989
|
+
http = EventMachine::HttpRequest.new('http://[::1]:8091/').get
|
990
|
+
|
991
|
+
http.errback { failed(http) }
|
992
|
+
http.callback {
|
993
|
+
http.response_header.status.should == 200
|
994
|
+
http.response.should match(/Hello IPv6/)
|
995
|
+
EventMachine.stop
|
996
|
+
}
|
997
|
+
}
|
998
|
+
end
|
999
|
+
end
|
945
1000
|
end
|
data/spec/dns_spec.rb
CHANGED
@@ -7,7 +7,7 @@ describe EventMachine::HttpRequest do
|
|
7
7
|
http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/redirect/badhost', :connect_timeout => 0.1).get :redirects => 1
|
8
8
|
http.callback { failed(http) }
|
9
9
|
http.errback {
|
10
|
-
http.error.should match(
|
10
|
+
http.error.should match(/unable to resolve (server |)address/)
|
11
11
|
EventMachine.stop
|
12
12
|
}
|
13
13
|
}
|
@@ -31,7 +31,7 @@ describe EventMachine::HttpRequest do
|
|
31
31
|
http = EventMachine::HttpRequest.new('http://somethinglocal/', :connect_timeout => 0.1).get
|
32
32
|
http.callback { failed(http) }
|
33
33
|
http.errback {
|
34
|
-
http.error.should match(/unable to resolve server address/)
|
34
|
+
http.error.should match(/unable to resolve (server |)address/)
|
35
35
|
http.response_header.status.should == 0
|
36
36
|
EventMachine.stop
|
37
37
|
}
|
data/spec/http_proxy_spec.rb
CHANGED
@@ -1,107 +1,268 @@
|
|
1
1
|
require 'helper'
|
2
2
|
|
3
|
+
shared_examples "*_PROXY var (through proxy)" do
|
4
|
+
it "should use HTTP proxy" do
|
5
|
+
EventMachine.run {
|
6
|
+
http = EventMachine::HttpRequest.new("#{proxy_test_scheme}://127.0.0.1:8090/?q=test").get
|
7
|
+
|
8
|
+
http.errback { failed(http) }
|
9
|
+
http.callback {
|
10
|
+
http.response_header.status.should == 200
|
11
|
+
http.response_header.should_not include("X_PROXY_AUTH")
|
12
|
+
http.response.should match('test')
|
13
|
+
EventMachine.stop
|
14
|
+
}
|
15
|
+
}
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
shared_examples "*_PROXY var (testing var)" do
|
20
|
+
subject { HttpConnectionOptions.new("#{proxy_test_scheme}://example.com", {}) }
|
21
|
+
it { expect(subject.proxy_from_env).to eq({ :host => "127.0.0.1", :port => 8083, :type => :http }) }
|
22
|
+
it { expect(subject.host).to eq "127.0.0.1" }
|
23
|
+
it { expect(subject.port).to be 8083 }
|
24
|
+
it do
|
25
|
+
case proxy_test_scheme.to_sym
|
26
|
+
when :http
|
27
|
+
expect(subject.http_proxy?).to be_truthy
|
28
|
+
when :https
|
29
|
+
expect(subject.connect_proxy?).to be_truthy
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
3
34
|
describe EventMachine::HttpRequest do
|
4
35
|
|
5
36
|
context "connections via" do
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
http.
|
16
|
-
|
17
|
-
|
18
|
-
|
37
|
+
context "without *_PROXY env" do
|
38
|
+
let(:proxy) { {:proxy => { :host => '127.0.0.1', :port => 8083 }} }
|
39
|
+
let(:authenticated_proxy) { {:proxy => { :host => '127.0.0.1', :port => 8083, :authorization => ["user", "name"] } } }
|
40
|
+
|
41
|
+
it "should use HTTP proxy" do
|
42
|
+
EventMachine.run {
|
43
|
+
http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/?q=test', proxy).get
|
44
|
+
|
45
|
+
http.errback { failed(http) }
|
46
|
+
http.callback {
|
47
|
+
http.response_header.status.should == 200
|
48
|
+
http.response_header.should_not include("X_PROXY_AUTH")
|
49
|
+
http.response.should match('test')
|
50
|
+
EventMachine.stop
|
51
|
+
}
|
19
52
|
}
|
20
|
-
|
21
|
-
end
|
53
|
+
end
|
22
54
|
|
23
|
-
|
24
|
-
|
25
|
-
|
55
|
+
it "should use HTTP proxy with authentication" do
|
56
|
+
EventMachine.run {
|
57
|
+
http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/proxyauth?q=test', authenticated_proxy).get
|
26
58
|
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
59
|
+
http.errback { failed(http) }
|
60
|
+
http.callback {
|
61
|
+
http.response_header.status.should == 200
|
62
|
+
http.response_header['X_PROXY_AUTH'].should == "Proxy-Authorization: Basic dXNlcjpuYW1l"
|
63
|
+
http.response.should match('test')
|
64
|
+
EventMachine.stop
|
65
|
+
}
|
33
66
|
}
|
34
|
-
|
35
|
-
end
|
67
|
+
end
|
36
68
|
|
37
|
-
|
38
|
-
|
69
|
+
it "should send absolute URIs to the proxy server" do
|
70
|
+
EventMachine.run {
|
39
71
|
|
40
|
-
|
72
|
+
http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/?q=test', proxy).get
|
41
73
|
|
42
|
-
|
43
|
-
|
44
|
-
|
74
|
+
http.errback { failed(http) }
|
75
|
+
http.callback {
|
76
|
+
http.response_header.status.should == 200
|
45
77
|
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
78
|
+
# The test proxy server gives the requested uri back in this header
|
79
|
+
http.response_header['X_THE_REQUESTED_URI'].should == 'http://127.0.0.1:8090/?q=test'
|
80
|
+
http.response_header['X_THE_REQUESTED_URI'].should_not == '/?q=test'
|
81
|
+
http.response.should match('test')
|
82
|
+
EventMachine.stop
|
83
|
+
}
|
51
84
|
}
|
52
|
-
|
53
|
-
end
|
85
|
+
end
|
54
86
|
|
55
|
-
|
56
|
-
|
87
|
+
it "should strip basic auth from before the host in URI sent to proxy" do
|
88
|
+
EventMachine.run {
|
57
89
|
|
58
|
-
|
90
|
+
http = EventMachine::HttpRequest.new('http://user:pass@127.0.0.1:8090/echo_authorization_header', proxy).get
|
59
91
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
92
|
+
http.errback { failed(http) }
|
93
|
+
http.callback {
|
94
|
+
http.response_header.status.should == 200
|
95
|
+
# The test proxy server gives the requested uri back in this header
|
96
|
+
http.response_header['X_THE_REQUESTED_URI'].should == 'http://127.0.0.1:8090/echo_authorization_header'
|
97
|
+
# Ensure the basic auth was converted to a header correctly
|
98
|
+
http.response.should match('authorization:Basic dXNlcjpwYXNz')
|
99
|
+
EventMachine.stop
|
100
|
+
}
|
68
101
|
}
|
69
|
-
|
70
|
-
end
|
102
|
+
end
|
71
103
|
|
72
|
-
|
73
|
-
|
74
|
-
|
104
|
+
it "should include query parameters specified in the options" do
|
105
|
+
EventMachine.run {
|
106
|
+
http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/', proxy).get :query => { 'q' => 'test' }
|
75
107
|
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
108
|
+
http.errback { failed(http) }
|
109
|
+
http.callback {
|
110
|
+
http.response_header.status.should == 200
|
111
|
+
http.response.should match('test')
|
112
|
+
EventMachine.stop
|
113
|
+
}
|
81
114
|
}
|
82
|
-
|
83
|
-
end
|
115
|
+
end
|
84
116
|
|
85
|
-
|
86
|
-
|
87
|
-
|
117
|
+
it "should use HTTP proxy while redirecting" do
|
118
|
+
EventMachine.run {
|
119
|
+
http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/redirect', proxy).get :redirects => 1
|
88
120
|
|
89
|
-
|
90
|
-
|
91
|
-
|
121
|
+
http.errback { failed(http) }
|
122
|
+
http.callback {
|
123
|
+
http.response_header.status.should == 200
|
92
124
|
|
93
|
-
|
94
|
-
|
125
|
+
http.response_header['X_THE_REQUESTED_URI'].should == 'http://127.0.0.1:8090/gzip'
|
126
|
+
http.response_header['X_THE_REQUESTED_URI'].should_not == '/redirect'
|
95
127
|
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
128
|
+
http.response_header["CONTENT_ENCODING"].should == "gzip"
|
129
|
+
http.response.should == "compressed"
|
130
|
+
http.last_effective_url.to_s.should == 'http://127.0.0.1:8090/gzip'
|
131
|
+
http.redirects.should == 1
|
100
132
|
|
101
|
-
|
133
|
+
EventMachine.stop
|
134
|
+
}
|
102
135
|
}
|
103
|
-
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
139
|
+
context "when parsing *_PROXY var (through proxy)s" do
|
140
|
+
context "with $HTTP_PROXY env" do
|
141
|
+
let(:proxy_test_scheme) { :http }
|
142
|
+
|
143
|
+
before(:all) do
|
144
|
+
PROXY_ENV_VARS.each {|k| ENV.delete k }
|
145
|
+
ENV['HTTP_PROXY'] = 'http://127.0.0.1:8083'
|
146
|
+
end
|
147
|
+
|
148
|
+
include_examples "*_PROXY var (through proxy)"
|
149
|
+
end
|
150
|
+
|
151
|
+
context "with $http_proxy env" do
|
152
|
+
let(:proxy_test_scheme) { :http }
|
153
|
+
|
154
|
+
before(:all) do
|
155
|
+
PROXY_ENV_VARS.each {|k| ENV.delete k }
|
156
|
+
ENV['http_proxy'] = 'http://127.0.0.1:8083'
|
157
|
+
end
|
158
|
+
|
159
|
+
include_examples "*_PROXY var (through proxy)"
|
160
|
+
end
|
161
|
+
|
162
|
+
## TODO: Use a Mongrel HTTP server that can handle SSL:
|
163
|
+
context "with $HTTPS_PROXY env", skip: "Mongrel isn't configured to handle HTTPS, currently" do
|
164
|
+
let(:proxy_test_scheme) { :https }
|
165
|
+
|
166
|
+
before(:all) do
|
167
|
+
PROXY_ENV_VARS.each {|k| ENV.delete k }
|
168
|
+
ENV['HTTPS_PROXY'] = 'http://127.0.0.1:8083'
|
169
|
+
end
|
170
|
+
|
171
|
+
include_examples "*_PROXY var (through proxy)"
|
172
|
+
end
|
173
|
+
|
174
|
+
## TODO: Use a Mongrel HTTP server that can handle SSL:
|
175
|
+
context "with $https_proxy env", skip: "Mongrel isn't configured to handle HTTPS, currently" do
|
176
|
+
let(:proxy_test_scheme) { :https }
|
177
|
+
|
178
|
+
before(:all) do
|
179
|
+
PROXY_ENV_VARS.each {|k| ENV.delete k }
|
180
|
+
ENV['https_proxy'] = 'http://127.0.0.1:8083'
|
181
|
+
end
|
182
|
+
|
183
|
+
include_examples "*_PROXY var (through proxy)"
|
184
|
+
end
|
185
|
+
|
186
|
+
context "with $ALL_PROXY env" do
|
187
|
+
let(:proxy_test_scheme) { :http }
|
188
|
+
|
189
|
+
before(:all) do
|
190
|
+
PROXY_ENV_VARS.each {|k| ENV.delete k }
|
191
|
+
ENV['ALL_PROXY'] = 'http://127.0.0.1:8083'
|
192
|
+
end
|
193
|
+
|
194
|
+
include_examples "*_PROXY var (through proxy)"
|
195
|
+
end
|
104
196
|
end
|
105
197
|
end
|
106
198
|
|
199
|
+
context "when parsing *_PROXY vars" do
|
200
|
+
context "without a *_PROXY var" do
|
201
|
+
before(:all) do
|
202
|
+
PROXY_ENV_VARS.each {|k| ENV.delete k }
|
203
|
+
end
|
204
|
+
|
205
|
+
subject { HttpConnectionOptions.new("http://example.com", {}) }
|
206
|
+
it { expect(subject.proxy_from_env).to be_nil }
|
207
|
+
it { expect(subject.host).to eq "example.com" }
|
208
|
+
it { expect(subject.port).to be 80 }
|
209
|
+
it { expect(subject.http_proxy?).to be_falsey }
|
210
|
+
it { expect(subject.connect_proxy?).to be_falsey }
|
211
|
+
end
|
212
|
+
|
213
|
+
context "with $HTTP_PROXY env" do
|
214
|
+
let(:proxy_test_scheme) { :http }
|
215
|
+
|
216
|
+
before(:each) do
|
217
|
+
PROXY_ENV_VARS.each {|k| ENV.delete k }
|
218
|
+
ENV['HTTP_PROXY'] = 'http://127.0.0.1:8083'
|
219
|
+
end
|
220
|
+
|
221
|
+
include_examples "*_PROXY var (testing var)"
|
222
|
+
end
|
223
|
+
|
224
|
+
context "with $http_proxy env" do
|
225
|
+
let(:proxy_test_scheme) { :http }
|
226
|
+
|
227
|
+
before(:each) do
|
228
|
+
PROXY_ENV_VARS.each {|k| ENV.delete k }
|
229
|
+
ENV['http_proxy'] = 'http://127.0.0.1:8083'
|
230
|
+
end
|
231
|
+
|
232
|
+
include_examples "*_PROXY var (testing var)"
|
233
|
+
end
|
234
|
+
|
235
|
+
context "with $HTTPS_PROXY env" do
|
236
|
+
let(:proxy_test_scheme) { :https }
|
237
|
+
|
238
|
+
before(:each) do
|
239
|
+
PROXY_ENV_VARS.each {|k| ENV.delete k }
|
240
|
+
ENV['HTTPS_PROXY'] = 'http://127.0.0.1:8083'
|
241
|
+
end
|
242
|
+
|
243
|
+
include_examples "*_PROXY var (testing var)"
|
244
|
+
end
|
245
|
+
|
246
|
+
context "with $https_proxy env" do
|
247
|
+
let(:proxy_test_scheme) { :https }
|
248
|
+
|
249
|
+
before(:each) do
|
250
|
+
PROXY_ENV_VARS.each {|k| ENV.delete k }
|
251
|
+
ENV['https_proxy'] = 'http://127.0.0.1:8083'
|
252
|
+
end
|
253
|
+
|
254
|
+
include_examples "*_PROXY var (testing var)"
|
255
|
+
end
|
256
|
+
|
257
|
+
context "with $ALL_PROXY env" do
|
258
|
+
let(:proxy_test_scheme) { :https }
|
259
|
+
|
260
|
+
before(:each) do
|
261
|
+
PROXY_ENV_VARS.each {|k| ENV.delete k }
|
262
|
+
ENV['ALL_PROXY'] = 'http://127.0.0.1:8083'
|
263
|
+
end
|
264
|
+
|
265
|
+
include_examples "*_PROXY var (testing var)"
|
266
|
+
end
|
267
|
+
end
|
107
268
|
end
|
data/spec/spec_helper.rb
CHANGED
@@ -1,8 +1,25 @@
|
|
1
|
+
PROXY_ENV_VARS = %w[HTTP_PROXY http_proxy HTTPS_PROXY https_proxy ALL_PROXY]
|
2
|
+
|
1
3
|
RSpec.configure do |config|
|
4
|
+
proxy_envs = {}
|
5
|
+
|
2
6
|
config.mock_with :rspec do |c|
|
3
7
|
c.syntax = [:should, :expect]
|
4
8
|
end
|
5
9
|
config.expect_with :rspec do |c|
|
6
10
|
c.syntax = [:should, :expect]
|
7
11
|
end
|
8
|
-
|
12
|
+
|
13
|
+
config.before :all do
|
14
|
+
# Back-up ENV *_PROXY vars
|
15
|
+
orig_proxy_envs = Hash[
|
16
|
+
PROXY_ENV_VARS.select {|k| ENV.key? k }.map {|k| [k, ENV.delete(k)] }
|
17
|
+
]
|
18
|
+
proxy_envs.replace(orig_proxy_envs)
|
19
|
+
end
|
20
|
+
|
21
|
+
config.after :all do
|
22
|
+
# Restore ENV *_PROXY vars
|
23
|
+
ENV.update(proxy_envs)
|
24
|
+
end
|
25
|
+
end
|
data/spec/ssl_spec.rb
CHANGED
@@ -3,7 +3,6 @@ require 'helper'
|
|
3
3
|
requires_connection do
|
4
4
|
|
5
5
|
describe EventMachine::HttpRequest do
|
6
|
-
|
7
6
|
it "should initiate SSL/TLS on HTTPS connections" do
|
8
7
|
EventMachine.run {
|
9
8
|
http = EventMachine::HttpRequest.new('https://mail.google.com:443/mail/').get
|
@@ -15,6 +14,58 @@ requires_connection do
|
|
15
14
|
}
|
16
15
|
}
|
17
16
|
end
|
17
|
+
|
18
|
+
describe "TLS hostname verification" do
|
19
|
+
before do
|
20
|
+
@cve_warning = "[WARNING; em-http-request] TLS hostname validation is disabled (use 'tls: {verify_peer: true}'), see" +
|
21
|
+
" CVE-2020-13482 and https://github.com/igrigorik/em-http-request/issues/339 for details"
|
22
|
+
@orig_stderr = $stderr
|
23
|
+
$stderr = StringIO.new
|
24
|
+
end
|
25
|
+
|
26
|
+
after do
|
27
|
+
$stderr = @orig_stderr
|
28
|
+
end
|
29
|
+
|
30
|
+
it "should not warn if verify_peer is specified" do
|
31
|
+
EventMachine.run {
|
32
|
+
http = EventMachine::HttpRequest.new('https://mail.google.com:443/mail', {tls: {verify_peer: false}}).get
|
33
|
+
|
34
|
+
http.callback {
|
35
|
+
$stderr.rewind
|
36
|
+
$stderr.string.chomp.should_not eq(@cve_warning)
|
37
|
+
|
38
|
+
EventMachine.stop
|
39
|
+
}
|
40
|
+
}
|
41
|
+
end
|
42
|
+
|
43
|
+
it "should not warn if verify_peer is true" do
|
44
|
+
EventMachine.run {
|
45
|
+
http = EventMachine::HttpRequest.new('https://mail.google.com:443/mail', {tls: {verify_peer: true}}).get
|
46
|
+
|
47
|
+
http.callback {
|
48
|
+
$stderr.rewind
|
49
|
+
$stderr.string.chomp.should_not eq(@cve_warning)
|
50
|
+
|
51
|
+
EventMachine.stop
|
52
|
+
}
|
53
|
+
}
|
54
|
+
end
|
55
|
+
|
56
|
+
it "should warn if verify_peer is unspecified" do
|
57
|
+
EventMachine.run {
|
58
|
+
http = EventMachine::HttpRequest.new('https://mail.google.com:443/mail').get
|
59
|
+
|
60
|
+
http.callback {
|
61
|
+
$stderr.rewind
|
62
|
+
$stderr.string.chomp.should eq(@cve_warning)
|
63
|
+
|
64
|
+
EventMachine.stop
|
65
|
+
}
|
66
|
+
}
|
67
|
+
end
|
68
|
+
end
|
18
69
|
end
|
19
70
|
|
20
71
|
end
|
data/spec/stallion.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: em-http-request
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.1.
|
4
|
+
version: 1.1.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ilya Grigorik
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-08-31 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: addressable
|
@@ -112,16 +112,16 @@ dependencies:
|
|
112
112
|
name: rack
|
113
113
|
requirement: !ruby/object:Gem::Requirement
|
114
114
|
requirements:
|
115
|
-
- - "
|
115
|
+
- - "<"
|
116
116
|
- !ruby/object:Gem::Version
|
117
|
-
version: '0'
|
117
|
+
version: '2.0'
|
118
118
|
type: :development
|
119
119
|
prerelease: false
|
120
120
|
version_requirements: !ruby/object:Gem::Requirement
|
121
121
|
requirements:
|
122
|
-
- - "
|
122
|
+
- - "<"
|
123
123
|
- !ruby/object:Gem::Version
|
124
|
-
version: '0'
|
124
|
+
version: '2.0'
|
125
125
|
- !ruby/object:Gem::Dependency
|
126
126
|
name: rake
|
127
127
|
requirement: !ruby/object:Gem::Requirement
|
@@ -197,6 +197,7 @@ files:
|
|
197
197
|
- lib/em-http/multi.rb
|
198
198
|
- lib/em-http/request.rb
|
199
199
|
- lib/em-http/version.rb
|
200
|
+
- lib/em/io_streamer.rb
|
200
201
|
- spec/client_fiber_spec.rb
|
201
202
|
- spec/client_spec.rb
|
202
203
|
- spec/digest_auth_spec.rb
|
@@ -222,7 +223,7 @@ homepage: http://github.com/igrigorik/em-http-request
|
|
222
223
|
licenses:
|
223
224
|
- MIT
|
224
225
|
metadata: {}
|
225
|
-
post_install_message:
|
226
|
+
post_install_message:
|
226
227
|
rdoc_options: []
|
227
228
|
require_paths:
|
228
229
|
- lib
|
@@ -237,9 +238,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
237
238
|
- !ruby/object:Gem::Version
|
238
239
|
version: '0'
|
239
240
|
requirements: []
|
240
|
-
|
241
|
-
|
242
|
-
signing_key:
|
241
|
+
rubygems_version: 3.0.3
|
242
|
+
signing_key:
|
243
243
|
specification_version: 4
|
244
244
|
summary: EventMachine based, async HTTP Request client
|
245
245
|
test_files:
|
@@ -264,4 +264,3 @@ test_files:
|
|
264
264
|
- spec/ssl_spec.rb
|
265
265
|
- spec/stallion.rb
|
266
266
|
- spec/stub_server.rb
|
267
|
-
has_rdoc:
|