em-http-request 1.1.5 → 1.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/README.md +1 -1
- data/em-http-request.gemspec +1 -1
- data/lib/em/io_streamer.rb +49 -0
- data/lib/em-http/client.rb +25 -6
- data/lib/em-http/http_client_options.rb +1 -1
- data/lib/em-http/http_connection.rb +77 -5
- data/lib/em-http/http_connection_options.rb +29 -4
- data/lib/em-http/version.rb +1 -1
- data/lib/em-http.rb +1 -0
- data/spec/client_spec.rb +55 -0
- data/spec/dns_spec.rb +2 -2
- data/spec/http_proxy_spec.rb +237 -76
- data/spec/spec_helper.rb +18 -1
- data/spec/ssl_spec.rb +52 -1
- data/spec/stallion.rb +1 -1
- metadata +11 -12
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
2
|
+
SHA256:
|
3
|
+
metadata.gz: c7562e4d20c35c54a9319250e665a883c810546cb657d8f897591e113999ed3a
|
4
|
+
data.tar.gz: 643bf26ea7bfa2a85d6e4257a475295c16eca044ff0d04341537793f07d5bd04
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 9d9d1f441081a034f29447cb99b26c73ef7767a989c31df007e1ecfc6ea8db1f4cbe00fb26c0f81b59108f53b54dfbccd51ea9140fd5286efaca6095b45943ad
|
7
|
+
data.tar.gz: 690dc944373085313c41c484edd25366036a3da46855b6a153aab65c19aed961d94d202de07cb3f5f8406585bb5c3a66998913debcba4d02796fb1c8f04be7c1
|
data/README.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# EM-HTTP-Request
|
2
2
|
|
3
|
-
[](http://rubygems.org/gems/em-http-request) [](https://travis-ci.org/igrigorik/em-http-request)
|
4
4
|
|
5
5
|
Async (EventMachine) HTTP client, with support for:
|
6
6
|
|
data/em-http-request.gemspec
CHANGED
@@ -23,7 +23,7 @@ Gem::Specification.new do |s|
|
|
23
23
|
|
24
24
|
s.add_development_dependency 'mongrel', '~> 1.2.0.pre2'
|
25
25
|
s.add_development_dependency 'multi_json'
|
26
|
-
s.add_development_dependency 'rack'
|
26
|
+
s.add_development_dependency 'rack', '< 2.0'
|
27
27
|
s.add_development_dependency 'rake'
|
28
28
|
s.add_development_dependency 'rspec'
|
29
29
|
|
@@ -0,0 +1,49 @@
|
|
1
|
+
require 'em/streamer'
|
2
|
+
|
3
|
+
# similar to EventMachine::FileStreamer, but for any IO object
|
4
|
+
module EventMachine
|
5
|
+
class IOStreamer
|
6
|
+
include Deferrable
|
7
|
+
CHUNK_SIZE = 16384
|
8
|
+
|
9
|
+
# @param [EventMachine::Connection] connection
|
10
|
+
# @param [IO] io Data source
|
11
|
+
# @param [Integer] Data size
|
12
|
+
#
|
13
|
+
# @option opts [Boolean] :http_chunks (false) Use HTTP 1.1 style chunked-encoding semantics.
|
14
|
+
def initialize(connection, io, opts = {})
|
15
|
+
@connection = connection
|
16
|
+
@io = io
|
17
|
+
@http_chunks = opts[:http_chunks]
|
18
|
+
|
19
|
+
@buff = String.new
|
20
|
+
@io.binmode if @io.respond_to?(:binmode)
|
21
|
+
stream_one_chunk
|
22
|
+
end
|
23
|
+
|
24
|
+
private
|
25
|
+
|
26
|
+
# Used internally to stream one chunk at a time over multiple reactor ticks
|
27
|
+
# @private
|
28
|
+
def stream_one_chunk
|
29
|
+
loop do
|
30
|
+
if @io.eof?
|
31
|
+
@connection.send_data "0\r\n\r\n" if @http_chunks
|
32
|
+
succeed
|
33
|
+
break
|
34
|
+
end
|
35
|
+
|
36
|
+
if @connection.respond_to?(:get_outbound_data_size) && (@connection.get_outbound_data_size > FileStreamer::BackpressureLevel)
|
37
|
+
EventMachine::next_tick { stream_one_chunk }
|
38
|
+
break
|
39
|
+
end
|
40
|
+
|
41
|
+
if @io.read(CHUNK_SIZE, @buff)
|
42
|
+
@connection.send_data("#{@buff.length.to_s(16)}\r\n") if @http_chunks
|
43
|
+
@connection.send_data(@buff)
|
44
|
+
@connection.send_data("\r\n") if @http_chunks
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
data/lib/em-http/client.rb
CHANGED
@@ -182,7 +182,7 @@ module EventMachine
|
|
182
182
|
# Set the Content-Length if body is given,
|
183
183
|
# or we're doing an empty post or put
|
184
184
|
if body
|
185
|
-
head['content-length']
|
185
|
+
head['content-length'] ||= body.respond_to?(:bytesize) ? body.bytesize : body.size
|
186
186
|
elsif @req.method == 'POST' or @req.method == 'PUT'
|
187
187
|
# wont happen if body is set and we already set content-length above
|
188
188
|
head['content-length'] ||= 0
|
@@ -198,11 +198,8 @@ module EventMachine
|
|
198
198
|
request_header << CRLF
|
199
199
|
@conn.send_data request_header
|
200
200
|
|
201
|
-
|
202
|
-
|
203
|
-
elsif @req.file
|
204
|
-
@conn.stream_file_data @req.file, :http_chunks => false
|
205
|
-
end
|
201
|
+
@req_body = body || (@req.file && Pathname.new(@req.file))
|
202
|
+
send_request_body unless @req.headers['expect'] == '100-continue'
|
206
203
|
end
|
207
204
|
|
208
205
|
def on_body_data(data)
|
@@ -226,6 +223,28 @@ module EventMachine
|
|
226
223
|
end
|
227
224
|
end
|
228
225
|
|
226
|
+
def request_body_pending?
|
227
|
+
!!@req_body
|
228
|
+
end
|
229
|
+
|
230
|
+
def send_request_body
|
231
|
+
return if @req_body.nil?
|
232
|
+
|
233
|
+
if @req_body.is_a?(String)
|
234
|
+
@conn.send_data @req_body
|
235
|
+
|
236
|
+
elsif @req_body.is_a?(Pathname)
|
237
|
+
@conn.stream_file_data @req_body.to_path, http_chunks: false
|
238
|
+
|
239
|
+
elsif @req_body.respond_to?(:read) && @req_body.respond_to?(:eof?) # IO or IO-like object
|
240
|
+
@conn.stream_data @req_body
|
241
|
+
|
242
|
+
else
|
243
|
+
raise "Don't know how to send request body: #{@req_body.inspect}"
|
244
|
+
end
|
245
|
+
@req_body = nil
|
246
|
+
end
|
247
|
+
|
229
248
|
def parse_response_header(header, version, status)
|
230
249
|
@response_header.raw = header
|
231
250
|
header.each do |key, val|
|
@@ -1,3 +1,5 @@
|
|
1
|
+
require 'em/io_streamer'
|
2
|
+
|
1
3
|
module EventMachine
|
2
4
|
|
3
5
|
module HTTPMethods
|
@@ -20,7 +22,11 @@ module EventMachine
|
|
20
22
|
end
|
21
23
|
|
22
24
|
def receive_data(data)
|
23
|
-
|
25
|
+
begin
|
26
|
+
@parent.receive_data data
|
27
|
+
rescue EventMachine::Connectify::CONNECTError => e
|
28
|
+
@parent.close(e.message)
|
29
|
+
end
|
24
30
|
end
|
25
31
|
|
26
32
|
def connection_completed
|
@@ -30,6 +36,62 @@ module EventMachine
|
|
30
36
|
def unbind(reason=nil)
|
31
37
|
@parent.unbind(reason)
|
32
38
|
end
|
39
|
+
|
40
|
+
# TLS verification support, original implementation by Mislav Marohnić
|
41
|
+
# https://github.com/lostisland/faraday/blob/63cf47c95b573539f047c729bd9ad67560bc83ff/lib/faraday/adapter/em_http_ssl_patch.rb
|
42
|
+
def ssl_verify_peer(cert_string)
|
43
|
+
cert = nil
|
44
|
+
begin
|
45
|
+
cert = OpenSSL::X509::Certificate.new(cert_string)
|
46
|
+
rescue OpenSSL::X509::CertificateError
|
47
|
+
return false
|
48
|
+
end
|
49
|
+
|
50
|
+
@last_seen_cert = cert
|
51
|
+
|
52
|
+
if certificate_store.verify(@last_seen_cert)
|
53
|
+
begin
|
54
|
+
certificate_store.add_cert(@last_seen_cert)
|
55
|
+
rescue OpenSSL::X509::StoreError => e
|
56
|
+
raise e unless e.message == 'cert already in hash table'
|
57
|
+
end
|
58
|
+
true
|
59
|
+
else
|
60
|
+
raise OpenSSL::SSL::SSLError.new(%(unable to verify the server certificate for "#{host}"))
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
def ssl_handshake_completed
|
65
|
+
unless verify_peer?
|
66
|
+
warn "[WARNING; em-http-request] TLS hostname validation is disabled (use 'tls: {verify_peer: true}'), see" +
|
67
|
+
" CVE-2020-13482 and https://github.com/igrigorik/em-http-request/issues/339 for details" unless parent.connopts.tls.has_key?(:verify_peer)
|
68
|
+
return true
|
69
|
+
end
|
70
|
+
|
71
|
+
unless OpenSSL::SSL.verify_certificate_identity(@last_seen_cert, host)
|
72
|
+
raise OpenSSL::SSL::SSLError.new(%(host "#{host}" does not match the server certificate))
|
73
|
+
else
|
74
|
+
true
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
def verify_peer?
|
79
|
+
parent.connopts.tls[:verify_peer]
|
80
|
+
end
|
81
|
+
|
82
|
+
def host
|
83
|
+
parent.connopts.host
|
84
|
+
end
|
85
|
+
|
86
|
+
def certificate_store
|
87
|
+
@certificate_store ||= begin
|
88
|
+
store = OpenSSL::X509::Store.new
|
89
|
+
store.set_default_paths
|
90
|
+
ca_file = parent.connopts.tls[:cert_chain_file]
|
91
|
+
store.add_file(ca_file) if ca_file
|
92
|
+
store
|
93
|
+
end
|
94
|
+
end
|
33
95
|
end
|
34
96
|
|
35
97
|
class HttpConnection
|
@@ -115,8 +177,13 @@ module EventMachine
|
|
115
177
|
@p.header_value_type = :mixed
|
116
178
|
@p.on_headers_complete = proc do |h|
|
117
179
|
if client
|
118
|
-
|
119
|
-
|
180
|
+
if @p.status_code == 100
|
181
|
+
client.send_request_body
|
182
|
+
@p.reset!
|
183
|
+
else
|
184
|
+
client.parse_response_header(h, @p.http_version, @p.status_code)
|
185
|
+
:reset if client.req.no_body?
|
186
|
+
end
|
120
187
|
else
|
121
188
|
# if we receive unexpected data without a pending client request
|
122
189
|
# reset the parser to avoid firing any further callbacks and close
|
@@ -160,9 +227,9 @@ module EventMachine
|
|
160
227
|
@peer = @conn.get_peername
|
161
228
|
|
162
229
|
if @connopts.socks_proxy?
|
163
|
-
socksify(client.req.uri.
|
230
|
+
socksify(client.req.uri.hostname, client.req.uri.inferred_port, *@connopts.proxy[:authorization]) { start }
|
164
231
|
elsif @connopts.connect_proxy?
|
165
|
-
connectify(client.req.uri.
|
232
|
+
connectify(client.req.uri.hostname, client.req.uri.inferred_port, *@connopts.proxy[:authorization]) { start }
|
166
233
|
else
|
167
234
|
start
|
168
235
|
end
|
@@ -185,6 +252,7 @@ module EventMachine
|
|
185
252
|
conn = HttpConnection.new
|
186
253
|
client.conn = conn
|
187
254
|
conn.connopts = @connopts
|
255
|
+
conn.connopts.https = new_location.scheme == "https"
|
188
256
|
conn.uri = client.req.uri
|
189
257
|
conn.activate_connection(client)
|
190
258
|
|
@@ -240,6 +308,10 @@ module EventMachine
|
|
240
308
|
@conn.stream_file_data filename, args
|
241
309
|
end
|
242
310
|
|
311
|
+
def stream_data(io, opts = {})
|
312
|
+
EventMachine::IOStreamer.new(self, io, opts)
|
313
|
+
end
|
314
|
+
|
243
315
|
private
|
244
316
|
|
245
317
|
def client
|
@@ -1,13 +1,13 @@
|
|
1
1
|
class HttpConnectionOptions
|
2
2
|
attr_reader :host, :port, :tls, :proxy, :bind, :bind_port
|
3
3
|
attr_reader :connect_timeout, :inactivity_timeout
|
4
|
+
attr_writer :https
|
4
5
|
|
5
6
|
def initialize(uri, options)
|
6
7
|
@connect_timeout = options[:connect_timeout] || 5 # default connection setup timeout
|
7
8
|
@inactivity_timeout = options[:inactivity_timeout] ||= 10 # default connection inactivity (post-setup) timeout
|
8
9
|
|
9
10
|
@tls = options[:tls] || options[:ssl] || {}
|
10
|
-
@proxy = options[:proxy]
|
11
11
|
|
12
12
|
if bind = options[:bind]
|
13
13
|
@bind = bind[:host] || '0.0.0.0'
|
@@ -20,13 +20,15 @@ class HttpConnectionOptions
|
|
20
20
|
uri = uri.kind_of?(Addressable::URI) ? uri : Addressable::URI::parse(uri.to_s)
|
21
21
|
@https = uri.scheme == "https"
|
22
22
|
uri.port ||= (@https ? 443 : 80)
|
23
|
-
@tls[:sni_hostname] = uri.
|
23
|
+
@tls[:sni_hostname] = uri.hostname
|
24
24
|
|
25
|
-
|
25
|
+
@proxy = options[:proxy] || proxy_from_env
|
26
|
+
|
27
|
+
if proxy
|
26
28
|
@host = proxy[:host]
|
27
29
|
@port = proxy[:port]
|
28
30
|
else
|
29
|
-
@host = uri.
|
31
|
+
@host = uri.hostname
|
30
32
|
@port = uri.port
|
31
33
|
end
|
32
34
|
end
|
@@ -42,4 +44,27 @@ class HttpConnectionOptions
|
|
42
44
|
def socks_proxy?
|
43
45
|
@proxy && (@proxy[:type] == :socks5)
|
44
46
|
end
|
47
|
+
|
48
|
+
def proxy_from_env
|
49
|
+
# TODO: Add support for $http_no_proxy or $no_proxy ?
|
50
|
+
proxy_str = if @https
|
51
|
+
ENV['HTTPS_PROXY'] || ENV['https_proxy']
|
52
|
+
else
|
53
|
+
ENV['HTTP_PROXY'] || ENV['http_proxy']
|
54
|
+
|
55
|
+
# Fall-back to $ALL_PROXY if none of the above env-vars have values
|
56
|
+
end || ENV['ALL_PROXY']
|
57
|
+
|
58
|
+
# Addressable::URI::parse will return `nil` if given `nil` and an empty URL for an empty string
|
59
|
+
# so, let's short-circuit that:
|
60
|
+
return if !proxy_str || proxy_str.empty?
|
61
|
+
|
62
|
+
proxy_env_uri = Addressable::URI::parse(proxy_str)
|
63
|
+
{ :host => proxy_env_uri.host, :port => proxy_env_uri.port, :type => :http }
|
64
|
+
|
65
|
+
rescue Addressable::URI::InvalidURIError
|
66
|
+
# An invalid env-var shouldn't crash the config step, IMHO.
|
67
|
+
# We should somehow log / warn about this, perhaps...
|
68
|
+
return
|
69
|
+
end
|
45
70
|
end
|
data/lib/em-http/version.rb
CHANGED
data/lib/em-http.rb
CHANGED
data/spec/client_spec.rb
CHANGED
@@ -256,6 +256,19 @@ describe EventMachine::HttpRequest do
|
|
256
256
|
}
|
257
257
|
end
|
258
258
|
|
259
|
+
xit "should support expect-continue header" do
|
260
|
+
EventMachine.run {
|
261
|
+
http = EventMachine::HttpRequest.new('http://127.0.0.1:8090').post :body => "data", :head => { 'expect' => '100-continue' }
|
262
|
+
|
263
|
+
http.errback { failed(http) }
|
264
|
+
http.callback {
|
265
|
+
http.response_header.status.should == 200
|
266
|
+
http.response.should == "data"
|
267
|
+
EventMachine.stop
|
268
|
+
}
|
269
|
+
}
|
270
|
+
end
|
271
|
+
|
259
272
|
it "should perform successful GET with custom header" do
|
260
273
|
EventMachine.run {
|
261
274
|
http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/').get :head => {'if-none-match' => 'evar!'}
|
@@ -789,6 +802,28 @@ describe EventMachine::HttpRequest do
|
|
789
802
|
}
|
790
803
|
end
|
791
804
|
|
805
|
+
it "streams POST request from disk via Pathname" do
|
806
|
+
EventMachine.run {
|
807
|
+
http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/').post :body => Pathname.new('spec/fixtures/google.ca')
|
808
|
+
http.errback { failed(http) }
|
809
|
+
http.callback {
|
810
|
+
http.response.should match('google')
|
811
|
+
EventMachine.stop
|
812
|
+
}
|
813
|
+
}
|
814
|
+
end
|
815
|
+
|
816
|
+
it "streams POST request from IO object" do
|
817
|
+
EventMachine.run {
|
818
|
+
http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/').post :body => StringIO.new(File.read('spec/fixtures/google.ca'))
|
819
|
+
http.errback { failed(http) }
|
820
|
+
http.callback {
|
821
|
+
http.response.should match('google')
|
822
|
+
EventMachine.stop
|
823
|
+
}
|
824
|
+
}
|
825
|
+
end
|
826
|
+
|
792
827
|
it "should reconnect if connection was closed between requests" do
|
793
828
|
EventMachine.run {
|
794
829
|
conn = EM::HttpRequest.new('http://127.0.0.1:8090/')
|
@@ -942,4 +977,24 @@ describe EventMachine::HttpRequest do
|
|
942
977
|
}
|
943
978
|
end
|
944
979
|
end
|
980
|
+
|
981
|
+
context "IPv6" do
|
982
|
+
it "should perform successful GET" do
|
983
|
+
EventMachine.run {
|
984
|
+
@s = StubServer.new({
|
985
|
+
response: "HTTP/1.1 200 OK\r\n\r\nHello IPv6",
|
986
|
+
port: 8091,
|
987
|
+
host: '::1',
|
988
|
+
})
|
989
|
+
http = EventMachine::HttpRequest.new('http://[::1]:8091/').get
|
990
|
+
|
991
|
+
http.errback { failed(http) }
|
992
|
+
http.callback {
|
993
|
+
http.response_header.status.should == 200
|
994
|
+
http.response.should match(/Hello IPv6/)
|
995
|
+
EventMachine.stop
|
996
|
+
}
|
997
|
+
}
|
998
|
+
end
|
999
|
+
end
|
945
1000
|
end
|
data/spec/dns_spec.rb
CHANGED
@@ -7,7 +7,7 @@ describe EventMachine::HttpRequest do
|
|
7
7
|
http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/redirect/badhost', :connect_timeout => 0.1).get :redirects => 1
|
8
8
|
http.callback { failed(http) }
|
9
9
|
http.errback {
|
10
|
-
http.error.should match(
|
10
|
+
http.error.should match(/unable to resolve (server |)address/)
|
11
11
|
EventMachine.stop
|
12
12
|
}
|
13
13
|
}
|
@@ -31,7 +31,7 @@ describe EventMachine::HttpRequest do
|
|
31
31
|
http = EventMachine::HttpRequest.new('http://somethinglocal/', :connect_timeout => 0.1).get
|
32
32
|
http.callback { failed(http) }
|
33
33
|
http.errback {
|
34
|
-
http.error.should match(/unable to resolve server address/)
|
34
|
+
http.error.should match(/unable to resolve (server |)address/)
|
35
35
|
http.response_header.status.should == 0
|
36
36
|
EventMachine.stop
|
37
37
|
}
|
data/spec/http_proxy_spec.rb
CHANGED
@@ -1,107 +1,268 @@
|
|
1
1
|
require 'helper'
|
2
2
|
|
3
|
+
shared_examples "*_PROXY var (through proxy)" do
|
4
|
+
it "should use HTTP proxy" do
|
5
|
+
EventMachine.run {
|
6
|
+
http = EventMachine::HttpRequest.new("#{proxy_test_scheme}://127.0.0.1:8090/?q=test").get
|
7
|
+
|
8
|
+
http.errback { failed(http) }
|
9
|
+
http.callback {
|
10
|
+
http.response_header.status.should == 200
|
11
|
+
http.response_header.should_not include("X_PROXY_AUTH")
|
12
|
+
http.response.should match('test')
|
13
|
+
EventMachine.stop
|
14
|
+
}
|
15
|
+
}
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
shared_examples "*_PROXY var (testing var)" do
|
20
|
+
subject { HttpConnectionOptions.new("#{proxy_test_scheme}://example.com", {}) }
|
21
|
+
it { expect(subject.proxy_from_env).to eq({ :host => "127.0.0.1", :port => 8083, :type => :http }) }
|
22
|
+
it { expect(subject.host).to eq "127.0.0.1" }
|
23
|
+
it { expect(subject.port).to be 8083 }
|
24
|
+
it do
|
25
|
+
case proxy_test_scheme.to_sym
|
26
|
+
when :http
|
27
|
+
expect(subject.http_proxy?).to be_truthy
|
28
|
+
when :https
|
29
|
+
expect(subject.connect_proxy?).to be_truthy
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
3
34
|
describe EventMachine::HttpRequest do
|
4
35
|
|
5
36
|
context "connections via" do
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
http.
|
16
|
-
|
17
|
-
|
18
|
-
|
37
|
+
context "without *_PROXY env" do
|
38
|
+
let(:proxy) { {:proxy => { :host => '127.0.0.1', :port => 8083 }} }
|
39
|
+
let(:authenticated_proxy) { {:proxy => { :host => '127.0.0.1', :port => 8083, :authorization => ["user", "name"] } } }
|
40
|
+
|
41
|
+
it "should use HTTP proxy" do
|
42
|
+
EventMachine.run {
|
43
|
+
http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/?q=test', proxy).get
|
44
|
+
|
45
|
+
http.errback { failed(http) }
|
46
|
+
http.callback {
|
47
|
+
http.response_header.status.should == 200
|
48
|
+
http.response_header.should_not include("X_PROXY_AUTH")
|
49
|
+
http.response.should match('test')
|
50
|
+
EventMachine.stop
|
51
|
+
}
|
19
52
|
}
|
20
|
-
|
21
|
-
end
|
53
|
+
end
|
22
54
|
|
23
|
-
|
24
|
-
|
25
|
-
|
55
|
+
it "should use HTTP proxy with authentication" do
|
56
|
+
EventMachine.run {
|
57
|
+
http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/proxyauth?q=test', authenticated_proxy).get
|
26
58
|
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
59
|
+
http.errback { failed(http) }
|
60
|
+
http.callback {
|
61
|
+
http.response_header.status.should == 200
|
62
|
+
http.response_header['X_PROXY_AUTH'].should == "Proxy-Authorization: Basic dXNlcjpuYW1l"
|
63
|
+
http.response.should match('test')
|
64
|
+
EventMachine.stop
|
65
|
+
}
|
33
66
|
}
|
34
|
-
|
35
|
-
end
|
67
|
+
end
|
36
68
|
|
37
|
-
|
38
|
-
|
69
|
+
it "should send absolute URIs to the proxy server" do
|
70
|
+
EventMachine.run {
|
39
71
|
|
40
|
-
|
72
|
+
http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/?q=test', proxy).get
|
41
73
|
|
42
|
-
|
43
|
-
|
44
|
-
|
74
|
+
http.errback { failed(http) }
|
75
|
+
http.callback {
|
76
|
+
http.response_header.status.should == 200
|
45
77
|
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
78
|
+
# The test proxy server gives the requested uri back in this header
|
79
|
+
http.response_header['X_THE_REQUESTED_URI'].should == 'http://127.0.0.1:8090/?q=test'
|
80
|
+
http.response_header['X_THE_REQUESTED_URI'].should_not == '/?q=test'
|
81
|
+
http.response.should match('test')
|
82
|
+
EventMachine.stop
|
83
|
+
}
|
51
84
|
}
|
52
|
-
|
53
|
-
end
|
85
|
+
end
|
54
86
|
|
55
|
-
|
56
|
-
|
87
|
+
it "should strip basic auth from before the host in URI sent to proxy" do
|
88
|
+
EventMachine.run {
|
57
89
|
|
58
|
-
|
90
|
+
http = EventMachine::HttpRequest.new('http://user:pass@127.0.0.1:8090/echo_authorization_header', proxy).get
|
59
91
|
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
92
|
+
http.errback { failed(http) }
|
93
|
+
http.callback {
|
94
|
+
http.response_header.status.should == 200
|
95
|
+
# The test proxy server gives the requested uri back in this header
|
96
|
+
http.response_header['X_THE_REQUESTED_URI'].should == 'http://127.0.0.1:8090/echo_authorization_header'
|
97
|
+
# Ensure the basic auth was converted to a header correctly
|
98
|
+
http.response.should match('authorization:Basic dXNlcjpwYXNz')
|
99
|
+
EventMachine.stop
|
100
|
+
}
|
68
101
|
}
|
69
|
-
|
70
|
-
end
|
102
|
+
end
|
71
103
|
|
72
|
-
|
73
|
-
|
74
|
-
|
104
|
+
it "should include query parameters specified in the options" do
|
105
|
+
EventMachine.run {
|
106
|
+
http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/', proxy).get :query => { 'q' => 'test' }
|
75
107
|
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
108
|
+
http.errback { failed(http) }
|
109
|
+
http.callback {
|
110
|
+
http.response_header.status.should == 200
|
111
|
+
http.response.should match('test')
|
112
|
+
EventMachine.stop
|
113
|
+
}
|
81
114
|
}
|
82
|
-
|
83
|
-
end
|
115
|
+
end
|
84
116
|
|
85
|
-
|
86
|
-
|
87
|
-
|
117
|
+
it "should use HTTP proxy while redirecting" do
|
118
|
+
EventMachine.run {
|
119
|
+
http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/redirect', proxy).get :redirects => 1
|
88
120
|
|
89
|
-
|
90
|
-
|
91
|
-
|
121
|
+
http.errback { failed(http) }
|
122
|
+
http.callback {
|
123
|
+
http.response_header.status.should == 200
|
92
124
|
|
93
|
-
|
94
|
-
|
125
|
+
http.response_header['X_THE_REQUESTED_URI'].should == 'http://127.0.0.1:8090/gzip'
|
126
|
+
http.response_header['X_THE_REQUESTED_URI'].should_not == '/redirect'
|
95
127
|
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
128
|
+
http.response_header["CONTENT_ENCODING"].should == "gzip"
|
129
|
+
http.response.should == "compressed"
|
130
|
+
http.last_effective_url.to_s.should == 'http://127.0.0.1:8090/gzip'
|
131
|
+
http.redirects.should == 1
|
100
132
|
|
101
|
-
|
133
|
+
EventMachine.stop
|
134
|
+
}
|
102
135
|
}
|
103
|
-
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
139
|
+
context "when parsing *_PROXY var (through proxy)s" do
|
140
|
+
context "with $HTTP_PROXY env" do
|
141
|
+
let(:proxy_test_scheme) { :http }
|
142
|
+
|
143
|
+
before(:all) do
|
144
|
+
PROXY_ENV_VARS.each {|k| ENV.delete k }
|
145
|
+
ENV['HTTP_PROXY'] = 'http://127.0.0.1:8083'
|
146
|
+
end
|
147
|
+
|
148
|
+
include_examples "*_PROXY var (through proxy)"
|
149
|
+
end
|
150
|
+
|
151
|
+
context "with $http_proxy env" do
|
152
|
+
let(:proxy_test_scheme) { :http }
|
153
|
+
|
154
|
+
before(:all) do
|
155
|
+
PROXY_ENV_VARS.each {|k| ENV.delete k }
|
156
|
+
ENV['http_proxy'] = 'http://127.0.0.1:8083'
|
157
|
+
end
|
158
|
+
|
159
|
+
include_examples "*_PROXY var (through proxy)"
|
160
|
+
end
|
161
|
+
|
162
|
+
## TODO: Use a Mongrel HTTP server that can handle SSL:
|
163
|
+
context "with $HTTPS_PROXY env", skip: "Mongrel isn't configured to handle HTTPS, currently" do
|
164
|
+
let(:proxy_test_scheme) { :https }
|
165
|
+
|
166
|
+
before(:all) do
|
167
|
+
PROXY_ENV_VARS.each {|k| ENV.delete k }
|
168
|
+
ENV['HTTPS_PROXY'] = 'http://127.0.0.1:8083'
|
169
|
+
end
|
170
|
+
|
171
|
+
include_examples "*_PROXY var (through proxy)"
|
172
|
+
end
|
173
|
+
|
174
|
+
## TODO: Use a Mongrel HTTP server that can handle SSL:
|
175
|
+
context "with $https_proxy env", skip: "Mongrel isn't configured to handle HTTPS, currently" do
|
176
|
+
let(:proxy_test_scheme) { :https }
|
177
|
+
|
178
|
+
before(:all) do
|
179
|
+
PROXY_ENV_VARS.each {|k| ENV.delete k }
|
180
|
+
ENV['https_proxy'] = 'http://127.0.0.1:8083'
|
181
|
+
end
|
182
|
+
|
183
|
+
include_examples "*_PROXY var (through proxy)"
|
184
|
+
end
|
185
|
+
|
186
|
+
context "with $ALL_PROXY env" do
|
187
|
+
let(:proxy_test_scheme) { :http }
|
188
|
+
|
189
|
+
before(:all) do
|
190
|
+
PROXY_ENV_VARS.each {|k| ENV.delete k }
|
191
|
+
ENV['ALL_PROXY'] = 'http://127.0.0.1:8083'
|
192
|
+
end
|
193
|
+
|
194
|
+
include_examples "*_PROXY var (through proxy)"
|
195
|
+
end
|
104
196
|
end
|
105
197
|
end
|
106
198
|
|
199
|
+
context "when parsing *_PROXY vars" do
|
200
|
+
context "without a *_PROXY var" do
|
201
|
+
before(:all) do
|
202
|
+
PROXY_ENV_VARS.each {|k| ENV.delete k }
|
203
|
+
end
|
204
|
+
|
205
|
+
subject { HttpConnectionOptions.new("http://example.com", {}) }
|
206
|
+
it { expect(subject.proxy_from_env).to be_nil }
|
207
|
+
it { expect(subject.host).to eq "example.com" }
|
208
|
+
it { expect(subject.port).to be 80 }
|
209
|
+
it { expect(subject.http_proxy?).to be_falsey }
|
210
|
+
it { expect(subject.connect_proxy?).to be_falsey }
|
211
|
+
end
|
212
|
+
|
213
|
+
context "with $HTTP_PROXY env" do
|
214
|
+
let(:proxy_test_scheme) { :http }
|
215
|
+
|
216
|
+
before(:each) do
|
217
|
+
PROXY_ENV_VARS.each {|k| ENV.delete k }
|
218
|
+
ENV['HTTP_PROXY'] = 'http://127.0.0.1:8083'
|
219
|
+
end
|
220
|
+
|
221
|
+
include_examples "*_PROXY var (testing var)"
|
222
|
+
end
|
223
|
+
|
224
|
+
context "with $http_proxy env" do
|
225
|
+
let(:proxy_test_scheme) { :http }
|
226
|
+
|
227
|
+
before(:each) do
|
228
|
+
PROXY_ENV_VARS.each {|k| ENV.delete k }
|
229
|
+
ENV['http_proxy'] = 'http://127.0.0.1:8083'
|
230
|
+
end
|
231
|
+
|
232
|
+
include_examples "*_PROXY var (testing var)"
|
233
|
+
end
|
234
|
+
|
235
|
+
context "with $HTTPS_PROXY env" do
|
236
|
+
let(:proxy_test_scheme) { :https }
|
237
|
+
|
238
|
+
before(:each) do
|
239
|
+
PROXY_ENV_VARS.each {|k| ENV.delete k }
|
240
|
+
ENV['HTTPS_PROXY'] = 'http://127.0.0.1:8083'
|
241
|
+
end
|
242
|
+
|
243
|
+
include_examples "*_PROXY var (testing var)"
|
244
|
+
end
|
245
|
+
|
246
|
+
context "with $https_proxy env" do
|
247
|
+
let(:proxy_test_scheme) { :https }
|
248
|
+
|
249
|
+
before(:each) do
|
250
|
+
PROXY_ENV_VARS.each {|k| ENV.delete k }
|
251
|
+
ENV['https_proxy'] = 'http://127.0.0.1:8083'
|
252
|
+
end
|
253
|
+
|
254
|
+
include_examples "*_PROXY var (testing var)"
|
255
|
+
end
|
256
|
+
|
257
|
+
context "with $ALL_PROXY env" do
|
258
|
+
let(:proxy_test_scheme) { :https }
|
259
|
+
|
260
|
+
before(:each) do
|
261
|
+
PROXY_ENV_VARS.each {|k| ENV.delete k }
|
262
|
+
ENV['ALL_PROXY'] = 'http://127.0.0.1:8083'
|
263
|
+
end
|
264
|
+
|
265
|
+
include_examples "*_PROXY var (testing var)"
|
266
|
+
end
|
267
|
+
end
|
107
268
|
end
|
data/spec/spec_helper.rb
CHANGED
@@ -1,8 +1,25 @@
|
|
1
|
+
PROXY_ENV_VARS = %w[HTTP_PROXY http_proxy HTTPS_PROXY https_proxy ALL_PROXY]
|
2
|
+
|
1
3
|
RSpec.configure do |config|
|
4
|
+
proxy_envs = {}
|
5
|
+
|
2
6
|
config.mock_with :rspec do |c|
|
3
7
|
c.syntax = [:should, :expect]
|
4
8
|
end
|
5
9
|
config.expect_with :rspec do |c|
|
6
10
|
c.syntax = [:should, :expect]
|
7
11
|
end
|
8
|
-
|
12
|
+
|
13
|
+
config.before :all do
|
14
|
+
# Back-up ENV *_PROXY vars
|
15
|
+
orig_proxy_envs = Hash[
|
16
|
+
PROXY_ENV_VARS.select {|k| ENV.key? k }.map {|k| [k, ENV.delete(k)] }
|
17
|
+
]
|
18
|
+
proxy_envs.replace(orig_proxy_envs)
|
19
|
+
end
|
20
|
+
|
21
|
+
config.after :all do
|
22
|
+
# Restore ENV *_PROXY vars
|
23
|
+
ENV.update(proxy_envs)
|
24
|
+
end
|
25
|
+
end
|
data/spec/ssl_spec.rb
CHANGED
@@ -3,7 +3,6 @@ require 'helper'
|
|
3
3
|
requires_connection do
|
4
4
|
|
5
5
|
describe EventMachine::HttpRequest do
|
6
|
-
|
7
6
|
it "should initiate SSL/TLS on HTTPS connections" do
|
8
7
|
EventMachine.run {
|
9
8
|
http = EventMachine::HttpRequest.new('https://mail.google.com:443/mail/').get
|
@@ -15,6 +14,58 @@ requires_connection do
|
|
15
14
|
}
|
16
15
|
}
|
17
16
|
end
|
17
|
+
|
18
|
+
describe "TLS hostname verification" do
|
19
|
+
before do
|
20
|
+
@cve_warning = "[WARNING; em-http-request] TLS hostname validation is disabled (use 'tls: {verify_peer: true}'), see" +
|
21
|
+
" CVE-2020-13482 and https://github.com/igrigorik/em-http-request/issues/339 for details"
|
22
|
+
@orig_stderr = $stderr
|
23
|
+
$stderr = StringIO.new
|
24
|
+
end
|
25
|
+
|
26
|
+
after do
|
27
|
+
$stderr = @orig_stderr
|
28
|
+
end
|
29
|
+
|
30
|
+
it "should not warn if verify_peer is specified" do
|
31
|
+
EventMachine.run {
|
32
|
+
http = EventMachine::HttpRequest.new('https://mail.google.com:443/mail', {tls: {verify_peer: false}}).get
|
33
|
+
|
34
|
+
http.callback {
|
35
|
+
$stderr.rewind
|
36
|
+
$stderr.string.chomp.should_not eq(@cve_warning)
|
37
|
+
|
38
|
+
EventMachine.stop
|
39
|
+
}
|
40
|
+
}
|
41
|
+
end
|
42
|
+
|
43
|
+
it "should not warn if verify_peer is true" do
|
44
|
+
EventMachine.run {
|
45
|
+
http = EventMachine::HttpRequest.new('https://mail.google.com:443/mail', {tls: {verify_peer: true}}).get
|
46
|
+
|
47
|
+
http.callback {
|
48
|
+
$stderr.rewind
|
49
|
+
$stderr.string.chomp.should_not eq(@cve_warning)
|
50
|
+
|
51
|
+
EventMachine.stop
|
52
|
+
}
|
53
|
+
}
|
54
|
+
end
|
55
|
+
|
56
|
+
it "should warn if verify_peer is unspecified" do
|
57
|
+
EventMachine.run {
|
58
|
+
http = EventMachine::HttpRequest.new('https://mail.google.com:443/mail').get
|
59
|
+
|
60
|
+
http.callback {
|
61
|
+
$stderr.rewind
|
62
|
+
$stderr.string.chomp.should eq(@cve_warning)
|
63
|
+
|
64
|
+
EventMachine.stop
|
65
|
+
}
|
66
|
+
}
|
67
|
+
end
|
68
|
+
end
|
18
69
|
end
|
19
70
|
|
20
71
|
end
|
data/spec/stallion.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: em-http-request
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.1.
|
4
|
+
version: 1.1.7
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ilya Grigorik
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-08-31 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: addressable
|
@@ -112,16 +112,16 @@ dependencies:
|
|
112
112
|
name: rack
|
113
113
|
requirement: !ruby/object:Gem::Requirement
|
114
114
|
requirements:
|
115
|
-
- - "
|
115
|
+
- - "<"
|
116
116
|
- !ruby/object:Gem::Version
|
117
|
-
version: '0'
|
117
|
+
version: '2.0'
|
118
118
|
type: :development
|
119
119
|
prerelease: false
|
120
120
|
version_requirements: !ruby/object:Gem::Requirement
|
121
121
|
requirements:
|
122
|
-
- - "
|
122
|
+
- - "<"
|
123
123
|
- !ruby/object:Gem::Version
|
124
|
-
version: '0'
|
124
|
+
version: '2.0'
|
125
125
|
- !ruby/object:Gem::Dependency
|
126
126
|
name: rake
|
127
127
|
requirement: !ruby/object:Gem::Requirement
|
@@ -197,6 +197,7 @@ files:
|
|
197
197
|
- lib/em-http/multi.rb
|
198
198
|
- lib/em-http/request.rb
|
199
199
|
- lib/em-http/version.rb
|
200
|
+
- lib/em/io_streamer.rb
|
200
201
|
- spec/client_fiber_spec.rb
|
201
202
|
- spec/client_spec.rb
|
202
203
|
- spec/digest_auth_spec.rb
|
@@ -222,7 +223,7 @@ homepage: http://github.com/igrigorik/em-http-request
|
|
222
223
|
licenses:
|
223
224
|
- MIT
|
224
225
|
metadata: {}
|
225
|
-
post_install_message:
|
226
|
+
post_install_message:
|
226
227
|
rdoc_options: []
|
227
228
|
require_paths:
|
228
229
|
- lib
|
@@ -237,9 +238,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
237
238
|
- !ruby/object:Gem::Version
|
238
239
|
version: '0'
|
239
240
|
requirements: []
|
240
|
-
|
241
|
-
|
242
|
-
signing_key:
|
241
|
+
rubygems_version: 3.0.3
|
242
|
+
signing_key:
|
243
243
|
specification_version: 4
|
244
244
|
summary: EventMachine based, async HTTP Request client
|
245
245
|
test_files:
|
@@ -264,4 +264,3 @@ test_files:
|
|
264
264
|
- spec/ssl_spec.rb
|
265
265
|
- spec/stallion.rb
|
266
266
|
- spec/stub_server.rb
|
267
|
-
has_rdoc:
|