ruby-proxy-headers 0.2.2 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +49 -0
- data/README.md +48 -1
- data/docs/core-api.md +184 -0
- data/docs/excon.md +29 -0
- data/docs/faraday.md +29 -0
- data/docs/getting-started.md +66 -0
- data/docs/http-gem.md +126 -0
- data/docs/httparty.md +35 -0
- data/docs/index.md +109 -0
- data/docs/net-http.md +29 -0
- data/docs/rest-client.md +96 -0
- data/docs/testing.md +105 -0
- data/docs/typhoeus.md +47 -0
- data/lib/ruby_proxy_headers/connection.rb +1 -8
- data/lib/ruby_proxy_headers/faraday.rb +19 -16
- data/lib/ruby_proxy_headers/http_gem.rb +61 -40
- data/lib/ruby_proxy_headers/net_http.rb +76 -131
- data/lib/ruby_proxy_headers/rest_client.rb +76 -60
- data/lib/ruby_proxy_headers/typhoeus.rb +81 -74
- data/lib/ruby_proxy_headers/version.rb +1 -1
- data/lib/ruby_proxy_headers.rb +45 -0
- metadata +70 -10
- data/DEFERRED.md +0 -29
- data/IMPLEMENTATION_PRIORITY.md +0 -72
- data/LIBRARY_RESEARCH.md +0 -71
|
@@ -1,16 +1,20 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require 'net/http'
|
|
4
|
-
require 'net/protocol'
|
|
5
4
|
require 'openssl'
|
|
6
|
-
|
|
7
|
-
|
|
5
|
+
|
|
6
|
+
# Loaded for header validation. Requiring this file first is safe: the parent
|
|
7
|
+
# requires it back and Ruby drops the cycle.
|
|
8
|
+
require_relative '../ruby_proxy_headers'
|
|
8
9
|
|
|
9
10
|
module RubyProxyHeaders
|
|
10
|
-
# Patches Net::HTTP
|
|
11
|
-
#
|
|
12
|
-
#
|
|
13
|
-
#
|
|
11
|
+
# Patches Net::HTTP so an HTTPS request through a proxy can send extra CONNECT
|
|
12
|
+
# headers and keep the proxy's CONNECT response.
|
|
13
|
+
#
|
|
14
|
+
# The patch does not copy +Net::HTTP#connect+. The installed net-http still
|
|
15
|
+
# owns the socket, +timeouted_connect+, and HTTPS-to-proxy (+proxy_use_ssl+).
|
|
16
|
+
# A prepend validates the target, then rewrites the CONNECT bytes that
|
|
17
|
+
# upstream writes and records the CONNECT response.
|
|
14
18
|
#
|
|
15
19
|
# @example
|
|
16
20
|
# require 'ruby_proxy_headers/net_http'
|
|
@@ -20,148 +24,89 @@ module RubyProxyHeaders
|
|
|
20
24
|
# http.use_ssl = true
|
|
21
25
|
# http.proxy_connect_request_headers = { 'X-ProxyMesh-IP' => '203.0.113.1' }
|
|
22
26
|
# res = http.request(Net::HTTP::Get.new(uri))
|
|
23
|
-
# p http.last_proxy_connect_response_headers
|
|
27
|
+
# p http.last_proxy_connect_response_headers['X-ProxyMesh-IP']
|
|
24
28
|
module NetHTTP
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
module Extension
|
|
30
|
-
attr_accessor :proxy_connect_request_headers
|
|
31
|
-
attr_reader :last_proxy_connect_response_headers
|
|
29
|
+
# Inserts +headers+ before the blank line that ends a CONNECT request.
|
|
30
|
+
def self.inject_connect_headers(buf, headers)
|
|
31
|
+
return buf if headers.nil? || headers.empty?
|
|
32
32
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
else
|
|
37
|
-
RubyProxyHeaders::NetHTTP::ORIGINAL_CONNECT.bind_call(self)
|
|
38
|
-
end
|
|
33
|
+
extra = String.new(encoding: buf.encoding)
|
|
34
|
+
headers.each do |key, value|
|
|
35
|
+
extra << "#{key}: #{value}\r\n".encode(buf.encoding)
|
|
39
36
|
end
|
|
37
|
+
"#{buf.sub(/\r\n\r\n\z/, "\r\n")}#{extra}\r\n"
|
|
38
|
+
end
|
|
40
39
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
conn_addr = proxy_address
|
|
53
|
-
conn_port = proxy_port
|
|
54
|
-
else
|
|
55
|
-
conn_addr = conn_address
|
|
56
|
-
conn_port = port
|
|
40
|
+
# Rewrites a CONNECT write while a patched connection is opening.
|
|
41
|
+
module ConnectRequestWrite
|
|
42
|
+
def write(*strs)
|
|
43
|
+
http = Thread.current[:ruby_proxy_headers_active_http]
|
|
44
|
+
if http && strs.first.start_with?('CONNECT ')
|
|
45
|
+
strs = [
|
|
46
|
+
RubyProxyHeaders::NetHTTP.inject_connect_headers(
|
|
47
|
+
strs.join,
|
|
48
|
+
http.proxy_connect_request_headers
|
|
49
|
+
)
|
|
50
|
+
]
|
|
57
51
|
end
|
|
52
|
+
super(*strs)
|
|
53
|
+
end
|
|
54
|
+
end
|
|
58
55
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
56
|
+
# Records CONNECT response headers. Active only while #connect is running,
|
|
57
|
+
# so the later origin response is left alone.
|
|
58
|
+
module ConnectResponseRead
|
|
59
|
+
def read_new(sock)
|
|
60
|
+
response = super
|
|
61
|
+
http = Thread.current[:ruby_proxy_headers_active_http]
|
|
62
|
+
if http
|
|
63
|
+
headers = HeaderMap.new
|
|
64
|
+
response.each_header { |key, value| headers[key] = value }
|
|
65
|
+
http.last_proxy_connect_response_headers = headers
|
|
66
|
+
Thread.current[:ruby_proxy_headers_connect_headers] = headers.dup
|
|
67
67
|
end
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
if use_ssl?
|
|
72
|
-
if proxy?
|
|
73
|
-
plain_sock = Net::BufferedIO.new(s, read_timeout: @read_timeout,
|
|
74
|
-
write_timeout: @write_timeout,
|
|
75
|
-
continue_timeout: @continue_timeout,
|
|
76
|
-
debug_output: @debug_output)
|
|
77
|
-
buf = +"CONNECT #{conn_address}:#{@port} HTTP/#{::Net::HTTP::HTTPVersion}\r\n" \
|
|
78
|
-
"Host: #{@address}:#{@port}\r\n"
|
|
79
|
-
if proxy_user
|
|
80
|
-
credential = ["#{proxy_user}:#{proxy_pass}"].pack('m0')
|
|
81
|
-
buf << "Proxy-Authorization: Basic #{credential}\r\n"
|
|
82
|
-
end
|
|
83
|
-
if proxy_connect_request_headers&.any?
|
|
84
|
-
proxy_connect_request_headers.each do |k, v|
|
|
85
|
-
RubyProxyHeaders.validate_header!(k, v)
|
|
86
|
-
buf << "#{k}: #{v}\r\n"
|
|
87
|
-
end
|
|
88
|
-
end
|
|
89
|
-
buf << "\r\n"
|
|
90
|
-
plain_sock.write(buf)
|
|
91
|
-
|
|
92
|
-
proxy_res = ::Net::HTTPResponse.read_new(plain_sock)
|
|
93
|
-
@last_proxy_connect_response_headers = {}
|
|
94
|
-
proxy_res.each_header do |k, v|
|
|
95
|
-
@last_proxy_connect_response_headers[k] = v
|
|
96
|
-
end
|
|
97
|
-
Thread.current[:ruby_proxy_headers_connect_headers] = @last_proxy_connect_response_headers.dup
|
|
98
|
-
proxy_res.value
|
|
99
|
-
end
|
|
100
|
-
|
|
101
|
-
ssl_parameters = {}
|
|
102
|
-
iv_list = instance_variables
|
|
103
|
-
Net::HTTP::SSL_IVNAMES.each_with_index do |ivname, i|
|
|
104
|
-
if iv_list.include?(ivname)
|
|
105
|
-
value = instance_variable_get(ivname)
|
|
106
|
-
unless value.nil?
|
|
107
|
-
ssl_parameters[Net::HTTP::SSL_ATTRIBUTES[i]] = value
|
|
108
|
-
end
|
|
109
|
-
end
|
|
110
|
-
end
|
|
111
|
-
@ssl_context.set_params(ssl_parameters)
|
|
112
|
-
unless @ssl_context.session_cache_mode.nil?
|
|
113
|
-
@ssl_context.session_cache_mode =
|
|
114
|
-
OpenSSL::SSL::SSLContext::SESSION_CACHE_CLIENT |
|
|
115
|
-
OpenSSL::SSL::SSLContext::SESSION_CACHE_NO_INTERNAL_STORE
|
|
116
|
-
end
|
|
117
|
-
if @ssl_context.respond_to?(:session_new_cb)
|
|
118
|
-
@ssl_context.session_new_cb = proc { |sock, sess| @ssl_session = sess }
|
|
119
|
-
end
|
|
68
|
+
response
|
|
69
|
+
end
|
|
70
|
+
end
|
|
120
71
|
|
|
121
|
-
|
|
72
|
+
# Instance methods prepended onto Net::HTTP.
|
|
73
|
+
module Extension
|
|
74
|
+
attr_accessor :proxy_connect_request_headers, :last_proxy_connect_response_headers
|
|
122
75
|
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
@ssl_context.verify_hostname = false
|
|
126
|
-
else
|
|
127
|
-
ssl_host_address = @address
|
|
128
|
-
end
|
|
76
|
+
def connect
|
|
77
|
+
return super unless use_ssl? && proxy?
|
|
129
78
|
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
79
|
+
# Reject smuggled targets and header values before any socket I/O.
|
|
80
|
+
RubyProxyHeaders.validate_connect_target!(conn_address, @port)
|
|
81
|
+
RubyProxyHeaders.validate_connect_target!(@address, @port)
|
|
82
|
+
proxy_connect_request_headers&.each do |key, value|
|
|
83
|
+
RubyProxyHeaders.validate_header!(key, value)
|
|
84
|
+
end
|
|
134
85
|
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
86
|
+
@last_proxy_connect_response_headers = nil
|
|
87
|
+
previous_http = Thread.current[:ruby_proxy_headers_active_http]
|
|
88
|
+
previous_headers = Thread.current[:ruby_proxy_headers_connect_headers]
|
|
89
|
+
begin
|
|
90
|
+
Thread.current[:ruby_proxy_headers_active_http] = self
|
|
91
|
+
Thread.current[:ruby_proxy_headers_connect_headers] = nil
|
|
92
|
+
super
|
|
93
|
+
ensure
|
|
94
|
+
Thread.current[:ruby_proxy_headers_active_http] = previous_http
|
|
95
|
+
if Thread.current[:ruby_proxy_headers_connect_headers].nil?
|
|
96
|
+
Thread.current[:ruby_proxy_headers_connect_headers] = previous_headers
|
|
138
97
|
end
|
|
139
|
-
ssl_socket_connect(s, @open_timeout)
|
|
140
|
-
if (@ssl_context.verify_mode != OpenSSL::SSL::VERIFY_NONE) && verify_hostname
|
|
141
|
-
s.post_connection_check(@address)
|
|
142
|
-
end
|
|
143
|
-
debug "SSL established, protocol: #{s.ssl_version}, cipher: #{s.cipher[0]}"
|
|
144
|
-
end
|
|
145
|
-
@socket = Net::BufferedIO.new(s, read_timeout: @read_timeout,
|
|
146
|
-
write_timeout: @write_timeout,
|
|
147
|
-
continue_timeout: @continue_timeout,
|
|
148
|
-
debug_output: @debug_output)
|
|
149
|
-
@last_communicated = nil
|
|
150
|
-
on_connect
|
|
151
|
-
rescue StandardError => e
|
|
152
|
-
if s
|
|
153
|
-
debug "Conn close because of connect error #{e}"
|
|
154
|
-
s.close
|
|
155
98
|
end
|
|
156
|
-
raise
|
|
157
99
|
end
|
|
158
100
|
end
|
|
159
101
|
|
|
160
102
|
class << self
|
|
161
103
|
def patch!
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
::Net::
|
|
104
|
+
::Net::HTTP.prepend(Extension) unless ::Net::HTTP.ancestors.include?(Extension)
|
|
105
|
+
io_hooked = ::Net::BufferedIO.ancestors.include?(ConnectRequestWrite)
|
|
106
|
+
::Net::BufferedIO.prepend(ConnectRequestWrite) unless io_hooked
|
|
107
|
+
response_class = ::Net::HTTPResponse.singleton_class
|
|
108
|
+
response_hooked = response_class.ancestors.include?(ConnectResponseRead)
|
|
109
|
+
response_class.prepend(ConnectResponseRead) unless response_hooked
|
|
165
110
|
@patched = true
|
|
166
111
|
end
|
|
167
112
|
|
|
@@ -1,89 +1,105 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'delegate'
|
|
4
|
+
require 'openssl'
|
|
5
|
+
require 'uri'
|
|
6
|
+
require_relative '../ruby_proxy_headers'
|
|
7
|
+
|
|
8
|
+
begin
|
|
9
|
+
require 'rest-client'
|
|
10
|
+
rescue LoadError => e
|
|
11
|
+
raise LoadError, "rest-client is required for ruby_proxy_headers/rest_client (#{e.message})"
|
|
12
|
+
end
|
|
13
|
+
|
|
3
14
|
module RubyProxyHeaders
|
|
4
|
-
# RestClient
|
|
15
|
+
# RestClient uses Net::HTTP. HTTPS requests go through that client so
|
|
16
|
+
# redirects and payloads stay RestClient's. The Net::HTTP patch attaches
|
|
17
|
+
# CONNECT headers and records +X-ProxyMesh-IP+.
|
|
5
18
|
#
|
|
6
19
|
# @example
|
|
7
20
|
# response = RubyProxyHeaders::RestClient.get(
|
|
8
21
|
# 'https://example.com',
|
|
9
|
-
# proxy: 'http://user:pass@proxy:
|
|
22
|
+
# proxy: 'http://user:pass@proxy.example.com:31280',
|
|
10
23
|
# proxy_headers: { 'X-ProxyMesh-Country' => 'US' }
|
|
11
24
|
# )
|
|
12
|
-
# puts response.proxy_response_headers
|
|
13
|
-
#
|
|
25
|
+
# puts response.proxy_response_headers['X-ProxyMesh-IP']
|
|
14
26
|
module RestClient
|
|
15
|
-
#
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
end
|
|
24
|
-
|
|
25
|
-
# Make a PUT request with proxy headers.
|
|
26
|
-
def self.put(url, proxy:, proxy_headers: {}, payload: nil, **options)
|
|
27
|
-
request(:put, url, proxy: proxy, proxy_headers: proxy_headers, body: payload, **options)
|
|
28
|
-
end
|
|
29
|
-
|
|
30
|
-
# Make a DELETE request with proxy headers.
|
|
31
|
-
def self.delete(url, proxy:, proxy_headers: {}, **options)
|
|
32
|
-
request(:delete, url, proxy: proxy, proxy_headers: proxy_headers, **options)
|
|
27
|
+
# Sets CONNECT headers on the Net::HTTP object RestClient is about to use.
|
|
28
|
+
module NetHttpHook
|
|
29
|
+
def net_http_object(hostname, port)
|
|
30
|
+
http = super
|
|
31
|
+
headers = Thread.current[:ruby_proxy_headers_rest_client_headers]
|
|
32
|
+
http.proxy_connect_request_headers = headers if headers&.any?
|
|
33
|
+
http
|
|
34
|
+
end
|
|
33
35
|
end
|
|
34
36
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
37
|
+
class << self
|
|
38
|
+
def get(url, proxy:, proxy_headers: {}, **options)
|
|
39
|
+
request(:get, url, proxy: proxy, proxy_headers: proxy_headers, **options)
|
|
40
|
+
end
|
|
38
41
|
|
|
39
|
-
|
|
42
|
+
def post(url, proxy:, proxy_headers: {}, payload: nil, **options)
|
|
43
|
+
request(:post, url, proxy: proxy, proxy_headers: proxy_headers, payload: payload, **options)
|
|
44
|
+
end
|
|
40
45
|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
response = RubyProxyHeaders::NetHTTP.request(
|
|
44
|
-
method,
|
|
45
|
-
url,
|
|
46
|
-
proxy: proxy,
|
|
47
|
-
proxy_headers: proxy_headers,
|
|
48
|
-
headers: options[:headers],
|
|
49
|
-
body: options[:body]
|
|
50
|
-
)
|
|
51
|
-
return response
|
|
46
|
+
def put(url, proxy:, proxy_headers: {}, payload: nil, **options)
|
|
47
|
+
request(:put, url, proxy: proxy, proxy_headers: proxy_headers, payload: payload, **options)
|
|
52
48
|
end
|
|
53
49
|
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
ProxyResponse.new(response, {})
|
|
58
|
-
end
|
|
50
|
+
def delete(url, proxy:, proxy_headers: {}, **options)
|
|
51
|
+
request(:delete, url, proxy: proxy, proxy_headers: proxy_headers, **options)
|
|
52
|
+
end
|
|
59
53
|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
54
|
+
def request(method, url, proxy:, proxy_headers: {}, **options)
|
|
55
|
+
install!
|
|
56
|
+
headers = proxy_headers || {}
|
|
57
|
+
https = URI.parse(url).scheme == 'https'
|
|
58
|
+
headers.each { |key, value| RubyProxyHeaders.validate_header!(key, value) } if https
|
|
63
59
|
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
60
|
+
previous = Thread.current[:ruby_proxy_headers_rest_client_headers]
|
|
61
|
+
Thread.current[:ruby_proxy_headers_rest_client_headers] = https ? headers : nil
|
|
62
|
+
response = ::RestClient::Request.execute(execute_options(method, url, proxy, options))
|
|
63
|
+
captured = RubyProxyHeaders.proxy_connect_response_headers
|
|
64
|
+
ProxyResponse.new(response, captured ? captured.dup : HeaderMap.new)
|
|
65
|
+
ensure
|
|
66
|
+
Thread.current[:ruby_proxy_headers_rest_client_headers] = previous
|
|
67
67
|
end
|
|
68
68
|
|
|
69
|
-
def
|
|
70
|
-
|
|
71
|
-
|
|
69
|
+
def install!
|
|
70
|
+
RubyProxyHeaders::NetHTTP.patch!
|
|
71
|
+
return if ::RestClient::Request.ancestors.include?(NetHttpHook)
|
|
72
72
|
|
|
73
|
-
|
|
74
|
-
@response.body
|
|
73
|
+
::RestClient::Request.prepend(NetHttpHook)
|
|
75
74
|
end
|
|
76
75
|
|
|
77
|
-
|
|
78
|
-
@response.headers
|
|
79
|
-
end
|
|
76
|
+
private
|
|
80
77
|
|
|
81
|
-
def
|
|
82
|
-
|
|
78
|
+
def execute_options(method, url, proxy, options)
|
|
79
|
+
payload = options.key?(:payload) ? options[:payload] : options[:body]
|
|
80
|
+
verify_ssl = options[:verify_ssl]
|
|
81
|
+
verify_ssl = OpenSSL::SSL::VERIFY_PEER if verify_ssl == true
|
|
82
|
+
verify_ssl = OpenSSL::SSL::VERIFY_NONE if verify_ssl == false
|
|
83
|
+
{
|
|
84
|
+
method: method,
|
|
85
|
+
url: url,
|
|
86
|
+
proxy: proxy,
|
|
87
|
+
headers: options[:headers],
|
|
88
|
+
payload: payload,
|
|
89
|
+
open_timeout: options[:open_timeout],
|
|
90
|
+
read_timeout: options[:read_timeout],
|
|
91
|
+
verify_ssl: verify_ssl
|
|
92
|
+
}.compact
|
|
83
93
|
end
|
|
94
|
+
end
|
|
84
95
|
|
|
85
|
-
|
|
86
|
-
|
|
96
|
+
# RestClient response plus CONNECT response headers.
|
|
97
|
+
class ProxyResponse < SimpleDelegator
|
|
98
|
+
attr_reader :proxy_response_headers
|
|
99
|
+
|
|
100
|
+
def initialize(response, proxy_headers)
|
|
101
|
+
super(response)
|
|
102
|
+
@proxy_response_headers = proxy_headers
|
|
87
103
|
end
|
|
88
104
|
end
|
|
89
105
|
end
|
|
@@ -1,101 +1,108 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require 'delegate'
|
|
4
|
+
require 'uri'
|
|
5
|
+
require_relative '../ruby_proxy_headers'
|
|
6
|
+
|
|
7
|
+
begin
|
|
8
|
+
require 'typhoeus'
|
|
9
|
+
rescue LoadError => e
|
|
10
|
+
raise LoadError, "typhoeus ~> 1.7 is required for ruby_proxy_headers/typhoeus (#{e.message})"
|
|
11
|
+
end
|
|
12
|
+
|
|
3
13
|
module RubyProxyHeaders
|
|
4
|
-
# Typhoeus
|
|
5
|
-
#
|
|
14
|
+
# Typhoeus adapter using libcurl's proxy-header list.
|
|
15
|
+
#
|
|
16
|
+
# Typhoeus 1.7 / Ethon 0.19 expose +:proxyheader+ (`CURLOPT_PROXYHEADER`) and
|
|
17
|
+
# +:headeropt+ (`CURLOPT_HEADEROPT`). With +headeropt: :separate+, those
|
|
18
|
+
# headers are sent only to the proxy. On HTTPS that is the CONNECT request.
|
|
19
|
+
# +:headers+ stay on the tunneled origin request.
|
|
20
|
+
#
|
|
21
|
+
# libcurl records the CONNECT response as the first header block of
|
|
22
|
+
# +response_headers+ and the origin response after it. Typhoeus's
|
|
23
|
+
# +response.headers+ is only the last block. On HTTPS through a proxy,
|
|
24
|
+
# +proxy_response_headers+ is that first block when its status is 2xx.
|
|
6
25
|
#
|
|
7
26
|
# @example
|
|
8
27
|
# response = RubyProxyHeaders::Typhoeus.get(
|
|
9
|
-
# 'https://
|
|
10
|
-
# proxy: 'http://user:pass@proxy:
|
|
28
|
+
# 'https://api.ipify.org?format=json',
|
|
29
|
+
# proxy: 'http://user:pass@proxy.example.com:31280',
|
|
11
30
|
# proxy_headers: { 'X-ProxyMesh-Country' => 'US' }
|
|
12
31
|
# )
|
|
13
|
-
# puts response.proxy_response_headers
|
|
14
|
-
#
|
|
32
|
+
# puts response.proxy_response_headers['X-ProxyMesh-IP']
|
|
15
33
|
module Typhoeus
|
|
16
|
-
|
|
17
|
-
# @param url [String] Target URL
|
|
18
|
-
# @param proxy [String] Proxy URL
|
|
19
|
-
# @param proxy_headers [Hash] Custom headers to send to proxy
|
|
20
|
-
# @param options [Hash] Additional Typhoeus options
|
|
21
|
-
# @return [ProxyResponse]
|
|
22
|
-
def self.get(url, proxy:, proxy_headers: {}, **options)
|
|
23
|
-
request(:get, url, proxy: proxy, proxy_headers: proxy_headers, **options)
|
|
24
|
-
end
|
|
34
|
+
HTTP_STATUS = %r{\AHTTP/\d+(?:\.\d+)?\s+(\d{3})\b}i
|
|
25
35
|
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
36
|
+
class << self
|
|
37
|
+
def get(url, proxy:, proxy_headers: {}, **options)
|
|
38
|
+
request(:get, url, proxy: proxy, proxy_headers: proxy_headers, **options)
|
|
39
|
+
end
|
|
30
40
|
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
# @param url [String] Target URL
|
|
34
|
-
# @param proxy [String] Proxy URL
|
|
35
|
-
# @param proxy_headers [Hash] Custom headers to send to proxy
|
|
36
|
-
# @param options [Hash] Additional Typhoeus options
|
|
37
|
-
# @return [ProxyResponse]
|
|
38
|
-
def self.request(method, url, proxy:, proxy_headers: {}, **options)
|
|
39
|
-
require 'typhoeus'
|
|
40
|
-
|
|
41
|
-
uri = URI.parse(url)
|
|
42
|
-
|
|
43
|
-
# For HTTPS, we need custom handling since Typhoeus doesn't expose CURLOPT_PROXYHEADER
|
|
44
|
-
if uri.scheme == 'https' && !proxy_headers.empty?
|
|
45
|
-
# Use our core connection for now
|
|
46
|
-
# TODO: Extend Ethon to expose CURLOPT_PROXYHEADER for native support
|
|
47
|
-
response = RubyProxyHeaders::NetHTTP.request(
|
|
48
|
-
method,
|
|
49
|
-
url,
|
|
50
|
-
proxy: proxy,
|
|
51
|
-
proxy_headers: proxy_headers,
|
|
52
|
-
headers: options[:headers],
|
|
53
|
-
body: options[:body]
|
|
54
|
-
)
|
|
55
|
-
return response
|
|
41
|
+
def post(url, proxy:, proxy_headers: {}, body: nil, **options)
|
|
42
|
+
request(:post, url, proxy: proxy, proxy_headers: proxy_headers, body: body, **options)
|
|
56
43
|
end
|
|
57
44
|
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
45
|
+
def request(method, url, proxy:, proxy_headers: {}, **options)
|
|
46
|
+
typhoeus_options = options.merge(proxy: proxy, method: method)
|
|
47
|
+
unless proxy_headers.empty?
|
|
48
|
+
lines = proxy_headers.map do |name, value|
|
|
49
|
+
RubyProxyHeaders.validate_header!(name, value)
|
|
50
|
+
"#{name}: #{value}"
|
|
51
|
+
end
|
|
52
|
+
# The Slist has to stay in the options Typhoeus keeps. libcurl does not
|
|
53
|
+
# copy CURLOPT_PROXYHEADER, and Ethon clears its own reference on reset.
|
|
54
|
+
typhoeus_options[:proxyheader] = ::Ethon::Curl::Slist.new(::Ethon::Curl::Slist.build(lines))
|
|
55
|
+
typhoeus_options[:headeropt] = :separate
|
|
56
|
+
end
|
|
63
57
|
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
58
|
+
response = ::Typhoeus::Request.new(url, typhoeus_options).run
|
|
59
|
+
ProxyResponse.new(response, connect_headers(response, url, proxy))
|
|
60
|
+
end
|
|
67
61
|
|
|
68
|
-
|
|
69
|
-
class ProxyResponse
|
|
70
|
-
attr_reader :proxy_response_headers
|
|
62
|
+
private
|
|
71
63
|
|
|
72
|
-
def
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
64
|
+
def connect_headers(response, url, proxy)
|
|
65
|
+
blocks = response.response_headers.to_s.split(/\r?\n\r?\n/).map(&:strip).select do |part|
|
|
66
|
+
part.start_with?('HTTP/')
|
|
67
|
+
end
|
|
68
|
+
block = blocks.first
|
|
69
|
+
return HeaderMap.new unless proxy_connect_block?(block, blocks.size, url, proxy)
|
|
76
70
|
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
71
|
+
headers = HeaderMap.new
|
|
72
|
+
block.each_line do |line|
|
|
73
|
+
stripped = line.strip
|
|
74
|
+
next if stripped.empty? || stripped.start_with?('HTTP/')
|
|
80
75
|
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
end
|
|
76
|
+
name, value = stripped.split(':', 2)
|
|
77
|
+
next if name.nil? || value.nil?
|
|
84
78
|
|
|
85
|
-
|
|
86
|
-
|
|
79
|
+
name = name.strip
|
|
80
|
+
value = value.strip
|
|
81
|
+
headers[name] = headers.key?(name) ? "#{headers[name]}, #{value}" : value
|
|
82
|
+
end
|
|
83
|
+
headers
|
|
87
84
|
end
|
|
88
85
|
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
86
|
+
# Some proxies send "200 OK" rather than "200 Connection established".
|
|
87
|
+
# The CONNECT block is the first of at least two when the origin responded.
|
|
88
|
+
def proxy_connect_block?(block, block_count, url, proxy)
|
|
89
|
+
return false if block.nil? || proxy.nil? || block_count < 2
|
|
90
|
+
return false unless URI.parse(url.to_s).scheme == 'https'
|
|
92
91
|
|
|
93
|
-
|
|
94
|
-
|
|
92
|
+
code = block.lines.first.to_s[HTTP_STATUS, 1]
|
|
93
|
+
code.to_i.between?(200, 299)
|
|
94
|
+
rescue URI::InvalidURIError
|
|
95
|
+
false
|
|
95
96
|
end
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# Typhoeus response plus the CONNECT response headers.
|
|
100
|
+
class ProxyResponse < SimpleDelegator
|
|
101
|
+
attr_reader :proxy_response_headers
|
|
96
102
|
|
|
97
|
-
def
|
|
98
|
-
|
|
103
|
+
def initialize(response, proxy_headers)
|
|
104
|
+
super(response)
|
|
105
|
+
@proxy_response_headers = proxy_headers
|
|
99
106
|
end
|
|
100
107
|
end
|
|
101
108
|
end
|
data/lib/ruby_proxy_headers.rb
CHANGED
|
@@ -7,6 +7,38 @@ module RubyProxyHeaders
|
|
|
7
7
|
INVALID_HEADER_NAME_RE = /[\r\n\0]/
|
|
8
8
|
INVALID_HEADER_VALUE_RE = /[\r\n\0]/
|
|
9
9
|
|
|
10
|
+
# HTTP header map with case-insensitive lookup. Names are stored downcased,
|
|
11
|
+
# which is how Net::HTTP yields them. +X-ProxyMesh-IP+ and +x-proxymesh-ip+
|
|
12
|
+
# read the same value. +[]+, +[]=+, +fetch+, +delete+, and +key?+ normalize
|
|
13
|
+
# names. Other Hash methods still use the stored downcased key.
|
|
14
|
+
class HeaderMap < Hash
|
|
15
|
+
def [](key)
|
|
16
|
+
super(normalize(key))
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def []=(key, value)
|
|
20
|
+
super(normalize(key), value)
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def fetch(key, *args, &block)
|
|
24
|
+
super(normalize(key), *args, &block)
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def delete(key)
|
|
28
|
+
super(normalize(key))
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def key?(key)
|
|
32
|
+
super(normalize(key))
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
private
|
|
36
|
+
|
|
37
|
+
def normalize(key)
|
|
38
|
+
key.to_s.downcase
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
10
42
|
# CONNECT response headers from the last Net::HTTP-based proxied HTTPS request on this thread.
|
|
11
43
|
def self.proxy_connect_response_headers
|
|
12
44
|
Thread.current[:ruby_proxy_headers_connect_headers]
|
|
@@ -22,4 +54,17 @@ module RubyProxyHeaders
|
|
|
22
54
|
raise ArgumentError, "proxy CONNECT header value contains invalid characters (CR, LF, or NUL): #{value.inspect}"
|
|
23
55
|
end
|
|
24
56
|
end
|
|
57
|
+
|
|
58
|
+
# Raises ArgumentError if +host+ or +port+ contain CR, LF, or NUL bytes that
|
|
59
|
+
# would allow HTTP request smuggling when interpolated into a CONNECT line.
|
|
60
|
+
def self.validate_connect_target!(host, port)
|
|
61
|
+
if INVALID_HEADER_VALUE_RE.match?(host.to_s)
|
|
62
|
+
raise ArgumentError,
|
|
63
|
+
"CONNECT target host contains invalid characters (CR, LF, or NUL): #{host.inspect}"
|
|
64
|
+
end
|
|
65
|
+
if INVALID_HEADER_VALUE_RE.match?(port.to_s)
|
|
66
|
+
raise ArgumentError,
|
|
67
|
+
"CONNECT target port contains invalid characters (CR, LF, or NUL): #{port.inspect}"
|
|
68
|
+
end
|
|
69
|
+
end
|
|
25
70
|
end
|