requests_ruby 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +63 -0
- data/LICENSE +21 -0
- data/README.md +331 -0
- data/lib/cacert.pem +5689 -0
- data/lib/requests/adapters.rb +114 -0
- data/lib/requests/api.rb +25 -0
- data/lib/requests/auth.rb +74 -0
- data/lib/requests/cookies.rb +55 -0
- data/lib/requests/exceptions.rb +56 -0
- data/lib/requests/models.rb +118 -0
- data/lib/requests/sessions.rb +222 -0
- data/lib/requests/status_codes.rb +31 -0
- data/lib/requests/structures.rb +54 -0
- data/lib/requests/utils.rb +55 -0
- data/lib/requests/version.rb +5 -0
- data/lib/requests.rb +19 -0
- data/lib/requests_ruby.rb +3 -0
- metadata +75 -0
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'net/http'
|
|
4
|
+
require 'openssl'
|
|
5
|
+
require 'uri'
|
|
6
|
+
require 'zlib'
|
|
7
|
+
require 'stringio'
|
|
8
|
+
|
|
9
|
+
module Requests
|
|
10
|
+
class HTTPAdapter
|
|
11
|
+
METHS = {
|
|
12
|
+
'GET' => Net::HTTP::Get,
|
|
13
|
+
'POST' => Net::HTTP::Post,
|
|
14
|
+
'PUT' => Net::HTTP::Put,
|
|
15
|
+
'PATCH' => Net::HTTP::Patch,
|
|
16
|
+
'DELETE' => Net::HTTP::Delete,
|
|
17
|
+
'HEAD' => Net::HTTP::Head,
|
|
18
|
+
'OPTIONS' => Net::HTTP::Options}.freeze
|
|
19
|
+
attr_accessor :max_retries, :backoff_factor
|
|
20
|
+
def initialize(max_retries: 0, backoff_factor: 0)
|
|
21
|
+
@max_retries = max_retries
|
|
22
|
+
@backoff_factor = backoff_factor
|
|
23
|
+
end
|
|
24
|
+
def send_once(meth, url, hdrs, body, timeout, proxies, verify, cert, auth)
|
|
25
|
+
attempt = 0
|
|
26
|
+
begin
|
|
27
|
+
do_send(meth, url, hdrs, body, timeout, proxies, verify, cert, auth)
|
|
28
|
+
rescue Requests::ConnectionError, Requests::ConnectTimeout => e
|
|
29
|
+
attempt += 1
|
|
30
|
+
if attempt <= @max_retries && idempotent?(meth)
|
|
31
|
+
sleep(@backoff_factor * attempt) if @backoff_factor.to_f > 0
|
|
32
|
+
retry
|
|
33
|
+
end
|
|
34
|
+
raise e
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
def decode_body(net_resp)
|
|
38
|
+
raw = net_resp.body
|
|
39
|
+
return '' if raw.nil?
|
|
40
|
+
case net_resp['content-encoding'].to_s.downcase
|
|
41
|
+
when 'gzip', 'x-gzip'
|
|
42
|
+
Zlib::GzipReader.new(StringIO.new(raw)).read
|
|
43
|
+
when 'deflate'
|
|
44
|
+
begin
|
|
45
|
+
Zlib::Inflate.inflate(raw)
|
|
46
|
+
rescue Zlib::DataError
|
|
47
|
+
Zlib::Inflate.new(-Zlib::MAX_WBITS).inflate(raw)
|
|
48
|
+
end
|
|
49
|
+
else
|
|
50
|
+
raw
|
|
51
|
+
end
|
|
52
|
+
rescue Zlib::Error => e
|
|
53
|
+
raise Requests::ContentDecodingError, "couldn't decode response body: #{e.message}"
|
|
54
|
+
end
|
|
55
|
+
private
|
|
56
|
+
def idempotent?(meth)
|
|
57
|
+
%w[GET HEAD OPTIONS PUT DELETE].include?(meth.to_s.upcase)
|
|
58
|
+
end
|
|
59
|
+
def do_send(meth, url, hdrs, body, timeout, proxies, verify, cert, auth)
|
|
60
|
+
u = URI.parse(url)
|
|
61
|
+
auth.call(hdrs) if auth.respond_to?(:call) && !auth.is_a?(Requests::DigestAuth)
|
|
62
|
+
popt = proxy_opts(u, proxies)
|
|
63
|
+
open_t, read_t = split_timeout(timeout)
|
|
64
|
+
http = Net::HTTP.new(u.host, u.port, *popt)
|
|
65
|
+
configure_ssl(http, u, verify, cert)
|
|
66
|
+
http.open_timeout = open_t if open_t
|
|
67
|
+
http.read_timeout = read_t if read_t
|
|
68
|
+
begin
|
|
69
|
+
http.start do |h|
|
|
70
|
+
klass = METHS[meth] || Net::HTTP::Get
|
|
71
|
+
req = klass.new(u.request_uri)
|
|
72
|
+
hdrs.each { |k, v| req[k] = v }
|
|
73
|
+
req.body = body if body
|
|
74
|
+
h.request(req)
|
|
75
|
+
end
|
|
76
|
+
rescue Net::OpenTimeout
|
|
77
|
+
raise Requests::ConnectTimeout, "connect timeout: #{url}"
|
|
78
|
+
rescue Net::ReadTimeout
|
|
79
|
+
raise Requests::ReadTimeout, "read timeout: #{url}"
|
|
80
|
+
rescue OpenSSL::SSL::SSLError => e
|
|
81
|
+
raise Requests::SSLError, e.message
|
|
82
|
+
rescue SocketError, Errno::ECONNREFUSED, Errno::EHOSTUNREACH, Errno::ETIMEDOUT, Errno::ECONNRESET => e
|
|
83
|
+
raise Requests::ConnectionError, e.message
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
def configure_ssl(http, uri, verify, cert)
|
|
87
|
+
return unless uri.scheme == 'https'
|
|
88
|
+
http.use_ssl = true
|
|
89
|
+
if verify == false
|
|
90
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
|
91
|
+
else
|
|
92
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_PEER
|
|
93
|
+
http.ca_file = verify.is_a?(String) ? verify : Requests::CA_FILE
|
|
94
|
+
end
|
|
95
|
+
return unless cert
|
|
96
|
+
http.cert = OpenSSL::X509::Certificate.new(File.read(cert[0]))
|
|
97
|
+
http.key = OpenSSL::PKey::RSA.new(File.read(cert[1]))
|
|
98
|
+
end
|
|
99
|
+
def proxy_opts(uri, proxies)
|
|
100
|
+
return [] unless proxies && !proxies.empty?
|
|
101
|
+
proxy_url = proxies[uri.scheme] || proxies[uri.scheme.to_sym]
|
|
102
|
+
return [] unless proxy_url
|
|
103
|
+
pu = URI.parse(proxy_url)
|
|
104
|
+
[pu.host, pu.port, pu.user, pu.password]
|
|
105
|
+
end
|
|
106
|
+
def split_timeout(t)
|
|
107
|
+
case t
|
|
108
|
+
when nil then [nil, nil]
|
|
109
|
+
when Array then [t[0], t[1]]
|
|
110
|
+
else [t, t]
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
end
|
data/lib/requests/api.rb
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Requests
|
|
4
|
+
def self.request(method, url, **kw)
|
|
5
|
+
Session.new.request(method, url, **kw)
|
|
6
|
+
end
|
|
7
|
+
def self.get(url, **kw); request('GET', url, **kw); end
|
|
8
|
+
def self.post(url, **kw); request('POST', url, **kw); end
|
|
9
|
+
def self.put(url, **kw); request('PUT', url, **kw); end
|
|
10
|
+
def self.patch(url, **kw); request('PATCH', url, **kw); end
|
|
11
|
+
def self.delete(url, **kw); request('DELETE', url, **kw); end
|
|
12
|
+
def self.options(url, **kw); request('OPTIONS', url, **kw); end
|
|
13
|
+
def self.head(url, **kw)
|
|
14
|
+
kw[:allow_redirects] = kw.fetch(:allow_redirects, false)
|
|
15
|
+
request('HEAD', url, **kw)
|
|
16
|
+
end
|
|
17
|
+
def self.session
|
|
18
|
+
Session.new
|
|
19
|
+
end
|
|
20
|
+
def self.download(url, to:, **kw)
|
|
21
|
+
resp = get(url, **kw)
|
|
22
|
+
resp.raise_for_status
|
|
23
|
+
resp.save_to(to)
|
|
24
|
+
end
|
|
25
|
+
end
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'base64'
|
|
4
|
+
require 'digest/md5'
|
|
5
|
+
require 'securerandom'
|
|
6
|
+
|
|
7
|
+
module Requests
|
|
8
|
+
module AuthBase
|
|
9
|
+
def call(_headers)
|
|
10
|
+
raise NotImplementedError, 'subclass must implement #call'
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
class BasicAuth
|
|
14
|
+
include AuthBase
|
|
15
|
+
def initialize(user, pass)
|
|
16
|
+
@user = user
|
|
17
|
+
@pass = pass
|
|
18
|
+
end
|
|
19
|
+
def call(hdrs)
|
|
20
|
+
hdrs['Authorization'] = 'Basic ' + Base64.strict_encode64("#{@user}:#{@pass}")
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
class BearerAuth
|
|
24
|
+
include AuthBase
|
|
25
|
+
def initialize(token)
|
|
26
|
+
@token = token
|
|
27
|
+
end
|
|
28
|
+
def call(hdrs)
|
|
29
|
+
hdrs['Authorization'] = "Bearer #{@token}"
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
class DigestAuth
|
|
33
|
+
include AuthBase
|
|
34
|
+
def initialize(user, pass)
|
|
35
|
+
@user = user
|
|
36
|
+
@pass = pass
|
|
37
|
+
end
|
|
38
|
+
def call(hdrs, meth: nil, url: nil, prev_resp: nil)
|
|
39
|
+
return unless prev_resp
|
|
40
|
+
wa = prev_resp.headers['www-authenticate']
|
|
41
|
+
return unless wa && wa =~ /\ADigest/
|
|
42
|
+
pr = parse_challenge(wa)
|
|
43
|
+
realm = pr['realm']
|
|
44
|
+
nonce = pr['nonce']
|
|
45
|
+
qop = pr['qop']
|
|
46
|
+
path = URI.parse(url).request_uri
|
|
47
|
+
ha1 = Digest::MD5.hexdigest("#{@user}:#{realm}:#{@pass}")
|
|
48
|
+
ha2 = Digest::MD5.hexdigest("#{meth}:#{path}")
|
|
49
|
+
if qop
|
|
50
|
+
nc = '00000001'
|
|
51
|
+
cnonce = SecureRandom.hex(8)
|
|
52
|
+
response = Digest::MD5.hexdigest("#{ha1}:#{nonce}:#{nc}:#{cnonce}:#{qop}:#{ha2}")
|
|
53
|
+
hdrs['Authorization'] =
|
|
54
|
+
"Digest username=\"#{@user}\", realm=\"#{realm}\", nonce=\"#{nonce}\", " \
|
|
55
|
+
"uri=\"#{path}\", qop=#{qop}, nc=#{nc}, cnonce=\"#{cnonce}\", response=\"#{response}\""
|
|
56
|
+
else
|
|
57
|
+
response = Digest::MD5.hexdigest("#{ha1}:#{nonce}:#{ha2}")
|
|
58
|
+
hdrs['Authorization'] =
|
|
59
|
+
"Digest username=\"#{@user}\", realm=\"#{realm}\", nonce=\"#{nonce}\", " \
|
|
60
|
+
"uri=\"#{path}\", response=\"#{response}\""
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
private
|
|
64
|
+
def parse_challenge(www_authenticate)
|
|
65
|
+
pr = {}
|
|
66
|
+
www_authenticate.sub(/\ADigest\s*/, '').split(',').each do |pair|
|
|
67
|
+
k, v = pair.strip.split('=', 2)
|
|
68
|
+
next unless k
|
|
69
|
+
pr[k] = v.to_s.gsub('"', '')
|
|
70
|
+
end
|
|
71
|
+
pr
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
end
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Requests
|
|
4
|
+
class Jar
|
|
5
|
+
include Enumerable
|
|
6
|
+
def initialize
|
|
7
|
+
@c = {}
|
|
8
|
+
end
|
|
9
|
+
def [](n)
|
|
10
|
+
@c[n]
|
|
11
|
+
end
|
|
12
|
+
def []=(n, v)
|
|
13
|
+
@c[n] = v
|
|
14
|
+
end
|
|
15
|
+
def get(name, default = nil)
|
|
16
|
+
@c.fetch(name, default)
|
|
17
|
+
end
|
|
18
|
+
def set(name, value)
|
|
19
|
+
@c[name] = value
|
|
20
|
+
self
|
|
21
|
+
end
|
|
22
|
+
def delete(name)
|
|
23
|
+
@c.delete(name)
|
|
24
|
+
end
|
|
25
|
+
def clear
|
|
26
|
+
@c.clear
|
|
27
|
+
end
|
|
28
|
+
def to_h
|
|
29
|
+
@c.dup
|
|
30
|
+
end
|
|
31
|
+
def to_a
|
|
32
|
+
@c.to_a
|
|
33
|
+
end
|
|
34
|
+
def each(&b)
|
|
35
|
+
@c.each(&b)
|
|
36
|
+
end
|
|
37
|
+
def empty?
|
|
38
|
+
@c.empty?
|
|
39
|
+
end
|
|
40
|
+
def to_header
|
|
41
|
+
@c.map { |k, v| "#{k}=#{v}" }.join('; ')
|
|
42
|
+
end
|
|
43
|
+
def update(net_resp)
|
|
44
|
+
lines = net_resp.get_fields('set-cookie') || []
|
|
45
|
+
lines.each do |l|
|
|
46
|
+
kv = l.split(';').first
|
|
47
|
+
next unless kv
|
|
48
|
+
k, v = kv.split('=', 2)
|
|
49
|
+
next unless k
|
|
50
|
+
@c[k.strip] = v.to_s.strip
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
CookieJar = Jar
|
|
55
|
+
end
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Requests
|
|
4
|
+
module Timeoutable; end
|
|
5
|
+
class RequestException < StandardError
|
|
6
|
+
attr_reader :response, :request
|
|
7
|
+
def initialize(msg = nil, response: nil, request: nil)
|
|
8
|
+
@response = response
|
|
9
|
+
@request = request
|
|
10
|
+
super(msg || 'request error')
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
class InvalidJSONError < RequestException
|
|
15
|
+
end
|
|
16
|
+
class JSONDecodeError < InvalidJSONError
|
|
17
|
+
end
|
|
18
|
+
class HTTPError < RequestException
|
|
19
|
+
end
|
|
20
|
+
class ConnectionError < RequestException
|
|
21
|
+
end
|
|
22
|
+
class ProxyError < ConnectionError
|
|
23
|
+
end
|
|
24
|
+
class SSLError < ConnectionError
|
|
25
|
+
end
|
|
26
|
+
class Timeout < RequestException
|
|
27
|
+
include Timeoutable
|
|
28
|
+
end
|
|
29
|
+
class ConnectTimeout < ConnectionError
|
|
30
|
+
include Timeoutable
|
|
31
|
+
end
|
|
32
|
+
class ReadTimeout < Timeout
|
|
33
|
+
end
|
|
34
|
+
class URLRequired < RequestException
|
|
35
|
+
end
|
|
36
|
+
class TooManyRedirects < RequestException
|
|
37
|
+
end
|
|
38
|
+
class MissingSchema < RequestException
|
|
39
|
+
end
|
|
40
|
+
class InvalidSchema < RequestException
|
|
41
|
+
end
|
|
42
|
+
class InvalidURL < RequestException
|
|
43
|
+
end
|
|
44
|
+
class InvalidProxyURL < InvalidURL
|
|
45
|
+
end
|
|
46
|
+
class ChunkedEncodingError < RequestException
|
|
47
|
+
end
|
|
48
|
+
class ContentDecodingError < RequestException
|
|
49
|
+
end
|
|
50
|
+
class StreamConsumedError < RequestException
|
|
51
|
+
end
|
|
52
|
+
class InvalidHeader < RequestException
|
|
53
|
+
end
|
|
54
|
+
class RetryError < RequestException
|
|
55
|
+
end
|
|
56
|
+
end
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
|
|
5
|
+
module Requests
|
|
6
|
+
class PreparedRequest
|
|
7
|
+
attr_reader :method, :url, :headers, :body
|
|
8
|
+
def initialize(method:, url:, headers: nil, body: nil)
|
|
9
|
+
@method = method
|
|
10
|
+
@url = url
|
|
11
|
+
@headers = headers
|
|
12
|
+
@body = body
|
|
13
|
+
end
|
|
14
|
+
def [](key)
|
|
15
|
+
case key.to_sym
|
|
16
|
+
when :method then @method
|
|
17
|
+
when :url then @url
|
|
18
|
+
when :headers then @headers
|
|
19
|
+
when :body then @body
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
def to_h
|
|
23
|
+
{ method: @method, url: @url, headers: @headers, body: @body }
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
class Response
|
|
27
|
+
attr_accessor :status_code, :headers, :raw_body, :url, :history,
|
|
28
|
+
:elapsed, :request, :cookies, :encoding, :reason
|
|
29
|
+
def initialize
|
|
30
|
+
@history = []
|
|
31
|
+
end
|
|
32
|
+
def ok?
|
|
33
|
+
status_code.to_i < 400
|
|
34
|
+
end
|
|
35
|
+
alias_method :ok, :ok?
|
|
36
|
+
def status
|
|
37
|
+
status_code
|
|
38
|
+
end
|
|
39
|
+
def redirect?
|
|
40
|
+
[301, 302, 303, 307, 308].include?(status_code)
|
|
41
|
+
end
|
|
42
|
+
alias_method :is_redirect?, :redirect?
|
|
43
|
+
def permanent_redirect?
|
|
44
|
+
[301, 308].include?(status_code)
|
|
45
|
+
end
|
|
46
|
+
alias_method :is_permanent_redirect?, :permanent_redirect?
|
|
47
|
+
def client_error?
|
|
48
|
+
status_code.to_i.between?(400, 499)
|
|
49
|
+
end
|
|
50
|
+
def server_error?
|
|
51
|
+
status_code.to_i >= 500
|
|
52
|
+
end
|
|
53
|
+
def content
|
|
54
|
+
raw_body
|
|
55
|
+
end
|
|
56
|
+
def text
|
|
57
|
+
enc = encoding || 'UTF-8'
|
|
58
|
+
s = raw_body.to_s.dup
|
|
59
|
+
begin
|
|
60
|
+
s.force_encoding(enc).encode('UTF-8', invalid: :replace, undef: :replace)
|
|
61
|
+
rescue ArgumentError, Encoding::ConverterNotFoundError
|
|
62
|
+
s.force_encoding('UTF-8')
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
def json(**kw)
|
|
66
|
+
JSON.parse(text, **kw)
|
|
67
|
+
rescue JSON::ParserError => e
|
|
68
|
+
raise Requests::JSONDecodeError.new(build_json_error_message(e), response: self)
|
|
69
|
+
end
|
|
70
|
+
def raise_for_status
|
|
71
|
+
return self if ok?
|
|
72
|
+
kind = status_code.to_i >= 500 ? 'Server' : 'Client'
|
|
73
|
+
msg = "#{status_code} #{kind} Error: #{reason} for url: #{url}"
|
|
74
|
+
raise Requests::HTTPError.new(msg, response: self)
|
|
75
|
+
end
|
|
76
|
+
alias_method :raise_for_status!, :raise_for_status
|
|
77
|
+
|
|
78
|
+
def iter_lines(chunk: nil)
|
|
79
|
+
return enum_for(:iter_lines, chunk: chunk) unless block_given?
|
|
80
|
+
text.each_line { |l| yield l.chomp }
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def iter_content(chunk_size: 1024)
|
|
84
|
+
return enum_for(:iter_content, chunk_size: chunk_size) unless block_given?
|
|
85
|
+
s = content.to_s
|
|
86
|
+
i = 0
|
|
87
|
+
while i < s.bytesize
|
|
88
|
+
yield s.byteslice(i, chunk_size)
|
|
89
|
+
i += chunk_size
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
def save_to(path)
|
|
93
|
+
File.open(path, 'wb') { |f| f.write(raw_body.to_s.dup.force_encoding(Encoding::ASCII_8BIT)) }
|
|
94
|
+
path
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def apparent_encoding
|
|
98
|
+
encoding
|
|
99
|
+
end
|
|
100
|
+
def links
|
|
101
|
+
Requests::Utils.parse_header_links(headers['link'])
|
|
102
|
+
end
|
|
103
|
+
private
|
|
104
|
+
def build_json_error_message(err)
|
|
105
|
+
body_preview = Requests::Utils.truncate(text, 200)
|
|
106
|
+
if Requests::Utils.looks_like_html?(text)
|
|
107
|
+
"expected JSON but got an HTML page back (status #{status_code} from #{url}). " \
|
|
108
|
+
"you probably hit an error page or got redirected somewhere unexpected, " \
|
|
109
|
+
"not the JSON endpoint you meant to call. body starts with: #{body_preview.inspect}"
|
|
110
|
+
elsif text.to_s.strip.empty?
|
|
111
|
+
"expected JSON but the response body was empty (status #{status_code} from #{url})"
|
|
112
|
+
else
|
|
113
|
+
"couldn't parse response body as JSON (status #{status_code} from #{url}): " \
|
|
114
|
+
"#{err.message}. body starts with: #{body_preview.inspect}"
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
end
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'securerandom'
|
|
4
|
+
require 'time'
|
|
5
|
+
|
|
6
|
+
module Requests
|
|
7
|
+
class Session
|
|
8
|
+
attr_accessor :headers, :cookies, :auth, :params, :proxies, :verify,
|
|
9
|
+
:cert, :max_redirects, :timeout, :retries, :backoff_factor
|
|
10
|
+
def initialize
|
|
11
|
+
@headers = Requests::Utils.default_headers
|
|
12
|
+
@cookies = Jar.new
|
|
13
|
+
@auth = nil
|
|
14
|
+
@params = nil
|
|
15
|
+
@proxies = {}
|
|
16
|
+
@verify = true
|
|
17
|
+
@cert = nil
|
|
18
|
+
@max_redirects = 30
|
|
19
|
+
@timeout = nil
|
|
20
|
+
@retries = 0
|
|
21
|
+
@backoff_factor = 0
|
|
22
|
+
@hooks = { response: [] }
|
|
23
|
+
@adapters = { 'https://' => HTTPAdapter.new, 'http://' => HTTPAdapter.new }
|
|
24
|
+
end
|
|
25
|
+
def hooks
|
|
26
|
+
@hooks
|
|
27
|
+
end
|
|
28
|
+
def mount(prefix, adapter)
|
|
29
|
+
@adapters[prefix] = adapter
|
|
30
|
+
self
|
|
31
|
+
end
|
|
32
|
+
def get(url, **kw); request('GET', url, **kw); end
|
|
33
|
+
def post(url, **kw); request('POST', url, **kw); end
|
|
34
|
+
def put(url, **kw); request('PUT', url, **kw); end
|
|
35
|
+
def patch(url, **kw); request('PATCH', url, **kw); end
|
|
36
|
+
def delete(url, **kw); request('DELETE', url, **kw); end
|
|
37
|
+
def options(url, **kw); request('OPTIONS', url, **kw); end
|
|
38
|
+
def head(url, **kw)
|
|
39
|
+
kw[:allow_redirects] = kw.fetch(:allow_redirects, false)
|
|
40
|
+
request('HEAD', url, **kw)
|
|
41
|
+
end
|
|
42
|
+
def get!(url, **kw); get(url, **kw).raise_for_status; end
|
|
43
|
+
def post!(url, **kw); post(url, **kw).raise_for_status; end
|
|
44
|
+
def close
|
|
45
|
+
true
|
|
46
|
+
end
|
|
47
|
+
def request(method, url, params: nil, data: nil, json: nil, headers: nil, cookies: nil, files: nil,
|
|
48
|
+
auth: nil, timeout: nil, allow_redirects: true, proxies: nil, verify: nil, stream: false,
|
|
49
|
+
cert: nil, hooks: nil, retries: nil)
|
|
50
|
+
validate_url!(url)
|
|
51
|
+
use_auth = auth || @auth
|
|
52
|
+
merged_params = merge_hash(@params, params)
|
|
53
|
+
original_url = build_url(url, merged_params)
|
|
54
|
+
original_host = URI.parse(original_url).host
|
|
55
|
+
cur_url = original_url
|
|
56
|
+
cur_method = method.to_s.upcase
|
|
57
|
+
hist = []
|
|
58
|
+
redirs = 0
|
|
59
|
+
body, content_type = build_body(data: data, json: json, files: files)
|
|
60
|
+
loop do
|
|
61
|
+
hdrs = CIHash.new(@headers.to_h)
|
|
62
|
+
hdrs.merge!(headers)
|
|
63
|
+
hdrs['Content-Type'] = content_type if content_type && !hdrs.key?('content-type')
|
|
64
|
+
hdrs.delete('Authorization') if redirs > 0 && URI.parse(cur_url).host != original_host
|
|
65
|
+
jar = merge_jar(@cookies, cookies)
|
|
66
|
+
hdrs['Cookie'] = jar.to_header unless jar.empty?
|
|
67
|
+
adapter = adapter_for(cur_url)
|
|
68
|
+
adapter.max_retries = retries.nil? ? @retries : retries
|
|
69
|
+
adapter.backoff_factor = @backoff_factor
|
|
70
|
+
t0 = Time.now
|
|
71
|
+
net_resp = adapter.send_once(cur_method, cur_url, hdrs, body, timeout || @timeout,proxies || @proxies, verify.nil? ? @verify : verify,cert || @cert, use_auth)
|
|
72
|
+
elapsed = Time.now - t0
|
|
73
|
+
resp = to_response(adapter, net_resp, cur_url, elapsed, cur_method, hdrs, body)
|
|
74
|
+
@cookies.update(net_resp)
|
|
75
|
+
resp.cookies = @cookies
|
|
76
|
+
if use_auth.is_a?(DigestAuth) && resp.status_code == 401 && redirs.zero? && hist.empty?
|
|
77
|
+
use_auth.call(hdrs, meth: cur_method, url: cur_url, prev_resp: resp)
|
|
78
|
+
t1 = Time.now
|
|
79
|
+
net_resp = adapter.send_once(cur_method, cur_url, hdrs, body, timeout || @timeout,proxies || @proxies, verify.nil? ? @verify : verify,cert || @cert, nil)
|
|
80
|
+
resp = to_response(adapter, net_resp, cur_url, Time.now - t1, cur_method, hdrs, body)
|
|
81
|
+
@cookies.update(net_resp)
|
|
82
|
+
resp.cookies = @cookies
|
|
83
|
+
end
|
|
84
|
+
hist << resp
|
|
85
|
+
if allow_redirects && resp.redirect? && resp.headers['location']
|
|
86
|
+
redirs += 1
|
|
87
|
+
raise Requests::TooManyRedirects, "too many redirects: #{url}" if redirs > @max_redirects
|
|
88
|
+
cur_url = URI.join(cur_url, resp.headers['location']).to_s
|
|
89
|
+
if resp.status_code == 303 || ([301, 302].include?(resp.status_code) && cur_method == 'POST')
|
|
90
|
+
cur_method = 'GET'
|
|
91
|
+
body = nil
|
|
92
|
+
end
|
|
93
|
+
next
|
|
94
|
+
end
|
|
95
|
+
resp.history = hist[0..-2]
|
|
96
|
+
run_hooks(resp, hooks)
|
|
97
|
+
return resp
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
private
|
|
101
|
+
def run_hooks(resp, per_request_hooks)
|
|
102
|
+
all = Array(@hooks[:response]) + Array(per_request_hooks && per_request_hooks[:response])
|
|
103
|
+
all.each { |h| h.call(resp) }
|
|
104
|
+
end
|
|
105
|
+
def validate_url!(url)
|
|
106
|
+
s = url.to_s
|
|
107
|
+
raise Requests::MissingSchema, "invalid url, no scheme: #{url}" unless s.include?('://')
|
|
108
|
+
raise Requests::InvalidSchema, "unsupported scheme: #{url}" unless s =~ %r{\Ahttps?://}
|
|
109
|
+
end
|
|
110
|
+
def adapter_for(url)
|
|
111
|
+
match = @adapters.keys.select { |prefix| url.start_with?(prefix) }.max_by(&:length)
|
|
112
|
+
match ? @adapters[match] : @adapters['https://']
|
|
113
|
+
end
|
|
114
|
+
def merge_hash(a, b)
|
|
115
|
+
return b unless a
|
|
116
|
+
return a unless b
|
|
117
|
+
a.merge(b)
|
|
118
|
+
end
|
|
119
|
+
def merge_jar(session_jar, extra)
|
|
120
|
+
j = Jar.new
|
|
121
|
+
session_jar.each { |k, v| j[k] = v }
|
|
122
|
+
extra&.each { |k, v| j[k] = v }
|
|
123
|
+
j
|
|
124
|
+
end
|
|
125
|
+
def build_url(base, params)
|
|
126
|
+
return base if params.nil? || (params.respond_to?(:empty?) && params.empty?)
|
|
127
|
+
qs = build_qs(params)
|
|
128
|
+
return base if qs.nil? || qs.empty?
|
|
129
|
+
sep = base.include?('?') ? '&' : '?'
|
|
130
|
+
base + sep + qs
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
def build_qs(p)
|
|
134
|
+
case p
|
|
135
|
+
when String then p
|
|
136
|
+
when Hash
|
|
137
|
+
parts = []
|
|
138
|
+
p.each do |k, v|
|
|
139
|
+
next if v.nil?
|
|
140
|
+
if v.is_a?(Array)
|
|
141
|
+
v.each { |vv| parts << "#{esc(k)}=#{esc(vv)}" }
|
|
142
|
+
else
|
|
143
|
+
parts << "#{esc(k)}=#{esc(v)}"
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
parts.join('&')
|
|
147
|
+
when Array
|
|
148
|
+
p.map { |k, v| "#{esc(k)}=#{esc(v)}" }.join('&')
|
|
149
|
+
else
|
|
150
|
+
''
|
|
151
|
+
end
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
def esc(v)
|
|
155
|
+
URI.encode_www_form_component(v.to_s)
|
|
156
|
+
end
|
|
157
|
+
def build_body(data: nil, json: nil, files: nil)
|
|
158
|
+
if files
|
|
159
|
+
build_multipart(data, files)
|
|
160
|
+
elsif json
|
|
161
|
+
[JSON.generate(json), 'application/json']
|
|
162
|
+
elsif data.is_a?(Hash) || data.is_a?(Array)
|
|
163
|
+
[build_qs(data), 'application/x-www-form-urlencoded']
|
|
164
|
+
elsif data.is_a?(String)
|
|
165
|
+
[data, nil]
|
|
166
|
+
elsif data.respond_to?(:read)
|
|
167
|
+
[data.read, nil]
|
|
168
|
+
else
|
|
169
|
+
[nil, nil]
|
|
170
|
+
end
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
def build_multipart(data, files)
|
|
174
|
+
boundary = "----RubyReq#{SecureRandom.hex(10)}"
|
|
175
|
+
out = String.new(encoding: Encoding::ASCII_8BIT)
|
|
176
|
+
(data || {}).each do |k, v|
|
|
177
|
+
out << "--#{boundary}\r\n"
|
|
178
|
+
out << "Content-Disposition: form-data; name=\"#{k}\"\r\n\r\n"
|
|
179
|
+
out << v.to_s.dup.force_encoding(Encoding::ASCII_8BIT)
|
|
180
|
+
out << "\r\n"
|
|
181
|
+
end
|
|
182
|
+
files.each do |field, val|
|
|
183
|
+
fname, content, ctype = extract_file(field, val)
|
|
184
|
+
out << "--#{boundary}\r\n"
|
|
185
|
+
out << "Content-Disposition: form-data; name=\"#{field}\"; filename=\"#{fname}\"\r\n"
|
|
186
|
+
out << "Content-Type: #{ctype}\r\n\r\n"
|
|
187
|
+
out << content.to_s.dup.force_encoding(Encoding::ASCII_8BIT)
|
|
188
|
+
out << "\r\n"
|
|
189
|
+
end
|
|
190
|
+
out << "--#{boundary}--\r\n"
|
|
191
|
+
[out, "multipart/form-data; boundary=#{boundary}"]
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
def extract_file(field, val)
|
|
195
|
+
if val.is_a?(Array)
|
|
196
|
+
fname = val[0]
|
|
197
|
+
content = val[1].respond_to?(:read) ? val[1].read : val[1].to_s
|
|
198
|
+
ctype = val[2] || 'application/octet-stream'
|
|
199
|
+
[fname, content, ctype]
|
|
200
|
+
elsif val.respond_to?(:read)
|
|
201
|
+
[Requests::Utils.guess_filename(val, field), val.read, 'application/octet-stream']
|
|
202
|
+
else
|
|
203
|
+
[field.to_s, val.to_s, 'application/octet-stream']
|
|
204
|
+
end
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
def to_response(adapter, net_resp, url, elapsed, method, req_headers, req_body)
|
|
208
|
+
r = Response.new
|
|
209
|
+
r.status_code = net_resp.code.to_i
|
|
210
|
+
rh = CIHash.new
|
|
211
|
+
net_resp.each_header { |k, v| rh[k] = v }
|
|
212
|
+
r.headers = rh
|
|
213
|
+
r.raw_body = adapter.decode_body(net_resp)
|
|
214
|
+
r.url = url
|
|
215
|
+
r.elapsed = elapsed
|
|
216
|
+
r.encoding = Requests::Utils.encoding_from_headers(rh)
|
|
217
|
+
r.reason = net_resp.message
|
|
218
|
+
r.request = PreparedRequest.new(method: method, url: url, headers: req_headers.to_h, body: req_body)
|
|
219
|
+
r
|
|
220
|
+
end
|
|
221
|
+
end
|
|
222
|
+
end
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Requests
|
|
4
|
+
STATUS_CODES = {100 => :continue,101 => :switching_protocols,102 => :processing,103 => :early_hints,200 => :ok,201 => :created,202 => :accepted,203 => :non_authoritative_info,204 => :no_content,205 => :reset_content,206 => :partial_content,207 => :multi_status,208 => :already_reported,226 => :im_used,300 => :multiple_choices,301 => :moved_permanently,302 => :found,303 => :see_other,304 => :not_modified,305 => :use_proxy,307 => :temporary_redirect,308 => :permanent_redirect,400 => :bad_request,401 => :unauthorized,402 => :payment_required,403 => :forbidden,404 => :not_found,405 => :method_not_allowed,406 => :not_acceptable,407 => :proxy_authentication_required,408 => :request_timeout,409 => :conflict,410 => :gone,411 => :length_required,412 => :precondition_failed,413 => :payload_too_large,414 => :uri_too_long,415 => :unsupported_media_type,416 => :range_not_satisfiable,417 => :expectation_failed,418 => :im_a_teapot,421 => :misdirected_request,422 => :unprocessable_entity,423 => :locked,424 => :failed_dependency,425 => :too_early,426 => :upgrade_required,428 => :precondition_required,429 => :too_many_requests,431 => :request_header_fields_too_large,451 => :unavailable_for_legal_reasons,500 => :internal_server_error,501 => :not_implemented,502 => :bad_gateway,503 => :service_unavailable,504 => :gateway_timeout,505 => :http_version_not_supported,506 => :variant_also_negotiates,507 => :insufficient_storage,508 => :loop_detected,509 => :bandwidth_limit_exceeded,510 => :not_extended,511 => :network_authentication_required}.freeze
|
|
5
|
+
STATUS_ALIASES = {okay: 200,all_ok: 200,all_good: 200,request_uri_too_long: 414,unprocessable: 422,server_error: 500,unauthorised: 401}.freeze
|
|
6
|
+
class LookupDict
|
|
7
|
+
def initialize(code_map, aliases = {})
|
|
8
|
+
@by_name = {}
|
|
9
|
+
code_map.each { |code, name| @by_name[name] = code }
|
|
10
|
+
aliases.each { |name, code| @by_name[name] ||= code }
|
|
11
|
+
end
|
|
12
|
+
def method_missing(name, *_args)
|
|
13
|
+
@by_name.key?(name) ? @by_name[name] : super
|
|
14
|
+
end
|
|
15
|
+
def respond_to_missing?(name, include_private = false)
|
|
16
|
+
@by_name.key?(name) || super
|
|
17
|
+
end
|
|
18
|
+
def [](name)
|
|
19
|
+
@by_name[name.to_sym]
|
|
20
|
+
end
|
|
21
|
+
def name_for(code)
|
|
22
|
+
STATUS_CODES[code]
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
CODES = LookupDict.new(STATUS_CODES, STATUS_ALIASES)
|
|
27
|
+
|
|
28
|
+
def self.codes
|
|
29
|
+
CODES
|
|
30
|
+
end
|
|
31
|
+
end
|