ruby-requests 0.0.1.a1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +5 -0
- data/.gitlab-ci.yml +37 -0
- data/.rubocop-disables.yml +96 -0
- data/.rubocop.yml +9 -0
- data/Gemfile +9 -0
- data/LICENSE +21 -0
- data/Makefile +17 -0
- data/README.md +151 -0
- data/lib/requests.rb +95 -0
- data/lib/requests/adapters.rb +124 -0
- data/lib/requests/cookies.rb +98 -0
- data/lib/requests/exceptions.rb +32 -0
- data/lib/requests/http_methods.rb +36 -0
- data/lib/requests/logger.rb +37 -0
- data/lib/requests/request.rb +94 -0
- data/lib/requests/response.rb +93 -0
- data/lib/requests/session.rb +120 -0
- data/lib/requests/utils.rb +162 -0
- data/lib/requests/version.rb +4 -0
- data/ruby-requests.gemspec +23 -0
- data/spec/docker-compose.yml +23 -0
- data/spec/docker_test_entrypoint.sh +12 -0
- data/spec/docker_tests.sh +6 -0
- data/spec/functional/proxies/proxies_spec.rb +84 -0
- data/spec/functional/proxies/squid_conf/passwords +2 -0
- data/spec/functional/proxies/squid_conf/squid.conf +11 -0
- data/spec/functional/requests_spec.rb +326 -0
- data/spec/functional/session_spec.rb +111 -0
- data/spec/spec_helper.rb +44 -0
- data/spec/unit/api_spec.rb +42 -0
- data/spec/unit/utils_spec.rb +112 -0
- metadata +131 -0
@@ -0,0 +1,120 @@
|
|
1
|
+
module Requests
|
2
|
+
class Session
|
3
|
+
include HttpMethods
|
4
|
+
|
5
|
+
attr_accessor :headers, :cookies, :auth, :proxies, :params, :verify,
|
6
|
+
:cert, :stream, :max_redirects, :trust_env
|
7
|
+
attr_reader :logger
|
8
|
+
private :logger
|
9
|
+
|
10
|
+
DEFAULT_REDIRECT_LIMIT = 30
|
11
|
+
|
12
|
+
def initialize
|
13
|
+
@logger = Requests.logger
|
14
|
+
@headers = Utils.default_headers
|
15
|
+
@cookies = CookieJar.new
|
16
|
+
@proxies = {}
|
17
|
+
@params = {}
|
18
|
+
@verify = true
|
19
|
+
@stream = false
|
20
|
+
@trust_env = true
|
21
|
+
@max_redirects = DEFAULT_REDIRECT_LIMIT
|
22
|
+
end
|
23
|
+
|
24
|
+
def request(method, url, params: {}, data: nil, headers: {},
|
25
|
+
cookies: {}, files: nil, proxies: {}, stream: false,
|
26
|
+
auth: nil, timeout: nil, json: nil, allow_redirects: true,
|
27
|
+
verify: true)
|
28
|
+
headers = @headers.merge(headers)
|
29
|
+
req = Request.new(method.upcase,
|
30
|
+
url,
|
31
|
+
headers: headers,
|
32
|
+
files: files,
|
33
|
+
params: params,
|
34
|
+
data: data,
|
35
|
+
json: json,
|
36
|
+
cookies: cookies,
|
37
|
+
auth: auth)
|
38
|
+
|
39
|
+
prepare_request(req)
|
40
|
+
settings = merge_environment_settings(
|
41
|
+
req.url, proxies, stream, verify, cert
|
42
|
+
)
|
43
|
+
send_kwargs = {
|
44
|
+
:timeout => timeout,
|
45
|
+
:allow_redirects => allow_redirects,
|
46
|
+
}.merge(settings)
|
47
|
+
send_request(req, **send_kwargs)
|
48
|
+
end
|
49
|
+
|
50
|
+
def prepare_request(request)
|
51
|
+
merged_cookies = CookieJar.new
|
52
|
+
merged_cookies.merge!(@cookies)
|
53
|
+
merged_cookies.merge!(request.cookies)
|
54
|
+
|
55
|
+
request.headers = merge_setting(request.headers, @headers,
|
56
|
+
Utils::InsensitiveDict)
|
57
|
+
request.params = merge_setting(request.params, @params)
|
58
|
+
request.auth = merge_setting(request.auth, @auth)
|
59
|
+
request.cookies = merged_cookies
|
60
|
+
request.prepare
|
61
|
+
request
|
62
|
+
end
|
63
|
+
|
64
|
+
def send_request(request, **kwargs)
|
65
|
+
kwargs[:stream] = kwargs.fetch(:stream, @stream)
|
66
|
+
kwargs[:verify] = kwargs.fetch(:verify, @verify)
|
67
|
+
kwargs[:cert] = kwargs.fetch(:cert, @cert)
|
68
|
+
kwargs[:proxies] = kwargs.fetch(:proxies, @proxies)
|
69
|
+
|
70
|
+
## Use the following line once implementing redirects:
|
71
|
+
# allow_redirects = kwargs.delete(:allow_redirects) || true
|
72
|
+
kwargs.delete(:allow_redirects)
|
73
|
+
|
74
|
+
response = adapter.send_request(request, **kwargs)
|
75
|
+
@cookies.merge!(response.cookies)
|
76
|
+
response
|
77
|
+
end
|
78
|
+
|
79
|
+
def merge_environment_settings(_url, proxies, stream, verify, cert)
|
80
|
+
# TODO: Implement actually using environment settings
|
81
|
+
proxies = merge_setting(proxies, @proxies)
|
82
|
+
stream = merge_setting(stream, @stream)
|
83
|
+
verify = merge_setting(verify, @verify)
|
84
|
+
cert = merge_setting(cert, @cert)
|
85
|
+
|
86
|
+
{verify: verify,
|
87
|
+
proxies: proxies,
|
88
|
+
stream: stream,
|
89
|
+
cert: cert}
|
90
|
+
end
|
91
|
+
|
92
|
+
##
|
93
|
+
# Determine appropriate setting for a given request, taking into account
|
94
|
+
# the explicit setting on that request, and the setting in the session.
|
95
|
+
# If a is a Hash or InsensitiveDict they will be merged together using
|
96
|
+
# `dict_class`
|
97
|
+
|
98
|
+
def merge_setting(request_setting, session_setting, dict_class=Hash)
|
99
|
+
return request_setting if session_setting.nil?
|
100
|
+
return session_setting if request_setting.nil?
|
101
|
+
|
102
|
+
map_types = [Hash, Requests::Utils::InsensitiveDict]
|
103
|
+
if !(map_types.include?(request_setting.class) &&
|
104
|
+
map_types.include?(session_setting.class))
|
105
|
+
return request_setting
|
106
|
+
end
|
107
|
+
|
108
|
+
merged_setting = dict_class.new.merge(session_setting)
|
109
|
+
merged_setting.merge!(request_setting)
|
110
|
+
|
111
|
+
nil_keys = merged_setting.select { |_k, v| v.nil? }.map(&:first)
|
112
|
+
nil_keys.each { |key| merged_setting.delete(key) }
|
113
|
+
merged_setting
|
114
|
+
end
|
115
|
+
|
116
|
+
def adapter
|
117
|
+
HTTPAdapter.new
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
@@ -0,0 +1,162 @@
|
|
1
|
+
require 'forwardable'
|
2
|
+
|
3
|
+
|
4
|
+
module Requests
|
5
|
+
module Utils
|
6
|
+
##
|
7
|
+
# Dictionary that stores keys as case-insensitive strings.
|
8
|
+
# Only Strings can be used as keys in this structure; attempts
|
9
|
+
# to use anything else as a key will raise a TypeError.
|
10
|
+
|
11
|
+
class InsensitiveDict
|
12
|
+
attr_accessor :store, :dict
|
13
|
+
|
14
|
+
extend Forwardable
|
15
|
+
include Enumerable
|
16
|
+
|
17
|
+
def_delegators :@dict, :each
|
18
|
+
|
19
|
+
def initialize(init_dict={})
|
20
|
+
@store = {}
|
21
|
+
@dict = {}
|
22
|
+
init_dict.each do |k, v|
|
23
|
+
self[k] = v
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
##
|
28
|
+
# Can compare to another InsensitiveDict or to a regular Hash.
|
29
|
+
|
30
|
+
def ==(compare)
|
31
|
+
compare.each do |k, v|
|
32
|
+
return false if !k.is_a?(String) || self[k.downcase] != v
|
33
|
+
end
|
34
|
+
true
|
35
|
+
end
|
36
|
+
|
37
|
+
def []=(k, v)
|
38
|
+
check_key_type(k)
|
39
|
+
@store[k.downcase] = [k, v]
|
40
|
+
@dict[k.downcase] = v
|
41
|
+
end
|
42
|
+
|
43
|
+
def [](k)
|
44
|
+
check_key_type(k)
|
45
|
+
@dict[k.downcase]
|
46
|
+
end
|
47
|
+
|
48
|
+
def merge!(hash)
|
49
|
+
hash.each do |k, v|
|
50
|
+
self[k] = v
|
51
|
+
end
|
52
|
+
self
|
53
|
+
end
|
54
|
+
|
55
|
+
def merge(new)
|
56
|
+
hash = to_h
|
57
|
+
new.each do |k, v|
|
58
|
+
hash[k] = v
|
59
|
+
end
|
60
|
+
hash
|
61
|
+
end
|
62
|
+
|
63
|
+
def to_h
|
64
|
+
hash = {}
|
65
|
+
@store.each_value do |i|
|
66
|
+
orig, value = i
|
67
|
+
hash[orig] = value
|
68
|
+
end
|
69
|
+
hash
|
70
|
+
end
|
71
|
+
|
72
|
+
def inspect
|
73
|
+
to_h.inspect
|
74
|
+
end
|
75
|
+
|
76
|
+
def to_s
|
77
|
+
to_h.to_s
|
78
|
+
end
|
79
|
+
|
80
|
+
def delete(key)
|
81
|
+
if key.is_a?(String)
|
82
|
+
key = key.downcase
|
83
|
+
@store.delete(key)
|
84
|
+
@dict.delete(key)
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
private def check_key_type(key)
|
89
|
+
if !key.is_a?(String)
|
90
|
+
err = "Key for #{self.class.name} object " \
|
91
|
+
"must be String, not #{key.class.name}"
|
92
|
+
raise(TypeError, err)
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
def self.default_user_agent
|
98
|
+
"ruby-requests/#{Requests::VERSION}"
|
99
|
+
end
|
100
|
+
|
101
|
+
def self.default_accept_encoding
|
102
|
+
'gzip, deflate'
|
103
|
+
end
|
104
|
+
|
105
|
+
def self.default_headers
|
106
|
+
headers = {
|
107
|
+
'User-Agent' => default_user_agent,
|
108
|
+
'Accept-Encoding' => default_accept_encoding,
|
109
|
+
'Accept' => '*/*',
|
110
|
+
'Connection' => 'keep-alive',
|
111
|
+
}
|
112
|
+
InsensitiveDict.new(headers)
|
113
|
+
end
|
114
|
+
|
115
|
+
module HeadersMixin
|
116
|
+
##
|
117
|
+
# Set @headers to InsensitiveDict keyed by String with Strings as
|
118
|
+
# values. When passed the result of Net::HTTPResponse.to_hash,
|
119
|
+
# which is a Hash keyed by String with Arrays as values, it will
|
120
|
+
# join the elements in the Array by ', ' as the header value.
|
121
|
+
# 'Set-Cookie' is handled specially because a comma is valid in a
|
122
|
+
# cookie definition, so it is joined by '; ' instead
|
123
|
+
|
124
|
+
def headers=(hash)
|
125
|
+
@headers = Utils::InsensitiveDict.new
|
126
|
+
hash.each do |k, v|
|
127
|
+
delimiter = ', '
|
128
|
+
@headers[k] = v.is_a?(Array) ? v.join(delimiter) : v
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
132
|
+
def headers
|
133
|
+
@headers
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
##
|
138
|
+
# Given a string indicating an HTTP method, return the
|
139
|
+
# corresponding Net::HTTP class
|
140
|
+
|
141
|
+
def self.http_method_class(method)
|
142
|
+
{
|
143
|
+
'GET' => Net::HTTP::Get,
|
144
|
+
'POST' => Net::HTTP::Post,
|
145
|
+
'PUT' => Net::HTTP::Put,
|
146
|
+
'DELETE' => Net::HTTP::Delete,
|
147
|
+
'HEAD' => Net::HTTP::Head,
|
148
|
+
'OPTIONS' => Net::HTTP::Options,
|
149
|
+
'PATCH' => Net::HTTP::Patch,
|
150
|
+
}[method.upcase]
|
151
|
+
end
|
152
|
+
|
153
|
+
def self.charset_from_content_type(header)
|
154
|
+
charset = nil
|
155
|
+
if header
|
156
|
+
match = header.match('charset=([^\s;]+)')
|
157
|
+
charset = match.captures[0] if match
|
158
|
+
end
|
159
|
+
charset
|
160
|
+
end
|
161
|
+
end
|
162
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
require File.expand_path('lib/requests/version', __dir__)
|
2
|
+
|
3
|
+
|
4
|
+
Gem::Specification.new do |s|
|
5
|
+
s.name = 'ruby-requests'
|
6
|
+
s.version = Requests::VERSION
|
7
|
+
s.authors = ['Daniel Hones']
|
8
|
+
s.description = "A library for HTTP requests that aims to provide " \
|
9
|
+
"the same API as the Requests library in Python"
|
10
|
+
s.summary = 'A Ruby version of the Requests library for Python'
|
11
|
+
|
12
|
+
s.license = 'MIT'
|
13
|
+
s.homepage = 'https://gitlab.com/danielhones/ruby_requests'
|
14
|
+
|
15
|
+
s.files = `git ls-files`.split("\n")
|
16
|
+
s.test_files = `git ls-files -- {spec}/**/*`.split("\n")
|
17
|
+
s.required_ruby_version = ">= 2.3.0"
|
18
|
+
|
19
|
+
s.add_dependency('http-cookie', '~> 1.0')
|
20
|
+
s.add_development_dependency('rspec', '~> 3.7')
|
21
|
+
s.add_development_dependency('rubocop', '~> 0.54.0')
|
22
|
+
s.add_development_dependency('simplecov', '~> 0.15')
|
23
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
version: '3'
|
2
|
+
|
3
|
+
services:
|
4
|
+
proxy:
|
5
|
+
image: sameersbn/squid
|
6
|
+
volumes:
|
7
|
+
- ./functional/proxies/squid_conf:/etc/squid3
|
8
|
+
- ./functional/proxies/squid_logs:/var/log/squid3
|
9
|
+
ports:
|
10
|
+
- "3128:3128"
|
11
|
+
|
12
|
+
ruby:
|
13
|
+
image: ruby
|
14
|
+
command: /code/spec/docker_test_entrypoint.sh
|
15
|
+
environment:
|
16
|
+
- REPO_ROOT_DIR=/code
|
17
|
+
- RUNNING_IN_DOCKER=true
|
18
|
+
- PROXY_HOST=proxy
|
19
|
+
- PROXY_PORT=3128
|
20
|
+
volumes:
|
21
|
+
- ../:/code
|
22
|
+
depends_on:
|
23
|
+
- "proxy"
|
@@ -0,0 +1,12 @@
|
|
1
|
+
#!/bin/bash
|
2
|
+
#
|
3
|
+
# This is the script used as the entrypoint for running proxy
|
4
|
+
# tests from within a docker container as part of docker-compose
|
5
|
+
|
6
|
+
cd $REPO_ROOT_DIR
|
7
|
+
gem install -g Gemfile
|
8
|
+
rspec spec/
|
9
|
+
result=$?
|
10
|
+
|
11
|
+
echo "ENTRYPOINT: Rspec returned, exiting with code $result"
|
12
|
+
exit $result
|
@@ -0,0 +1,84 @@
|
|
1
|
+
require_relative '../../spec_helper'
|
2
|
+
|
3
|
+
##
|
4
|
+
# These tests will run from inside the docker container named
|
5
|
+
# "ruby", set up by the docker-compose called in spec/docker_tests.sh
|
6
|
+
|
7
|
+
RSpec.describe 'proxies', :if => running_in_docker do
|
8
|
+
Requests.enable_dev_mode
|
9
|
+
|
10
|
+
let(:proxy) { ENV['PROXY_HOST'] + ":" + ENV['PROXY_PORT'] }
|
11
|
+
|
12
|
+
# These are hard-coded in the squid_conf/passwords file which gets
|
13
|
+
# placed on the squid proxy container:
|
14
|
+
let(:proxy_user) { "foobar" }
|
15
|
+
let(:proxy_pass) { "SecretPass" }
|
16
|
+
|
17
|
+
before(:each) do
|
18
|
+
# The squid_logs directory is shared in a volume with both
|
19
|
+
# the proxy and ruby docker containers:
|
20
|
+
@proxy_log_file = File.join(__dir__, 'squid_logs/access.log')
|
21
|
+
if File.exist?(@proxy_log_file)
|
22
|
+
@proxy_log = File.open(@proxy_log_file)
|
23
|
+
@proxy_log.read
|
24
|
+
end
|
25
|
+
proxy_creds = "http://#{proxy_user}:#{proxy_pass}@#{proxy}"
|
26
|
+
proxy_no_creds = "http://#{proxy}"
|
27
|
+
@proxies = {'http' => proxy_creds, 'https' => proxy_creds}
|
28
|
+
@proxies_no_creds = {'http' => proxy_no_creds,
|
29
|
+
'https' => proxy_no_creds}
|
30
|
+
end
|
31
|
+
|
32
|
+
after(:each) do
|
33
|
+
@proxy_log.close if @proxy_log
|
34
|
+
end
|
35
|
+
|
36
|
+
context "HTTP" do
|
37
|
+
before(:each) do
|
38
|
+
@url = HTTPBIN_URLS['HTTP'] + '/anything'
|
39
|
+
end
|
40
|
+
|
41
|
+
it 'through proxy' do
|
42
|
+
r = Requests.get(@url, proxies: @proxies)
|
43
|
+
sleep(1) # wait for log to be written to file
|
44
|
+
new_log_entries = @proxy_log.readlines
|
45
|
+
expect(r.status_code).to eq(200)
|
46
|
+
msg = /GET #{@url} #{proxy_user}/
|
47
|
+
expect(new_log_entries.any? { |x| x =~ msg }).to eq(true)
|
48
|
+
end
|
49
|
+
|
50
|
+
it "through proxy, wrong credentials returns 407 response" do
|
51
|
+
r = Requests.get(@url, proxies: @proxies_no_creds)
|
52
|
+
sleep(1) # wait for log to be written to file
|
53
|
+
new_log_entries = @proxy_log.readlines
|
54
|
+
expect(r.status_code).to eq(407)
|
55
|
+
msg = /TCP_DENIED\/407 .* GET #{@url}/
|
56
|
+
expect(new_log_entries.any? { |x| x =~ msg }).to eq(true)
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
context "HTTPS" do
|
61
|
+
before(:each) do
|
62
|
+
@url = HTTPBIN_URLS['HTTPS'] + '/anything'
|
63
|
+
end
|
64
|
+
|
65
|
+
it 'through proxy' do
|
66
|
+
r = Requests.get(@url, proxies: @proxies)
|
67
|
+
sleep(1) # wait for log to be written to file
|
68
|
+
new_log_entries = @proxy_log.readlines
|
69
|
+
expect(r.status_code).to eq(200)
|
70
|
+
msg = /CONNECT #{URI(@url).host}/
|
71
|
+
expect(new_log_entries.any? { |x| x =~ msg }).to eq(true)
|
72
|
+
end
|
73
|
+
|
74
|
+
it "through proxy, wrong credentials raises ProxyAuthError" do
|
75
|
+
expect {
|
76
|
+
Requests.get(@url, proxies: @proxies_no_creds)
|
77
|
+
}.to raise_error(Requests::ProxyAuthError)
|
78
|
+
sleep(1) # wait for log to be written to file
|
79
|
+
new_log_entries = @proxy_log.readlines
|
80
|
+
msg = /TCP_DENIED\/407 .* CONNECT #{URI(@url).host}/
|
81
|
+
expect(new_log_entries.any? { |x| x =~ msg }).to eq(true)
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|