httpx 0.11.3 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +2 -2
- data/doc/release_notes/0_11_1.md +5 -1
- data/doc/release_notes/0_12_0.md +55 -0
- data/lib/httpx.rb +2 -1
- data/lib/httpx/adapters/faraday.rb +4 -6
- data/lib/httpx/altsvc.rb +1 -0
- data/lib/httpx/connection.rb +63 -15
- data/lib/httpx/connection/http1.rb +8 -7
- data/lib/httpx/connection/http2.rb +32 -25
- data/lib/httpx/io.rb +16 -3
- data/lib/httpx/io/ssl.rb +7 -9
- data/lib/httpx/io/tcp.rb +9 -8
- data/lib/httpx/io/tls.rb +218 -0
- data/lib/httpx/io/tls/box.rb +365 -0
- data/lib/httpx/io/tls/context.rb +199 -0
- data/lib/httpx/io/tls/ffi.rb +390 -0
- data/lib/httpx/parser/http1.rb +4 -4
- data/lib/httpx/plugins/aws_sdk_authentication.rb +81 -0
- data/lib/httpx/plugins/aws_sigv4.rb +218 -0
- data/lib/httpx/plugins/compression/deflate.rb +2 -5
- data/lib/httpx/plugins/internal_telemetry.rb +93 -0
- data/lib/httpx/plugins/multipart.rb +2 -0
- data/lib/httpx/plugins/multipart/encoder.rb +4 -9
- data/lib/httpx/plugins/proxy.rb +1 -1
- data/lib/httpx/plugins/proxy/http.rb +1 -1
- data/lib/httpx/plugins/proxy/socks4.rb +8 -0
- data/lib/httpx/plugins/proxy/socks5.rb +8 -0
- data/lib/httpx/plugins/push_promise.rb +3 -2
- data/lib/httpx/plugins/retries.rb +1 -1
- data/lib/httpx/plugins/stream.rb +3 -5
- data/lib/httpx/pool.rb +0 -1
- data/lib/httpx/registry.rb +1 -7
- data/lib/httpx/request.rb +11 -1
- data/lib/httpx/resolver/https.rb +3 -11
- data/lib/httpx/response.rb +9 -2
- data/lib/httpx/selector.rb +5 -0
- data/lib/httpx/session.rb +25 -2
- data/lib/httpx/transcoder/body.rb +3 -5
- data/lib/httpx/version.rb +1 -1
- data/sig/connection/http1.rbs +2 -2
- data/sig/connection/http2.rbs +5 -3
- data/sig/plugins/aws_sdk_authentication.rbs +17 -0
- data/sig/plugins/aws_sigv4.rbs +65 -0
- data/sig/plugins/push_promise.rbs +1 -1
- metadata +13 -2
data/lib/httpx/parser/http1.rb
CHANGED
@@ -66,7 +66,6 @@ module HTTPX
|
|
66
66
|
@status_code = code.to_i
|
67
67
|
raise(Error, "wrong status code (#{@status_code})") unless (100..599).cover?(@status_code)
|
68
68
|
|
69
|
-
# @buffer.slice!(0, idx + 1)
|
70
69
|
@buffer = @buffer.byteslice((idx + 1)..-1)
|
71
70
|
nextstate(:headers)
|
72
71
|
end
|
@@ -74,7 +73,8 @@ module HTTPX
|
|
74
73
|
def parse_headers
|
75
74
|
headers = @headers
|
76
75
|
while (idx = @buffer.index("\n"))
|
77
|
-
line = @buffer.
|
76
|
+
line = @buffer.byteslice(0..idx).sub(/\s+\z/, "")
|
77
|
+
@buffer = @buffer.byteslice((idx + 1)..-1)
|
78
78
|
if line.empty?
|
79
79
|
case @state
|
80
80
|
when :headers
|
@@ -96,11 +96,11 @@ module HTTPX
|
|
96
96
|
separator_index = line.index(":")
|
97
97
|
raise Error, "wrong header format" unless separator_index
|
98
98
|
|
99
|
-
key = line
|
99
|
+
key = line.byteslice(0..(separator_index - 1))
|
100
100
|
raise Error, "wrong header format" if key.start_with?("\s", "\t")
|
101
101
|
|
102
102
|
key.strip!
|
103
|
-
value = line
|
103
|
+
value = line.byteslice((separator_index + 1)..-1)
|
104
104
|
value.strip!
|
105
105
|
raise Error, "wrong header format" if value.nil?
|
106
106
|
|
@@ -0,0 +1,81 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module HTTPX
|
4
|
+
module Plugins
|
5
|
+
#
|
6
|
+
# This plugin applies AWS Sigv4 to requests, using the AWS SDK credentials and configuration.
|
7
|
+
#
|
8
|
+
# It requires the "aws-sdk-core" gem.
|
9
|
+
#
|
10
|
+
module AwsSdkAuthentication
|
11
|
+
#
|
12
|
+
# encapsulates access to an AWS SDK credentials store.
|
13
|
+
#
|
14
|
+
class Credentials
|
15
|
+
def initialize(aws_credentials)
|
16
|
+
@aws_credentials = aws_credentials
|
17
|
+
end
|
18
|
+
|
19
|
+
def username
|
20
|
+
@aws_credentials.access_key_id
|
21
|
+
end
|
22
|
+
|
23
|
+
def password
|
24
|
+
@aws_credentials.secret_access_key
|
25
|
+
end
|
26
|
+
|
27
|
+
def security_token
|
28
|
+
@aws_credentials.session_token
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
class << self
|
33
|
+
attr_reader :credentials, :region
|
34
|
+
|
35
|
+
def load_dependencies(klass)
|
36
|
+
require "aws-sdk-core"
|
37
|
+
klass.plugin(:aws_sigv4)
|
38
|
+
|
39
|
+
client = Class.new(Seahorse::Client::Base) do
|
40
|
+
@identifier = :httpx
|
41
|
+
set_api(Aws::S3::ClientApi::API)
|
42
|
+
add_plugin(Aws::Plugins::CredentialsConfiguration)
|
43
|
+
add_plugin(Aws::Plugins::RegionalEndpoint)
|
44
|
+
class << self
|
45
|
+
attr_reader :identifier
|
46
|
+
end
|
47
|
+
end.new
|
48
|
+
|
49
|
+
@credentials = Credentials.new(client.config[:credentials])
|
50
|
+
@region = client.config[:region]
|
51
|
+
end
|
52
|
+
|
53
|
+
def extra_options(options)
|
54
|
+
options.merge(max_concurrent_requests: 1)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
module InstanceMethods
|
59
|
+
#
|
60
|
+
# aws_authentication
|
61
|
+
# aws_authentication(credentials: Aws::Credentials.new('akid', 'secret'))
|
62
|
+
# aws_authentication()
|
63
|
+
#
|
64
|
+
def aws_sdk_authentication(**options)
|
65
|
+
credentials = AwsSdkAuthentication.credentials
|
66
|
+
region = AwsSdkAuthentication.region
|
67
|
+
|
68
|
+
aws_sigv4_authentication(
|
69
|
+
credentials: credentials,
|
70
|
+
region: region,
|
71
|
+
provider_prefix: "aws",
|
72
|
+
header_provider_field: "amz",
|
73
|
+
**options
|
74
|
+
)
|
75
|
+
end
|
76
|
+
alias_method :aws_auth, :aws_sdk_authentication
|
77
|
+
end
|
78
|
+
end
|
79
|
+
register_plugin :aws_sdk_authentication, AwsSdkAuthentication
|
80
|
+
end
|
81
|
+
end
|
@@ -0,0 +1,218 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "set"
|
4
|
+
require "aws-sdk-s3"
|
5
|
+
|
6
|
+
module HTTPX
|
7
|
+
module Plugins
|
8
|
+
#
|
9
|
+
# This plugin adds AWS Sigv4 authentication.
|
10
|
+
#
|
11
|
+
# https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html
|
12
|
+
#
|
13
|
+
# https://gitlab.com/honeyryderchuck/httpx/wikis/AWS-SigV4
|
14
|
+
#
|
15
|
+
module AWSSigV4
|
16
|
+
Credentials = Struct.new(:username, :password, :security_token)
|
17
|
+
|
18
|
+
class Signer
|
19
|
+
def initialize(
|
20
|
+
service:,
|
21
|
+
region:,
|
22
|
+
credentials: nil,
|
23
|
+
username: nil,
|
24
|
+
password: nil,
|
25
|
+
security_token: nil,
|
26
|
+
provider_prefix: "aws",
|
27
|
+
header_provider_field: "amz",
|
28
|
+
unsigned_headers: [],
|
29
|
+
apply_checksum_header: true,
|
30
|
+
algorithm: "SHA256"
|
31
|
+
)
|
32
|
+
@credentials = credentials || Credentials.new(username, password, security_token)
|
33
|
+
@service = service
|
34
|
+
@region = region
|
35
|
+
|
36
|
+
@unsigned_headers = Set.new(unsigned_headers.map(&:downcase))
|
37
|
+
@unsigned_headers << "authorization"
|
38
|
+
@unsigned_headers << "x-amzn-trace-id"
|
39
|
+
@unsigned_headers << "expect"
|
40
|
+
|
41
|
+
@apply_checksum_header = apply_checksum_header
|
42
|
+
@provider_prefix = provider_prefix
|
43
|
+
@header_provider_field = header_provider_field
|
44
|
+
|
45
|
+
@algorithm = algorithm
|
46
|
+
end
|
47
|
+
|
48
|
+
def sign!(request)
|
49
|
+
lower_provider_prefix = "#{@provider_prefix}4"
|
50
|
+
upper_provider_prefix = lower_provider_prefix.upcase
|
51
|
+
|
52
|
+
downcased_algorithm = @algorithm.downcase
|
53
|
+
|
54
|
+
datetime = (request.headers["x-#{@header_provider_field}-date"] ||= Time.now.utc.strftime("%Y%m%dT%H%M%SZ"))
|
55
|
+
date = datetime[0, 8]
|
56
|
+
|
57
|
+
content_hashed = request.headers["x-#{@header_provider_field}-content-#{downcased_algorithm}"] || hexdigest(request.body)
|
58
|
+
|
59
|
+
request.headers["x-#{@header_provider_field}-content-#{downcased_algorithm}"] ||= content_hashed if @apply_checksum_header
|
60
|
+
request.headers["x-#{@header_provider_field}-security-token"] ||= @credentials.security_token if @credentials.security_token
|
61
|
+
|
62
|
+
signature_headers = request.headers.each.reject do |k, _|
|
63
|
+
@unsigned_headers.include?(k)
|
64
|
+
end
|
65
|
+
# aws sigv4 needs to declare the host, regardless of protocol version
|
66
|
+
signature_headers << ["host", request.authority] unless request.headers.key?("host")
|
67
|
+
signature_headers.sort_by!(&:first)
|
68
|
+
|
69
|
+
signed_headers = signature_headers.map(&:first).join(";")
|
70
|
+
|
71
|
+
canonical_headers = signature_headers.map do |k, v|
|
72
|
+
# eliminate whitespace between value fields, unless it's a quoted value
|
73
|
+
"#{k}:#{v.start_with?("\"") && v.end_with?("\"") ? v : v.gsub(/\s+/, " ").strip}\n"
|
74
|
+
end.join
|
75
|
+
|
76
|
+
# canonical request
|
77
|
+
creq = "#{request.verb.to_s.upcase}" \
|
78
|
+
"\n#{request.canonical_path}" \
|
79
|
+
"\n#{request.canonical_query}" \
|
80
|
+
"\n#{canonical_headers}" \
|
81
|
+
"\n#{signed_headers}" \
|
82
|
+
"\n#{content_hashed}"
|
83
|
+
|
84
|
+
credential_scope = "#{date}" \
|
85
|
+
"/#{@region}" \
|
86
|
+
"/#{@service}" \
|
87
|
+
"/#{lower_provider_prefix}_request"
|
88
|
+
|
89
|
+
algo_line = "#{upper_provider_prefix}-HMAC-#{@algorithm}"
|
90
|
+
# string to sign
|
91
|
+
sts = "#{algo_line}" \
|
92
|
+
"\n#{datetime}" \
|
93
|
+
"\n#{credential_scope}" \
|
94
|
+
"\n#{hexdigest(creq)}"
|
95
|
+
|
96
|
+
# signature
|
97
|
+
k_date = hmac("#{upper_provider_prefix}#{@credentials.password}", date)
|
98
|
+
k_region = hmac(k_date, @region)
|
99
|
+
k_service = hmac(k_region, @service)
|
100
|
+
k_credentials = hmac(k_service, "#{lower_provider_prefix}_request")
|
101
|
+
sig = hexhmac(k_credentials, sts)
|
102
|
+
|
103
|
+
credential = "#{@credentials.username}/#{credential_scope}"
|
104
|
+
# apply signature
|
105
|
+
request.headers["authorization"] =
|
106
|
+
"#{algo_line} " \
|
107
|
+
"Credential=#{credential}, " \
|
108
|
+
"SignedHeaders=#{signed_headers}, " \
|
109
|
+
"Signature=#{sig}"
|
110
|
+
end
|
111
|
+
|
112
|
+
private
|
113
|
+
|
114
|
+
def hexdigest(value)
|
115
|
+
if value.respond_to?(:to_path)
|
116
|
+
# files, pathnames
|
117
|
+
OpenSSL::Digest.new(@algorithm).file(value.to_path).hexdigest
|
118
|
+
elsif value.respond_to?(:each)
|
119
|
+
digest = OpenSSL::Digest.new(@algorithm)
|
120
|
+
|
121
|
+
mb_buffer = value.each.each_with_object("".b) do |chunk, buffer|
|
122
|
+
buffer << chunk
|
123
|
+
break if buffer.bytesize >= 1024 * 1024
|
124
|
+
end
|
125
|
+
|
126
|
+
digest.update(mb_buffer)
|
127
|
+
value.rewind
|
128
|
+
digest.hexdigest
|
129
|
+
else
|
130
|
+
OpenSSL::Digest.new(@algorithm).hexdigest(value)
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
def hmac(key, value)
|
135
|
+
OpenSSL::HMAC.digest(OpenSSL::Digest.new(@algorithm), key, value)
|
136
|
+
end
|
137
|
+
|
138
|
+
def hexhmac(key, value)
|
139
|
+
OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new(@algorithm), key, value)
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
class << self
|
144
|
+
def extra_options(options)
|
145
|
+
Class.new(options.class) do
|
146
|
+
def_option(:sigv4_signer) do |signer|
|
147
|
+
signer.is_a?(Signer) ? signer : Signer.new(signer)
|
148
|
+
end
|
149
|
+
end.new.merge(options)
|
150
|
+
end
|
151
|
+
|
152
|
+
def load_dependencies(klass)
|
153
|
+
require "openssl"
|
154
|
+
klass.plugin(:expect)
|
155
|
+
klass.plugin(:compression)
|
156
|
+
end
|
157
|
+
end
|
158
|
+
|
159
|
+
module InstanceMethods
|
160
|
+
def aws_sigv4_authentication(**options)
|
161
|
+
with(sigv4_signer: Signer.new(**options))
|
162
|
+
end
|
163
|
+
|
164
|
+
def build_request(*, _)
|
165
|
+
request = super
|
166
|
+
|
167
|
+
return request if request.headers.key?("authorization")
|
168
|
+
|
169
|
+
signer = request.options.sigv4_signer
|
170
|
+
|
171
|
+
return request unless signer
|
172
|
+
|
173
|
+
signer.sign!(request)
|
174
|
+
|
175
|
+
request
|
176
|
+
end
|
177
|
+
end
|
178
|
+
|
179
|
+
module RequestMethods
|
180
|
+
def canonical_path
|
181
|
+
path = uri.path.dup
|
182
|
+
path << "/" if path.empty?
|
183
|
+
path.gsub(%r{[^/]+}) { |part| CGI.escape(part.encode("UTF-8")).gsub("+", "%20").gsub("%7E", "~") }
|
184
|
+
end
|
185
|
+
|
186
|
+
def canonical_query
|
187
|
+
params = query.split("&")
|
188
|
+
# params = params.map { |p| p.match(/=/) ? p : p + '=' }
|
189
|
+
# From: https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
|
190
|
+
# Sort the parameter names by character code point in ascending order.
|
191
|
+
# Parameters with duplicate names should be sorted by value.
|
192
|
+
#
|
193
|
+
# Default sort <=> in JRuby will swap members
|
194
|
+
# occasionally when <=> is 0 (considered still sorted), but this
|
195
|
+
# causes our normalized query string to not match the sent querystring.
|
196
|
+
# When names match, we then sort by their values. When values also
|
197
|
+
# match then we sort by their original order
|
198
|
+
params.each.with_index.sort do |a, b|
|
199
|
+
a, a_offset = a
|
200
|
+
b, b_offset = b
|
201
|
+
a_name, a_value = a.split("=")
|
202
|
+
b_name, b_value = b.split("=")
|
203
|
+
if a_name == b_name
|
204
|
+
if a_value == b_value
|
205
|
+
a_offset <=> b_offset
|
206
|
+
else
|
207
|
+
a_value <=> b_value
|
208
|
+
end
|
209
|
+
else
|
210
|
+
a_name <=> b_name
|
211
|
+
end
|
212
|
+
end.map(&:first).join("&")
|
213
|
+
end
|
214
|
+
end
|
215
|
+
end
|
216
|
+
register_plugin :aws_sigv4, AWSSigV4
|
217
|
+
end
|
218
|
+
end
|
@@ -18,10 +18,7 @@ module HTTPX
|
|
18
18
|
module_function
|
19
19
|
|
20
20
|
def deflate(raw, buffer, chunk_size:)
|
21
|
-
deflater = Zlib::Deflate.new
|
22
|
-
Zlib::MAX_WBITS,
|
23
|
-
Zlib::MAX_MEM_LEVEL,
|
24
|
-
Zlib::HUFFMAN_ONLY)
|
21
|
+
deflater = Zlib::Deflate.new
|
25
22
|
while (chunk = raw.read(chunk_size))
|
26
23
|
compressed = deflater.deflate(chunk)
|
27
24
|
buffer << compressed
|
@@ -31,7 +28,7 @@ module HTTPX
|
|
31
28
|
buffer << last
|
32
29
|
yield last if block_given?
|
33
30
|
ensure
|
34
|
-
deflater.close
|
31
|
+
deflater.close if deflater
|
35
32
|
end
|
36
33
|
end
|
37
34
|
|
@@ -0,0 +1,93 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module HTTPX
|
4
|
+
module Plugins
|
5
|
+
#
|
6
|
+
# The InternalTelemetry plugin is for internal use only. It is therefore undocumented, and
|
7
|
+
# its use is disencouraged, as API compatiblity will **not be guaranteed**.
|
8
|
+
#
|
9
|
+
# The gist of it is: when debug_level of logger is enabled to 3 or greater, considered internal-only
|
10
|
+
# supported log levels, it'll be loaded by default.
|
11
|
+
#
|
12
|
+
# Against a specific point of time, which will be by default the session initialization, but can be set
|
13
|
+
# by the end user in $http_init_time, different diff metrics can be shown. The "point of time" is calculated
|
14
|
+
# using the monotonic clock.
|
15
|
+
module InternalTelemetry
|
16
|
+
module TrackTimeMethods
|
17
|
+
private
|
18
|
+
|
19
|
+
def elapsed_time
|
20
|
+
yield
|
21
|
+
ensure
|
22
|
+
meter_elapsed_time("#{self.class.superclass}##{caller_locations(1, 1)[0].label}")
|
23
|
+
end
|
24
|
+
|
25
|
+
def meter_elapsed_time(label)
|
26
|
+
$http_init_time ||= Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
|
27
|
+
prev_time = $http_init_time
|
28
|
+
after_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
|
29
|
+
# $http_init_time = after_time
|
30
|
+
elapsed = after_time - prev_time
|
31
|
+
warn(+"\e[31m" << "[ELAPSED TIME]: #{label}: #{elapsed} (ms)" << "\e[0m")
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
module InstanceMethods
|
36
|
+
def self.included(klass)
|
37
|
+
klass.prepend TrackTimeMethods
|
38
|
+
super
|
39
|
+
end
|
40
|
+
|
41
|
+
def initialize(*)
|
42
|
+
meter_elapsed_time("Session: initializing...")
|
43
|
+
super
|
44
|
+
meter_elapsed_time("Session: initialized!!!")
|
45
|
+
end
|
46
|
+
|
47
|
+
private
|
48
|
+
|
49
|
+
def build_requests(*)
|
50
|
+
elapsed_time { super }
|
51
|
+
end
|
52
|
+
|
53
|
+
def fetch_response(*)
|
54
|
+
response = super
|
55
|
+
meter_elapsed_time("Session -> response") if response
|
56
|
+
response
|
57
|
+
end
|
58
|
+
|
59
|
+
def close(*)
|
60
|
+
super
|
61
|
+
meter_elapsed_time("Session -> close")
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
module RequestMethods
|
66
|
+
def self.included(klass)
|
67
|
+
klass.prepend TrackTimeMethods
|
68
|
+
super
|
69
|
+
end
|
70
|
+
|
71
|
+
def transition(nextstate)
|
72
|
+
state = @state
|
73
|
+
super
|
74
|
+
meter_elapsed_time("Request[#{@verb} #{@uri}: #{state}] -> #{nextstate}") if nextstate == @state
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
module ConnectionMethods
|
79
|
+
def self.included(klass)
|
80
|
+
klass.prepend TrackTimeMethods
|
81
|
+
super
|
82
|
+
end
|
83
|
+
|
84
|
+
def transition(nextstate)
|
85
|
+
state = @state
|
86
|
+
super
|
87
|
+
meter_elapsed_time("Connection[#{@origin}]: #{state} -> #{nextstate}") if nextstate == @state
|
88
|
+
end
|
89
|
+
end
|
90
|
+
end
|
91
|
+
register_plugin :internal_telemetry, InternalTelemetry
|
92
|
+
end
|
93
|
+
end
|
@@ -23,6 +23,7 @@ module HTTPX
|
|
23
23
|
end
|
24
24
|
|
25
25
|
def load_dependencies(*)
|
26
|
+
# :nocov:
|
26
27
|
begin
|
27
28
|
unless defined?(HTTP::FormData)
|
28
29
|
# in order not to break legacy code, we'll keep loading http/form_data for them.
|
@@ -33,6 +34,7 @@ module HTTPX
|
|
33
34
|
end
|
34
35
|
rescue LoadError
|
35
36
|
end
|
37
|
+
# :nocov:
|
36
38
|
require "httpx/plugins/multipart/encoder"
|
37
39
|
require "httpx/plugins/multipart/part"
|
38
40
|
require "httpx/plugins/multipart/mime_type_detector"
|