httpx 0.11.3 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +2 -2
  3. data/doc/release_notes/0_11_1.md +5 -1
  4. data/doc/release_notes/0_12_0.md +55 -0
  5. data/lib/httpx.rb +2 -1
  6. data/lib/httpx/adapters/faraday.rb +4 -6
  7. data/lib/httpx/altsvc.rb +1 -0
  8. data/lib/httpx/connection.rb +63 -15
  9. data/lib/httpx/connection/http1.rb +8 -7
  10. data/lib/httpx/connection/http2.rb +32 -25
  11. data/lib/httpx/io.rb +16 -3
  12. data/lib/httpx/io/ssl.rb +7 -9
  13. data/lib/httpx/io/tcp.rb +9 -8
  14. data/lib/httpx/io/tls.rb +218 -0
  15. data/lib/httpx/io/tls/box.rb +365 -0
  16. data/lib/httpx/io/tls/context.rb +199 -0
  17. data/lib/httpx/io/tls/ffi.rb +390 -0
  18. data/lib/httpx/parser/http1.rb +4 -4
  19. data/lib/httpx/plugins/aws_sdk_authentication.rb +81 -0
  20. data/lib/httpx/plugins/aws_sigv4.rb +218 -0
  21. data/lib/httpx/plugins/compression/deflate.rb +2 -5
  22. data/lib/httpx/plugins/internal_telemetry.rb +93 -0
  23. data/lib/httpx/plugins/multipart.rb +2 -0
  24. data/lib/httpx/plugins/multipart/encoder.rb +4 -9
  25. data/lib/httpx/plugins/proxy.rb +1 -1
  26. data/lib/httpx/plugins/proxy/http.rb +1 -1
  27. data/lib/httpx/plugins/proxy/socks4.rb +8 -0
  28. data/lib/httpx/plugins/proxy/socks5.rb +8 -0
  29. data/lib/httpx/plugins/push_promise.rb +3 -2
  30. data/lib/httpx/plugins/retries.rb +1 -1
  31. data/lib/httpx/plugins/stream.rb +3 -5
  32. data/lib/httpx/pool.rb +0 -1
  33. data/lib/httpx/registry.rb +1 -7
  34. data/lib/httpx/request.rb +11 -1
  35. data/lib/httpx/resolver/https.rb +3 -11
  36. data/lib/httpx/response.rb +9 -2
  37. data/lib/httpx/selector.rb +5 -0
  38. data/lib/httpx/session.rb +25 -2
  39. data/lib/httpx/transcoder/body.rb +3 -5
  40. data/lib/httpx/version.rb +1 -1
  41. data/sig/connection/http1.rbs +2 -2
  42. data/sig/connection/http2.rbs +5 -3
  43. data/sig/plugins/aws_sdk_authentication.rbs +17 -0
  44. data/sig/plugins/aws_sigv4.rbs +65 -0
  45. data/sig/plugins/push_promise.rbs +1 -1
  46. metadata +13 -2
@@ -66,7 +66,6 @@ module HTTPX
66
66
  @status_code = code.to_i
67
67
  raise(Error, "wrong status code (#{@status_code})") unless (100..599).cover?(@status_code)
68
68
 
69
- # @buffer.slice!(0, idx + 1)
70
69
  @buffer = @buffer.byteslice((idx + 1)..-1)
71
70
  nextstate(:headers)
72
71
  end
@@ -74,7 +73,8 @@ module HTTPX
74
73
  def parse_headers
75
74
  headers = @headers
76
75
  while (idx = @buffer.index("\n"))
77
- line = @buffer.slice!(0, idx + 1).sub(/\s+\z/, "")
76
+ line = @buffer.byteslice(0..idx).sub(/\s+\z/, "")
77
+ @buffer = @buffer.byteslice((idx + 1)..-1)
78
78
  if line.empty?
79
79
  case @state
80
80
  when :headers
@@ -96,11 +96,11 @@ module HTTPX
96
96
  separator_index = line.index(":")
97
97
  raise Error, "wrong header format" unless separator_index
98
98
 
99
- key = line[0..separator_index - 1]
99
+ key = line.byteslice(0..(separator_index - 1))
100
100
  raise Error, "wrong header format" if key.start_with?("\s", "\t")
101
101
 
102
102
  key.strip!
103
- value = line[separator_index + 1..-1]
103
+ value = line.byteslice((separator_index + 1)..-1)
104
104
  value.strip!
105
105
  raise Error, "wrong header format" if value.nil?
106
106
 
@@ -0,0 +1,81 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HTTPX
4
+ module Plugins
5
+ #
6
+ # This plugin applies AWS Sigv4 to requests, using the AWS SDK credentials and configuration.
7
+ #
8
+ # It requires the "aws-sdk-core" gem.
9
+ #
10
+ module AwsSdkAuthentication
11
+ #
12
+ # encapsulates access to an AWS SDK credentials store.
13
+ #
14
+ class Credentials
15
+ def initialize(aws_credentials)
16
+ @aws_credentials = aws_credentials
17
+ end
18
+
19
+ def username
20
+ @aws_credentials.access_key_id
21
+ end
22
+
23
+ def password
24
+ @aws_credentials.secret_access_key
25
+ end
26
+
27
+ def security_token
28
+ @aws_credentials.session_token
29
+ end
30
+ end
31
+
32
+ class << self
33
+ attr_reader :credentials, :region
34
+
35
+ def load_dependencies(klass)
36
+ require "aws-sdk-core"
37
+ klass.plugin(:aws_sigv4)
38
+
39
+ client = Class.new(Seahorse::Client::Base) do
40
+ @identifier = :httpx
41
+ set_api(Aws::S3::ClientApi::API)
42
+ add_plugin(Aws::Plugins::CredentialsConfiguration)
43
+ add_plugin(Aws::Plugins::RegionalEndpoint)
44
+ class << self
45
+ attr_reader :identifier
46
+ end
47
+ end.new
48
+
49
+ @credentials = Credentials.new(client.config[:credentials])
50
+ @region = client.config[:region]
51
+ end
52
+
53
+ def extra_options(options)
54
+ options.merge(max_concurrent_requests: 1)
55
+ end
56
+ end
57
+
58
+ module InstanceMethods
59
+ #
60
+ # aws_authentication
61
+ # aws_authentication(credentials: Aws::Credentials.new('akid', 'secret'))
62
+ # aws_authentication()
63
+ #
64
+ def aws_sdk_authentication(**options)
65
+ credentials = AwsSdkAuthentication.credentials
66
+ region = AwsSdkAuthentication.region
67
+
68
+ aws_sigv4_authentication(
69
+ credentials: credentials,
70
+ region: region,
71
+ provider_prefix: "aws",
72
+ header_provider_field: "amz",
73
+ **options
74
+ )
75
+ end
76
+ alias_method :aws_auth, :aws_sdk_authentication
77
+ end
78
+ end
79
+ register_plugin :aws_sdk_authentication, AwsSdkAuthentication
80
+ end
81
+ end
@@ -0,0 +1,218 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "set"
4
+ require "aws-sdk-s3"
5
+
6
+ module HTTPX
7
+ module Plugins
8
+ #
9
+ # This plugin adds AWS Sigv4 authentication.
10
+ #
11
+ # https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html
12
+ #
13
+ # https://gitlab.com/honeyryderchuck/httpx/wikis/AWS-SigV4
14
+ #
15
+ module AWSSigV4
16
+ Credentials = Struct.new(:username, :password, :security_token)
17
+
18
+ class Signer
19
+ def initialize(
20
+ service:,
21
+ region:,
22
+ credentials: nil,
23
+ username: nil,
24
+ password: nil,
25
+ security_token: nil,
26
+ provider_prefix: "aws",
27
+ header_provider_field: "amz",
28
+ unsigned_headers: [],
29
+ apply_checksum_header: true,
30
+ algorithm: "SHA256"
31
+ )
32
+ @credentials = credentials || Credentials.new(username, password, security_token)
33
+ @service = service
34
+ @region = region
35
+
36
+ @unsigned_headers = Set.new(unsigned_headers.map(&:downcase))
37
+ @unsigned_headers << "authorization"
38
+ @unsigned_headers << "x-amzn-trace-id"
39
+ @unsigned_headers << "expect"
40
+
41
+ @apply_checksum_header = apply_checksum_header
42
+ @provider_prefix = provider_prefix
43
+ @header_provider_field = header_provider_field
44
+
45
+ @algorithm = algorithm
46
+ end
47
+
48
+ def sign!(request)
49
+ lower_provider_prefix = "#{@provider_prefix}4"
50
+ upper_provider_prefix = lower_provider_prefix.upcase
51
+
52
+ downcased_algorithm = @algorithm.downcase
53
+
54
+ datetime = (request.headers["x-#{@header_provider_field}-date"] ||= Time.now.utc.strftime("%Y%m%dT%H%M%SZ"))
55
+ date = datetime[0, 8]
56
+
57
+ content_hashed = request.headers["x-#{@header_provider_field}-content-#{downcased_algorithm}"] || hexdigest(request.body)
58
+
59
+ request.headers["x-#{@header_provider_field}-content-#{downcased_algorithm}"] ||= content_hashed if @apply_checksum_header
60
+ request.headers["x-#{@header_provider_field}-security-token"] ||= @credentials.security_token if @credentials.security_token
61
+
62
+ signature_headers = request.headers.each.reject do |k, _|
63
+ @unsigned_headers.include?(k)
64
+ end
65
+ # aws sigv4 needs to declare the host, regardless of protocol version
66
+ signature_headers << ["host", request.authority] unless request.headers.key?("host")
67
+ signature_headers.sort_by!(&:first)
68
+
69
+ signed_headers = signature_headers.map(&:first).join(";")
70
+
71
+ canonical_headers = signature_headers.map do |k, v|
72
+ # eliminate whitespace between value fields, unless it's a quoted value
73
+ "#{k}:#{v.start_with?("\"") && v.end_with?("\"") ? v : v.gsub(/\s+/, " ").strip}\n"
74
+ end.join
75
+
76
+ # canonical request
77
+ creq = "#{request.verb.to_s.upcase}" \
78
+ "\n#{request.canonical_path}" \
79
+ "\n#{request.canonical_query}" \
80
+ "\n#{canonical_headers}" \
81
+ "\n#{signed_headers}" \
82
+ "\n#{content_hashed}"
83
+
84
+ credential_scope = "#{date}" \
85
+ "/#{@region}" \
86
+ "/#{@service}" \
87
+ "/#{lower_provider_prefix}_request"
88
+
89
+ algo_line = "#{upper_provider_prefix}-HMAC-#{@algorithm}"
90
+ # string to sign
91
+ sts = "#{algo_line}" \
92
+ "\n#{datetime}" \
93
+ "\n#{credential_scope}" \
94
+ "\n#{hexdigest(creq)}"
95
+
96
+ # signature
97
+ k_date = hmac("#{upper_provider_prefix}#{@credentials.password}", date)
98
+ k_region = hmac(k_date, @region)
99
+ k_service = hmac(k_region, @service)
100
+ k_credentials = hmac(k_service, "#{lower_provider_prefix}_request")
101
+ sig = hexhmac(k_credentials, sts)
102
+
103
+ credential = "#{@credentials.username}/#{credential_scope}"
104
+ # apply signature
105
+ request.headers["authorization"] =
106
+ "#{algo_line} " \
107
+ "Credential=#{credential}, " \
108
+ "SignedHeaders=#{signed_headers}, " \
109
+ "Signature=#{sig}"
110
+ end
111
+
112
+ private
113
+
114
+ def hexdigest(value)
115
+ if value.respond_to?(:to_path)
116
+ # files, pathnames
117
+ OpenSSL::Digest.new(@algorithm).file(value.to_path).hexdigest
118
+ elsif value.respond_to?(:each)
119
+ digest = OpenSSL::Digest.new(@algorithm)
120
+
121
+ mb_buffer = value.each.each_with_object("".b) do |chunk, buffer|
122
+ buffer << chunk
123
+ break if buffer.bytesize >= 1024 * 1024
124
+ end
125
+
126
+ digest.update(mb_buffer)
127
+ value.rewind
128
+ digest.hexdigest
129
+ else
130
+ OpenSSL::Digest.new(@algorithm).hexdigest(value)
131
+ end
132
+ end
133
+
134
+ def hmac(key, value)
135
+ OpenSSL::HMAC.digest(OpenSSL::Digest.new(@algorithm), key, value)
136
+ end
137
+
138
+ def hexhmac(key, value)
139
+ OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new(@algorithm), key, value)
140
+ end
141
+ end
142
+
143
+ class << self
144
+ def extra_options(options)
145
+ Class.new(options.class) do
146
+ def_option(:sigv4_signer) do |signer|
147
+ signer.is_a?(Signer) ? signer : Signer.new(signer)
148
+ end
149
+ end.new.merge(options)
150
+ end
151
+
152
+ def load_dependencies(klass)
153
+ require "openssl"
154
+ klass.plugin(:expect)
155
+ klass.plugin(:compression)
156
+ end
157
+ end
158
+
159
+ module InstanceMethods
160
+ def aws_sigv4_authentication(**options)
161
+ with(sigv4_signer: Signer.new(**options))
162
+ end
163
+
164
+ def build_request(*, _)
165
+ request = super
166
+
167
+ return request if request.headers.key?("authorization")
168
+
169
+ signer = request.options.sigv4_signer
170
+
171
+ return request unless signer
172
+
173
+ signer.sign!(request)
174
+
175
+ request
176
+ end
177
+ end
178
+
179
+ module RequestMethods
180
+ def canonical_path
181
+ path = uri.path.dup
182
+ path << "/" if path.empty?
183
+ path.gsub(%r{[^/]+}) { |part| CGI.escape(part.encode("UTF-8")).gsub("+", "%20").gsub("%7E", "~") }
184
+ end
185
+
186
+ def canonical_query
187
+ params = query.split("&")
188
+ # params = params.map { |p| p.match(/=/) ? p : p + '=' }
189
+ # From: https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
190
+ # Sort the parameter names by character code point in ascending order.
191
+ # Parameters with duplicate names should be sorted by value.
192
+ #
193
+ # Default sort <=> in JRuby will swap members
194
+ # occasionally when <=> is 0 (considered still sorted), but this
195
+ # causes our normalized query string to not match the sent querystring.
196
+ # When names match, we then sort by their values. When values also
197
+ # match then we sort by their original order
198
+ params.each.with_index.sort do |a, b|
199
+ a, a_offset = a
200
+ b, b_offset = b
201
+ a_name, a_value = a.split("=")
202
+ b_name, b_value = b.split("=")
203
+ if a_name == b_name
204
+ if a_value == b_value
205
+ a_offset <=> b_offset
206
+ else
207
+ a_value <=> b_value
208
+ end
209
+ else
210
+ a_name <=> b_name
211
+ end
212
+ end.map(&:first).join("&")
213
+ end
214
+ end
215
+ end
216
+ register_plugin :aws_sigv4, AWSSigV4
217
+ end
218
+ end
@@ -18,10 +18,7 @@ module HTTPX
18
18
  module_function
19
19
 
20
20
  def deflate(raw, buffer, chunk_size:)
21
- deflater = Zlib::Deflate.new(Zlib::BEST_COMPRESSION,
22
- Zlib::MAX_WBITS,
23
- Zlib::MAX_MEM_LEVEL,
24
- Zlib::HUFFMAN_ONLY)
21
+ deflater = Zlib::Deflate.new
25
22
  while (chunk = raw.read(chunk_size))
26
23
  compressed = deflater.deflate(chunk)
27
24
  buffer << compressed
@@ -31,7 +28,7 @@ module HTTPX
31
28
  buffer << last
32
29
  yield last if block_given?
33
30
  ensure
34
- deflater.close
31
+ deflater.close if deflater
35
32
  end
36
33
  end
37
34
 
@@ -0,0 +1,93 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HTTPX
4
+ module Plugins
5
+ #
6
+ # The InternalTelemetry plugin is for internal use only. It is therefore undocumented, and
7
+ # its use is disencouraged, as API compatiblity will **not be guaranteed**.
8
+ #
9
+ # The gist of it is: when debug_level of logger is enabled to 3 or greater, considered internal-only
10
+ # supported log levels, it'll be loaded by default.
11
+ #
12
+ # Against a specific point of time, which will be by default the session initialization, but can be set
13
+ # by the end user in $http_init_time, different diff metrics can be shown. The "point of time" is calculated
14
+ # using the monotonic clock.
15
+ module InternalTelemetry
16
+ module TrackTimeMethods
17
+ private
18
+
19
+ def elapsed_time
20
+ yield
21
+ ensure
22
+ meter_elapsed_time("#{self.class.superclass}##{caller_locations(1, 1)[0].label}")
23
+ end
24
+
25
+ def meter_elapsed_time(label)
26
+ $http_init_time ||= Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
27
+ prev_time = $http_init_time
28
+ after_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
29
+ # $http_init_time = after_time
30
+ elapsed = after_time - prev_time
31
+ warn(+"\e[31m" << "[ELAPSED TIME]: #{label}: #{elapsed} (ms)" << "\e[0m")
32
+ end
33
+ end
34
+
35
+ module InstanceMethods
36
+ def self.included(klass)
37
+ klass.prepend TrackTimeMethods
38
+ super
39
+ end
40
+
41
+ def initialize(*)
42
+ meter_elapsed_time("Session: initializing...")
43
+ super
44
+ meter_elapsed_time("Session: initialized!!!")
45
+ end
46
+
47
+ private
48
+
49
+ def build_requests(*)
50
+ elapsed_time { super }
51
+ end
52
+
53
+ def fetch_response(*)
54
+ response = super
55
+ meter_elapsed_time("Session -> response") if response
56
+ response
57
+ end
58
+
59
+ def close(*)
60
+ super
61
+ meter_elapsed_time("Session -> close")
62
+ end
63
+ end
64
+
65
+ module RequestMethods
66
+ def self.included(klass)
67
+ klass.prepend TrackTimeMethods
68
+ super
69
+ end
70
+
71
+ def transition(nextstate)
72
+ state = @state
73
+ super
74
+ meter_elapsed_time("Request[#{@verb} #{@uri}: #{state}] -> #{nextstate}") if nextstate == @state
75
+ end
76
+ end
77
+
78
+ module ConnectionMethods
79
+ def self.included(klass)
80
+ klass.prepend TrackTimeMethods
81
+ super
82
+ end
83
+
84
+ def transition(nextstate)
85
+ state = @state
86
+ super
87
+ meter_elapsed_time("Connection[#{@origin}]: #{state} -> #{nextstate}") if nextstate == @state
88
+ end
89
+ end
90
+ end
91
+ register_plugin :internal_telemetry, InternalTelemetry
92
+ end
93
+ end
@@ -23,6 +23,7 @@ module HTTPX
23
23
  end
24
24
 
25
25
  def load_dependencies(*)
26
+ # :nocov:
26
27
  begin
27
28
  unless defined?(HTTP::FormData)
28
29
  # in order not to break legacy code, we'll keep loading http/form_data for them.
@@ -33,6 +34,7 @@ module HTTPX
33
34
  end
34
35
  rescue LoadError
35
36
  end
37
+ # :nocov:
36
38
  require "httpx/plugins/multipart/encoder"
37
39
  require "httpx/plugins/multipart/part"
38
40
  require "httpx/plugins/multipart/mime_type_detector"