httpx 0.11.1 → 0.13.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +2 -2
  3. data/doc/release_notes/0_11_1.md +5 -1
  4. data/doc/release_notes/0_11_2.md +5 -0
  5. data/doc/release_notes/0_11_3.md +5 -0
  6. data/doc/release_notes/0_12_0.md +55 -0
  7. data/doc/release_notes/0_13_0.md +58 -0
  8. data/doc/release_notes/0_13_1.md +5 -0
  9. data/lib/httpx.rb +2 -1
  10. data/lib/httpx/adapters/faraday.rb +4 -6
  11. data/lib/httpx/altsvc.rb +1 -0
  12. data/lib/httpx/chainable.rb +2 -2
  13. data/lib/httpx/connection.rb +80 -28
  14. data/lib/httpx/connection/http1.rb +19 -6
  15. data/lib/httpx/connection/http2.rb +32 -25
  16. data/lib/httpx/io.rb +16 -3
  17. data/lib/httpx/io/ssl.rb +35 -24
  18. data/lib/httpx/io/tcp.rb +50 -28
  19. data/lib/httpx/io/tls.rb +218 -0
  20. data/lib/httpx/io/tls/box.rb +365 -0
  21. data/lib/httpx/io/tls/context.rb +199 -0
  22. data/lib/httpx/io/tls/ffi.rb +390 -0
  23. data/lib/httpx/io/unix.rb +27 -12
  24. data/lib/httpx/options.rb +11 -23
  25. data/lib/httpx/parser/http1.rb +4 -4
  26. data/lib/httpx/plugins/aws_sdk_authentication.rb +81 -0
  27. data/lib/httpx/plugins/aws_sigv4.rb +218 -0
  28. data/lib/httpx/plugins/compression.rb +20 -8
  29. data/lib/httpx/plugins/compression/brotli.rb +8 -6
  30. data/lib/httpx/plugins/compression/deflate.rb +4 -7
  31. data/lib/httpx/plugins/compression/gzip.rb +2 -2
  32. data/lib/httpx/plugins/cookies/set_cookie_parser.rb +1 -1
  33. data/lib/httpx/plugins/digest_authentication.rb +1 -1
  34. data/lib/httpx/plugins/follow_redirects.rb +1 -1
  35. data/lib/httpx/plugins/h2c.rb +43 -58
  36. data/lib/httpx/plugins/internal_telemetry.rb +93 -0
  37. data/lib/httpx/plugins/multipart.rb +2 -0
  38. data/lib/httpx/plugins/multipart/encoder.rb +4 -9
  39. data/lib/httpx/plugins/proxy.rb +1 -1
  40. data/lib/httpx/plugins/proxy/http.rb +1 -1
  41. data/lib/httpx/plugins/proxy/socks4.rb +8 -0
  42. data/lib/httpx/plugins/proxy/socks5.rb +8 -0
  43. data/lib/httpx/plugins/push_promise.rb +3 -2
  44. data/lib/httpx/plugins/retries.rb +2 -2
  45. data/lib/httpx/plugins/stream.rb +6 -6
  46. data/lib/httpx/plugins/upgrade.rb +83 -0
  47. data/lib/httpx/plugins/upgrade/h2.rb +54 -0
  48. data/lib/httpx/pool.rb +14 -6
  49. data/lib/httpx/registry.rb +1 -7
  50. data/lib/httpx/request.rb +11 -1
  51. data/lib/httpx/resolver/https.rb +3 -11
  52. data/lib/httpx/response.rb +14 -7
  53. data/lib/httpx/selector.rb +5 -0
  54. data/lib/httpx/session.rb +25 -2
  55. data/lib/httpx/transcoder/body.rb +3 -5
  56. data/lib/httpx/version.rb +1 -1
  57. data/sig/chainable.rbs +2 -1
  58. data/sig/connection/http1.rbs +3 -2
  59. data/sig/connection/http2.rbs +5 -3
  60. data/sig/options.rbs +7 -20
  61. data/sig/plugins/aws_sdk_authentication.rbs +17 -0
  62. data/sig/plugins/aws_sigv4.rbs +64 -0
  63. data/sig/plugins/compression.rbs +5 -3
  64. data/sig/plugins/compression/brotli.rbs +1 -1
  65. data/sig/plugins/compression/deflate.rbs +1 -1
  66. data/sig/plugins/compression/gzip.rbs +1 -1
  67. data/sig/plugins/cookies.rbs +0 -1
  68. data/sig/plugins/digest_authentication.rbs +0 -1
  69. data/sig/plugins/expect.rbs +0 -2
  70. data/sig/plugins/follow_redirects.rbs +0 -2
  71. data/sig/plugins/h2c.rbs +5 -10
  72. data/sig/plugins/persistent.rbs +0 -1
  73. data/sig/plugins/proxy.rbs +0 -1
  74. data/sig/plugins/push_promise.rbs +1 -1
  75. data/sig/plugins/retries.rbs +0 -4
  76. data/sig/plugins/upgrade.rbs +23 -0
  77. data/sig/response.rbs +3 -1
  78. metadata +24 -2
data/lib/httpx/options.rb CHANGED
@@ -6,11 +6,6 @@ module HTTPX
6
6
  MAX_BODY_THRESHOLD_SIZE = (1 << 10) * 112 # 112K
7
7
 
8
8
  class << self
9
- def inherited(klass)
10
- super
11
- klass.instance_variable_set(:@defined_options, @defined_options.dup)
12
- end
13
-
14
9
  def new(options = {})
15
10
  # let enhanced options go through
16
11
  return options if self == Options && options.class > self
@@ -19,13 +14,7 @@ module HTTPX
19
14
  super
20
15
  end
21
16
 
22
- def defined_options
23
- @defined_options ||= []
24
- end
25
-
26
17
  def def_option(name, &interpreter)
27
- defined_options << name.to_sym
28
-
29
18
  attr_reader name
30
19
 
31
20
  if interpreter
@@ -34,16 +23,8 @@ module HTTPX
34
23
 
35
24
  instance_variable_set(:"@#{name}", instance_exec(value, &interpreter))
36
25
  end
37
-
38
- define_method(:"with_#{name}") do |value|
39
- merge(name => instance_exec(value, &interpreter))
40
- end
41
26
  else
42
27
  attr_writer name
43
-
44
- define_method(:"with_#{name}") do |value|
45
- merge(name => value)
46
- end
47
28
  end
48
29
 
49
30
  protected :"#{name}="
@@ -69,6 +50,7 @@ module HTTPX
69
50
  :connection_class => Class.new(Connection),
70
51
  :transport => nil,
71
52
  :transport_options => nil,
53
+ :addresses => nil,
72
54
  :persistent => false,
73
55
  :resolver_class => (ENV["HTTPX_RESOLVER"] || :native).to_sym,
74
56
  :resolver_options => { cache: true },
@@ -121,6 +103,10 @@ module HTTPX
121
103
  transport
122
104
  end
123
105
 
106
+ def_option(:addresses) do |addrs|
107
+ Array(addrs)
108
+ end
109
+
124
110
  %w[
125
111
  params form json body ssl http2_settings
126
112
  request_class response_class headers_class request_body_class response_body_class connection_class
@@ -153,6 +139,8 @@ module HTTPX
153
139
 
154
140
  h1 = to_hash
155
141
 
142
+ return self if h1 == h2
143
+
156
144
  merged = h1.merge(h2) do |k, v1, v2|
157
145
  case k
158
146
  when :headers, :ssl, :http2_settings, :timeout
@@ -166,10 +154,10 @@ module HTTPX
166
154
  end
167
155
 
168
156
  def to_hash
169
- hash_pairs = self.class
170
- .defined_options
171
- .flat_map { |opt_name| [opt_name, send(opt_name)] }
172
- Hash[*hash_pairs]
157
+ hash_pairs = instance_variables.map do |ivar|
158
+ [ivar[1..-1].to_sym, instance_variable_get(ivar)]
159
+ end
160
+ Hash[hash_pairs]
173
161
  end
174
162
 
175
163
  def initialize_dup(other)
@@ -66,7 +66,6 @@ module HTTPX
66
66
  @status_code = code.to_i
67
67
  raise(Error, "wrong status code (#{@status_code})") unless (100..599).cover?(@status_code)
68
68
 
69
- # @buffer.slice!(0, idx + 1)
70
69
  @buffer = @buffer.byteslice((idx + 1)..-1)
71
70
  nextstate(:headers)
72
71
  end
@@ -74,7 +73,8 @@ module HTTPX
74
73
  def parse_headers
75
74
  headers = @headers
76
75
  while (idx = @buffer.index("\n"))
77
- line = @buffer.slice!(0, idx + 1).sub(/\s+\z/, "")
76
+ line = @buffer.byteslice(0..idx).sub(/\s+\z/, "")
77
+ @buffer = @buffer.byteslice((idx + 1)..-1)
78
78
  if line.empty?
79
79
  case @state
80
80
  when :headers
@@ -96,11 +96,11 @@ module HTTPX
96
96
  separator_index = line.index(":")
97
97
  raise Error, "wrong header format" unless separator_index
98
98
 
99
- key = line[0..separator_index - 1]
99
+ key = line.byteslice(0..(separator_index - 1))
100
100
  raise Error, "wrong header format" if key.start_with?("\s", "\t")
101
101
 
102
102
  key.strip!
103
- value = line[separator_index + 1..-1]
103
+ value = line.byteslice((separator_index + 1)..-1)
104
104
  value.strip!
105
105
  raise Error, "wrong header format" if value.nil?
106
106
 
@@ -0,0 +1,81 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HTTPX
4
+ module Plugins
5
+ #
6
+ # This plugin applies AWS Sigv4 to requests, using the AWS SDK credentials and configuration.
7
+ #
8
+ # It requires the "aws-sdk-core" gem.
9
+ #
10
+ module AwsSdkAuthentication
11
+ #
12
+ # encapsulates access to an AWS SDK credentials store.
13
+ #
14
+ class Credentials
15
+ def initialize(aws_credentials)
16
+ @aws_credentials = aws_credentials
17
+ end
18
+
19
+ def username
20
+ @aws_credentials.access_key_id
21
+ end
22
+
23
+ def password
24
+ @aws_credentials.secret_access_key
25
+ end
26
+
27
+ def security_token
28
+ @aws_credentials.session_token
29
+ end
30
+ end
31
+
32
+ class << self
33
+ attr_reader :credentials, :region
34
+
35
+ def load_dependencies(klass)
36
+ require "aws-sdk-core"
37
+ klass.plugin(:aws_sigv4)
38
+
39
+ client = Class.new(Seahorse::Client::Base) do
40
+ @identifier = :httpx
41
+ set_api(Aws::S3::ClientApi::API)
42
+ add_plugin(Aws::Plugins::CredentialsConfiguration)
43
+ add_plugin(Aws::Plugins::RegionalEndpoint)
44
+ class << self
45
+ attr_reader :identifier
46
+ end
47
+ end.new
48
+
49
+ @credentials = Credentials.new(client.config[:credentials])
50
+ @region = client.config[:region]
51
+ end
52
+
53
+ def extra_options(options)
54
+ options.merge(max_concurrent_requests: 1)
55
+ end
56
+ end
57
+
58
+ module InstanceMethods
59
+ #
60
+ # aws_authentication
61
+ # aws_authentication(credentials: Aws::Credentials.new('akid', 'secret'))
62
+ # aws_authentication()
63
+ #
64
+ def aws_sdk_authentication(**options)
65
+ credentials = AwsSdkAuthentication.credentials
66
+ region = AwsSdkAuthentication.region
67
+
68
+ aws_sigv4_authentication(
69
+ credentials: credentials,
70
+ region: region,
71
+ provider_prefix: "aws",
72
+ header_provider_field: "amz",
73
+ **options
74
+ )
75
+ end
76
+ alias_method :aws_auth, :aws_sdk_authentication
77
+ end
78
+ end
79
+ register_plugin :aws_sdk_authentication, AwsSdkAuthentication
80
+ end
81
+ end
@@ -0,0 +1,218 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "set"
4
+ require "aws-sdk-s3"
5
+
6
+ module HTTPX
7
+ module Plugins
8
+ #
9
+ # This plugin adds AWS Sigv4 authentication.
10
+ #
11
+ # https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html
12
+ #
13
+ # https://gitlab.com/honeyryderchuck/httpx/wikis/AWS-SigV4
14
+ #
15
+ module AWSSigV4
16
+ Credentials = Struct.new(:username, :password, :security_token)
17
+
18
+ class Signer
19
+ def initialize(
20
+ service:,
21
+ region:,
22
+ credentials: nil,
23
+ username: nil,
24
+ password: nil,
25
+ security_token: nil,
26
+ provider_prefix: "aws",
27
+ header_provider_field: "amz",
28
+ unsigned_headers: [],
29
+ apply_checksum_header: true,
30
+ algorithm: "SHA256"
31
+ )
32
+ @credentials = credentials || Credentials.new(username, password, security_token)
33
+ @service = service
34
+ @region = region
35
+
36
+ @unsigned_headers = Set.new(unsigned_headers.map(&:downcase))
37
+ @unsigned_headers << "authorization"
38
+ @unsigned_headers << "x-amzn-trace-id"
39
+ @unsigned_headers << "expect"
40
+
41
+ @apply_checksum_header = apply_checksum_header
42
+ @provider_prefix = provider_prefix
43
+ @header_provider_field = header_provider_field
44
+
45
+ @algorithm = algorithm
46
+ end
47
+
48
+ def sign!(request)
49
+ lower_provider_prefix = "#{@provider_prefix}4"
50
+ upper_provider_prefix = lower_provider_prefix.upcase
51
+
52
+ downcased_algorithm = @algorithm.downcase
53
+
54
+ datetime = (request.headers["x-#{@header_provider_field}-date"] ||= Time.now.utc.strftime("%Y%m%dT%H%M%SZ"))
55
+ date = datetime[0, 8]
56
+
57
+ content_hashed = request.headers["x-#{@header_provider_field}-content-#{downcased_algorithm}"] || hexdigest(request.body)
58
+
59
+ request.headers["x-#{@header_provider_field}-content-#{downcased_algorithm}"] ||= content_hashed if @apply_checksum_header
60
+ request.headers["x-#{@header_provider_field}-security-token"] ||= @credentials.security_token if @credentials.security_token
61
+
62
+ signature_headers = request.headers.each.reject do |k, _|
63
+ @unsigned_headers.include?(k)
64
+ end
65
+ # aws sigv4 needs to declare the host, regardless of protocol version
66
+ signature_headers << ["host", request.authority] unless request.headers.key?("host")
67
+ signature_headers.sort_by!(&:first)
68
+
69
+ signed_headers = signature_headers.map(&:first).join(";")
70
+
71
+ canonical_headers = signature_headers.map do |k, v|
72
+ # eliminate whitespace between value fields, unless it's a quoted value
73
+ "#{k}:#{v.start_with?("\"") && v.end_with?("\"") ? v : v.gsub(/\s+/, " ").strip}\n"
74
+ end.join
75
+
76
+ # canonical request
77
+ creq = "#{request.verb.to_s.upcase}" \
78
+ "\n#{request.canonical_path}" \
79
+ "\n#{request.canonical_query}" \
80
+ "\n#{canonical_headers}" \
81
+ "\n#{signed_headers}" \
82
+ "\n#{content_hashed}"
83
+
84
+ credential_scope = "#{date}" \
85
+ "/#{@region}" \
86
+ "/#{@service}" \
87
+ "/#{lower_provider_prefix}_request"
88
+
89
+ algo_line = "#{upper_provider_prefix}-HMAC-#{@algorithm}"
90
+ # string to sign
91
+ sts = "#{algo_line}" \
92
+ "\n#{datetime}" \
93
+ "\n#{credential_scope}" \
94
+ "\n#{hexdigest(creq)}"
95
+
96
+ # signature
97
+ k_date = hmac("#{upper_provider_prefix}#{@credentials.password}", date)
98
+ k_region = hmac(k_date, @region)
99
+ k_service = hmac(k_region, @service)
100
+ k_credentials = hmac(k_service, "#{lower_provider_prefix}_request")
101
+ sig = hexhmac(k_credentials, sts)
102
+
103
+ credential = "#{@credentials.username}/#{credential_scope}"
104
+ # apply signature
105
+ request.headers["authorization"] =
106
+ "#{algo_line} " \
107
+ "Credential=#{credential}, " \
108
+ "SignedHeaders=#{signed_headers}, " \
109
+ "Signature=#{sig}"
110
+ end
111
+
112
+ private
113
+
114
+ def hexdigest(value)
115
+ if value.respond_to?(:to_path)
116
+ # files, pathnames
117
+ OpenSSL::Digest.new(@algorithm).file(value.to_path).hexdigest
118
+ elsif value.respond_to?(:each)
119
+ digest = OpenSSL::Digest.new(@algorithm)
120
+
121
+ mb_buffer = value.each.each_with_object("".b) do |chunk, buffer|
122
+ buffer << chunk
123
+ break if buffer.bytesize >= 1024 * 1024
124
+ end
125
+
126
+ digest.update(mb_buffer)
127
+ value.rewind
128
+ digest.hexdigest
129
+ else
130
+ OpenSSL::Digest.new(@algorithm).hexdigest(value)
131
+ end
132
+ end
133
+
134
+ def hmac(key, value)
135
+ OpenSSL::HMAC.digest(OpenSSL::Digest.new(@algorithm), key, value)
136
+ end
137
+
138
+ def hexhmac(key, value)
139
+ OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new(@algorithm), key, value)
140
+ end
141
+ end
142
+
143
+ class << self
144
+ def extra_options(options)
145
+ Class.new(options.class) do
146
+ def_option(:sigv4_signer) do |signer|
147
+ signer.is_a?(Signer) ? signer : Signer.new(signer)
148
+ end
149
+ end.new.merge(options)
150
+ end
151
+
152
+ def load_dependencies(klass)
153
+ require "openssl"
154
+ klass.plugin(:expect)
155
+ klass.plugin(:compression)
156
+ end
157
+ end
158
+
159
+ module InstanceMethods
160
+ def aws_sigv4_authentication(**options)
161
+ with(sigv4_signer: Signer.new(**options))
162
+ end
163
+
164
+ def build_request(*, _)
165
+ request = super
166
+
167
+ return request if request.headers.key?("authorization")
168
+
169
+ signer = request.options.sigv4_signer
170
+
171
+ return request unless signer
172
+
173
+ signer.sign!(request)
174
+
175
+ request
176
+ end
177
+ end
178
+
179
+ module RequestMethods
180
+ def canonical_path
181
+ path = uri.path.dup
182
+ path << "/" if path.empty?
183
+ path.gsub(%r{[^/]+}) { |part| CGI.escape(part.encode("UTF-8")).gsub("+", "%20").gsub("%7E", "~") }
184
+ end
185
+
186
+ def canonical_query
187
+ params = query.split("&")
188
+ # params = params.map { |p| p.match(/=/) ? p : p + '=' }
189
+ # From: https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
190
+ # Sort the parameter names by character code point in ascending order.
191
+ # Parameters with duplicate names should be sorted by value.
192
+ #
193
+ # Default sort <=> in JRuby will swap members
194
+ # occasionally when <=> is 0 (considered still sorted), but this
195
+ # causes our normalized query string to not match the sent querystring.
196
+ # When names match, we then sort by their values. When values also
197
+ # match then we sort by their original order
198
+ params.each.with_index.sort do |a, b|
199
+ a, a_offset = a
200
+ b, b_offset = b
201
+ a_name, a_value = a.split("=")
202
+ b_name, b_value = b.split("=")
203
+ if a_name == b_name
204
+ if a_value == b_value
205
+ a_offset <=> b_offset
206
+ else
207
+ a_value <=> b_value
208
+ end
209
+ else
210
+ a_name <=> b_name
211
+ end
212
+ end.map(&:first).join("&")
213
+ end
214
+ end
215
+ end
216
+ register_plugin :aws_sigv4, AWSSigV4
217
+ end
218
+ end
@@ -13,15 +13,17 @@ module HTTPX
13
13
  # https://gitlab.com/honeyryderchuck/httpx/wikis/Compression
14
14
  #
15
15
  module Compression
16
- extend Registry
17
-
18
16
  class << self
19
- def load_dependencies(klass)
17
+ def configure(klass)
20
18
  klass.plugin(:"compression/gzip")
21
19
  klass.plugin(:"compression/deflate")
22
20
  end
23
21
 
24
22
  def extra_options(options)
23
+ encodings = Module.new do
24
+ extend Registry
25
+ end
26
+
25
27
  Class.new(options.class) do
26
28
  def_option(:compression_threshold_size) do |bytes|
27
29
  bytes = Integer(bytes)
@@ -29,7 +31,13 @@ module HTTPX
29
31
 
30
32
  bytes
31
33
  end
32
- end.new(options).merge(headers: { "accept-encoding" => Compression.registry.keys })
34
+
35
+ def_option(:encodings) do |encs|
36
+ raise Error, ":encodings must be a registry" unless encs.respond_to?(:registry)
37
+
38
+ encs
39
+ end
40
+ end.new(options).merge(encodings: encodings)
33
41
  end
34
42
  end
35
43
 
@@ -37,7 +45,11 @@ module HTTPX
37
45
  def initialize(*)
38
46
  super
39
47
  # forego compression in the Range cases
40
- @headers.delete("accept-encoding") if @headers.key?("range")
48
+ if @headers.key?("range")
49
+ @headers.delete("accept-encoding")
50
+ else
51
+ @headers["accept-encoding"] ||= @options.encodings.registry.keys
52
+ end
41
53
  end
42
54
  end
43
55
 
@@ -52,7 +64,7 @@ module HTTPX
52
64
  @headers.get("content-encoding").each do |encoding|
53
65
  next if encoding == "identity"
54
66
 
55
- @body = Encoder.new(@body, Compression.registry(encoding).deflater)
67
+ @body = Encoder.new(@body, options.encodings.registry(encoding).deflater)
56
68
  end
57
69
  @headers["content-length"] = @body.bytesize unless chunked?
58
70
  end
@@ -61,7 +73,7 @@ module HTTPX
61
73
  module ResponseBodyMethods
62
74
  attr_reader :encodings
63
75
 
64
- def initialize(*, **)
76
+ def initialize(*)
65
77
  @encodings = []
66
78
 
67
79
  super
@@ -80,7 +92,7 @@ module HTTPX
80
92
  @_inflaters = @headers.get("content-encoding").map do |encoding|
81
93
  next if encoding == "identity"
82
94
 
83
- inflater = Compression.registry(encoding).inflater(compressed_length)
95
+ inflater = @options.encodings.registry(encoding).inflater(compressed_length)
84
96
  # do not uncompress if there is no decoder available. In fact, we can't reliably
85
97
  # continue decompressing beyond that, so ignore.
86
98
  break unless inflater