httpx 0.10.2 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +13 -5
  3. data/doc/release_notes/0_11_0.md +76 -0
  4. data/doc/release_notes/0_11_1.md +5 -0
  5. data/doc/release_notes/0_11_2.md +5 -0
  6. data/doc/release_notes/0_11_3.md +5 -0
  7. data/doc/release_notes/0_12_0.md +55 -0
  8. data/lib/httpx.rb +2 -1
  9. data/lib/httpx/adapters/datadog.rb +205 -0
  10. data/lib/httpx/adapters/faraday.rb +4 -8
  11. data/lib/httpx/adapters/webmock.rb +123 -0
  12. data/lib/httpx/altsvc.rb +1 -0
  13. data/lib/httpx/chainable.rb +1 -1
  14. data/lib/httpx/connection.rb +63 -15
  15. data/lib/httpx/connection/http1.rb +16 -5
  16. data/lib/httpx/connection/http2.rb +36 -29
  17. data/lib/httpx/domain_name.rb +1 -3
  18. data/lib/httpx/errors.rb +2 -0
  19. data/lib/httpx/headers.rb +1 -0
  20. data/lib/httpx/io.rb +16 -3
  21. data/lib/httpx/io/ssl.rb +7 -13
  22. data/lib/httpx/io/tcp.rb +9 -8
  23. data/lib/httpx/io/tls.rb +218 -0
  24. data/lib/httpx/io/tls/box.rb +365 -0
  25. data/lib/httpx/io/tls/context.rb +199 -0
  26. data/lib/httpx/io/tls/ffi.rb +390 -0
  27. data/lib/httpx/io/udp.rb +4 -3
  28. data/lib/httpx/parser/http1.rb +4 -4
  29. data/lib/httpx/plugins/aws_sdk_authentication.rb +81 -0
  30. data/lib/httpx/plugins/aws_sigv4.rb +218 -0
  31. data/lib/httpx/plugins/compression.rb +1 -1
  32. data/lib/httpx/plugins/compression/deflate.rb +2 -5
  33. data/lib/httpx/plugins/cookies/set_cookie_parser.rb +1 -1
  34. data/lib/httpx/plugins/expect.rb +33 -8
  35. data/lib/httpx/plugins/internal_telemetry.rb +93 -0
  36. data/lib/httpx/plugins/multipart.rb +42 -35
  37. data/lib/httpx/plugins/multipart/encoder.rb +110 -0
  38. data/lib/httpx/plugins/multipart/mime_type_detector.rb +64 -0
  39. data/lib/httpx/plugins/multipart/part.rb +34 -0
  40. data/lib/httpx/plugins/proxy.rb +1 -1
  41. data/lib/httpx/plugins/proxy/http.rb +1 -1
  42. data/lib/httpx/plugins/proxy/socks4.rb +8 -0
  43. data/lib/httpx/plugins/proxy/socks5.rb +11 -2
  44. data/lib/httpx/plugins/push_promise.rb +5 -4
  45. data/lib/httpx/plugins/retries.rb +1 -1
  46. data/lib/httpx/plugins/stream.rb +3 -5
  47. data/lib/httpx/pool.rb +0 -1
  48. data/lib/httpx/registry.rb +1 -7
  49. data/lib/httpx/request.rb +32 -12
  50. data/lib/httpx/resolver.rb +7 -4
  51. data/lib/httpx/resolver/https.rb +7 -13
  52. data/lib/httpx/resolver/native.rb +10 -6
  53. data/lib/httpx/resolver/system.rb +1 -1
  54. data/lib/httpx/response.rb +9 -2
  55. data/lib/httpx/selector.rb +6 -0
  56. data/lib/httpx/session.rb +40 -20
  57. data/lib/httpx/transcoder.rb +6 -4
  58. data/lib/httpx/transcoder/body.rb +3 -5
  59. data/lib/httpx/version.rb +1 -1
  60. data/sig/connection/http1.rbs +2 -2
  61. data/sig/connection/http2.rbs +8 -7
  62. data/sig/headers.rbs +3 -0
  63. data/sig/plugins/aws_sdk_authentication.rbs +17 -0
  64. data/sig/plugins/aws_sigv4.rbs +65 -0
  65. data/sig/plugins/multipart.rbs +27 -4
  66. data/sig/plugins/push_promise.rbs +1 -1
  67. data/sig/request.rbs +1 -1
  68. data/sig/resolver/https.rbs +2 -0
  69. data/sig/response.rbs +1 -1
  70. data/sig/session.rbs +1 -1
  71. data/sig/transcoder.rbs +2 -2
  72. data/sig/transcoder/body.rbs +2 -0
  73. data/sig/transcoder/form.rbs +7 -1
  74. data/sig/transcoder/json.rbs +3 -1
  75. metadata +50 -47
  76. data/sig/missing.rbs +0 -12
data/lib/httpx/io/udp.rb CHANGED
@@ -25,7 +25,7 @@ module HTTPX
25
25
  true
26
26
  end
27
27
 
28
- if RUBY_VERSION < "2.2"
28
+ if RUBY_VERSION < "2.3"
29
29
  # :nocov:
30
30
  def close
31
31
  @io.close
@@ -40,14 +40,15 @@ module HTTPX
40
40
  end
41
41
 
42
42
  def write(buffer)
43
- siz = @io.send(buffer, 0, @host, @port)
43
+ siz = @io.send(buffer.to_s, 0, @host, @port)
44
44
  log { "WRITE: #{siz} bytes..." }
45
45
  buffer.shift!(siz)
46
46
  siz
47
47
  end
48
48
 
49
49
  # :nocov:
50
- if RUBY_VERSION < "2.3"
50
+ if (RUBY_ENGINE == "truffleruby" && RUBY_ENGINE_VERSION < "21.1.0") ||
51
+ RUBY_VERSION < "2.3"
51
52
  def read(size, buffer)
52
53
  data, _ = @io.recvfrom_nonblock(size)
53
54
  buffer.replace(data)
@@ -66,7 +66,6 @@ module HTTPX
66
66
  @status_code = code.to_i
67
67
  raise(Error, "wrong status code (#{@status_code})") unless (100..599).cover?(@status_code)
68
68
 
69
- # @buffer.slice!(0, idx + 1)
70
69
  @buffer = @buffer.byteslice((idx + 1)..-1)
71
70
  nextstate(:headers)
72
71
  end
@@ -74,7 +73,8 @@ module HTTPX
74
73
  def parse_headers
75
74
  headers = @headers
76
75
  while (idx = @buffer.index("\n"))
77
- line = @buffer.slice!(0, idx + 1).sub(/\s+\z/, "")
76
+ line = @buffer.byteslice(0..idx).sub(/\s+\z/, "")
77
+ @buffer = @buffer.byteslice((idx + 1)..-1)
78
78
  if line.empty?
79
79
  case @state
80
80
  when :headers
@@ -96,11 +96,11 @@ module HTTPX
96
96
  separator_index = line.index(":")
97
97
  raise Error, "wrong header format" unless separator_index
98
98
 
99
- key = line[0..separator_index - 1]
99
+ key = line.byteslice(0..(separator_index - 1))
100
100
  raise Error, "wrong header format" if key.start_with?("\s", "\t")
101
101
 
102
102
  key.strip!
103
- value = line[separator_index + 1..-1]
103
+ value = line.byteslice((separator_index + 1)..-1)
104
104
  value.strip!
105
105
  raise Error, "wrong header format" if value.nil?
106
106
 
@@ -0,0 +1,81 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HTTPX
4
+ module Plugins
5
+ #
6
+ # This plugin applies AWS Sigv4 to requests, using the AWS SDK credentials and configuration.
7
+ #
8
+ # It requires the "aws-sdk-core" gem.
9
+ #
10
+ module AwsSdkAuthentication
11
+ #
12
+ # encapsulates access to an AWS SDK credentials store.
13
+ #
14
+ class Credentials
15
+ def initialize(aws_credentials)
16
+ @aws_credentials = aws_credentials
17
+ end
18
+
19
+ def username
20
+ @aws_credentials.access_key_id
21
+ end
22
+
23
+ def password
24
+ @aws_credentials.secret_access_key
25
+ end
26
+
27
+ def security_token
28
+ @aws_credentials.session_token
29
+ end
30
+ end
31
+
32
+ class << self
33
+ attr_reader :credentials, :region
34
+
35
+ def load_dependencies(klass)
36
+ require "aws-sdk-core"
37
+ klass.plugin(:aws_sigv4)
38
+
39
+ client = Class.new(Seahorse::Client::Base) do
40
+ @identifier = :httpx
41
+ set_api(Aws::S3::ClientApi::API)
42
+ add_plugin(Aws::Plugins::CredentialsConfiguration)
43
+ add_plugin(Aws::Plugins::RegionalEndpoint)
44
+ class << self
45
+ attr_reader :identifier
46
+ end
47
+ end.new
48
+
49
+ @credentials = Credentials.new(client.config[:credentials])
50
+ @region = client.config[:region]
51
+ end
52
+
53
+ def extra_options(options)
54
+ options.merge(max_concurrent_requests: 1)
55
+ end
56
+ end
57
+
58
+ module InstanceMethods
59
+ #
60
+ # aws_authentication
61
+ # aws_authentication(credentials: Aws::Credentials.new('akid', 'secret'))
62
+ # aws_authentication()
63
+ #
64
+ def aws_sdk_authentication(**options)
65
+ credentials = AwsSdkAuthentication.credentials
66
+ region = AwsSdkAuthentication.region
67
+
68
+ aws_sigv4_authentication(
69
+ credentials: credentials,
70
+ region: region,
71
+ provider_prefix: "aws",
72
+ header_provider_field: "amz",
73
+ **options
74
+ )
75
+ end
76
+ alias_method :aws_auth, :aws_sdk_authentication
77
+ end
78
+ end
79
+ register_plugin :aws_sdk_authentication, AwsSdkAuthentication
80
+ end
81
+ end
@@ -0,0 +1,218 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "set"
4
+ require "aws-sdk-s3"
5
+
6
+ module HTTPX
7
+ module Plugins
8
+ #
9
+ # This plugin adds AWS Sigv4 authentication.
10
+ #
11
+ # https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html
12
+ #
13
+ # https://gitlab.com/honeyryderchuck/httpx/wikis/AWS-SigV4
14
+ #
15
+ module AWSSigV4
16
+ Credentials = Struct.new(:username, :password, :security_token)
17
+
18
+ class Signer
19
+ def initialize(
20
+ service:,
21
+ region:,
22
+ credentials: nil,
23
+ username: nil,
24
+ password: nil,
25
+ security_token: nil,
26
+ provider_prefix: "aws",
27
+ header_provider_field: "amz",
28
+ unsigned_headers: [],
29
+ apply_checksum_header: true,
30
+ algorithm: "SHA256"
31
+ )
32
+ @credentials = credentials || Credentials.new(username, password, security_token)
33
+ @service = service
34
+ @region = region
35
+
36
+ @unsigned_headers = Set.new(unsigned_headers.map(&:downcase))
37
+ @unsigned_headers << "authorization"
38
+ @unsigned_headers << "x-amzn-trace-id"
39
+ @unsigned_headers << "expect"
40
+
41
+ @apply_checksum_header = apply_checksum_header
42
+ @provider_prefix = provider_prefix
43
+ @header_provider_field = header_provider_field
44
+
45
+ @algorithm = algorithm
46
+ end
47
+
48
+ def sign!(request)
49
+ lower_provider_prefix = "#{@provider_prefix}4"
50
+ upper_provider_prefix = lower_provider_prefix.upcase
51
+
52
+ downcased_algorithm = @algorithm.downcase
53
+
54
+ datetime = (request.headers["x-#{@header_provider_field}-date"] ||= Time.now.utc.strftime("%Y%m%dT%H%M%SZ"))
55
+ date = datetime[0, 8]
56
+
57
+ content_hashed = request.headers["x-#{@header_provider_field}-content-#{downcased_algorithm}"] || hexdigest(request.body)
58
+
59
+ request.headers["x-#{@header_provider_field}-content-#{downcased_algorithm}"] ||= content_hashed if @apply_checksum_header
60
+ request.headers["x-#{@header_provider_field}-security-token"] ||= @credentials.security_token if @credentials.security_token
61
+
62
+ signature_headers = request.headers.each.reject do |k, _|
63
+ @unsigned_headers.include?(k)
64
+ end
65
+ # aws sigv4 needs to declare the host, regardless of protocol version
66
+ signature_headers << ["host", request.authority] unless request.headers.key?("host")
67
+ signature_headers.sort_by!(&:first)
68
+
69
+ signed_headers = signature_headers.map(&:first).join(";")
70
+
71
+ canonical_headers = signature_headers.map do |k, v|
72
+ # eliminate whitespace between value fields, unless it's a quoted value
73
+ "#{k}:#{v.start_with?("\"") && v.end_with?("\"") ? v : v.gsub(/\s+/, " ").strip}\n"
74
+ end.join
75
+
76
+ # canonical request
77
+ creq = "#{request.verb.to_s.upcase}" \
78
+ "\n#{request.canonical_path}" \
79
+ "\n#{request.canonical_query}" \
80
+ "\n#{canonical_headers}" \
81
+ "\n#{signed_headers}" \
82
+ "\n#{content_hashed}"
83
+
84
+ credential_scope = "#{date}" \
85
+ "/#{@region}" \
86
+ "/#{@service}" \
87
+ "/#{lower_provider_prefix}_request"
88
+
89
+ algo_line = "#{upper_provider_prefix}-HMAC-#{@algorithm}"
90
+ # string to sign
91
+ sts = "#{algo_line}" \
92
+ "\n#{datetime}" \
93
+ "\n#{credential_scope}" \
94
+ "\n#{hexdigest(creq)}"
95
+
96
+ # signature
97
+ k_date = hmac("#{upper_provider_prefix}#{@credentials.password}", date)
98
+ k_region = hmac(k_date, @region)
99
+ k_service = hmac(k_region, @service)
100
+ k_credentials = hmac(k_service, "#{lower_provider_prefix}_request")
101
+ sig = hexhmac(k_credentials, sts)
102
+
103
+ credential = "#{@credentials.username}/#{credential_scope}"
104
+ # apply signature
105
+ request.headers["authorization"] =
106
+ "#{algo_line} " \
107
+ "Credential=#{credential}, " \
108
+ "SignedHeaders=#{signed_headers}, " \
109
+ "Signature=#{sig}"
110
+ end
111
+
112
+ private
113
+
114
+ def hexdigest(value)
115
+ if value.respond_to?(:to_path)
116
+ # files, pathnames
117
+ OpenSSL::Digest.new(@algorithm).file(value.to_path).hexdigest
118
+ elsif value.respond_to?(:each)
119
+ digest = OpenSSL::Digest.new(@algorithm)
120
+
121
+ mb_buffer = value.each.each_with_object("".b) do |chunk, buffer|
122
+ buffer << chunk
123
+ break if buffer.bytesize >= 1024 * 1024
124
+ end
125
+
126
+ digest.update(mb_buffer)
127
+ value.rewind
128
+ digest.hexdigest
129
+ else
130
+ OpenSSL::Digest.new(@algorithm).hexdigest(value)
131
+ end
132
+ end
133
+
134
+ def hmac(key, value)
135
+ OpenSSL::HMAC.digest(OpenSSL::Digest.new(@algorithm), key, value)
136
+ end
137
+
138
+ def hexhmac(key, value)
139
+ OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new(@algorithm), key, value)
140
+ end
141
+ end
142
+
143
+ class << self
144
+ def extra_options(options)
145
+ Class.new(options.class) do
146
+ def_option(:sigv4_signer) do |signer|
147
+ signer.is_a?(Signer) ? signer : Signer.new(signer)
148
+ end
149
+ end.new.merge(options)
150
+ end
151
+
152
+ def load_dependencies(klass)
153
+ require "openssl"
154
+ klass.plugin(:expect)
155
+ klass.plugin(:compression)
156
+ end
157
+ end
158
+
159
+ module InstanceMethods
160
+ def aws_sigv4_authentication(**options)
161
+ with(sigv4_signer: Signer.new(**options))
162
+ end
163
+
164
+ def build_request(*, _)
165
+ request = super
166
+
167
+ return request if request.headers.key?("authorization")
168
+
169
+ signer = request.options.sigv4_signer
170
+
171
+ return request unless signer
172
+
173
+ signer.sign!(request)
174
+
175
+ request
176
+ end
177
+ end
178
+
179
+ module RequestMethods
180
+ def canonical_path
181
+ path = uri.path.dup
182
+ path << "/" if path.empty?
183
+ path.gsub(%r{[^/]+}) { |part| CGI.escape(part.encode("UTF-8")).gsub("+", "%20").gsub("%7E", "~") }
184
+ end
185
+
186
+ def canonical_query
187
+ params = query.split("&")
188
+ # params = params.map { |p| p.match(/=/) ? p : p + '=' }
189
+ # From: https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
190
+ # Sort the parameter names by character code point in ascending order.
191
+ # Parameters with duplicate names should be sorted by value.
192
+ #
193
+ # Default sort <=> in JRuby will swap members
194
+ # occasionally when <=> is 0 (considered still sorted), but this
195
+ # causes our normalized query string to not match the sent querystring.
196
+ # When names match, we then sort by their values. When values also
197
+ # match then we sort by their original order
198
+ params.each.with_index.sort do |a, b|
199
+ a, a_offset = a
200
+ b, b_offset = b
201
+ a_name, a_value = a.split("=")
202
+ b_name, b_value = b.split("=")
203
+ if a_name == b_name
204
+ if a_value == b_value
205
+ a_offset <=> b_offset
206
+ else
207
+ a_value <=> b_value
208
+ end
209
+ else
210
+ a_name <=> b_name
211
+ end
212
+ end.map(&:first).join("&")
213
+ end
214
+ end
215
+ end
216
+ register_plugin :aws_sigv4, AWSSigV4
217
+ end
218
+ end
@@ -94,7 +94,7 @@ module HTTPX
94
94
  end
95
95
 
96
96
  def write(chunk)
97
- return super unless defined?(@_inflaters)
97
+ return super unless defined?(@_inflaters) && !chunk.empty?
98
98
 
99
99
  chunk = decompress(chunk)
100
100
  super(chunk)
@@ -18,10 +18,7 @@ module HTTPX
18
18
  module_function
19
19
 
20
20
  def deflate(raw, buffer, chunk_size:)
21
- deflater = Zlib::Deflate.new(Zlib::BEST_COMPRESSION,
22
- Zlib::MAX_WBITS,
23
- Zlib::MAX_MEM_LEVEL,
24
- Zlib::HUFFMAN_ONLY)
21
+ deflater = Zlib::Deflate.new
25
22
  while (chunk = raw.read(chunk_size))
26
23
  compressed = deflater.deflate(chunk)
27
24
  buffer << compressed
@@ -31,7 +28,7 @@ module HTTPX
31
28
  buffer << last
32
29
  yield last if block_given?
33
30
  ensure
34
- deflater.close
31
+ deflater.close if deflater
35
32
  end
36
33
  end
37
34
 
@@ -107,7 +107,7 @@ module HTTPX
107
107
  case aname
108
108
  when "expires"
109
109
  # RFC 6265 5.2.1
110
- (avalue &&= Time.httpdate(avalue)) || next
110
+ (avalue &&= Time.parse(avalue)) || next
111
111
  when "max-age"
112
112
  # RFC 6265 5.2.2
113
113
  next unless /\A-?\d+\z/.match?(avalue)
@@ -10,6 +10,10 @@ module HTTPX
10
10
  module Expect
11
11
  EXPECT_TIMEOUT = 2
12
12
 
13
+ def self.no_expect_store
14
+ @no_expect_store ||= []
15
+ end
16
+
13
17
  def self.extra_options(options)
14
18
  Class.new(options.class) do
15
19
  def_option(:expect_timeout) do |seconds|
@@ -28,25 +32,46 @@ module HTTPX
28
32
  end.new(options).merge(expect_timeout: EXPECT_TIMEOUT)
29
33
  end
30
34
 
31
- module RequestBodyMethods
32
- def initialize(*, options)
35
+ module RequestMethods
36
+ def initialize(*)
33
37
  super
34
- return if @body.nil?
38
+ return if @body.empty?
35
39
 
36
- threshold = options.expect_threshold_size
37
- return if threshold && !unbounded_body? && @body.bytesize < threshold
40
+ threshold = @options.expect_threshold_size
41
+ return if threshold && !@body.unbounded_body? && @body.bytesize < threshold
42
+
43
+ return if Expect.no_expect_store.include?(origin)
38
44
 
39
45
  @headers["expect"] = "100-continue"
40
46
  end
47
+
48
+ def response=(response)
49
+ if response && response.status == 100 &&
50
+ !@headers.key?("expect") &&
51
+ (@state == :body || @state == :done)
52
+
53
+ # if we're past this point, this means that we just received a 100-Continue response,
54
+ # but the request doesn't have the expect flag, and is already flushing (or flushed) the body.
55
+ #
56
+ # this means that expect was deactivated for this request too soon, i.e. response took longer.
57
+ #
58
+ # so we have to reactivate it again.
59
+ @headers["expect"] = "100-continue"
60
+ @informational_status = 100
61
+ Expect.no_expect_store.delete(origin)
62
+ end
63
+ super
64
+ end
41
65
  end
42
66
 
43
67
  module ConnectionMethods
44
68
  def send(request)
45
- request.once(:expects) do
69
+ request.once(:expect) do
46
70
  @timers.after(@options.expect_timeout) do
47
- if request.state == :expects && !request.expects?
71
+ if request.state == :expect && !request.expects?
72
+ Expect.no_expect_store << request.origin
48
73
  request.headers.delete("expect")
49
- handle(request)
74
+ consume
50
75
  end
51
76
  end
52
77
  end