httpx 0.18.7 → 0.19.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +5 -1
  3. data/doc/release_notes/0_19_0.md +39 -0
  4. data/doc/release_notes/0_19_1.md +5 -0
  5. data/doc/release_notes/0_19_2.md +7 -0
  6. data/doc/release_notes/0_19_3.md +6 -0
  7. data/lib/httpx/adapters/faraday.rb +7 -3
  8. data/lib/httpx/connection.rb +14 -10
  9. data/lib/httpx/extensions.rb +16 -0
  10. data/lib/httpx/headers.rb +0 -2
  11. data/lib/httpx/io/tcp.rb +24 -5
  12. data/lib/httpx/options.rb +44 -11
  13. data/lib/httpx/plugins/cookies.rb +5 -7
  14. data/lib/httpx/plugins/proxy.rb +2 -5
  15. data/lib/httpx/plugins/retries.rb +2 -2
  16. data/lib/httpx/pool.rb +40 -20
  17. data/lib/httpx/resolver/https.rb +32 -42
  18. data/lib/httpx/resolver/multi.rb +79 -0
  19. data/lib/httpx/resolver/native.rb +24 -39
  20. data/lib/httpx/resolver/resolver.rb +95 -0
  21. data/lib/httpx/resolver/system.rb +175 -19
  22. data/lib/httpx/resolver.rb +37 -11
  23. data/lib/httpx/response.rb +4 -2
  24. data/lib/httpx/session_extensions.rb +9 -2
  25. data/lib/httpx/timers.rb +1 -1
  26. data/lib/httpx/transcoder/chunker.rb +0 -1
  27. data/lib/httpx/version.rb +1 -1
  28. data/lib/httpx.rb +2 -0
  29. data/sig/errors.rbs +8 -0
  30. data/sig/headers.rbs +0 -2
  31. data/sig/httpx.rbs +4 -0
  32. data/sig/options.rbs +10 -7
  33. data/sig/parser/http1.rbs +14 -5
  34. data/sig/pool.rbs +17 -9
  35. data/sig/registry.rbs +3 -0
  36. data/sig/request.rbs +11 -0
  37. data/sig/resolver/https.rbs +15 -27
  38. data/sig/resolver/multi.rbs +7 -0
  39. data/sig/resolver/native.rbs +3 -12
  40. data/sig/resolver/resolver.rbs +36 -0
  41. data/sig/resolver/system.rbs +3 -9
  42. data/sig/resolver.rbs +12 -10
  43. data/sig/response.rbs +15 -5
  44. data/sig/selector.rbs +3 -3
  45. data/sig/timers.rbs +5 -2
  46. data/sig/transcoder/chunker.rbs +16 -5
  47. data/sig/transcoder/json.rbs +5 -0
  48. data/sig/transcoder.rbs +3 -1
  49. metadata +14 -4
  50. data/lib/httpx/resolver/resolver_mixin.rb +0 -75
  51. data/sig/resolver/resolver_mixin.rbs +0 -26
@@ -6,32 +6,23 @@ require "cgi"
6
6
  require "forwardable"
7
7
 
8
8
  module HTTPX
9
- class Resolver::HTTPS
9
+ class Resolver::HTTPS < Resolver::Resolver
10
10
  extend Forwardable
11
- include Resolver::ResolverMixin
12
11
  using URIExtensions
12
+ using StringExtensions
13
13
 
14
14
  NAMESERVER = "https://1.1.1.1/dns-query"
15
15
 
16
- RECORD_TYPES = {
17
- "A" => Resolv::DNS::Resource::IN::A,
18
- "AAAA" => Resolv::DNS::Resource::IN::AAAA,
19
- }.freeze
20
-
21
16
  DEFAULTS = {
22
17
  uri: NAMESERVER,
23
18
  use_get: false,
24
- record_types: RECORD_TYPES.keys,
25
19
  }.freeze
26
20
 
27
21
  def_delegators :@resolver_connection, :state, :connecting?, :to_io, :call, :close
28
22
 
29
- attr_writer :pool
30
-
31
- def initialize(options)
32
- @options = HTTPX::Options.new(options)
23
+ def initialize(_, options)
24
+ super
33
25
  @resolver_options = DEFAULTS.merge(@options.resolver_options)
34
- @_record_types = Hash.new { |types, host| types[host] = @resolver_options[:record_types].dup }
35
26
  @queries = {}
36
27
  @requests = {}
37
28
  @connections = []
@@ -44,7 +35,7 @@ module HTTPX
44
35
  def <<(connection)
45
36
  return if @uri.origin == connection.origin.to_s
46
37
 
47
- @uri_addresses ||= ip_resolve(@uri.host) || system_resolve(@uri.host) || @resolver.getaddresses(@uri.host)
38
+ @uri_addresses ||= HTTPX::Resolver.nolookup_resolve(@uri.host) || @resolver.getaddresses(@uri.host)
48
39
 
49
40
  if @uri_addresses.empty?
50
41
  ex = ResolveError.new("Can't resolve DNS server #{@uri.host}")
@@ -52,7 +43,7 @@ module HTTPX
52
43
  throw(:resolve_error, ex)
53
44
  end
54
45
 
55
- early_resolve(connection) || resolve(connection)
46
+ resolve(connection)
56
47
  end
57
48
 
58
49
  def closed?
@@ -63,21 +54,22 @@ module HTTPX
63
54
  true
64
55
  end
65
56
 
66
- private
67
-
68
57
  def resolver_connection
69
58
  @resolver_connection ||= @pool.find_connection(@uri, @options) || begin
70
59
  @building_connection = true
71
60
  connection = @options.connection_class.new("ssl", @uri, @options.merge(ssl: { alpn_protocols: %w[h2] }))
72
61
  @pool.init_connection(connection, @options)
73
- emit_addresses(connection, @uri_addresses)
62
+ emit_addresses(connection, @family, @uri_addresses)
74
63
  @building_connection = false
75
64
  connection
76
65
  end
77
66
  end
78
67
 
68
+ private
69
+
79
70
  def resolve(connection = @connections.first, hostname = nil)
80
71
  return if @building_connection
72
+ return unless connection
81
73
 
82
74
  hostname ||= @queries.key(connection)
83
75
 
@@ -85,17 +77,16 @@ module HTTPX
85
77
  hostname = connection.origin.host
86
78
  log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
87
79
  end
88
- type = @_record_types[hostname].first || "A"
89
- log { "resolver: query #{type} for #{hostname}" }
80
+ log { "resolver: query #{FAMILY_TYPES[RECORD_TYPES[@family]]} for #{hostname}" }
90
81
  begin
91
- request = build_request(hostname, type)
82
+ request = build_request(hostname)
92
83
  request.on(:response, &method(:on_response).curry(2)[request])
93
84
  request.on(:promise, &method(:on_promise))
94
85
  @requests[request] = connection
95
86
  resolver_connection.send(request)
96
87
  @queries[hostname] = connection
97
88
  @connections << connection
98
- rescue Resolv::DNS::EncodeError, JSON::JSONError => e
89
+ rescue ResolveError, Resolv::DNS::EncodeError, JSON::JSONError => e
99
90
  emit_resolve_error(connection, hostname, e)
100
91
  end
101
92
  end
@@ -107,6 +98,7 @@ module HTTPX
107
98
  hostname = @queries.key(connection)
108
99
  emit_resolve_error(connection, hostname, e)
109
100
  else
101
+ # @type var response: HTTPX::Response
110
102
  parse(response)
111
103
  ensure
112
104
  @requests.delete(request)
@@ -122,21 +114,15 @@ module HTTPX
122
114
  answers = decode_response_body(response)
123
115
  rescue Resolv::DNS::DecodeError, JSON::JSONError => e
124
116
  host, connection = @queries.first
125
- if @_record_types[host].empty?
126
- @queries.delete(host)
127
- emit_resolve_error(connection, host, e)
128
- return
129
- end
117
+ @queries.delete(host)
118
+ emit_resolve_error(connection, host, e)
119
+ return
130
120
  end
131
121
  if answers.nil? || answers.empty?
132
122
  host, connection = @queries.first
133
- @_record_types[host].shift
134
- if @_record_types[host].empty?
135
- @queries.delete(host)
136
- @_record_types.delete(host)
137
- emit_resolve_error(connection, host)
138
- return
139
- end
123
+ @queries.delete(host)
124
+ emit_resolve_error(connection, host)
125
+ return
140
126
  else
141
127
  answers = answers.group_by { |answer| answer["name"] }
142
128
  answers.each do |hostname, addresses|
@@ -146,8 +132,12 @@ module HTTPX
146
132
  if alias_address.nil?
147
133
  connection = @queries[hostname]
148
134
  @queries.delete(address["name"])
149
- resolve(connection, address["alias"])
150
- return # rubocop:disable Lint/NonLocalExitFromIterator
135
+ if catch(:coalesced) { early_resolve(connection, hostname: address["alias"]) }
136
+ @connections.delete(connection)
137
+ else
138
+ resolve(connection, address["alias"])
139
+ return # rubocop:disable Lint/NonLocalExitFromIterator
140
+ end
151
141
  else
152
142
  alias_address
153
143
  end
@@ -157,13 +147,13 @@ module HTTPX
157
147
  end.compact
158
148
  next if addresses.empty?
159
149
 
160
- hostname = hostname[0..-2] if hostname.end_with?(".")
150
+ hostname.delete_suffix!(".") if hostname.end_with?(".")
161
151
  connection = @queries.delete(hostname)
162
152
  next unless connection # probably a retried query for which there's an answer
163
153
 
164
154
  @connections.delete(connection)
165
- Resolver.cached_lookup_set(hostname, addresses) if @resolver_options[:cache]
166
- emit_addresses(connection, addresses.map { |addr| addr["data"] })
155
+ Resolver.cached_lookup_set(hostname, @family, addresses) if @resolver_options[:cache]
156
+ emit_addresses(connection, @family, addresses.map { |addr| addr["data"] })
167
157
  end
168
158
  end
169
159
  return if @connections.empty?
@@ -171,14 +161,14 @@ module HTTPX
171
161
  resolve
172
162
  end
173
163
 
174
- def build_request(hostname, type)
164
+ def build_request(hostname)
175
165
  uri = @uri.dup
176
166
  rklass = @options.request_class
177
- payload = Resolver.encode_dns_query(hostname, type: RECORD_TYPES[type])
167
+ payload = Resolver.encode_dns_query(hostname, type: @record_type)
178
168
 
179
169
  if @resolver_options[:use_get]
180
170
  params = URI.decode_www_form(uri.query.to_s)
181
- params << ["type", type]
171
+ params << ["type", FAMILY_TYPES[@record_type]]
182
172
  params << ["dns", Base64.urlsafe_encode64(payload, padding: false)]
183
173
  uri.query = URI.encode_www_form(params)
184
174
  request = rklass.new("GET", uri, @options)
@@ -0,0 +1,79 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "forwardable"
4
+ require "resolv"
5
+
6
+ module HTTPX
7
+ class Resolver::Multi
8
+ include Callbacks
9
+ using ArrayExtensions
10
+
11
+ attr_reader :resolvers
12
+
13
+ def initialize(resolver_type, options)
14
+ @options = options
15
+ @resolver_options = @options.resolver_options
16
+
17
+ @resolvers = options.ip_families.map do |ip_family|
18
+ resolver = resolver_type.new(ip_family, options)
19
+ resolver.on(:resolve, &method(:on_resolver_connection))
20
+ resolver.on(:error, &method(:on_resolver_error))
21
+ resolver.on(:close) { on_resolver_close(resolver) }
22
+ resolver
23
+ end
24
+
25
+ @errors = Hash.new { |hs, k| hs[k] = [] }
26
+ end
27
+
28
+ def closed?
29
+ @resolvers.all?(&:closed?)
30
+ end
31
+
32
+ def timeout
33
+ @resolvers.filter_map(&:timeout).min
34
+ end
35
+
36
+ def close
37
+ @resolvers.each(&:close)
38
+ end
39
+
40
+ def connections
41
+ @resolvers.filter_map { |r| r.resolver_connection if r.respond_to?(:resolver_connection) }
42
+ end
43
+
44
+ def early_resolve(connection)
45
+ hostname = connection.origin.host
46
+ addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
47
+ return unless addresses
48
+
49
+ addresses = addresses.group_by(&:family)
50
+
51
+ @resolvers.each do |resolver|
52
+ addrs = addresses[resolver.family]
53
+
54
+ next if !addrs || addrs.empty?
55
+
56
+ resolver.emit_addresses(connection, resolver.family, addrs)
57
+ end
58
+ end
59
+
60
+ private
61
+
62
+ def on_resolver_connection(connection)
63
+ emit(:resolve, connection)
64
+ end
65
+
66
+ def on_resolver_error(connection, error)
67
+ @errors[connection] << error
68
+
69
+ return unless @errors[connection].size >= @resolvers.size
70
+
71
+ errors = @errors.delete(connection)
72
+ emit(:error, connection, errors.first)
73
+ end
74
+
75
+ def on_resolver_close(resolver)
76
+ emit(:close, resolver)
77
+ end
78
+ end
79
+ end
@@ -4,22 +4,15 @@ require "forwardable"
4
4
  require "resolv"
5
5
 
6
6
  module HTTPX
7
- class Resolver::Native
7
+ class Resolver::Native < Resolver::Resolver
8
8
  extend Forwardable
9
- include Resolver::ResolverMixin
10
9
  using URIExtensions
11
10
 
12
- RECORD_TYPES = {
13
- "A" => Resolv::DNS::Resource::IN::A,
14
- "AAAA" => Resolv::DNS::Resource::IN::AAAA,
15
- }.freeze
16
-
17
11
  DEFAULTS = if RUBY_VERSION < "2.2"
18
12
  {
19
13
  **Resolv::DNS::Config.default_config_hash,
20
14
  packet_size: 512,
21
15
  timeouts: Resolver::RESOLVE_TIMEOUT,
22
- record_types: RECORD_TYPES.keys,
23
16
  }.freeze
24
17
  else
25
18
  {
@@ -27,7 +20,6 @@ module HTTPX
27
20
  **Resolv::DNS::Config.default_config_hash,
28
21
  packet_size: 512,
29
22
  timeouts: Resolver::RESOLVE_TIMEOUT,
30
- record_types: RECORD_TYPES.keys,
31
23
  }.freeze
32
24
  end
33
25
 
@@ -49,14 +41,13 @@ module HTTPX
49
41
 
50
42
  attr_reader :state
51
43
 
52
- def initialize(options)
53
- @options = HTTPX::Options.new(options)
44
+ def initialize(_, options)
45
+ super
54
46
  @ns_index = 0
55
47
  @resolver_options = DEFAULTS.merge(@options.resolver_options)
56
48
  @nameserver = @resolver_options[:nameserver]
57
49
  @_timeouts = Array(@resolver_options[:timeouts])
58
50
  @timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
59
- @_record_types = Hash.new { |types, host| types[host] = @resolver_options[:record_types].dup }
60
51
  @connections = []
61
52
  @queries = {}
62
53
  @read_buffer = "".b
@@ -107,8 +98,6 @@ module HTTPX
107
98
  end
108
99
 
109
100
  def <<(connection)
110
- return if early_resolve(connection)
111
-
112
101
  if @nameserver.nil?
113
102
  ex = ResolveError.new("No available nameserver")
114
103
  ex.set_backtrace(caller)
@@ -140,7 +129,7 @@ module HTTPX
140
129
  end
141
130
 
142
131
  def do_retry
143
- return if @queries.empty?
132
+ return if @queries.empty? || !@start_timeout
144
133
 
145
134
  loop_time = Utils.elapsed_time(@start_timeout)
146
135
  connections = []
@@ -160,7 +149,7 @@ module HTTPX
160
149
  @connections.delete(connection)
161
150
  # This loop_time passed to the exception is bogus. Ideally we would pass the total
162
151
  # resolve timeout, including from the previous retries.
163
- raise ResolveTimeoutError.new(loop_time, "Timed out")
152
+ raise ResolveTimeoutError.new(loop_time, "Timed out while resolving #{host}")
164
153
  # raise NativeResolveError.new(connection, host)
165
154
  else
166
155
  log { "resolver: timeout after #{timeout}s, retry(#{@timeouts[host].first}) #{host}..." }
@@ -198,27 +187,21 @@ module HTTPX
198
187
  addresses = Resolver.decode_dns_answer(buffer)
199
188
  rescue Resolv::DNS::DecodeError => e
200
189
  hostname, connection = @queries.first
201
- if @_record_types[hostname].empty?
202
- @queries.delete(hostname)
203
- @timeouts.delete(hostname)
204
- @connections.delete(connection)
205
- ex = NativeResolveError.new(connection, hostname, e.message)
206
- ex.set_backtrace(e.backtrace)
207
- raise ex
208
- end
190
+ @queries.delete(hostname)
191
+ @timeouts.delete(hostname)
192
+ @connections.delete(connection)
193
+ ex = NativeResolveError.new(connection, hostname, e.message)
194
+ ex.set_backtrace(e.backtrace)
195
+ raise ex
209
196
  end
210
197
 
211
198
  if addresses.nil? || addresses.empty?
212
199
  hostname, connection = @queries.first
213
- @_record_types[hostname].shift
214
- if @_record_types[hostname].empty?
215
- @queries.delete(hostname)
216
- @_record_types.delete(hostname)
217
- @timeouts.delete(hostname)
218
- @connections.delete(connection)
200
+ @queries.delete(hostname)
201
+ @timeouts.delete(hostname)
202
+ @connections.delete(connection)
219
203
 
220
- raise NativeResolveError.new(connection, hostname)
221
- end
204
+ raise NativeResolveError.new(connection, hostname)
222
205
  else
223
206
  address = addresses.first
224
207
  name = address["name"]
@@ -239,20 +222,20 @@ module HTTPX
239
222
 
240
223
  if address.key?("alias") # CNAME
241
224
  # clean up intermediate queries
242
- @timeouts.delete(address["name"]) unless connection.origin.host == address["name"]
225
+ @timeouts.delete(name) unless connection.origin.host == name
243
226
 
244
227
  if catch(:coalesced) { early_resolve(connection, hostname: address["alias"]) }
245
- @timeouts.delete(connection.origin.host)
246
228
  @connections.delete(connection)
247
229
  else
248
230
  resolve(connection, address["alias"])
249
231
  return
250
232
  end
251
233
  else
234
+ @timeouts.delete(name)
252
235
  @timeouts.delete(connection.origin.host)
253
236
  @connections.delete(connection)
254
- Resolver.cached_lookup_set(connection.origin.host, addresses) if @resolver_options[:cache]
255
- emit_addresses(connection, addresses.map { |addr| addr["data"] })
237
+ Resolver.cached_lookup_set(connection.origin.host, @family, addresses) if @resolver_options[:cache]
238
+ emit_addresses(connection, @family, addresses.map { |addr| addr["data"] })
256
239
  end
257
240
  end
258
241
  return emit(:close) if @connections.empty?
@@ -271,10 +254,9 @@ module HTTPX
271
254
  log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
272
255
  end
273
256
  @queries[hostname] = connection
274
- type = @_record_types[hostname].first || "A"
275
- log { "resolver: query #{type} for #{hostname}" }
257
+ log { "resolver: query #{@record_type.name.split("::").last} for #{hostname}" }
276
258
  begin
277
- @write_buffer << Resolver.encode_dns_query(hostname, type: RECORD_TYPES[type])
259
+ @write_buffer << Resolver.encode_dns_query(hostname, type: @record_type)
278
260
  rescue Resolv::DNS::EncodeError => e
279
261
  emit_resolve_error(connection, hostname, e)
280
262
  end
@@ -313,6 +295,9 @@ module HTTPX
313
295
  return unless @state == :open
314
296
 
315
297
  @io.close if @io
298
+ @start_timeout = nil
299
+ @write_buffer.clear
300
+ @read_buffer.clear
316
301
  end
317
302
  @state = nextstate
318
303
  end
@@ -0,0 +1,95 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "resolv"
4
+ require "ipaddr"
5
+
6
+ module HTTPX
7
+ class Resolver::Resolver
8
+ include Callbacks
9
+ include Loggable
10
+
11
+ RECORD_TYPES = {
12
+ Socket::AF_INET6 => Resolv::DNS::Resource::IN::AAAA,
13
+ Socket::AF_INET => Resolv::DNS::Resource::IN::A,
14
+ }.freeze
15
+
16
+ FAMILY_TYPES = {
17
+ Resolv::DNS::Resource::IN::AAAA => "AAAA",
18
+ Resolv::DNS::Resource::IN::A => "A",
19
+ }.freeze
20
+
21
+ class << self
22
+ def multi?
23
+ true
24
+ end
25
+ end
26
+
27
+ attr_reader :family
28
+
29
+ attr_writer :pool
30
+
31
+ def initialize(family, options)
32
+ @family = family
33
+ @record_type = RECORD_TYPES[family]
34
+ @options = Options.new(options)
35
+ end
36
+
37
+ def close; end
38
+
39
+ def closed?
40
+ true
41
+ end
42
+
43
+ def empty?
44
+ true
45
+ end
46
+
47
+ def emit_addresses(connection, family, addresses)
48
+ addresses.map! do |address|
49
+ address.is_a?(IPAddr) ? address : IPAddr.new(address.to_s)
50
+ end
51
+ log { "resolver: answer #{connection.origin.host}: #{addresses.inspect}" }
52
+ if @pool && # if triggered by early resolve, pool may not be here yet
53
+ !connection.io &&
54
+ connection.options.ip_families.size > 1 &&
55
+ family == Socket::AF_INET &&
56
+ addresses.first.to_s != connection.origin.host.to_s
57
+ log { "resolver: A response, applying resolution delay..." }
58
+ @pool.after(0.05) do
59
+ connection.addresses = addresses
60
+ emit(:resolve, connection)
61
+ end
62
+ else
63
+ connection.addresses = addresses
64
+ emit(:resolve, connection)
65
+ end
66
+ end
67
+
68
+ private
69
+
70
+ def early_resolve(connection, hostname: connection.origin.host)
71
+ addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
72
+
73
+ return unless addresses
74
+
75
+ addresses.select! { |addr| addr.family == @family }
76
+
77
+ return if addresses.empty?
78
+
79
+ emit_addresses(connection, @family, addresses)
80
+ end
81
+
82
+ def emit_resolve_error(connection, hostname = connection.origin.host, ex = nil)
83
+ emit(:error, connection, resolve_error(hostname, ex))
84
+ end
85
+
86
+ def resolve_error(hostname, ex = nil)
87
+ return ex if ex.is_a?(ResolveError)
88
+
89
+ message = ex ? ex.message : "Can't resolve #{hostname}"
90
+ error = ResolveError.new(message)
91
+ error.set_backtrace(ex ? ex.backtrace : caller)
92
+ error
93
+ end
94
+ end
95
+ end