httpx 0.18.7 → 0.19.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +5 -1
  3. data/doc/release_notes/0_19_0.md +39 -0
  4. data/doc/release_notes/0_19_1.md +5 -0
  5. data/doc/release_notes/0_19_2.md +7 -0
  6. data/doc/release_notes/0_19_3.md +6 -0
  7. data/lib/httpx/adapters/faraday.rb +7 -3
  8. data/lib/httpx/connection.rb +14 -10
  9. data/lib/httpx/extensions.rb +16 -0
  10. data/lib/httpx/headers.rb +0 -2
  11. data/lib/httpx/io/tcp.rb +24 -5
  12. data/lib/httpx/options.rb +44 -11
  13. data/lib/httpx/plugins/cookies.rb +5 -7
  14. data/lib/httpx/plugins/proxy.rb +2 -5
  15. data/lib/httpx/plugins/retries.rb +2 -2
  16. data/lib/httpx/pool.rb +40 -20
  17. data/lib/httpx/resolver/https.rb +32 -42
  18. data/lib/httpx/resolver/multi.rb +79 -0
  19. data/lib/httpx/resolver/native.rb +24 -39
  20. data/lib/httpx/resolver/resolver.rb +95 -0
  21. data/lib/httpx/resolver/system.rb +175 -19
  22. data/lib/httpx/resolver.rb +37 -11
  23. data/lib/httpx/response.rb +4 -2
  24. data/lib/httpx/session_extensions.rb +9 -2
  25. data/lib/httpx/timers.rb +1 -1
  26. data/lib/httpx/transcoder/chunker.rb +0 -1
  27. data/lib/httpx/version.rb +1 -1
  28. data/lib/httpx.rb +2 -0
  29. data/sig/errors.rbs +8 -0
  30. data/sig/headers.rbs +0 -2
  31. data/sig/httpx.rbs +4 -0
  32. data/sig/options.rbs +10 -7
  33. data/sig/parser/http1.rbs +14 -5
  34. data/sig/pool.rbs +17 -9
  35. data/sig/registry.rbs +3 -0
  36. data/sig/request.rbs +11 -0
  37. data/sig/resolver/https.rbs +15 -27
  38. data/sig/resolver/multi.rbs +7 -0
  39. data/sig/resolver/native.rbs +3 -12
  40. data/sig/resolver/resolver.rbs +36 -0
  41. data/sig/resolver/system.rbs +3 -9
  42. data/sig/resolver.rbs +12 -10
  43. data/sig/response.rbs +15 -5
  44. data/sig/selector.rbs +3 -3
  45. data/sig/timers.rbs +5 -2
  46. data/sig/transcoder/chunker.rbs +16 -5
  47. data/sig/transcoder/json.rbs +5 -0
  48. data/sig/transcoder.rbs +3 -1
  49. metadata +14 -4
  50. data/lib/httpx/resolver/resolver_mixin.rb +0 -75
  51. data/sig/resolver/resolver_mixin.rbs +0 -26
@@ -6,32 +6,23 @@ require "cgi"
6
6
  require "forwardable"
7
7
 
8
8
  module HTTPX
9
- class Resolver::HTTPS
9
+ class Resolver::HTTPS < Resolver::Resolver
10
10
  extend Forwardable
11
- include Resolver::ResolverMixin
12
11
  using URIExtensions
12
+ using StringExtensions
13
13
 
14
14
  NAMESERVER = "https://1.1.1.1/dns-query"
15
15
 
16
- RECORD_TYPES = {
17
- "A" => Resolv::DNS::Resource::IN::A,
18
- "AAAA" => Resolv::DNS::Resource::IN::AAAA,
19
- }.freeze
20
-
21
16
  DEFAULTS = {
22
17
  uri: NAMESERVER,
23
18
  use_get: false,
24
- record_types: RECORD_TYPES.keys,
25
19
  }.freeze
26
20
 
27
21
  def_delegators :@resolver_connection, :state, :connecting?, :to_io, :call, :close
28
22
 
29
- attr_writer :pool
30
-
31
- def initialize(options)
32
- @options = HTTPX::Options.new(options)
23
+ def initialize(_, options)
24
+ super
33
25
  @resolver_options = DEFAULTS.merge(@options.resolver_options)
34
- @_record_types = Hash.new { |types, host| types[host] = @resolver_options[:record_types].dup }
35
26
  @queries = {}
36
27
  @requests = {}
37
28
  @connections = []
@@ -44,7 +35,7 @@ module HTTPX
44
35
  def <<(connection)
45
36
  return if @uri.origin == connection.origin.to_s
46
37
 
47
- @uri_addresses ||= ip_resolve(@uri.host) || system_resolve(@uri.host) || @resolver.getaddresses(@uri.host)
38
+ @uri_addresses ||= HTTPX::Resolver.nolookup_resolve(@uri.host) || @resolver.getaddresses(@uri.host)
48
39
 
49
40
  if @uri_addresses.empty?
50
41
  ex = ResolveError.new("Can't resolve DNS server #{@uri.host}")
@@ -52,7 +43,7 @@ module HTTPX
52
43
  throw(:resolve_error, ex)
53
44
  end
54
45
 
55
- early_resolve(connection) || resolve(connection)
46
+ resolve(connection)
56
47
  end
57
48
 
58
49
  def closed?
@@ -63,21 +54,22 @@ module HTTPX
63
54
  true
64
55
  end
65
56
 
66
- private
67
-
68
57
  def resolver_connection
69
58
  @resolver_connection ||= @pool.find_connection(@uri, @options) || begin
70
59
  @building_connection = true
71
60
  connection = @options.connection_class.new("ssl", @uri, @options.merge(ssl: { alpn_protocols: %w[h2] }))
72
61
  @pool.init_connection(connection, @options)
73
- emit_addresses(connection, @uri_addresses)
62
+ emit_addresses(connection, @family, @uri_addresses)
74
63
  @building_connection = false
75
64
  connection
76
65
  end
77
66
  end
78
67
 
68
+ private
69
+
79
70
  def resolve(connection = @connections.first, hostname = nil)
80
71
  return if @building_connection
72
+ return unless connection
81
73
 
82
74
  hostname ||= @queries.key(connection)
83
75
 
@@ -85,17 +77,16 @@ module HTTPX
85
77
  hostname = connection.origin.host
86
78
  log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
87
79
  end
88
- type = @_record_types[hostname].first || "A"
89
- log { "resolver: query #{type} for #{hostname}" }
80
+ log { "resolver: query #{FAMILY_TYPES[RECORD_TYPES[@family]]} for #{hostname}" }
90
81
  begin
91
- request = build_request(hostname, type)
82
+ request = build_request(hostname)
92
83
  request.on(:response, &method(:on_response).curry(2)[request])
93
84
  request.on(:promise, &method(:on_promise))
94
85
  @requests[request] = connection
95
86
  resolver_connection.send(request)
96
87
  @queries[hostname] = connection
97
88
  @connections << connection
98
- rescue Resolv::DNS::EncodeError, JSON::JSONError => e
89
+ rescue ResolveError, Resolv::DNS::EncodeError, JSON::JSONError => e
99
90
  emit_resolve_error(connection, hostname, e)
100
91
  end
101
92
  end
@@ -107,6 +98,7 @@ module HTTPX
107
98
  hostname = @queries.key(connection)
108
99
  emit_resolve_error(connection, hostname, e)
109
100
  else
101
+ # @type var response: HTTPX::Response
110
102
  parse(response)
111
103
  ensure
112
104
  @requests.delete(request)
@@ -122,21 +114,15 @@ module HTTPX
122
114
  answers = decode_response_body(response)
123
115
  rescue Resolv::DNS::DecodeError, JSON::JSONError => e
124
116
  host, connection = @queries.first
125
- if @_record_types[host].empty?
126
- @queries.delete(host)
127
- emit_resolve_error(connection, host, e)
128
- return
129
- end
117
+ @queries.delete(host)
118
+ emit_resolve_error(connection, host, e)
119
+ return
130
120
  end
131
121
  if answers.nil? || answers.empty?
132
122
  host, connection = @queries.first
133
- @_record_types[host].shift
134
- if @_record_types[host].empty?
135
- @queries.delete(host)
136
- @_record_types.delete(host)
137
- emit_resolve_error(connection, host)
138
- return
139
- end
123
+ @queries.delete(host)
124
+ emit_resolve_error(connection, host)
125
+ return
140
126
  else
141
127
  answers = answers.group_by { |answer| answer["name"] }
142
128
  answers.each do |hostname, addresses|
@@ -146,8 +132,12 @@ module HTTPX
146
132
  if alias_address.nil?
147
133
  connection = @queries[hostname]
148
134
  @queries.delete(address["name"])
149
- resolve(connection, address["alias"])
150
- return # rubocop:disable Lint/NonLocalExitFromIterator
135
+ if catch(:coalesced) { early_resolve(connection, hostname: address["alias"]) }
136
+ @connections.delete(connection)
137
+ else
138
+ resolve(connection, address["alias"])
139
+ return # rubocop:disable Lint/NonLocalExitFromIterator
140
+ end
151
141
  else
152
142
  alias_address
153
143
  end
@@ -157,13 +147,13 @@ module HTTPX
157
147
  end.compact
158
148
  next if addresses.empty?
159
149
 
160
- hostname = hostname[0..-2] if hostname.end_with?(".")
150
+ hostname.delete_suffix!(".") if hostname.end_with?(".")
161
151
  connection = @queries.delete(hostname)
162
152
  next unless connection # probably a retried query for which there's an answer
163
153
 
164
154
  @connections.delete(connection)
165
- Resolver.cached_lookup_set(hostname, addresses) if @resolver_options[:cache]
166
- emit_addresses(connection, addresses.map { |addr| addr["data"] })
155
+ Resolver.cached_lookup_set(hostname, @family, addresses) if @resolver_options[:cache]
156
+ emit_addresses(connection, @family, addresses.map { |addr| addr["data"] })
167
157
  end
168
158
  end
169
159
  return if @connections.empty?
@@ -171,14 +161,14 @@ module HTTPX
171
161
  resolve
172
162
  end
173
163
 
174
- def build_request(hostname, type)
164
+ def build_request(hostname)
175
165
  uri = @uri.dup
176
166
  rklass = @options.request_class
177
- payload = Resolver.encode_dns_query(hostname, type: RECORD_TYPES[type])
167
+ payload = Resolver.encode_dns_query(hostname, type: @record_type)
178
168
 
179
169
  if @resolver_options[:use_get]
180
170
  params = URI.decode_www_form(uri.query.to_s)
181
- params << ["type", type]
171
+ params << ["type", FAMILY_TYPES[@record_type]]
182
172
  params << ["dns", Base64.urlsafe_encode64(payload, padding: false)]
183
173
  uri.query = URI.encode_www_form(params)
184
174
  request = rklass.new("GET", uri, @options)
@@ -0,0 +1,79 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "forwardable"
4
+ require "resolv"
5
+
6
+ module HTTPX
7
+ class Resolver::Multi
8
+ include Callbacks
9
+ using ArrayExtensions
10
+
11
+ attr_reader :resolvers
12
+
13
+ def initialize(resolver_type, options)
14
+ @options = options
15
+ @resolver_options = @options.resolver_options
16
+
17
+ @resolvers = options.ip_families.map do |ip_family|
18
+ resolver = resolver_type.new(ip_family, options)
19
+ resolver.on(:resolve, &method(:on_resolver_connection))
20
+ resolver.on(:error, &method(:on_resolver_error))
21
+ resolver.on(:close) { on_resolver_close(resolver) }
22
+ resolver
23
+ end
24
+
25
+ @errors = Hash.new { |hs, k| hs[k] = [] }
26
+ end
27
+
28
+ def closed?
29
+ @resolvers.all?(&:closed?)
30
+ end
31
+
32
+ def timeout
33
+ @resolvers.filter_map(&:timeout).min
34
+ end
35
+
36
+ def close
37
+ @resolvers.each(&:close)
38
+ end
39
+
40
+ def connections
41
+ @resolvers.filter_map { |r| r.resolver_connection if r.respond_to?(:resolver_connection) }
42
+ end
43
+
44
+ def early_resolve(connection)
45
+ hostname = connection.origin.host
46
+ addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
47
+ return unless addresses
48
+
49
+ addresses = addresses.group_by(&:family)
50
+
51
+ @resolvers.each do |resolver|
52
+ addrs = addresses[resolver.family]
53
+
54
+ next if !addrs || addrs.empty?
55
+
56
+ resolver.emit_addresses(connection, resolver.family, addrs)
57
+ end
58
+ end
59
+
60
+ private
61
+
62
+ def on_resolver_connection(connection)
63
+ emit(:resolve, connection)
64
+ end
65
+
66
+ def on_resolver_error(connection, error)
67
+ @errors[connection] << error
68
+
69
+ return unless @errors[connection].size >= @resolvers.size
70
+
71
+ errors = @errors.delete(connection)
72
+ emit(:error, connection, errors.first)
73
+ end
74
+
75
+ def on_resolver_close(resolver)
76
+ emit(:close, resolver)
77
+ end
78
+ end
79
+ end
@@ -4,22 +4,15 @@ require "forwardable"
4
4
  require "resolv"
5
5
 
6
6
  module HTTPX
7
- class Resolver::Native
7
+ class Resolver::Native < Resolver::Resolver
8
8
  extend Forwardable
9
- include Resolver::ResolverMixin
10
9
  using URIExtensions
11
10
 
12
- RECORD_TYPES = {
13
- "A" => Resolv::DNS::Resource::IN::A,
14
- "AAAA" => Resolv::DNS::Resource::IN::AAAA,
15
- }.freeze
16
-
17
11
  DEFAULTS = if RUBY_VERSION < "2.2"
18
12
  {
19
13
  **Resolv::DNS::Config.default_config_hash,
20
14
  packet_size: 512,
21
15
  timeouts: Resolver::RESOLVE_TIMEOUT,
22
- record_types: RECORD_TYPES.keys,
23
16
  }.freeze
24
17
  else
25
18
  {
@@ -27,7 +20,6 @@ module HTTPX
27
20
  **Resolv::DNS::Config.default_config_hash,
28
21
  packet_size: 512,
29
22
  timeouts: Resolver::RESOLVE_TIMEOUT,
30
- record_types: RECORD_TYPES.keys,
31
23
  }.freeze
32
24
  end
33
25
 
@@ -49,14 +41,13 @@ module HTTPX
49
41
 
50
42
  attr_reader :state
51
43
 
52
- def initialize(options)
53
- @options = HTTPX::Options.new(options)
44
+ def initialize(_, options)
45
+ super
54
46
  @ns_index = 0
55
47
  @resolver_options = DEFAULTS.merge(@options.resolver_options)
56
48
  @nameserver = @resolver_options[:nameserver]
57
49
  @_timeouts = Array(@resolver_options[:timeouts])
58
50
  @timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
59
- @_record_types = Hash.new { |types, host| types[host] = @resolver_options[:record_types].dup }
60
51
  @connections = []
61
52
  @queries = {}
62
53
  @read_buffer = "".b
@@ -107,8 +98,6 @@ module HTTPX
107
98
  end
108
99
 
109
100
  def <<(connection)
110
- return if early_resolve(connection)
111
-
112
101
  if @nameserver.nil?
113
102
  ex = ResolveError.new("No available nameserver")
114
103
  ex.set_backtrace(caller)
@@ -140,7 +129,7 @@ module HTTPX
140
129
  end
141
130
 
142
131
  def do_retry
143
- return if @queries.empty?
132
+ return if @queries.empty? || !@start_timeout
144
133
 
145
134
  loop_time = Utils.elapsed_time(@start_timeout)
146
135
  connections = []
@@ -160,7 +149,7 @@ module HTTPX
160
149
  @connections.delete(connection)
161
150
  # This loop_time passed to the exception is bogus. Ideally we would pass the total
162
151
  # resolve timeout, including from the previous retries.
163
- raise ResolveTimeoutError.new(loop_time, "Timed out")
152
+ raise ResolveTimeoutError.new(loop_time, "Timed out while resolving #{host}")
164
153
  # raise NativeResolveError.new(connection, host)
165
154
  else
166
155
  log { "resolver: timeout after #{timeout}s, retry(#{@timeouts[host].first}) #{host}..." }
@@ -198,27 +187,21 @@ module HTTPX
198
187
  addresses = Resolver.decode_dns_answer(buffer)
199
188
  rescue Resolv::DNS::DecodeError => e
200
189
  hostname, connection = @queries.first
201
- if @_record_types[hostname].empty?
202
- @queries.delete(hostname)
203
- @timeouts.delete(hostname)
204
- @connections.delete(connection)
205
- ex = NativeResolveError.new(connection, hostname, e.message)
206
- ex.set_backtrace(e.backtrace)
207
- raise ex
208
- end
190
+ @queries.delete(hostname)
191
+ @timeouts.delete(hostname)
192
+ @connections.delete(connection)
193
+ ex = NativeResolveError.new(connection, hostname, e.message)
194
+ ex.set_backtrace(e.backtrace)
195
+ raise ex
209
196
  end
210
197
 
211
198
  if addresses.nil? || addresses.empty?
212
199
  hostname, connection = @queries.first
213
- @_record_types[hostname].shift
214
- if @_record_types[hostname].empty?
215
- @queries.delete(hostname)
216
- @_record_types.delete(hostname)
217
- @timeouts.delete(hostname)
218
- @connections.delete(connection)
200
+ @queries.delete(hostname)
201
+ @timeouts.delete(hostname)
202
+ @connections.delete(connection)
219
203
 
220
- raise NativeResolveError.new(connection, hostname)
221
- end
204
+ raise NativeResolveError.new(connection, hostname)
222
205
  else
223
206
  address = addresses.first
224
207
  name = address["name"]
@@ -239,20 +222,20 @@ module HTTPX
239
222
 
240
223
  if address.key?("alias") # CNAME
241
224
  # clean up intermediate queries
242
- @timeouts.delete(address["name"]) unless connection.origin.host == address["name"]
225
+ @timeouts.delete(name) unless connection.origin.host == name
243
226
 
244
227
  if catch(:coalesced) { early_resolve(connection, hostname: address["alias"]) }
245
- @timeouts.delete(connection.origin.host)
246
228
  @connections.delete(connection)
247
229
  else
248
230
  resolve(connection, address["alias"])
249
231
  return
250
232
  end
251
233
  else
234
+ @timeouts.delete(name)
252
235
  @timeouts.delete(connection.origin.host)
253
236
  @connections.delete(connection)
254
- Resolver.cached_lookup_set(connection.origin.host, addresses) if @resolver_options[:cache]
255
- emit_addresses(connection, addresses.map { |addr| addr["data"] })
237
+ Resolver.cached_lookup_set(connection.origin.host, @family, addresses) if @resolver_options[:cache]
238
+ emit_addresses(connection, @family, addresses.map { |addr| addr["data"] })
256
239
  end
257
240
  end
258
241
  return emit(:close) if @connections.empty?
@@ -271,10 +254,9 @@ module HTTPX
271
254
  log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
272
255
  end
273
256
  @queries[hostname] = connection
274
- type = @_record_types[hostname].first || "A"
275
- log { "resolver: query #{type} for #{hostname}" }
257
+ log { "resolver: query #{@record_type.name.split("::").last} for #{hostname}" }
276
258
  begin
277
- @write_buffer << Resolver.encode_dns_query(hostname, type: RECORD_TYPES[type])
259
+ @write_buffer << Resolver.encode_dns_query(hostname, type: @record_type)
278
260
  rescue Resolv::DNS::EncodeError => e
279
261
  emit_resolve_error(connection, hostname, e)
280
262
  end
@@ -313,6 +295,9 @@ module HTTPX
313
295
  return unless @state == :open
314
296
 
315
297
  @io.close if @io
298
+ @start_timeout = nil
299
+ @write_buffer.clear
300
+ @read_buffer.clear
316
301
  end
317
302
  @state = nextstate
318
303
  end
@@ -0,0 +1,95 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "resolv"
4
+ require "ipaddr"
5
+
6
+ module HTTPX
7
+ class Resolver::Resolver
8
+ include Callbacks
9
+ include Loggable
10
+
11
+ RECORD_TYPES = {
12
+ Socket::AF_INET6 => Resolv::DNS::Resource::IN::AAAA,
13
+ Socket::AF_INET => Resolv::DNS::Resource::IN::A,
14
+ }.freeze
15
+
16
+ FAMILY_TYPES = {
17
+ Resolv::DNS::Resource::IN::AAAA => "AAAA",
18
+ Resolv::DNS::Resource::IN::A => "A",
19
+ }.freeze
20
+
21
+ class << self
22
+ def multi?
23
+ true
24
+ end
25
+ end
26
+
27
+ attr_reader :family
28
+
29
+ attr_writer :pool
30
+
31
+ def initialize(family, options)
32
+ @family = family
33
+ @record_type = RECORD_TYPES[family]
34
+ @options = Options.new(options)
35
+ end
36
+
37
+ def close; end
38
+
39
+ def closed?
40
+ true
41
+ end
42
+
43
+ def empty?
44
+ true
45
+ end
46
+
47
+ def emit_addresses(connection, family, addresses)
48
+ addresses.map! do |address|
49
+ address.is_a?(IPAddr) ? address : IPAddr.new(address.to_s)
50
+ end
51
+ log { "resolver: answer #{connection.origin.host}: #{addresses.inspect}" }
52
+ if @pool && # if triggered by early resolve, pool may not be here yet
53
+ !connection.io &&
54
+ connection.options.ip_families.size > 1 &&
55
+ family == Socket::AF_INET &&
56
+ addresses.first.to_s != connection.origin.host.to_s
57
+ log { "resolver: A response, applying resolution delay..." }
58
+ @pool.after(0.05) do
59
+ connection.addresses = addresses
60
+ emit(:resolve, connection)
61
+ end
62
+ else
63
+ connection.addresses = addresses
64
+ emit(:resolve, connection)
65
+ end
66
+ end
67
+
68
+ private
69
+
70
+ def early_resolve(connection, hostname: connection.origin.host)
71
+ addresses = @resolver_options[:cache] && (connection.addresses || HTTPX::Resolver.nolookup_resolve(hostname))
72
+
73
+ return unless addresses
74
+
75
+ addresses.select! { |addr| addr.family == @family }
76
+
77
+ return if addresses.empty?
78
+
79
+ emit_addresses(connection, @family, addresses)
80
+ end
81
+
82
+ def emit_resolve_error(connection, hostname = connection.origin.host, ex = nil)
83
+ emit(:error, connection, resolve_error(hostname, ex))
84
+ end
85
+
86
+ def resolve_error(hostname, ex = nil)
87
+ return ex if ex.is_a?(ResolveError)
88
+
89
+ message = ex ? ex.message : "Can't resolve #{hostname}"
90
+ error = ResolveError.new(message)
91
+ error.set_backtrace(ex ? ex.backtrace : caller)
92
+ error
93
+ end
94
+ end
95
+ end