httpx 0.19.4 → 0.19.7

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: c1c8ae2cc46f5a4033ea5c776bef1a7703cfc27834818cded1487101b8a6bbbe
4
- data.tar.gz: ea445112ce42d2d81c4465f646687d605da855e3248838d26075388463250d9b
3
+ metadata.gz: ca4deb7487ed711d1241a9d332fde1fbb5492d58497c26b4022aa255d3b0788b
4
+ data.tar.gz: 594ad514fff0cb9e98e9464b2091626583ecdd2f3f7b0754e2e5c14734560708
5
5
  SHA512:
6
- metadata.gz: effaefc6365aa24fe86a62bca9bdbfed701dc9c19354c6bf7f046ea2bf84b94f908e1924381f2e27cfe748c7ad4e0b8d97dfa0a6d695775cec8d1b8a92760fe7
7
- data.tar.gz: 55f846f8412acb99008cd18834ce50fe73ae072a0446d5d4fdf76fe2c48be4b2a863721b18191ca406a60b6883a8e35cba2f856c07ce9fa8ba808476b5b66887
6
+ metadata.gz: c852596dbab3faaf2c1849fd16c933879a452d82425fdbf398d289c4968f56a3e681f5ef25a5f64d5db2ed12a7f47469ce0bb3052e231514372db8d528e86e56
7
+ data.tar.gz: f79a5abca5e1bd3b26e729e69977e0324c1434cd740cdd177c8d058c6ac50f97c60a81e0d3cee50d336811b3c03a0f527e987bea423667ce8ddb42ed72db2f00
@@ -1,4 +1,4 @@
1
- # 0.19.3
1
+ # 0.19.4
2
2
 
3
3
  ## Improvements
4
4
 
@@ -10,4 +10,5 @@ The (optional) FFI-based TLS module for jruby was deleted. Besides it being cumb
10
10
 
11
11
  * `webmock` integration was fixed to take the mocked URI query string into account.
12
12
  * fix internal codepath where mergeable-but-not-coalescable connections were still triggering the coalesce branch.
13
- * fixed after-use mutation of connection addresses array which was making it empty after initial usage.
13
+ * fixed after-use mutation of connection addresses array which was making it empty after initial usage.
14
+ * fixed a "busy loop" caused by long-running native resolver not signaling it had "nothing to do".
@@ -0,0 +1,13 @@
1
+ # 0.19.5
2
+
3
+ ## Features
4
+
5
+ ### DNS: resolv.conf search/ndots options support (native/https resolvers)
6
+
7
+ Both the native (default) as well as the HTTPS (DoH) resolvers now support the "search" and "ndots" options, which adds domain "suffixes" under certain conditions to be used in name resolutions (this is a quite common feature found in kubernetes pods).
8
+
9
+ (While this means a new feature is being shipped in a patch release, one can argue that this feature "fixes" DNS in `httpx`.)
10
+
11
+ ## Bugfixes
12
+
13
+ * skipping headers comparison in HTTPX::Options#==; this had the unintended consequence of breaking connection reuse when crafting requests in a certain way, thereby making every request to the same origin issue their own connection, resulting, in multi-request scenarios (and with the `:persistent` plugin), in the process exhausting the max amount of allowed file descriptors.
@@ -0,0 +1,5 @@
1
+ # 0.19.6
2
+
3
+ ## Bugfixes
4
+
5
+ * fixed the root cause of the `faraday` adapter failing when used alongside the `webmock` integration.
@@ -0,0 +1,5 @@
1
+ # 0.19.7
2
+
3
+ ## Bugfixes
4
+
5
+ * fix: in `:proxy` plugin, user/pass credentials were not being used when passed as options (only when embedded in the URL). This was a regression introduced in the 0.19.x series.
@@ -99,33 +99,31 @@ module Faraday
99
99
  end
100
100
  end
101
101
 
102
- class Session < ::HTTPX::Session
103
- plugin(:compression)
104
- plugin(:persistent)
105
-
106
- module ReasonPlugin
102
+ module ReasonPlugin
103
+ if RUBY_VERSION < "2.5"
104
+ def self.load_dependencies(*)
105
+ require "webrick"
106
+ end
107
+ else
108
+ def self.load_dependencies(*)
109
+ require "net/http/status"
110
+ end
111
+ end
112
+ module ResponseMethods
107
113
  if RUBY_VERSION < "2.5"
108
- def self.load_dependencies(*)
109
- require "webrick"
114
+ def reason
115
+ WEBrick::HTTPStatus::StatusMessage.fetch(@status)
110
116
  end
111
117
  else
112
- def self.load_dependencies(*)
113
- require "net/http/status"
114
- end
115
- end
116
- module ResponseMethods
117
- if RUBY_VERSION < "2.5"
118
- def reason
119
- WEBrick::HTTPStatus::StatusMessage.fetch(@status)
120
- end
121
- else
122
- def reason
123
- Net::HTTP::STATUS_CODES.fetch(@status)
124
- end
118
+ def reason
119
+ Net::HTTP::STATUS_CODES.fetch(@status)
125
120
  end
126
121
  end
127
122
  end
128
- plugin(ReasonPlugin)
123
+ end
124
+
125
+ def self.session
126
+ @session ||= ::HTTPX.plugin(:compression).plugin(:persistent).plugin(ReasonPlugin)
129
127
  end
130
128
 
131
129
  class ParallelManager
@@ -161,7 +159,6 @@ module Faraday
161
159
  include RequestMixin
162
160
 
163
161
  def initialize
164
- @session = Session.new
165
162
  @handlers = []
166
163
  end
167
164
 
@@ -174,7 +171,7 @@ module Faraday
174
171
  def run
175
172
  env = @handlers.last.env
176
173
 
177
- session = @session.with(options_from_env(env))
174
+ session = HTTPX.session.with(options_from_env(env))
178
175
  session = session.plugin(:proxy).with(proxy: { uri: env.request.proxy }) if env.request.proxy
179
176
  session = session.plugin(OnDataPlugin) if env.request.stream_response?
180
177
 
@@ -205,7 +202,7 @@ module Faraday
205
202
 
206
203
  def initialize(app, options = {})
207
204
  super(app)
208
- @session = Session.new(options)
205
+ @session_options = options
209
206
  end
210
207
 
211
208
  def call(env)
@@ -221,7 +218,9 @@ module Faraday
221
218
  return handler
222
219
  end
223
220
 
224
- session = @session.with(options_from_env(env))
221
+ session = HTTPX.session
222
+ session = session.with(@session_options) unless @session_options.empty?
223
+ session = session.with(options_from_env(env))
225
224
  session = session.plugin(:proxy).with(proxy: { uri: env.request.proxy }) if env.request.proxy
226
225
  session = session.plugin(OnDataPlugin) if env.request.stream_response?
227
226
 
data/lib/httpx/headers.rb CHANGED
@@ -42,6 +42,8 @@ module HTTPX
42
42
  def same_headers?(headers)
43
43
  @headers.empty? || begin
44
44
  headers.each do |k, v|
45
+ next unless key?(k)
46
+
45
47
  return false unless v == self[k]
46
48
  end
47
49
  true
data/lib/httpx/options.rb CHANGED
@@ -209,8 +209,9 @@ module HTTPX
209
209
  ivars.all? do |ivar|
210
210
  case ivar
211
211
  when :@headers
212
- headers = instance_variable_get(ivar)
213
- headers.same_headers?(other.instance_variable_get(ivar))
212
+ # currently, this is used to pick up an available matching connection.
213
+ # the headers do not play a role, as they are relevant only for the request.
214
+ true
214
215
  when *REQUEST_IVARS
215
216
  true
216
217
  else
@@ -78,12 +78,6 @@ module HTTPX
78
78
  end
79
79
 
80
80
  module ConnectionMethods
81
- def match?(uri, options)
82
- return super unless @options.proxy
83
-
84
- super && @options.proxy == options.proxy
85
- end
86
-
87
81
  # should not coalesce connections here, as the IP is the IP of the proxy
88
82
  def coalescable?(*)
89
83
  return super unless @options.proxy
@@ -81,7 +81,11 @@ module HTTPX
81
81
  end
82
82
  uris
83
83
  end
84
- { uri: @_proxy_uris.first } unless @_proxy_uris.empty?
84
+ return if @_proxy_uris.empty?
85
+
86
+ proxy_opts = { uri: @_proxy_uris.first }
87
+ proxy_opts = options.proxy.merge(proxy_opts) if options.proxy
88
+ proxy_opts
85
89
  end
86
90
 
87
91
  def find_connection(request, connections, options)
@@ -174,12 +178,6 @@ module HTTPX
174
178
  @origin.port = proxy_uri.port
175
179
  end
176
180
 
177
- def match?(uri, options)
178
- return super unless @options.proxy
179
-
180
- super && @options.proxy == options.proxy
181
- end
182
-
183
181
  def coalescable?(connection)
184
182
  return super unless @options.proxy
185
183
 
@@ -11,6 +11,15 @@ module HTTPX
11
11
  using URIExtensions
12
12
  using StringExtensions
13
13
 
14
+ module DNSExtensions
15
+ refine Resolv::DNS do
16
+ def generate_candidates(name)
17
+ @config.generate_candidates(name)
18
+ end
19
+ end
20
+ end
21
+ using DNSExtensions
22
+
14
23
  NAMESERVER = "https://1.1.1.1/dns-query"
15
24
 
16
25
  DEFAULTS = {
@@ -76,30 +85,37 @@ module HTTPX
76
85
  if hostname.nil?
77
86
  hostname = connection.origin.host
78
87
  log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
88
+
89
+ hostname = @resolver.generate_candidates(hostname).each do |name|
90
+ @queries[name.to_s] = connection
91
+ end.first.to_s
92
+ else
93
+ @queries[hostname] = connection
79
94
  end
80
95
  log { "resolver: query #{FAMILY_TYPES[RECORD_TYPES[@family]]} for #{hostname}" }
96
+
81
97
  begin
82
98
  request = build_request(hostname)
83
99
  request.on(:response, &method(:on_response).curry(2)[request])
84
100
  request.on(:promise, &method(:on_promise))
85
- @requests[request] = connection
101
+ @requests[request] = hostname
86
102
  resolver_connection.send(request)
87
- @queries[hostname] = connection
88
103
  @connections << connection
89
104
  rescue ResolveError, Resolv::DNS::EncodeError, JSON::JSONError => e
90
- emit_resolve_error(connection, hostname, e)
105
+ @queries.delete(hostname)
106
+ emit_resolve_error(connection, connection.origin.host, e)
91
107
  end
92
108
  end
93
109
 
94
110
  def on_response(request, response)
95
111
  response.raise_for_status
96
112
  rescue StandardError => e
97
- connection = @requests[request]
98
- hostname = @queries.key(connection)
99
- emit_resolve_error(connection, hostname, e)
113
+ hostname = @requests.delete(request)
114
+ connection = @queries.delete(hostname)
115
+ emit_resolve_error(connection, connection.origin.host, e)
100
116
  else
101
117
  # @type var response: HTTPX::Response
102
- parse(response)
118
+ parse(request, response)
103
119
  ensure
104
120
  @requests.delete(request)
105
121
  end
@@ -109,20 +125,21 @@ module HTTPX
109
125
  stream.refuse
110
126
  end
111
127
 
112
- def parse(response)
128
+ def parse(request, response)
113
129
  begin
114
130
  answers = decode_response_body(response)
115
131
  rescue Resolv::DNS::DecodeError, JSON::JSONError => e
116
132
  host, connection = @queries.first
117
133
  @queries.delete(host)
118
- emit_resolve_error(connection, host, e)
134
+ emit_resolve_error(connection, connection.origin.host, e)
119
135
  return
120
136
  end
121
137
  if answers.nil? || answers.empty?
122
- host, connection = @queries.first
123
- @queries.delete(host)
124
- emit_resolve_error(connection, host)
138
+ host = @requests.delete(request)
139
+ connection = @queries.delete(host)
140
+ emit_resolve_error(connection)
125
141
  return
142
+
126
143
  else
127
144
  answers = answers.group_by { |answer| answer["name"] }
128
145
  answers.each do |hostname, addresses|
@@ -130,7 +147,6 @@ module HTTPX
130
147
  if address.key?("alias")
131
148
  alias_address = answers[address["alias"]]
132
149
  if alias_address.nil?
133
- connection = @queries[hostname]
134
150
  @queries.delete(address["name"])
135
151
  if catch(:coalesced) { early_resolve(connection, hostname: address["alias"]) }
136
152
  @connections.delete(connection)
@@ -152,6 +168,10 @@ module HTTPX
152
168
  next unless connection # probably a retried query for which there's an answer
153
169
 
154
170
  @connections.delete(connection)
171
+
172
+ # eliminate other candidates
173
+ @queries.delete_if { |_, conn| connection == conn }
174
+
155
175
  Resolver.cached_lookup_set(hostname, @family, addresses) if @resolver_options[:cache]
156
176
  emit_addresses(connection, @family, addresses.map { |addr| addr["data"] })
157
177
  end
@@ -46,6 +46,8 @@ module HTTPX
46
46
  @ns_index = 0
47
47
  @resolver_options = DEFAULTS.merge(@options.resolver_options)
48
48
  @nameserver = @resolver_options[:nameserver]
49
+ @ndots = @resolver_options[:ndots]
50
+ @search = Array(@resolver_options[:search]).map { |srch| srch.scan(/[^.]+/) }
49
51
  @_timeouts = Array(@resolver_options[:timeouts])
50
52
  @timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
51
53
  @connections = []
@@ -136,33 +138,32 @@ module HTTPX
136
138
  return if @queries.empty? || !@start_timeout
137
139
 
138
140
  loop_time = Utils.elapsed_time(@start_timeout)
139
- connections = []
140
- queries = {}
141
- while (query = @queries.shift)
142
- h, connection = query
143
- host = connection.origin.host
144
- timeout = (@timeouts[host][0] -= loop_time)
145
- unless timeout.negative?
146
- queries[h] = connection
147
- next
148
- end
149
141
 
150
- @timeouts[host].shift
151
- if @timeouts[host].empty?
152
- @timeouts.delete(host)
153
- @connections.delete(connection)
154
- # This loop_time passed to the exception is bogus. Ideally we would pass the total
155
- # resolve timeout, including from the previous retries.
156
- raise ResolveTimeoutError.new(loop_time, "Timed out while resolving #{host}")
157
- # raise NativeResolveError.new(connection, host)
158
- else
159
- log { "resolver: timeout after #{timeout}s, retry(#{@timeouts[host].first}) #{host}..." }
160
- connections << connection
161
- queries[h] = connection
162
- end
142
+ query = @queries.first
143
+
144
+ return unless query
145
+
146
+ h, connection = query
147
+ host = connection.origin.host
148
+ timeout = (@timeouts[host][0] -= loop_time)
149
+
150
+ return unless timeout.negative?
151
+
152
+ @timeouts[host].shift
153
+ if @timeouts[host].empty?
154
+ @timeouts.delete(host)
155
+ @queries.delete(h)
156
+
157
+ return unless @queries.empty?
158
+
159
+ @connections.delete(connection)
160
+ # This loop_time passed to the exception is bogus. Ideally we would pass the total
161
+ # resolve timeout, including from the previous retries.
162
+ raise ResolveTimeoutError.new(loop_time, "Timed out while resolving #{connection.origin.host}")
163
+ else
164
+ log { "resolver: timeout after #{timeout}s, retry(#{@timeouts[host].first}) #{host}..." }
165
+ resolve(connection)
163
166
  end
164
- @queries = queries
165
- connections.each { |ch| resolve(ch) }
166
167
  end
167
168
 
168
169
  def dread(wsize = @resolver_options[:packet_size])
@@ -194,7 +195,7 @@ module HTTPX
194
195
  @queries.delete(hostname)
195
196
  @timeouts.delete(hostname)
196
197
  @connections.delete(connection)
197
- ex = NativeResolveError.new(connection, hostname, e.message)
198
+ ex = NativeResolveError.new(connection, connection.origin.host, e.message)
198
199
  ex.set_backtrace(e.backtrace)
199
200
  raise ex
200
201
  end
@@ -203,9 +204,11 @@ module HTTPX
203
204
  hostname, connection = @queries.first
204
205
  @queries.delete(hostname)
205
206
  @timeouts.delete(hostname)
206
- @connections.delete(connection)
207
207
 
208
- raise NativeResolveError.new(connection, hostname)
208
+ unless @queries.value?(connection)
209
+ @connections.delete(connection)
210
+ raise NativeResolveError.new(connection, connection.origin.host)
211
+ end
209
212
  else
210
213
  address = addresses.first
211
214
  name = address["name"]
@@ -224,6 +227,9 @@ module HTTPX
224
227
  connection = @queries.delete(name)
225
228
  end
226
229
 
230
+ # eliminate other candidates
231
+ @queries.delete_if { |_, conn| connection == conn }
232
+
227
233
  if address.key?("alias") # CNAME
228
234
  # clean up intermediate queries
229
235
  @timeouts.delete(name) unless connection.origin.host == name
@@ -256,8 +262,13 @@ module HTTPX
256
262
  if hostname.nil?
257
263
  hostname = connection.origin.host
258
264
  log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
265
+
266
+ hostname = generate_candidates(hostname).each do |name|
267
+ @queries[name] = connection
268
+ end.first
269
+ else
270
+ @queries[hostname] = connection
259
271
  end
260
- @queries[hostname] = connection
261
272
  log { "resolver: query #{@record_type.name.split("::").last} for #{hostname}" }
262
273
  begin
263
274
  @write_buffer << Resolver.encode_dns_query(hostname, type: @record_type)
@@ -266,6 +277,18 @@ module HTTPX
266
277
  end
267
278
  end
268
279
 
280
+ def generate_candidates(name)
281
+ return [name] if name.end_with?(".")
282
+
283
+ candidates = []
284
+ name_parts = name.scan(/[^.]+/)
285
+ candidates = [name] if @ndots <= name_parts.size - 1
286
+ candidates.concat(@search.map { |domain| [*name_parts, *domain].join(".") })
287
+ candidates << name unless candidates.include?(name)
288
+
289
+ candidates
290
+ end
291
+
269
292
  def build_socket
270
293
  return if @io
271
294
 
@@ -82,6 +82,8 @@ module HTTPX
82
82
 
83
83
  _, connection = @queries.first
84
84
 
85
+ return unless connection
86
+
85
87
  @timeouts[connection.origin.host].first
86
88
  end
87
89
 
data/lib/httpx/version.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module HTTPX
4
- VERSION = "0.19.4"
4
+ VERSION = "0.19.7"
5
5
  end
@@ -8,7 +8,7 @@ module HTTPX
8
8
 
9
9
  @family: ip_family
10
10
  @options: Options
11
- @requests: Hash[Request, Connection]
11
+ @requests: Hash[Request, String]
12
12
  @connections: Array[Connection]
13
13
  @uri: URI::Generic
14
14
  @uri_addresses: Array[String]?
@@ -29,7 +29,7 @@ module HTTPX
29
29
 
30
30
  def on_response: (Request, response) -> void
31
31
 
32
- def parse: (Response response) -> void
32
+ def parse: (Request request, Response response) -> void
33
33
 
34
34
  def build_request: (String hostname) -> Request
35
35
 
@@ -46,6 +46,8 @@ module HTTPX
46
46
 
47
47
  def resolve: (?Connection connection, ?String hostname) -> void
48
48
 
49
+ def generate_candidates: (String) -> Array[String]
50
+
49
51
  def build_socket: () -> void
50
52
 
51
53
  def transition: (Symbol nextstate) -> void
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: httpx
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.19.4
4
+ version: 0.19.7
5
5
  platform: ruby
6
6
  authors:
7
7
  - Tiago Cardoso
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2022-03-06 00:00:00.000000000 Z
11
+ date: 2022-04-08 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: http-2-next
@@ -75,6 +75,9 @@ extra_rdoc_files:
75
75
  - doc/release_notes/0_19_2.md
76
76
  - doc/release_notes/0_19_3.md
77
77
  - doc/release_notes/0_19_4.md
78
+ - doc/release_notes/0_19_5.md
79
+ - doc/release_notes/0_19_6.md
80
+ - doc/release_notes/0_19_7.md
78
81
  - doc/release_notes/0_1_0.md
79
82
  - doc/release_notes/0_2_0.md
80
83
  - doc/release_notes/0_2_1.md
@@ -143,6 +146,9 @@ files:
143
146
  - doc/release_notes/0_19_2.md
144
147
  - doc/release_notes/0_19_3.md
145
148
  - doc/release_notes/0_19_4.md
149
+ - doc/release_notes/0_19_5.md
150
+ - doc/release_notes/0_19_6.md
151
+ - doc/release_notes/0_19_7.md
146
152
  - doc/release_notes/0_1_0.md
147
153
  - doc/release_notes/0_2_0.md
148
154
  - doc/release_notes/0_2_1.md