httpx 0.7.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: e0f44b6fc0f644a3f70b58a7051e963c5f74f9a5cb8ce6ccde0f6d2aa8909b14
4
- data.tar.gz: d341c978e49312b3abe26c24845b672be5055fd4a4a2cab9e2aa9d1acf79aa63
3
+ metadata.gz: 4ac832ee93155b950d27dde3d7295fae2b817a8b2c55c881ec07f73e97736c59
4
+ data.tar.gz: 28251067c648230eed4435b9df7cf75bfc1c552162f37928a8ee6c5b485d6c3b
5
5
  SHA512:
6
- metadata.gz: c6a9e8caa8f2424c8628153535deda5f1d8623253801ae329884ee51f10c5f0e5237d4c6e2548faa40d14c0e0e3dd982c050e6f67521a8b2a56e39a1bbab40aa
7
- data.tar.gz: db1b34ff9023343b4f9580b1a72c54765cf7d446f59f96bea22ec9230825ce42cd42f0ac26ec168cd3b721e75e6cf5c2964a05afea1895a778d8f27932f61d65
6
+ metadata.gz: d7b9f1419af3bb823ff58de80dca6a933bf25a756e27ff35f76c5ad7948ad1aab1c1f63f8eaf20bc9fc05bb0b2ed4932fd5ff1e33bcd7a8cda061bee2c54e4ac
7
+ data.tar.gz: f410653e0e1646f36eeaf90746bfae17d8c344abdf44eb92df2a992897d24843d16e370ec54f6441bedfe779981d17e4b9c5316f73115adc77e0639cb8dba53f
@@ -42,7 +42,23 @@ module HTTPX
42
42
 
43
43
  origin = request.origin
44
44
  host = request.uri.host
45
- parse(response.headers["alt-svc"]) do |alt_origin, alt_params|
45
+
46
+ altsvc = response.headers["alt-svc"]
47
+
48
+ # https://tools.ietf.org/html/rfc7838#section-3
49
+ # A field value containing the special value "clear" indicates that the
50
+ # origin requests all alternatives for that origin to be invalidated
51
+ # (including those specified in the same response, in case of an
52
+ # invalid reply containing both "clear" and alternative services).
53
+ if altsvc == "clear"
54
+ @altsvc_mutex.synchronize do
55
+ @altsvcs[origin].clear
56
+ end
57
+
58
+ return
59
+ end
60
+
61
+ parse(altsvc) do |alt_origin, alt_params|
46
62
  alt_origin.host ||= host
47
63
  yield(alt_origin, origin, alt_params)
48
64
  end
@@ -73,7 +89,7 @@ module HTTPX
73
89
  alt_proto, alt_origin = alt_origin.split("=")
74
90
  alt_origin = alt_origin[1..-2] if alt_origin.start_with?("\"") && alt_origin.end_with?("\"")
75
91
  if alt_origin.start_with?(":")
76
- alt_origin = "dummy#{alt_origin}"
92
+ alt_origin = "#{alt_proto}://dummy#{alt_origin}"
77
93
  uri = URI.parse(alt_origin)
78
94
  uri.host = nil
79
95
  uri
@@ -3,9 +3,11 @@
3
3
  module HTTPX
4
4
  module Chainable
5
5
  %i[head get post put delete trace options connect patch].each do |meth|
6
- define_method meth do |*uri, **options|
7
- request(meth, uri, **options)
8
- end
6
+ class_eval(<<-MOD, __FILE__, __LINE__ + 1)
7
+ def #{meth}(*uri, **options)
8
+ request(:#{meth}, uri, **options)
9
+ end
10
+ MOD
9
11
  end
10
12
 
11
13
  def request(verb, uri, **options)
@@ -67,6 +67,10 @@ module HTTPX
67
67
  else
68
68
  transition(:idle)
69
69
  end
70
+
71
+ @inflight = 0
72
+ @keep_alive_timeout = options.timeout.keep_alive_timeout
73
+ @keep_alive_timer = nil
70
74
  end
71
75
 
72
76
  # this is a semi-private method, to be used by the resolver
@@ -84,6 +88,8 @@ module HTTPX
84
88
 
85
89
  return false if exhausted?
86
90
 
91
+ return false if @keep_alive_timer && @keep_alive_timer.fires_in.negative?
92
+
87
93
  (
88
94
  (
89
95
  @origins.include?(uri.origin) &&
@@ -101,6 +107,8 @@ module HTTPX
101
107
 
102
108
  return false if exhausted?
103
109
 
110
+ return false if @keep_alive_timer && @keep_alive_timer.fires_in.negative?
111
+
104
112
  !(@io.addresses & connection.addresses).empty? && @options == connection.options
105
113
  end
106
114
 
@@ -140,7 +148,10 @@ module HTTPX
140
148
 
141
149
  def purge_pending
142
150
  pendings = []
143
- pendings << @parser.pending if @parser
151
+ if @parser
152
+ @inflight -= @parser.pending.size
153
+ pendings << @parser.pending
154
+ end
144
155
  pendings << @pending
145
156
  pendings.each do |pending|
146
157
  pending.reject! do |request|
@@ -168,22 +179,45 @@ module HTTPX
168
179
  end
169
180
 
170
181
  def interests
171
- return :w if @state == :idle
182
+ # connecting
183
+ if connecting?
184
+ connect
172
185
 
173
- :rw
186
+ return @io.interests if connecting?
187
+ end
188
+
189
+ # if the write buffer is full, we drain it
190
+ return :w if @write_buffer.full?
191
+
192
+ return @parser.interests if @parser
193
+
194
+ nil
174
195
  end
175
196
 
176
197
  def to_io
198
+ @io.to_io
199
+ end
200
+
201
+ def call
177
202
  case @state
178
- when :idle
179
- transition(:open)
203
+ when :closed
204
+ return
205
+ when :closing
206
+ consume
207
+ transition(:closed)
208
+ emit(:close)
209
+ when :open
210
+ consume
180
211
  end
181
- @io.to_io
212
+ nil
182
213
  end
183
214
 
184
215
  def close
185
216
  @parser.close if @parser
186
- transition(:closing)
217
+ return unless @keep_alive_timer
218
+
219
+ @keep_alive_timer.cancel
220
+ remove_instance_variable(:@keep_alive_timer)
187
221
  end
188
222
 
189
223
  def reset
@@ -195,26 +229,14 @@ module HTTPX
195
229
  def send(request)
196
230
  if @parser && !@write_buffer.full?
197
231
  request.headers["alt-used"] = @origin.authority if match_altsvcs?(request.uri)
232
+ @inflight += 1
233
+ @keep_alive_timer.pause if @keep_alive_timer
198
234
  parser.send(request)
199
235
  else
200
236
  @pending << request
201
237
  end
202
238
  end
203
239
 
204
- def call
205
- case @state
206
- when :closed
207
- return
208
- when :closing
209
- dwrite
210
- transition(:closed)
211
- emit(:close)
212
- when :open
213
- consume
214
- end
215
- nil
216
- end
217
-
218
240
  def timeout
219
241
  return @timeout if defined?(@timeout)
220
242
 
@@ -225,54 +247,94 @@ module HTTPX
225
247
 
226
248
  private
227
249
 
250
+ def connect
251
+ transition(:open)
252
+ end
253
+
228
254
  def exhausted?
229
255
  @parser && parser.exhausted?
230
256
  end
231
257
 
232
258
  def consume
233
259
  catch(:called) do
234
- dread
235
- dwrite
236
- parser.consume
237
- end
238
- end
239
-
240
- def dread(wsize = @window_size)
241
- loop do
242
- siz = @io.read(wsize, @read_buffer)
243
- unless siz
244
- ex = EOFError.new("descriptor closed")
245
- ex.set_backtrace(caller)
246
- on_error(ex)
247
- return
248
- end
249
- return if siz.zero?
250
-
251
- log { "READ: #{siz} bytes..." }
252
- parser << @read_buffer.to_s
253
- return if @state == :closing || @state == :closed
254
- end
255
- end
256
-
257
- def dwrite
258
- loop do
259
- return if @write_buffer.empty?
260
-
261
- siz = @io.write(@write_buffer)
262
- unless siz
263
- ex = EOFError.new("descriptor closed")
264
- ex.set_backtrace(caller)
265
- on_error(ex)
266
- return
260
+ loop do
261
+ parser.consume
262
+
263
+ # we exit if there's no more data to process
264
+ if @pending.size.zero? && @inflight.zero?
265
+ log(level: 3) { "NO MORE REQUESTS..." }
266
+ return
267
+ end
268
+
269
+ @timeout = @current_timeout
270
+
271
+ read_drained = false
272
+ write_drained = nil
273
+
274
+ # dread
275
+ loop do
276
+ siz = @io.read(@window_size, @read_buffer)
277
+ unless siz
278
+ ex = EOFError.new("descriptor closed")
279
+ ex.set_backtrace(caller)
280
+ on_error(ex)
281
+ return
282
+ end
283
+
284
+ log { "READ: #{siz} bytes..." }
285
+
286
+ if siz.zero?
287
+ read_drained = @read_buffer.empty?
288
+ break
289
+ end
290
+
291
+ parser << @read_buffer.to_s
292
+
293
+ break if @state == :closing || @state == :closed
294
+
295
+ # for HTTP/2, we just want to write goaway frame
296
+ end unless @state == :closing
297
+
298
+ # dwrite
299
+ loop do
300
+ if @write_buffer.empty?
301
+ # we only mark as drained on the first loop
302
+ write_drained = write_drained.nil? && @inflight.positive?
303
+ break
304
+ end
305
+
306
+ siz = @io.write(@write_buffer)
307
+ unless siz
308
+ ex = EOFError.new("descriptor closed")
309
+ ex.set_backtrace(caller)
310
+ on_error(ex)
311
+ return
312
+ end
313
+ log { "WRITE: #{siz} bytes..." }
314
+
315
+ if siz.zero?
316
+ write_drained = !@write_buffer.empty?
317
+ break
318
+ end
319
+
320
+ break if @state == :closing || @state == :closed
321
+
322
+ write_drained = false
323
+ end
324
+
325
+ # return if socket is drained
326
+ if read_drained && write_drained
327
+ log(level: 3) { "WAITING FOR EVENTS..." }
328
+ return
329
+ end
267
330
  end
268
- log { "WRITE: #{siz} bytes..." }
269
- return if siz.zero?
270
- return if @state == :closing || @state == :closed
271
331
  end
272
332
  end
273
333
 
274
334
  def send_pending
275
335
  while !@write_buffer.full? && (request = @pending.shift)
336
+ @inflight += 1
337
+ @keep_alive_timer.pause if @keep_alive_timer
276
338
  parser.send(request)
277
339
  end
278
340
  end
@@ -292,6 +354,7 @@ module HTTPX
292
354
  AltSvc.emit(request, response) do |alt_origin, origin, alt_params|
293
355
  emit(:altsvc, alt_origin, origin, alt_params)
294
356
  end
357
+ handle_response
295
358
  request.emit(:response, response)
296
359
  end
297
360
  parser.on(:altsvc) do |alt_origin, origin, alt_params|
@@ -307,14 +370,21 @@ module HTTPX
307
370
  parser.on(:origin) do |origin|
308
371
  @origins << origin
309
372
  end
310
- parser.on(:close) do
373
+ parser.on(:close) do |force|
311
374
  transition(:closing)
375
+ if force
376
+ transition(:closed)
377
+ emit(:close)
378
+ end
312
379
  end
313
380
  parser.on(:reset) do
314
- transition(:closing)
315
- unless parser.empty?
381
+ if parser.empty?
382
+ reset
383
+ else
384
+ transition(:closing)
316
385
  transition(:closed)
317
386
  emit(:reset)
387
+ @parser.reset if @parser
318
388
  transition(:idle)
319
389
  transition(:open)
320
390
  end
@@ -335,6 +405,9 @@ module HTTPX
335
405
 
336
406
  def transition(nextstate)
337
407
  case nextstate
408
+ when :idle
409
+ @timeout = @current_timeout = @options.timeout.connect_timeout
410
+
338
411
  when :open
339
412
  return if @state == :closed
340
413
 
@@ -344,6 +417,8 @@ module HTTPX
344
417
  return unless @io.connected?
345
418
 
346
419
  send_pending
420
+
421
+ @timeout = @current_timeout = @options.timeout.operation_timeout
347
422
  emit(:open)
348
423
  when :closing
349
424
  return unless @state == :open
@@ -358,6 +433,11 @@ module HTTPX
358
433
 
359
434
  @io.close
360
435
  @read_buffer.clear
436
+ if @keep_alive_timer
437
+ @keep_alive_timer.cancel
438
+ remove_instance_variable(:@keep_alive_timer)
439
+ end
440
+
361
441
  remove_instance_variable(:@timeout) if defined?(@timeout)
362
442
  when :already_open
363
443
  nextstate = :open
@@ -379,21 +459,43 @@ module HTTPX
379
459
  emit(:close)
380
460
  end
381
461
 
382
- def on_error(ex)
383
- handle_error(ex)
384
- reset
462
+ def handle_response
463
+ @inflight -= 1
464
+ return unless @inflight.zero?
465
+
466
+ if @keep_alive_timer
467
+ @keep_alive_timer.resume
468
+ @keep_alive_timer.reset
469
+ else
470
+ @keep_alive_timer = @timers.after(@keep_alive_timeout) do
471
+ unless @inflight.zero?
472
+ log { "(#{object_id})) keep alive timeout expired, closing..." }
473
+ reset
474
+ end
475
+ end
476
+ end
385
477
  end
386
478
 
387
- def handle_error(error)
479
+ def on_error(error)
388
480
  if error.instance_of?(TimeoutError)
389
481
  if @timeout
390
482
  @timeout -= error.timeout
391
483
  return unless @timeout <= 0
392
484
  end
393
485
 
394
- error = error.to_connection_error if connecting?
486
+ if @total_timeout && @total_timeout.fires_in.negative?
487
+ ex = TotalTimeoutError.new(@total_timeout.interval, "Timed out after #{@total_timeout.interval} seconds")
488
+ ex.set_backtrace(error.backtrace)
489
+ error = ex
490
+ elsif connecting?
491
+ error = error.to_connection_error
492
+ end
395
493
  end
494
+ handle_error(error)
495
+ reset
496
+ end
396
497
 
498
+ def handle_error(error)
397
499
  parser.handle_error(error) if @parser && parser.respond_to?(:handle_error)
398
500
  while (request = @pending.shift)
399
501
  request.emit(:response, ErrorResponse.new(request, error, @options))
@@ -408,8 +510,8 @@ module HTTPX
408
510
  @total_timeout ||= @timers.after(total) do
409
511
  ex = TotalTimeoutError.new(total, "Timed out after #{total} seconds")
410
512
  ex.set_backtrace(caller)
411
- @parser.close if @parser
412
513
  on_error(ex)
514
+ @parser.close if @parser
413
515
  end
414
516
  end
415
517
  end
@@ -23,6 +23,19 @@ module HTTPX
23
23
  @requests = []
24
24
  end
25
25
 
26
+ def interests
27
+ # this means we're processing incoming response already
28
+ return :r if @request
29
+
30
+ return if @requests.empty?
31
+
32
+ request = @requests.first
33
+
34
+ return :w if request.interests == :w || !@buffer.empty?
35
+
36
+ :r
37
+ end
38
+
26
39
  def reset
27
40
  @max_requests = @options.max_requests || MAX_REQUESTS
28
41
  @parser.reset!
@@ -30,7 +43,7 @@ module HTTPX
30
43
 
31
44
  def close
32
45
  reset
33
- emit(:close)
46
+ emit(:close, true)
34
47
  end
35
48
 
36
49
  def exhausted?
@@ -53,16 +66,18 @@ module HTTPX
53
66
  return
54
67
  end
55
68
 
56
- unless @requests.include?(request)
57
- @requests << request
58
- @pipelining = true if @requests.size > 1
59
- end
69
+ return if @requests.include?(request)
60
70
 
61
- handle(request)
71
+ @requests << request
72
+ @pipelining = true if @requests.size > 1
62
73
  end
63
74
 
64
75
  def consume
65
- @requests.each do |request|
76
+ requests_limit = [@max_concurrent_requests, @max_requests, @requests.size].min
77
+ @requests.each_with_index do |request, idx|
78
+ break if idx >= requests_limit
79
+ next if request.state == :done
80
+
66
81
  handle(request)
67
82
  end
68
83
  end
@@ -121,7 +136,7 @@ module HTTPX
121
136
 
122
137
  def dispatch
123
138
  if @request.expects?
124
- reset
139
+ @parser.reset!
125
140
  return handle(@request)
126
141
  end
127
142
 
@@ -136,10 +151,10 @@ module HTTPX
136
151
  throw(:called)
137
152
  end
138
153
 
139
- reset
154
+ @parser.reset!
140
155
  @max_requests -= 1
141
- send(@pending.shift) unless @pending.empty?
142
156
  manage_connection(response)
157
+ send(@pending.shift) unless @pending.empty?
143
158
  end
144
159
 
145
160
  def handle_error(ex)
@@ -149,6 +164,9 @@ module HTTPX
149
164
  @requests.each do |request|
150
165
  emit(:error, request, ex)
151
166
  end
167
+ @pending.each do |request|
168
+ emit(:error, request, ex)
169
+ end
152
170
  end
153
171
  end
154
172