httpx 0.6.7 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +7 -5
  3. data/doc/release_notes/0_0_1.md +7 -0
  4. data/doc/release_notes/0_0_2.md +9 -0
  5. data/doc/release_notes/0_0_3.md +9 -0
  6. data/doc/release_notes/0_0_4.md +7 -0
  7. data/doc/release_notes/0_0_5.md +5 -0
  8. data/doc/release_notes/0_1_0.md +9 -0
  9. data/doc/release_notes/0_2_0.md +5 -0
  10. data/doc/release_notes/0_2_1.md +16 -0
  11. data/doc/release_notes/0_3_0.md +12 -0
  12. data/doc/release_notes/0_3_1.md +6 -0
  13. data/doc/release_notes/0_4_0.md +51 -0
  14. data/doc/release_notes/0_4_1.md +3 -0
  15. data/doc/release_notes/0_5_0.md +15 -0
  16. data/doc/release_notes/0_5_1.md +14 -0
  17. data/doc/release_notes/0_6_0.md +5 -0
  18. data/doc/release_notes/0_6_1.md +6 -0
  19. data/doc/release_notes/0_6_2.md +6 -0
  20. data/doc/release_notes/0_6_3.md +13 -0
  21. data/doc/release_notes/0_6_4.md +21 -0
  22. data/doc/release_notes/0_6_5.md +22 -0
  23. data/doc/release_notes/0_6_6.md +19 -0
  24. data/doc/release_notes/0_6_7.md +5 -0
  25. data/doc/release_notes/0_7_0.md +46 -0
  26. data/doc/release_notes/0_8_0.md +27 -0
  27. data/doc/release_notes/0_8_1.md +8 -0
  28. data/doc/release_notes/0_8_2.md +7 -0
  29. data/doc/release_notes/0_9_0.md +38 -0
  30. data/lib/httpx/adapters/faraday.rb +2 -2
  31. data/lib/httpx/altsvc.rb +18 -2
  32. data/lib/httpx/chainable.rb +27 -9
  33. data/lib/httpx/connection.rb +215 -65
  34. data/lib/httpx/connection/http1.rb +54 -18
  35. data/lib/httpx/connection/http2.rb +100 -37
  36. data/lib/httpx/extensions.rb +2 -2
  37. data/lib/httpx/headers.rb +2 -2
  38. data/lib/httpx/io/ssl.rb +11 -3
  39. data/lib/httpx/io/tcp.rb +12 -2
  40. data/lib/httpx/loggable.rb +6 -6
  41. data/lib/httpx/options.rb +43 -28
  42. data/lib/httpx/plugins/authentication.rb +1 -1
  43. data/lib/httpx/plugins/compression.rb +28 -8
  44. data/lib/httpx/plugins/compression/gzip.rb +22 -12
  45. data/lib/httpx/plugins/cookies.rb +12 -8
  46. data/lib/httpx/plugins/digest_authentication.rb +2 -0
  47. data/lib/httpx/plugins/expect.rb +79 -0
  48. data/lib/httpx/plugins/follow_redirects.rb +1 -2
  49. data/lib/httpx/plugins/h2c.rb +0 -1
  50. data/lib/httpx/plugins/proxy.rb +23 -20
  51. data/lib/httpx/plugins/proxy/http.rb +9 -6
  52. data/lib/httpx/plugins/proxy/socks4.rb +1 -1
  53. data/lib/httpx/plugins/proxy/socks5.rb +5 -1
  54. data/lib/httpx/plugins/proxy/ssh.rb +0 -4
  55. data/lib/httpx/plugins/push_promise.rb +2 -2
  56. data/lib/httpx/plugins/retries.rb +32 -29
  57. data/lib/httpx/pool.rb +15 -10
  58. data/lib/httpx/registry.rb +2 -1
  59. data/lib/httpx/request.rb +8 -6
  60. data/lib/httpx/resolver.rb +7 -8
  61. data/lib/httpx/resolver/https.rb +15 -3
  62. data/lib/httpx/resolver/native.rb +22 -32
  63. data/lib/httpx/resolver/options.rb +2 -2
  64. data/lib/httpx/resolver/resolver_mixin.rb +1 -1
  65. data/lib/httpx/response.rb +17 -3
  66. data/lib/httpx/selector.rb +96 -95
  67. data/lib/httpx/session.rb +33 -34
  68. data/lib/httpx/timeout.rb +7 -1
  69. data/lib/httpx/version.rb +1 -1
  70. metadata +77 -20
@@ -0,0 +1,38 @@
1
+ # 0.9.0
2
+
3
+ ## Features
4
+
5
+ ### Multiple requests with specific options
6
+
7
+ You can now pass a third element to the "request element" of an array to `.request`.
8
+
9
+ ```ruby
10
+ requests = [
11
+ [:post, "https://url/post", { form: { foo: "bar" } }],
12
+ [:post, "https://url/post", { form: { foo: "bar2" } }]
13
+ ]
14
+ HTTPX.request(requests)
15
+ # or, if you want to pass options common to all requests
16
+ HTTPX.request(requests, max_concurrent_requests: 1)
17
+ ```
18
+
19
+
20
+ ### HTTPX::Session#build_request
21
+
22
+ `HTTPX::Session::build_request` is now public API from a session. You can now build requests before you send them. These request objects are still considered somewhat "internal", so consider them immutable and **do not rely on its API**. Just pass them forward.
23
+
24
+ Note: this API is only available for instantiated session, so there is no `HTTPX.build_request`.
25
+
26
+
27
+ ```ruby
28
+
29
+ HTTPX.wrap do |http|
30
+ requests = [
31
+ http.build_request(:post, "https://url/post", { form: { foo: "bar" } }),
32
+ http.build_request(:post, "https://url/post", { form: { foo: "bar2" } })
33
+ ]
34
+ http.request(requests)
35
+ # or, if you want to pass options common to all requests
36
+ http.request(requests, max_concurrent_requests: 1)
37
+ end
38
+ ```
@@ -153,7 +153,7 @@ module Faraday
153
153
  proxy_options = { uri: env.request.proxy }
154
154
 
155
155
  session = @session.with(options_from_env(env))
156
- session = session.plugin(:proxy).with_proxy(proxy_options) if env.request.proxy
156
+ session = session.plugin(:proxy).with(proxy: proxy_options) if env.request.proxy
157
157
 
158
158
  responses = session.request(requests)
159
159
  Array(responses).each_with_index do |response, index|
@@ -192,7 +192,7 @@ module Faraday
192
192
  meth, uri, request_options = build_request(env)
193
193
 
194
194
  session = @session.with(options_from_env(env))
195
- session = session.plugin(:proxy).with_proxy(proxy_options) if env.request.proxy
195
+ session = session.plugin(:proxy).with(proxy: proxy_options) if env.request.proxy
196
196
  response = session.__send__(meth, uri, **request_options)
197
197
  response.raise_for_status unless response.is_a?(::HTTPX::Response)
198
198
  save_response(env, response.status, response.body.to_s, response.headers, response.reason) do |response_headers|
@@ -42,7 +42,23 @@ module HTTPX
42
42
 
43
43
  origin = request.origin
44
44
  host = request.uri.host
45
- parse(response.headers["alt-svc"]) do |alt_origin, alt_params|
45
+
46
+ altsvc = response.headers["alt-svc"]
47
+
48
+ # https://tools.ietf.org/html/rfc7838#section-3
49
+ # A field value containing the special value "clear" indicates that the
50
+ # origin requests all alternatives for that origin to be invalidated
51
+ # (including those specified in the same response, in case of an
52
+ # invalid reply containing both "clear" and alternative services).
53
+ if altsvc == "clear"
54
+ @altsvc_mutex.synchronize do
55
+ @altsvcs[origin].clear
56
+ end
57
+
58
+ return
59
+ end
60
+
61
+ parse(altsvc) do |alt_origin, alt_params|
46
62
  alt_origin.host ||= host
47
63
  yield(alt_origin, origin, alt_params)
48
64
  end
@@ -73,7 +89,7 @@ module HTTPX
73
89
  alt_proto, alt_origin = alt_origin.split("=")
74
90
  alt_origin = alt_origin[1..-2] if alt_origin.start_with?("\"") && alt_origin.end_with?("\"")
75
91
  if alt_origin.start_with?(":")
76
- alt_origin = "dummy#{alt_origin}"
92
+ alt_origin = "#{alt_proto}://dummy#{alt_origin}"
77
93
  uri = URI.parse(alt_origin)
78
94
  uri.host = nil
79
95
  uri
@@ -3,25 +3,31 @@
3
3
  module HTTPX
4
4
  module Chainable
5
5
  %i[head get post put delete trace options connect patch].each do |meth|
6
- define_method meth do |*uri, **options|
7
- request(meth, uri, **options)
8
- end
6
+ class_eval(<<-MOD, __FILE__, __LINE__ + 1)
7
+ def #{meth}(*uri, **options)
8
+ request(:#{meth}, uri, **options)
9
+ end
10
+ MOD
9
11
  end
10
12
 
11
- def request(verb, uri, **options)
12
- branch(default_options).request(verb, uri, **options)
13
+ def request(*args, **options)
14
+ branch(default_options).request(*args, **options)
13
15
  end
14
16
 
17
+ # :nocov:
15
18
  def timeout(**args)
16
- branch(default_options.with_timeout(args))
19
+ warn ":#{__method__} is deprecated, use :with_timeout instead"
20
+ branch(default_options.with(timeout: args))
17
21
  end
18
22
 
19
23
  def headers(headers)
20
- branch(default_options.with_headers(headers))
24
+ warn ":#{__method__} is deprecated, use :with_headers instead"
25
+ branch(default_options.with(headers: headers))
21
26
  end
27
+ # :nocov:
22
28
 
23
29
  def accept(type)
24
- headers("accept" => String(type))
30
+ with(headers: { "accept" => String(type) })
25
31
  end
26
32
 
27
33
  def wrap(&blk)
@@ -53,11 +59,23 @@ module HTTPX
53
59
  @options || Options.new
54
60
  end
55
61
 
56
- # :nodoc:
57
62
  def branch(options, &blk)
58
63
  return self.class.new(options, &blk) if is_a?(Session)
59
64
 
60
65
  Session.new(options, &blk)
61
66
  end
67
+
68
+ def method_missing(meth, *args, **options)
69
+ if meth =~ /\Awith_(.+)/
70
+ option = Regexp.last_match(1).to_sym
71
+ with(option => (args.first || options))
72
+ else
73
+ super
74
+ end
75
+ end
76
+
77
+ def respond_to_missing?(meth, *args)
78
+ default_options.respond_to?(meth, *args) || super
79
+ end
62
80
  end
63
81
  end
@@ -46,6 +46,8 @@ module HTTPX
46
46
 
47
47
  attr_reader :origin, :state, :pending, :options
48
48
 
49
+ attr_writer :timers
50
+
49
51
  def initialize(type, uri, options)
50
52
  @type = type
51
53
  @origins = [uri.origin]
@@ -65,6 +67,10 @@ module HTTPX
65
67
  else
66
68
  transition(:idle)
67
69
  end
70
+
71
+ @inflight = 0
72
+ @keep_alive_timeout = options.timeout.keep_alive_timeout
73
+ @keep_alive_timer = nil
68
74
  end
69
75
 
70
76
  # this is a semi-private method, to be used by the resolver
@@ -80,6 +86,8 @@ module HTTPX
80
86
  def match?(uri, options)
81
87
  return false if @state == :closing || @state == :closed
82
88
 
89
+ return false if exhausted?
90
+
83
91
  (
84
92
  (
85
93
  @origins.include?(uri.origin) &&
@@ -95,6 +103,8 @@ module HTTPX
95
103
  def mergeable?(connection)
96
104
  return false if @state == :closing || @state == :closed || !@io
97
105
 
106
+ return false if exhausted?
107
+
98
108
  !(@io.addresses & connection.addresses).empty? && @options == connection.options
99
109
  end
100
110
 
@@ -110,10 +120,13 @@ module HTTPX
110
120
  end
111
121
  end
112
122
 
123
+ def create_idle
124
+ self.class.new(@type, @origin, @options)
125
+ end
126
+
113
127
  def merge(connection)
114
128
  @origins += connection.instance_variable_get(:@origins)
115
- pending = connection.instance_variable_get(:@pending)
116
- pending.each do |req|
129
+ connection.purge_pending do |req|
117
130
  send(req)
118
131
  end
119
132
  end
@@ -130,7 +143,13 @@ module HTTPX
130
143
  end
131
144
 
132
145
  def purge_pending
133
- [*@parser.pending, *@pending].each do |pending|
146
+ pendings = []
147
+ if @parser
148
+ @inflight -= @parser.pending.size
149
+ pendings << @parser.pending
150
+ end
151
+ pendings << @pending
152
+ pendings.each do |pending|
134
153
  pending.reject! do |request|
135
154
  yield request
136
155
  end
@@ -156,22 +175,45 @@ module HTTPX
156
175
  end
157
176
 
158
177
  def interests
159
- return :w if @state == :idle
178
+ # connecting
179
+ if connecting?
180
+ connect
181
+
182
+ return @io.interests if connecting?
183
+ end
184
+
185
+ # if the write buffer is full, we drain it
186
+ return :w if @write_buffer.full?
187
+
188
+ return @parser.interests if @parser
160
189
 
161
- :rw
190
+ nil
162
191
  end
163
192
 
164
193
  def to_io
194
+ @io.to_io
195
+ end
196
+
197
+ def call
165
198
  case @state
166
- when :idle
167
- transition(:open)
199
+ when :closed
200
+ return
201
+ when :closing
202
+ consume
203
+ transition(:closed)
204
+ emit(:close)
205
+ when :open
206
+ consume
168
207
  end
169
- @io.to_io
208
+ nil
170
209
  end
171
210
 
172
211
  def close
173
212
  @parser.close if @parser
174
- transition(:closing)
213
+ return unless @keep_alive_timer
214
+
215
+ @keep_alive_timer.cancel
216
+ remove_instance_variable(:@keep_alive_timer)
175
217
  end
176
218
 
177
219
  def reset
@@ -183,26 +225,25 @@ module HTTPX
183
225
  def send(request)
184
226
  if @parser && !@write_buffer.full?
185
227
  request.headers["alt-used"] = @origin.authority if match_altsvcs?(request.uri)
228
+ if @keep_alive_timer
229
+ # when pushing a request into an existing connection, we have to check whether there
230
+ # is the possibility that the connection might have extended the keep alive timeout.
231
+ # for such cases, we want to ping for availability before deciding to shovel requests.
232
+ if @keep_alive_timer.fires_in.negative?
233
+ @pending << request
234
+ parser.ping
235
+ return
236
+ end
237
+
238
+ @keep_alive_timer.pause
239
+ end
240
+ @inflight += 1
186
241
  parser.send(request)
187
242
  else
188
243
  @pending << request
189
244
  end
190
245
  end
191
246
 
192
- def call
193
- case @state
194
- when :closed
195
- return
196
- when :closing
197
- dwrite
198
- transition(:closed)
199
- emit(:close)
200
- when :open
201
- consume
202
- end
203
- nil
204
- end
205
-
206
247
  def timeout
207
248
  return @timeout if defined?(@timeout)
208
249
 
@@ -213,50 +254,94 @@ module HTTPX
213
254
 
214
255
  private
215
256
 
216
- def consume
217
- catch(:called) do
218
- dread
219
- dwrite
220
- parser.consume
221
- end
257
+ def connect
258
+ transition(:open)
222
259
  end
223
260
 
224
- def dread(wsize = @window_size)
225
- loop do
226
- siz = @io.read(wsize, @read_buffer)
227
- unless siz
228
- ex = EOFError.new("descriptor closed")
229
- ex.set_backtrace(caller)
230
- on_error(ex)
231
- return
232
- end
233
- return if siz.zero?
234
-
235
- log { "READ: #{siz} bytes..." }
236
- parser << @read_buffer.to_s
237
- return if @state == :closing || @state == :closed
238
- end
261
+ def exhausted?
262
+ @parser && parser.exhausted?
239
263
  end
240
264
 
241
- def dwrite
242
- loop do
243
- return if @write_buffer.empty?
244
-
245
- siz = @io.write(@write_buffer)
246
- unless siz
247
- ex = EOFError.new("descriptor closed")
248
- ex.set_backtrace(caller)
249
- on_error(ex)
250
- return
265
+ def consume
266
+ catch(:called) do
267
+ loop do
268
+ parser.consume
269
+
270
+ # we exit if there's no more data to process
271
+ if @pending.size.zero? && @inflight.zero?
272
+ log(level: 3) { "NO MORE REQUESTS..." }
273
+ return
274
+ end
275
+
276
+ @timeout = @current_timeout
277
+
278
+ read_drained = false
279
+ write_drained = nil
280
+
281
+ # dread
282
+ loop do
283
+ siz = @io.read(@window_size, @read_buffer)
284
+ unless siz
285
+ ex = EOFError.new("descriptor closed")
286
+ ex.set_backtrace(caller)
287
+ on_error(ex)
288
+ return
289
+ end
290
+
291
+ if siz.zero?
292
+ read_drained = @read_buffer.empty?
293
+ break
294
+ end
295
+
296
+ log { "READ: #{siz} bytes..." }
297
+
298
+ parser << @read_buffer.to_s
299
+
300
+ break if @state == :closing || @state == :closed
301
+
302
+ # for HTTP/2, we just want to write goaway frame
303
+ end unless @state == :closing
304
+
305
+ # dwrite
306
+ loop do
307
+ if @write_buffer.empty?
308
+ # we only mark as drained on the first loop
309
+ write_drained = write_drained.nil? && @inflight.positive?
310
+ break
311
+ end
312
+
313
+ siz = @io.write(@write_buffer)
314
+ unless siz
315
+ ex = EOFError.new("descriptor closed")
316
+ ex.set_backtrace(caller)
317
+ on_error(ex)
318
+ return
319
+ end
320
+ log { "WRITE: #{siz} bytes..." }
321
+
322
+ if siz.zero?
323
+ write_drained = !@write_buffer.empty?
324
+ break
325
+ end
326
+
327
+ break if @state == :closing || @state == :closed
328
+
329
+ write_drained = false
330
+ end
331
+
332
+ # return if socket is drained
333
+ if read_drained && write_drained
334
+ log(level: 3) { "WAITING FOR EVENTS..." }
335
+ return
336
+ end
251
337
  end
252
- log { "WRITE: #{siz} bytes..." }
253
- return if siz.zero?
254
- return if @state == :closing || @state == :closed
255
338
  end
256
339
  end
257
340
 
258
341
  def send_pending
259
342
  while !@write_buffer.full? && (request = @pending.shift)
343
+ @inflight += 1
344
+ @keep_alive_timer.pause if @keep_alive_timer
260
345
  parser.send(request)
261
346
  end
262
347
  end
@@ -276,26 +361,39 @@ module HTTPX
276
361
  AltSvc.emit(request, response) do |alt_origin, origin, alt_params|
277
362
  emit(:altsvc, alt_origin, origin, alt_params)
278
363
  end
364
+ handle_response
279
365
  request.emit(:response, response)
280
366
  end
281
367
  parser.on(:altsvc) do |alt_origin, origin, alt_params|
282
368
  emit(:altsvc, alt_origin, origin, alt_params)
283
369
  end
284
370
 
371
+ parser.on(:pong, &method(:send_pending))
372
+
285
373
  parser.on(:promise) do |request, stream|
286
374
  request.emit(:promise, parser, stream)
287
375
  end
376
+ parser.on(:exhausted) do
377
+ emit(:exhausted)
378
+ end
288
379
  parser.on(:origin) do |origin|
289
380
  @origins << origin
290
381
  end
291
- parser.on(:close) do
382
+ parser.on(:close) do |force|
292
383
  transition(:closing)
384
+ if force
385
+ transition(:closed)
386
+ emit(:close)
387
+ end
293
388
  end
294
389
  parser.on(:reset) do
295
- transition(:closing)
296
- unless parser.empty?
390
+ if parser.empty?
391
+ reset
392
+ else
393
+ transition(:closing)
297
394
  transition(:closed)
298
395
  emit(:reset)
396
+ @parser.reset if @parser
299
397
  transition(:idle)
300
398
  transition(:open)
301
399
  end
@@ -316,13 +414,20 @@ module HTTPX
316
414
 
317
415
  def transition(nextstate)
318
416
  case nextstate
417
+ when :idle
418
+ @timeout = @current_timeout = @options.timeout.connect_timeout
419
+
319
420
  when :open
320
421
  return if @state == :closed
321
422
 
423
+ total_timeout
424
+
322
425
  @io.connect
323
426
  return unless @io.connected?
324
427
 
325
428
  send_pending
429
+
430
+ @timeout = @current_timeout = @options.timeout.operation_timeout
326
431
  emit(:open)
327
432
  when :closing
328
433
  return unless @state == :open
@@ -330,8 +435,18 @@ module HTTPX
330
435
  return unless @state == :closing
331
436
  return unless @write_buffer.empty?
332
437
 
438
+ if @total_timeout
439
+ @total_timeout.cancel
440
+ remove_instance_variable(:@total_timeout)
441
+ end
442
+
333
443
  @io.close
334
444
  @read_buffer.clear
445
+ if @keep_alive_timer
446
+ @keep_alive_timer.cancel
447
+ remove_instance_variable(:@keep_alive_timer)
448
+ end
449
+
335
450
  remove_instance_variable(:@timeout) if defined?(@timeout)
336
451
  when :already_open
337
452
  nextstate = :open
@@ -353,25 +468,60 @@ module HTTPX
353
468
  emit(:close)
354
469
  end
355
470
 
356
- def on_error(ex)
357
- handle_error(ex)
358
- reset
471
+ def handle_response
472
+ @inflight -= 1
473
+ return unless @inflight.zero?
474
+
475
+ if @keep_alive_timer
476
+ @keep_alive_timer.resume
477
+ @keep_alive_timer.reset
478
+ else
479
+ @keep_alive_timer = @timers.after(@keep_alive_timeout) do
480
+ unless @inflight.zero?
481
+ log { "(#{@origin}): keep alive timeout expired" }
482
+ parser.ping
483
+ end
484
+ end
485
+ end
359
486
  end
360
487
 
361
- def handle_error(error)
488
+ def on_error(error)
362
489
  if error.instance_of?(TimeoutError)
363
490
  if @timeout
364
491
  @timeout -= error.timeout
365
492
  return unless @timeout <= 0
366
493
  end
367
494
 
368
- error = error.to_connection_error if connecting?
495
+ if @total_timeout && @total_timeout.fires_in.negative?
496
+ ex = TotalTimeoutError.new(@total_timeout.interval, "Timed out after #{@total_timeout.interval} seconds")
497
+ ex.set_backtrace(error.backtrace)
498
+ error = ex
499
+ elsif connecting?
500
+ error = error.to_connection_error
501
+ end
369
502
  end
503
+ handle_error(error)
504
+ reset
505
+ end
370
506
 
507
+ def handle_error(error)
371
508
  parser.handle_error(error) if @parser && parser.respond_to?(:handle_error)
372
509
  while (request = @pending.shift)
373
510
  request.emit(:response, ErrorResponse.new(request, error, @options))
374
511
  end
375
512
  end
513
+
514
+ def total_timeout
515
+ total = @options.timeout.total_timeout
516
+
517
+ return unless total
518
+
519
+ @total_timeout ||= @timers.after(total) do
520
+ ex = TotalTimeoutError.new(total, "Timed out after #{total} seconds")
521
+ ex.set_backtrace(caller)
522
+ on_error(ex)
523
+ @parser.close if @parser
524
+ end
525
+ end
376
526
  end
377
527
  end