httpx 0.11.2 → 0.13.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +2 -2
  3. data/doc/release_notes/0_11_1.md +5 -1
  4. data/doc/release_notes/0_11_3.md +5 -0
  5. data/doc/release_notes/0_12_0.md +55 -0
  6. data/doc/release_notes/0_13_0.md +58 -0
  7. data/doc/release_notes/0_13_1.md +5 -0
  8. data/doc/release_notes/0_13_2.md +9 -0
  9. data/lib/httpx.rb +2 -1
  10. data/lib/httpx/adapters/faraday.rb +4 -6
  11. data/lib/httpx/altsvc.rb +1 -0
  12. data/lib/httpx/chainable.rb +2 -2
  13. data/lib/httpx/connection.rb +80 -28
  14. data/lib/httpx/connection/http1.rb +19 -6
  15. data/lib/httpx/connection/http2.rb +32 -25
  16. data/lib/httpx/io.rb +16 -3
  17. data/lib/httpx/io/ssl.rb +35 -24
  18. data/lib/httpx/io/tcp.rb +50 -28
  19. data/lib/httpx/io/tls.rb +218 -0
  20. data/lib/httpx/io/tls/box.rb +365 -0
  21. data/lib/httpx/io/tls/context.rb +199 -0
  22. data/lib/httpx/io/tls/ffi.rb +390 -0
  23. data/lib/httpx/io/udp.rb +31 -7
  24. data/lib/httpx/io/unix.rb +27 -12
  25. data/lib/httpx/options.rb +11 -23
  26. data/lib/httpx/parser/http1.rb +4 -4
  27. data/lib/httpx/plugins/aws_sdk_authentication.rb +81 -0
  28. data/lib/httpx/plugins/aws_sigv4.rb +219 -0
  29. data/lib/httpx/plugins/compression.rb +20 -8
  30. data/lib/httpx/plugins/compression/brotli.rb +8 -6
  31. data/lib/httpx/plugins/compression/deflate.rb +4 -7
  32. data/lib/httpx/plugins/compression/gzip.rb +2 -2
  33. data/lib/httpx/plugins/digest_authentication.rb +1 -1
  34. data/lib/httpx/plugins/follow_redirects.rb +1 -1
  35. data/lib/httpx/plugins/h2c.rb +43 -58
  36. data/lib/httpx/plugins/internal_telemetry.rb +93 -0
  37. data/lib/httpx/plugins/multipart.rb +2 -0
  38. data/lib/httpx/plugins/multipart/encoder.rb +4 -9
  39. data/lib/httpx/plugins/proxy.rb +1 -1
  40. data/lib/httpx/plugins/proxy/http.rb +1 -1
  41. data/lib/httpx/plugins/proxy/socks4.rb +8 -0
  42. data/lib/httpx/plugins/proxy/socks5.rb +8 -0
  43. data/lib/httpx/plugins/push_promise.rb +3 -2
  44. data/lib/httpx/plugins/retries.rb +2 -2
  45. data/lib/httpx/plugins/stream.rb +6 -6
  46. data/lib/httpx/plugins/upgrade.rb +84 -0
  47. data/lib/httpx/plugins/upgrade/h2.rb +54 -0
  48. data/lib/httpx/pool.rb +14 -6
  49. data/lib/httpx/registry.rb +1 -7
  50. data/lib/httpx/request.rb +11 -1
  51. data/lib/httpx/resolver/https.rb +3 -11
  52. data/lib/httpx/resolver/native.rb +7 -3
  53. data/lib/httpx/response.rb +14 -7
  54. data/lib/httpx/selector.rb +5 -0
  55. data/lib/httpx/session.rb +25 -2
  56. data/lib/httpx/transcoder/body.rb +3 -5
  57. data/lib/httpx/version.rb +1 -1
  58. data/sig/chainable.rbs +2 -1
  59. data/sig/connection/http1.rbs +3 -2
  60. data/sig/connection/http2.rbs +5 -3
  61. data/sig/options.rbs +7 -20
  62. data/sig/plugins/aws_sdk_authentication.rbs +17 -0
  63. data/sig/plugins/aws_sigv4.rbs +64 -0
  64. data/sig/plugins/compression.rbs +5 -3
  65. data/sig/plugins/compression/brotli.rbs +1 -1
  66. data/sig/plugins/compression/deflate.rbs +1 -1
  67. data/sig/plugins/compression/gzip.rbs +1 -1
  68. data/sig/plugins/cookies.rbs +0 -1
  69. data/sig/plugins/digest_authentication.rbs +0 -1
  70. data/sig/plugins/expect.rbs +0 -2
  71. data/sig/plugins/follow_redirects.rbs +0 -2
  72. data/sig/plugins/h2c.rbs +5 -10
  73. data/sig/plugins/persistent.rbs +0 -1
  74. data/sig/plugins/proxy.rbs +0 -1
  75. data/sig/plugins/push_promise.rbs +1 -1
  76. data/sig/plugins/retries.rbs +0 -4
  77. data/sig/plugins/upgrade.rbs +23 -0
  78. data/sig/response.rbs +3 -1
  79. metadata +24 -2
@@ -10,7 +10,7 @@ module HTTPX
10
10
  MAX_REQUESTS = 100
11
11
  CRLF = "\r\n"
12
12
 
13
- attr_reader :pending
13
+ attr_reader :pending, :requests
14
14
 
15
15
  def initialize(buffer, options)
16
16
  @options = Options.new(options)
@@ -74,9 +74,10 @@ module HTTPX
74
74
  end
75
75
 
76
76
  def consume
77
- requests_limit = [@max_concurrent_requests, @max_requests, @requests.size].min
77
+ requests_limit = [@max_requests, @requests.size].min
78
+ concurrent_requests_limit = [@max_concurrent_requests, requests_limit].min
78
79
  @requests.each_with_index do |request, idx|
79
- break if idx >= requests_limit
80
+ break if idx >= concurrent_requests_limit
80
81
  next if request.state == :done
81
82
 
82
83
  request.headers["connection"] ||= request.options.persistent || idx < requests_limit - 1 ? "keep-alive" : "close"
@@ -115,7 +116,7 @@ module HTTPX
115
116
  response = @request.response
116
117
  log(level: 2) { "trailer headers received" }
117
118
 
118
- log(color: :yellow) { h.each.map { |f, v| "-> HEADER: #{f}: #{v}" }.join("\n") }
119
+ log(color: :yellow) { h.each.map { |f, v| "-> HEADER: #{f}: #{v.join(", ")}" }.join("\n") }
119
120
  response.merge_headers(h)
120
121
  end
121
122
 
@@ -161,6 +162,16 @@ module HTTPX
161
162
  end
162
163
 
163
164
  def handle_error(ex)
165
+ if (ex.is_a?(EOFError) || ex.is_a?(TimeoutError)) && @request && @request.response &&
166
+ !@request.response.headers.key?("content-length") &&
167
+ !@request.response.headers.key?("transfer-encoding")
168
+ # if the response does not contain a content-length header, the server closing the
169
+ # connnection is the indicator of response consumed.
170
+ # https://greenbytes.de/tech/webdav/rfc2616.html#rfc.section.4.4
171
+ catch(:called) { on_complete }
172
+ return
173
+ end
174
+
164
175
  if @pipelining
165
176
  disable
166
177
  else
@@ -224,6 +235,8 @@ module HTTPX
224
235
 
225
236
  def disable_pipelining
226
237
  return if @requests.empty?
238
+ # do not disable pipelining if already set to 1 request at a time
239
+ return if @max_concurrent_requests == 1
227
240
 
228
241
  @requests.each do |r|
229
242
  r.transition(:idle)
@@ -240,7 +253,7 @@ module HTTPX
240
253
  @pipelining = false
241
254
  end
242
255
 
243
- def set_request_headers(request)
256
+ def set_protocol_headers(request)
244
257
  request.headers["host"] ||= request.authority
245
258
  request.headers["connection"] ||= request.options.persistent ? "keep-alive" : "close"
246
259
  if !request.headers.key?("content-length") &&
@@ -254,7 +267,6 @@ module HTTPX
254
267
  end
255
268
 
256
269
  def handle(request)
257
- set_request_headers(request)
258
270
  catch(:buffer_full) do
259
271
  request.transition(:headers)
260
272
  join_headers(request) if request.state == :headers
@@ -270,6 +282,7 @@ module HTTPX
270
282
  log(color: :yellow) { "<- HEADLINE: #{buffer.chomp.inspect}" }
271
283
  @buffer << buffer
272
284
  buffer.clear
285
+ set_protocol_headers(request)
273
286
  request.headers.each do |field, value|
274
287
  buffer << "#{capitalized(field)}: #{value}" << CRLF
275
288
  log(color: :yellow) { "<- HEADER: #{buffer.chomp}" }
@@ -42,11 +42,15 @@ module HTTPX
42
42
  return @buffer.empty? ? :r : :rw
43
43
  end
44
44
 
45
- return :w unless @pending.empty?
45
+ return :w if !@pending.empty? && can_buffer_more_requests?
46
46
 
47
47
  return :w if @streams.each_key.any? { |r| r.interests == :w }
48
48
 
49
- return :r if @buffer.empty?
49
+ if @buffer.empty?
50
+ return if @streams.empty? && @pings.empty?
51
+
52
+ return :r
53
+ end
50
54
 
51
55
  :rw
52
56
  end
@@ -70,10 +74,14 @@ module HTTPX
70
74
  @connection << data
71
75
  end
72
76
 
77
+ def can_buffer_more_requests?
78
+ @handshake_completed &&
79
+ @streams.size < @max_concurrent_requests &&
80
+ @streams.size < @max_requests
81
+ end
82
+
73
83
  def send(request)
74
- if !@handshake_completed ||
75
- @streams.size >= @max_concurrent_requests ||
76
- @streams.size >= @max_requests
84
+ unless can_buffer_more_requests?
77
85
  @pending << request
78
86
  return
79
87
  end
@@ -126,8 +134,6 @@ module HTTPX
126
134
  request.path
127
135
  end
128
136
 
129
- def set_request_headers(request); end
130
-
131
137
  def handle(request, stream)
132
138
  catch(:buffer_full) do
133
139
  request.transition(:headers)
@@ -172,18 +178,19 @@ module HTTPX
172
178
  stream.on(:data, &method(:on_stream_data).curry(3)[stream, request])
173
179
  end
174
180
 
181
+ def set_protocol_headers(request)
182
+ request.headers[":scheme"] = request.scheme
183
+ request.headers[":method"] = request.verb.to_s.upcase
184
+ request.headers[":path"] = headline_uri(request)
185
+ request.headers[":authority"] = request.authority
186
+ end
187
+
175
188
  def join_headers(stream, request)
176
- set_request_headers(request)
177
- headers = {}
178
- headers[":scheme"] = request.scheme
179
- headers[":method"] = request.verb.to_s.upcase
180
- headers[":path"] = headline_uri(request)
181
- headers[":authority"] = request.authority
182
- headers = headers.merge(request.headers)
189
+ set_protocol_headers(request)
183
190
  log(level: 1, color: :yellow) do
184
- headers.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{v}" }.join("\n")
191
+ request.headers.each.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{v}" }.join("\n")
185
192
  end
186
- stream.headers(headers, end_stream: request.empty?)
193
+ stream.headers(request.headers.each, end_stream: request.empty?)
187
194
  end
188
195
 
189
196
  def join_body(stream, request)
@@ -227,10 +234,15 @@ module HTTPX
227
234
  end
228
235
 
229
236
  def on_stream_close(stream, request, error)
237
+ log(level: 2) { "#{stream.id}: closing stream" }
238
+ @drains.delete(request)
239
+ @streams.delete(request)
240
+
230
241
  if error && error != :no_error
231
242
  ex = Error.new(stream.id, error)
232
243
  ex.set_backtrace(caller)
233
- emit(:error, request, ex)
244
+ response = ErrorResponse.new(request, ex, request.options)
245
+ emit(:response, request, response)
234
246
  else
235
247
  response = request.response
236
248
  if response.status == 421
@@ -241,9 +253,6 @@ module HTTPX
241
253
  emit(:response, request, response)
242
254
  end
243
255
  end
244
- log(level: 2) { "#{stream.id}: closing stream" }
245
-
246
- @streams.delete(request)
247
256
  send(@pending.shift) unless @pending.empty?
248
257
  return unless @streams.empty? && exhausted?
249
258
 
@@ -328,11 +337,9 @@ module HTTPX
328
337
  end
329
338
 
330
339
  def method_missing(meth, *args, &blk)
331
- if @connection.respond_to?(meth)
332
- @connection.__send__(meth, *args, &blk)
333
- else
334
- super
335
- end
340
+ return super unless @connection.respond_to?(meth)
341
+
342
+ @connection.__send__(meth, *args, &blk)
336
343
  end
337
344
  end
338
345
  Connection.register "h2", Connection::HTTP2
data/lib/httpx/io.rb CHANGED
@@ -2,16 +2,29 @@
2
2
 
3
3
  require "socket"
4
4
  require "httpx/io/tcp"
5
- require "httpx/io/ssl"
6
5
  require "httpx/io/unix"
7
6
  require "httpx/io/udp"
8
7
 
9
8
  module HTTPX
10
9
  module IO
11
10
  extend Registry
12
- register "tcp", TCP
13
- register "ssl", SSL
14
11
  register "udp", UDP
15
12
  register "unix", HTTPX::UNIX
13
+ register "tcp", TCP
14
+
15
+ if RUBY_ENGINE == "jruby"
16
+ begin
17
+ require "httpx/io/tls"
18
+ register "ssl", TLS
19
+ rescue LoadError
20
+ # :nocov:
21
+ require "httpx/io/ssl"
22
+ register "ssl", SSL
23
+ # :nocov:
24
+ end
25
+ else
26
+ require "httpx/io/ssl"
27
+ register "ssl", SSL
28
+ end
16
29
  end
17
30
  end
data/lib/httpx/io/ssl.rb CHANGED
@@ -3,6 +3,8 @@
3
3
  require "openssl"
4
4
 
5
5
  module HTTPX
6
+ TLSError = OpenSSL::SSL::SSLError
7
+
6
8
  class SSL < TCP
7
9
  TLS_OPTIONS = if OpenSSL::SSL::SSLContext.instance_methods.include?(:alpn_protocols)
8
10
  { alpn_protocols: %w[h2 http/1.1] }
@@ -11,19 +13,14 @@ module HTTPX
11
13
  end
12
14
 
13
15
  def initialize(_, _, options)
16
+ super
14
17
  @ctx = OpenSSL::SSL::SSLContext.new
15
18
  ctx_options = TLS_OPTIONS.merge(options.ssl)
16
- @tls_hostname = ctx_options.delete(:hostname)
19
+ @sni_hostname = ctx_options.delete(:hostname) || @hostname
17
20
  @ctx.set_params(ctx_options) unless ctx_options.empty?
18
- super
19
- @tls_hostname ||= @hostname
20
21
  @state = :negotiated if @keep_open
21
22
  end
22
23
 
23
- def interests
24
- @interests || super
25
- end
26
-
27
24
  def protocol
28
25
  @io.alpn_protocol || super
29
26
  rescue StandardError
@@ -50,28 +47,30 @@ module HTTPX
50
47
 
51
48
  def connect
52
49
  super
53
- if @keep_open
54
- @state = :negotiated
55
- return
56
- end
57
50
  return if @state == :negotiated ||
58
51
  @state != :connected
59
52
 
60
53
  unless @io.is_a?(OpenSSL::SSL::SSLSocket)
61
54
  @io = OpenSSL::SSL::SSLSocket.new(@io, @ctx)
62
- @io.hostname = @tls_hostname
55
+ @io.hostname = @sni_hostname
63
56
  @io.sync_close = true
64
57
  end
65
- @io.connect_nonblock
66
- @io.post_connection_check(@tls_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
67
- transition(:negotiated)
68
- rescue ::IO::WaitReadable
69
- @interests = :r
70
- rescue ::IO::WaitWritable
71
- @interests = :w
58
+ try_ssl_connect
72
59
  end
73
60
 
74
61
  if RUBY_VERSION < "2.3"
62
+ # :nocov:
63
+ def try_ssl_connect
64
+ @io.connect_nonblock
65
+ @io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
66
+ transition(:negotiated)
67
+ @interests = :w
68
+ rescue ::IO::WaitReadable
69
+ @interests = :r
70
+ rescue ::IO::WaitWritable
71
+ @interests = :w
72
+ end
73
+
75
74
  def read(_, buffer)
76
75
  super
77
76
  rescue ::IO::WaitWritable
@@ -84,7 +83,23 @@ module HTTPX
84
83
  rescue ::IO::WaitReadable
85
84
  0
86
85
  end
86
+ # :nocov:
87
87
  else
88
+ def try_ssl_connect
89
+ case @io.connect_nonblock(exception: false)
90
+ when :wait_readable
91
+ @interests = :r
92
+ return
93
+ when :wait_writable
94
+ @interests = :w
95
+ return
96
+ end
97
+ @io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
98
+ transition(:negotiated)
99
+ @interests = :w
100
+ end
101
+
102
+ # :nocov:
88
103
  if OpenSSL::VERSION < "2.0.6"
89
104
  def read(size, buffer)
90
105
  @io.read_nonblock(size, buffer)
@@ -97,11 +112,7 @@ module HTTPX
97
112
  nil
98
113
  end
99
114
  end
100
- end
101
-
102
- def inspect
103
- id = @io.closed? ? "closed" : @io.to_io.fileno
104
- "#<SSL(fd: #{id}): #{@ip}:#{@port} state: #{@state}>"
115
+ # :nocov:
105
116
  end
106
117
 
107
118
  private
data/lib/httpx/io/tcp.rb CHANGED
@@ -7,17 +7,19 @@ module HTTPX
7
7
  class TCP
8
8
  include Loggable
9
9
 
10
- attr_reader :ip, :port, :addresses, :state
10
+ using URIExtensions
11
+
12
+ attr_reader :ip, :port, :addresses, :state, :interests
11
13
 
12
14
  alias_method :host, :ip
13
15
 
14
16
  def initialize(origin, addresses, options)
15
17
  @state = :idle
16
18
  @hostname = origin.host
17
- @addresses = addresses
18
19
  @options = Options.new(options)
19
20
  @fallback_protocol = @options.fallback_protocol
20
21
  @port = origin.port
22
+ @interests = :w
21
23
  if @options.io
22
24
  @io = case @options.io
23
25
  when Hash
@@ -25,24 +27,20 @@ module HTTPX
25
27
  else
26
28
  @options.io
27
29
  end
30
+ raise Error, "Given IO objects do not match the request authority" unless @io
31
+
28
32
  _, _, _, @ip = @io.addr
29
33
  @addresses ||= [@ip]
30
34
  @ip_index = @addresses.size - 1
31
- unless @io.nil?
32
- @keep_open = true
33
- @state = :connected
34
- end
35
+ @keep_open = true
36
+ @state = :connected
35
37
  else
36
- @ip_index = @addresses.size - 1
37
- @ip = @addresses[@ip_index]
38
+ @addresses = addresses.map { |addr| addr.is_a?(IPAddr) ? addr : IPAddr.new(addr) }
38
39
  end
40
+ @ip_index = @addresses.size - 1
39
41
  @io ||= build_socket
40
42
  end
41
43
 
42
- def interests
43
- :w
44
- end
45
-
46
44
  def to_io
47
45
  @io.to_io
48
46
  end
@@ -54,32 +52,40 @@ module HTTPX
54
52
  def connect
55
53
  return unless closed?
56
54
 
57
- begin
58
- if @io.closed?
59
- transition(:idle)
60
- @io = build_socket
61
- end
62
- @io.connect_nonblock(Socket.sockaddr_in(@port, @ip.to_s))
63
- rescue Errno::EISCONN
55
+ if @io.closed?
56
+ transition(:idle)
57
+ @io = build_socket
64
58
  end
65
- transition(:connected)
59
+ try_connect
66
60
  rescue Errno::EHOSTUNREACH => e
67
61
  raise e if @ip_index <= 0
68
62
 
69
63
  @ip_index -= 1
70
64
  retry
71
65
  rescue Errno::ETIMEDOUT => e
72
- raise ConnectTimeoutError, e.message if @ip_index <= 0
66
+ raise ConnectTimeoutError.new(@options.timeout.connect_timeout, e.message) if @ip_index <= 0
73
67
 
74
68
  @ip_index -= 1
75
69
  retry
76
- rescue Errno::EINPROGRESS,
77
- Errno::EALREADY,
78
- ::IO::WaitReadable
79
70
  end
80
71
 
81
72
  if RUBY_VERSION < "2.3"
82
73
  # :nocov:
74
+ def try_connect
75
+ @io.connect_nonblock(Socket.sockaddr_in(@port, @ip.to_s))
76
+ rescue ::IO::WaitWritable, Errno::EALREADY
77
+ @interests = :w
78
+ rescue ::IO::WaitReadable
79
+ @interests = :r
80
+ rescue Errno::EISCONN
81
+ transition(:connected)
82
+ @interests = :w
83
+ else
84
+ transition(:connected)
85
+ @interests = :w
86
+ end
87
+ private :try_connect
88
+
83
89
  def read(size, buffer)
84
90
  @io.read_nonblock(size, buffer)
85
91
  log { "READ: #{buffer.bytesize} bytes..." }
@@ -103,6 +109,22 @@ module HTTPX
103
109
  end
104
110
  # :nocov:
105
111
  else
112
+ def try_connect
113
+ case @io.connect_nonblock(Socket.sockaddr_in(@port, @ip.to_s), exception: false)
114
+ when :wait_readable
115
+ @interests = :r
116
+ return
117
+ when :wait_writable
118
+ @interests = :w
119
+ return
120
+ end
121
+ transition(:connected)
122
+ @interests = :w
123
+ rescue Errno::EALREADY
124
+ @interests = :w
125
+ end
126
+ private :try_connect
127
+
106
128
  def read(size, buffer)
107
129
  ret = @io.read_nonblock(size, buffer, exception: false)
108
130
  if ret == :wait_readable
@@ -147,14 +169,14 @@ module HTTPX
147
169
 
148
170
  # :nocov:
149
171
  def inspect
150
- id = @io.closed? ? "closed" : @io.fileno
151
- "#<TCP(fd: #{id}): #{@ip}:#{@port} (state: #{@state})>"
172
+ "#<#{self.class}: #{@ip}:#{@port} (state: #{@state})>"
152
173
  end
153
174
  # :nocov:
154
175
 
155
176
  private
156
177
 
157
178
  def build_socket
179
+ @ip = @addresses[@ip_index]
158
180
  Socket.new(@ip.family, :STREAM, 0)
159
181
  end
160
182
 
@@ -177,9 +199,9 @@ module HTTPX
177
199
  def log_transition_state(nextstate)
178
200
  case nextstate
179
201
  when :connected
180
- "Connected to #{@hostname} (#{@ip}) port #{@port} (##{@io.fileno})"
202
+ "Connected to #{host} (##{@io.fileno})"
181
203
  else
182
- "#{@ip}:#{@port} #{@state} -> #{nextstate}"
204
+ "#{host} #{@state} -> #{nextstate}"
183
205
  end
184
206
  end
185
207
  end