httpx 0.12.0 → 0.14.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. checksums.yaml +4 -4
  2. data/doc/release_notes/0_10_1.md +1 -1
  3. data/doc/release_notes/0_13_0.md +58 -0
  4. data/doc/release_notes/0_13_1.md +5 -0
  5. data/doc/release_notes/0_13_2.md +9 -0
  6. data/doc/release_notes/0_14_0.md +79 -0
  7. data/doc/release_notes/0_14_1.md +7 -0
  8. data/lib/httpx.rb +1 -2
  9. data/lib/httpx/callbacks.rb +12 -3
  10. data/lib/httpx/chainable.rb +2 -2
  11. data/lib/httpx/connection.rb +29 -22
  12. data/lib/httpx/connection/http1.rb +35 -15
  13. data/lib/httpx/connection/http2.rb +61 -15
  14. data/lib/httpx/headers.rb +7 -3
  15. data/lib/httpx/io/ssl.rb +30 -17
  16. data/lib/httpx/io/tcp.rb +48 -27
  17. data/lib/httpx/io/udp.rb +31 -7
  18. data/lib/httpx/io/unix.rb +27 -12
  19. data/lib/httpx/options.rb +97 -74
  20. data/lib/httpx/plugins/aws_sdk_authentication.rb +5 -2
  21. data/lib/httpx/plugins/aws_sigv4.rb +5 -4
  22. data/lib/httpx/plugins/basic_authentication.rb +8 -3
  23. data/lib/httpx/plugins/compression.rb +24 -12
  24. data/lib/httpx/plugins/compression/brotli.rb +10 -7
  25. data/lib/httpx/plugins/compression/deflate.rb +6 -5
  26. data/lib/httpx/plugins/compression/gzip.rb +4 -3
  27. data/lib/httpx/plugins/cookies.rb +3 -7
  28. data/lib/httpx/plugins/digest_authentication.rb +5 -5
  29. data/lib/httpx/plugins/expect.rb +6 -6
  30. data/lib/httpx/plugins/follow_redirects.rb +4 -4
  31. data/lib/httpx/plugins/grpc.rb +247 -0
  32. data/lib/httpx/plugins/grpc/call.rb +62 -0
  33. data/lib/httpx/plugins/grpc/message.rb +85 -0
  34. data/lib/httpx/plugins/h2c.rb +43 -58
  35. data/lib/httpx/plugins/internal_telemetry.rb +1 -1
  36. data/lib/httpx/plugins/multipart/part.rb +2 -2
  37. data/lib/httpx/plugins/proxy.rb +3 -7
  38. data/lib/httpx/plugins/proxy/http.rb +5 -4
  39. data/lib/httpx/plugins/proxy/ssh.rb +3 -3
  40. data/lib/httpx/plugins/rate_limiter.rb +1 -1
  41. data/lib/httpx/plugins/retries.rb +14 -15
  42. data/lib/httpx/plugins/stream.rb +99 -75
  43. data/lib/httpx/plugins/upgrade.rb +84 -0
  44. data/lib/httpx/plugins/upgrade/h2.rb +54 -0
  45. data/lib/httpx/pool.rb +14 -5
  46. data/lib/httpx/request.rb +25 -2
  47. data/lib/httpx/resolver/native.rb +7 -3
  48. data/lib/httpx/response.rb +9 -5
  49. data/lib/httpx/session.rb +17 -7
  50. data/lib/httpx/transcoder/chunker.rb +1 -1
  51. data/lib/httpx/version.rb +1 -1
  52. data/sig/callbacks.rbs +2 -0
  53. data/sig/chainable.rbs +2 -1
  54. data/sig/connection/http1.rbs +6 -1
  55. data/sig/connection/http2.rbs +6 -2
  56. data/sig/headers.rbs +2 -2
  57. data/sig/options.rbs +16 -22
  58. data/sig/plugins/aws_sdk_authentication.rbs +2 -0
  59. data/sig/plugins/aws_sigv4.rbs +0 -1
  60. data/sig/plugins/basic_authentication.rbs +2 -0
  61. data/sig/plugins/compression.rbs +7 -5
  62. data/sig/plugins/compression/brotli.rbs +1 -1
  63. data/sig/plugins/compression/deflate.rbs +1 -1
  64. data/sig/plugins/compression/gzip.rbs +1 -1
  65. data/sig/plugins/cookies.rbs +0 -1
  66. data/sig/plugins/digest_authentication.rbs +0 -1
  67. data/sig/plugins/expect.rbs +0 -2
  68. data/sig/plugins/follow_redirects.rbs +0 -2
  69. data/sig/plugins/h2c.rbs +5 -10
  70. data/sig/plugins/persistent.rbs +0 -1
  71. data/sig/plugins/proxy.rbs +0 -1
  72. data/sig/plugins/retries.rbs +0 -4
  73. data/sig/plugins/stream.rbs +17 -16
  74. data/sig/plugins/upgrade.rbs +23 -0
  75. data/sig/request.rbs +7 -2
  76. data/sig/response.rbs +4 -1
  77. data/sig/session.rbs +4 -0
  78. metadata +21 -7
  79. data/lib/httpx/timeout.rb +0 -67
  80. data/sig/timeout.rbs +0 -29
@@ -21,14 +21,16 @@ module HTTPX
21
21
 
22
22
  def initialize(buffer, options)
23
23
  @options = Options.new(options)
24
- @max_concurrent_requests = @options.max_concurrent_requests || MAX_CONCURRENT_REQUESTS
25
- @max_requests = @options.max_requests || 0
24
+ @settings = @options.http2_settings
26
25
  @pending = []
27
26
  @streams = {}
28
27
  @drains = {}
29
28
  @pings = []
30
29
  @buffer = buffer
31
30
  @handshake_completed = false
31
+ @wait_for_handshake = @settings.key?(:wait_for_handshake) ? @settings.delete(:wait_for_handshake) : true
32
+ @max_concurrent_requests = @options.max_concurrent_requests || MAX_CONCURRENT_REQUESTS
33
+ @max_requests = @options.max_requests || 0
32
34
  init_connection
33
35
  end
34
36
 
@@ -36,7 +38,11 @@ module HTTPX
36
38
  # waiting for WINDOW_UPDATE frames
37
39
  return :r if @buffer.full?
38
40
 
39
- return :w if @connection.state == :closed
41
+ if @connection.state == :closed
42
+ return unless @handshake_completed
43
+
44
+ return :w
45
+ end
40
46
 
41
47
  unless (@connection.state == :connected && @handshake_completed)
42
48
  return @buffer.empty? ? :r : :rw
@@ -75,9 +81,13 @@ module HTTPX
75
81
  end
76
82
 
77
83
  def can_buffer_more_requests?
78
- @handshake_completed &&
84
+ if @handshake_completed
79
85
  @streams.size < @max_concurrent_requests &&
80
- @streams.size < @max_requests
86
+ @streams.size < @max_requests
87
+ else
88
+ !@wait_for_handshake &&
89
+ @streams.size < @max_concurrent_requests
90
+ end
81
91
  end
82
92
 
83
93
  def send(request)
@@ -91,7 +101,6 @@ module HTTPX
91
101
  @streams[request] = stream
92
102
  @max_requests -= 1
93
103
  end
94
- request.once(:headers, &method(:set_protocol_headers))
95
104
  handle(request, stream)
96
105
  true
97
106
  rescue HTTP2Next::Error::StreamLimitExceeded
@@ -141,12 +150,14 @@ module HTTPX
141
150
  join_headers(stream, request) if request.state == :headers
142
151
  request.transition(:body)
143
152
  join_body(stream, request) if request.state == :body
153
+ request.transition(:trailers)
154
+ join_trailers(stream, request) if request.state == :trailers && !request.body.empty?
144
155
  request.transition(:done)
145
156
  end
146
157
  end
147
158
 
148
159
  def init_connection
149
- @connection = HTTP2Next::Client.new(@options.http2_settings)
160
+ @connection = HTTP2Next::Client.new(@settings)
150
161
  @connection.max_streams = @max_requests if @connection.respond_to?(:max_streams=) && @max_requests.positive?
151
162
  @connection.on(:frame, &method(:on_frame))
152
163
  @connection.on(:frame_sent, &method(:on_frame_sent))
@@ -170,6 +181,7 @@ module HTTPX
170
181
  public :reset
171
182
 
172
183
  def handle_stream(stream, request)
184
+ request.on(:refuse, &method(:on_stream_refuse).curry(3)[stream, request])
173
185
  stream.on(:close, &method(:on_stream_close).curry(3)[stream, request])
174
186
  stream.on(:half_close) do
175
187
  log(level: 2) { "#{stream.id}: waiting for response..." }
@@ -180,17 +192,32 @@ module HTTPX
180
192
  end
181
193
 
182
194
  def set_protocol_headers(request)
183
- request.headers[":scheme"] = request.scheme
184
- request.headers[":method"] = request.verb.to_s.upcase
185
- request.headers[":path"] = headline_uri(request)
186
- request.headers[":authority"] = request.authority
195
+ {
196
+ ":scheme" => request.scheme,
197
+ ":method" => request.verb.to_s.upcase,
198
+ ":path" => headline_uri(request),
199
+ ":authority" => request.authority,
200
+ }
187
201
  end
188
202
 
189
203
  def join_headers(stream, request)
204
+ extra_headers = set_protocol_headers(request)
205
+ log(level: 1, color: :yellow) do
206
+ request.headers.merge(extra_headers).each.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{v}" }.join("\n")
207
+ end
208
+ stream.headers(request.headers.each(extra_headers), end_stream: request.empty?)
209
+ end
210
+
211
+ def join_trailers(stream, request)
212
+ unless request.trailers?
213
+ stream.data("", end_stream: true) if request.callbacks_for?(:trailers)
214
+ return
215
+ end
216
+
190
217
  log(level: 1, color: :yellow) do
191
- request.headers.each.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{v}" }.join("\n")
218
+ request.trailers.each.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{v}" }.join("\n")
192
219
  end
193
- stream.headers(request.headers.each, end_stream: request.empty?)
220
+ stream.headers(request.trailers.each, end_stream: true)
194
221
  end
195
222
 
196
223
  def join_body(stream, request)
@@ -201,13 +228,15 @@ module HTTPX
201
228
  next_chunk = request.drain_body
202
229
  log(level: 1, color: :green) { "#{stream.id}: -> DATA: #{chunk.bytesize} bytes..." }
203
230
  log(level: 2, color: :green) { "#{stream.id}: -> #{chunk.inspect}" }
204
- stream.data(chunk, end_stream: !next_chunk)
205
- if next_chunk && @buffer.full?
231
+ stream.data(chunk, end_stream: !(next_chunk || request.trailers? || request.callbacks_for?(:trailers)))
232
+ if next_chunk && (@buffer.full? || request.body.unbounded_body?)
206
233
  @drains[request] = next_chunk
207
234
  throw(:buffer_full)
208
235
  end
209
236
  chunk = next_chunk
210
237
  end
238
+
239
+ on_stream_refuse(stream, request, request.drain_error) if request.drain_error
211
240
  end
212
241
 
213
242
  ######
@@ -215,6 +244,11 @@ module HTTPX
215
244
  ######
216
245
 
217
246
  def on_stream_headers(stream, request, h)
247
+ if request.response && request.response.version == "2.0"
248
+ on_stream_trailers(stream, request, h)
249
+ return
250
+ end
251
+
218
252
  log(color: :yellow) do
219
253
  h.map { |k, v| "#{stream.id}: <- HEADER: #{k}: #{v}" }.join("\n")
220
254
  end
@@ -227,12 +261,24 @@ module HTTPX
227
261
  handle(request, stream) if request.expects?
228
262
  end
229
263
 
264
+ def on_stream_trailers(stream, request, h)
265
+ log(color: :yellow) do
266
+ h.map { |k, v| "#{stream.id}: <- HEADER: #{k}: #{v}" }.join("\n")
267
+ end
268
+ request.response.merge_headers(h)
269
+ end
270
+
230
271
  def on_stream_data(stream, request, data)
231
272
  log(level: 1, color: :green) { "#{stream.id}: <- DATA: #{data.bytesize} bytes..." }
232
273
  log(level: 2, color: :green) { "#{stream.id}: <- #{data.inspect}" }
233
274
  request.response << data
234
275
  end
235
276
 
277
+ def on_stream_refuse(stream, request, error)
278
+ stream.close
279
+ on_stream_close(stream, request, error)
280
+ end
281
+
236
282
  def on_stream_close(stream, request, error)
237
283
  log(level: 2) { "#{stream.id}: closing stream" }
238
284
  @drains.delete(request)
data/lib/httpx/headers.rb CHANGED
@@ -103,16 +103,20 @@ module HTTPX
103
103
  # returns the enumerable headers store in pairs of header field + the values in
104
104
  # the comma-separated string format
105
105
  #
106
- def each
107
- return enum_for(__method__) { @headers.size } unless block_given?
106
+ def each(extra_headers = nil)
107
+ return enum_for(__method__, extra_headers) { @headers.size } unless block_given?
108
108
 
109
109
  @headers.each do |field, value|
110
110
  yield(field, value.join(", ")) unless value.empty?
111
111
  end
112
+
113
+ extra_headers.each do |field, value|
114
+ yield(field, value) unless value.empty?
115
+ end if extra_headers
112
116
  end
113
117
 
114
118
  def ==(other)
115
- to_hash == Headers.new(other).to_hash
119
+ other == to_hash
116
120
  end
117
121
 
118
122
  # the headers store in Hash format
data/lib/httpx/io/ssl.rb CHANGED
@@ -47,10 +47,6 @@ module HTTPX
47
47
 
48
48
  def connect
49
49
  super
50
- if @keep_open
51
- @state = :negotiated
52
- return
53
- end
54
50
  return if @state == :negotiated ||
55
51
  @state != :connected
56
52
 
@@ -59,17 +55,22 @@ module HTTPX
59
55
  @io.hostname = @sni_hostname
60
56
  @io.sync_close = true
61
57
  end
62
- @io.connect_nonblock
63
- @io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
64
- transition(:negotiated)
65
- @interests = :w
66
- rescue ::IO::WaitReadable
67
- @interests = :r
68
- rescue ::IO::WaitWritable
69
- @interests = :w
58
+ try_ssl_connect
70
59
  end
71
60
 
72
61
  if RUBY_VERSION < "2.3"
62
+ # :nocov:
63
+ def try_ssl_connect
64
+ @io.connect_nonblock
65
+ @io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
66
+ transition(:negotiated)
67
+ @interests = :w
68
+ rescue ::IO::WaitReadable
69
+ @interests = :r
70
+ rescue ::IO::WaitWritable
71
+ @interests = :w
72
+ end
73
+
73
74
  def read(_, buffer)
74
75
  super
75
76
  rescue ::IO::WaitWritable
@@ -82,7 +83,23 @@ module HTTPX
82
83
  rescue ::IO::WaitReadable
83
84
  0
84
85
  end
86
+ # :nocov:
85
87
  else
88
+ def try_ssl_connect
89
+ case @io.connect_nonblock(exception: false)
90
+ when :wait_readable
91
+ @interests = :r
92
+ return
93
+ when :wait_writable
94
+ @interests = :w
95
+ return
96
+ end
97
+ @io.post_connection_check(@sni_hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
98
+ transition(:negotiated)
99
+ @interests = :w
100
+ end
101
+
102
+ # :nocov:
86
103
  if OpenSSL::VERSION < "2.0.6"
87
104
  def read(size, buffer)
88
105
  @io.read_nonblock(size, buffer)
@@ -95,11 +112,7 @@ module HTTPX
95
112
  nil
96
113
  end
97
114
  end
98
- end
99
-
100
- def inspect
101
- id = @io.closed? ? "closed" : @io.to_io.fileno
102
- "#<SSL(fd: #{id}): #{@ip}:#{@port} state: #{@state}>"
115
+ # :nocov:
103
116
  end
104
117
 
105
118
  private
data/lib/httpx/io/tcp.rb CHANGED
@@ -7,6 +7,8 @@ module HTTPX
7
7
  class TCP
8
8
  include Loggable
9
9
 
10
+ using URIExtensions
11
+
10
12
  attr_reader :ip, :port, :addresses, :state, :interests
11
13
 
12
14
  alias_method :host, :ip
@@ -14,7 +16,6 @@ module HTTPX
14
16
  def initialize(origin, addresses, options)
15
17
  @state = :idle
16
18
  @hostname = origin.host
17
- @addresses = addresses
18
19
  @options = Options.new(options)
19
20
  @fallback_protocol = @options.fallback_protocol
20
21
  @port = origin.port
@@ -26,17 +27,17 @@ module HTTPX
26
27
  else
27
28
  @options.io
28
29
  end
30
+ raise Error, "Given IO objects do not match the request authority" unless @io
31
+
29
32
  _, _, _, @ip = @io.addr
30
33
  @addresses ||= [@ip]
31
34
  @ip_index = @addresses.size - 1
32
- unless @io.nil?
33
- @keep_open = true
34
- @state = :connected
35
- end
35
+ @keep_open = true
36
+ @state = :connected
36
37
  else
37
- @ip_index = @addresses.size - 1
38
- @ip = @addresses[@ip_index]
38
+ @addresses = addresses.map { |addr| addr.is_a?(IPAddr) ? addr : IPAddr.new(addr) }
39
39
  end
40
+ @ip_index = @addresses.size - 1
40
41
  @io ||= build_socket
41
42
  end
42
43
 
@@ -51,36 +52,40 @@ module HTTPX
51
52
  def connect
52
53
  return unless closed?
53
54
 
54
- begin
55
- if @io.closed?
56
- transition(:idle)
57
- @io = build_socket
58
- end
59
- @io.connect_nonblock(Socket.sockaddr_in(@port, @ip.to_s))
60
- rescue Errno::EISCONN
55
+ if @io.closed?
56
+ transition(:idle)
57
+ @io = build_socket
61
58
  end
62
- @interests = :w
63
-
64
- transition(:connected)
59
+ try_connect
65
60
  rescue Errno::EHOSTUNREACH => e
66
61
  raise e if @ip_index <= 0
67
62
 
68
63
  @ip_index -= 1
69
64
  retry
70
65
  rescue Errno::ETIMEDOUT => e
71
- raise ConnectTimeoutError.new(@options.timeout.connect_timeout, e.message) if @ip_index <= 0
66
+ raise ConnectTimeoutError.new(@options.timeout[:connect_timeout], e.message) if @ip_index <= 0
72
67
 
73
68
  @ip_index -= 1
74
69
  retry
75
- rescue Errno::EINPROGRESS,
76
- Errno::EALREADY
77
- @interests = :w
78
- rescue ::IO::WaitReadable
79
- @interests = :r
80
70
  end
81
71
 
82
72
  if RUBY_VERSION < "2.3"
83
73
  # :nocov:
74
+ def try_connect
75
+ @io.connect_nonblock(Socket.sockaddr_in(@port, @ip.to_s))
76
+ rescue ::IO::WaitWritable, Errno::EALREADY
77
+ @interests = :w
78
+ rescue ::IO::WaitReadable
79
+ @interests = :r
80
+ rescue Errno::EISCONN
81
+ transition(:connected)
82
+ @interests = :w
83
+ else
84
+ transition(:connected)
85
+ @interests = :w
86
+ end
87
+ private :try_connect
88
+
84
89
  def read(size, buffer)
85
90
  @io.read_nonblock(size, buffer)
86
91
  log { "READ: #{buffer.bytesize} bytes..." }
@@ -104,6 +109,22 @@ module HTTPX
104
109
  end
105
110
  # :nocov:
106
111
  else
112
+ def try_connect
113
+ case @io.connect_nonblock(Socket.sockaddr_in(@port, @ip.to_s), exception: false)
114
+ when :wait_readable
115
+ @interests = :r
116
+ return
117
+ when :wait_writable
118
+ @interests = :w
119
+ return
120
+ end
121
+ transition(:connected)
122
+ @interests = :w
123
+ rescue Errno::EALREADY
124
+ @interests = :w
125
+ end
126
+ private :try_connect
127
+
107
128
  def read(size, buffer)
108
129
  ret = @io.read_nonblock(size, buffer, exception: false)
109
130
  if ret == :wait_readable
@@ -148,14 +169,14 @@ module HTTPX
148
169
 
149
170
  # :nocov:
150
171
  def inspect
151
- id = @io.closed? ? "closed" : @io.fileno
152
- "#<TCP(fd: #{id}): #{@ip}:#{@port} (state: #{@state})>"
172
+ "#<#{self.class}: #{@ip}:#{@port} (state: #{@state})>"
153
173
  end
154
174
  # :nocov:
155
175
 
156
176
  private
157
177
 
158
178
  def build_socket
179
+ @ip = @addresses[@ip_index]
159
180
  Socket.new(@ip.family, :STREAM, 0)
160
181
  end
161
182
 
@@ -178,9 +199,9 @@ module HTTPX
178
199
  def log_transition_state(nextstate)
179
200
  case nextstate
180
201
  when :connected
181
- "Connected to #{@hostname} (#{@ip}) port #{@port} (##{@io.fileno})"
202
+ "Connected to #{host} (##{@io.fileno})"
182
203
  else
183
- "#{@ip}:#{@port} #{@state} -> #{nextstate}"
204
+ "#{host} #{@state} -> #{nextstate}"
184
205
  end
185
206
  end
186
207
  end
data/lib/httpx/io/udp.rb CHANGED
@@ -39,16 +39,20 @@ module HTTPX
39
39
  end
40
40
  end
41
41
 
42
- def write(buffer)
43
- siz = @io.send(buffer.to_s, 0, @host, @port)
44
- log { "WRITE: #{siz} bytes..." }
45
- buffer.shift!(siz)
46
- siz
47
- end
48
-
49
42
  # :nocov:
50
43
  if (RUBY_ENGINE == "truffleruby" && RUBY_ENGINE_VERSION < "21.1.0") ||
51
44
  RUBY_VERSION < "2.3"
45
+ def write(buffer)
46
+ siz = @io.sendmsg_nonblock(buffer.to_s, 0, Socket.sockaddr_in(@port, @host.to_s))
47
+ log { "WRITE: #{siz} bytes..." }
48
+ buffer.shift!(siz)
49
+ siz
50
+ rescue ::IO::WaitWritable
51
+ 0
52
+ rescue EOFError
53
+ nil
54
+ end
55
+
52
56
  def read(size, buffer)
53
57
  data, _ = @io.recvfrom_nonblock(size)
54
58
  buffer.replace(data)
@@ -59,6 +63,18 @@ module HTTPX
59
63
  rescue IOError
60
64
  end
61
65
  else
66
+
67
+ def write(buffer)
68
+ siz = @io.sendmsg_nonblock(buffer.to_s, 0, Socket.sockaddr_in(@port, @host.to_s), exception: false)
69
+ return 0 if siz == :wait_writable
70
+ return if siz.nil?
71
+
72
+ log { "WRITE: #{siz} bytes..." }
73
+
74
+ buffer.shift!(siz)
75
+ siz
76
+ end
77
+
62
78
  def read(size, buffer)
63
79
  ret = @io.recvfrom_nonblock(size, 0, buffer, exception: false)
64
80
  return 0 if ret == :wait_readable
@@ -68,6 +84,14 @@ module HTTPX
68
84
  rescue IOError
69
85
  end
70
86
  end
87
+
88
+ # In JRuby, sendmsg_nonblock is not implemented
89
+ def write(buffer)
90
+ siz = @io.send(buffer.to_s, 0, @host, @port)
91
+ log { "WRITE: #{siz} bytes..." }
92
+ buffer.shift!(siz)
93
+ siz
94
+ end if RUBY_ENGINE == "jruby"
71
95
  # :nocov:
72
96
  end
73
97
  end