httpx 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/httpx/altsvc.rb +18 -2
- data/lib/httpx/chainable.rb +5 -3
- data/lib/httpx/connection.rb +168 -66
- data/lib/httpx/connection/http1.rb +28 -10
- data/lib/httpx/connection/http2.rb +55 -32
- data/lib/httpx/extensions.rb +1 -1
- data/lib/httpx/io/ssl.rb +11 -3
- data/lib/httpx/io/tcp.rb +12 -2
- data/lib/httpx/loggable.rb +6 -6
- data/lib/httpx/options.rb +17 -14
- data/lib/httpx/plugins/compression.rb +5 -1
- data/lib/httpx/plugins/compression/gzip.rb +22 -12
- data/lib/httpx/plugins/digest_authentication.rb +2 -0
- data/lib/httpx/plugins/proxy.rb +16 -12
- data/lib/httpx/plugins/proxy/http.rb +7 -2
- data/lib/httpx/plugins/proxy/socks4.rb +1 -1
- data/lib/httpx/plugins/proxy/socks5.rb +5 -1
- data/lib/httpx/plugins/push_promise.rb +2 -2
- data/lib/httpx/plugins/retries.rb +11 -5
- data/lib/httpx/pool.rb +7 -12
- data/lib/httpx/registry.rb +2 -1
- data/lib/httpx/request.rb +7 -0
- data/lib/httpx/resolver/https.rb +15 -3
- data/lib/httpx/resolver/native.rb +22 -32
- data/lib/httpx/resolver/options.rb +2 -2
- data/lib/httpx/resolver/resolver_mixin.rb +1 -1
- data/lib/httpx/response.rb +15 -1
- data/lib/httpx/selector.rb +96 -95
- data/lib/httpx/session.rb +1 -1
- data/lib/httpx/timeout.rb +7 -1
- data/lib/httpx/version.rb +1 -1
- metadata +16 -16
@@ -30,8 +30,30 @@ module HTTPX
|
|
30
30
|
init_connection
|
31
31
|
end
|
32
32
|
|
33
|
+
def interests
|
34
|
+
# waiting for WINDOW_UPDATE frames
|
35
|
+
return :r if @buffer.full?
|
36
|
+
|
37
|
+
return :w if @connection.state == :closed
|
38
|
+
|
39
|
+
return :r unless (@connection.state == :connected && @handshake_completed)
|
40
|
+
|
41
|
+
return :w unless @pending.empty?
|
42
|
+
|
43
|
+
return :w if @streams.each_key.any? { |r| r.interests == :w }
|
44
|
+
|
45
|
+
return :r if @buffer.empty?
|
46
|
+
|
47
|
+
:rw
|
48
|
+
end
|
49
|
+
|
50
|
+
def reset
|
51
|
+
init_connection
|
52
|
+
end
|
53
|
+
|
33
54
|
def close
|
34
55
|
@connection.goaway unless @connection.state == :closed
|
56
|
+
emit(:close)
|
35
57
|
end
|
36
58
|
|
37
59
|
def empty?
|
@@ -39,6 +61,8 @@ module HTTPX
|
|
39
61
|
end
|
40
62
|
|
41
63
|
def exhausted?
|
64
|
+
return false if @max_requests.zero? && @connection.active_stream_count.zero?
|
65
|
+
|
42
66
|
@connection.active_stream_count >= @max_requests
|
43
67
|
end
|
44
68
|
|
@@ -68,6 +92,8 @@ module HTTPX
|
|
68
92
|
|
69
93
|
def consume
|
70
94
|
@streams.each do |request, stream|
|
95
|
+
next if request.state == :done
|
96
|
+
|
71
97
|
handle(request, stream)
|
72
98
|
end
|
73
99
|
end
|
@@ -128,7 +154,7 @@ module HTTPX
|
|
128
154
|
def handle_stream(stream, request)
|
129
155
|
stream.on(:close, &method(:on_stream_close).curry[stream, request])
|
130
156
|
stream.on(:half_close) do
|
131
|
-
log(level: 2
|
157
|
+
log(level: 2) { "#{stream.id}: waiting for response..." }
|
132
158
|
end
|
133
159
|
stream.on(:altsvc, &method(:on_altsvc).curry[request.origin])
|
134
160
|
stream.on(:headers, &method(:on_stream_headers).curry[stream, request])
|
@@ -143,8 +169,8 @@ module HTTPX
|
|
143
169
|
headers[":path"] = headline_uri(request)
|
144
170
|
headers[":authority"] = request.authority
|
145
171
|
headers = headers.merge(request.headers)
|
146
|
-
log(level: 1,
|
147
|
-
headers.map { |k, v| "-> HEADER: #{k}: #{v}" }.join("\n")
|
172
|
+
log(level: 1, color: :yellow) do
|
173
|
+
headers.map { |k, v| "#{stream.id}: -> HEADER: #{k}: #{v}" }.join("\n")
|
148
174
|
end
|
149
175
|
stream.headers(headers, end_stream: request.empty?)
|
150
176
|
end
|
@@ -155,8 +181,8 @@ module HTTPX
|
|
155
181
|
chunk = @drains.delete(request) || request.drain_body
|
156
182
|
while chunk
|
157
183
|
next_chunk = request.drain_body
|
158
|
-
log(level: 1,
|
159
|
-
log(level: 2,
|
184
|
+
log(level: 1, color: :green) { "#{stream.id}: -> DATA: #{chunk.bytesize} bytes..." }
|
185
|
+
log(level: 2, color: :green) { "#{stream.id}: -> #{chunk.inspect}" }
|
160
186
|
stream.data(chunk, end_stream: !next_chunk)
|
161
187
|
if next_chunk && @buffer.full?
|
162
188
|
@drains[request] = next_chunk
|
@@ -171,25 +197,25 @@ module HTTPX
|
|
171
197
|
######
|
172
198
|
|
173
199
|
def on_stream_headers(stream, request, h)
|
174
|
-
log(
|
175
|
-
h.map { |k, v| "<- HEADER: #{k}: #{v}" }.join("\n")
|
200
|
+
log(color: :yellow) do
|
201
|
+
h.map { |k, v| "#{stream.id}: <- HEADER: #{k}: #{v}" }.join("\n")
|
176
202
|
end
|
177
203
|
_, status = h.shift
|
178
204
|
headers = request.options.headers_class.new(h)
|
179
205
|
response = request.options.response_class.new(request, status, "2.0", headers)
|
180
206
|
request.response = response
|
181
207
|
@streams[request] = stream
|
208
|
+
|
209
|
+
handle(request, stream) if request.expects?
|
182
210
|
end
|
183
211
|
|
184
212
|
def on_stream_data(stream, request, data)
|
185
|
-
log(level: 1,
|
186
|
-
log(level: 2,
|
213
|
+
log(level: 1, color: :green) { "#{stream.id}: <- DATA: #{data.bytesize} bytes..." }
|
214
|
+
log(level: 2, color: :green) { "#{stream.id}: <- #{data.inspect}" }
|
187
215
|
request.response << data
|
188
216
|
end
|
189
217
|
|
190
218
|
def on_stream_close(stream, request, error)
|
191
|
-
return handle(request, stream) if request.expects?
|
192
|
-
|
193
219
|
if error && error != :no_error
|
194
220
|
ex = Error.new(stream.id, error)
|
195
221
|
ex.set_backtrace(caller)
|
@@ -204,14 +230,13 @@ module HTTPX
|
|
204
230
|
emit(:response, request, response)
|
205
231
|
end
|
206
232
|
end
|
207
|
-
log(level: 2
|
233
|
+
log(level: 2) { "#{stream.id}: closing stream" }
|
208
234
|
|
209
235
|
@streams.delete(request)
|
210
236
|
send(@pending.shift) unless @pending.empty?
|
211
237
|
return unless @streams.empty? && exhausted?
|
212
238
|
|
213
239
|
close
|
214
|
-
emit(:close)
|
215
240
|
emit(:exhausted) unless @pending.empty?
|
216
241
|
end
|
217
242
|
|
@@ -222,7 +247,11 @@ module HTTPX
|
|
222
247
|
def on_settings(*)
|
223
248
|
@handshake_completed = true
|
224
249
|
|
225
|
-
|
250
|
+
if @max_requests.zero?
|
251
|
+
@max_requests = @connection.remote_settings[:settings_max_concurrent_streams]
|
252
|
+
|
253
|
+
@connection.max_streams = @max_requests if @connection.respond_to?(:max_streams=) && @max_requests.positive?
|
254
|
+
end
|
226
255
|
|
227
256
|
@max_concurrent_requests = [@max_concurrent_requests, @max_requests].min
|
228
257
|
send_pending
|
@@ -242,32 +271,26 @@ module HTTPX
|
|
242
271
|
end
|
243
272
|
|
244
273
|
def on_frame_sent(frame)
|
245
|
-
log(level: 2
|
246
|
-
log(level: 2,
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
else
|
251
|
-
frame.inspect
|
252
|
-
end
|
274
|
+
log(level: 2) { "#{frame[:stream]}: frame was sent!" }
|
275
|
+
log(level: 2, color: :blue) do
|
276
|
+
payload = frame
|
277
|
+
payload = payload.merge(payload: frame[:payload].bytesize) if frame[:type] == :data
|
278
|
+
"#{frame[:stream]}: #{payload}"
|
253
279
|
end
|
254
280
|
end
|
255
281
|
|
256
282
|
def on_frame_received(frame)
|
257
|
-
log(level: 2
|
258
|
-
log(level: 2,
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
else
|
263
|
-
frame.inspect
|
264
|
-
end
|
283
|
+
log(level: 2) { "#{frame[:stream]}: frame was received!" }
|
284
|
+
log(level: 2, color: :magenta) do
|
285
|
+
payload = frame
|
286
|
+
payload = payload.merge(payload: frame[:payload].bytesize) if frame[:type] == :data
|
287
|
+
"#{frame[:stream]}: #{payload}"
|
265
288
|
end
|
266
289
|
end
|
267
290
|
|
268
291
|
def on_altsvc(origin, frame)
|
269
|
-
log(level: 2
|
270
|
-
log(level: 2
|
292
|
+
log(level: 2) { "#{frame[:stream]}: altsvc frame was received" }
|
293
|
+
log(level: 2) { "#{frame[:stream]}: #{frame.inspect}" }
|
271
294
|
alt_origin = URI.parse("#{frame[:proto]}://#{frame[:host]}:#{frame[:port]}")
|
272
295
|
params = { "ma" => frame[:max_age] }
|
273
296
|
emit(:altsvc, origin, alt_origin, origin, params)
|
data/lib/httpx/extensions.rb
CHANGED
data/lib/httpx/io/ssl.rb
CHANGED
@@ -20,6 +20,10 @@ module HTTPX
|
|
20
20
|
@state = :negotiated if @keep_open
|
21
21
|
end
|
22
22
|
|
23
|
+
def interests
|
24
|
+
@interests || super
|
25
|
+
end
|
26
|
+
|
23
27
|
def protocol
|
24
28
|
@io.alpn_protocol || super
|
25
29
|
rescue StandardError
|
@@ -62,15 +66,18 @@ module HTTPX
|
|
62
66
|
@io.connect_nonblock
|
63
67
|
@io.post_connection_check(@hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
|
64
68
|
transition(:negotiated)
|
65
|
-
rescue ::IO::WaitReadable
|
66
|
-
|
69
|
+
rescue ::IO::WaitReadable
|
70
|
+
@interests = :r
|
71
|
+
rescue ::IO::WaitWritable
|
72
|
+
@interests = :w
|
67
73
|
end
|
68
74
|
|
69
75
|
# :nocov:
|
70
76
|
if RUBY_VERSION < "2.3"
|
71
|
-
def read(
|
77
|
+
def read(_, buffer)
|
72
78
|
super
|
73
79
|
rescue ::IO::WaitWritable
|
80
|
+
buffer.clear
|
74
81
|
0
|
75
82
|
end
|
76
83
|
|
@@ -86,6 +93,7 @@ module HTTPX
|
|
86
93
|
buffer.bytesize
|
87
94
|
rescue ::IO::WaitReadable,
|
88
95
|
::IO::WaitWritable
|
96
|
+
buffer.clear
|
89
97
|
0
|
90
98
|
rescue EOFError
|
91
99
|
nil
|
data/lib/httpx/io/tcp.rb
CHANGED
@@ -11,6 +11,8 @@ module HTTPX
|
|
11
11
|
|
12
12
|
attr_reader :addresses
|
13
13
|
|
14
|
+
attr_reader :state
|
15
|
+
|
14
16
|
alias_method :host, :ip
|
15
17
|
|
16
18
|
def initialize(origin, addresses, options)
|
@@ -41,6 +43,10 @@ module HTTPX
|
|
41
43
|
@io ||= build_socket
|
42
44
|
end
|
43
45
|
|
46
|
+
def interests
|
47
|
+
:w
|
48
|
+
end
|
49
|
+
|
44
50
|
def to_io
|
45
51
|
@io.to_io
|
46
52
|
end
|
@@ -67,7 +73,7 @@ module HTTPX
|
|
67
73
|
@ip_index -= 1
|
68
74
|
retry
|
69
75
|
rescue Errno::ETIMEDOUT => e
|
70
|
-
raise
|
76
|
+
raise ConnectTimeoutError, e.message if @ip_index <= 0
|
71
77
|
|
72
78
|
@ip_index -= 1
|
73
79
|
retry
|
@@ -82,6 +88,7 @@ module HTTPX
|
|
82
88
|
@io.read_nonblock(size, buffer)
|
83
89
|
buffer.bytesize
|
84
90
|
rescue ::IO::WaitReadable
|
91
|
+
buffer.clear
|
85
92
|
0
|
86
93
|
rescue EOFError
|
87
94
|
nil
|
@@ -100,7 +107,10 @@ module HTTPX
|
|
100
107
|
else
|
101
108
|
def read(size, buffer)
|
102
109
|
ret = @io.read_nonblock(size, buffer, exception: false)
|
103
|
-
|
110
|
+
if ret == :wait_readable
|
111
|
+
buffer.clear
|
112
|
+
return 0
|
113
|
+
end
|
104
114
|
return if ret.nil?
|
105
115
|
|
106
116
|
buffer.bytesize
|
data/lib/httpx/loggable.rb
CHANGED
@@ -13,35 +13,35 @@ module HTTPX
|
|
13
13
|
white: 37,
|
14
14
|
}.freeze
|
15
15
|
|
16
|
-
def log(level: @options.debug_level,
|
16
|
+
def log(level: @options.debug_level, color: nil, &msg)
|
17
17
|
return unless @options.debug
|
18
18
|
return unless @options.debug_level >= level
|
19
19
|
|
20
20
|
debug_stream = @options.debug
|
21
21
|
|
22
|
-
message = (+
|
22
|
+
message = (+"" << msg.call << "\n")
|
23
23
|
message = "\e[#{COLORS[color]}m#{message}\e[0m" if debug_stream.respond_to?(:isatty) && debug_stream.isatty
|
24
24
|
debug_stream << message
|
25
25
|
end
|
26
26
|
|
27
27
|
if !Exception.instance_methods.include?(:full_message)
|
28
28
|
|
29
|
-
def log_exception(ex, level: @options.debug_level,
|
29
|
+
def log_exception(ex, level: @options.debug_level, color: nil)
|
30
30
|
return unless @options.debug
|
31
31
|
return unless @options.debug_level >= level
|
32
32
|
|
33
33
|
message = +"#{ex.message} (#{ex.class})"
|
34
34
|
message << "\n" << ex.backtrace.join("\n") unless ex.backtrace.nil?
|
35
|
-
log(level: level,
|
35
|
+
log(level: level, color: color) { message }
|
36
36
|
end
|
37
37
|
|
38
38
|
else
|
39
39
|
|
40
|
-
def log_exception(ex, level: @options.debug_level,
|
40
|
+
def log_exception(ex, level: @options.debug_level, color: nil)
|
41
41
|
return unless @options.debug
|
42
42
|
return unless @options.debug_level >= level
|
43
43
|
|
44
|
-
log(level: level,
|
44
|
+
log(level: level, color: color) { ex.full_message }
|
45
45
|
end
|
46
46
|
|
47
47
|
end
|
data/lib/httpx/options.rb
CHANGED
@@ -25,23 +25,28 @@ module HTTPX
|
|
25
25
|
|
26
26
|
def def_option(name, &interpreter)
|
27
27
|
defined_options << name.to_sym
|
28
|
-
interpreter ||= ->(v) { v }
|
29
28
|
|
30
29
|
attr_reader name
|
31
30
|
|
32
|
-
|
33
|
-
|
31
|
+
if interpreter
|
32
|
+
define_method(:"#{name}=") do |value|
|
33
|
+
return if value.nil?
|
34
34
|
|
35
|
-
|
36
|
-
|
35
|
+
instance_variable_set(:"@#{name}", instance_exec(value, &interpreter))
|
36
|
+
end
|
37
37
|
|
38
|
-
|
38
|
+
define_method(:"with_#{name}") do |value|
|
39
|
+
merge(name => instance_exec(value, &interpreter))
|
40
|
+
end
|
41
|
+
else
|
42
|
+
attr_writer name
|
39
43
|
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
other
|
44
|
+
define_method(:"with_#{name}") do |value|
|
45
|
+
merge(name => value)
|
46
|
+
end
|
44
47
|
end
|
48
|
+
|
49
|
+
protected :"#{name}="
|
45
50
|
end
|
46
51
|
end
|
47
52
|
|
@@ -70,7 +75,6 @@ module HTTPX
|
|
70
75
|
}
|
71
76
|
|
72
77
|
defaults.merge!(options)
|
73
|
-
defaults[:headers] = Headers.new(defaults[:headers])
|
74
78
|
defaults.each do |(k, v)|
|
75
79
|
__send__(:"#{k}=", v)
|
76
80
|
end
|
@@ -80,7 +84,7 @@ module HTTPX
|
|
80
84
|
if self.headers
|
81
85
|
self.headers.merge(headers)
|
82
86
|
else
|
83
|
-
headers
|
87
|
+
Headers.new(headers)
|
84
88
|
end
|
85
89
|
end
|
86
90
|
|
@@ -116,8 +120,7 @@ module HTTPX
|
|
116
120
|
end
|
117
121
|
|
118
122
|
%w[
|
119
|
-
params form json body
|
120
|
-
follow ssl http2_settings
|
123
|
+
params form json body ssl http2_settings
|
121
124
|
request_class response_class headers_class request_body_class response_body_class connection_class
|
122
125
|
io fallback_protocol debug debug_level transport_options resolver_class resolver_options
|
123
126
|
persistent
|
@@ -37,6 +37,8 @@ module HTTPX
|
|
37
37
|
return if @body.nil?
|
38
38
|
|
39
39
|
@headers.get("content-encoding").each do |encoding|
|
40
|
+
next if encoding == "identity"
|
41
|
+
|
40
42
|
@body = Encoder.new(@body, Compression.registry(encoding).encoder)
|
41
43
|
end
|
42
44
|
@headers["content-length"] = @body.bytesize unless chunked?
|
@@ -54,6 +56,8 @@ module HTTPX
|
|
54
56
|
return unless @headers.key?("content-encoding")
|
55
57
|
|
56
58
|
@_decoders = @headers.get("content-encoding").map do |encoding|
|
59
|
+
next if encoding == "identity"
|
60
|
+
|
57
61
|
decoder = Compression.registry(encoding).decoder
|
58
62
|
# do not uncompress if there is no decoder available. In fact, we can't reliably
|
59
63
|
# continue decompressing beyond that, so ignore.
|
@@ -61,7 +65,7 @@ module HTTPX
|
|
61
65
|
|
62
66
|
@encodings << encoding
|
63
67
|
decoder
|
64
|
-
end
|
68
|
+
end.compact
|
65
69
|
|
66
70
|
# remove encodings that we are able to decode
|
67
71
|
@headers["content-encoding"] = @headers.get("content-encoding") - @encodings
|
@@ -15,31 +15,41 @@ module HTTPX
|
|
15
15
|
end
|
16
16
|
|
17
17
|
class Encoder
|
18
|
+
def initialize
|
19
|
+
@compressed_chunk = "".b
|
20
|
+
end
|
21
|
+
|
18
22
|
def deflate(raw, buffer, chunk_size:)
|
19
23
|
gzip = Zlib::GzipWriter.new(self)
|
20
24
|
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
25
|
+
begin
|
26
|
+
while (chunk = raw.read(chunk_size))
|
27
|
+
gzip.write(chunk)
|
28
|
+
gzip.flush
|
29
|
+
compressed = compressed_chunk
|
30
|
+
buffer << compressed
|
31
|
+
yield compressed if block_given?
|
32
|
+
end
|
33
|
+
ensure
|
34
|
+
gzip.close
|
27
35
|
end
|
28
|
-
|
29
|
-
|
36
|
+
|
37
|
+
return unless (compressed = compressed_chunk)
|
38
|
+
|
39
|
+
buffer << compressed
|
40
|
+
yield compressed if block_given?
|
30
41
|
end
|
31
42
|
|
32
43
|
private
|
33
44
|
|
34
45
|
def write(chunk)
|
35
|
-
@compressed_chunk
|
46
|
+
@compressed_chunk << chunk
|
36
47
|
end
|
37
48
|
|
38
49
|
def compressed_chunk
|
39
|
-
|
40
|
-
compressed
|
50
|
+
@compressed_chunk.dup
|
41
51
|
ensure
|
42
|
-
@compressed_chunk
|
52
|
+
@compressed_chunk.clear
|
43
53
|
end
|
44
54
|
end
|
45
55
|
|