httpx 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/httpx/altsvc.rb +18 -2
- data/lib/httpx/chainable.rb +5 -3
- data/lib/httpx/connection.rb +168 -66
- data/lib/httpx/connection/http1.rb +28 -10
- data/lib/httpx/connection/http2.rb +55 -32
- data/lib/httpx/extensions.rb +1 -1
- data/lib/httpx/io/ssl.rb +11 -3
- data/lib/httpx/io/tcp.rb +12 -2
- data/lib/httpx/loggable.rb +6 -6
- data/lib/httpx/options.rb +17 -14
- data/lib/httpx/plugins/compression.rb +5 -1
- data/lib/httpx/plugins/compression/gzip.rb +22 -12
- data/lib/httpx/plugins/digest_authentication.rb +2 -0
- data/lib/httpx/plugins/proxy.rb +16 -12
- data/lib/httpx/plugins/proxy/http.rb +7 -2
- data/lib/httpx/plugins/proxy/socks4.rb +1 -1
- data/lib/httpx/plugins/proxy/socks5.rb +5 -1
- data/lib/httpx/plugins/push_promise.rb +2 -2
- data/lib/httpx/plugins/retries.rb +11 -5
- data/lib/httpx/pool.rb +7 -12
- data/lib/httpx/registry.rb +2 -1
- data/lib/httpx/request.rb +7 -0
- data/lib/httpx/resolver/https.rb +15 -3
- data/lib/httpx/resolver/native.rb +22 -32
- data/lib/httpx/resolver/options.rb +2 -2
- data/lib/httpx/resolver/resolver_mixin.rb +1 -1
- data/lib/httpx/response.rb +15 -1
- data/lib/httpx/selector.rb +96 -95
- data/lib/httpx/session.rb +1 -1
- data/lib/httpx/timeout.rb +7 -1
- data/lib/httpx/version.rb +1 -1
- metadata +16 -16
data/lib/httpx/plugins/proxy.rb
CHANGED
@@ -179,20 +179,9 @@ module HTTPX
|
|
179
179
|
super || @state == :connecting || @state == :connected
|
180
180
|
end
|
181
181
|
|
182
|
-
def to_io
|
183
|
-
return super unless @options.proxy
|
184
|
-
|
185
|
-
case @state
|
186
|
-
when :idle
|
187
|
-
transition(:connecting)
|
188
|
-
when :connected
|
189
|
-
transition(:open)
|
190
|
-
end
|
191
|
-
@io.to_io
|
192
|
-
end
|
193
|
-
|
194
182
|
def call
|
195
183
|
super
|
184
|
+
|
196
185
|
return unless @options.proxy
|
197
186
|
|
198
187
|
case @state
|
@@ -210,11 +199,26 @@ module HTTPX
|
|
210
199
|
emit(:close)
|
211
200
|
end
|
212
201
|
|
202
|
+
private
|
203
|
+
|
204
|
+
def connect
|
205
|
+
return super unless @options.proxy
|
206
|
+
|
207
|
+
case @state
|
208
|
+
when :idle
|
209
|
+
transition(:connecting)
|
210
|
+
when :connected
|
211
|
+
transition(:open)
|
212
|
+
end
|
213
|
+
end
|
214
|
+
|
213
215
|
def transition(nextstate)
|
214
216
|
return super unless @options.proxy
|
215
217
|
|
216
218
|
case nextstate
|
217
219
|
when :closing
|
220
|
+
# this is a hack so that we can use the super method
|
221
|
+
# and it'll thing that the current state is open
|
218
222
|
@state = :open if @state == :connecting
|
219
223
|
end
|
220
224
|
super
|
@@ -7,6 +7,10 @@ module HTTPX
|
|
7
7
|
module Proxy
|
8
8
|
module HTTP
|
9
9
|
module ConnectionMethods
|
10
|
+
def connecting?
|
11
|
+
super || @state == :connecting || @state == :connected
|
12
|
+
end
|
13
|
+
|
10
14
|
private
|
11
15
|
|
12
16
|
def transition(nextstate)
|
@@ -34,7 +38,6 @@ module HTTPX
|
|
34
38
|
when :idle
|
35
39
|
@parser = ProxyParser.new(@write_buffer, @options)
|
36
40
|
set_parser_callbacks(@parser)
|
37
|
-
@parser.on(:close) { transition(:closing) }
|
38
41
|
end
|
39
42
|
end
|
40
43
|
super
|
@@ -48,7 +51,7 @@ module HTTPX
|
|
48
51
|
#
|
49
52
|
if req.uri.scheme == "https"
|
50
53
|
connect_request = ConnectRequest.new(req.uri, @options)
|
51
|
-
|
54
|
+
@inflight += 1
|
52
55
|
parser.send(connect_request)
|
53
56
|
else
|
54
57
|
transition(:connected)
|
@@ -56,6 +59,7 @@ module HTTPX
|
|
56
59
|
end
|
57
60
|
|
58
61
|
def __http_on_connect(_, response)
|
62
|
+
@inflight -= 1
|
59
63
|
if response.status == 200
|
60
64
|
req = @pending.first
|
61
65
|
request_uri = req.uri
|
@@ -67,6 +71,7 @@ module HTTPX
|
|
67
71
|
while (req = pending.shift)
|
68
72
|
req.emit(:response, response)
|
69
73
|
end
|
74
|
+
reset
|
70
75
|
end
|
71
76
|
end
|
72
77
|
end
|
@@ -39,7 +39,7 @@ module HTTPX
|
|
39
39
|
|
40
40
|
@parser = nil
|
41
41
|
end
|
42
|
-
log(level: 1
|
42
|
+
log(level: 1) { "SOCKS4: #{nextstate}: #{@write_buffer.to_s.inspect}" } unless nextstate == :open
|
43
43
|
super
|
44
44
|
end
|
45
45
|
|
@@ -31,6 +31,10 @@ module HTTPX
|
|
31
31
|
end
|
32
32
|
end
|
33
33
|
|
34
|
+
def connecting?
|
35
|
+
super || @state == :authenticating || @state == :negotiating
|
36
|
+
end
|
37
|
+
|
34
38
|
private
|
35
39
|
|
36
40
|
def transition(nextstate)
|
@@ -60,7 +64,7 @@ module HTTPX
|
|
60
64
|
|
61
65
|
@parser = nil
|
62
66
|
end
|
63
|
-
log(level: 1
|
67
|
+
log(level: 1) { "SOCKS5: #{nextstate}: #{@write_buffer.to_s.inspect}" } unless nextstate == :open
|
64
68
|
super
|
65
69
|
end
|
66
70
|
|
@@ -43,9 +43,9 @@ module HTTPX
|
|
43
43
|
end
|
44
44
|
|
45
45
|
def __on_promise_request(parser, stream, h)
|
46
|
-
log(level: 1
|
46
|
+
log(level: 1) do
|
47
47
|
# :nocov:
|
48
|
-
h.map { |k, v| "-> PROMISE HEADER: #{k}: #{v}" }.join("\n")
|
48
|
+
h.map { |k, v| "#{stream.id}: -> PROMISE HEADER: #{k}: #{v}" }.join("\n")
|
49
49
|
# :nocov:
|
50
50
|
end
|
51
51
|
headers = @options.headers_class.new(h)
|
@@ -63,13 +63,19 @@ module HTTPX
|
|
63
63
|
def fetch_response(request, connections, options)
|
64
64
|
response = super
|
65
65
|
|
66
|
-
|
67
|
-
|
68
|
-
if response.is_a?(ErrorResponse) &&
|
66
|
+
if response &&
|
69
67
|
request.retries.positive? &&
|
70
68
|
__repeatable_request?(request, options) &&
|
71
|
-
|
72
|
-
|
69
|
+
(
|
70
|
+
# rubocop:disable Style/MultilineTernaryOperator
|
71
|
+
options.retry_on ?
|
72
|
+
options.retry_on.call(response) :
|
73
|
+
(
|
74
|
+
response.is_a?(ErrorResponse) && __retryable_error?(response.error)
|
75
|
+
)
|
76
|
+
# rubocop:enable Style/MultilineTernaryOperator
|
77
|
+
)
|
78
|
+
|
73
79
|
request.retries -= 1
|
74
80
|
log { "failed to get response, #{request.retries} tries to go..." }
|
75
81
|
request.transition(:idle)
|
data/lib/httpx/pool.rb
CHANGED
@@ -14,7 +14,7 @@ module HTTPX
|
|
14
14
|
|
15
15
|
def initialize
|
16
16
|
@resolvers = {}
|
17
|
-
@
|
17
|
+
@_resolver_ios = {}
|
18
18
|
@timers = Timers::Group.new
|
19
19
|
@selector = Selector.new
|
20
20
|
@connections = []
|
@@ -28,15 +28,13 @@ module HTTPX
|
|
28
28
|
def next_tick
|
29
29
|
catch(:jump_tick) do
|
30
30
|
timeout = [next_timeout, @timers.wait_interval].compact.min
|
31
|
-
if timeout.negative?
|
31
|
+
if timeout && timeout.negative?
|
32
32
|
@timers.fire
|
33
33
|
throw(:jump_tick)
|
34
34
|
end
|
35
35
|
|
36
|
-
@selector.select(timeout)
|
37
|
-
|
38
|
-
monitor.interests = monitor.io.interests
|
39
|
-
end
|
36
|
+
@selector.select(timeout, &:call)
|
37
|
+
|
40
38
|
@timers.fire
|
41
39
|
end
|
42
40
|
rescue StandardError => e
|
@@ -86,7 +84,7 @@ module HTTPX
|
|
86
84
|
resolver << connection
|
87
85
|
return if resolver.empty?
|
88
86
|
|
89
|
-
@
|
87
|
+
@_resolver_ios[resolver] ||= @selector.register(resolver)
|
90
88
|
end
|
91
89
|
|
92
90
|
def on_resolver_connection(connection)
|
@@ -118,8 +116,7 @@ module HTTPX
|
|
118
116
|
@resolvers.delete(resolver_type)
|
119
117
|
|
120
118
|
@selector.deregister(resolver)
|
121
|
-
|
122
|
-
monitor.close if monitor
|
119
|
+
@_resolver_ios.delete(resolver)
|
123
120
|
resolver.close unless resolver.closed?
|
124
121
|
end
|
125
122
|
|
@@ -128,10 +125,8 @@ module HTTPX
|
|
128
125
|
# if open, an IO was passed upstream, therefore
|
129
126
|
# consider it connected already.
|
130
127
|
@connected_connections += 1
|
131
|
-
@selector.register(connection, :rw)
|
132
|
-
else
|
133
|
-
@selector.register(connection, :w)
|
134
128
|
end
|
129
|
+
@selector.register(connection)
|
135
130
|
connection.on(:close) do
|
136
131
|
unregister_connection(connection)
|
137
132
|
end
|
data/lib/httpx/registry.rb
CHANGED
data/lib/httpx/request.rb
CHANGED
@@ -58,6 +58,12 @@ module HTTPX
|
|
58
58
|
@state = :idle
|
59
59
|
end
|
60
60
|
|
61
|
+
def interests
|
62
|
+
return :r if @state == :done || @state == :expect
|
63
|
+
|
64
|
+
:w
|
65
|
+
end
|
66
|
+
|
61
67
|
# :nocov:
|
62
68
|
if RUBY_VERSION < "2.2"
|
63
69
|
# rubocop: disable Lint/UriEscapeUnescape:
|
@@ -219,6 +225,7 @@ module HTTPX
|
|
219
225
|
case nextstate
|
220
226
|
when :idle
|
221
227
|
@response = nil
|
228
|
+
@drainer = nil
|
222
229
|
when :headers
|
223
230
|
return unless @state == :idle
|
224
231
|
when :body
|
data/lib/httpx/resolver/https.rb
CHANGED
@@ -25,7 +25,7 @@ module HTTPX
|
|
25
25
|
|
26
26
|
def_delegator :@connections, :empty?
|
27
27
|
|
28
|
-
def_delegators :@resolver_connection, :to_io, :call, :
|
28
|
+
def_delegators :@resolver_connection, :connecting?, :to_io, :call, :close
|
29
29
|
|
30
30
|
def initialize(options)
|
31
31
|
@options = Options.new(options)
|
@@ -62,8 +62,20 @@ module HTTPX
|
|
62
62
|
resolver_connection.closed?
|
63
63
|
end
|
64
64
|
|
65
|
+
def interests
|
66
|
+
return if @queries.empty?
|
67
|
+
|
68
|
+
resolver_connection.__send__(__method__)
|
69
|
+
end
|
70
|
+
|
65
71
|
private
|
66
72
|
|
73
|
+
def connect
|
74
|
+
return if @queries.empty?
|
75
|
+
|
76
|
+
resolver_connection.__send__(__method__)
|
77
|
+
end
|
78
|
+
|
67
79
|
def pool
|
68
80
|
Thread.current[:httpx_connection_pool] ||= Pool.new
|
69
81
|
end
|
@@ -84,7 +96,7 @@ module HTTPX
|
|
84
96
|
|
85
97
|
hostname = hostname || @queries.key(connection) || connection.origin.host
|
86
98
|
type = @_record_types[hostname].first
|
87
|
-
log
|
99
|
+
log { "resolver: query #{type} for #{hostname}" }
|
88
100
|
begin
|
89
101
|
request = build_request(hostname, type)
|
90
102
|
@requests[request] = connection
|
@@ -111,7 +123,7 @@ module HTTPX
|
|
111
123
|
end
|
112
124
|
|
113
125
|
def on_promise(_, stream)
|
114
|
-
log(level: 2
|
126
|
+
log(level: 2) { "#{stream.id}: refusing stream!" }
|
115
127
|
stream.refuse
|
116
128
|
end
|
117
129
|
|
@@ -73,14 +73,6 @@ module HTTPX
|
|
73
73
|
end
|
74
74
|
|
75
75
|
def to_io
|
76
|
-
case @state
|
77
|
-
when :idle
|
78
|
-
transition(:open)
|
79
|
-
when :closed
|
80
|
-
transition(:idle)
|
81
|
-
transition(:open)
|
82
|
-
end
|
83
|
-
resolve if @queries.empty?
|
84
76
|
@io.to_io
|
85
77
|
end
|
86
78
|
|
@@ -93,11 +85,7 @@ module HTTPX
|
|
93
85
|
rescue Errno::EHOSTUNREACH => e
|
94
86
|
@ns_index += 1
|
95
87
|
if @ns_index < @nameserver.size
|
96
|
-
log
|
97
|
-
# :nocov:
|
98
|
-
"failed resolving on nameserver #{@nameserver[@ns_index - 1]} (#{e.message})"
|
99
|
-
# :nocov:
|
100
|
-
end
|
88
|
+
log { "resolver: failed resolving on nameserver #{@nameserver[@ns_index - 1]} (#{e.message})" }
|
101
89
|
transition(:idle)
|
102
90
|
else
|
103
91
|
handle_error(e)
|
@@ -107,6 +95,14 @@ module HTTPX
|
|
107
95
|
end
|
108
96
|
|
109
97
|
def interests
|
98
|
+
case @state
|
99
|
+
when :idle
|
100
|
+
transition(:open)
|
101
|
+
when :closed
|
102
|
+
transition(:idle)
|
103
|
+
transition(:open)
|
104
|
+
end
|
105
|
+
|
110
106
|
!@write_buffer.empty? || @queries.empty? ? :w : :r
|
111
107
|
end
|
112
108
|
|
@@ -160,11 +156,7 @@ module HTTPX
|
|
160
156
|
raise NativeResolveError.new(connection, host)
|
161
157
|
else
|
162
158
|
connections << connection
|
163
|
-
log
|
164
|
-
# :nocov:
|
165
|
-
"timeout after #{prev_timeout}s, retry(#{timeouts.first}) #{host}..."
|
166
|
-
# :nocov:
|
167
|
-
end
|
159
|
+
log { "resolver: timeout after #{prev_timeout}s, retry(#{timeouts.first}) #{host}..." }
|
168
160
|
end
|
169
161
|
end
|
170
162
|
@queries = queries
|
@@ -174,14 +166,11 @@ module HTTPX
|
|
174
166
|
def dread(wsize = @resolver_options.packet_size)
|
175
167
|
loop do
|
176
168
|
siz = @io.read(wsize, @read_buffer)
|
177
|
-
unless siz
|
178
|
-
emit(:close)
|
179
|
-
return
|
180
|
-
end
|
181
|
-
return if siz.zero?
|
169
|
+
return unless siz && siz.positive?
|
182
170
|
|
183
|
-
log
|
171
|
+
log { "resolver: READ: #{siz} bytes..." }
|
184
172
|
parse(@read_buffer)
|
173
|
+
return if @state == :closed
|
185
174
|
end
|
186
175
|
end
|
187
176
|
|
@@ -190,12 +179,10 @@ module HTTPX
|
|
190
179
|
return if @write_buffer.empty?
|
191
180
|
|
192
181
|
siz = @io.write(@write_buffer)
|
193
|
-
unless siz
|
194
|
-
|
195
|
-
|
196
|
-
|
197
|
-
log(label: "resolver: ") { "WRITE: #{siz} bytes..." }
|
198
|
-
return if siz.zero?
|
182
|
+
return unless siz && siz.positive?
|
183
|
+
|
184
|
+
log { "resolver: WRITE: #{siz} bytes..." }
|
185
|
+
return if @state == :closed
|
199
186
|
end
|
200
187
|
end
|
201
188
|
|
@@ -253,7 +240,7 @@ module HTTPX
|
|
253
240
|
hostname = hostname || @queries.key(connection) || connection.origin.host
|
254
241
|
@queries[hostname] = connection
|
255
242
|
type = @_record_types[hostname].first
|
256
|
-
log
|
243
|
+
log { "resolver: query #{type} for #{hostname}" }
|
257
244
|
begin
|
258
245
|
@write_buffer << Resolver.encode_dns_query(hostname, type: RECORD_TYPES[type])
|
259
246
|
rescue Resolv::DNS::EncodeError => e
|
@@ -269,7 +256,7 @@ module HTTPX
|
|
269
256
|
uri = URI::Generic.build(scheme: "udp", port: port)
|
270
257
|
uri.hostname = ip
|
271
258
|
type = IO.registry(uri.scheme)
|
272
|
-
log
|
259
|
+
log { "resolver: server: #{uri}..." }
|
273
260
|
@io = type.new(uri, [IPAddr.new(ip)], @options)
|
274
261
|
end
|
275
262
|
|
@@ -285,8 +272,11 @@ module HTTPX
|
|
285
272
|
return unless @state == :idle
|
286
273
|
|
287
274
|
build_socket
|
275
|
+
|
288
276
|
@io.connect
|
289
277
|
return unless @io.connected?
|
278
|
+
|
279
|
+
resolve if @queries.empty?
|
290
280
|
when :closed
|
291
281
|
return unless @state == :open
|
292
282
|
|
@@ -6,7 +6,7 @@ module HTTPX
|
|
6
6
|
@options = options
|
7
7
|
end
|
8
8
|
|
9
|
-
def method_missing(m,
|
9
|
+
def method_missing(m, *, &block)
|
10
10
|
if @options.key?(m)
|
11
11
|
@options[m]
|
12
12
|
else
|
@@ -14,7 +14,7 @@ module HTTPX
|
|
14
14
|
end
|
15
15
|
end
|
16
16
|
|
17
|
-
def respond_to_missing?(m)
|
17
|
+
def respond_to_missing?(m, *)
|
18
18
|
@options.key?(m) || super
|
19
19
|
end
|
20
20
|
|
@@ -30,7 +30,7 @@ module HTTPX
|
|
30
30
|
addresses.map! do |address|
|
31
31
|
address.is_a?(IPAddr) ? address : IPAddr.new(address.to_s)
|
32
32
|
end
|
33
|
-
log
|
33
|
+
log { "resolver: answer #{connection.origin.host}: #{addresses.inspect}" }
|
34
34
|
connection.addresses = addresses
|
35
35
|
catch(:coalesced) { emit(:resolve, connection) }
|
36
36
|
end
|