httpx 0.2.1 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/httpx.rb +1 -0
- data/lib/httpx/altsvc.rb +76 -0
- data/lib/httpx/chainable.rb +4 -0
- data/lib/httpx/channel.rb +72 -20
- data/lib/httpx/channel/http1.rb +71 -52
- data/lib/httpx/channel/http2.rb +43 -5
- data/lib/httpx/client.rb +32 -1
- data/lib/httpx/connection.rb +21 -8
- data/lib/httpx/errors.rb +18 -1
- data/lib/httpx/extensions.rb +72 -41
- data/lib/httpx/parser/http1.rb +171 -0
- data/lib/httpx/plugins/multipart.rb +50 -0
- data/lib/httpx/plugins/proxy.rb +6 -1
- data/lib/httpx/request.rb +8 -3
- data/lib/httpx/response.rb +15 -6
- data/lib/httpx/selector.rb +1 -1
- data/lib/httpx/timeout.rb +37 -8
- data/lib/httpx/transcoder/chunker.rb +76 -3
- data/lib/httpx/transcoder/form.rb +7 -11
- data/lib/httpx/version.rb +1 -1
- metadata +7 -18
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: bed8cbd465eb9d1b565024395767f619aaaff1bf2ee35d33dc15e9fe4eeec13e
|
4
|
+
data.tar.gz: 65ac8fee9f8c773d6b9a7894e68641547f75ee69159f01e60c8cdc1f30b648f0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1e891362a86b73f6d2670eb14a1fd0815a0fab0173f6adf58bdb8e8225d83012560492c5563291d10d03e05ab9e2d9c786ba0cc238e886832276639bab06a8aa
|
7
|
+
data.tar.gz: 539853ff0c6ca209f32d56b8f5eb6209ef60a4df02f0073a588b915a5a583753b7bebe66cd91b76e599a55bb214c6d56a9d1a4d47d6d7a4d5ea2e9b18254af3c
|
data/lib/httpx.rb
CHANGED
data/lib/httpx/altsvc.rb
ADDED
@@ -0,0 +1,76 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module HTTPX
|
4
|
+
module AltSvc
|
5
|
+
@altsvc_mutex = Mutex.new
|
6
|
+
@altsvcs = Hash.new { |h, k| h[k] = [] }
|
7
|
+
|
8
|
+
module_function
|
9
|
+
|
10
|
+
def cached_altsvc(origin)
|
11
|
+
now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
12
|
+
@altsvc_mutex.synchronize do
|
13
|
+
lookup(origin, now)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def cached_altsvc_set(origin, entry)
|
18
|
+
now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
19
|
+
@altsvc_mutex.synchronize do
|
20
|
+
return if @altsvcs[origin].any? { |altsvc| altsvc["origin"] == entry["origin"] }
|
21
|
+
entry["TTL"] = Integer(entry["ma"]) + now if entry.key?("ma")
|
22
|
+
@altsvcs[origin] << entry
|
23
|
+
entry
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def lookup(origin, ttl)
|
28
|
+
return [] unless @altsvcs.key?(origin)
|
29
|
+
@altsvcs[origin] = @altsvcs[origin].select do |entry|
|
30
|
+
!entry.key?("TTL") || entry["TTL"] > ttl
|
31
|
+
end
|
32
|
+
@altsvcs[origin].reject { |entry| entry["noop"] }
|
33
|
+
end
|
34
|
+
|
35
|
+
def emit(request, response)
|
36
|
+
# Alt-Svc
|
37
|
+
return unless response.headers.key?("alt-svc")
|
38
|
+
origin = request.origin
|
39
|
+
host = request.uri.host
|
40
|
+
parse(response.headers["alt-svc"]) do |alt_origin, alt_params|
|
41
|
+
alt_origin.host ||= host
|
42
|
+
yield(alt_origin, origin, alt_params)
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def parse(altsvc)
|
47
|
+
return enum_for(__method__, altsvc) unless block_given?
|
48
|
+
alt_origins, *alt_params = altsvc.split(/ *; */)
|
49
|
+
alt_params = Hash[alt_params.map { |field| field.split("=") }]
|
50
|
+
alt_origins.split(/ *, */).each do |alt_origin|
|
51
|
+
yield(parse_altsvc_origin(alt_origin), alt_params)
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
if RUBY_VERSION < "2.2"
|
56
|
+
def parse_altsvc_origin(alt_origin)
|
57
|
+
alt_proto, alt_origin = alt_origin.split("=")
|
58
|
+
alt_origin = alt_origin[1..-2] if alt_origin.start_with?("\"") && alt_origin.end_with?("\"")
|
59
|
+
if alt_origin.start_with?(":")
|
60
|
+
alt_origin = "dummy#{alt_origin}"
|
61
|
+
uri = URI.parse(alt_origin)
|
62
|
+
uri.host = nil
|
63
|
+
uri
|
64
|
+
else
|
65
|
+
URI.parse("#{alt_proto}://#{alt_origin}")
|
66
|
+
end
|
67
|
+
end
|
68
|
+
else
|
69
|
+
def parse_altsvc_origin(alt_origin)
|
70
|
+
alt_proto, alt_origin = alt_origin.split("=")
|
71
|
+
alt_origin = alt_origin[1..-2] if alt_origin.start_with?("\"") && alt_origin.end_with?("\"")
|
72
|
+
URI.parse("#{alt_proto}://#{alt_origin}")
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
data/lib/httpx/chainable.rb
CHANGED
data/lib/httpx/channel.rb
CHANGED
@@ -35,6 +35,8 @@ module HTTPX
|
|
35
35
|
include Loggable
|
36
36
|
include Callbacks
|
37
37
|
|
38
|
+
using URIExtensions
|
39
|
+
|
38
40
|
require "httpx/channel/http2"
|
39
41
|
require "httpx/channel/http1"
|
40
42
|
|
@@ -44,10 +46,15 @@ module HTTPX
|
|
44
46
|
def by(uri, options)
|
45
47
|
type = options.transport || begin
|
46
48
|
case uri.scheme
|
47
|
-
when "http"
|
48
|
-
|
49
|
+
when "http"
|
50
|
+
"tcp"
|
51
|
+
when "https"
|
52
|
+
"ssl"
|
53
|
+
when "h2"
|
54
|
+
options = options.merge(ssl: { alpn_protocols: %(h2) })
|
55
|
+
"ssl"
|
49
56
|
else
|
50
|
-
raise
|
57
|
+
raise UnsupportedSchemeError, "#{uri}: #{uri.scheme}: unsupported URI scheme"
|
51
58
|
end
|
52
59
|
end
|
53
60
|
new(type, uri, options)
|
@@ -58,19 +65,19 @@ module HTTPX
|
|
58
65
|
|
59
66
|
def_delegator :@write_buffer, :empty?
|
60
67
|
|
61
|
-
attr_reader :uri, :state
|
68
|
+
attr_reader :uri, :state, :pending
|
62
69
|
|
63
70
|
def initialize(type, uri, options)
|
64
71
|
@type = type
|
65
72
|
@uri = uri
|
66
|
-
@
|
73
|
+
@origins = [@uri.origin]
|
67
74
|
@options = Options.new(options)
|
68
75
|
@window_size = @options.window_size
|
69
76
|
@read_buffer = Buffer.new(BUFFER_SIZE)
|
70
77
|
@write_buffer = Buffer.new(BUFFER_SIZE)
|
71
78
|
@pending = []
|
72
|
-
@state = :idle
|
73
79
|
on(:error) { |ex| on_error(ex) }
|
80
|
+
transition(:idle)
|
74
81
|
end
|
75
82
|
|
76
83
|
def addresses=(addrs)
|
@@ -88,14 +95,12 @@ module HTTPX
|
|
88
95
|
if @io.protocol == "h2" && @uri.scheme == "https"
|
89
96
|
@io.verify_hostname(channel.uri.host)
|
90
97
|
else
|
91
|
-
@uri.
|
92
|
-
@uri.port == channel.uri.port &&
|
93
|
-
@uri.scheme == channel.uri.scheme
|
98
|
+
@uri.origin == channel.uri.origin
|
94
99
|
end
|
95
100
|
end
|
96
101
|
|
97
102
|
def merge(channel)
|
98
|
-
@
|
103
|
+
@origins += channel.instance_variable_get(:@origins)
|
99
104
|
pending = channel.instance_variable_get(:@pending)
|
100
105
|
pending.each do |req, args|
|
101
106
|
send(req, args)
|
@@ -103,14 +108,20 @@ module HTTPX
|
|
103
108
|
end
|
104
109
|
|
105
110
|
def unmerge(channel)
|
106
|
-
@
|
111
|
+
@origins -= channel.instance_variable_get(:@origins)
|
112
|
+
purge_pending do |request, args|
|
113
|
+
request.uri == channel.uri && begin
|
114
|
+
request.transition(:idle)
|
115
|
+
channel.send(request, *args)
|
116
|
+
true
|
117
|
+
end
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
def purge_pending
|
107
122
|
[@parser.pending, @pending].each do |pending|
|
108
|
-
pending.reject! do |request|
|
109
|
-
request
|
110
|
-
request.transition(:idle)
|
111
|
-
channel.send(request)
|
112
|
-
true
|
113
|
-
end
|
123
|
+
pending.reject! do |request, *args|
|
124
|
+
yield(request, args)
|
114
125
|
end
|
115
126
|
end
|
116
127
|
end
|
@@ -118,9 +129,20 @@ module HTTPX
|
|
118
129
|
def match?(uri)
|
119
130
|
return false if @state == :closing
|
120
131
|
|
121
|
-
@
|
122
|
-
|
123
|
-
|
132
|
+
@origins.include?(uri.origin) || match_altsvcs?(uri)
|
133
|
+
end
|
134
|
+
|
135
|
+
# checks if this is channel is an alternative service of
|
136
|
+
# +uri+
|
137
|
+
def match_altsvcs?(uri)
|
138
|
+
AltSvc.cached_altsvc(@uri.origin).any? do |altsvc|
|
139
|
+
origin = altsvc["origin"]
|
140
|
+
origin.altsvc_match?(uri.origin)
|
141
|
+
end
|
142
|
+
end
|
143
|
+
|
144
|
+
def connecting?
|
145
|
+
@state == :idle
|
124
146
|
end
|
125
147
|
|
126
148
|
def interests
|
@@ -157,6 +179,7 @@ module HTTPX
|
|
157
179
|
if @error_response
|
158
180
|
emit(:response, request, @error_response)
|
159
181
|
elsif @parser && !@write_buffer.full?
|
182
|
+
request.headers["alt-used"] = @uri.authority if match_altsvcs?(request.uri)
|
160
183
|
parser.send(request, **args)
|
161
184
|
else
|
162
185
|
@pending << [request, args]
|
@@ -164,6 +187,7 @@ module HTTPX
|
|
164
187
|
end
|
165
188
|
|
166
189
|
def call
|
190
|
+
@timeout = @timeout_threshold
|
167
191
|
case @state
|
168
192
|
when :closed
|
169
193
|
return
|
@@ -182,6 +206,17 @@ module HTTPX
|
|
182
206
|
@parser = build_parser(protocol)
|
183
207
|
end
|
184
208
|
|
209
|
+
def handle_timeout_error(e)
|
210
|
+
return emit(:error, e) unless @timeout
|
211
|
+
@timeout -= e.timeout
|
212
|
+
return unless @timeout <= 0
|
213
|
+
if connecting?
|
214
|
+
emit(:error, e.to_connection_error)
|
215
|
+
else
|
216
|
+
emit(:error, e)
|
217
|
+
end
|
218
|
+
end
|
219
|
+
|
185
220
|
private
|
186
221
|
|
187
222
|
def consume
|
@@ -204,6 +239,7 @@ module HTTPX
|
|
204
239
|
return if siz.zero?
|
205
240
|
log { "READ: #{siz} bytes..." }
|
206
241
|
parser << @read_buffer.to_s
|
242
|
+
return if @state == :closing || @state == :closed
|
207
243
|
end
|
208
244
|
end
|
209
245
|
|
@@ -219,6 +255,7 @@ module HTTPX
|
|
219
255
|
end
|
220
256
|
log { "WRITE: #{siz} bytes..." }
|
221
257
|
return if siz.zero?
|
258
|
+
return if @state == :closing || @state == :closed
|
222
259
|
end
|
223
260
|
end
|
224
261
|
|
@@ -236,8 +273,15 @@ module HTTPX
|
|
236
273
|
def build_parser(protocol = @io.protocol)
|
237
274
|
parser = registry(protocol).new(@write_buffer, @options)
|
238
275
|
parser.on(:response) do |*args|
|
276
|
+
AltSvc.emit(*args) do |alt_origin, origin, alt_params|
|
277
|
+
emit(:altsvc, alt_origin, origin, alt_params)
|
278
|
+
end
|
239
279
|
emit(:response, *args)
|
240
280
|
end
|
281
|
+
parser.on(:altsvc) do |alt_origin, origin, alt_params|
|
282
|
+
emit(:altsvc, alt_origin, origin, alt_params)
|
283
|
+
end
|
284
|
+
|
241
285
|
parser.on(:promise) do |*args|
|
242
286
|
emit(:promise, *args)
|
243
287
|
end
|
@@ -248,10 +292,14 @@ module HTTPX
|
|
248
292
|
transition(:closing)
|
249
293
|
unless parser.empty?
|
250
294
|
transition(:closed)
|
295
|
+
emit(:reset)
|
251
296
|
transition(:idle)
|
252
297
|
transition(:open)
|
253
298
|
end
|
254
299
|
end
|
300
|
+
parser.on(:timeout) do |timeout|
|
301
|
+
@timeout = timeout
|
302
|
+
end
|
255
303
|
parser.on(:error) do |request, ex|
|
256
304
|
case ex
|
257
305
|
when MisdirectedRequestError
|
@@ -269,11 +317,15 @@ module HTTPX
|
|
269
317
|
# when :idle
|
270
318
|
when :idle
|
271
319
|
@error_response = nil
|
320
|
+
@timeout_threshold = @options.timeout.connect_timeout
|
321
|
+
@timeout = @timeout_threshold
|
272
322
|
when :open
|
273
323
|
return if @state == :closed
|
274
324
|
@io.connect
|
275
325
|
return unless @io.connected?
|
276
326
|
send_pending
|
327
|
+
@timeout_threshold = @options.timeout.operation_timeout
|
328
|
+
@timeout = @timeout_threshold
|
277
329
|
emit(:open)
|
278
330
|
when :closing
|
279
331
|
return unless @state == :open
|
data/lib/httpx/channel/http1.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require "
|
3
|
+
require "httpx/parser/http1"
|
4
4
|
|
5
5
|
module HTTPX
|
6
6
|
class Channel::HTTP1
|
@@ -12,18 +12,16 @@ module HTTPX
|
|
12
12
|
def initialize(buffer, options)
|
13
13
|
@options = Options.new(options)
|
14
14
|
@max_concurrent_requests = @options.max_concurrent_requests
|
15
|
-
@
|
16
|
-
@parser
|
15
|
+
@max_requests = Float::INFINITY
|
16
|
+
@parser = Parser::HTTP1.new(self)
|
17
17
|
@buffer = buffer
|
18
18
|
@version = [1, 1]
|
19
19
|
@pending = []
|
20
20
|
@requests = []
|
21
|
-
@has_response = false
|
22
21
|
end
|
23
22
|
|
24
23
|
def reset
|
25
24
|
@parser.reset!
|
26
|
-
@has_response = false
|
27
25
|
end
|
28
26
|
|
29
27
|
def close
|
@@ -39,15 +37,18 @@ module HTTPX
|
|
39
37
|
|
40
38
|
def <<(data)
|
41
39
|
@parser << data
|
42
|
-
dispatch if @has_response
|
43
40
|
end
|
44
41
|
|
45
42
|
def send(request, **)
|
46
|
-
if @
|
43
|
+
if @max_requests.positive? &&
|
44
|
+
@requests.size >= @max_concurrent_requests
|
47
45
|
@pending << request
|
48
46
|
return
|
49
47
|
end
|
50
|
-
|
48
|
+
unless @requests.include?(request)
|
49
|
+
@requests << request
|
50
|
+
@pipelining = true if @requests.size > 1
|
51
|
+
end
|
51
52
|
handle(request)
|
52
53
|
end
|
53
54
|
|
@@ -69,69 +70,59 @@ module HTTPX
|
|
69
70
|
#
|
70
71
|
# must be public methods, or else they won't be reachable
|
71
72
|
|
72
|
-
def
|
73
|
+
def on_start
|
73
74
|
log(level: 2) { "parsing begins" }
|
74
75
|
end
|
75
76
|
|
76
|
-
def
|
77
|
-
|
78
|
-
|
79
|
-
# callback is called 2 times when chunked
|
80
|
-
request = @requests.first
|
81
|
-
return if request.response
|
77
|
+
def on_headers(h)
|
78
|
+
@request = @requests.first
|
79
|
+
return if @request.response
|
82
80
|
|
83
81
|
log(level: 2) { "headers received" }
|
84
82
|
headers = @options.headers_class.new(h)
|
85
|
-
response = @options.response_class.new(@
|
83
|
+
response = @options.response_class.new(@request,
|
86
84
|
@parser.status_code,
|
87
85
|
@parser.http_version.join("."),
|
88
86
|
headers, @options)
|
89
87
|
log(color: :yellow) { "-> HEADLINE: #{response.status} HTTP/#{@parser.http_version.join(".")}" }
|
90
88
|
log(color: :yellow) { response.headers.each.map { |f, v| "-> HEADER: #{f}: #{v}" }.join("\n") }
|
91
89
|
|
92
|
-
request.response = response
|
90
|
+
@request.response = response
|
91
|
+
on_complete if response.complete?
|
92
|
+
end
|
93
|
+
|
94
|
+
def on_trailers(h)
|
95
|
+
return unless @request
|
96
|
+
response = @request.response
|
97
|
+
log(level: 2) { "trailer headers received" }
|
93
98
|
|
94
|
-
|
99
|
+
log(color: :yellow) { h.each.map { |f, v| "-> HEADER: #{f}: #{v}" }.join("\n") }
|
100
|
+
response.merge_headers(h)
|
95
101
|
end
|
96
102
|
|
97
|
-
def
|
103
|
+
def on_data(chunk)
|
104
|
+
return unless @request
|
98
105
|
log(color: :green) { "-> DATA: #{chunk.bytesize} bytes..." }
|
99
106
|
log(level: 2, color: :green) { "-> #{chunk.inspect}" }
|
100
|
-
response = @
|
107
|
+
response = @request.response
|
101
108
|
|
102
109
|
response << chunk
|
103
|
-
|
104
|
-
@has_response = response.complete?
|
105
110
|
end
|
106
111
|
|
107
|
-
def
|
112
|
+
def on_complete
|
113
|
+
return unless @request
|
108
114
|
log(level: 2) { "parsing complete" }
|
109
|
-
|
110
|
-
response = request.response
|
111
|
-
|
112
|
-
if !@parser_trailers && response.headers.key?("trailer")
|
113
|
-
@parser_trailers = true
|
114
|
-
# this is needed, because the parser can't accept further headers.
|
115
|
-
# we need to reset it and artificially move it to receive headers state,
|
116
|
-
# hence the bogus headline
|
117
|
-
#
|
118
|
-
@parser.reset!
|
119
|
-
@parser << "#{request.verb.to_s.upcase} #{request.path} HTTP/#{response.version}#{CRLF}"
|
120
|
-
else
|
121
|
-
@has_response = true
|
122
|
-
end
|
123
|
-
end
|
124
|
-
|
125
|
-
def on_trailer_headers_complete(h)
|
126
|
-
response = @requests.first.response
|
127
|
-
|
128
|
-
response.merge_headers(h)
|
115
|
+
dispatch
|
129
116
|
end
|
130
117
|
|
131
118
|
def dispatch
|
132
|
-
|
133
|
-
|
119
|
+
if @request.expects?
|
120
|
+
reset
|
121
|
+
return handle(@request)
|
122
|
+
end
|
134
123
|
|
124
|
+
request = @request
|
125
|
+
@request = nil
|
135
126
|
@requests.shift
|
136
127
|
response = request.response
|
137
128
|
emit(:response, request, response)
|
@@ -140,28 +131,57 @@ module HTTPX
|
|
140
131
|
response << @parser.upgrade_data
|
141
132
|
throw(:called)
|
142
133
|
end
|
134
|
+
|
143
135
|
reset
|
136
|
+
@max_requests -= 1
|
144
137
|
send(@pending.shift) unless @pending.empty?
|
145
|
-
|
146
|
-
disable_concurrency
|
147
|
-
emit(:reset)
|
138
|
+
manage_connection(response)
|
148
139
|
end
|
149
140
|
|
150
141
|
def handle_error(ex)
|
151
|
-
@
|
152
|
-
|
142
|
+
if @pipelining
|
143
|
+
disable_pipelining
|
144
|
+
emit(:reset)
|
145
|
+
throw(:called)
|
146
|
+
else
|
147
|
+
@requests.each do |request|
|
148
|
+
emit(:error, request, ex)
|
149
|
+
end
|
153
150
|
end
|
154
151
|
end
|
155
152
|
|
156
153
|
private
|
157
154
|
|
158
|
-
def
|
155
|
+
def manage_connection(response)
|
156
|
+
connection = response.headers["connection"]
|
157
|
+
case connection
|
158
|
+
when /keep\-alive/i
|
159
|
+
keep_alive = response.headers["keep-alive"]
|
160
|
+
return unless keep_alive
|
161
|
+
parameters = Hash[keep_alive.split(/ *, */).map do |pair|
|
162
|
+
pair.split(/ *= */)
|
163
|
+
end]
|
164
|
+
@max_requests = parameters["max"].to_i if parameters.key?("max")
|
165
|
+
if parameters.key?("timeout")
|
166
|
+
keep_alive_timeout = parameters["timeout"].to_i
|
167
|
+
emit(:timeout, keep_alive_timeout)
|
168
|
+
end
|
169
|
+
# TODO: on keep alive timeout, reset
|
170
|
+
when /close/i, nil
|
171
|
+
disable_pipelining
|
172
|
+
@max_requests = Float::INFINITY
|
173
|
+
emit(:reset)
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
177
|
+
def disable_pipelining
|
159
178
|
return if @requests.empty?
|
160
179
|
@requests.each { |r| r.transition(:idle) }
|
161
180
|
# server doesn't handle pipelining, and probably
|
162
181
|
# doesn't support keep-alive. Fallback to send only
|
163
182
|
# 1 keep alive request.
|
164
183
|
@max_concurrent_requests = 1
|
184
|
+
@pipelining = false
|
165
185
|
end
|
166
186
|
|
167
187
|
def set_request_headers(request)
|
@@ -178,7 +198,6 @@ module HTTPX
|
|
178
198
|
end
|
179
199
|
|
180
200
|
def handle(request)
|
181
|
-
@has_response = false
|
182
201
|
set_request_headers(request)
|
183
202
|
catch(:buffer_full) do
|
184
203
|
request.transition(:headers)
|