httpx 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/httpx.rb +8 -2
- data/lib/httpx/adapters/faraday.rb +203 -0
- data/lib/httpx/altsvc.rb +4 -0
- data/lib/httpx/callbacks.rb +1 -4
- data/lib/httpx/chainable.rb +4 -3
- data/lib/httpx/connection.rb +326 -104
- data/lib/httpx/{channel → connection}/http1.rb +29 -15
- data/lib/httpx/{channel → connection}/http2.rb +12 -6
- data/lib/httpx/errors.rb +2 -0
- data/lib/httpx/headers.rb +4 -1
- data/lib/httpx/io/ssl.rb +5 -1
- data/lib/httpx/io/tcp.rb +13 -7
- data/lib/httpx/io/udp.rb +1 -0
- data/lib/httpx/io/unix.rb +1 -0
- data/lib/httpx/loggable.rb +34 -9
- data/lib/httpx/options.rb +57 -31
- data/lib/httpx/parser/http1.rb +8 -0
- data/lib/httpx/plugins/authentication.rb +4 -0
- data/lib/httpx/plugins/basic_authentication.rb +4 -0
- data/lib/httpx/plugins/compression.rb +22 -5
- data/lib/httpx/plugins/cookies.rb +89 -36
- data/lib/httpx/plugins/digest_authentication.rb +45 -26
- data/lib/httpx/plugins/follow_redirects.rb +61 -62
- data/lib/httpx/plugins/h2c.rb +78 -39
- data/lib/httpx/plugins/multipart.rb +5 -0
- data/lib/httpx/plugins/persistent.rb +29 -0
- data/lib/httpx/plugins/proxy.rb +125 -78
- data/lib/httpx/plugins/proxy/http.rb +31 -27
- data/lib/httpx/plugins/proxy/socks4.rb +30 -24
- data/lib/httpx/plugins/proxy/socks5.rb +49 -39
- data/lib/httpx/plugins/proxy/ssh.rb +81 -0
- data/lib/httpx/plugins/push_promise.rb +18 -9
- data/lib/httpx/plugins/retries.rb +43 -15
- data/lib/httpx/pool.rb +159 -0
- data/lib/httpx/registry.rb +2 -0
- data/lib/httpx/request.rb +10 -0
- data/lib/httpx/resolver.rb +2 -1
- data/lib/httpx/resolver/https.rb +62 -56
- data/lib/httpx/resolver/native.rb +48 -37
- data/lib/httpx/resolver/resolver_mixin.rb +16 -11
- data/lib/httpx/resolver/system.rb +11 -7
- data/lib/httpx/response.rb +24 -10
- data/lib/httpx/selector.rb +32 -39
- data/lib/httpx/{client.rb → session.rb} +99 -62
- data/lib/httpx/timeout.rb +7 -15
- data/lib/httpx/transcoder/body.rb +4 -0
- data/lib/httpx/transcoder/chunker.rb +4 -0
- data/lib/httpx/version.rb +1 -1
- metadata +10 -8
- data/lib/httpx/channel.rb +0 -367
@@ -1,45 +1,47 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
module HTTPX
|
4
|
-
class
|
4
|
+
class Session
|
5
5
|
include Loggable
|
6
6
|
include Chainable
|
7
7
|
|
8
8
|
def initialize(options = {}, &blk)
|
9
9
|
@options = self.class.default_options.merge(options)
|
10
|
-
@connection = Connection.new(@options)
|
11
10
|
@responses = {}
|
12
|
-
@
|
11
|
+
@persistent = @options.persistent
|
13
12
|
wrap(&blk) if block_given?
|
14
13
|
end
|
15
14
|
|
16
15
|
def wrap
|
17
16
|
return unless block_given?
|
17
|
+
|
18
18
|
begin
|
19
|
-
|
20
|
-
@
|
19
|
+
prev_persistent = @persistent
|
20
|
+
@persistent = true
|
21
21
|
yield self
|
22
22
|
ensure
|
23
|
-
@
|
24
|
-
close
|
23
|
+
@persistent = prev_persistent
|
25
24
|
end
|
26
25
|
end
|
27
26
|
|
28
|
-
def close
|
29
|
-
|
27
|
+
def close(*args)
|
28
|
+
pool.close(*args)
|
30
29
|
end
|
31
30
|
|
32
|
-
def request(*args,
|
33
|
-
requests =
|
34
|
-
responses =
|
31
|
+
def request(*args, **options)
|
32
|
+
requests = build_requests(*args, options)
|
33
|
+
responses = send_requests(*requests, options)
|
35
34
|
return responses.first if responses.size == 1
|
35
|
+
|
36
36
|
responses
|
37
|
-
ensure
|
38
|
-
close unless keep_open
|
39
37
|
end
|
40
38
|
|
41
39
|
private
|
42
40
|
|
41
|
+
def pool
|
42
|
+
Thread.current[:httpx_connection_pool] ||= Pool.new
|
43
|
+
end
|
44
|
+
|
43
45
|
def on_response(request, response)
|
44
46
|
@responses[request] = response
|
45
47
|
end
|
@@ -49,55 +51,54 @@ module HTTPX
|
|
49
51
|
stream.refuse
|
50
52
|
end
|
51
53
|
|
52
|
-
def fetch_response(request)
|
54
|
+
def fetch_response(request, _, _)
|
53
55
|
@responses.delete(request)
|
54
56
|
end
|
55
57
|
|
56
|
-
def
|
58
|
+
def find_connection(request, connections, options)
|
57
59
|
uri = URI(request.uri)
|
58
|
-
|
60
|
+
connection = pool.find_connection(uri, options) || build_connection(uri, options)
|
61
|
+
unless connections.nil? || connections.include?(connection)
|
62
|
+
connections << connection
|
63
|
+
set_connection_callbacks(connection, options)
|
64
|
+
end
|
65
|
+
connection
|
59
66
|
end
|
60
67
|
|
61
|
-
def
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
other_channel = build_channel(uncoalesced_uri, options)
|
66
|
-
channel.unmerge(other_channel)
|
68
|
+
def set_connection_callbacks(connection, options)
|
69
|
+
connection.on(:uncoalesce) do |uncoalesced_uri|
|
70
|
+
other_connection = build_connection(uncoalesced_uri, options)
|
71
|
+
connection.unmerge(other_connection)
|
67
72
|
end
|
68
|
-
|
69
|
-
|
73
|
+
connection.on(:altsvc) do |alt_origin, origin, alt_params|
|
74
|
+
build_altsvc_connection(connection, alt_origin, origin, alt_params, options)
|
70
75
|
end
|
71
76
|
end
|
72
77
|
|
73
|
-
def
|
74
|
-
channel = @connection.build_channel(uri, **options)
|
75
|
-
set_channel_callbacks(channel, options)
|
76
|
-
channel
|
77
|
-
end
|
78
|
-
|
79
|
-
def build_altsvc_channel(existing_channel, alt_origin, origin, alt_params, options)
|
78
|
+
def build_altsvc_connection(existing_connection, alt_origin, origin, alt_params, options)
|
80
79
|
altsvc = AltSvc.cached_altsvc_set(origin, alt_params.merge("origin" => alt_origin))
|
81
80
|
|
82
81
|
# altsvc already exists, somehow it wasn't advertised, probably noop
|
83
82
|
return unless altsvc
|
84
83
|
|
85
|
-
|
84
|
+
connection = pool.find_connection(alt_origin, options) || build_connection(alt_origin, options)
|
86
85
|
# advertised altsvc is the same origin being used, ignore
|
87
|
-
return if
|
86
|
+
return if connection == existing_connection
|
87
|
+
|
88
|
+
set_connection_callbacks(connection, options)
|
88
89
|
|
89
90
|
log(level: 1) { "#{origin} alt-svc: #{alt_origin}" }
|
90
91
|
|
91
92
|
# get uninitialized requests
|
92
93
|
# incidentally, all requests will be re-routed to the first
|
93
94
|
# advertised alt-svc, which incidentally follows the spec.
|
94
|
-
|
95
|
+
existing_connection.purge_pending do |request|
|
95
96
|
is_idle = request.origin == origin &&
|
96
97
|
request.state == :idle &&
|
97
98
|
!request.headers.key?("alt-used")
|
98
99
|
if is_idle
|
99
100
|
log(level: 1) { "#{origin} alt-svc: sending #{request.uri} to #{alt_origin}" }
|
100
|
-
|
101
|
+
connection.send(request)
|
101
102
|
end
|
102
103
|
is_idle
|
103
104
|
end
|
@@ -105,57 +106,92 @@ module HTTPX
|
|
105
106
|
altsvc["noop"] = true
|
106
107
|
end
|
107
108
|
|
108
|
-
def
|
109
|
+
def build_requests(*args, options)
|
110
|
+
request_options = @options.merge(options)
|
111
|
+
|
109
112
|
requests = case args.size
|
110
113
|
when 1
|
111
114
|
reqs = args.first
|
112
115
|
reqs.map do |verb, uri|
|
113
|
-
|
116
|
+
build_request(verb, uri, request_options)
|
114
117
|
end
|
115
118
|
when 2, 3
|
116
119
|
verb, uris = args
|
117
120
|
if uris.respond_to?(:each)
|
118
121
|
uris.map do |uri, **opts|
|
119
|
-
|
122
|
+
build_request(verb, uri, request_options.merge(opts))
|
120
123
|
end
|
121
124
|
else
|
122
|
-
[
|
125
|
+
[build_request(verb, uris, request_options)]
|
123
126
|
end
|
124
127
|
else
|
125
128
|
raise ArgumentError, "unsupported number of arguments"
|
126
129
|
end
|
127
130
|
raise ArgumentError, "wrong number of URIs (given 0, expect 1..+1)" if requests.empty?
|
131
|
+
|
128
132
|
requests
|
129
133
|
end
|
130
134
|
|
131
|
-
def
|
135
|
+
def build_connection(uri, options)
|
136
|
+
type = options.transport || begin
|
137
|
+
case uri.scheme
|
138
|
+
when "http"
|
139
|
+
"tcp"
|
140
|
+
when "https"
|
141
|
+
"ssl"
|
142
|
+
when "h2"
|
143
|
+
options = options.merge(ssl: { alpn_protocols: %w[h2] })
|
144
|
+
"ssl"
|
145
|
+
else
|
146
|
+
raise UnsupportedSchemeError, "#{uri}: #{uri.scheme}: unsupported URI scheme"
|
147
|
+
end
|
148
|
+
end
|
149
|
+
connection = options.connection_class.new(type, uri, options)
|
150
|
+
pool.init_connection(connection, options)
|
151
|
+
connection
|
152
|
+
end
|
153
|
+
|
154
|
+
def send_requests(*requests, options)
|
155
|
+
connections = []
|
156
|
+
request_options = @options.merge(options)
|
157
|
+
timeout = request_options.timeout
|
158
|
+
|
132
159
|
requests.each do |request|
|
133
|
-
|
134
|
-
|
160
|
+
connection = find_connection(request, connections, request_options)
|
161
|
+
connection.send(request)
|
135
162
|
end
|
163
|
+
|
136
164
|
responses = []
|
137
165
|
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
166
|
+
begin
|
167
|
+
# guarantee ordered responses
|
168
|
+
loop do
|
169
|
+
begin
|
170
|
+
request = requests.first
|
171
|
+
pool.next_tick(timeout) until (response = fetch_response(request, connections, request_options))
|
143
172
|
|
144
|
-
|
145
|
-
|
173
|
+
responses << response
|
174
|
+
requests.shift
|
146
175
|
|
147
|
-
|
176
|
+
break if requests.empty? || pool.empty?
|
177
|
+
end
|
148
178
|
end
|
179
|
+
responses
|
180
|
+
ensure
|
181
|
+
close(connections) unless @persistent
|
149
182
|
end
|
150
|
-
responses
|
151
183
|
end
|
152
184
|
|
153
|
-
def
|
185
|
+
def build_request(verb, uri, options)
|
154
186
|
rklass = @options.request_class
|
155
|
-
rklass.new(verb, uri, @options.merge(options))
|
187
|
+
request = rklass.new(verb, uri, @options.merge(options))
|
188
|
+
request.on(:response, &method(:on_response).curry[request])
|
189
|
+
request.on(:promise, &method(:on_promise))
|
190
|
+
request
|
156
191
|
end
|
157
192
|
|
158
193
|
@default_options = Options.new
|
194
|
+
@default_options.freeze
|
159
195
|
@plugins = []
|
160
196
|
|
161
197
|
class << self
|
@@ -163,7 +199,7 @@ module HTTPX
|
|
163
199
|
|
164
200
|
def inherited(klass)
|
165
201
|
super
|
166
|
-
klass.instance_variable_set(:@default_options, @default_options
|
202
|
+
klass.instance_variable_set(:@default_options, @default_options)
|
167
203
|
klass.instance_variable_set(:@plugins, @plugins.dup)
|
168
204
|
end
|
169
205
|
|
@@ -173,15 +209,13 @@ module HTTPX
|
|
173
209
|
unless @plugins.include?(pl)
|
174
210
|
@plugins << pl
|
175
211
|
pl.load_dependencies(self, *args, &block) if pl.respond_to?(:load_dependencies)
|
212
|
+
@default_options = @default_options.dup
|
213
|
+
@default_options = pl.extra_options(@default_options) if pl.respond_to?(:extra_options)
|
214
|
+
|
176
215
|
include(pl::InstanceMethods) if defined?(pl::InstanceMethods)
|
177
216
|
extend(pl::ClassMethods) if defined?(pl::ClassMethods)
|
178
|
-
|
179
|
-
|
180
|
-
options_klass.extend(pl::OptionsClassMethods) if defined?(pl::OptionsClassMethods)
|
181
|
-
options_klass.__send__(:include, pl::OptionsMethods) if defined?(pl::OptionsMethods)
|
182
|
-
@default_options = options_klass.new
|
183
|
-
end
|
184
|
-
opts = default_options
|
217
|
+
|
218
|
+
opts = @default_options
|
185
219
|
opts.request_class.__send__(:include, pl::RequestMethods) if defined?(pl::RequestMethods)
|
186
220
|
opts.request_class.extend(pl::RequestClassMethods) if defined?(pl::RequestClassMethods)
|
187
221
|
opts.response_class.__send__(:include, pl::ResponseMethods) if defined?(pl::ResponseMethods)
|
@@ -192,7 +226,10 @@ module HTTPX
|
|
192
226
|
opts.request_body_class.extend(pl::RequestBodyClassMethods) if defined?(pl::RequestBodyClassMethods)
|
193
227
|
opts.response_body_class.__send__(:include, pl::ResponseBodyMethods) if defined?(pl::ResponseBodyMethods)
|
194
228
|
opts.response_body_class.extend(pl::ResponseBodyClassMethods) if defined?(pl::ResponseBodyClassMethods)
|
229
|
+
opts.connection_class.__send__(:include, pl::ConnectionMethods) if defined?(pl::ConnectionMethods)
|
195
230
|
pl.configure(self, *args, &block) if pl.respond_to?(:configure)
|
231
|
+
|
232
|
+
@default_options.freeze
|
196
233
|
end
|
197
234
|
self
|
198
235
|
end
|
data/lib/httpx/timeout.rb
CHANGED
@@ -9,6 +9,7 @@ module HTTPX
|
|
9
9
|
|
10
10
|
def self.new(opts = {})
|
11
11
|
return opts if opts.is_a?(Timeout)
|
12
|
+
|
12
13
|
super
|
13
14
|
end
|
14
15
|
|
@@ -26,12 +27,10 @@ module HTTPX
|
|
26
27
|
@operation_timeout = loop_timeout
|
27
28
|
end
|
28
29
|
reset_counter
|
29
|
-
@state = :idle # this is here not to trigger the log
|
30
|
-
transition(:idle)
|
31
30
|
end
|
32
31
|
|
33
|
-
def
|
34
|
-
@
|
32
|
+
def total_timeout
|
33
|
+
@total_timeout
|
35
34
|
ensure
|
36
35
|
log_time
|
37
36
|
end
|
@@ -63,16 +62,8 @@ module HTTPX
|
|
63
62
|
end
|
64
63
|
end
|
65
64
|
|
66
|
-
def
|
67
|
-
|
68
|
-
case nextstate
|
69
|
-
# when :idle
|
70
|
-
when :idle
|
71
|
-
@timeout = @connect_timeout
|
72
|
-
when :open
|
73
|
-
@timeout = @operation_timeout
|
74
|
-
end
|
75
|
-
@state = nextstate
|
65
|
+
def no_time_left?
|
66
|
+
@time_left <= 0
|
76
67
|
end
|
77
68
|
|
78
69
|
private
|
@@ -88,8 +79,9 @@ module HTTPX
|
|
88
79
|
def log_time
|
89
80
|
return unless @time_left
|
90
81
|
return reset_timer unless @started
|
82
|
+
|
91
83
|
@time_left -= (Process.clock_gettime(Process::CLOCK_MONOTONIC) - @started)
|
92
|
-
raise
|
84
|
+
raise TotalTimeoutError.new(@total_timeout, "Timed out after #{@total_timeout} seconds") if no_time_left?
|
93
85
|
|
94
86
|
reset_timer
|
95
87
|
end
|
@@ -4,6 +4,8 @@ require "forwardable"
|
|
4
4
|
|
5
5
|
module HTTPX::Transcoder
|
6
6
|
module Body
|
7
|
+
Error = Class.new(HTTPX::Error)
|
8
|
+
|
7
9
|
module_function
|
8
10
|
|
9
11
|
class Encoder
|
@@ -22,6 +24,8 @@ module HTTPX::Transcoder
|
|
22
24
|
@raw.map(&:bytesize).reduce(0, :+)
|
23
25
|
elsif @raw.respond_to?(:size)
|
24
26
|
@raw.size || Float::INFINITY
|
27
|
+
elsif @raw.respond_to?(:length)
|
28
|
+
@raw.length || Float::INFINITY
|
25
29
|
elsif @raw.respond_to?(:each)
|
26
30
|
Float::INFINITY
|
27
31
|
else
|
@@ -18,6 +18,7 @@ module HTTPX::Transcoder
|
|
18
18
|
|
19
19
|
def each
|
20
20
|
return enum_for(__method__) unless block_given?
|
21
|
+
|
21
22
|
@raw.each do |chunk|
|
22
23
|
yield "#{chunk.bytesize.to_s(16)}#{CRLF}#{chunk}#{CRLF}"
|
23
24
|
end
|
@@ -57,6 +58,7 @@ module HTTPX::Transcoder
|
|
57
58
|
when :length
|
58
59
|
index = @buffer.index(CRLF)
|
59
60
|
return unless index && index.positive?
|
61
|
+
|
60
62
|
# Read hex-length
|
61
63
|
hexlen = @buffer.slice!(0, index)
|
62
64
|
hexlen[/\h/] || raise(Error, "wrong chunk size line: #{hexlen}")
|
@@ -69,11 +71,13 @@ module HTTPX::Transcoder
|
|
69
71
|
# consume CRLF
|
70
72
|
return if @buffer.bytesize < crlf_size
|
71
73
|
raise Error, "wrong chunked encoding format" unless @buffer.start_with?(CRLF * (crlf_size / 2))
|
74
|
+
|
72
75
|
@buffer.slice!(0, crlf_size)
|
73
76
|
if @chunk_length.nil?
|
74
77
|
nextstate(:length)
|
75
78
|
else
|
76
79
|
return if @finished
|
80
|
+
|
77
81
|
nextstate(:data)
|
78
82
|
end
|
79
83
|
when :data
|
data/lib/httpx/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: httpx
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.4.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Tiago Cardoso
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2019-05-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: http-2
|
@@ -68,15 +68,14 @@ files:
|
|
68
68
|
- LICENSE.txt
|
69
69
|
- README.md
|
70
70
|
- lib/httpx.rb
|
71
|
+
- lib/httpx/adapters/faraday.rb
|
71
72
|
- lib/httpx/altsvc.rb
|
72
73
|
- lib/httpx/buffer.rb
|
73
74
|
- lib/httpx/callbacks.rb
|
74
75
|
- lib/httpx/chainable.rb
|
75
|
-
- lib/httpx/channel.rb
|
76
|
-
- lib/httpx/channel/http1.rb
|
77
|
-
- lib/httpx/channel/http2.rb
|
78
|
-
- lib/httpx/client.rb
|
79
76
|
- lib/httpx/connection.rb
|
77
|
+
- lib/httpx/connection/http1.rb
|
78
|
+
- lib/httpx/connection/http2.rb
|
80
79
|
- lib/httpx/errors.rb
|
81
80
|
- lib/httpx/extensions.rb
|
82
81
|
- lib/httpx/headers.rb
|
@@ -99,13 +98,16 @@ files:
|
|
99
98
|
- lib/httpx/plugins/follow_redirects.rb
|
100
99
|
- lib/httpx/plugins/h2c.rb
|
101
100
|
- lib/httpx/plugins/multipart.rb
|
101
|
+
- lib/httpx/plugins/persistent.rb
|
102
102
|
- lib/httpx/plugins/proxy.rb
|
103
103
|
- lib/httpx/plugins/proxy/http.rb
|
104
104
|
- lib/httpx/plugins/proxy/socks4.rb
|
105
105
|
- lib/httpx/plugins/proxy/socks5.rb
|
106
|
+
- lib/httpx/plugins/proxy/ssh.rb
|
106
107
|
- lib/httpx/plugins/push_promise.rb
|
107
108
|
- lib/httpx/plugins/retries.rb
|
108
109
|
- lib/httpx/plugins/stream.rb
|
110
|
+
- lib/httpx/pool.rb
|
109
111
|
- lib/httpx/registry.rb
|
110
112
|
- lib/httpx/request.rb
|
111
113
|
- lib/httpx/resolver.rb
|
@@ -116,6 +118,7 @@ files:
|
|
116
118
|
- lib/httpx/resolver/system.rb
|
117
119
|
- lib/httpx/response.rb
|
118
120
|
- lib/httpx/selector.rb
|
121
|
+
- lib/httpx/session.rb
|
119
122
|
- lib/httpx/timeout.rb
|
120
123
|
- lib/httpx/transcoder.rb
|
121
124
|
- lib/httpx/transcoder/body.rb
|
@@ -142,8 +145,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
142
145
|
- !ruby/object:Gem::Version
|
143
146
|
version: '0'
|
144
147
|
requirements: []
|
145
|
-
|
146
|
-
rubygems_version: 2.7.8
|
148
|
+
rubygems_version: 3.0.1
|
147
149
|
signing_key:
|
148
150
|
specification_version: 4
|
149
151
|
summary: HTTPX, to the future, and beyond
|