httpx 0.10.0 → 0.11.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +11 -3
- data/doc/release_notes/0_10_1.md +37 -0
- data/doc/release_notes/0_10_2.md +5 -0
- data/doc/release_notes/0_11_0.md +76 -0
- data/doc/release_notes/0_11_1.md +1 -0
- data/doc/release_notes/0_11_2.md +5 -0
- data/lib/httpx/adapters/datadog.rb +205 -0
- data/lib/httpx/adapters/faraday.rb +0 -2
- data/lib/httpx/adapters/webmock.rb +123 -0
- data/lib/httpx/chainable.rb +8 -7
- data/lib/httpx/connection.rb +4 -15
- data/lib/httpx/connection/http1.rb +14 -1
- data/lib/httpx/connection/http2.rb +15 -16
- data/lib/httpx/domain_name.rb +1 -3
- data/lib/httpx/errors.rb +3 -1
- data/lib/httpx/headers.rb +1 -0
- data/lib/httpx/io/ssl.rb +4 -8
- data/lib/httpx/io/udp.rb +4 -3
- data/lib/httpx/plugins/compression.rb +1 -1
- data/lib/httpx/plugins/cookies/set_cookie_parser.rb +1 -1
- data/lib/httpx/plugins/expect.rb +33 -8
- data/lib/httpx/plugins/multipart.rb +42 -23
- data/lib/httpx/plugins/multipart/encoder.rb +115 -0
- data/lib/httpx/plugins/multipart/mime_type_detector.rb +64 -0
- data/lib/httpx/plugins/multipart/part.rb +34 -0
- data/lib/httpx/plugins/proxy.rb +16 -2
- data/lib/httpx/plugins/proxy/socks4.rb +14 -16
- data/lib/httpx/plugins/proxy/socks5.rb +3 -2
- data/lib/httpx/plugins/push_promise.rb +2 -2
- data/lib/httpx/pool.rb +8 -14
- data/lib/httpx/request.rb +22 -12
- data/lib/httpx/resolver.rb +7 -6
- data/lib/httpx/resolver/https.rb +18 -23
- data/lib/httpx/resolver/native.rb +22 -19
- data/lib/httpx/resolver/resolver_mixin.rb +4 -2
- data/lib/httpx/resolver/system.rb +3 -3
- data/lib/httpx/selector.rb +9 -13
- data/lib/httpx/session.rb +24 -21
- data/lib/httpx/transcoder.rb +20 -0
- data/lib/httpx/transcoder/form.rb +9 -1
- data/lib/httpx/version.rb +1 -1
- data/sig/connection.rbs +84 -1
- data/sig/connection/http1.rbs +66 -0
- data/sig/connection/http2.rbs +73 -0
- data/sig/headers.rbs +3 -0
- data/sig/httpx.rbs +1 -0
- data/sig/options.rbs +3 -3
- data/sig/plugins/basic_authentication.rbs +1 -1
- data/sig/plugins/compression.rbs +1 -1
- data/sig/plugins/compression/brotli.rbs +1 -1
- data/sig/plugins/compression/deflate.rbs +1 -1
- data/sig/plugins/compression/gzip.rbs +1 -1
- data/sig/plugins/h2c.rbs +1 -1
- data/sig/plugins/multipart.rbs +29 -4
- data/sig/plugins/persistent.rbs +1 -1
- data/sig/plugins/proxy.rbs +2 -2
- data/sig/plugins/proxy/ssh.rbs +1 -1
- data/sig/plugins/rate_limiter.rbs +1 -1
- data/sig/pool.rbs +36 -2
- data/sig/request.rbs +2 -2
- data/sig/resolver.rbs +26 -0
- data/sig/resolver/https.rbs +51 -0
- data/sig/resolver/native.rbs +60 -0
- data/sig/resolver/resolver_mixin.rbs +27 -0
- data/sig/resolver/system.rbs +17 -0
- data/sig/response.rbs +2 -2
- data/sig/selector.rbs +20 -0
- data/sig/session.rbs +3 -3
- data/sig/transcoder.rbs +4 -2
- data/sig/transcoder/body.rbs +2 -0
- data/sig/transcoder/form.rbs +8 -2
- data/sig/transcoder/json.rbs +3 -1
- metadata +47 -48
- data/lib/httpx/resolver/options.rb +0 -25
- data/sig/missing.rbs +0 -12
- data/sig/test.rbs +0 -9
@@ -0,0 +1,115 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module HTTPX::Plugins
|
4
|
+
module Multipart
|
5
|
+
class Encoder
|
6
|
+
attr_reader :bytesize
|
7
|
+
|
8
|
+
def initialize(form)
|
9
|
+
@boundary = ("-" * 21) << SecureRandom.hex(21)
|
10
|
+
@part_index = 0
|
11
|
+
@buffer = "".b
|
12
|
+
|
13
|
+
@form = form
|
14
|
+
@parts = to_parts(form)
|
15
|
+
end
|
16
|
+
|
17
|
+
def content_type
|
18
|
+
"multipart/form-data; boundary=#{@boundary}"
|
19
|
+
end
|
20
|
+
|
21
|
+
def read(length = nil, outbuf = nil)
|
22
|
+
data = outbuf.clear.force_encoding(Encoding::BINARY) if outbuf
|
23
|
+
data ||= "".b
|
24
|
+
|
25
|
+
read_chunks(data, length)
|
26
|
+
|
27
|
+
data unless length && data.empty?
|
28
|
+
end
|
29
|
+
|
30
|
+
def rewind
|
31
|
+
form = @form.each_with_object([]) do |(key, val), aux|
|
32
|
+
v = case val
|
33
|
+
when File
|
34
|
+
val = val.reopen(val.path, File::RDONLY) if val.closed?
|
35
|
+
val.rewind
|
36
|
+
val
|
37
|
+
else
|
38
|
+
v
|
39
|
+
end
|
40
|
+
aux << [key, v]
|
41
|
+
end
|
42
|
+
@form = form
|
43
|
+
@parts = to_parts(form)
|
44
|
+
end
|
45
|
+
|
46
|
+
private
|
47
|
+
|
48
|
+
def to_parts(form)
|
49
|
+
@bytesize = 0
|
50
|
+
params = form.each_with_object([]) do |(key, val), aux|
|
51
|
+
Multipart.normalize_keys(key, val) do |k, v|
|
52
|
+
next if v.nil?
|
53
|
+
|
54
|
+
value, content_type, filename = Part.call(v)
|
55
|
+
|
56
|
+
header = header_part(k, content_type, filename)
|
57
|
+
@bytesize += header.size
|
58
|
+
aux << header
|
59
|
+
|
60
|
+
@bytesize += value.size
|
61
|
+
aux << value
|
62
|
+
|
63
|
+
delimiter = StringIO.new("\r\n")
|
64
|
+
@bytesize += delimiter.size
|
65
|
+
aux << delimiter
|
66
|
+
end
|
67
|
+
end
|
68
|
+
final_delimiter = StringIO.new("--#{@boundary}--\r\n")
|
69
|
+
@bytesize += final_delimiter.size
|
70
|
+
params << final_delimiter
|
71
|
+
|
72
|
+
params
|
73
|
+
end
|
74
|
+
|
75
|
+
def header_part(key, content_type, filename)
|
76
|
+
header = "--#{@boundary}\r\n".b
|
77
|
+
header << "Content-Disposition: form-data; name=#{key.inspect}".b
|
78
|
+
header << "; filename=#{filename.inspect}" if filename
|
79
|
+
header << "\r\nContent-Type: #{content_type}\r\n\r\n"
|
80
|
+
StringIO.new(header)
|
81
|
+
end
|
82
|
+
|
83
|
+
def read_chunks(buffer, length = nil)
|
84
|
+
while @part_index < @parts.size
|
85
|
+
chunk = read_from_part(length)
|
86
|
+
|
87
|
+
next unless chunk
|
88
|
+
|
89
|
+
buffer << chunk.force_encoding(Encoding::BINARY)
|
90
|
+
|
91
|
+
next unless length
|
92
|
+
|
93
|
+
length -= chunk.bytesize
|
94
|
+
|
95
|
+
break if length.zero?
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
# if there's a current part to read from, tries to read a chunk.
|
100
|
+
def read_from_part(max_length = nil)
|
101
|
+
part = @parts[@part_index]
|
102
|
+
|
103
|
+
chunk = part.read(max_length, @buffer)
|
104
|
+
|
105
|
+
return chunk if chunk && !chunk.empty?
|
106
|
+
|
107
|
+
part.close if part.respond_to?(:close)
|
108
|
+
|
109
|
+
@part_index += 1
|
110
|
+
|
111
|
+
nil
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|
@@ -0,0 +1,64 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module HTTPX
|
4
|
+
module Plugins::Multipart
|
5
|
+
module MimeTypeDetector
|
6
|
+
module_function
|
7
|
+
|
8
|
+
DEFAULT_MIMETYPE = "application/octet-stream"
|
9
|
+
|
10
|
+
# inspired by https://github.com/shrinerb/shrine/blob/master/lib/shrine/plugins/determine_mime_type.rb
|
11
|
+
if defined?(MIME::Types)
|
12
|
+
|
13
|
+
def call(_file, filename)
|
14
|
+
mime = MIME::Types.of(filename).first
|
15
|
+
mime.content_type if mime
|
16
|
+
end
|
17
|
+
|
18
|
+
elsif defined?(MimeMagic)
|
19
|
+
|
20
|
+
def call(file, *)
|
21
|
+
mime = MimeMagic.by_magic(file)
|
22
|
+
mime.type if mime
|
23
|
+
end
|
24
|
+
|
25
|
+
elsif system("which file", out: File::NULL)
|
26
|
+
require "open3"
|
27
|
+
|
28
|
+
def call(file, *)
|
29
|
+
return if file.eof? # file command returns "application/x-empty" for empty files
|
30
|
+
|
31
|
+
Open3.popen3(*%w[file --mime-type --brief -]) do |stdin, stdout, stderr, thread|
|
32
|
+
begin
|
33
|
+
::IO.copy_stream(file, stdin.binmode)
|
34
|
+
rescue Errno::EPIPE
|
35
|
+
end
|
36
|
+
file.rewind
|
37
|
+
stdin.close
|
38
|
+
|
39
|
+
status = thread.value
|
40
|
+
|
41
|
+
# call to file command failed
|
42
|
+
if status.nil? || !status.success?
|
43
|
+
$stderr.print(stderr.read)
|
44
|
+
else
|
45
|
+
|
46
|
+
output = stdout.read.strip
|
47
|
+
|
48
|
+
if output.include?("cannot open")
|
49
|
+
$stderr.print(output)
|
50
|
+
else
|
51
|
+
output
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
else
|
58
|
+
|
59
|
+
def call(*); end
|
60
|
+
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module HTTPX
|
4
|
+
module Plugins::Multipart
|
5
|
+
module Part
|
6
|
+
module_function
|
7
|
+
|
8
|
+
def call(value)
|
9
|
+
# take out specialized objects of the way
|
10
|
+
if value.respond_to?(:filename) && value.respond_to?(:content_type) && value.respond_to?(:read)
|
11
|
+
return [value, value.content_type, value.filename]
|
12
|
+
end
|
13
|
+
|
14
|
+
content_type = filename = nil
|
15
|
+
|
16
|
+
if value.is_a?(Hash)
|
17
|
+
content_type = value[:content_type]
|
18
|
+
filename = value[:filename]
|
19
|
+
value = value[:body]
|
20
|
+
end
|
21
|
+
|
22
|
+
value = value.open(:binmode => true) if value.is_a?(Pathname)
|
23
|
+
|
24
|
+
if value.is_a?(File)
|
25
|
+
filename ||= File.basename(value.path)
|
26
|
+
content_type ||= MimeTypeDetector.call(value, filename) || "application/octet-stream"
|
27
|
+
[value, content_type, filename]
|
28
|
+
else
|
29
|
+
[StringIO.new(value.to_s), "text/plain"]
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
data/lib/httpx/plugins/proxy.rb
CHANGED
@@ -121,8 +121,7 @@ module HTTPX
|
|
121
121
|
def fetch_response(request, connections, options)
|
122
122
|
response = super
|
123
123
|
if response.is_a?(ErrorResponse) &&
|
124
|
-
|
125
|
-
PROXY_ERRORS.any? { |ex| response.error.is_a?(ex) } && !@_proxy_uris.empty?
|
124
|
+
__proxy_error?(response) && !@_proxy_uris.empty?
|
126
125
|
@_proxy_uris.shift
|
127
126
|
log { "failed connecting to proxy, trying next..." }
|
128
127
|
request.transition(:idle)
|
@@ -139,6 +138,21 @@ module HTTPX
|
|
139
138
|
|
140
139
|
super
|
141
140
|
end
|
141
|
+
|
142
|
+
def __proxy_error?(response)
|
143
|
+
error = response.error
|
144
|
+
case error
|
145
|
+
when ResolveError
|
146
|
+
# failed resolving proxy domain
|
147
|
+
proxy_uri = error.connection.options.proxy.uri
|
148
|
+
proxy_uri.to_s == @_proxy_uris.first
|
149
|
+
when *PROXY_ERRORS
|
150
|
+
# timeout errors connecting to proxy
|
151
|
+
true
|
152
|
+
else
|
153
|
+
false
|
154
|
+
end
|
155
|
+
end
|
142
156
|
end
|
143
157
|
|
144
158
|
module ConnectionMethods
|
@@ -10,7 +10,7 @@ module HTTPX
|
|
10
10
|
module Socks4
|
11
11
|
VERSION = 4
|
12
12
|
CONNECT = 1
|
13
|
-
GRANTED =
|
13
|
+
GRANTED = 0x5A
|
14
14
|
PROTOCOLS = %w[socks4 socks4a].freeze
|
15
15
|
|
16
16
|
Error = Socks4Error
|
@@ -91,27 +91,25 @@ module HTTPX
|
|
91
91
|
end
|
92
92
|
|
93
93
|
module Packet
|
94
|
-
using(RegexpExtensions) unless Regexp.method_defined?(:match?)
|
95
|
-
|
96
94
|
module_function
|
97
95
|
|
98
96
|
def connect(parameters, uri)
|
99
97
|
packet = [VERSION, CONNECT, uri.port].pack("CCn")
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
else
|
111
|
-
packet << "\x0\x0\x0\x1" << "\x7\x0" << uri.host
|
98
|
+
|
99
|
+
case parameters.uri.scheme
|
100
|
+
when "socks4"
|
101
|
+
socks_host = uri.host
|
102
|
+
begin
|
103
|
+
ip = IPAddr.new(socks_host)
|
104
|
+
packet << ip.hton
|
105
|
+
rescue IPAddr::InvalidAddressError
|
106
|
+
socks_host = Resolv.getaddress(socks_host)
|
107
|
+
retry
|
112
108
|
end
|
109
|
+
packet << [parameters.username].pack("Z*")
|
110
|
+
when "socks4a"
|
111
|
+
packet << "\x0\x0\x0\x1" << [parameters.username].pack("Z*") << uri.host << "\x0"
|
113
112
|
end
|
114
|
-
packet << [parameters.username].pack("Z*")
|
115
113
|
packet
|
116
114
|
end
|
117
115
|
end
|
@@ -159,9 +159,10 @@ module HTTPX
|
|
159
159
|
packet = [VERSION, CONNECT, 0].pack("C*")
|
160
160
|
begin
|
161
161
|
ip = IPAddr.new(uri.host)
|
162
|
-
raise Error, "Socks4 connection to #{ip} not supported" unless ip.ipv4?
|
163
162
|
|
164
|
-
|
163
|
+
ipcode = ip.ipv6? ? IPV6 : IPV4
|
164
|
+
|
165
|
+
packet << [ipcode].pack("C") << ip.hton
|
165
166
|
rescue IPAddr::InvalidAddressError
|
166
167
|
packet << [DOMAIN, uri.host.bytesize, uri.host].pack("CCA*")
|
167
168
|
end
|
@@ -70,8 +70,8 @@ module HTTPX
|
|
70
70
|
request.transition(:done)
|
71
71
|
response = request.response
|
72
72
|
response.mark_as_pushed!
|
73
|
-
stream.on(:data, &parser.method(:on_stream_data).curry[stream, request])
|
74
|
-
stream.on(:close, &parser.method(:on_stream_close).curry[stream, request])
|
73
|
+
stream.on(:data, &parser.method(:on_stream_data).curry(3)[stream, request])
|
74
|
+
stream.on(:close, &parser.method(:on_stream_close).curry(3)[stream, request])
|
75
75
|
end
|
76
76
|
end
|
77
77
|
end
|
data/lib/httpx/pool.rb
CHANGED
@@ -108,10 +108,10 @@ module HTTPX
|
|
108
108
|
end
|
109
109
|
end
|
110
110
|
|
111
|
-
def on_resolver_error(
|
112
|
-
|
111
|
+
def on_resolver_error(connection, error)
|
112
|
+
connection.emit(:error, error)
|
113
113
|
# must remove connection by hand, hasn't been started yet
|
114
|
-
unregister_connection(
|
114
|
+
unregister_connection(connection)
|
115
115
|
end
|
116
116
|
|
117
117
|
def on_resolver_close(resolver)
|
@@ -144,12 +144,12 @@ module HTTPX
|
|
144
144
|
@connected_connections -= 1
|
145
145
|
end
|
146
146
|
|
147
|
-
def coalesce_connections(
|
148
|
-
if
|
149
|
-
|
150
|
-
@connections.delete(
|
147
|
+
def coalesce_connections(conn1, conn2)
|
148
|
+
if conn1.coalescable?(conn2)
|
149
|
+
conn1.merge(conn2)
|
150
|
+
@connections.delete(conn2)
|
151
151
|
else
|
152
|
-
register_connection(
|
152
|
+
register_connection(conn2)
|
153
153
|
end
|
154
154
|
end
|
155
155
|
|
@@ -168,12 +168,6 @@ module HTTPX
|
|
168
168
|
resolver.on(:error, &method(:on_resolver_error))
|
169
169
|
resolver.on(:close) { on_resolver_close(resolver) }
|
170
170
|
resolver
|
171
|
-
rescue ArgumentError
|
172
|
-
# this block is here because of an error which happens on CI from time to time
|
173
|
-
warn "tried resolver: #{resolver_type}"
|
174
|
-
warn "initialize: #{resolver_type.instance_method(:initialize).source_location}"
|
175
|
-
warn "new: #{resolver_type.method(:new).source_location}"
|
176
|
-
raise
|
177
171
|
end
|
178
172
|
end
|
179
173
|
end
|
data/lib/httpx/request.rb
CHANGED
@@ -81,6 +81,10 @@ module HTTPX
|
|
81
81
|
def response=(response)
|
82
82
|
return unless response
|
83
83
|
|
84
|
+
if response.status == 100
|
85
|
+
@informational_status = response.status
|
86
|
+
return
|
87
|
+
end
|
84
88
|
@response = response
|
85
89
|
end
|
86
90
|
|
@@ -106,7 +110,7 @@ module HTTPX
|
|
106
110
|
|
107
111
|
query = []
|
108
112
|
if (q = @options.params)
|
109
|
-
query <<
|
113
|
+
query << Transcoder.registry("form").encode(q)
|
110
114
|
end
|
111
115
|
query << @uri.query if @uri.query
|
112
116
|
@query = query.join("&")
|
@@ -170,6 +174,12 @@ module HTTPX
|
|
170
174
|
end
|
171
175
|
end
|
172
176
|
|
177
|
+
def rewind
|
178
|
+
return if empty?
|
179
|
+
|
180
|
+
@body.rewind if @body.respond_to?(:rewind)
|
181
|
+
end
|
182
|
+
|
173
183
|
def empty?
|
174
184
|
return true if @body.nil?
|
175
185
|
return false if chunked?
|
@@ -212,6 +222,7 @@ module HTTPX
|
|
212
222
|
def transition(nextstate)
|
213
223
|
case nextstate
|
214
224
|
when :idle
|
225
|
+
@body.rewind
|
215
226
|
@response = nil
|
216
227
|
@drainer = nil
|
217
228
|
when :headers
|
@@ -221,15 +232,15 @@ module HTTPX
|
|
221
232
|
@state == :expect
|
222
233
|
|
223
234
|
if @headers.key?("expect")
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
235
|
+
if @informational_status && @informational_status == 100
|
236
|
+
# check for 100 Continue response, and deallocate the var
|
237
|
+
# if @informational_status == 100
|
238
|
+
# @response = nil
|
239
|
+
# end
|
240
|
+
else
|
241
|
+
return if @state == :expect # do not re-set it
|
242
|
+
|
243
|
+
nextstate = :expect
|
233
244
|
end
|
234
245
|
end
|
235
246
|
when :done
|
@@ -241,8 +252,7 @@ module HTTPX
|
|
241
252
|
end
|
242
253
|
|
243
254
|
def expects?
|
244
|
-
@headers["expect"] == "100-continue" &&
|
245
|
-
@response && @response.status == 100
|
255
|
+
@headers["expect"] == "100-continue" && @informational_status == 100 && !@response
|
246
256
|
end
|
247
257
|
|
248
258
|
class ProcIO
|
data/lib/httpx/resolver.rb
CHANGED
@@ -1,15 +1,18 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
require "resolv"
|
4
|
-
require "httpx/resolver/resolver_mixin"
|
5
|
-
require "httpx/resolver/system"
|
6
|
-
require "httpx/resolver/native"
|
7
|
-
require "httpx/resolver/https"
|
8
4
|
|
9
5
|
module HTTPX
|
10
6
|
module Resolver
|
11
7
|
extend Registry
|
12
8
|
|
9
|
+
RESOLVE_TIMEOUT = 5
|
10
|
+
|
11
|
+
require "httpx/resolver/resolver_mixin"
|
12
|
+
require "httpx/resolver/system"
|
13
|
+
require "httpx/resolver/native"
|
14
|
+
require "httpx/resolver/https"
|
15
|
+
|
13
16
|
register :system, System
|
14
17
|
register :native, Native
|
15
18
|
register :https, HTTPS
|
@@ -101,5 +104,3 @@ module HTTPX
|
|
101
104
|
end
|
102
105
|
end
|
103
106
|
end
|
104
|
-
|
105
|
-
require "httpx/resolver/options"
|