httpx 0.6.7 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +7 -5
- data/doc/release_notes/0_0_1.md +7 -0
- data/doc/release_notes/0_0_2.md +9 -0
- data/doc/release_notes/0_0_3.md +9 -0
- data/doc/release_notes/0_0_4.md +7 -0
- data/doc/release_notes/0_0_5.md +5 -0
- data/doc/release_notes/0_1_0.md +9 -0
- data/doc/release_notes/0_2_0.md +5 -0
- data/doc/release_notes/0_2_1.md +16 -0
- data/doc/release_notes/0_3_0.md +12 -0
- data/doc/release_notes/0_3_1.md +6 -0
- data/doc/release_notes/0_4_0.md +51 -0
- data/doc/release_notes/0_4_1.md +3 -0
- data/doc/release_notes/0_5_0.md +15 -0
- data/doc/release_notes/0_5_1.md +14 -0
- data/doc/release_notes/0_6_0.md +5 -0
- data/doc/release_notes/0_6_1.md +6 -0
- data/doc/release_notes/0_6_2.md +6 -0
- data/doc/release_notes/0_6_3.md +13 -0
- data/doc/release_notes/0_6_4.md +21 -0
- data/doc/release_notes/0_6_5.md +22 -0
- data/doc/release_notes/0_6_6.md +19 -0
- data/doc/release_notes/0_6_7.md +5 -0
- data/doc/release_notes/0_7_0.md +46 -0
- data/doc/release_notes/0_8_0.md +27 -0
- data/doc/release_notes/0_8_1.md +8 -0
- data/doc/release_notes/0_8_2.md +7 -0
- data/doc/release_notes/0_9_0.md +38 -0
- data/lib/httpx/adapters/faraday.rb +2 -2
- data/lib/httpx/altsvc.rb +18 -2
- data/lib/httpx/chainable.rb +27 -9
- data/lib/httpx/connection.rb +215 -65
- data/lib/httpx/connection/http1.rb +54 -18
- data/lib/httpx/connection/http2.rb +100 -37
- data/lib/httpx/extensions.rb +2 -2
- data/lib/httpx/headers.rb +2 -2
- data/lib/httpx/io/ssl.rb +11 -3
- data/lib/httpx/io/tcp.rb +12 -2
- data/lib/httpx/loggable.rb +6 -6
- data/lib/httpx/options.rb +43 -28
- data/lib/httpx/plugins/authentication.rb +1 -1
- data/lib/httpx/plugins/compression.rb +28 -8
- data/lib/httpx/plugins/compression/gzip.rb +22 -12
- data/lib/httpx/plugins/cookies.rb +12 -8
- data/lib/httpx/plugins/digest_authentication.rb +2 -0
- data/lib/httpx/plugins/expect.rb +79 -0
- data/lib/httpx/plugins/follow_redirects.rb +1 -2
- data/lib/httpx/plugins/h2c.rb +0 -1
- data/lib/httpx/plugins/proxy.rb +23 -20
- data/lib/httpx/plugins/proxy/http.rb +9 -6
- data/lib/httpx/plugins/proxy/socks4.rb +1 -1
- data/lib/httpx/plugins/proxy/socks5.rb +5 -1
- data/lib/httpx/plugins/proxy/ssh.rb +0 -4
- data/lib/httpx/plugins/push_promise.rb +2 -2
- data/lib/httpx/plugins/retries.rb +32 -29
- data/lib/httpx/pool.rb +15 -10
- data/lib/httpx/registry.rb +2 -1
- data/lib/httpx/request.rb +8 -6
- data/lib/httpx/resolver.rb +7 -8
- data/lib/httpx/resolver/https.rb +15 -3
- data/lib/httpx/resolver/native.rb +22 -32
- data/lib/httpx/resolver/options.rb +2 -2
- data/lib/httpx/resolver/resolver_mixin.rb +1 -1
- data/lib/httpx/response.rb +17 -3
- data/lib/httpx/selector.rb +96 -95
- data/lib/httpx/session.rb +33 -34
- data/lib/httpx/timeout.rb +7 -1
- data/lib/httpx/version.rb +1 -1
- metadata +77 -20
data/lib/httpx/headers.rb
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
|
3
3
|
module HTTPX
|
4
4
|
class Headers
|
5
|
-
EMPTY = [].freeze
|
5
|
+
EMPTY = [].freeze
|
6
6
|
|
7
7
|
class << self
|
8
8
|
def new(headers = nil)
|
@@ -67,7 +67,7 @@ module HTTPX
|
|
67
67
|
#
|
68
68
|
def [](field)
|
69
69
|
a = @headers[downcased(field)] || return
|
70
|
-
a.join(",")
|
70
|
+
a.join(", ")
|
71
71
|
end
|
72
72
|
|
73
73
|
# sets +value+ (if not nil) as single value for the +field+ header.
|
data/lib/httpx/io/ssl.rb
CHANGED
@@ -20,6 +20,10 @@ module HTTPX
|
|
20
20
|
@state = :negotiated if @keep_open
|
21
21
|
end
|
22
22
|
|
23
|
+
def interests
|
24
|
+
@interests || super
|
25
|
+
end
|
26
|
+
|
23
27
|
def protocol
|
24
28
|
@io.alpn_protocol || super
|
25
29
|
rescue StandardError
|
@@ -62,15 +66,18 @@ module HTTPX
|
|
62
66
|
@io.connect_nonblock
|
63
67
|
@io.post_connection_check(@hostname) if @ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
|
64
68
|
transition(:negotiated)
|
65
|
-
rescue ::IO::WaitReadable
|
66
|
-
|
69
|
+
rescue ::IO::WaitReadable
|
70
|
+
@interests = :r
|
71
|
+
rescue ::IO::WaitWritable
|
72
|
+
@interests = :w
|
67
73
|
end
|
68
74
|
|
69
75
|
# :nocov:
|
70
76
|
if RUBY_VERSION < "2.3"
|
71
|
-
def read(
|
77
|
+
def read(_, buffer)
|
72
78
|
super
|
73
79
|
rescue ::IO::WaitWritable
|
80
|
+
buffer.clear
|
74
81
|
0
|
75
82
|
end
|
76
83
|
|
@@ -86,6 +93,7 @@ module HTTPX
|
|
86
93
|
buffer.bytesize
|
87
94
|
rescue ::IO::WaitReadable,
|
88
95
|
::IO::WaitWritable
|
96
|
+
buffer.clear
|
89
97
|
0
|
90
98
|
rescue EOFError
|
91
99
|
nil
|
data/lib/httpx/io/tcp.rb
CHANGED
@@ -11,6 +11,8 @@ module HTTPX
|
|
11
11
|
|
12
12
|
attr_reader :addresses
|
13
13
|
|
14
|
+
attr_reader :state
|
15
|
+
|
14
16
|
alias_method :host, :ip
|
15
17
|
|
16
18
|
def initialize(origin, addresses, options)
|
@@ -41,6 +43,10 @@ module HTTPX
|
|
41
43
|
@io ||= build_socket
|
42
44
|
end
|
43
45
|
|
46
|
+
def interests
|
47
|
+
:w
|
48
|
+
end
|
49
|
+
|
44
50
|
def to_io
|
45
51
|
@io.to_io
|
46
52
|
end
|
@@ -67,7 +73,7 @@ module HTTPX
|
|
67
73
|
@ip_index -= 1
|
68
74
|
retry
|
69
75
|
rescue Errno::ETIMEDOUT => e
|
70
|
-
raise
|
76
|
+
raise ConnectTimeoutError, e.message if @ip_index <= 0
|
71
77
|
|
72
78
|
@ip_index -= 1
|
73
79
|
retry
|
@@ -82,6 +88,7 @@ module HTTPX
|
|
82
88
|
@io.read_nonblock(size, buffer)
|
83
89
|
buffer.bytesize
|
84
90
|
rescue ::IO::WaitReadable
|
91
|
+
buffer.clear
|
85
92
|
0
|
86
93
|
rescue EOFError
|
87
94
|
nil
|
@@ -100,7 +107,10 @@ module HTTPX
|
|
100
107
|
else
|
101
108
|
def read(size, buffer)
|
102
109
|
ret = @io.read_nonblock(size, buffer, exception: false)
|
103
|
-
|
110
|
+
if ret == :wait_readable
|
111
|
+
buffer.clear
|
112
|
+
return 0
|
113
|
+
end
|
104
114
|
return if ret.nil?
|
105
115
|
|
106
116
|
buffer.bytesize
|
data/lib/httpx/loggable.rb
CHANGED
@@ -13,35 +13,35 @@ module HTTPX
|
|
13
13
|
white: 37,
|
14
14
|
}.freeze
|
15
15
|
|
16
|
-
def log(level: @options.debug_level,
|
16
|
+
def log(level: @options.debug_level, color: nil, &msg)
|
17
17
|
return unless @options.debug
|
18
18
|
return unless @options.debug_level >= level
|
19
19
|
|
20
20
|
debug_stream = @options.debug
|
21
21
|
|
22
|
-
message = (+
|
22
|
+
message = (+"" << msg.call << "\n")
|
23
23
|
message = "\e[#{COLORS[color]}m#{message}\e[0m" if debug_stream.respond_to?(:isatty) && debug_stream.isatty
|
24
24
|
debug_stream << message
|
25
25
|
end
|
26
26
|
|
27
27
|
if !Exception.instance_methods.include?(:full_message)
|
28
28
|
|
29
|
-
def log_exception(ex, level: @options.debug_level,
|
29
|
+
def log_exception(ex, level: @options.debug_level, color: nil)
|
30
30
|
return unless @options.debug
|
31
31
|
return unless @options.debug_level >= level
|
32
32
|
|
33
33
|
message = +"#{ex.message} (#{ex.class})"
|
34
34
|
message << "\n" << ex.backtrace.join("\n") unless ex.backtrace.nil?
|
35
|
-
log(level: level,
|
35
|
+
log(level: level, color: color) { message }
|
36
36
|
end
|
37
37
|
|
38
38
|
else
|
39
39
|
|
40
|
-
def log_exception(ex, level: @options.debug_level,
|
40
|
+
def log_exception(ex, level: @options.debug_level, color: nil)
|
41
41
|
return unless @options.debug
|
42
42
|
return unless @options.debug_level >= level
|
43
43
|
|
44
|
-
log(level: level,
|
44
|
+
log(level: level, color: color) { ex.full_message }
|
45
45
|
end
|
46
46
|
|
47
47
|
end
|
data/lib/httpx/options.rb
CHANGED
@@ -2,7 +2,6 @@
|
|
2
2
|
|
3
3
|
module HTTPX
|
4
4
|
class Options
|
5
|
-
MAX_CONCURRENT_REQUESTS = 100
|
6
5
|
WINDOW_SIZE = 1 << 14 # 16K
|
7
6
|
MAX_BODY_THRESHOLD_SIZE = (1 << 10) * 112 # 112K
|
8
7
|
|
@@ -26,16 +25,28 @@ module HTTPX
|
|
26
25
|
|
27
26
|
def def_option(name, &interpreter)
|
28
27
|
defined_options << name.to_sym
|
29
|
-
interpreter ||= ->(v) { v }
|
30
28
|
|
31
|
-
|
32
|
-
|
29
|
+
attr_reader name
|
30
|
+
|
31
|
+
if interpreter
|
32
|
+
define_method(:"#{name}=") do |value|
|
33
|
+
return if value.nil?
|
34
|
+
|
35
|
+
instance_variable_set(:"@#{name}", instance_exec(value, &interpreter))
|
36
|
+
end
|
37
|
+
|
38
|
+
define_method(:"with_#{name}") do |value|
|
39
|
+
merge(name => instance_exec(value, &interpreter))
|
40
|
+
end
|
41
|
+
else
|
42
|
+
attr_writer name
|
33
43
|
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
other
|
44
|
+
define_method(:"with_#{name}") do |value|
|
45
|
+
merge(name => value)
|
46
|
+
end
|
38
47
|
end
|
48
|
+
|
49
|
+
protected :"#{name}="
|
39
50
|
end
|
40
51
|
end
|
41
52
|
|
@@ -48,7 +59,6 @@ module HTTPX
|
|
48
59
|
:fallback_protocol => "http/1.1",
|
49
60
|
:timeout => Timeout.new,
|
50
61
|
:headers => {},
|
51
|
-
:max_concurrent_requests => MAX_CONCURRENT_REQUESTS,
|
52
62
|
:window_size => WINDOW_SIZE,
|
53
63
|
:body_threshold_size => MAX_BODY_THRESHOLD_SIZE,
|
54
64
|
:request_class => Class.new(Request),
|
@@ -65,43 +75,52 @@ module HTTPX
|
|
65
75
|
}
|
66
76
|
|
67
77
|
defaults.merge!(options)
|
68
|
-
defaults
|
69
|
-
|
78
|
+
defaults.each do |(k, v)|
|
79
|
+
__send__(:"#{k}=", v)
|
80
|
+
end
|
70
81
|
end
|
71
82
|
|
72
83
|
def_option(:headers) do |headers|
|
73
|
-
self.headers
|
84
|
+
if self.headers
|
85
|
+
self.headers.merge(headers)
|
86
|
+
else
|
87
|
+
Headers.new(headers)
|
88
|
+
end
|
74
89
|
end
|
75
90
|
|
76
91
|
def_option(:timeout) do |opts|
|
77
|
-
|
92
|
+
Timeout.new(opts)
|
78
93
|
end
|
79
94
|
|
80
95
|
def_option(:max_concurrent_requests) do |num|
|
81
|
-
|
82
|
-
raise Error, ":max_concurrent_requests must be positive" unless max.positive?
|
96
|
+
raise Error, ":max_concurrent_requests must be positive" unless num.positive?
|
83
97
|
|
84
|
-
|
98
|
+
num
|
99
|
+
end
|
100
|
+
|
101
|
+
def_option(:max_requests) do |num|
|
102
|
+
raise Error, ":max_requests must be positive" unless num.positive?
|
103
|
+
|
104
|
+
num
|
85
105
|
end
|
86
106
|
|
87
107
|
def_option(:window_size) do |num|
|
88
|
-
|
108
|
+
Integer(num)
|
89
109
|
end
|
90
110
|
|
91
111
|
def_option(:body_threshold_size) do |num|
|
92
|
-
|
112
|
+
Integer(num)
|
93
113
|
end
|
94
114
|
|
95
115
|
def_option(:transport) do |tr|
|
96
116
|
transport = tr.to_s
|
97
117
|
raise Error, "#{transport} is an unsupported transport type" unless IO.registry.key?(transport)
|
98
118
|
|
99
|
-
|
119
|
+
transport
|
100
120
|
end
|
101
121
|
|
102
122
|
%w[
|
103
|
-
params form json body
|
104
|
-
follow ssl http2_settings
|
123
|
+
params form json body ssl http2_settings
|
105
124
|
request_class response_class headers_class request_body_class response_body_class connection_class
|
106
125
|
io fallback_protocol debug debug_level transport_options resolver_class resolver_options
|
107
126
|
persistent
|
@@ -127,8 +146,10 @@ module HTTPX
|
|
127
146
|
end
|
128
147
|
|
129
148
|
def merge(other)
|
130
|
-
h1 = to_hash
|
131
149
|
h2 = other.to_hash
|
150
|
+
return self if h2.empty?
|
151
|
+
|
152
|
+
h1 = to_hash
|
132
153
|
|
133
154
|
merged = h1.merge(h2) do |k, v1, v2|
|
134
155
|
case k
|
@@ -172,11 +193,5 @@ module HTTPX
|
|
172
193
|
response_body_class.freeze
|
173
194
|
connection_class.freeze
|
174
195
|
end
|
175
|
-
|
176
|
-
protected
|
177
|
-
|
178
|
-
def []=(option, val)
|
179
|
-
send(:"#{option}=", val)
|
180
|
-
end
|
181
196
|
end
|
182
197
|
end
|
@@ -14,13 +14,23 @@ module HTTPX
|
|
14
14
|
#
|
15
15
|
module Compression
|
16
16
|
extend Registry
|
17
|
-
def self.load_dependencies(klass)
|
18
|
-
klass.plugin(:"compression/gzip")
|
19
|
-
klass.plugin(:"compression/deflate")
|
20
|
-
end
|
21
17
|
|
22
|
-
|
23
|
-
|
18
|
+
class << self
|
19
|
+
def load_dependencies(klass)
|
20
|
+
klass.plugin(:"compression/gzip")
|
21
|
+
klass.plugin(:"compression/deflate")
|
22
|
+
end
|
23
|
+
|
24
|
+
def extra_options(options)
|
25
|
+
Class.new(options.class) do
|
26
|
+
def_option(:compression_threshold_size) do |bytes|
|
27
|
+
bytes = Integer(bytes)
|
28
|
+
raise Error, ":expect_threshold_size must be positive" unless bytes.positive?
|
29
|
+
|
30
|
+
bytes
|
31
|
+
end
|
32
|
+
end.new(options).merge(headers: { "accept-encoding" => Compression.registry.keys })
|
33
|
+
end
|
24
34
|
end
|
25
35
|
|
26
36
|
module RequestMethods
|
@@ -32,11 +42,19 @@ module HTTPX
|
|
32
42
|
end
|
33
43
|
|
34
44
|
module RequestBodyMethods
|
35
|
-
def initialize(
|
45
|
+
def initialize(*, options)
|
36
46
|
super
|
37
47
|
return if @body.nil?
|
38
48
|
|
49
|
+
if (threshold = options.compression_threshold_size)
|
50
|
+
unless unbounded_body?
|
51
|
+
return if @body.bytesize < threshold
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
39
55
|
@headers.get("content-encoding").each do |encoding|
|
56
|
+
next if encoding == "identity"
|
57
|
+
|
40
58
|
@body = Encoder.new(@body, Compression.registry(encoding).encoder)
|
41
59
|
end
|
42
60
|
@headers["content-length"] = @body.bytesize unless chunked?
|
@@ -54,6 +72,8 @@ module HTTPX
|
|
54
72
|
return unless @headers.key?("content-encoding")
|
55
73
|
|
56
74
|
@_decoders = @headers.get("content-encoding").map do |encoding|
|
75
|
+
next if encoding == "identity"
|
76
|
+
|
57
77
|
decoder = Compression.registry(encoding).decoder
|
58
78
|
# do not uncompress if there is no decoder available. In fact, we can't reliably
|
59
79
|
# continue decompressing beyond that, so ignore.
|
@@ -61,7 +81,7 @@ module HTTPX
|
|
61
81
|
|
62
82
|
@encodings << encoding
|
63
83
|
decoder
|
64
|
-
end
|
84
|
+
end.compact
|
65
85
|
|
66
86
|
# remove encodings that we are able to decode
|
67
87
|
@headers["content-encoding"] = @headers.get("content-encoding") - @encodings
|
@@ -15,31 +15,41 @@ module HTTPX
|
|
15
15
|
end
|
16
16
|
|
17
17
|
class Encoder
|
18
|
+
def initialize
|
19
|
+
@compressed_chunk = "".b
|
20
|
+
end
|
21
|
+
|
18
22
|
def deflate(raw, buffer, chunk_size:)
|
19
23
|
gzip = Zlib::GzipWriter.new(self)
|
20
24
|
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
25
|
+
begin
|
26
|
+
while (chunk = raw.read(chunk_size))
|
27
|
+
gzip.write(chunk)
|
28
|
+
gzip.flush
|
29
|
+
compressed = compressed_chunk
|
30
|
+
buffer << compressed
|
31
|
+
yield compressed if block_given?
|
32
|
+
end
|
33
|
+
ensure
|
34
|
+
gzip.close
|
27
35
|
end
|
28
|
-
|
29
|
-
|
36
|
+
|
37
|
+
return unless (compressed = compressed_chunk)
|
38
|
+
|
39
|
+
buffer << compressed
|
40
|
+
yield compressed if block_given?
|
30
41
|
end
|
31
42
|
|
32
43
|
private
|
33
44
|
|
34
45
|
def write(chunk)
|
35
|
-
@compressed_chunk
|
46
|
+
@compressed_chunk << chunk
|
36
47
|
end
|
37
48
|
|
38
49
|
def compressed_chunk
|
39
|
-
|
40
|
-
compressed
|
50
|
+
@compressed_chunk.dup
|
41
51
|
ensure
|
42
|
-
@compressed_chunk
|
52
|
+
@compressed_chunk.clear
|
43
53
|
end
|
44
54
|
end
|
45
55
|
|
@@ -17,14 +17,22 @@ module HTTPX
|
|
17
17
|
def self.extra_options(options)
|
18
18
|
Class.new(options.class) do
|
19
19
|
def_option(:cookies) do |cookies|
|
20
|
-
|
21
|
-
|
22
|
-
|
20
|
+
if cookies.is_a?(Store)
|
21
|
+
cookies
|
22
|
+
else
|
23
|
+
Store.new(cookies)
|
24
|
+
end
|
23
25
|
end
|
24
26
|
end.new(options)
|
25
27
|
end
|
26
28
|
|
27
29
|
class Store
|
30
|
+
def self.new(cookies = nil)
|
31
|
+
return cookies if cookies.is_a?(self)
|
32
|
+
|
33
|
+
super
|
34
|
+
end
|
35
|
+
|
28
36
|
def initialize(cookies = nil)
|
29
37
|
@store = Hash.new { |hash, origin| hash[origin] = HTTP::CookieJar.new }
|
30
38
|
return unless cookies
|
@@ -74,10 +82,6 @@ module HTTPX
|
|
74
82
|
super({ cookies: Store.new }.merge(options), &blk)
|
75
83
|
end
|
76
84
|
|
77
|
-
def with_cookies(cookies)
|
78
|
-
branch(default_options.with_cookies(cookies))
|
79
|
-
end
|
80
|
-
|
81
85
|
def wrap
|
82
86
|
return super unless block_given?
|
83
87
|
|
@@ -86,7 +90,7 @@ module HTTPX
|
|
86
90
|
begin
|
87
91
|
yield session
|
88
92
|
ensure
|
89
|
-
@options = @options.
|
93
|
+
@options = @options.with(cookies: old_cookies_store)
|
90
94
|
end
|
91
95
|
end
|
92
96
|
end
|