httpx 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/LICENSE.txt +48 -0
- data/README.md +2 -0
- data/doc/release_notes/0_10_0.md +66 -0
- data/lib/httpx.rb +2 -0
- data/lib/httpx/adapters/faraday.rb +1 -1
- data/lib/httpx/chainable.rb +2 -2
- data/lib/httpx/connection.rb +3 -9
- data/lib/httpx/connection/http1.rb +1 -1
- data/lib/httpx/domain_name.rb +440 -0
- data/lib/httpx/errors.rb +1 -0
- data/lib/httpx/extensions.rb +21 -1
- data/lib/httpx/io/ssl.rb +0 -1
- data/lib/httpx/io/tcp.rb +6 -5
- data/lib/httpx/io/udp.rb +4 -1
- data/lib/httpx/options.rb +2 -0
- data/lib/httpx/parser/http1.rb +14 -17
- data/lib/httpx/plugins/compression.rb +28 -63
- data/lib/httpx/plugins/compression/brotli.rb +10 -14
- data/lib/httpx/plugins/compression/deflate.rb +7 -6
- data/lib/httpx/plugins/compression/gzip.rb +23 -5
- data/lib/httpx/plugins/cookies.rb +21 -60
- data/lib/httpx/plugins/cookies/cookie.rb +173 -0
- data/lib/httpx/plugins/cookies/jar.rb +74 -0
- data/lib/httpx/plugins/cookies/set_cookie_parser.rb +142 -0
- data/lib/httpx/plugins/expect.rb +3 -5
- data/lib/httpx/plugins/follow_redirects.rb +20 -2
- data/lib/httpx/plugins/h2c.rb +1 -1
- data/lib/httpx/plugins/multipart.rb +0 -8
- data/lib/httpx/plugins/persistent.rb +6 -1
- data/lib/httpx/plugins/proxy/socks4.rb +3 -1
- data/lib/httpx/plugins/rate_limiter.rb +51 -0
- data/lib/httpx/plugins/retries.rb +3 -2
- data/lib/httpx/plugins/stream.rb +109 -13
- data/lib/httpx/pool.rb +6 -6
- data/lib/httpx/request.rb +7 -19
- data/lib/httpx/resolver/https.rb +7 -2
- data/lib/httpx/resolver/native.rb +7 -3
- data/lib/httpx/response.rb +16 -23
- data/lib/httpx/selector.rb +2 -4
- data/lib/httpx/session.rb +17 -11
- data/lib/httpx/transcoder/chunker.rb +0 -2
- data/lib/httpx/transcoder/form.rb +0 -6
- data/lib/httpx/transcoder/json.rb +0 -4
- data/lib/httpx/utils.rb +45 -0
- data/lib/httpx/version.rb +1 -1
- data/sig/buffer.rbs +24 -0
- data/sig/callbacks.rbs +14 -0
- data/sig/chainable.rbs +37 -0
- data/sig/connection.rbs +2 -0
- data/sig/connection/http2.rbs +4 -0
- data/sig/domain_name.rbs +17 -0
- data/sig/errors.rbs +3 -0
- data/sig/headers.rbs +42 -0
- data/sig/httpx.rbs +14 -0
- data/sig/loggable.rbs +11 -0
- data/sig/missing.rbs +12 -0
- data/sig/options.rbs +118 -0
- data/sig/parser/http1.rbs +50 -0
- data/sig/plugins/authentication.rbs +11 -0
- data/sig/plugins/basic_authentication.rbs +13 -0
- data/sig/plugins/compression.rbs +55 -0
- data/sig/plugins/compression/brotli.rbs +21 -0
- data/sig/plugins/compression/deflate.rbs +17 -0
- data/sig/plugins/compression/gzip.rbs +29 -0
- data/sig/plugins/cookies.rbs +26 -0
- data/sig/plugins/cookies/cookie.rbs +50 -0
- data/sig/plugins/cookies/jar.rbs +27 -0
- data/sig/plugins/digest_authentication.rbs +33 -0
- data/sig/plugins/expect.rbs +19 -0
- data/sig/plugins/follow_redirects.rbs +37 -0
- data/sig/plugins/h2c.rbs +26 -0
- data/sig/plugins/multipart.rbs +19 -0
- data/sig/plugins/persistent.rbs +17 -0
- data/sig/plugins/proxy.rbs +47 -0
- data/sig/plugins/proxy/http.rbs +14 -0
- data/sig/plugins/proxy/socks4.rbs +33 -0
- data/sig/plugins/proxy/socks5.rbs +36 -0
- data/sig/plugins/proxy/ssh.rbs +18 -0
- data/sig/plugins/push_promise.rbs +22 -0
- data/sig/plugins/rate_limiter.rbs +11 -0
- data/sig/plugins/retries.rbs +48 -0
- data/sig/plugins/stream.rbs +39 -0
- data/sig/pool.rbs +2 -0
- data/sig/registry.rbs +9 -0
- data/sig/request.rbs +61 -0
- data/sig/response.rbs +87 -0
- data/sig/session.rbs +49 -0
- data/sig/test.rbs +9 -0
- data/sig/timeout.rbs +29 -0
- data/sig/transcoder.rbs +16 -0
- data/sig/transcoder/body.rbs +18 -0
- data/sig/transcoder/chunker.rbs +32 -0
- data/sig/transcoder/form.rbs +16 -0
- data/sig/transcoder/json.rbs +14 -0
- metadata +60 -17
data/lib/httpx/errors.rb
CHANGED
data/lib/httpx/extensions.rb
CHANGED
@@ -54,11 +54,31 @@ module HTTPX
|
|
54
54
|
Numeric.__send__(:include, NegMethods)
|
55
55
|
end
|
56
56
|
|
57
|
+
module RegexpExtensions
|
58
|
+
# If you wonder why this is there: the oauth feature uses a refinement to enhance the
|
59
|
+
# Regexp class locally with #match? , but this is never tested, because ActiveSupport
|
60
|
+
# monkey-patches the same method... Please ActiveSupport, stop being so intrusive!
|
61
|
+
# :nocov:
|
62
|
+
refine(Regexp) do
|
63
|
+
def match?(*args)
|
64
|
+
!match(*args).nil?
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
57
69
|
module URIExtensions
|
58
70
|
refine URI::Generic do
|
71
|
+
def non_ascii_hostname
|
72
|
+
@non_ascii_hostname
|
73
|
+
end
|
74
|
+
|
75
|
+
def non_ascii_hostname=(hostname)
|
76
|
+
@non_ascii_hostname = hostname
|
77
|
+
end
|
78
|
+
|
59
79
|
def authority
|
60
80
|
port_string = port == default_port ? nil : ":#{port}"
|
61
|
-
"#{host}#{port_string}"
|
81
|
+
"#{@non_ascii_hostname || host}#{port_string}"
|
62
82
|
end
|
63
83
|
|
64
84
|
def origin
|
data/lib/httpx/io/ssl.rb
CHANGED
data/lib/httpx/io/tcp.rb
CHANGED
@@ -7,11 +7,7 @@ module HTTPX
|
|
7
7
|
class TCP
|
8
8
|
include Loggable
|
9
9
|
|
10
|
-
attr_reader :ip, :port
|
11
|
-
|
12
|
-
attr_reader :addresses
|
13
|
-
|
14
|
-
attr_reader :state
|
10
|
+
attr_reader :ip, :port, :addresses, :state
|
15
11
|
|
16
12
|
alias_method :host, :ip
|
17
13
|
|
@@ -86,6 +82,7 @@ module HTTPX
|
|
86
82
|
# :nocov:
|
87
83
|
def read(size, buffer)
|
88
84
|
@io.read_nonblock(size, buffer)
|
85
|
+
log { "READ: #{buffer.bytesize} bytes..." }
|
89
86
|
buffer.bytesize
|
90
87
|
rescue ::IO::WaitReadable
|
91
88
|
buffer.clear
|
@@ -96,6 +93,7 @@ module HTTPX
|
|
96
93
|
|
97
94
|
def write(buffer)
|
98
95
|
siz = @io.write_nonblock(buffer)
|
96
|
+
log { "WRITE: #{siz} bytes..." }
|
99
97
|
buffer.shift!(siz)
|
100
98
|
siz
|
101
99
|
rescue ::IO::WaitWritable
|
@@ -113,6 +111,7 @@ module HTTPX
|
|
113
111
|
end
|
114
112
|
return if ret.nil?
|
115
113
|
|
114
|
+
log { "READ: #{buffer.bytesize} bytes..." }
|
116
115
|
buffer.bytesize
|
117
116
|
end
|
118
117
|
|
@@ -121,6 +120,8 @@ module HTTPX
|
|
121
120
|
return 0 if siz == :wait_writable
|
122
121
|
return if siz.nil?
|
123
122
|
|
123
|
+
log { "WRITE: #{siz} bytes..." }
|
124
|
+
|
124
125
|
buffer.shift!(siz)
|
125
126
|
siz
|
126
127
|
end
|
data/lib/httpx/io/udp.rb
CHANGED
@@ -7,11 +7,12 @@ module HTTPX
|
|
7
7
|
class UDP
|
8
8
|
include Loggable
|
9
9
|
|
10
|
-
def initialize(uri, _,
|
10
|
+
def initialize(uri, _, options)
|
11
11
|
ip = IPAddr.new(uri.host)
|
12
12
|
@host = ip.to_s
|
13
13
|
@port = uri.port
|
14
14
|
@io = UDPSocket.new(ip.family)
|
15
|
+
@options = options
|
15
16
|
end
|
16
17
|
|
17
18
|
def to_io
|
@@ -40,6 +41,7 @@ module HTTPX
|
|
40
41
|
|
41
42
|
def write(buffer)
|
42
43
|
siz = @io.send(buffer, 0, @host, @port)
|
44
|
+
log { "WRITE: #{siz} bytes..." }
|
43
45
|
buffer.shift!(siz)
|
44
46
|
siz
|
45
47
|
end
|
@@ -49,6 +51,7 @@ module HTTPX
|
|
49
51
|
def read(size, buffer)
|
50
52
|
data, _ = @io.recvfrom_nonblock(size)
|
51
53
|
buffer.replace(data)
|
54
|
+
log { "READ: #{buffer.bytesize} bytes..." }
|
52
55
|
buffer.bytesize
|
53
56
|
rescue ::IO::WaitReadable
|
54
57
|
0
|
data/lib/httpx/options.rb
CHANGED
data/lib/httpx/parser/http1.rb
CHANGED
@@ -9,10 +9,9 @@ module HTTPX
|
|
9
9
|
|
10
10
|
attr_reader :status_code, :http_version, :headers
|
11
11
|
|
12
|
-
def initialize(observer
|
12
|
+
def initialize(observer)
|
13
13
|
@observer = observer
|
14
14
|
@state = :idle
|
15
|
-
@header_separator = header_separator
|
16
15
|
@buffer = "".b
|
17
16
|
@headers = {}
|
18
17
|
end
|
@@ -40,25 +39,25 @@ module HTTPX
|
|
40
39
|
private
|
41
40
|
|
42
41
|
def parse
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
42
|
+
loop do
|
43
|
+
state = @state
|
44
|
+
case @state
|
45
|
+
when :idle
|
46
|
+
parse_headline
|
47
|
+
when :headers, :trailers
|
48
|
+
parse_headers
|
49
|
+
when :data
|
50
|
+
parse_data
|
51
|
+
end
|
52
|
+
return if @buffer.empty? || state == @state
|
53
53
|
end
|
54
|
-
parse if !@buffer.empty? && state != @state
|
55
54
|
end
|
56
55
|
|
57
56
|
def parse_headline
|
58
57
|
idx = @buffer.index("\n")
|
59
58
|
return unless idx
|
60
59
|
|
61
|
-
(m = %r{\AHTTP(
|
60
|
+
(m = %r{\AHTTP(?:/(\d+\.\d+))?\s+(\d\d\d)(?:\s+(.*))?}in.match(@buffer)) ||
|
62
61
|
raise(Error, "wrong head line format")
|
63
62
|
version, code, _ = m.captures
|
64
63
|
raise(Error, "unsupported HTTP version (HTTP/#{version})") unless VERSIONS.include?(version)
|
@@ -91,12 +90,10 @@ module HTTPX
|
|
91
90
|
@observer.on_trailers(headers)
|
92
91
|
headers.clear
|
93
92
|
nextstate(:complete)
|
94
|
-
else
|
95
|
-
raise Error, "wrong header format"
|
96
93
|
end
|
97
94
|
return
|
98
95
|
end
|
99
|
-
separator_index = line.index(
|
96
|
+
separator_index = line.index(":")
|
100
97
|
raise Error, "wrong header format" unless separator_index
|
101
98
|
|
102
99
|
key = line[0..separator_index - 1]
|
@@ -46,16 +46,13 @@ module HTTPX
|
|
46
46
|
super
|
47
47
|
return if @body.nil?
|
48
48
|
|
49
|
-
|
50
|
-
|
51
|
-
return if @body.bytesize < threshold
|
52
|
-
end
|
53
|
-
end
|
49
|
+
threshold = options.compression_threshold_size
|
50
|
+
return if threshold && !unbounded_body? && @body.bytesize < threshold
|
54
51
|
|
55
52
|
@headers.get("content-encoding").each do |encoding|
|
56
53
|
next if encoding == "identity"
|
57
54
|
|
58
|
-
@body = Encoder.new(@body, Compression.registry(encoding).
|
55
|
+
@body = Encoder.new(@body, Compression.registry(encoding).deflater)
|
59
56
|
end
|
60
57
|
@headers["content-length"] = @body.bytesize unless chunked?
|
61
58
|
end
|
@@ -71,57 +68,53 @@ module HTTPX
|
|
71
68
|
|
72
69
|
return unless @headers.key?("content-encoding")
|
73
70
|
|
74
|
-
|
71
|
+
# remove encodings that we are able to decode
|
72
|
+
@headers["content-encoding"] = @headers.get("content-encoding") - @encodings
|
73
|
+
|
74
|
+
compressed_length = if @headers.key?("content-length")
|
75
|
+
@headers["content-length"].to_i
|
76
|
+
else
|
77
|
+
Float::INFINITY
|
78
|
+
end
|
79
|
+
|
80
|
+
@_inflaters = @headers.get("content-encoding").map do |encoding|
|
75
81
|
next if encoding == "identity"
|
76
82
|
|
77
|
-
|
83
|
+
inflater = Compression.registry(encoding).inflater(compressed_length)
|
78
84
|
# do not uncompress if there is no decoder available. In fact, we can't reliably
|
79
85
|
# continue decompressing beyond that, so ignore.
|
80
|
-
break unless
|
86
|
+
break unless inflater
|
81
87
|
|
82
88
|
@encodings << encoding
|
83
|
-
|
89
|
+
inflater
|
84
90
|
end.compact
|
85
91
|
|
86
|
-
#
|
87
|
-
|
88
|
-
|
89
|
-
@_compressed_length = if @headers.key?("content-length")
|
90
|
-
@headers["content-length"].to_i
|
91
|
-
else
|
92
|
-
Float::INFINITY
|
93
|
-
end
|
92
|
+
# this can happen if the only declared encoding is "identity"
|
93
|
+
remove_instance_variable(:@_inflaters) if @_inflaters.empty?
|
94
94
|
end
|
95
95
|
|
96
96
|
def write(chunk)
|
97
|
-
return super unless defined?(@
|
97
|
+
return super unless defined?(@_inflaters)
|
98
98
|
|
99
|
-
@_compressed_length -= chunk.bytesize
|
100
99
|
chunk = decompress(chunk)
|
101
100
|
super(chunk)
|
102
101
|
end
|
103
102
|
|
104
|
-
def close
|
105
|
-
super
|
106
|
-
|
107
|
-
return unless defined?(@_decoders)
|
108
|
-
|
109
|
-
@_decoders.each(&:close)
|
110
|
-
end
|
111
|
-
|
112
103
|
private
|
113
104
|
|
114
105
|
def decompress(buffer)
|
115
|
-
@
|
116
|
-
buffer =
|
117
|
-
buffer << decoder.finish if @_compressed_length <= 0
|
106
|
+
@_inflaters.reverse_each do |inflater|
|
107
|
+
buffer = inflater.inflate(buffer)
|
118
108
|
end
|
119
109
|
buffer
|
120
110
|
end
|
121
111
|
end
|
122
112
|
|
123
113
|
class Encoder
|
114
|
+
attr_reader :content_type
|
115
|
+
|
124
116
|
def initialize(body, deflater)
|
117
|
+
@content_type = body.content_type
|
125
118
|
@body = body.respond_to?(:read) ? body : StringIO.new(body.to_s)
|
126
119
|
@buffer = StringIO.new("".b, File::RDWR)
|
127
120
|
@deflater = deflater
|
@@ -130,11 +123,10 @@ module HTTPX
|
|
130
123
|
def each(&blk)
|
131
124
|
return enum_for(__method__) unless block_given?
|
132
125
|
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
deflate(&blk)
|
126
|
+
return deflate(&blk) if @buffer.size.zero?
|
127
|
+
|
128
|
+
@buffer.rewind
|
129
|
+
@buffer.each(&blk)
|
138
130
|
end
|
139
131
|
|
140
132
|
def bytesize
|
@@ -142,17 +134,6 @@ module HTTPX
|
|
142
134
|
@buffer.size
|
143
135
|
end
|
144
136
|
|
145
|
-
def to_s
|
146
|
-
deflate
|
147
|
-
@buffer.rewind
|
148
|
-
@buffer.read
|
149
|
-
end
|
150
|
-
|
151
|
-
def close
|
152
|
-
@buffer.close
|
153
|
-
@body.close
|
154
|
-
end
|
155
|
-
|
156
137
|
private
|
157
138
|
|
158
139
|
def deflate(&blk)
|
@@ -162,22 +143,6 @@ module HTTPX
|
|
162
143
|
@deflater.deflate(@body, @buffer, chunk_size: 16_384, &blk)
|
163
144
|
end
|
164
145
|
end
|
165
|
-
|
166
|
-
class Decoder
|
167
|
-
extend Forwardable
|
168
|
-
|
169
|
-
def_delegator :@inflater, :finish
|
170
|
-
|
171
|
-
def_delegator :@inflater, :close
|
172
|
-
|
173
|
-
def initialize(inflater)
|
174
|
-
@inflater = inflater
|
175
|
-
end
|
176
|
-
|
177
|
-
def decode(chunk)
|
178
|
-
@inflater.inflate(chunk)
|
179
|
-
end
|
180
|
-
end
|
181
146
|
end
|
182
147
|
register_plugin :compression, Compression
|
183
148
|
end
|
@@ -13,7 +13,7 @@ module HTTPX
|
|
13
13
|
Compression.register "br", self
|
14
14
|
end
|
15
15
|
|
16
|
-
module
|
16
|
+
module Deflater
|
17
17
|
module_function
|
18
18
|
|
19
19
|
def deflate(raw, buffer, chunk_size:)
|
@@ -25,28 +25,24 @@ module HTTPX
|
|
25
25
|
end
|
26
26
|
end
|
27
27
|
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
def inflate(text)
|
32
|
-
::Brotli.inflate(text)
|
28
|
+
class Inflater
|
29
|
+
def initialize(bytesize)
|
30
|
+
@bytesize = bytesize
|
33
31
|
end
|
34
32
|
|
35
|
-
def
|
36
|
-
|
37
|
-
def finish
|
38
|
-
""
|
33
|
+
def inflate(chunk)
|
34
|
+
::Brotli.inflate(chunk)
|
39
35
|
end
|
40
36
|
end
|
41
37
|
|
42
38
|
module_function
|
43
39
|
|
44
|
-
def
|
45
|
-
|
40
|
+
def deflater
|
41
|
+
Deflater
|
46
42
|
end
|
47
43
|
|
48
|
-
def
|
49
|
-
|
44
|
+
def inflater(bytesize)
|
45
|
+
Inflater.new(bytesize)
|
50
46
|
end
|
51
47
|
end
|
52
48
|
end
|
@@ -4,16 +4,17 @@ module HTTPX
|
|
4
4
|
module Plugins
|
5
5
|
module Compression
|
6
6
|
module Deflate
|
7
|
-
def self.load_dependencies(
|
7
|
+
def self.load_dependencies(klass)
|
8
8
|
require "stringio"
|
9
9
|
require "zlib"
|
10
|
+
klass.plugin(:"compression/gzip")
|
10
11
|
end
|
11
12
|
|
12
13
|
def self.configure(*)
|
13
14
|
Compression.register "deflate", self
|
14
15
|
end
|
15
16
|
|
16
|
-
module
|
17
|
+
module Deflater
|
17
18
|
module_function
|
18
19
|
|
19
20
|
def deflate(raw, buffer, chunk_size:)
|
@@ -36,12 +37,12 @@ module HTTPX
|
|
36
37
|
|
37
38
|
module_function
|
38
39
|
|
39
|
-
def
|
40
|
-
|
40
|
+
def deflater
|
41
|
+
Deflater
|
41
42
|
end
|
42
43
|
|
43
|
-
def
|
44
|
-
|
44
|
+
def inflater(bytesize)
|
45
|
+
GZIP::Inflater.new(bytesize)
|
45
46
|
end
|
46
47
|
end
|
47
48
|
end
|
@@ -14,7 +14,7 @@ module HTTPX
|
|
14
14
|
Compression.register "gzip", self
|
15
15
|
end
|
16
16
|
|
17
|
-
class
|
17
|
+
class Deflater
|
18
18
|
def initialize
|
19
19
|
@compressed_chunk = "".b
|
20
20
|
end
|
@@ -53,14 +53,32 @@ module HTTPX
|
|
53
53
|
end
|
54
54
|
end
|
55
55
|
|
56
|
+
class Inflater
|
57
|
+
def initialize(bytesize)
|
58
|
+
@inflater = Zlib::Inflate.new(32 + Zlib::MAX_WBITS)
|
59
|
+
@bytesize = bytesize
|
60
|
+
@buffer = nil
|
61
|
+
end
|
62
|
+
|
63
|
+
def inflate(chunk)
|
64
|
+
buffer = @inflater.inflate(chunk)
|
65
|
+
@bytesize -= chunk.bytesize
|
66
|
+
if @bytesize <= 0
|
67
|
+
buffer << @inflater.finish
|
68
|
+
@inflater.close
|
69
|
+
end
|
70
|
+
buffer
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
56
74
|
module_function
|
57
75
|
|
58
|
-
def
|
59
|
-
|
76
|
+
def deflater
|
77
|
+
Deflater.new
|
60
78
|
end
|
61
79
|
|
62
|
-
def
|
63
|
-
|
80
|
+
def inflater(bytesize)
|
81
|
+
Inflater.new(bytesize)
|
64
82
|
end
|
65
83
|
end
|
66
84
|
end
|