httpx 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE.txt +191 -0
- data/README.md +119 -0
- data/lib/httpx.rb +50 -0
- data/lib/httpx/buffer.rb +34 -0
- data/lib/httpx/callbacks.rb +32 -0
- data/lib/httpx/chainable.rb +51 -0
- data/lib/httpx/channel.rb +222 -0
- data/lib/httpx/channel/http1.rb +220 -0
- data/lib/httpx/channel/http2.rb +224 -0
- data/lib/httpx/client.rb +173 -0
- data/lib/httpx/connection.rb +74 -0
- data/lib/httpx/errors.rb +7 -0
- data/lib/httpx/extensions.rb +52 -0
- data/lib/httpx/headers.rb +152 -0
- data/lib/httpx/io.rb +240 -0
- data/lib/httpx/loggable.rb +11 -0
- data/lib/httpx/options.rb +138 -0
- data/lib/httpx/plugins/authentication.rb +14 -0
- data/lib/httpx/plugins/basic_authentication.rb +20 -0
- data/lib/httpx/plugins/compression.rb +123 -0
- data/lib/httpx/plugins/compression/brotli.rb +55 -0
- data/lib/httpx/plugins/compression/deflate.rb +50 -0
- data/lib/httpx/plugins/compression/gzip.rb +59 -0
- data/lib/httpx/plugins/cookies.rb +63 -0
- data/lib/httpx/plugins/digest_authentication.rb +141 -0
- data/lib/httpx/plugins/follow_redirects.rb +72 -0
- data/lib/httpx/plugins/h2c.rb +85 -0
- data/lib/httpx/plugins/proxy.rb +108 -0
- data/lib/httpx/plugins/proxy/http.rb +115 -0
- data/lib/httpx/plugins/proxy/socks4.rb +110 -0
- data/lib/httpx/plugins/proxy/socks5.rb +152 -0
- data/lib/httpx/plugins/push_promise.rb +67 -0
- data/lib/httpx/plugins/stream.rb +33 -0
- data/lib/httpx/registry.rb +88 -0
- data/lib/httpx/request.rb +222 -0
- data/lib/httpx/response.rb +225 -0
- data/lib/httpx/selector.rb +155 -0
- data/lib/httpx/timeout.rb +68 -0
- data/lib/httpx/transcoder.rb +12 -0
- data/lib/httpx/transcoder/body.rb +56 -0
- data/lib/httpx/transcoder/chunker.rb +38 -0
- data/lib/httpx/transcoder/form.rb +41 -0
- data/lib/httpx/transcoder/json.rb +36 -0
- data/lib/httpx/version.rb +5 -0
- metadata +150 -0
data/lib/httpx/io.rb
ADDED
@@ -0,0 +1,240 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "socket"
|
4
|
+
require "openssl"
|
5
|
+
require "ipaddr"
|
6
|
+
|
7
|
+
module HTTPX
|
8
|
+
class TCP
|
9
|
+
include Loggable
|
10
|
+
|
11
|
+
attr_reader :ip, :port
|
12
|
+
|
13
|
+
def initialize(hostname, port, options)
|
14
|
+
@state = :idle
|
15
|
+
@options = Options.new(options)
|
16
|
+
@fallback_protocol = @options.fallback_protocol
|
17
|
+
@port = port
|
18
|
+
if @options.io
|
19
|
+
@io = case @options.io
|
20
|
+
when Hash
|
21
|
+
@ip = TCPSocket.getaddress(hostname)
|
22
|
+
@options.io[@ip] || @options.io["#{@ip}:#{@port}"]
|
23
|
+
else
|
24
|
+
@ip = hostname
|
25
|
+
@options.io
|
26
|
+
end
|
27
|
+
unless @io.nil?
|
28
|
+
@keep_open = true
|
29
|
+
@state = :connected
|
30
|
+
end
|
31
|
+
else
|
32
|
+
@ip = TCPSocket.getaddress(hostname)
|
33
|
+
end
|
34
|
+
@io ||= build_socket
|
35
|
+
end
|
36
|
+
|
37
|
+
def scheme
|
38
|
+
"http"
|
39
|
+
end
|
40
|
+
|
41
|
+
def to_io
|
42
|
+
@io.to_io
|
43
|
+
end
|
44
|
+
|
45
|
+
def protocol
|
46
|
+
@fallback_protocol
|
47
|
+
end
|
48
|
+
|
49
|
+
def connect
|
50
|
+
return unless closed?
|
51
|
+
begin
|
52
|
+
if @io.closed?
|
53
|
+
transition(:idle)
|
54
|
+
@io = build_socket
|
55
|
+
end
|
56
|
+
@io.connect_nonblock(Socket.sockaddr_in(@port, @ip))
|
57
|
+
rescue Errno::EISCONN
|
58
|
+
end
|
59
|
+
transition(:connected)
|
60
|
+
rescue Errno::EINPROGRESS,
|
61
|
+
Errno::EALREADY,
|
62
|
+
::IO::WaitReadable
|
63
|
+
end
|
64
|
+
|
65
|
+
if RUBY_VERSION < "2.3"
|
66
|
+
def read(size, buffer)
|
67
|
+
@io.read_nonblock(size, buffer)
|
68
|
+
buffer.bytesize
|
69
|
+
rescue ::IO::WaitReadable
|
70
|
+
0
|
71
|
+
rescue EOFError
|
72
|
+
nil
|
73
|
+
end
|
74
|
+
|
75
|
+
def write(buffer)
|
76
|
+
siz = @io.write_nonblock(buffer)
|
77
|
+
buffer.slice!(0, siz)
|
78
|
+
siz
|
79
|
+
rescue ::IO::WaitWritable
|
80
|
+
0
|
81
|
+
rescue EOFError
|
82
|
+
nil
|
83
|
+
end
|
84
|
+
else
|
85
|
+
def read(size, buffer)
|
86
|
+
ret = @io.read_nonblock(size, buffer, exception: false)
|
87
|
+
return 0 if ret == :wait_readable
|
88
|
+
return if ret.nil?
|
89
|
+
buffer.bytesize
|
90
|
+
end
|
91
|
+
|
92
|
+
def write(buffer)
|
93
|
+
siz = @io.write_nonblock(buffer, exception: false)
|
94
|
+
return 0 if siz == :wait_writable
|
95
|
+
return if siz.nil?
|
96
|
+
buffer.slice!(0, siz)
|
97
|
+
siz
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
def close
|
102
|
+
return if @keep_open || closed?
|
103
|
+
begin
|
104
|
+
@io.close
|
105
|
+
ensure
|
106
|
+
transition(:closed)
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
def connected?
|
111
|
+
@state == :connected
|
112
|
+
end
|
113
|
+
|
114
|
+
def closed?
|
115
|
+
@state == :idle || @state == :closed
|
116
|
+
end
|
117
|
+
|
118
|
+
def inspect
|
119
|
+
id = @io.closed? ? "closed" : @io.fileno
|
120
|
+
"#<TCP(fd: #{id}): #{@ip}:#{@port} (state: #{@state})>"
|
121
|
+
end
|
122
|
+
|
123
|
+
private
|
124
|
+
|
125
|
+
def build_socket
|
126
|
+
addr = IPAddr.new(@ip)
|
127
|
+
Socket.new(addr.family, :STREAM, 0)
|
128
|
+
end
|
129
|
+
|
130
|
+
def transition(nextstate)
|
131
|
+
case nextstate
|
132
|
+
# when :idle
|
133
|
+
when :connected
|
134
|
+
return unless @state == :idle
|
135
|
+
when :closed
|
136
|
+
return unless @state == :connected
|
137
|
+
end
|
138
|
+
log(1, "#{inspect}: ") { nextstate.to_s }
|
139
|
+
@state = nextstate
|
140
|
+
end
|
141
|
+
end
|
142
|
+
|
143
|
+
class SSL < TCP
|
144
|
+
TLS_OPTIONS = if OpenSSL::SSL::SSLContext.instance_methods.include?(:alpn_protocols)
|
145
|
+
{ alpn_protocols: %w[h2 http/1.1] }
|
146
|
+
else
|
147
|
+
{}
|
148
|
+
end
|
149
|
+
|
150
|
+
def initialize(_, _, options)
|
151
|
+
@ctx = OpenSSL::SSL::SSLContext.new
|
152
|
+
ctx_options = TLS_OPTIONS.merge(options.ssl)
|
153
|
+
@ctx.set_params(ctx_options) unless ctx_options.empty?
|
154
|
+
super
|
155
|
+
@state = :negotiated if @keep_open
|
156
|
+
end
|
157
|
+
|
158
|
+
def scheme
|
159
|
+
"https"
|
160
|
+
end
|
161
|
+
|
162
|
+
def protocol
|
163
|
+
@io.alpn_protocol || super
|
164
|
+
rescue StandardError
|
165
|
+
super
|
166
|
+
end
|
167
|
+
|
168
|
+
def close
|
169
|
+
super
|
170
|
+
# allow reconnections
|
171
|
+
# connect only works if initial @io is a socket
|
172
|
+
@io = @io.io if @io.respond_to?(:io)
|
173
|
+
@negotiated = false
|
174
|
+
end
|
175
|
+
|
176
|
+
def connect
|
177
|
+
super
|
178
|
+
if @keep_open
|
179
|
+
@state = :negotiated
|
180
|
+
return
|
181
|
+
end
|
182
|
+
return if @state == :negotiated ||
|
183
|
+
@state != :connected
|
184
|
+
@io = OpenSSL::SSL::SSLSocket.new(@io, @ctx)
|
185
|
+
@io.hostname = @hostname
|
186
|
+
@io.sync_close = true
|
187
|
+
@io.connect
|
188
|
+
transition(:negotiated)
|
189
|
+
end
|
190
|
+
|
191
|
+
if RUBY_VERSION < "2.3"
|
192
|
+
def read(*)
|
193
|
+
super
|
194
|
+
rescue ::IO::WaitWritable
|
195
|
+
0
|
196
|
+
end
|
197
|
+
|
198
|
+
def write(*)
|
199
|
+
super
|
200
|
+
rescue ::IO::WaitReadable
|
201
|
+
0
|
202
|
+
end
|
203
|
+
else
|
204
|
+
if OpenSSL::VERSION < "2.0.6"
|
205
|
+
def read(size, buffer)
|
206
|
+
@io.read_nonblock(size, buffer)
|
207
|
+
buffer.bytesize
|
208
|
+
rescue ::IO::WaitReadable,
|
209
|
+
::IO::WaitWritable
|
210
|
+
0
|
211
|
+
rescue EOFError
|
212
|
+
nil
|
213
|
+
end
|
214
|
+
end
|
215
|
+
end
|
216
|
+
|
217
|
+
def inspect
|
218
|
+
id = @io.closed? ? "closed" : @io.to_io.fileno
|
219
|
+
"#<SSL(fd: #{id}): #{@ip}:#{@port} state: #{@state}>"
|
220
|
+
end
|
221
|
+
|
222
|
+
private
|
223
|
+
|
224
|
+
def transition(nextstate)
|
225
|
+
case nextstate
|
226
|
+
when :negotiated
|
227
|
+
return unless @state == :connected
|
228
|
+
when :closed
|
229
|
+
return unless @state == :negotiated ||
|
230
|
+
@state == :connected
|
231
|
+
end
|
232
|
+
super
|
233
|
+
end
|
234
|
+
end
|
235
|
+
module IO
|
236
|
+
extend Registry
|
237
|
+
register "tcp", TCP
|
238
|
+
register "ssl", SSL
|
239
|
+
end
|
240
|
+
end
|
@@ -0,0 +1,11 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module HTTPX
|
4
|
+
module Loggable
|
5
|
+
def log(level = @options.debug_level, label = "", &msg)
|
6
|
+
return unless @options.debug
|
7
|
+
return unless @options.debug_level >= level
|
8
|
+
@options.debug << (+label << msg.call << "\n")
|
9
|
+
end
|
10
|
+
end
|
11
|
+
end
|
@@ -0,0 +1,138 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module HTTPX
|
4
|
+
class Options
|
5
|
+
MAX_CONCURRENT_REQUESTS = 100
|
6
|
+
MAX_RETRIES = 3
|
7
|
+
WINDOW_SIZE = 1 << 14 # 16K
|
8
|
+
MAX_BODY_THRESHOLD_SIZE = (1 << 10) * 112 # 112K
|
9
|
+
|
10
|
+
class << self
|
11
|
+
def inherited(klass)
|
12
|
+
super
|
13
|
+
klass.instance_variable_set(:@defined_options, @defined_options.dup)
|
14
|
+
end
|
15
|
+
|
16
|
+
def new(options = {})
|
17
|
+
# let enhanced options go through
|
18
|
+
return options if self == Options && options.class > self
|
19
|
+
return options if options.is_a?(self)
|
20
|
+
super
|
21
|
+
end
|
22
|
+
|
23
|
+
def defined_options
|
24
|
+
@defined_options ||= []
|
25
|
+
end
|
26
|
+
|
27
|
+
def def_option(name, &interpreter)
|
28
|
+
defined_options << name.to_sym
|
29
|
+
interpreter ||= ->(v) { v }
|
30
|
+
|
31
|
+
attr_accessor name
|
32
|
+
protected :"#{name}="
|
33
|
+
|
34
|
+
define_method(:"with_#{name}") do |value|
|
35
|
+
dup { |opts| opts.send(:"#{name}=", instance_exec(value, &interpreter)) }
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def initialize(options = {})
|
41
|
+
defaults = {
|
42
|
+
:debug => ENV.key?("HTTPX_DEBUG") ? $stderr : nil,
|
43
|
+
:debug_level => (ENV["HTTPX_DEBUG"] || 1).to_i,
|
44
|
+
:ssl => {},
|
45
|
+
:http2_settings => { settings_enable_push: 0 },
|
46
|
+
:fallback_protocol => "http/1.1",
|
47
|
+
:timeout => Timeout.new,
|
48
|
+
:headers => {},
|
49
|
+
:max_concurrent_requests => MAX_CONCURRENT_REQUESTS,
|
50
|
+
:max_retries => MAX_RETRIES,
|
51
|
+
:window_size => WINDOW_SIZE,
|
52
|
+
:body_threshold_size => MAX_BODY_THRESHOLD_SIZE,
|
53
|
+
:request_class => Class.new(Request),
|
54
|
+
:response_class => Class.new(Response),
|
55
|
+
:headers_class => Class.new(Headers),
|
56
|
+
:request_body_class => Class.new(Request::Body),
|
57
|
+
:response_body_class => Class.new(Response::Body),
|
58
|
+
}
|
59
|
+
|
60
|
+
defaults.merge!(options)
|
61
|
+
defaults[:headers] = Headers.new(defaults[:headers])
|
62
|
+
defaults.each { |(k, v)| self[k] = v }
|
63
|
+
end
|
64
|
+
|
65
|
+
def_option(:headers) do |headers|
|
66
|
+
self.headers.merge(headers)
|
67
|
+
end
|
68
|
+
|
69
|
+
def_option(:timeout) do |opts|
|
70
|
+
self.timeout = Timeout.new(opts)
|
71
|
+
end
|
72
|
+
|
73
|
+
def_option(:max_concurrent_requests) do |num|
|
74
|
+
max = Integer(num)
|
75
|
+
raise Error, ":max_concurrent_requests must be positive" unless max.positive?
|
76
|
+
self.max_concurrent_requests = max
|
77
|
+
end
|
78
|
+
|
79
|
+
def_option(:window_size) do |num|
|
80
|
+
self.window_size = Integer(num)
|
81
|
+
end
|
82
|
+
|
83
|
+
def_option(:body_threshold_size) do |num|
|
84
|
+
self.body_threshold_size = Integer(num)
|
85
|
+
end
|
86
|
+
|
87
|
+
%w[
|
88
|
+
params form json body
|
89
|
+
follow ssl http2_settings max_retries
|
90
|
+
request_class response_class headers_class request_body_class response_body_class
|
91
|
+
io fallback_protocol debug debug_level
|
92
|
+
].each do |method_name|
|
93
|
+
def_option(method_name)
|
94
|
+
end
|
95
|
+
|
96
|
+
def merge(other)
|
97
|
+
h1 = to_hash
|
98
|
+
h2 = other.to_hash
|
99
|
+
|
100
|
+
merged = h1.merge(h2) do |k, v1, v2|
|
101
|
+
case k
|
102
|
+
when :headers, :ssl, :http2_settings, :timeout
|
103
|
+
v1.merge(v2)
|
104
|
+
else
|
105
|
+
v2
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
self.class.new(merged)
|
110
|
+
end
|
111
|
+
|
112
|
+
def to_hash
|
113
|
+
hash_pairs = self.class
|
114
|
+
.defined_options
|
115
|
+
.flat_map { |opt_name| [opt_name, send(opt_name)] }
|
116
|
+
Hash[*hash_pairs]
|
117
|
+
end
|
118
|
+
|
119
|
+
def dup
|
120
|
+
dupped = super
|
121
|
+
dupped.headers = headers.dup
|
122
|
+
dupped.ssl = ssl.dup
|
123
|
+
dupped.request_class = request_class.dup
|
124
|
+
dupped.response_class = response_class.dup
|
125
|
+
dupped.headers_class = headers_class.dup
|
126
|
+
dupped.request_body_class = request_body_class.dup
|
127
|
+
dupped.response_body_class = response_body_class.dup
|
128
|
+
yield(dupped) if block_given?
|
129
|
+
dupped
|
130
|
+
end
|
131
|
+
|
132
|
+
protected
|
133
|
+
|
134
|
+
def []=(option, val)
|
135
|
+
send(:"#{option}=", val)
|
136
|
+
end
|
137
|
+
end
|
138
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module HTTPX
|
4
|
+
module Plugins
|
5
|
+
module Authentication
|
6
|
+
module InstanceMethods
|
7
|
+
def authentication(token)
|
8
|
+
headers("authorization" => token)
|
9
|
+
end
|
10
|
+
end
|
11
|
+
end
|
12
|
+
register_plugin :authentication, Authentication
|
13
|
+
end
|
14
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module HTTPX
|
4
|
+
module Plugins
|
5
|
+
module BasicAuthentication
|
6
|
+
def self.load_dependencies(klass, *)
|
7
|
+
require "base64"
|
8
|
+
klass.plugin(:authentication)
|
9
|
+
end
|
10
|
+
|
11
|
+
module InstanceMethods
|
12
|
+
def basic_authentication(user, password)
|
13
|
+
authentication("Basic #{Base64.strict_encode64("#{user}:#{password}")}")
|
14
|
+
end
|
15
|
+
alias_method :basic_auth, :basic_authentication
|
16
|
+
end
|
17
|
+
end
|
18
|
+
register_plugin :basic_authentication, BasicAuthentication
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,123 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module HTTPX
|
4
|
+
module Plugins
|
5
|
+
module Compression
|
6
|
+
extend Registry
|
7
|
+
def self.configure(klass, *)
|
8
|
+
klass.plugin(:"compression/gzip")
|
9
|
+
klass.plugin(:"compression/deflate")
|
10
|
+
end
|
11
|
+
|
12
|
+
module InstanceMethods
|
13
|
+
def initialize(opts = {})
|
14
|
+
super(opts.merge(headers: { "accept-encoding" => Compression.registry.keys }))
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
module RequestBodyMethods
|
19
|
+
def initialize(*)
|
20
|
+
super
|
21
|
+
return if @body.nil?
|
22
|
+
@headers.get("content-encoding").each do |encoding|
|
23
|
+
@body = Encoder.new(@body, Compression.registry(encoding).encoder)
|
24
|
+
end
|
25
|
+
@headers["content-length"] = @body.bytesize unless chunked?
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
module ResponseBodyMethods
|
30
|
+
def initialize(*)
|
31
|
+
super
|
32
|
+
@_decoders = @headers.get("content-encoding").map do |encoding|
|
33
|
+
Compression.registry(encoding).decoder
|
34
|
+
end
|
35
|
+
@_compressed_length = if @headers.key?("content-length")
|
36
|
+
@headers["content-length"].to_i
|
37
|
+
else
|
38
|
+
Float::INFINITY
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def write(chunk)
|
43
|
+
@_compressed_length -= chunk.bytesize
|
44
|
+
chunk = decompress(chunk)
|
45
|
+
super(chunk)
|
46
|
+
end
|
47
|
+
|
48
|
+
def close
|
49
|
+
super
|
50
|
+
@_decoders.each(&:close)
|
51
|
+
end
|
52
|
+
|
53
|
+
private
|
54
|
+
|
55
|
+
def decompress(buffer)
|
56
|
+
@_decoders.reverse_each do |decoder|
|
57
|
+
buffer = decoder.decode(buffer)
|
58
|
+
buffer << decoder.finish if @_compressed_length <= 0
|
59
|
+
end
|
60
|
+
buffer
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
class Encoder
|
65
|
+
def initialize(body, deflater)
|
66
|
+
@body = body.respond_to?(:read) ? body : StringIO.new(body.to_s)
|
67
|
+
@buffer = StringIO.new("".b, File::RDWR)
|
68
|
+
@deflater = deflater
|
69
|
+
end
|
70
|
+
|
71
|
+
def each(&blk)
|
72
|
+
return enum_for(__method__) unless block_given?
|
73
|
+
unless @buffer.size.zero?
|
74
|
+
@buffer.rewind
|
75
|
+
return @buffer.each(&blk)
|
76
|
+
end
|
77
|
+
deflate(&blk)
|
78
|
+
end
|
79
|
+
|
80
|
+
def bytesize
|
81
|
+
deflate
|
82
|
+
@buffer.size
|
83
|
+
end
|
84
|
+
|
85
|
+
def to_s
|
86
|
+
deflate
|
87
|
+
@buffer.rewind
|
88
|
+
@buffer.read
|
89
|
+
end
|
90
|
+
|
91
|
+
def close
|
92
|
+
@buffer.close
|
93
|
+
@body.close
|
94
|
+
end
|
95
|
+
|
96
|
+
private
|
97
|
+
|
98
|
+
def deflate(&blk)
|
99
|
+
return unless @buffer.size.zero?
|
100
|
+
@body.rewind
|
101
|
+
@deflater.deflate(@body, @buffer, chunk_size: 16_384, &blk)
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
105
|
+
class Decoder
|
106
|
+
extend Forwardable
|
107
|
+
|
108
|
+
def_delegator :@inflater, :finish
|
109
|
+
|
110
|
+
def_delegator :@inflater, :close
|
111
|
+
|
112
|
+
def initialize(inflater)
|
113
|
+
@inflater = inflater
|
114
|
+
end
|
115
|
+
|
116
|
+
def decode(chunk)
|
117
|
+
@inflater.inflate(chunk)
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
121
|
+
register_plugin :compression, Compression
|
122
|
+
end
|
123
|
+
end
|