httpx 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. checksums.yaml +7 -0
  2. data/LICENSE.txt +191 -0
  3. data/README.md +119 -0
  4. data/lib/httpx.rb +50 -0
  5. data/lib/httpx/buffer.rb +34 -0
  6. data/lib/httpx/callbacks.rb +32 -0
  7. data/lib/httpx/chainable.rb +51 -0
  8. data/lib/httpx/channel.rb +222 -0
  9. data/lib/httpx/channel/http1.rb +220 -0
  10. data/lib/httpx/channel/http2.rb +224 -0
  11. data/lib/httpx/client.rb +173 -0
  12. data/lib/httpx/connection.rb +74 -0
  13. data/lib/httpx/errors.rb +7 -0
  14. data/lib/httpx/extensions.rb +52 -0
  15. data/lib/httpx/headers.rb +152 -0
  16. data/lib/httpx/io.rb +240 -0
  17. data/lib/httpx/loggable.rb +11 -0
  18. data/lib/httpx/options.rb +138 -0
  19. data/lib/httpx/plugins/authentication.rb +14 -0
  20. data/lib/httpx/plugins/basic_authentication.rb +20 -0
  21. data/lib/httpx/plugins/compression.rb +123 -0
  22. data/lib/httpx/plugins/compression/brotli.rb +55 -0
  23. data/lib/httpx/plugins/compression/deflate.rb +50 -0
  24. data/lib/httpx/plugins/compression/gzip.rb +59 -0
  25. data/lib/httpx/plugins/cookies.rb +63 -0
  26. data/lib/httpx/plugins/digest_authentication.rb +141 -0
  27. data/lib/httpx/plugins/follow_redirects.rb +72 -0
  28. data/lib/httpx/plugins/h2c.rb +85 -0
  29. data/lib/httpx/plugins/proxy.rb +108 -0
  30. data/lib/httpx/plugins/proxy/http.rb +115 -0
  31. data/lib/httpx/plugins/proxy/socks4.rb +110 -0
  32. data/lib/httpx/plugins/proxy/socks5.rb +152 -0
  33. data/lib/httpx/plugins/push_promise.rb +67 -0
  34. data/lib/httpx/plugins/stream.rb +33 -0
  35. data/lib/httpx/registry.rb +88 -0
  36. data/lib/httpx/request.rb +222 -0
  37. data/lib/httpx/response.rb +225 -0
  38. data/lib/httpx/selector.rb +155 -0
  39. data/lib/httpx/timeout.rb +68 -0
  40. data/lib/httpx/transcoder.rb +12 -0
  41. data/lib/httpx/transcoder/body.rb +56 -0
  42. data/lib/httpx/transcoder/chunker.rb +38 -0
  43. data/lib/httpx/transcoder/form.rb +41 -0
  44. data/lib/httpx/transcoder/json.rb +36 -0
  45. data/lib/httpx/version.rb +5 -0
  46. metadata +150 -0
@@ -0,0 +1,240 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "socket"
4
+ require "openssl"
5
+ require "ipaddr"
6
+
7
+ module HTTPX
8
+ class TCP
9
+ include Loggable
10
+
11
+ attr_reader :ip, :port
12
+
13
+ def initialize(hostname, port, options)
14
+ @state = :idle
15
+ @options = Options.new(options)
16
+ @fallback_protocol = @options.fallback_protocol
17
+ @port = port
18
+ if @options.io
19
+ @io = case @options.io
20
+ when Hash
21
+ @ip = TCPSocket.getaddress(hostname)
22
+ @options.io[@ip] || @options.io["#{@ip}:#{@port}"]
23
+ else
24
+ @ip = hostname
25
+ @options.io
26
+ end
27
+ unless @io.nil?
28
+ @keep_open = true
29
+ @state = :connected
30
+ end
31
+ else
32
+ @ip = TCPSocket.getaddress(hostname)
33
+ end
34
+ @io ||= build_socket
35
+ end
36
+
37
+ def scheme
38
+ "http"
39
+ end
40
+
41
+ def to_io
42
+ @io.to_io
43
+ end
44
+
45
+ def protocol
46
+ @fallback_protocol
47
+ end
48
+
49
+ def connect
50
+ return unless closed?
51
+ begin
52
+ if @io.closed?
53
+ transition(:idle)
54
+ @io = build_socket
55
+ end
56
+ @io.connect_nonblock(Socket.sockaddr_in(@port, @ip))
57
+ rescue Errno::EISCONN
58
+ end
59
+ transition(:connected)
60
+ rescue Errno::EINPROGRESS,
61
+ Errno::EALREADY,
62
+ ::IO::WaitReadable
63
+ end
64
+
65
+ if RUBY_VERSION < "2.3"
66
+ def read(size, buffer)
67
+ @io.read_nonblock(size, buffer)
68
+ buffer.bytesize
69
+ rescue ::IO::WaitReadable
70
+ 0
71
+ rescue EOFError
72
+ nil
73
+ end
74
+
75
+ def write(buffer)
76
+ siz = @io.write_nonblock(buffer)
77
+ buffer.slice!(0, siz)
78
+ siz
79
+ rescue ::IO::WaitWritable
80
+ 0
81
+ rescue EOFError
82
+ nil
83
+ end
84
+ else
85
+ def read(size, buffer)
86
+ ret = @io.read_nonblock(size, buffer, exception: false)
87
+ return 0 if ret == :wait_readable
88
+ return if ret.nil?
89
+ buffer.bytesize
90
+ end
91
+
92
+ def write(buffer)
93
+ siz = @io.write_nonblock(buffer, exception: false)
94
+ return 0 if siz == :wait_writable
95
+ return if siz.nil?
96
+ buffer.slice!(0, siz)
97
+ siz
98
+ end
99
+ end
100
+
101
+ def close
102
+ return if @keep_open || closed?
103
+ begin
104
+ @io.close
105
+ ensure
106
+ transition(:closed)
107
+ end
108
+ end
109
+
110
+ def connected?
111
+ @state == :connected
112
+ end
113
+
114
+ def closed?
115
+ @state == :idle || @state == :closed
116
+ end
117
+
118
+ def inspect
119
+ id = @io.closed? ? "closed" : @io.fileno
120
+ "#<TCP(fd: #{id}): #{@ip}:#{@port} (state: #{@state})>"
121
+ end
122
+
123
+ private
124
+
125
+ def build_socket
126
+ addr = IPAddr.new(@ip)
127
+ Socket.new(addr.family, :STREAM, 0)
128
+ end
129
+
130
+ def transition(nextstate)
131
+ case nextstate
132
+ # when :idle
133
+ when :connected
134
+ return unless @state == :idle
135
+ when :closed
136
+ return unless @state == :connected
137
+ end
138
+ log(1, "#{inspect}: ") { nextstate.to_s }
139
+ @state = nextstate
140
+ end
141
+ end
142
+
143
+ class SSL < TCP
144
+ TLS_OPTIONS = if OpenSSL::SSL::SSLContext.instance_methods.include?(:alpn_protocols)
145
+ { alpn_protocols: %w[h2 http/1.1] }
146
+ else
147
+ {}
148
+ end
149
+
150
+ def initialize(_, _, options)
151
+ @ctx = OpenSSL::SSL::SSLContext.new
152
+ ctx_options = TLS_OPTIONS.merge(options.ssl)
153
+ @ctx.set_params(ctx_options) unless ctx_options.empty?
154
+ super
155
+ @state = :negotiated if @keep_open
156
+ end
157
+
158
+ def scheme
159
+ "https"
160
+ end
161
+
162
+ def protocol
163
+ @io.alpn_protocol || super
164
+ rescue StandardError
165
+ super
166
+ end
167
+
168
+ def close
169
+ super
170
+ # allow reconnections
171
+ # connect only works if initial @io is a socket
172
+ @io = @io.io if @io.respond_to?(:io)
173
+ @negotiated = false
174
+ end
175
+
176
+ def connect
177
+ super
178
+ if @keep_open
179
+ @state = :negotiated
180
+ return
181
+ end
182
+ return if @state == :negotiated ||
183
+ @state != :connected
184
+ @io = OpenSSL::SSL::SSLSocket.new(@io, @ctx)
185
+ @io.hostname = @hostname
186
+ @io.sync_close = true
187
+ @io.connect
188
+ transition(:negotiated)
189
+ end
190
+
191
+ if RUBY_VERSION < "2.3"
192
+ def read(*)
193
+ super
194
+ rescue ::IO::WaitWritable
195
+ 0
196
+ end
197
+
198
+ def write(*)
199
+ super
200
+ rescue ::IO::WaitReadable
201
+ 0
202
+ end
203
+ else
204
+ if OpenSSL::VERSION < "2.0.6"
205
+ def read(size, buffer)
206
+ @io.read_nonblock(size, buffer)
207
+ buffer.bytesize
208
+ rescue ::IO::WaitReadable,
209
+ ::IO::WaitWritable
210
+ 0
211
+ rescue EOFError
212
+ nil
213
+ end
214
+ end
215
+ end
216
+
217
+ def inspect
218
+ id = @io.closed? ? "closed" : @io.to_io.fileno
219
+ "#<SSL(fd: #{id}): #{@ip}:#{@port} state: #{@state}>"
220
+ end
221
+
222
+ private
223
+
224
+ def transition(nextstate)
225
+ case nextstate
226
+ when :negotiated
227
+ return unless @state == :connected
228
+ when :closed
229
+ return unless @state == :negotiated ||
230
+ @state == :connected
231
+ end
232
+ super
233
+ end
234
+ end
235
+ module IO
236
+ extend Registry
237
+ register "tcp", TCP
238
+ register "ssl", SSL
239
+ end
240
+ end
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HTTPX
4
+ module Loggable
5
+ def log(level = @options.debug_level, label = "", &msg)
6
+ return unless @options.debug
7
+ return unless @options.debug_level >= level
8
+ @options.debug << (+label << msg.call << "\n")
9
+ end
10
+ end
11
+ end
@@ -0,0 +1,138 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HTTPX
4
+ class Options
5
+ MAX_CONCURRENT_REQUESTS = 100
6
+ MAX_RETRIES = 3
7
+ WINDOW_SIZE = 1 << 14 # 16K
8
+ MAX_BODY_THRESHOLD_SIZE = (1 << 10) * 112 # 112K
9
+
10
+ class << self
11
+ def inherited(klass)
12
+ super
13
+ klass.instance_variable_set(:@defined_options, @defined_options.dup)
14
+ end
15
+
16
+ def new(options = {})
17
+ # let enhanced options go through
18
+ return options if self == Options && options.class > self
19
+ return options if options.is_a?(self)
20
+ super
21
+ end
22
+
23
+ def defined_options
24
+ @defined_options ||= []
25
+ end
26
+
27
+ def def_option(name, &interpreter)
28
+ defined_options << name.to_sym
29
+ interpreter ||= ->(v) { v }
30
+
31
+ attr_accessor name
32
+ protected :"#{name}="
33
+
34
+ define_method(:"with_#{name}") do |value|
35
+ dup { |opts| opts.send(:"#{name}=", instance_exec(value, &interpreter)) }
36
+ end
37
+ end
38
+ end
39
+
40
+ def initialize(options = {})
41
+ defaults = {
42
+ :debug => ENV.key?("HTTPX_DEBUG") ? $stderr : nil,
43
+ :debug_level => (ENV["HTTPX_DEBUG"] || 1).to_i,
44
+ :ssl => {},
45
+ :http2_settings => { settings_enable_push: 0 },
46
+ :fallback_protocol => "http/1.1",
47
+ :timeout => Timeout.new,
48
+ :headers => {},
49
+ :max_concurrent_requests => MAX_CONCURRENT_REQUESTS,
50
+ :max_retries => MAX_RETRIES,
51
+ :window_size => WINDOW_SIZE,
52
+ :body_threshold_size => MAX_BODY_THRESHOLD_SIZE,
53
+ :request_class => Class.new(Request),
54
+ :response_class => Class.new(Response),
55
+ :headers_class => Class.new(Headers),
56
+ :request_body_class => Class.new(Request::Body),
57
+ :response_body_class => Class.new(Response::Body),
58
+ }
59
+
60
+ defaults.merge!(options)
61
+ defaults[:headers] = Headers.new(defaults[:headers])
62
+ defaults.each { |(k, v)| self[k] = v }
63
+ end
64
+
65
+ def_option(:headers) do |headers|
66
+ self.headers.merge(headers)
67
+ end
68
+
69
+ def_option(:timeout) do |opts|
70
+ self.timeout = Timeout.new(opts)
71
+ end
72
+
73
+ def_option(:max_concurrent_requests) do |num|
74
+ max = Integer(num)
75
+ raise Error, ":max_concurrent_requests must be positive" unless max.positive?
76
+ self.max_concurrent_requests = max
77
+ end
78
+
79
+ def_option(:window_size) do |num|
80
+ self.window_size = Integer(num)
81
+ end
82
+
83
+ def_option(:body_threshold_size) do |num|
84
+ self.body_threshold_size = Integer(num)
85
+ end
86
+
87
+ %w[
88
+ params form json body
89
+ follow ssl http2_settings max_retries
90
+ request_class response_class headers_class request_body_class response_body_class
91
+ io fallback_protocol debug debug_level
92
+ ].each do |method_name|
93
+ def_option(method_name)
94
+ end
95
+
96
+ def merge(other)
97
+ h1 = to_hash
98
+ h2 = other.to_hash
99
+
100
+ merged = h1.merge(h2) do |k, v1, v2|
101
+ case k
102
+ when :headers, :ssl, :http2_settings, :timeout
103
+ v1.merge(v2)
104
+ else
105
+ v2
106
+ end
107
+ end
108
+
109
+ self.class.new(merged)
110
+ end
111
+
112
+ def to_hash
113
+ hash_pairs = self.class
114
+ .defined_options
115
+ .flat_map { |opt_name| [opt_name, send(opt_name)] }
116
+ Hash[*hash_pairs]
117
+ end
118
+
119
+ def dup
120
+ dupped = super
121
+ dupped.headers = headers.dup
122
+ dupped.ssl = ssl.dup
123
+ dupped.request_class = request_class.dup
124
+ dupped.response_class = response_class.dup
125
+ dupped.headers_class = headers_class.dup
126
+ dupped.request_body_class = request_body_class.dup
127
+ dupped.response_body_class = response_body_class.dup
128
+ yield(dupped) if block_given?
129
+ dupped
130
+ end
131
+
132
+ protected
133
+
134
+ def []=(option, val)
135
+ send(:"#{option}=", val)
136
+ end
137
+ end
138
+ end
@@ -0,0 +1,14 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HTTPX
4
+ module Plugins
5
+ module Authentication
6
+ module InstanceMethods
7
+ def authentication(token)
8
+ headers("authorization" => token)
9
+ end
10
+ end
11
+ end
12
+ register_plugin :authentication, Authentication
13
+ end
14
+ end
@@ -0,0 +1,20 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HTTPX
4
+ module Plugins
5
+ module BasicAuthentication
6
+ def self.load_dependencies(klass, *)
7
+ require "base64"
8
+ klass.plugin(:authentication)
9
+ end
10
+
11
+ module InstanceMethods
12
+ def basic_authentication(user, password)
13
+ authentication("Basic #{Base64.strict_encode64("#{user}:#{password}")}")
14
+ end
15
+ alias_method :basic_auth, :basic_authentication
16
+ end
17
+ end
18
+ register_plugin :basic_authentication, BasicAuthentication
19
+ end
20
+ end
@@ -0,0 +1,123 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HTTPX
4
+ module Plugins
5
+ module Compression
6
+ extend Registry
7
+ def self.configure(klass, *)
8
+ klass.plugin(:"compression/gzip")
9
+ klass.plugin(:"compression/deflate")
10
+ end
11
+
12
+ module InstanceMethods
13
+ def initialize(opts = {})
14
+ super(opts.merge(headers: { "accept-encoding" => Compression.registry.keys }))
15
+ end
16
+ end
17
+
18
+ module RequestBodyMethods
19
+ def initialize(*)
20
+ super
21
+ return if @body.nil?
22
+ @headers.get("content-encoding").each do |encoding|
23
+ @body = Encoder.new(@body, Compression.registry(encoding).encoder)
24
+ end
25
+ @headers["content-length"] = @body.bytesize unless chunked?
26
+ end
27
+ end
28
+
29
+ module ResponseBodyMethods
30
+ def initialize(*)
31
+ super
32
+ @_decoders = @headers.get("content-encoding").map do |encoding|
33
+ Compression.registry(encoding).decoder
34
+ end
35
+ @_compressed_length = if @headers.key?("content-length")
36
+ @headers["content-length"].to_i
37
+ else
38
+ Float::INFINITY
39
+ end
40
+ end
41
+
42
+ def write(chunk)
43
+ @_compressed_length -= chunk.bytesize
44
+ chunk = decompress(chunk)
45
+ super(chunk)
46
+ end
47
+
48
+ def close
49
+ super
50
+ @_decoders.each(&:close)
51
+ end
52
+
53
+ private
54
+
55
+ def decompress(buffer)
56
+ @_decoders.reverse_each do |decoder|
57
+ buffer = decoder.decode(buffer)
58
+ buffer << decoder.finish if @_compressed_length <= 0
59
+ end
60
+ buffer
61
+ end
62
+ end
63
+
64
+ class Encoder
65
+ def initialize(body, deflater)
66
+ @body = body.respond_to?(:read) ? body : StringIO.new(body.to_s)
67
+ @buffer = StringIO.new("".b, File::RDWR)
68
+ @deflater = deflater
69
+ end
70
+
71
+ def each(&blk)
72
+ return enum_for(__method__) unless block_given?
73
+ unless @buffer.size.zero?
74
+ @buffer.rewind
75
+ return @buffer.each(&blk)
76
+ end
77
+ deflate(&blk)
78
+ end
79
+
80
+ def bytesize
81
+ deflate
82
+ @buffer.size
83
+ end
84
+
85
+ def to_s
86
+ deflate
87
+ @buffer.rewind
88
+ @buffer.read
89
+ end
90
+
91
+ def close
92
+ @buffer.close
93
+ @body.close
94
+ end
95
+
96
+ private
97
+
98
+ def deflate(&blk)
99
+ return unless @buffer.size.zero?
100
+ @body.rewind
101
+ @deflater.deflate(@body, @buffer, chunk_size: 16_384, &blk)
102
+ end
103
+ end
104
+
105
+ class Decoder
106
+ extend Forwardable
107
+
108
+ def_delegator :@inflater, :finish
109
+
110
+ def_delegator :@inflater, :close
111
+
112
+ def initialize(inflater)
113
+ @inflater = inflater
114
+ end
115
+
116
+ def decode(chunk)
117
+ @inflater.inflate(chunk)
118
+ end
119
+ end
120
+ end
121
+ register_plugin :compression, Compression
122
+ end
123
+ end