httpx 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/lib/httpx.rb +8 -2
  3. data/lib/httpx/adapters/faraday.rb +203 -0
  4. data/lib/httpx/altsvc.rb +4 -0
  5. data/lib/httpx/callbacks.rb +1 -4
  6. data/lib/httpx/chainable.rb +4 -3
  7. data/lib/httpx/connection.rb +326 -104
  8. data/lib/httpx/{channel → connection}/http1.rb +29 -15
  9. data/lib/httpx/{channel → connection}/http2.rb +12 -6
  10. data/lib/httpx/errors.rb +2 -0
  11. data/lib/httpx/headers.rb +4 -1
  12. data/lib/httpx/io/ssl.rb +5 -1
  13. data/lib/httpx/io/tcp.rb +13 -7
  14. data/lib/httpx/io/udp.rb +1 -0
  15. data/lib/httpx/io/unix.rb +1 -0
  16. data/lib/httpx/loggable.rb +34 -9
  17. data/lib/httpx/options.rb +57 -31
  18. data/lib/httpx/parser/http1.rb +8 -0
  19. data/lib/httpx/plugins/authentication.rb +4 -0
  20. data/lib/httpx/plugins/basic_authentication.rb +4 -0
  21. data/lib/httpx/plugins/compression.rb +22 -5
  22. data/lib/httpx/plugins/cookies.rb +89 -36
  23. data/lib/httpx/plugins/digest_authentication.rb +45 -26
  24. data/lib/httpx/plugins/follow_redirects.rb +61 -62
  25. data/lib/httpx/plugins/h2c.rb +78 -39
  26. data/lib/httpx/plugins/multipart.rb +5 -0
  27. data/lib/httpx/plugins/persistent.rb +29 -0
  28. data/lib/httpx/plugins/proxy.rb +125 -78
  29. data/lib/httpx/plugins/proxy/http.rb +31 -27
  30. data/lib/httpx/plugins/proxy/socks4.rb +30 -24
  31. data/lib/httpx/plugins/proxy/socks5.rb +49 -39
  32. data/lib/httpx/plugins/proxy/ssh.rb +81 -0
  33. data/lib/httpx/plugins/push_promise.rb +18 -9
  34. data/lib/httpx/plugins/retries.rb +43 -15
  35. data/lib/httpx/pool.rb +159 -0
  36. data/lib/httpx/registry.rb +2 -0
  37. data/lib/httpx/request.rb +10 -0
  38. data/lib/httpx/resolver.rb +2 -1
  39. data/lib/httpx/resolver/https.rb +62 -56
  40. data/lib/httpx/resolver/native.rb +48 -37
  41. data/lib/httpx/resolver/resolver_mixin.rb +16 -11
  42. data/lib/httpx/resolver/system.rb +11 -7
  43. data/lib/httpx/response.rb +24 -10
  44. data/lib/httpx/selector.rb +32 -39
  45. data/lib/httpx/{client.rb → session.rb} +99 -62
  46. data/lib/httpx/timeout.rb +7 -15
  47. data/lib/httpx/transcoder/body.rb +4 -0
  48. data/lib/httpx/transcoder/chunker.rb +4 -0
  49. data/lib/httpx/version.rb +1 -1
  50. metadata +10 -8
  51. data/lib/httpx/channel.rb +0 -367
@@ -6,53 +6,56 @@ module HTTPX
6
6
  module Plugins
7
7
  module Proxy
8
8
  module HTTP
9
- class HTTPProxyChannel < ProxyChannel
9
+ module ConnectionMethods
10
10
  private
11
11
 
12
- def proxy_connect
13
- req, _ = @pending.first
14
- # if the first request after CONNECT is to an https address, it is assumed that
15
- # all requests in the queue are not only ALL HTTPS, but they also share the certificate,
16
- # and therefore, will share the connection.
17
- #
18
- if req.uri.scheme == "https"
19
- connect_request = ConnectRequest.new(req.uri)
20
- if @parameters.authenticated?
21
- connect_request.headers["proxy-authentication"] = "Basic #{@parameters.token_authentication}"
22
- end
23
- parser.send(connect_request)
24
- else
25
- transition(:connected)
26
- end
27
- end
28
-
29
12
  def transition(nextstate)
13
+ return super unless @options.proxy && @options.proxy.uri.scheme == "http"
14
+
30
15
  case nextstate
31
16
  when :connecting
32
17
  return unless @state == :idle
18
+
33
19
  @io.connect
34
20
  return unless @io.connected?
21
+
35
22
  @parser = ConnectProxyParser.new(@write_buffer, @options.merge(max_concurrent_requests: 1))
36
- @parser.once(:response, &method(:on_connect))
23
+ @parser.once(:response, &method(:__http_on_connect))
37
24
  @parser.on(:close) { transition(:closing) }
38
- proxy_connect
25
+ __http_proxy_connect
39
26
  return if @state == :connected
40
27
  when :connected
41
28
  return unless @state == :idle || @state == :connecting
29
+
42
30
  case @state
43
31
  when :connecting
44
32
  @parser.close
45
33
  @parser = nil
46
34
  when :idle
47
35
  @parser = ProxyParser.new(@write_buffer, @options)
48
- @parser.inherit_callbacks(self)
36
+ set_parser_callbacks(@parser)
49
37
  @parser.on(:close) { transition(:closing) }
50
38
  end
51
39
  end
52
40
  super
53
41
  end
54
42
 
55
- def on_connect(_request, response)
43
+ def __http_proxy_connect
44
+ req, _ = @pending.first
45
+ # if the first request after CONNECT is to an https address, it is assumed that
46
+ # all requests in the queue are not only ALL HTTPS, but they also share the certificate,
47
+ # and therefore, will share the connection.
48
+ #
49
+ if req.uri.scheme == "https"
50
+ connect_request = ConnectRequest.new(req.uri, @options)
51
+
52
+ parser.send(connect_request)
53
+ else
54
+ transition(:connected)
55
+ end
56
+ end
57
+
58
+ def __http_on_connect(_request, response)
56
59
  if response.status == 200
57
60
  req, _ = @pending.first
58
61
  request_uri = req.uri
@@ -68,7 +71,7 @@ module HTTPX
68
71
  end
69
72
  end
70
73
 
71
- class ProxyParser < Channel::HTTP1
74
+ class ProxyParser < Connection::HTTP1
72
75
  def headline_uri(request)
73
76
  request.uri.to_s
74
77
  end
@@ -85,6 +88,7 @@ module HTTPX
85
88
 
86
89
  def headline_uri(request)
87
90
  return super unless request.verb == :connect
91
+
88
92
  uri = request.uri
89
93
  tunnel = "#{uri.hostname}:#{uri.port}"
90
94
  log { "establishing HTTP proxy tunnel to #{tunnel}" }
@@ -98,8 +102,10 @@ module HTTPX
98
102
  end
99
103
 
100
104
  class ConnectRequest < Request
101
- def initialize(uri, options = {})
102
- super(:connect, uri, options)
105
+ def initialize(uri, options)
106
+ super(:connect, uri, {})
107
+ proxy_params = options.proxy
108
+ @headers["proxy-authentication"] = "Basic #{proxy_params.token_authentication}" if proxy_params.authenticated?
103
109
  @headers.delete("accept")
104
110
  end
105
111
 
@@ -107,8 +113,6 @@ module HTTPX
107
113
  "#{@uri.hostname}:#{@uri.port}"
108
114
  end
109
115
  end
110
-
111
- Parameters.register("http", HTTPProxyChannel)
112
116
  end
113
117
  end
114
118
  register_plugin :"proxy/http", Proxy::HTTP
@@ -10,58 +10,63 @@ module HTTPX
10
10
  VERSION = 4
11
11
  CONNECT = 1
12
12
  GRANTED = 90
13
+ PROTOCOLS = %w[socks4 socks4a].freeze
13
14
 
14
15
  Error = Class.new(Error)
15
16
 
16
- class Socks4ProxyChannel < ProxyChannel
17
+ module ConnectionMethods
17
18
  private
18
19
 
19
- def proxy_connect
20
- @parser = SocksParser.new(@write_buffer, @options)
21
- @parser.once(:packet, &method(:on_packet))
22
- end
23
-
24
- def on_packet(packet)
25
- _version, status, _port, _ip = packet.unpack("CCnN")
26
- if status == GRANTED
27
- req, _ = @pending.first
28
- request_uri = req.uri
29
- @io = ProxySSL.new(@io, request_uri, @options) if request_uri.scheme == "https"
30
- transition(:connected)
31
- throw(:called)
32
- else
33
- on_socks_error("socks error: #{status}")
34
- end
35
- end
36
-
37
20
  def transition(nextstate)
21
+ return super unless @options.proxy && PROTOCOLS.include?(@options.proxy.uri.scheme)
22
+
38
23
  case nextstate
39
24
  when :connecting
40
25
  return unless @state == :idle
26
+
41
27
  @io.connect
42
28
  return unless @io.connected?
29
+
43
30
  req, _ = @pending.first
44
31
  return unless req
32
+
45
33
  request_uri = req.uri
46
- @write_buffer << Packet.connect(@parameters, request_uri)
47
- proxy_connect
34
+ @write_buffer << Packet.connect(@options.proxy, request_uri)
35
+ __socks4_proxy_connect
48
36
  when :connected
49
37
  return unless @state == :connecting
38
+
50
39
  @parser = nil
51
40
  end
52
41
  log(level: 1, label: "SOCKS4: ") { "#{nextstate}: #{@write_buffer.to_s.inspect}" } unless nextstate == :open
53
42
  super
54
43
  end
55
44
 
56
- def on_socks_error(message)
45
+ def __socks4_proxy_connect
46
+ @parser = SocksParser.new(@write_buffer, @options)
47
+ @parser.once(:packet, &method(:__socks4_on_packet))
48
+ end
49
+
50
+ def __socks4_on_packet(packet)
51
+ _version, status, _port, _ip = packet.unpack("CCnN")
52
+ if status == GRANTED
53
+ req, _ = @pending.first
54
+ request_uri = req.uri
55
+ @io = ProxySSL.new(@io, request_uri, @options) if request_uri.scheme == "https"
56
+ transition(:connected)
57
+ throw(:called)
58
+ else
59
+ on_socks4_error("socks error: #{status}")
60
+ end
61
+ end
62
+
63
+ def on_socks4_error(message)
57
64
  ex = Error.new(message)
58
65
  ex.set_backtrace(caller)
59
66
  on_error(ex)
60
67
  throw(:called)
61
68
  end
62
69
  end
63
- Parameters.register("socks4", Socks4ProxyChannel)
64
- Parameters.register("socks4a", Socks4ProxyChannel)
65
70
 
66
71
  class SocksParser
67
72
  include Callbacks
@@ -92,6 +97,7 @@ module HTTPX
92
97
  begin
93
98
  ip = IPAddr.new(uri.host)
94
99
  raise Error, "Socks4 connection to #{ip} not supported" unless ip.ipv4?
100
+
95
101
  packet << [ip.to_i].pack("N")
96
102
  rescue IPAddr::InvalidAddressError
97
103
  if parameters.uri.scheme == "socks4"
@@ -16,9 +16,12 @@ module HTTPX
16
16
 
17
17
  Error = Class.new(Error)
18
18
 
19
- class Socks5ProxyChannel < ProxyChannel
19
+ module ConnectionMethods
20
20
  def call
21
21
  super
22
+
23
+ return unless @options.proxy && @options.proxy.uri.scheme == "socks5"
24
+
22
25
  case @state
23
26
  when :connecting,
24
27
  :negotiating,
@@ -29,35 +32,67 @@ module HTTPX
29
32
 
30
33
  private
31
34
 
32
- def proxy_connect
35
+ def transition(nextstate)
36
+ return super unless @options.proxy && @options.proxy.uri.scheme == "socks5"
37
+
38
+ case nextstate
39
+ when :connecting
40
+ return unless @state == :idle
41
+
42
+ @io.connect
43
+ return unless @io.connected?
44
+
45
+ @write_buffer << Packet.negotiate(@options.proxy)
46
+ __socks5_proxy_connect
47
+ when :authenticating
48
+ return unless @state == :connecting
49
+
50
+ @write_buffer << Packet.authenticate(@options.proxy)
51
+ when :negotiating
52
+ return unless @state == :connecting || @state == :authenticating
53
+
54
+ req, _ = @pending.first
55
+ request_uri = req.uri
56
+ @write_buffer << Packet.connect(request_uri)
57
+ when :connected
58
+ return unless @state == :negotiating
59
+
60
+ @parser = nil
61
+ end
62
+ log(level: 1, label: "SOCKS5: ") { "#{nextstate}: #{@write_buffer.to_s.inspect}" } unless nextstate == :open
63
+ super
64
+ end
65
+
66
+ def __socks5_proxy_connect
33
67
  @parser = SocksParser.new(@write_buffer, @options)
34
- @parser.on(:packet, &method(:on_packet))
68
+ @parser.on(:packet, &method(:__socks5_on_packet))
35
69
  transition(:negotiating)
36
70
  end
37
71
 
38
- def on_packet(packet)
72
+ def __socks5_on_packet(packet)
39
73
  case @state
40
74
  when :connecting
41
75
  version, method = packet.unpack("CC")
42
- check_version(version)
76
+ __socks5_check_version(version)
43
77
  case method
44
78
  when PASSWD
45
79
  transition(:authenticating)
46
80
  return
47
81
  when NONE
48
- on_socks_error("no supported authorization methods")
82
+ __on_socks5_error("no supported authorization methods")
49
83
  else
50
84
  transition(:negotiating)
51
85
  end
52
86
  when :authenticating
53
87
  version, status = packet.unpack("CC")
54
- check_version(version)
88
+ __socks5_check_version(version)
55
89
  return transition(:negotiating) if status == SUCCESS
56
- on_socks_error("socks authentication error: #{status}")
90
+
91
+ __on_socks5_error("socks authentication error: #{status}")
57
92
  when :negotiating
58
93
  version, reply, = packet.unpack("CC")
59
- check_version(version)
60
- on_socks_error("socks5 negotiation error: #{reply}") unless reply == SUCCESS
94
+ __socks5_check_version(version)
95
+ __on_socks5_error("socks5 negotiation error: #{reply}") unless reply == SUCCESS
61
96
  req, _ = @pending.first
62
97
  request_uri = req.uri
63
98
  @io = ProxySSL.new(@io, request_uri, @options) if request_uri.scheme == "https"
@@ -66,35 +101,11 @@ module HTTPX
66
101
  end
67
102
  end
68
103
 
69
- def transition(nextstate)
70
- case nextstate
71
- when :connecting
72
- return unless @state == :idle
73
- @io.connect
74
- return unless @io.connected?
75
- @write_buffer << Packet.negotiate(@parameters)
76
- proxy_connect
77
- when :authenticating
78
- return unless @state == :connecting
79
- @write_buffer << Packet.authenticate(@parameters)
80
- when :negotiating
81
- return unless @state == :connecting || @state == :authenticating
82
- req, _ = @pending.first
83
- request_uri = req.uri
84
- @write_buffer << Packet.connect(request_uri)
85
- when :connected
86
- return unless @state == :negotiating
87
- @parser = nil
88
- end
89
- log(level: 1, label: "SOCKS5: ") { "#{nextstate}: #{@write_buffer.to_s.inspect}" } unless nextstate == :open
90
- super
91
- end
92
-
93
- def check_version(version)
94
- on_socks_error("invalid SOCKS version (#{version})") if version != 5
104
+ def __socks5_check_version(version)
105
+ __on_socks5_error("invalid SOCKS version (#{version})") if version != 5
95
106
  end
96
107
 
97
- def on_socks_error(message)
108
+ def __on_socks5_error(message)
98
109
  ex = Error.new(message)
99
110
  ex.set_backtrace(caller)
100
111
  on_error(ex)
@@ -102,8 +113,6 @@ module HTTPX
102
113
  end
103
114
  end
104
115
 
105
- Parameters.register("socks5", Socks5ProxyChannel)
106
-
107
116
  class SocksParser
108
117
  include Callbacks
109
118
 
@@ -147,6 +156,7 @@ module HTTPX
147
156
  begin
148
157
  ip = IPAddr.new(uri.host)
149
158
  raise Error, "Socks4 connection to #{ip} not supported" unless ip.ipv4?
159
+
150
160
  packet << [IPV4, ip.to_i].pack("CN")
151
161
  rescue IPAddr::InvalidAddressError
152
162
  packet << [DOMAIN, uri.host.bytesize, uri.host].pack("CCA*")
@@ -0,0 +1,81 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "httpx/plugins/proxy"
4
+
5
+ module HTTPX
6
+ module Plugins
7
+ module Proxy
8
+ module SSH
9
+ def self.load_dependencies(_klass, *)
10
+ # klass.plugin(:proxy)
11
+ require "net/ssh/gateway"
12
+ end
13
+
14
+ def self.extra_options(options)
15
+ Class.new(options.class) do
16
+ def_option(:proxy) do |pr|
17
+ Hash[pr]
18
+ end
19
+ end.new(options)
20
+ end
21
+
22
+ module InstanceMethods
23
+ def with_proxy(*args)
24
+ branch(default_options.with_proxy(*args))
25
+ end
26
+
27
+ private
28
+
29
+ def send_requests(*requests, options)
30
+ request_options = @options.merge(options)
31
+
32
+ return super unless request_options.proxy
33
+
34
+ ssh_options = request_options.proxy
35
+ ssh_uris = ssh_options.delete(:uri)
36
+ ssh_uri = URI.parse(ssh_uris.shift)
37
+
38
+ return super unless ssh_uri.scheme == "ssh"
39
+
40
+ ssh_username = ssh_options.delete(:username)
41
+ ssh_options[:port] ||= ssh_uri.port || 22
42
+ if request_options.debug
43
+ ssh_options[:verbose] = request_options.debug_level == 2 ? :debug : :info
44
+ end
45
+ request_uri = URI(requests.first.uri)
46
+ @_gateway = Net::SSH::Gateway.new(ssh_uri.host, ssh_username, ssh_options)
47
+ begin
48
+ @_gateway.open(request_uri.host, request_uri.port) do |local_port|
49
+ io = build_gateway_socket(local_port, request_uri, request_options)
50
+ super(*requests, options.merge(io: io))
51
+ end
52
+ ensure
53
+ @_gateway.shutdown!
54
+ end
55
+ end
56
+
57
+ def build_gateway_socket(port, request_uri, options)
58
+ case request_uri.scheme
59
+ when "https"
60
+ ctx = OpenSSL::SSL::SSLContext.new
61
+ ctx_options = SSL::TLS_OPTIONS.merge(options.ssl)
62
+ ctx.set_params(ctx_options) unless ctx_options.empty?
63
+ sock = TCPSocket.open("localhost", port)
64
+ io = OpenSSL::SSL::SSLSocket.new(sock, ctx)
65
+ io.hostname = request_uri.host
66
+ io.sync_close = true
67
+ io.connect
68
+ io.post_connection_check(request_uri.host) if ctx.verify_mode != OpenSSL::SSL::VERIFY_NONE
69
+ io
70
+ when "http"
71
+ TCPSocket.open("localhost", port)
72
+ else
73
+ raise Error, "unexpected scheme: #{request_uri.scheme}"
74
+ end
75
+ end
76
+ end
77
+ end
78
+ end
79
+ register_plugin :"proxy/ssh", Proxy::SSH
80
+ end
81
+ end
@@ -2,9 +2,17 @@
2
2
 
3
3
  module HTTPX
4
4
  module Plugins
5
+ #
6
+ # This plugin adds support for HTTP/2 Push responses.
7
+ #
8
+ # In order to benefit from this, requests are sent one at a time, so that
9
+ # no push responses are received after corresponding request has been sent.
10
+ #
5
11
  module PushPromise
6
- PUSH_OPTIONS = { http2_settings: { settings_enable_push: 1 },
7
- max_concurrent_requests: 1 }.freeze
12
+ def self.extra_options(options)
13
+ options.merge(http2_settings: { settings_enable_push: 1 },
14
+ max_concurrent_requests: 1)
15
+ end
8
16
 
9
17
  module ResponseMethods
10
18
  def pushed?
@@ -17,13 +25,12 @@ module HTTPX
17
25
  end
18
26
 
19
27
  module InstanceMethods
20
- def initialize(opts = {})
21
- super(PUSH_OPTIONS.merge(opts))
22
- @promise_headers = {}
23
- end
24
-
25
28
  private
26
29
 
30
+ def promise_headers
31
+ @promise_headers ||= {}
32
+ end
33
+
27
34
  def on_promise(parser, stream)
28
35
  stream.on(:promise_headers) do |h|
29
36
  __on_promise_request(parser, stream, h)
@@ -40,10 +47,11 @@ module HTTPX
40
47
  headers = @options.headers_class.new(h)
41
48
  path = headers[":path"]
42
49
  authority = headers[":authority"]
50
+
43
51
  request = parser.pending.find { |r| r.authority == authority && r.path == path }
44
52
  if request
45
53
  request.merge_headers(headers)
46
- @promise_headers[stream] = request
54
+ promise_headers[stream] = request
47
55
  parser.pending.delete(request)
48
56
  else
49
57
  stream.refuse
@@ -51,8 +59,9 @@ module HTTPX
51
59
  end
52
60
 
53
61
  def __on_promise_response(parser, stream, h)
54
- request = @promise_headers.delete(stream)
62
+ request = promise_headers.delete(stream)
55
63
  return unless request
64
+
56
65
  parser.__send__(:on_stream_headers, stream, request, h)
57
66
  request.transition(:done)
58
67
  response = request.response