httpx 0.11.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +2 -2
  3. data/doc/release_notes/0_11_1.md +5 -0
  4. data/doc/release_notes/0_11_2.md +5 -0
  5. data/doc/release_notes/0_11_3.md +5 -0
  6. data/doc/release_notes/0_12_0.md +55 -0
  7. data/doc/release_notes/0_13_0.md +58 -0
  8. data/lib/httpx.rb +2 -1
  9. data/lib/httpx/adapters/faraday.rb +4 -6
  10. data/lib/httpx/altsvc.rb +1 -0
  11. data/lib/httpx/chainable.rb +2 -2
  12. data/lib/httpx/connection.rb +80 -28
  13. data/lib/httpx/connection/http1.rb +19 -6
  14. data/lib/httpx/connection/http2.rb +32 -25
  15. data/lib/httpx/io.rb +16 -3
  16. data/lib/httpx/io/ssl.rb +35 -24
  17. data/lib/httpx/io/tcp.rb +48 -28
  18. data/lib/httpx/io/tls.rb +218 -0
  19. data/lib/httpx/io/tls/box.rb +365 -0
  20. data/lib/httpx/io/tls/context.rb +199 -0
  21. data/lib/httpx/io/tls/ffi.rb +390 -0
  22. data/lib/httpx/io/udp.rb +3 -2
  23. data/lib/httpx/io/unix.rb +27 -12
  24. data/lib/httpx/options.rb +11 -23
  25. data/lib/httpx/parser/http1.rb +4 -4
  26. data/lib/httpx/plugins/aws_sdk_authentication.rb +81 -0
  27. data/lib/httpx/plugins/aws_sigv4.rb +218 -0
  28. data/lib/httpx/plugins/compression.rb +21 -9
  29. data/lib/httpx/plugins/compression/brotli.rb +8 -6
  30. data/lib/httpx/plugins/compression/deflate.rb +4 -7
  31. data/lib/httpx/plugins/compression/gzip.rb +2 -2
  32. data/lib/httpx/plugins/cookies/set_cookie_parser.rb +1 -1
  33. data/lib/httpx/plugins/digest_authentication.rb +1 -1
  34. data/lib/httpx/plugins/follow_redirects.rb +1 -1
  35. data/lib/httpx/plugins/h2c.rb +43 -58
  36. data/lib/httpx/plugins/internal_telemetry.rb +93 -0
  37. data/lib/httpx/plugins/multipart.rb +2 -0
  38. data/lib/httpx/plugins/multipart/encoder.rb +4 -9
  39. data/lib/httpx/plugins/proxy.rb +1 -1
  40. data/lib/httpx/plugins/proxy/http.rb +1 -1
  41. data/lib/httpx/plugins/proxy/socks4.rb +8 -0
  42. data/lib/httpx/plugins/proxy/socks5.rb +8 -0
  43. data/lib/httpx/plugins/push_promise.rb +3 -2
  44. data/lib/httpx/plugins/retries.rb +2 -2
  45. data/lib/httpx/plugins/stream.rb +6 -6
  46. data/lib/httpx/plugins/upgrade.rb +83 -0
  47. data/lib/httpx/plugins/upgrade/h2.rb +54 -0
  48. data/lib/httpx/pool.rb +14 -6
  49. data/lib/httpx/registry.rb +1 -7
  50. data/lib/httpx/request.rb +11 -1
  51. data/lib/httpx/resolver/https.rb +3 -11
  52. data/lib/httpx/response.rb +14 -7
  53. data/lib/httpx/selector.rb +5 -0
  54. data/lib/httpx/session.rb +25 -2
  55. data/lib/httpx/transcoder/body.rb +3 -5
  56. data/lib/httpx/version.rb +1 -1
  57. data/sig/chainable.rbs +2 -1
  58. data/sig/connection/http1.rbs +3 -2
  59. data/sig/connection/http2.rbs +5 -3
  60. data/sig/options.rbs +7 -20
  61. data/sig/plugins/aws_sdk_authentication.rbs +17 -0
  62. data/sig/plugins/aws_sigv4.rbs +64 -0
  63. data/sig/plugins/compression.rbs +5 -3
  64. data/sig/plugins/compression/brotli.rbs +1 -1
  65. data/sig/plugins/compression/deflate.rbs +1 -1
  66. data/sig/plugins/compression/gzip.rbs +1 -1
  67. data/sig/plugins/cookies.rbs +0 -1
  68. data/sig/plugins/digest_authentication.rbs +0 -1
  69. data/sig/plugins/expect.rbs +0 -2
  70. data/sig/plugins/follow_redirects.rbs +0 -2
  71. data/sig/plugins/h2c.rbs +5 -10
  72. data/sig/plugins/persistent.rbs +0 -1
  73. data/sig/plugins/proxy.rbs +0 -1
  74. data/sig/plugins/push_promise.rbs +1 -1
  75. data/sig/plugins/retries.rbs +0 -4
  76. data/sig/plugins/upgrade.rbs +23 -0
  77. data/sig/response.rbs +3 -1
  78. metadata +48 -26
@@ -4,13 +4,15 @@ module HTTPX
4
4
  module Plugins
5
5
  module Compression
6
6
  module Brotli
7
- def self.load_dependencies(klass)
8
- klass.plugin(:compression)
9
- require "brotli"
10
- end
7
+ class << self
8
+ def load_dependencies(klass)
9
+ klass.plugin(:compression)
10
+ require "brotli"
11
+ end
11
12
 
12
- def self.configure(*)
13
- Compression.register "br", self
13
+ def configure(klass)
14
+ klass.default_options.encodings.register "br", self
15
+ end
14
16
  end
15
17
 
16
18
  module Deflater
@@ -10,18 +10,15 @@ module HTTPX
10
10
  klass.plugin(:"compression/gzip")
11
11
  end
12
12
 
13
- def self.configure(*)
14
- Compression.register "deflate", self
13
+ def self.configure(klass)
14
+ klass.default_options.encodings.register "deflate", self
15
15
  end
16
16
 
17
17
  module Deflater
18
18
  module_function
19
19
 
20
20
  def deflate(raw, buffer, chunk_size:)
21
- deflater = Zlib::Deflate.new(Zlib::BEST_COMPRESSION,
22
- Zlib::MAX_WBITS,
23
- Zlib::MAX_MEM_LEVEL,
24
- Zlib::HUFFMAN_ONLY)
21
+ deflater = Zlib::Deflate.new
25
22
  while (chunk = raw.read(chunk_size))
26
23
  compressed = deflater.deflate(chunk)
27
24
  buffer << compressed
@@ -31,7 +28,7 @@ module HTTPX
31
28
  buffer << last
32
29
  yield last if block_given?
33
30
  ensure
34
- deflater.close
31
+ deflater.close if deflater
35
32
  end
36
33
  end
37
34
 
@@ -10,8 +10,8 @@ module HTTPX
10
10
  require "zlib"
11
11
  end
12
12
 
13
- def self.configure(*)
14
- Compression.register "gzip", self
13
+ def self.configure(klass)
14
+ klass.default_options.encodings.register "gzip", self
15
15
  end
16
16
 
17
17
  class Deflater
@@ -107,7 +107,7 @@ module HTTPX
107
107
  case aname
108
108
  when "expires"
109
109
  # RFC 6265 5.2.1
110
- (avalue &&= Time.httpdate(avalue)) || next
110
+ (avalue &&= Time.parse(avalue)) || next
111
111
  when "max-age"
112
112
  # RFC 6265 5.2.2
113
113
  next unless /\A-?\d+\z/.match?(avalue)
@@ -29,7 +29,7 @@ module HTTPX
29
29
 
30
30
  module InstanceMethods
31
31
  def digest_authentication(user, password)
32
- branch(default_options.with_digest(Digest.new(user, password)))
32
+ with(digest: Digest.new(user, password))
33
33
  end
34
34
 
35
35
  alias_method :digest_auth, :digest_authentication
@@ -32,7 +32,7 @@ module HTTPX
32
32
 
33
33
  module InstanceMethods
34
34
  def max_redirects(n)
35
- branch(default_options.with_max_redirects(n.to_i))
35
+ with(max_redirects: n.to_i)
36
36
  end
37
37
 
38
38
  private
@@ -6,68 +6,53 @@ module HTTPX
6
6
  # This plugin adds support for upgrading a plaintext HTTP/1.1 connection to HTTP/2
7
7
  # (https://tools.ietf.org/html/rfc7540#section-3.2)
8
8
  #
9
- # https://gitlab.com/honeyryderchuck/httpx/wikis/Follow-Redirects
9
+ # https://gitlab.com/honeyryderchuck/httpx/wikis/Upgrade#h2c
10
10
  #
11
11
  module H2C
12
- def self.load_dependencies(*)
13
- require "base64"
12
+ VALID_H2C_VERBS = %i[get options head].freeze
13
+
14
+ class << self
15
+ def load_dependencies(*)
16
+ require "base64"
17
+ end
18
+
19
+ def configure(klass)
20
+ klass.plugin(:upgrade)
21
+ klass.default_options.upgrade_handlers.register "h2c", self
22
+ end
23
+
24
+ def call(connection, request, response)
25
+ connection.upgrade_to_h2c(request, response)
26
+ end
14
27
  end
15
28
 
16
29
  module InstanceMethods
17
- def request(*args, **options)
18
- h2c_options = options.merge(fallback_protocol: "h2c")
30
+ def send_requests(*requests, options)
31
+ upgrade_request, *remainder = requests
32
+
33
+ return super unless VALID_H2C_VERBS.include?(upgrade_request.verb) && upgrade_request.scheme == "http"
19
34
 
20
- requests = build_requests(*args, h2c_options)
35
+ connection = pool.find_connection(upgrade_request.uri, @options.merge(options))
21
36
 
22
- upgrade_request = requests.first
23
- return super unless valid_h2c_upgrade_request?(upgrade_request)
37
+ return super if connection && connection.upgrade_protocol == :h2c
24
38
 
39
+ # build upgrade request
25
40
  upgrade_request.headers.add("connection", "upgrade")
26
41
  upgrade_request.headers.add("connection", "http2-settings")
27
42
  upgrade_request.headers["upgrade"] = "h2c"
28
43
  upgrade_request.headers["http2-settings"] = HTTP2Next::Client.settings_header(upgrade_request.options.http2_settings)
29
- wrap { send_requests(*upgrade_request, h2c_options).first }
30
44
 
31
- responses = send_requests(*requests, h2c_options)
32
-
33
- responses.size == 1 ? responses.first : responses
34
- end
35
-
36
- private
37
-
38
- def fetch_response(request, connections, options)
39
- response = super
40
- if response && valid_h2c_upgrade?(request, response, options)
41
- log { "upgrading to h2c..." }
42
- connection = find_connection(request, connections, options)
43
- connections << connection unless connections.include?(connection)
44
- connection.upgrade(request, response)
45
- end
46
- response
47
- end
48
-
49
- VALID_H2C_METHODS = %i[get options head].freeze
50
- private_constant :VALID_H2C_METHODS
51
-
52
- def valid_h2c_upgrade_request?(request)
53
- VALID_H2C_METHODS.include?(request.verb) &&
54
- request.scheme == "http"
55
- end
56
-
57
- def valid_h2c_upgrade?(request, response, options)
58
- options.fallback_protocol == "h2c" &&
59
- request.headers.get("connection").include?("upgrade") &&
60
- request.headers.get("upgrade").include?("h2c") &&
61
- response.status == 101
45
+ super(upgrade_request, *remainder, options.merge(max_concurrent_requests: 1))
62
46
  end
63
47
  end
64
48
 
65
49
  class H2CParser < Connection::HTTP2
66
50
  def upgrade(request, response)
67
- @connection.send_connection_preface
68
51
  # skip checks, it is assumed that this is the first
69
52
  # request in the connection
70
53
  stream = @connection.upgrade
54
+
55
+ # on_settings
71
56
  handle_stream(stream, request)
72
57
  @streams[request] = stream
73
58
 
@@ -81,29 +66,29 @@ module HTTPX
81
66
  module ConnectionMethods
82
67
  using URIExtensions
83
68
 
84
- def match?(uri, options)
85
- return super unless uri.scheme == "http" && @options.fallback_protocol == "h2c"
86
-
87
- super && options.fallback_protocol == "h2c"
88
- end
89
-
90
- def coalescable?(connection)
91
- return super unless @options.fallback_protocol == "h2c" && @origin.scheme == "http"
69
+ def upgrade_to_h2c(request, response)
70
+ prev_parser = @parser
92
71
 
93
- @origin == connection.origin && connection.options.fallback_protocol == "h2c"
94
- end
72
+ if prev_parser
73
+ prev_parser.reset
74
+ @inflight -= prev_parser.requests.size
75
+ end
95
76
 
96
- def upgrade(request, response)
97
- @parser.reset if @parser
98
- @parser = H2CParser.new(@write_buffer, @options)
77
+ parser_options = @options.merge(max_concurrent_requests: request.options.max_concurrent_requests)
78
+ @parser = H2CParser.new(@write_buffer, parser_options)
99
79
  set_parser_callbacks(@parser)
80
+ @inflight += 1
100
81
  @parser.upgrade(request, response)
101
- end
82
+ @upgrade_protocol = :h2c
102
83
 
103
- def build_parser(*)
104
- return super unless @origin.scheme == "http"
84
+ if request.options.max_concurrent_requests != @options.max_concurrent_requests
85
+ @options = @options.merge(max_concurrent_requests: nil)
86
+ end
105
87
 
106
- super("http/1.1")
88
+ prev_parser.requests.each do |req|
89
+ req.transition(:idle)
90
+ send(req)
91
+ end
107
92
  end
108
93
  end
109
94
  end
@@ -0,0 +1,93 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HTTPX
4
+ module Plugins
5
+ #
6
+ # The InternalTelemetry plugin is for internal use only. It is therefore undocumented, and
7
+ # its use is disencouraged, as API compatiblity will **not be guaranteed**.
8
+ #
9
+ # The gist of it is: when debug_level of logger is enabled to 3 or greater, considered internal-only
10
+ # supported log levels, it'll be loaded by default.
11
+ #
12
+ # Against a specific point of time, which will be by default the session initialization, but can be set
13
+ # by the end user in $http_init_time, different diff metrics can be shown. The "point of time" is calculated
14
+ # using the monotonic clock.
15
+ module InternalTelemetry
16
+ module TrackTimeMethods
17
+ private
18
+
19
+ def elapsed_time
20
+ yield
21
+ ensure
22
+ meter_elapsed_time("#{self.class.superclass}##{caller_locations(1, 1)[0].label}")
23
+ end
24
+
25
+ def meter_elapsed_time(label)
26
+ $http_init_time ||= Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
27
+ prev_time = $http_init_time
28
+ after_time = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
29
+ # $http_init_time = after_time
30
+ elapsed = after_time - prev_time
31
+ warn(+"\e[31m" << "[ELAPSED TIME]: #{label}: #{elapsed} (ms)" << "\e[0m")
32
+ end
33
+ end
34
+
35
+ module InstanceMethods
36
+ def self.included(klass)
37
+ klass.prepend TrackTimeMethods
38
+ super
39
+ end
40
+
41
+ def initialize(*)
42
+ meter_elapsed_time("Session: initializing...")
43
+ super
44
+ meter_elapsed_time("Session: initialized!!!")
45
+ end
46
+
47
+ private
48
+
49
+ def build_requests(*)
50
+ elapsed_time { super }
51
+ end
52
+
53
+ def fetch_response(*)
54
+ response = super
55
+ meter_elapsed_time("Session -> response") if response
56
+ response
57
+ end
58
+
59
+ def close(*)
60
+ super
61
+ meter_elapsed_time("Session -> close")
62
+ end
63
+ end
64
+
65
+ module RequestMethods
66
+ def self.included(klass)
67
+ klass.prepend TrackTimeMethods
68
+ super
69
+ end
70
+
71
+ def transition(nextstate)
72
+ state = @state
73
+ super
74
+ meter_elapsed_time("Request##{object_id}[#{@verb} #{@uri}: #{state}] -> #{nextstate}") if nextstate == @state
75
+ end
76
+ end
77
+
78
+ module ConnectionMethods
79
+ def self.included(klass)
80
+ klass.prepend TrackTimeMethods
81
+ super
82
+ end
83
+
84
+ def transition(nextstate)
85
+ state = @state
86
+ super
87
+ meter_elapsed_time("Connection[#{@origin}]: #{state} -> #{nextstate}") if nextstate == @state
88
+ end
89
+ end
90
+ end
91
+ register_plugin :internal_telemetry, InternalTelemetry
92
+ end
93
+ end
@@ -23,6 +23,7 @@ module HTTPX
23
23
  end
24
24
 
25
25
  def load_dependencies(*)
26
+ # :nocov:
26
27
  begin
27
28
  unless defined?(HTTP::FormData)
28
29
  # in order not to break legacy code, we'll keep loading http/form_data for them.
@@ -33,6 +34,7 @@ module HTTPX
33
34
  end
34
35
  rescue LoadError
35
36
  end
37
+ # :nocov:
36
38
  require "httpx/plugins/multipart/encoder"
37
39
  require "httpx/plugins/multipart/part"
38
40
  require "httpx/plugins/multipart/mime_type_detector"
@@ -29,18 +29,13 @@ module HTTPX::Plugins
29
29
 
30
30
  def rewind
31
31
  form = @form.each_with_object([]) do |(key, val), aux|
32
- v = case val
33
- when File
34
- val = val.reopen(val.path, File::RDONLY) if val.closed?
35
- val.rewind
36
- val
37
- else
38
- v
39
- end
40
- aux << [key, v]
32
+ val = val.reopen(val.path, File::RDONLY) if val.is_a?(File) && val.closed?
33
+ val.rewind if val.respond_to?(:rewind)
34
+ aux << [key, val]
41
35
  end
42
36
  @form = form
43
37
  @parts = to_parts(form)
38
+ @part_index = 0
44
39
  end
45
40
 
46
41
  private
@@ -242,7 +242,7 @@ module HTTPX
242
242
  register_plugin :proxy, Proxy
243
243
  end
244
244
 
245
- class ProxySSL < SSL
245
+ class ProxySSL < IO.registry["ssl"]
246
246
  def initialize(tcp, request_uri, options)
247
247
  @io = tcp.to_io
248
248
  super(request_uri, tcp.addresses, options)
@@ -81,7 +81,7 @@ module HTTPX
81
81
  request.uri.to_s
82
82
  end
83
83
 
84
- def set_request_headers(request)
84
+ def set_protocol_headers(request)
85
85
  super
86
86
  proxy_params = @options.proxy
87
87
  request.headers["proxy-authorization"] = "Basic #{proxy_params.token_authentication}" if proxy_params.authenticated?
@@ -16,6 +16,14 @@ module HTTPX
16
16
  Error = Socks4Error
17
17
 
18
18
  module ConnectionMethods
19
+ def interests
20
+ if @state == :connecting
21
+ return @write_buffer.empty? ? :r : :w
22
+ end
23
+
24
+ super
25
+ end
26
+
19
27
  private
20
28
 
21
29
  def transition(nextstate)
@@ -35,6 +35,14 @@ module HTTPX
35
35
  super || @state == :authenticating || @state == :negotiating
36
36
  end
37
37
 
38
+ def interests
39
+ if @state == :connecting || @state == :authenticating || @state == :negotiating
40
+ return @write_buffer.empty? ? :r : :w
41
+ end
42
+
43
+ super
44
+ end
45
+
38
46
  private
39
47
 
40
48
  def transition(nextstate)
@@ -43,7 +43,7 @@ module HTTPX
43
43
  end
44
44
 
45
45
  def __on_promise_request(parser, stream, h)
46
- log(level: 1) do
46
+ log(level: 1, color: :yellow) do
47
47
  # :nocov:
48
48
  h.map { |k, v| "#{stream.id}: -> PROMISE HEADER: #{k}: #{v}" }.join("\n")
49
49
  # :nocov:
@@ -57,6 +57,8 @@ module HTTPX
57
57
  request.merge_headers(headers)
58
58
  promise_headers[stream] = request
59
59
  parser.pending.delete(request)
60
+ parser.streams[request] = stream
61
+ request.transition(:done)
60
62
  else
61
63
  stream.refuse
62
64
  end
@@ -67,7 +69,6 @@ module HTTPX
67
69
  return unless request
68
70
 
69
71
  parser.__send__(:on_stream_headers, stream, request, h)
70
- request.transition(:done)
71
72
  response = request.response
72
73
  response.mark_as_pushed!
73
74
  stream.on(:data, &parser.method(:on_stream_data).curry(3)[stream, request])