httpx 0.12.0 → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. checksums.yaml +4 -4
  2. data/doc/release_notes/0_13_0.md +58 -0
  3. data/lib/httpx/chainable.rb +2 -2
  4. data/lib/httpx/connection.rb +17 -13
  5. data/lib/httpx/connection/http1.rb +4 -2
  6. data/lib/httpx/connection/http2.rb +1 -1
  7. data/lib/httpx/io/ssl.rb +30 -17
  8. data/lib/httpx/io/tcp.rb +45 -26
  9. data/lib/httpx/io/unix.rb +27 -12
  10. data/lib/httpx/options.rb +11 -23
  11. data/lib/httpx/plugins/compression.rb +20 -8
  12. data/lib/httpx/plugins/compression/brotli.rb +8 -6
  13. data/lib/httpx/plugins/compression/deflate.rb +2 -2
  14. data/lib/httpx/plugins/compression/gzip.rb +2 -2
  15. data/lib/httpx/plugins/digest_authentication.rb +1 -1
  16. data/lib/httpx/plugins/follow_redirects.rb +1 -1
  17. data/lib/httpx/plugins/h2c.rb +43 -58
  18. data/lib/httpx/plugins/internal_telemetry.rb +1 -1
  19. data/lib/httpx/plugins/retries.rb +1 -1
  20. data/lib/httpx/plugins/stream.rb +3 -1
  21. data/lib/httpx/plugins/upgrade.rb +83 -0
  22. data/lib/httpx/plugins/upgrade/h2.rb +54 -0
  23. data/lib/httpx/pool.rb +14 -5
  24. data/lib/httpx/response.rb +5 -5
  25. data/lib/httpx/version.rb +1 -1
  26. data/sig/chainable.rbs +2 -1
  27. data/sig/connection/http1.rbs +1 -0
  28. data/sig/options.rbs +7 -20
  29. data/sig/plugins/aws_sigv4.rbs +0 -1
  30. data/sig/plugins/compression.rbs +5 -3
  31. data/sig/plugins/compression/brotli.rbs +1 -1
  32. data/sig/plugins/compression/deflate.rbs +1 -1
  33. data/sig/plugins/compression/gzip.rbs +1 -1
  34. data/sig/plugins/cookies.rbs +0 -1
  35. data/sig/plugins/digest_authentication.rbs +0 -1
  36. data/sig/plugins/expect.rbs +0 -2
  37. data/sig/plugins/follow_redirects.rbs +0 -2
  38. data/sig/plugins/h2c.rbs +5 -10
  39. data/sig/plugins/persistent.rbs +0 -1
  40. data/sig/plugins/proxy.rbs +0 -1
  41. data/sig/plugins/retries.rbs +0 -4
  42. data/sig/plugins/upgrade.rbs +23 -0
  43. data/sig/response.rbs +3 -1
  44. metadata +7 -2
data/lib/httpx/options.rb CHANGED
@@ -6,11 +6,6 @@ module HTTPX
6
6
  MAX_BODY_THRESHOLD_SIZE = (1 << 10) * 112 # 112K
7
7
 
8
8
  class << self
9
- def inherited(klass)
10
- super
11
- klass.instance_variable_set(:@defined_options, @defined_options.dup)
12
- end
13
-
14
9
  def new(options = {})
15
10
  # let enhanced options go through
16
11
  return options if self == Options && options.class > self
@@ -19,13 +14,7 @@ module HTTPX
19
14
  super
20
15
  end
21
16
 
22
- def defined_options
23
- @defined_options ||= []
24
- end
25
-
26
17
  def def_option(name, &interpreter)
27
- defined_options << name.to_sym
28
-
29
18
  attr_reader name
30
19
 
31
20
  if interpreter
@@ -34,16 +23,8 @@ module HTTPX
34
23
 
35
24
  instance_variable_set(:"@#{name}", instance_exec(value, &interpreter))
36
25
  end
37
-
38
- define_method(:"with_#{name}") do |value|
39
- merge(name => instance_exec(value, &interpreter))
40
- end
41
26
  else
42
27
  attr_writer name
43
-
44
- define_method(:"with_#{name}") do |value|
45
- merge(name => value)
46
- end
47
28
  end
48
29
 
49
30
  protected :"#{name}="
@@ -69,6 +50,7 @@ module HTTPX
69
50
  :connection_class => Class.new(Connection),
70
51
  :transport => nil,
71
52
  :transport_options => nil,
53
+ :addresses => nil,
72
54
  :persistent => false,
73
55
  :resolver_class => (ENV["HTTPX_RESOLVER"] || :native).to_sym,
74
56
  :resolver_options => { cache: true },
@@ -121,6 +103,10 @@ module HTTPX
121
103
  transport
122
104
  end
123
105
 
106
+ def_option(:addresses) do |addrs|
107
+ Array(addrs)
108
+ end
109
+
124
110
  %w[
125
111
  params form json body ssl http2_settings
126
112
  request_class response_class headers_class request_body_class response_body_class connection_class
@@ -153,6 +139,8 @@ module HTTPX
153
139
 
154
140
  h1 = to_hash
155
141
 
142
+ return self if h1 == h2
143
+
156
144
  merged = h1.merge(h2) do |k, v1, v2|
157
145
  case k
158
146
  when :headers, :ssl, :http2_settings, :timeout
@@ -166,10 +154,10 @@ module HTTPX
166
154
  end
167
155
 
168
156
  def to_hash
169
- hash_pairs = self.class
170
- .defined_options
171
- .flat_map { |opt_name| [opt_name, send(opt_name)] }
172
- Hash[*hash_pairs]
157
+ hash_pairs = instance_variables.map do |ivar|
158
+ [ivar[1..-1].to_sym, instance_variable_get(ivar)]
159
+ end
160
+ Hash[hash_pairs]
173
161
  end
174
162
 
175
163
  def initialize_dup(other)
@@ -13,15 +13,17 @@ module HTTPX
13
13
  # https://gitlab.com/honeyryderchuck/httpx/wikis/Compression
14
14
  #
15
15
  module Compression
16
- extend Registry
17
-
18
16
  class << self
19
- def load_dependencies(klass)
17
+ def configure(klass)
20
18
  klass.plugin(:"compression/gzip")
21
19
  klass.plugin(:"compression/deflate")
22
20
  end
23
21
 
24
22
  def extra_options(options)
23
+ encodings = Module.new do
24
+ extend Registry
25
+ end
26
+
25
27
  Class.new(options.class) do
26
28
  def_option(:compression_threshold_size) do |bytes|
27
29
  bytes = Integer(bytes)
@@ -29,7 +31,13 @@ module HTTPX
29
31
 
30
32
  bytes
31
33
  end
32
- end.new(options).merge(headers: { "accept-encoding" => Compression.registry.keys })
34
+
35
+ def_option(:encodings) do |encs|
36
+ raise Error, ":encodings must be a registry" unless encs.respond_to?(:registry)
37
+
38
+ encs
39
+ end
40
+ end.new(options).merge(encodings: encodings)
33
41
  end
34
42
  end
35
43
 
@@ -37,7 +45,11 @@ module HTTPX
37
45
  def initialize(*)
38
46
  super
39
47
  # forego compression in the Range cases
40
- @headers.delete("accept-encoding") if @headers.key?("range")
48
+ if @headers.key?("range")
49
+ @headers.delete("accept-encoding")
50
+ else
51
+ @headers["accept-encoding"] ||= @options.encodings.registry.keys
52
+ end
41
53
  end
42
54
  end
43
55
 
@@ -52,7 +64,7 @@ module HTTPX
52
64
  @headers.get("content-encoding").each do |encoding|
53
65
  next if encoding == "identity"
54
66
 
55
- @body = Encoder.new(@body, Compression.registry(encoding).deflater)
67
+ @body = Encoder.new(@body, options.encodings.registry(encoding).deflater)
56
68
  end
57
69
  @headers["content-length"] = @body.bytesize unless chunked?
58
70
  end
@@ -61,7 +73,7 @@ module HTTPX
61
73
  module ResponseBodyMethods
62
74
  attr_reader :encodings
63
75
 
64
- def initialize(*, **)
76
+ def initialize(*)
65
77
  @encodings = []
66
78
 
67
79
  super
@@ -80,7 +92,7 @@ module HTTPX
80
92
  @_inflaters = @headers.get("content-encoding").map do |encoding|
81
93
  next if encoding == "identity"
82
94
 
83
- inflater = Compression.registry(encoding).inflater(compressed_length)
95
+ inflater = @options.encodings.registry(encoding).inflater(compressed_length)
84
96
  # do not uncompress if there is no decoder available. In fact, we can't reliably
85
97
  # continue decompressing beyond that, so ignore.
86
98
  break unless inflater
@@ -4,13 +4,15 @@ module HTTPX
4
4
  module Plugins
5
5
  module Compression
6
6
  module Brotli
7
- def self.load_dependencies(klass)
8
- klass.plugin(:compression)
9
- require "brotli"
10
- end
7
+ class << self
8
+ def load_dependencies(klass)
9
+ klass.plugin(:compression)
10
+ require "brotli"
11
+ end
11
12
 
12
- def self.configure(*)
13
- Compression.register "br", self
13
+ def configure(klass)
14
+ klass.default_options.encodings.register "br", self
15
+ end
14
16
  end
15
17
 
16
18
  module Deflater
@@ -10,8 +10,8 @@ module HTTPX
10
10
  klass.plugin(:"compression/gzip")
11
11
  end
12
12
 
13
- def self.configure(*)
14
- Compression.register "deflate", self
13
+ def self.configure(klass)
14
+ klass.default_options.encodings.register "deflate", self
15
15
  end
16
16
 
17
17
  module Deflater
@@ -10,8 +10,8 @@ module HTTPX
10
10
  require "zlib"
11
11
  end
12
12
 
13
- def self.configure(*)
14
- Compression.register "gzip", self
13
+ def self.configure(klass)
14
+ klass.default_options.encodings.register "gzip", self
15
15
  end
16
16
 
17
17
  class Deflater
@@ -29,7 +29,7 @@ module HTTPX
29
29
 
30
30
  module InstanceMethods
31
31
  def digest_authentication(user, password)
32
- branch(default_options.with_digest(Digest.new(user, password)))
32
+ with(digest: Digest.new(user, password))
33
33
  end
34
34
 
35
35
  alias_method :digest_auth, :digest_authentication
@@ -32,7 +32,7 @@ module HTTPX
32
32
 
33
33
  module InstanceMethods
34
34
  def max_redirects(n)
35
- branch(default_options.with_max_redirects(n.to_i))
35
+ with(max_redirects: n.to_i)
36
36
  end
37
37
 
38
38
  private
@@ -6,68 +6,53 @@ module HTTPX
6
6
  # This plugin adds support for upgrading a plaintext HTTP/1.1 connection to HTTP/2
7
7
  # (https://tools.ietf.org/html/rfc7540#section-3.2)
8
8
  #
9
- # https://gitlab.com/honeyryderchuck/httpx/wikis/Follow-Redirects
9
+ # https://gitlab.com/honeyryderchuck/httpx/wikis/Upgrade#h2c
10
10
  #
11
11
  module H2C
12
- def self.load_dependencies(*)
13
- require "base64"
12
+ VALID_H2C_VERBS = %i[get options head].freeze
13
+
14
+ class << self
15
+ def load_dependencies(*)
16
+ require "base64"
17
+ end
18
+
19
+ def configure(klass)
20
+ klass.plugin(:upgrade)
21
+ klass.default_options.upgrade_handlers.register "h2c", self
22
+ end
23
+
24
+ def call(connection, request, response)
25
+ connection.upgrade_to_h2c(request, response)
26
+ end
14
27
  end
15
28
 
16
29
  module InstanceMethods
17
- def request(*args, **options)
18
- h2c_options = options.merge(fallback_protocol: "h2c")
30
+ def send_requests(*requests, options)
31
+ upgrade_request, *remainder = requests
32
+
33
+ return super unless VALID_H2C_VERBS.include?(upgrade_request.verb) && upgrade_request.scheme == "http"
19
34
 
20
- requests = build_requests(*args, h2c_options)
35
+ connection = pool.find_connection(upgrade_request.uri, @options.merge(options))
21
36
 
22
- upgrade_request = requests.first
23
- return super unless valid_h2c_upgrade_request?(upgrade_request)
37
+ return super if connection && connection.upgrade_protocol == :h2c
24
38
 
39
+ # build upgrade request
25
40
  upgrade_request.headers.add("connection", "upgrade")
26
41
  upgrade_request.headers.add("connection", "http2-settings")
27
42
  upgrade_request.headers["upgrade"] = "h2c"
28
43
  upgrade_request.headers["http2-settings"] = HTTP2Next::Client.settings_header(upgrade_request.options.http2_settings)
29
- wrap { send_requests(*upgrade_request, h2c_options).first }
30
44
 
31
- responses = send_requests(*requests, h2c_options)
32
-
33
- responses.size == 1 ? responses.first : responses
34
- end
35
-
36
- private
37
-
38
- def fetch_response(request, connections, options)
39
- response = super
40
- if response && valid_h2c_upgrade?(request, response, options)
41
- log { "upgrading to h2c..." }
42
- connection = find_connection(request, connections, options)
43
- connections << connection unless connections.include?(connection)
44
- connection.upgrade(request, response)
45
- end
46
- response
47
- end
48
-
49
- VALID_H2C_METHODS = %i[get options head].freeze
50
- private_constant :VALID_H2C_METHODS
51
-
52
- def valid_h2c_upgrade_request?(request)
53
- VALID_H2C_METHODS.include?(request.verb) &&
54
- request.scheme == "http"
55
- end
56
-
57
- def valid_h2c_upgrade?(request, response, options)
58
- options.fallback_protocol == "h2c" &&
59
- request.headers.get("connection").include?("upgrade") &&
60
- request.headers.get("upgrade").include?("h2c") &&
61
- response.status == 101
45
+ super(upgrade_request, *remainder, options.merge(max_concurrent_requests: 1))
62
46
  end
63
47
  end
64
48
 
65
49
  class H2CParser < Connection::HTTP2
66
50
  def upgrade(request, response)
67
- @connection.send_connection_preface
68
51
  # skip checks, it is assumed that this is the first
69
52
  # request in the connection
70
53
  stream = @connection.upgrade
54
+
55
+ # on_settings
71
56
  handle_stream(stream, request)
72
57
  @streams[request] = stream
73
58
 
@@ -81,29 +66,29 @@ module HTTPX
81
66
  module ConnectionMethods
82
67
  using URIExtensions
83
68
 
84
- def match?(uri, options)
85
- return super unless uri.scheme == "http" && @options.fallback_protocol == "h2c"
86
-
87
- super && options.fallback_protocol == "h2c"
88
- end
89
-
90
- def coalescable?(connection)
91
- return super unless @options.fallback_protocol == "h2c" && @origin.scheme == "http"
69
+ def upgrade_to_h2c(request, response)
70
+ prev_parser = @parser
92
71
 
93
- @origin == connection.origin && connection.options.fallback_protocol == "h2c"
94
- end
72
+ if prev_parser
73
+ prev_parser.reset
74
+ @inflight -= prev_parser.requests.size
75
+ end
95
76
 
96
- def upgrade(request, response)
97
- @parser.reset if @parser
98
- @parser = H2CParser.new(@write_buffer, @options)
77
+ parser_options = @options.merge(max_concurrent_requests: request.options.max_concurrent_requests)
78
+ @parser = H2CParser.new(@write_buffer, parser_options)
99
79
  set_parser_callbacks(@parser)
80
+ @inflight += 1
100
81
  @parser.upgrade(request, response)
101
- end
82
+ @upgrade_protocol = :h2c
102
83
 
103
- def build_parser(*)
104
- return super unless @origin.scheme == "http"
84
+ if request.options.max_concurrent_requests != @options.max_concurrent_requests
85
+ @options = @options.merge(max_concurrent_requests: nil)
86
+ end
105
87
 
106
- super("http/1.1")
88
+ prev_parser.requests.each do |req|
89
+ req.transition(:idle)
90
+ send(req)
91
+ end
107
92
  end
108
93
  end
109
94
  end
@@ -71,7 +71,7 @@ module HTTPX
71
71
  def transition(nextstate)
72
72
  state = @state
73
73
  super
74
- meter_elapsed_time("Request[#{@verb} #{@uri}: #{state}] -> #{nextstate}") if nextstate == @state
74
+ meter_elapsed_time("Request##{object_id}[#{@verb} #{@uri}: #{state}] -> #{nextstate}") if nextstate == @state
75
75
  end
76
76
  end
77
77
 
@@ -55,7 +55,7 @@ module HTTPX
55
55
 
56
56
  module InstanceMethods
57
57
  def max_retries(n)
58
- branch(default_options.with_max_retries(n.to_i))
58
+ with(max_retries: n.to_i)
59
59
  end
60
60
 
61
61
  private
@@ -5,6 +5,8 @@ module HTTPX
5
5
  #
6
6
  # This plugin adds support for stream response (text/event-stream).
7
7
  #
8
+ # https://gitlab.com/honeyryderchuck/httpx/wikis/Stream
9
+ #
8
10
  module Stream
9
11
  module InstanceMethods
10
12
  private
@@ -31,7 +33,7 @@ module HTTPX
31
33
  end
32
34
 
33
35
  module ResponseBodyMethods
34
- def initialize(*, **)
36
+ def initialize(*)
35
37
  super
36
38
  @stream = @response.stream
37
39
  end
@@ -0,0 +1,83 @@
1
+ # frozen_string_literal: true
2
+
3
+ module HTTPX
4
+ module Plugins
5
+ #
6
+ # This plugin helps negotiating a new protocol from an HTTP/1.1 connection, via the
7
+ # Upgrade header.
8
+ #
9
+ # https://gitlab.com/honeyryderchuck/httpx/wikis/Upgrade
10
+ #
11
+ module Upgrade
12
+ class << self
13
+ def configure(klass)
14
+ klass.plugin(:"upgrade/h2")
15
+ end
16
+
17
+ def extra_options(options)
18
+ upgrade_handlers = Module.new do
19
+ extend Registry
20
+ end
21
+
22
+ Class.new(options.class) do
23
+ def_option(:upgrade_handlers) do |encs|
24
+ raise Error, ":upgrade_handlers must be a registry" unless encs.respond_to?(:registry)
25
+
26
+ encs
27
+ end
28
+ end.new(options).merge(upgrade_handlers: upgrade_handlers)
29
+ end
30
+ end
31
+
32
+ module InstanceMethods
33
+ def fetch_response(request, connections, options)
34
+ response = super
35
+
36
+ if response && response.headers.key?("upgrade")
37
+
38
+ upgrade_protocol = response.headers["upgrade"].split(/ *, */).first
39
+
40
+ return response unless upgrade_protocol && options.upgrade_handlers.registry.key?(upgrade_protocol)
41
+
42
+ protocol_handler = options.upgrade_handlers.registry(upgrade_protocol)
43
+
44
+ return response unless protocol_handler
45
+
46
+ log { "upgrading to #{upgrade_protocol}..." }
47
+ connection = find_connection(request, connections, options)
48
+ connections << connection unless connections.include?(connection)
49
+
50
+ # do not upgrade already upgraded connections
51
+ return if connection.upgrade_protocol == upgrade_protocol
52
+
53
+ protocol_handler.call(connection, request, response)
54
+
55
+ # keep in the loop if the server is switching, unless
56
+ # the connection has been hijacked, in which case you want
57
+ # to terminante immediately
58
+ return if response.status == 101 && !connection.hijacked
59
+ end
60
+
61
+ response
62
+ end
63
+
64
+ def close(*args)
65
+ return super if args.empty?
66
+
67
+ connections, = args
68
+
69
+ pool.close(connections.reject(&:hijacked))
70
+ end
71
+ end
72
+
73
+ module ConnectionMethods
74
+ attr_reader :upgrade_protocol, :hijacked
75
+
76
+ def hijack_io
77
+ @hijacked = true
78
+ end
79
+ end
80
+ end
81
+ register_plugin(:upgrade, Upgrade)
82
+ end
83
+ end