httpx 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/lib/httpx.rb +8 -2
  3. data/lib/httpx/adapters/faraday.rb +203 -0
  4. data/lib/httpx/altsvc.rb +4 -0
  5. data/lib/httpx/callbacks.rb +1 -4
  6. data/lib/httpx/chainable.rb +4 -3
  7. data/lib/httpx/connection.rb +326 -104
  8. data/lib/httpx/{channel → connection}/http1.rb +29 -15
  9. data/lib/httpx/{channel → connection}/http2.rb +12 -6
  10. data/lib/httpx/errors.rb +2 -0
  11. data/lib/httpx/headers.rb +4 -1
  12. data/lib/httpx/io/ssl.rb +5 -1
  13. data/lib/httpx/io/tcp.rb +13 -7
  14. data/lib/httpx/io/udp.rb +1 -0
  15. data/lib/httpx/io/unix.rb +1 -0
  16. data/lib/httpx/loggable.rb +34 -9
  17. data/lib/httpx/options.rb +57 -31
  18. data/lib/httpx/parser/http1.rb +8 -0
  19. data/lib/httpx/plugins/authentication.rb +4 -0
  20. data/lib/httpx/plugins/basic_authentication.rb +4 -0
  21. data/lib/httpx/plugins/compression.rb +22 -5
  22. data/lib/httpx/plugins/cookies.rb +89 -36
  23. data/lib/httpx/plugins/digest_authentication.rb +45 -26
  24. data/lib/httpx/plugins/follow_redirects.rb +61 -62
  25. data/lib/httpx/plugins/h2c.rb +78 -39
  26. data/lib/httpx/plugins/multipart.rb +5 -0
  27. data/lib/httpx/plugins/persistent.rb +29 -0
  28. data/lib/httpx/plugins/proxy.rb +125 -78
  29. data/lib/httpx/plugins/proxy/http.rb +31 -27
  30. data/lib/httpx/plugins/proxy/socks4.rb +30 -24
  31. data/lib/httpx/plugins/proxy/socks5.rb +49 -39
  32. data/lib/httpx/plugins/proxy/ssh.rb +81 -0
  33. data/lib/httpx/plugins/push_promise.rb +18 -9
  34. data/lib/httpx/plugins/retries.rb +43 -15
  35. data/lib/httpx/pool.rb +159 -0
  36. data/lib/httpx/registry.rb +2 -0
  37. data/lib/httpx/request.rb +10 -0
  38. data/lib/httpx/resolver.rb +2 -1
  39. data/lib/httpx/resolver/https.rb +62 -56
  40. data/lib/httpx/resolver/native.rb +48 -37
  41. data/lib/httpx/resolver/resolver_mixin.rb +16 -11
  42. data/lib/httpx/resolver/system.rb +11 -7
  43. data/lib/httpx/response.rb +24 -10
  44. data/lib/httpx/selector.rb +32 -39
  45. data/lib/httpx/{client.rb → session.rb} +99 -62
  46. data/lib/httpx/timeout.rb +7 -15
  47. data/lib/httpx/transcoder/body.rb +4 -0
  48. data/lib/httpx/transcoder/chunker.rb +4 -0
  49. data/lib/httpx/version.rb +1 -1
  50. metadata +10 -8
  51. data/lib/httpx/channel.rb +0 -367
@@ -2,9 +2,37 @@
2
2
 
3
3
  module HTTPX
4
4
  module Plugins
5
+ #
6
+ # This plugin adds support for retrying requests when certain errors happen.
7
+ #
5
8
  module Retries
6
9
  MAX_RETRIES = 3
10
+ # TODO: pass max_retries in a configure/load block
11
+
7
12
  IDEMPOTENT_METHODS = %i[get options head put delete].freeze
13
+ RETRYABLE_ERRORS = [IOError,
14
+ EOFError,
15
+ Errno::ECONNRESET,
16
+ Errno::ECONNABORTED,
17
+ Errno::EPIPE,
18
+ (OpenSSL::SSL::SSLError if defined?(OpenSSL)),
19
+ TimeoutError,
20
+ Parser::Error,
21
+ Errno::EINVAL,
22
+ Errno::ETIMEDOUT].freeze
23
+
24
+ def self.extra_options(options)
25
+ Class.new(options.class) do
26
+ def_option(:max_retries) do |num|
27
+ num = Integer(num)
28
+ raise Error, ":max_retries must be positive" unless num.positive?
29
+
30
+ num
31
+ end
32
+
33
+ def_option(:retry_change_requests)
34
+ end.new(options)
35
+ end
8
36
 
9
37
  module InstanceMethods
10
38
  def max_retries(n)
@@ -13,18 +41,29 @@ module HTTPX
13
41
 
14
42
  private
15
43
 
16
- def fetch_response(request)
44
+ def fetch_response(request, connections, options)
17
45
  response = super
18
46
  if response.is_a?(ErrorResponse) &&
19
47
  request.retries.positive? &&
20
- IDEMPOTENT_METHODS.include?(request.verb)
48
+ __repeatable_request?(request, options) &&
49
+ __retryable_error?(response.error)
21
50
  request.retries -= 1
22
- channel = find_channel(request)
23
- channel.send(request)
51
+ log { "failed to get response, #{request.retries} tries to go..." }
52
+ request.transition(:idle)
53
+ connection = find_connection(request, connections, options)
54
+ connection.send(request)
24
55
  return
25
56
  end
26
57
  response
27
58
  end
59
+
60
+ def __repeatable_request?(request, options)
61
+ IDEMPOTENT_METHODS.include?(request.verb) || options.retry_change_requests
62
+ end
63
+
64
+ def __retryable_error?(ex)
65
+ RETRYABLE_ERRORS.any? { |klass| ex.is_a?(klass) }
66
+ end
28
67
  end
29
68
 
30
69
  module RequestMethods
@@ -35,17 +74,6 @@ module HTTPX
35
74
  @retries = @options.max_retries || MAX_RETRIES
36
75
  end
37
76
  end
38
-
39
- module OptionsMethods
40
- def self.included(klass)
41
- super
42
- klass.def_option(:max_retries) do |num|
43
- num = Integer(num)
44
- raise Error, ":max_retries must be positive" unless num.positive?
45
- num
46
- end
47
- end
48
- end
49
77
  end
50
78
  register_plugin :retries, Retries
51
79
  end
@@ -0,0 +1,159 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "httpx/selector"
4
+ require "httpx/connection"
5
+ require "httpx/resolver"
6
+
7
+ module HTTPX
8
+ class Pool
9
+ def initialize
10
+ @resolvers = {}
11
+ @_resolver_monitors = {}
12
+ @selector = Selector.new
13
+ @connections = []
14
+ @connected_connections = 0
15
+ end
16
+
17
+ def empty?
18
+ @connections.empty?
19
+ end
20
+
21
+ def next_tick(timeout = nil)
22
+ catch(:jump_tick) do
23
+ tout = timeout.total_timeout if timeout
24
+
25
+ @selector.select(next_timeout || tout) do |monitor|
26
+ monitor.io.call
27
+ monitor.interests = monitor.io.interests
28
+ end
29
+ end
30
+ rescue StandardError => ex
31
+ @connections.each do |connection|
32
+ connection.emit(:error, ex)
33
+ end
34
+ end
35
+
36
+ def close(connections = @connections)
37
+ connections = connections.reject(&:inflight?)
38
+ connections.each(&:close)
39
+ next_tick until connections.none? { |c| @connections.include?(c) }
40
+ @resolvers.each_value do |resolver|
41
+ resolver.close unless resolver.closed?
42
+ end if @connections.empty?
43
+ end
44
+
45
+ def init_connection(connection, _options)
46
+ resolve_connection(connection)
47
+ connection.on(:open) do
48
+ @connected_connections += 1
49
+ end
50
+ connection.on(:unreachable) do
51
+ resolver = find_resolver_for(connection)
52
+ resolver.uncache(connection) if resolver
53
+ resolve_connection(connection)
54
+ end
55
+ end
56
+
57
+ # opens a connection to the IP reachable through +uri+.
58
+ # Many hostnames are reachable through the same IP, so we try to
59
+ # maximize pipelining by opening as few connections as possible.
60
+ #
61
+ def find_connection(uri, options)
62
+ @connections.find do |connection|
63
+ connection.match?(uri, options)
64
+ end
65
+ end
66
+
67
+ private
68
+
69
+ def resolve_connection(connection)
70
+ @connections << connection unless @connections.include?(connection)
71
+ resolver = find_resolver_for(connection)
72
+ resolver << connection
73
+ return if resolver.empty?
74
+
75
+ @_resolver_monitors[resolver] ||= @selector.register(resolver, :w)
76
+ end
77
+
78
+ def on_resolver_connection(connection)
79
+ found_connection = @connections.find do |ch|
80
+ ch != connection && ch.mergeable?(connection)
81
+ end
82
+ return register_connection(connection) unless found_connection
83
+
84
+ if found_connection.state == :open
85
+ coalesce_connections(found_connection, connection)
86
+ else
87
+ found_connection.once(:open) do
88
+ coalesce_connections(found_connection, connection)
89
+ end
90
+ end
91
+ end
92
+
93
+ def on_resolver_error(ch, error)
94
+ ch.emit(:error, error)
95
+ # must remove connection by hand, hasn't been started yet
96
+ unregister_connection(ch)
97
+ end
98
+
99
+ def on_resolver_close(resolver)
100
+ resolver_type = resolver.class
101
+ return unless @resolvers[resolver_type] == resolver
102
+
103
+ @resolvers.delete(resolver_type)
104
+
105
+ @selector.deregister(resolver)
106
+ monitor = @_resolver_monitors.delete(resolver)
107
+ monitor.close if monitor
108
+ resolver.close unless resolver.closed?
109
+ end
110
+
111
+ def register_connection(connection)
112
+ if connection.state == :open
113
+ # if open, an IO was passed upstream, therefore
114
+ # consider it connected already.
115
+ @connected_connections += 1
116
+ @selector.register(connection, :rw)
117
+ else
118
+ @selector.register(connection, :w)
119
+ end
120
+ connection.on(:close) do
121
+ unregister_connection(connection)
122
+ end
123
+ return if connection.state == :open
124
+ end
125
+
126
+ def unregister_connection(connection)
127
+ @connections.delete(connection)
128
+ @selector.deregister(connection)
129
+ @connected_connections -= 1
130
+ end
131
+
132
+ def coalesce_connections(ch1, ch2)
133
+ if ch1.coalescable?(ch2)
134
+ ch1.merge(ch2)
135
+ @connections.delete(ch2)
136
+ else
137
+ register_connection(ch2)
138
+ end
139
+ end
140
+
141
+ def next_timeout
142
+ @resolvers.values.reject(&:closed?).map(&:timeout).compact.min || @connections.map(&:timeout).compact.min
143
+ end
144
+
145
+ def find_resolver_for(connection)
146
+ connection_options = connection.options
147
+ resolver_type = connection_options.resolver_class
148
+ resolver_type = Resolver.registry(resolver_type) if resolver_type.is_a?(Symbol)
149
+
150
+ @resolvers[resolver_type] ||= begin
151
+ resolver = resolver_type.new(connection_options)
152
+ resolver.on(:resolve, &method(:on_resolver_connection))
153
+ resolver.on(:error, &method(:on_resolver_error))
154
+ resolver.on(:close) { on_resolver_close(resolver) }
155
+ resolver
156
+ end
157
+ end
158
+ end
159
+ end
@@ -58,8 +58,10 @@ module HTTPX
58
58
  def registry(tag = nil)
59
59
  @registry ||= {}
60
60
  return @registry if tag.nil?
61
+
61
62
  handler = @registry.fetch(tag)
62
63
  raise(Error, "#{tag} is not registered in #{self}") unless handler
64
+
63
65
  case handler
64
66
  when Symbol, String
65
67
  const_get(handler)
@@ -5,6 +5,7 @@ require "forwardable"
5
5
  module HTTPX
6
6
  class Request
7
7
  extend Forwardable
8
+ include Callbacks
8
9
  using URIExtensions
9
10
 
10
11
  METHODS = [
@@ -34,6 +35,8 @@ module HTTPX
34
35
 
35
36
  attr_reader :verb, :uri, :headers, :body, :state
36
37
 
38
+ attr_reader :options
39
+
37
40
  attr_accessor :response
38
41
 
39
42
  def_delegator :@body, :<<
@@ -84,6 +87,7 @@ module HTTPX
84
87
 
85
88
  def query
86
89
  return @query if defined?(@query)
90
+
87
91
  query = []
88
92
  if (q = @options.params)
89
93
  query << URI.encode_www_form(q)
@@ -94,6 +98,7 @@ module HTTPX
94
98
 
95
99
  def drain_body
96
100
  return nil if @body.nil?
101
+
97
102
  @drainer ||= @body.each
98
103
  chunk = @drainer.next
99
104
  chunk.dup
@@ -109,6 +114,7 @@ module HTTPX
109
114
  class << self
110
115
  def new(*, options)
111
116
  return options.body if options.body.is_a?(self)
117
+
112
118
  super
113
119
  end
114
120
  end
@@ -123,6 +129,7 @@ module HTTPX
123
129
  Transcoder.registry("json").encode(options.json)
124
130
  end
125
131
  return if @body.nil?
132
+
126
133
  @headers["content-type"] ||= @body.content_type
127
134
  @headers["content-length"] = @body.bytesize unless unbounded_body?
128
135
  end
@@ -130,6 +137,7 @@ module HTTPX
130
137
  def each(&block)
131
138
  return enum_for(__method__) unless block_given?
132
139
  return if @body.nil?
140
+
133
141
  body = stream(@body)
134
142
  if body.respond_to?(:read)
135
143
  ::IO.copy_stream(body, ProcIO.new(block))
@@ -143,11 +151,13 @@ module HTTPX
143
151
  def empty?
144
152
  return true if @body.nil?
145
153
  return false if chunked?
154
+
146
155
  bytesize.zero?
147
156
  end
148
157
 
149
158
  def bytesize
150
159
  return 0 if @body.nil?
160
+
151
161
  if @body.respond_to?(:bytesize)
152
162
  @body.bytesize
153
163
  elsif @body.respond_to?(:size)
@@ -52,6 +52,7 @@ module HTTPX
52
52
  # do not use directly!
53
53
  def lookup(hostname, ttl)
54
54
  return unless @lookups.key?(hostname)
55
+
55
56
  @lookups[hostname] = @lookups[hostname].select do |address|
56
57
  address["TTL"] > ttl
57
58
  end
@@ -92,7 +93,7 @@ module HTTPX
92
93
  Resolv::DNS::Resource::IN::AAAA
93
94
  addresses << {
94
95
  "name" => question.to_s,
95
- "TTL" => value.ttl,
96
+ "TTL" => value.ttl,
96
97
  "data" => value.address.to_s,
97
98
  }
98
99
  end
@@ -22,111 +22,111 @@ module HTTPX
22
22
  use_get: false,
23
23
  }.freeze
24
24
 
25
- def_delegator :@channels, :empty?
25
+ def_delegator :@connections, :empty?
26
26
 
27
- def_delegators :@resolver_channel, :to_io, :call, :interests, :close
27
+ def_delegators :@resolver_connection, :to_io, :call, :interests, :close
28
28
 
29
- def initialize(connection, options)
30
- @connection = connection
29
+ def initialize(options)
31
30
  @options = Options.new(options)
32
31
  @resolver_options = Resolver::Options.new(DEFAULTS.merge(@options.resolver_options || {}))
33
32
  @_record_types = Hash.new { |types, host| types[host] = RECORD_TYPES.keys.dup }
34
33
  @queries = {}
35
34
  @requests = {}
36
- @channels = []
35
+ @connections = []
37
36
  @uri = URI(@resolver_options.uri)
38
37
  @uri_addresses = nil
39
38
  end
40
39
 
41
- def <<(channel)
40
+ def <<(connection)
42
41
  @uri_addresses ||= Resolv.getaddresses(@uri.host)
43
42
  if @uri_addresses.empty?
44
- ex = ResolveError.new("Can't resolve #{channel.uri.host}")
43
+ ex = ResolveError.new("Can't resolve #{connection.origin.host}")
45
44
  ex.set_backtrace(caller)
46
- emit(:error, channel, ex)
45
+ emit(:error, connection, ex)
47
46
  else
48
- early_resolve(channel) || resolve(channel)
47
+ early_resolve(connection) || resolve(connection)
49
48
  end
50
49
  end
51
50
 
52
51
  def timeout
53
- timeout = @options.timeout
54
- timeout.timeout
52
+ @connections.map(&:timeout).min
55
53
  end
56
54
 
57
55
  def closed?
58
- return true unless @resolver_channel
59
- resolver_channel.closed?
56
+ return true unless @resolver_connection
57
+
58
+ resolver_connection.closed?
60
59
  end
61
60
 
62
61
  private
63
62
 
64
- def resolver_channel
65
- @resolver_channel ||= find_channel(@uri, @options)
63
+ def pool
64
+ Thread.current[:httpx_connection_pool] ||= Pool.new
66
65
  end
67
66
 
68
- def resolve(channel = @channels.first, hostname = nil)
69
- return if @building_channel
70
- hostname = hostname || @queries.key(channel) || channel.uri.host
67
+ def resolver_connection
68
+ @resolver_connection ||= pool.find_connection(@uri, @options) || begin
69
+ @building_connection = true
70
+ connection = @options.connection_class.new("ssl", @uri, @options.merge(ssl: { alpn_protocols: %w[h2] }))
71
+ pool.init_connection(connection, @options)
72
+ emit_addresses(connection, @uri_addresses)
73
+ @building_connection = false
74
+ connection
75
+ end
76
+ end
77
+
78
+ def resolve(connection = @connections.first, hostname = nil)
79
+ return if @building_connection
80
+
81
+ hostname = hostname || @queries.key(connection) || connection.origin.host
71
82
  type = @_record_types[hostname].first
72
83
  log(label: "resolver: ") { "query #{type} for #{hostname}" }
73
84
  begin
74
85
  request = build_request(hostname, type)
75
- @requests[request] = channel
76
- resolver_channel.send(request)
77
- @queries[hostname] = channel
78
- @channels << channel
86
+ @requests[request] = connection
87
+ resolver_connection.send(request)
88
+ @queries[hostname] = connection
89
+ @connections << connection
79
90
  rescue Resolv::DNS::EncodeError, JSON::JSONError => e
80
- emit_resolve_error(channel, hostname, e)
91
+ emit_resolve_error(connection, hostname, e)
81
92
  end
82
93
  end
83
94
 
84
- def find_channel(_request, **options)
85
- @connection.find_channel(@uri) || begin
86
- @building_channel = true
87
- channel = @connection.build_channel(@uri, **options)
88
- emit_addresses(channel, @uri_addresses)
89
- set_channel_callbacks(channel)
90
- @building_channel = false
91
- channel
92
- end
93
- end
94
-
95
- def set_channel_callbacks(channel)
96
- channel.on(:response, &method(:on_response))
97
- channel.on(:promise, &method(:on_response))
98
- end
99
-
100
95
  def on_response(request, response)
101
96
  response.raise_for_status
102
97
  rescue Error => ex
103
- channel = @requests[request]
104
- hostname = @queries.key(channel)
98
+ connection = @requests[request]
99
+ hostname = @queries.key(connection)
105
100
  error = ResolveError.new("Can't resolve #{hostname}: #{ex.message}")
106
101
  error.set_backtrace(ex.backtrace)
107
- emit(:error, channel, error)
102
+ emit(:error, connection, error)
108
103
  else
109
104
  parse(response)
110
105
  ensure
111
106
  @requests.delete(request)
112
107
  end
113
108
 
109
+ def on_promise(_, stream)
110
+ log(level: 2, label: "#{stream.id}: ") { "refusing stream!" }
111
+ stream.refuse
112
+ end
113
+
114
114
  def parse(response)
115
- answers = begin
116
- decode_response_body(response)
115
+ begin
116
+ answers = decode_response_body(response)
117
117
  rescue Resolv::DNS::DecodeError, JSON::JSONError => e
118
- host, channel = @queries.first
118
+ host, connection = @queries.first
119
119
  if @_record_types[host].empty?
120
- emit_resolve_error(channel, host, e)
120
+ emit_resolve_error(connection, host, e)
121
121
  return
122
122
  end
123
123
  end
124
124
  if answers.empty?
125
- host, channel = @queries.first
125
+ host, connection = @queries.first
126
126
  @_record_types[host].shift
127
127
  if @_record_types[host].empty?
128
128
  @_record_types.delete(host)
129
- emit_resolve_error(channel, host)
129
+ emit_resolve_error(connection, host)
130
130
  return
131
131
  end
132
132
  else
@@ -136,9 +136,9 @@ module HTTPX
136
136
  if address.key?("alias")
137
137
  alias_address = answers[address["alias"]]
138
138
  if alias_address.nil?
139
- channel = @queries[hostname]
139
+ connection = @queries[hostname]
140
140
  @queries.delete(address["name"])
141
- resolve(channel, address["alias"])
141
+ resolve(connection, address["alias"])
142
142
  return # rubocop:disable Lint/NonLocalExitFromIterator
143
143
  else
144
144
  alias_address
@@ -148,15 +148,18 @@ module HTTPX
148
148
  end
149
149
  end.compact
150
150
  next if addresses.empty?
151
+
151
152
  hostname = hostname[0..-2] if hostname.end_with?(".")
152
- channel = @queries.delete(hostname)
153
- next unless channel # probably a retried query for which there's an answer
154
- @channels.delete(channel)
153
+ connection = @queries.delete(hostname)
154
+ next unless connection # probably a retried query for which there's an answer
155
+
156
+ @connections.delete(connection)
155
157
  Resolver.cached_lookup_set(hostname, addresses)
156
- emit_addresses(channel, addresses.map { |addr| addr["data"] })
158
+ emit_addresses(connection, addresses.map { |addr| addr["data"] })
157
159
  end
158
160
  end
159
- return if @channels.empty?
161
+ return if @connections.empty?
162
+
160
163
  resolve
161
164
  end
162
165
 
@@ -175,6 +178,8 @@ module HTTPX
175
178
  request.headers["content-type"] = "application/dns-message"
176
179
  request.headers["accept"] = "application/dns-message"
177
180
  end
181
+ request.on(:response, &method(:on_response).curry[request])
182
+ request.on(:promise, &method(:on_promise))
178
183
  request
179
184
  end
180
185
 
@@ -188,7 +193,8 @@ module HTTPX
188
193
  when "application/dns-udpwireformat",
189
194
  "application/dns-message"
190
195
  Resolver.decode_dns_answer(response.to_s)
191
- # TODO: what about the rest?
196
+ else
197
+ raise Error, "unsupported DNS mime-type (#{response.headers["content-type"]})"
192
198
  end
193
199
  end
194
200
  end