httpx 0.10.0 → 0.11.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (77) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +11 -3
  3. data/doc/release_notes/0_10_1.md +37 -0
  4. data/doc/release_notes/0_10_2.md +5 -0
  5. data/doc/release_notes/0_11_0.md +76 -0
  6. data/doc/release_notes/0_11_1.md +1 -0
  7. data/doc/release_notes/0_11_2.md +5 -0
  8. data/lib/httpx/adapters/datadog.rb +205 -0
  9. data/lib/httpx/adapters/faraday.rb +0 -2
  10. data/lib/httpx/adapters/webmock.rb +123 -0
  11. data/lib/httpx/chainable.rb +8 -7
  12. data/lib/httpx/connection.rb +4 -15
  13. data/lib/httpx/connection/http1.rb +14 -1
  14. data/lib/httpx/connection/http2.rb +15 -16
  15. data/lib/httpx/domain_name.rb +1 -3
  16. data/lib/httpx/errors.rb +3 -1
  17. data/lib/httpx/headers.rb +1 -0
  18. data/lib/httpx/io/ssl.rb +4 -8
  19. data/lib/httpx/io/udp.rb +4 -3
  20. data/lib/httpx/plugins/compression.rb +1 -1
  21. data/lib/httpx/plugins/cookies/set_cookie_parser.rb +1 -1
  22. data/lib/httpx/plugins/expect.rb +33 -8
  23. data/lib/httpx/plugins/multipart.rb +42 -23
  24. data/lib/httpx/plugins/multipart/encoder.rb +115 -0
  25. data/lib/httpx/plugins/multipart/mime_type_detector.rb +64 -0
  26. data/lib/httpx/plugins/multipart/part.rb +34 -0
  27. data/lib/httpx/plugins/proxy.rb +16 -2
  28. data/lib/httpx/plugins/proxy/socks4.rb +14 -16
  29. data/lib/httpx/plugins/proxy/socks5.rb +3 -2
  30. data/lib/httpx/plugins/push_promise.rb +2 -2
  31. data/lib/httpx/pool.rb +8 -14
  32. data/lib/httpx/request.rb +22 -12
  33. data/lib/httpx/resolver.rb +7 -6
  34. data/lib/httpx/resolver/https.rb +18 -23
  35. data/lib/httpx/resolver/native.rb +22 -19
  36. data/lib/httpx/resolver/resolver_mixin.rb +4 -2
  37. data/lib/httpx/resolver/system.rb +3 -3
  38. data/lib/httpx/selector.rb +9 -13
  39. data/lib/httpx/session.rb +24 -21
  40. data/lib/httpx/transcoder.rb +20 -0
  41. data/lib/httpx/transcoder/form.rb +9 -1
  42. data/lib/httpx/version.rb +1 -1
  43. data/sig/connection.rbs +84 -1
  44. data/sig/connection/http1.rbs +66 -0
  45. data/sig/connection/http2.rbs +73 -0
  46. data/sig/headers.rbs +3 -0
  47. data/sig/httpx.rbs +1 -0
  48. data/sig/options.rbs +3 -3
  49. data/sig/plugins/basic_authentication.rbs +1 -1
  50. data/sig/plugins/compression.rbs +1 -1
  51. data/sig/plugins/compression/brotli.rbs +1 -1
  52. data/sig/plugins/compression/deflate.rbs +1 -1
  53. data/sig/plugins/compression/gzip.rbs +1 -1
  54. data/sig/plugins/h2c.rbs +1 -1
  55. data/sig/plugins/multipart.rbs +29 -4
  56. data/sig/plugins/persistent.rbs +1 -1
  57. data/sig/plugins/proxy.rbs +2 -2
  58. data/sig/plugins/proxy/ssh.rbs +1 -1
  59. data/sig/plugins/rate_limiter.rbs +1 -1
  60. data/sig/pool.rbs +36 -2
  61. data/sig/request.rbs +2 -2
  62. data/sig/resolver.rbs +26 -0
  63. data/sig/resolver/https.rbs +51 -0
  64. data/sig/resolver/native.rbs +60 -0
  65. data/sig/resolver/resolver_mixin.rbs +27 -0
  66. data/sig/resolver/system.rbs +17 -0
  67. data/sig/response.rbs +2 -2
  68. data/sig/selector.rbs +20 -0
  69. data/sig/session.rbs +3 -3
  70. data/sig/transcoder.rbs +4 -2
  71. data/sig/transcoder/body.rbs +2 -0
  72. data/sig/transcoder/form.rbs +8 -2
  73. data/sig/transcoder/json.rbs +3 -1
  74. metadata +47 -48
  75. data/lib/httpx/resolver/options.rb +0 -25
  76. data/sig/missing.rbs +0 -12
  77. data/sig/test.rbs +0 -9
@@ -21,6 +21,7 @@ module HTTPX
21
21
  DEFAULTS = {
22
22
  uri: NAMESERVER,
23
23
  use_get: false,
24
+ record_types: RECORD_TYPES.keys,
24
25
  }.freeze
25
26
 
26
27
  def_delegator :@connections, :empty?
@@ -29,27 +30,29 @@ module HTTPX
29
30
 
30
31
  def initialize(options)
31
32
  @options = Options.new(options)
32
- @resolver_options = Resolver::Options.new(DEFAULTS.merge(@options.resolver_options || {}))
33
- @_record_types = Hash.new { |types, host| types[host] = RECORD_TYPES.keys.dup }
33
+ @resolver_options = DEFAULTS.merge(@options.resolver_options)
34
+ @_record_types = Hash.new { |types, host| types[host] = @resolver_options[:record_types].dup }
34
35
  @queries = {}
35
36
  @requests = {}
36
37
  @connections = []
37
- @uri = URI(@resolver_options.uri)
38
+ @uri = URI(@resolver_options[:uri])
38
39
  @uri_addresses = nil
40
+ @resolver = Resolv::DNS.new
41
+ @resolver.timeouts = @resolver_options.fetch(:timeouts, Resolver::RESOLVE_TIMEOUT)
39
42
  end
40
43
 
41
44
  def <<(connection)
42
45
  return if @uri.origin == connection.origin.to_s
43
46
 
44
- @uri_addresses ||= Resolv.getaddresses(@uri.host)
47
+ @uri_addresses ||= ip_resolve(@uri.host) || system_resolve(@uri.host) || @resolver.getaddresses(@uri.host)
45
48
 
46
49
  if @uri_addresses.empty?
47
- ex = ResolveError.new("Can't resolve #{connection.origin.host}")
50
+ ex = ResolveError.new("Can't resolve DNS server #{@uri.host}")
48
51
  ex.set_backtrace(caller)
49
- emit(:error, connection, ex)
50
- else
51
- early_resolve(connection) || resolve(connection)
52
+ throw(:resolve_error, ex)
52
53
  end
54
+
55
+ early_resolve(connection) || resolve(connection)
53
56
  end
54
57
 
55
58
  def timeout
@@ -70,12 +73,6 @@ module HTTPX
70
73
 
71
74
  private
72
75
 
73
- def connect
74
- return if @queries.empty?
75
-
76
- resolver_connection.__send__(__method__)
77
- end
78
-
79
76
  def pool
80
77
  Thread.current[:httpx_connection_pool] ||= Pool.new
81
78
  end
@@ -100,10 +97,12 @@ module HTTPX
100
97
  hostname = connection.origin.host
101
98
  log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
102
99
  end
103
- type = @_record_types[hostname].first
100
+ type = @_record_types[hostname].first || "A"
104
101
  log { "resolver: query #{type} for #{hostname}" }
105
102
  begin
106
103
  request = build_request(hostname, type)
104
+ request.on(:response, &method(:on_response).curry(2)[request])
105
+ request.on(:promise, &method(:on_promise))
107
106
  @requests[request] = connection
108
107
  resolver_connection.send(request)
109
108
  @queries[hostname] = connection
@@ -118,9 +117,7 @@ module HTTPX
118
117
  rescue StandardError => e
119
118
  connection = @requests[request]
120
119
  hostname = @queries.key(connection)
121
- error = ResolveError.new("Can't resolve #{hostname}: #{e.message}")
122
- error.set_backtrace(e.backtrace)
123
- emit(:error, connection, error)
120
+ emit_resolve_error(connection, hostname, e)
124
121
  else
125
122
  parse(response)
126
123
  ensure
@@ -143,7 +140,7 @@ module HTTPX
143
140
  return
144
141
  end
145
142
  end
146
- if answers.empty?
143
+ if answers.nil? || answers.empty?
147
144
  host, connection = @queries.first
148
145
  @_record_types[host].shift
149
146
  if @_record_types[host].empty?
@@ -177,7 +174,7 @@ module HTTPX
177
174
  next unless connection # probably a retried query for which there's an answer
178
175
 
179
176
  @connections.delete(connection)
180
- Resolver.cached_lookup_set(hostname, addresses) if @resolver_options.cache
177
+ Resolver.cached_lookup_set(hostname, addresses) if @resolver_options[:cache]
181
178
  emit_addresses(connection, addresses.map { |addr| addr["data"] })
182
179
  end
183
180
  end
@@ -191,7 +188,7 @@ module HTTPX
191
188
  rklass = @options.request_class
192
189
  payload = Resolver.encode_dns_query(hostname, type: RECORD_TYPES[type])
193
190
 
194
- if @resolver_options.use_get
191
+ if @resolver_options[:use_get]
195
192
  params = URI.decode_www_form(uri.query.to_s)
196
193
  params << ["type", type]
197
194
  params << ["dns", Base64.urlsafe_encode64(payload, padding: false)]
@@ -202,8 +199,6 @@ module HTTPX
202
199
  request.headers["content-type"] = "application/dns-message"
203
200
  end
204
201
  request.headers["accept"] = "application/dns-message"
205
- request.on(:response, &method(:on_response).curry[request])
206
- request.on(:promise, &method(:on_promise))
207
202
  request
208
203
  end
209
204
 
@@ -9,18 +9,16 @@ module HTTPX
9
9
  include Resolver::ResolverMixin
10
10
  using URIExtensions
11
11
 
12
- RESOLVE_TIMEOUT = 5
13
12
  RECORD_TYPES = {
14
13
  "A" => Resolv::DNS::Resource::IN::A,
15
14
  "AAAA" => Resolv::DNS::Resource::IN::AAAA,
16
15
  }.freeze
17
16
 
18
- # :nocov:
19
17
  DEFAULTS = if RUBY_VERSION < "2.2"
20
18
  {
21
19
  **Resolv::DNS::Config.default_config_hash,
22
20
  packet_size: 512,
23
- timeouts: RESOLVE_TIMEOUT,
21
+ timeouts: Resolver::RESOLVE_TIMEOUT,
24
22
  record_types: RECORD_TYPES.keys,
25
23
  }.freeze
26
24
  else
@@ -28,7 +26,7 @@ module HTTPX
28
26
  nameserver: nil,
29
27
  **Resolv::DNS::Config.default_config_hash,
30
28
  packet_size: 512,
31
- timeouts: RESOLVE_TIMEOUT,
29
+ timeouts: Resolver::RESOLVE_TIMEOUT,
32
30
  record_types: RECORD_TYPES.keys,
33
31
  }.freeze
34
32
  end
@@ -44,7 +42,6 @@ module HTTPX
44
42
  false
45
43
  end
46
44
  end if DEFAULTS[:nameserver]
47
- # :nocov:
48
45
 
49
46
  DNS_PORT = 53
50
47
 
@@ -53,15 +50,15 @@ module HTTPX
53
50
  def initialize(options)
54
51
  @options = Options.new(options)
55
52
  @ns_index = 0
56
- @resolver_options = Resolver::Options.new(DEFAULTS.merge(@options.resolver_options || {}))
57
- @nameserver = @resolver_options.nameserver
58
- @_timeouts = Array(@resolver_options.timeouts)
53
+ @resolver_options = DEFAULTS.merge(@options.resolver_options)
54
+ @nameserver = @resolver_options[:nameserver]
55
+ @_timeouts = Array(@resolver_options[:timeouts])
59
56
  @timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
60
- @_record_types = Hash.new { |types, host| types[host] = @resolver_options.record_types.dup }
57
+ @_record_types = Hash.new { |types, host| types[host] = @resolver_options[:record_types].dup }
61
58
  @connections = []
62
59
  @queries = {}
63
60
  @read_buffer = "".b
64
- @write_buffer = Buffer.new(@resolver_options.packet_size)
61
+ @write_buffer = Buffer.new(@resolver_options[:packet_size])
65
62
  @state = :idle
66
63
  end
67
64
 
@@ -111,9 +108,9 @@ module HTTPX
111
108
  return if early_resolve(connection)
112
109
 
113
110
  if @nameserver.nil?
114
- ex = ResolveError.new("Can't resolve #{connection.origin.host}: no nameserver")
111
+ ex = ResolveError.new("No available nameserver")
115
112
  ex.set_backtrace(caller)
116
- emit(:error, connection, ex)
113
+ throw(:resolve_error, ex)
117
114
  else
118
115
  @connections << connection
119
116
  resolve
@@ -150,21 +147,26 @@ module HTTPX
150
147
  queries[h] = connection
151
148
  next
152
149
  end
150
+
153
151
  @timeouts[host].shift
154
152
  if @timeouts[host].empty?
155
153
  @timeouts.delete(host)
156
154
  @connections.delete(connection)
157
- raise NativeResolveError.new(connection, host)
155
+ # This loop_time passed to the exception is bogus. Ideally we would pass the total
156
+ # resolve timeout, including from the previous retries.
157
+ raise ResolveTimeoutError.new(loop_time, "Timed out")
158
+ # raise NativeResolveError.new(connection, host)
158
159
  else
160
+ log { "resolver: timeout after #{timeout}s, retry(#{@timeouts[host].first}) #{host}..." }
159
161
  connections << connection
160
- log { "resolver: timeout after #{prev_timeout}s, retry(#{timeouts.first}) #{host}..." }
162
+ queries[h] = connection
161
163
  end
162
164
  end
163
165
  @queries = queries
164
166
  connections.each { |ch| resolve(ch) }
165
167
  end
166
168
 
167
- def dread(wsize = @resolver_options.packet_size)
169
+ def dread(wsize = @resolver_options[:packet_size])
168
170
  loop do
169
171
  siz = @io.read(wsize, @read_buffer)
170
172
  return unless siz && siz.positive?
@@ -199,13 +201,14 @@ module HTTPX
199
201
  end
200
202
  end
201
203
 
202
- if addresses.empty?
204
+ if addresses.nil? || addresses.empty?
203
205
  hostname, connection = @queries.first
204
206
  @_record_types[hostname].shift
205
207
  if @_record_types[hostname].empty?
206
208
  @queries.delete(hostname)
207
209
  @_record_types.delete(hostname)
208
210
  @connections.delete(connection)
211
+
209
212
  raise NativeResolveError.new(connection, hostname)
210
213
  end
211
214
  else
@@ -223,7 +226,7 @@ module HTTPX
223
226
  end
224
227
  else
225
228
  @connections.delete(connection)
226
- Resolver.cached_lookup_set(connection.origin.host, addresses) if @resolver_options.cache
229
+ Resolver.cached_lookup_set(connection.origin.host, addresses) if @resolver_options[:cache]
227
230
  emit_addresses(connection, addresses.map { |addr| addr["data"] })
228
231
  end
229
232
  end
@@ -243,7 +246,7 @@ module HTTPX
243
246
  log { "resolver: resolve IDN #{connection.origin.non_ascii_hostname} as #{hostname}" } if connection.origin.non_ascii_hostname
244
247
  end
245
248
  @queries[hostname] = connection
246
- type = @_record_types[hostname].first
249
+ type = @_record_types[hostname].first || "A"
247
250
  log { "resolver: query #{type} for #{hostname}" }
248
251
  begin
249
252
  @write_buffer << Resolver.encode_dns_query(hostname, type: RECORD_TYPES[type])
@@ -280,7 +283,7 @@ module HTTPX
280
283
  @io.connect
281
284
  return unless @io.connected?
282
285
 
283
- resolve if @queries.empty?
286
+ resolve if @queries.empty? && !@connections.empty?
284
287
  when :closed
285
288
  return unless @state == :open
286
289
 
@@ -38,7 +38,7 @@ module HTTPX
38
38
  def early_resolve(connection, hostname: connection.origin.host)
39
39
  addresses = connection.addresses ||
40
40
  ip_resolve(hostname) ||
41
- (@resolver_options.cache && Resolver.cached_lookup(hostname)) ||
41
+ (@resolver_options[:cache] && Resolver.cached_lookup(hostname)) ||
42
42
  system_resolve(hostname)
43
43
  return unless addresses
44
44
 
@@ -57,11 +57,13 @@ module HTTPX
57
57
  ips.map { |ip| IPAddr.new(ip) }
58
58
  end
59
59
 
60
- def emit_resolve_error(connection, hostname, ex = nil)
60
+ def emit_resolve_error(connection, hostname = connection.origin.host, ex = nil)
61
61
  emit(:error, connection, resolve_error(hostname, ex))
62
62
  end
63
63
 
64
64
  def resolve_error(hostname, ex = nil)
65
+ return ex if ex.is_a?(ResolveError)
66
+
65
67
  message = ex ? ex.message : "Can't resolve #{hostname}"
66
68
  error = ResolveError.new(message)
67
69
  error.set_backtrace(ex ? ex.backtrace : caller)
@@ -14,13 +14,13 @@ module HTTPX
14
14
 
15
15
  def initialize(options)
16
16
  @options = Options.new(options)
17
- @resolver_options = Resolver::Options.new(@options.resolver_options)
17
+ @resolver_options = @options.resolver_options
18
18
  @state = :idle
19
- resolv_options = @resolver_options.to_h
19
+ resolv_options = @resolver_options.dup
20
20
  timeouts = resolv_options.delete(:timeouts)
21
21
  resolv_options.delete(:cache)
22
22
  @resolver = Resolv::DNS.new(resolv_options.empty? ? nil : resolv_options)
23
- @resolver.timeouts = timeouts if timeouts
23
+ @resolver.timeouts = timeouts || Resolver::RESOLVE_TIMEOUT
24
24
  end
25
25
 
26
26
  def closed?
@@ -4,19 +4,14 @@ require "io/wait"
4
4
 
5
5
  module IOExtensions
6
6
  refine IO do
7
- def wait(timeout = nil, mode = :read)
8
- case mode
9
- when :read
10
- wait_readable(timeout)
11
- when :write
12
- wait_writable(timeout)
13
- when :read_write
14
- r, w = IO.select([self], [self], nil, timeout)
15
-
16
- return unless r || w
17
-
18
- self
19
- end
7
+ # provides a fallback for rubies where IO#wait isn't implemented,
8
+ # but IO#wait_readable and IO#wait_writable are.
9
+ def wait(timeout = nil, _mode = :read_write)
10
+ r, w = IO.select([self], [self], nil, timeout)
11
+
12
+ return unless r || w
13
+
14
+ self
20
15
  end
21
16
  end
22
17
  end
@@ -122,6 +117,7 @@ class HTTPX::Selector
122
117
  yield io
123
118
  rescue IOError, SystemCallError
124
119
  @selectables.reject!(&:closed?)
120
+ raise unless @selectables.empty?
125
121
  end
126
122
 
127
123
  def select(interval, &block)
data/lib/httpx/session.rb CHANGED
@@ -41,7 +41,7 @@ module HTTPX
41
41
  def build_request(verb, uri, options = EMPTY_HASH)
42
42
  rklass = @options.request_class
43
43
  request = rklass.new(verb, uri, @options.merge(options).merge(persistent: @persistent))
44
- request.on(:response, &method(:on_response).curry[request])
44
+ request.on(:response, &method(:on_response).curry(2)[request])
45
45
  request.on(:promise, &method(:on_promise))
46
46
  request
47
47
  end
@@ -77,10 +77,16 @@ module HTTPX
77
77
  end
78
78
 
79
79
  def set_connection_callbacks(connection, connections, options)
80
- connection.on(:uncoalesce) do |uncoalesced_uri|
81
- other_connection = build_connection(uncoalesced_uri, options)
80
+ connection.on(:misdirected) do |misdirected_request|
81
+ other_connection = connection.create_idle(ssl: { alpn_protocols: %w[http/1.1] })
82
+ other_connection.merge(connection)
83
+ catch(:coalesced) do
84
+ pool.init_connection(other_connection, options)
85
+ end
86
+ set_connection_callbacks(other_connection, connections, options)
82
87
  connections << other_connection
83
- connection.unmerge(other_connection)
88
+ misdirected_request.transition(:idle)
89
+ other_connection.send(misdirected_request)
84
90
  end
85
91
  connection.on(:altsvc) do |alt_origin, origin, alt_params|
86
92
  other_connection = build_altsvc_connection(connection, connections, alt_origin, origin, alt_params, options)
@@ -130,23 +136,20 @@ module HTTPX
130
136
  def build_requests(*args, options)
131
137
  request_options = @options.merge(options)
132
138
 
133
- requests = case args.size
134
- when 1
135
- reqs = args.first
136
- reqs.map do |verb, uri, opts = EMPTY_HASH|
137
- build_request(verb, uri, request_options.merge(opts))
138
- end
139
- when 2
140
- verb, uris = args
141
- if uris.respond_to?(:each)
142
- uris.enum_for(:each).map do |uri, opts = EMPTY_HASH|
143
- build_request(verb, uri, request_options.merge(opts))
144
- end
145
- else
146
- [build_request(verb, uris, request_options)]
147
- end
148
- else
149
- raise ArgumentError, "unsupported number of arguments"
139
+ requests = if args.size == 1
140
+ reqs = args.first
141
+ reqs.map do |verb, uri, opts = EMPTY_HASH|
142
+ build_request(verb, uri, request_options.merge(opts))
143
+ end
144
+ else
145
+ verb, uris = args
146
+ if uris.respond_to?(:each)
147
+ uris.enum_for(:each).map do |uri, opts = EMPTY_HASH|
148
+ build_request(verb, uri, request_options.merge(opts))
149
+ end
150
+ else
151
+ [build_request(verb, uris, request_options)]
152
+ end
150
153
  end
151
154
  raise ArgumentError, "wrong number of URIs (given 0, expect 1..+1)" if requests.empty?
152
155
 
@@ -3,6 +3,26 @@
3
3
  module HTTPX
4
4
  module Transcoder
5
5
  extend Registry
6
+
7
+ def self.normalize_keys(key, value, cond = nil, &block)
8
+ if (cond && cond.call(value))
9
+ block.call(key.to_s, value)
10
+ elsif value.respond_to?(:to_ary)
11
+ if value.empty?
12
+ block.call("#{key}[]")
13
+ else
14
+ value.to_ary.each do |element|
15
+ normalize_keys("#{key}[]", element, cond, &block)
16
+ end
17
+ end
18
+ elsif value.respond_to?(:to_hash)
19
+ value.to_hash.each do |child_key, child_value|
20
+ normalize_keys("#{key}[#{child_key}]", child_value, cond, &block)
21
+ end
22
+ else
23
+ block.call(key.to_s, value)
24
+ end
25
+ end
6
26
  end
7
27
  end
8
28
 
@@ -12,10 +12,18 @@ module HTTPX::Transcoder
12
12
 
13
13
  def_delegator :@raw, :to_s
14
14
 
15
+ def_delegator :@raw, :to_str
16
+
15
17
  def_delegator :@raw, :bytesize
16
18
 
17
19
  def initialize(form)
18
- @raw = URI.encode_www_form(form)
20
+ @raw = form.each_with_object("".b) do |(key, val), buf|
21
+ HTTPX::Transcoder.normalize_keys(key, val) do |k, v|
22
+ buf << "&" unless buf.empty?
23
+ buf << URI.encode_www_form_component(k)
24
+ buf << "=#{URI.encode_www_form_component(v.to_s)}" unless v.nil?
25
+ end
26
+ end
19
27
  end
20
28
 
21
29
  def content_type