httpx 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/lib/httpx.rb +8 -2
  3. data/lib/httpx/adapters/faraday.rb +203 -0
  4. data/lib/httpx/altsvc.rb +4 -0
  5. data/lib/httpx/callbacks.rb +1 -4
  6. data/lib/httpx/chainable.rb +4 -3
  7. data/lib/httpx/connection.rb +326 -104
  8. data/lib/httpx/{channel → connection}/http1.rb +29 -15
  9. data/lib/httpx/{channel → connection}/http2.rb +12 -6
  10. data/lib/httpx/errors.rb +2 -0
  11. data/lib/httpx/headers.rb +4 -1
  12. data/lib/httpx/io/ssl.rb +5 -1
  13. data/lib/httpx/io/tcp.rb +13 -7
  14. data/lib/httpx/io/udp.rb +1 -0
  15. data/lib/httpx/io/unix.rb +1 -0
  16. data/lib/httpx/loggable.rb +34 -9
  17. data/lib/httpx/options.rb +57 -31
  18. data/lib/httpx/parser/http1.rb +8 -0
  19. data/lib/httpx/plugins/authentication.rb +4 -0
  20. data/lib/httpx/plugins/basic_authentication.rb +4 -0
  21. data/lib/httpx/plugins/compression.rb +22 -5
  22. data/lib/httpx/plugins/cookies.rb +89 -36
  23. data/lib/httpx/plugins/digest_authentication.rb +45 -26
  24. data/lib/httpx/plugins/follow_redirects.rb +61 -62
  25. data/lib/httpx/plugins/h2c.rb +78 -39
  26. data/lib/httpx/plugins/multipart.rb +5 -0
  27. data/lib/httpx/plugins/persistent.rb +29 -0
  28. data/lib/httpx/plugins/proxy.rb +125 -78
  29. data/lib/httpx/plugins/proxy/http.rb +31 -27
  30. data/lib/httpx/plugins/proxy/socks4.rb +30 -24
  31. data/lib/httpx/plugins/proxy/socks5.rb +49 -39
  32. data/lib/httpx/plugins/proxy/ssh.rb +81 -0
  33. data/lib/httpx/plugins/push_promise.rb +18 -9
  34. data/lib/httpx/plugins/retries.rb +43 -15
  35. data/lib/httpx/pool.rb +159 -0
  36. data/lib/httpx/registry.rb +2 -0
  37. data/lib/httpx/request.rb +10 -0
  38. data/lib/httpx/resolver.rb +2 -1
  39. data/lib/httpx/resolver/https.rb +62 -56
  40. data/lib/httpx/resolver/native.rb +48 -37
  41. data/lib/httpx/resolver/resolver_mixin.rb +16 -11
  42. data/lib/httpx/resolver/system.rb +11 -7
  43. data/lib/httpx/response.rb +24 -10
  44. data/lib/httpx/selector.rb +32 -39
  45. data/lib/httpx/{client.rb → session.rb} +99 -62
  46. data/lib/httpx/timeout.rb +7 -15
  47. data/lib/httpx/transcoder/body.rb +4 -0
  48. data/lib/httpx/transcoder/chunker.rb +4 -0
  49. data/lib/httpx/version.rb +1 -1
  50. metadata +10 -8
  51. data/lib/httpx/channel.rb +0 -367
@@ -33,9 +33,9 @@ module HTTPX
33
33
 
34
34
  DNS_PORT = 53
35
35
 
36
- def_delegator :@channels, :empty?
36
+ def_delegator :@connections, :empty?
37
37
 
38
- def initialize(_, options)
38
+ def initialize(options)
39
39
  @options = Options.new(options)
40
40
  @ns_index = 0
41
41
  @resolver_options = Resolver::Options.new(DEFAULTS.merge(@options.resolver_options || {}))
@@ -43,7 +43,7 @@ module HTTPX
43
43
  @_timeouts = Array(@resolver_options.timeouts)
44
44
  @timeouts = Hash.new { |timeouts, host| timeouts[host] = @_timeouts.dup }
45
45
  @_record_types = Hash.new { |types, host| types[host] = @resolver_options.record_types.dup }
46
- @channels = []
46
+ @connections = []
47
47
  @queries = {}
48
48
  @read_buffer = Buffer.new(@resolver_options.packet_size)
49
49
  @write_buffer = Buffer.new(@resolver_options.packet_size)
@@ -81,8 +81,8 @@ module HTTPX
81
81
  if @ns_index < @nameserver.size
82
82
  transition(:idle)
83
83
  else
84
- @queries.each do |host, channel|
85
- emit_resolve_error(channel, host, e)
84
+ @queries.each do |host, connection|
85
+ emit_resolve_error(connection, host, e)
86
86
  end
87
87
  end
88
88
  end
@@ -97,14 +97,16 @@ module HTTPX
97
97
  end
98
98
  end
99
99
 
100
- def <<(channel)
101
- return if early_resolve(channel)
100
+ def <<(connection)
101
+ return if early_resolve(connection)
102
+
102
103
  if @nameserver.nil?
103
- ex = ResolveError.new("Can't resolve #{channel.uri.host}: no nameserver")
104
+ ex = ResolveError.new("Can't resolve #{connection.origin.host}: no nameserver")
104
105
  ex.set_backtrace(caller)
105
- emit(:error, channel, ex)
106
+ emit(:error, connection, ex)
106
107
  else
107
- @channels << channel
108
+ @connections << connection
109
+ resolve
108
110
  end
109
111
  end
110
112
 
@@ -124,31 +126,32 @@ module HTTPX
124
126
 
125
127
  def do_retry
126
128
  return if @queries.empty?
129
+
127
130
  loop_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @start_timeout
128
- channels = []
131
+ connections = []
129
132
  queries = {}
130
133
  while (query = @queries.shift)
131
- h, channel = query
132
- host = channel.uri.host
134
+ h, connection = query
135
+ host = connection.origin.host
133
136
  timeout = (@timeouts[host][0] -= loop_time)
134
137
  unless timeout.negative?
135
- queries[h] = channel
138
+ queries[h] = connection
136
139
  next
137
140
  end
138
141
  @timeouts[host].shift
139
142
  if @timeouts[host].empty?
140
143
  @timeouts.delete(host)
141
- emit_resolve_error(channel, host)
144
+ emit_resolve_error(connection, host)
142
145
  return
143
146
  else
144
- channels << channel
147
+ connections << connection
145
148
  log(label: "resolver: ") do
146
149
  "timeout after #{prev_timeout}s, retry(#{timeouts.first}) #{host}..."
147
150
  end
148
151
  end
149
152
  end
150
153
  @queries = queries
151
- channels.each { |ch| resolve(ch) }
154
+ connections.each { |ch| resolve(ch) }
152
155
  end
153
156
 
154
157
  def dread(wsize = @read_buffer.limit)
@@ -159,6 +162,7 @@ module HTTPX
159
162
  return
160
163
  end
161
164
  return if siz.zero?
165
+
162
166
  log(label: "resolver: ") { "READ: #{siz} bytes..." }
163
167
  parse(@read_buffer.to_s)
164
168
  end
@@ -167,6 +171,7 @@ module HTTPX
167
171
  def dwrite
168
172
  loop do
169
173
  return if @write_buffer.empty?
174
+
170
175
  siz = @io.write(@write_buffer)
171
176
  unless siz
172
177
  emit(:close)
@@ -178,62 +183,66 @@ module HTTPX
178
183
  end
179
184
 
180
185
  def parse(buffer)
181
- addresses = begin
182
- Resolver.decode_dns_answer(buffer)
186
+ begin
187
+ addresses = Resolver.decode_dns_answer(buffer)
183
188
  rescue Resolv::DNS::DecodeError => e
184
- hostname, channel = @queries.first
189
+ hostname, connection = @queries.first
185
190
  if @_record_types[hostname].empty?
186
- emit_resolve_error(channel, hostname, e)
191
+ emit_resolve_error(connection, hostname, e)
187
192
  return
188
193
  end
189
194
  end
190
195
 
191
196
  if addresses.empty?
192
- hostname, channel = @queries.first
197
+ hostname, connection = @queries.first
193
198
  @_record_types[hostname].shift
194
199
  if @_record_types[hostname].empty?
195
200
  @_record_types.delete(hostname)
196
- emit_resolve_error(channel, hostname)
201
+ emit_resolve_error(connection, hostname)
197
202
  return
198
203
  end
199
204
  else
200
205
  address = addresses.first
201
- channel = @queries.delete(address["name"])
202
- return unless channel # probably a retried query for which there's an answer
206
+ connection = @queries.delete(address["name"])
207
+ return unless connection # probably a retried query for which there's an answer
208
+
203
209
  if address.key?("alias") # CNAME
204
- if early_resolve(channel, hostname: address["alias"])
205
- @channels.delete(channel)
210
+ if early_resolve(connection, hostname: address["alias"])
211
+ @connections.delete(connection)
206
212
  else
207
- resolve(channel, address["alias"])
213
+ resolve(connection, address["alias"])
208
214
  @queries.delete(address["name"])
209
215
  return
210
216
  end
211
217
  else
212
- @channels.delete(channel)
213
- Resolver.cached_lookup_set(channel.uri.host, addresses)
214
- emit_addresses(channel, addresses.map { |addr| addr["data"] })
218
+ @connections.delete(connection)
219
+ Resolver.cached_lookup_set(connection.origin.host, addresses)
220
+ emit_addresses(connection, addresses.map { |addr| addr["data"] })
215
221
  end
216
222
  end
217
- return emit(:close) if @channels.empty?
223
+ return emit(:close) if @connections.empty?
224
+
218
225
  resolve
219
226
  end
220
227
 
221
- def resolve(channel = @channels.first, hostname = nil)
222
- raise Error, "no URI to resolve" unless channel
228
+ def resolve(connection = @connections.first, hostname = nil)
229
+ raise Error, "no URI to resolve" unless connection
223
230
  return unless @write_buffer.empty?
224
- hostname = hostname || @queries.key(channel) || channel.uri.host
225
- @queries[hostname] = channel
231
+
232
+ hostname = hostname || @queries.key(connection) || connection.origin.host
233
+ @queries[hostname] = connection
226
234
  type = @_record_types[hostname].first
227
235
  log(label: "resolver: ") { "query #{type} for #{hostname}" }
228
236
  begin
229
237
  @write_buffer << Resolver.encode_dns_query(hostname, type: RECORD_TYPES[type])
230
238
  rescue Resolv::DNS::EncodeError => e
231
- emit_resolve_error(channel, hostname, e)
239
+ emit_resolve_error(connection, hostname, e)
232
240
  end
233
241
  end
234
242
 
235
243
  def build_socket
236
244
  return if @io
245
+
237
246
  ip, port = @nameserver[@ns_index]
238
247
  port ||= DNS_PORT
239
248
  uri = URI::Generic.build(scheme: "udp", port: port)
@@ -253,11 +262,13 @@ module HTTPX
253
262
  @timeouts.clear
254
263
  when :open
255
264
  return unless @state == :idle
265
+
256
266
  build_socket
257
267
  @io.connect
258
268
  return unless @io.connected?
259
269
  when :closed
260
270
  return unless @state == :open
271
+
261
272
  @io.close if @io
262
273
  end
263
274
  @state = nextstate
@@ -18,27 +18,31 @@ module HTTPX
18
18
  end
19
19
  end
20
20
 
21
- def uncache(channel)
22
- hostname = hostname || @queries.key(channel) || channel.uri.host
21
+ def uncache(connection)
22
+ hostname = hostname || @queries.key(connection) || connection.origin.host
23
23
  Resolver.uncache(hostname)
24
24
  @_record_types[hostname].shift
25
25
  end
26
26
 
27
27
  private
28
28
 
29
- def emit_addresses(channel, addresses)
29
+ def emit_addresses(connection, addresses)
30
30
  addresses.map! do |address|
31
31
  address.is_a?(IPAddr) ? address : IPAddr.new(address.to_s)
32
32
  end
33
- log(label: "resolver: ") { "answer #{channel.uri.host}: #{addresses.inspect}" }
34
- channel.addresses = addresses
35
- emit(:resolve, channel, addresses)
33
+ log(label: "resolver: ") { "answer #{connection.origin.host}: #{addresses.inspect}" }
34
+ connection.addresses = addresses
35
+ emit(:resolve, connection)
36
36
  end
37
37
 
38
- def early_resolve(channel, hostname: channel.uri.host)
39
- addresses = ip_resolve(hostname) || Resolver.cached_lookup(hostname) || system_resolve(hostname)
38
+ def early_resolve(connection, hostname: connection.origin.host)
39
+ addresses = connection.addresses ||
40
+ ip_resolve(hostname) ||
41
+ Resolver.cached_lookup(hostname) ||
42
+ system_resolve(hostname)
40
43
  return unless addresses
41
- emit_addresses(channel, addresses)
44
+
45
+ emit_addresses(connection, addresses)
42
46
  end
43
47
 
44
48
  def ip_resolve(hostname)
@@ -49,14 +53,15 @@ module HTTPX
49
53
  @system_resolver ||= Resolv::Hosts.new
50
54
  ips = @system_resolver.getaddresses(hostname)
51
55
  return if ips.empty?
56
+
52
57
  ips.map { |ip| IPAddr.new(ip) }
53
58
  end
54
59
 
55
- def emit_resolve_error(channel, hostname, ex = nil)
60
+ def emit_resolve_error(connection, hostname, ex = nil)
56
61
  message = ex ? ex.message : "Can't resolve #{hostname}"
57
62
  error = ResolveError.new(message)
58
63
  error.set_backtrace(ex ? ex.backtrace : caller)
59
- emit(:error, channel, error)
64
+ emit(:error, connection, error)
60
65
  end
61
66
  end
62
67
  end
@@ -12,7 +12,7 @@ module HTTPX
12
12
  Resolv::DNS::EncodeError,
13
13
  Resolv::DNS::DecodeError].freeze
14
14
 
15
- def initialize(_, options)
15
+ def initialize(options)
16
16
  @options = Options.new(options)
17
17
  roptions = @options.resolver_options
18
18
  @state = :idle
@@ -28,13 +28,17 @@ module HTTPX
28
28
  true
29
29
  end
30
30
 
31
- def <<(channel)
32
- hostname = channel.uri.host
33
- addresses = ip_resolve(hostname) || system_resolve(hostname) || @resolver.getaddresses(hostname)
34
- return emit_resolve_error(channel, hostname) if addresses.empty?
35
- emit_addresses(channel, addresses)
31
+ def <<(connection)
32
+ hostname = connection.origin.host
33
+ addresses = connection.addresses ||
34
+ ip_resolve(hostname) ||
35
+ system_resolve(hostname) ||
36
+ @resolver.getaddresses(hostname)
37
+ return emit_resolve_error(connection, hostname) if addresses.empty?
38
+
39
+ emit_addresses(connection, addresses)
36
40
  rescue Errno::EHOSTUNREACH, *RESOLV_ERRORS => e
37
- emit_resolve_error(channel, hostname, e)
41
+ emit_resolve_error(connection, hostname, e)
38
42
  end
39
43
 
40
44
  def uncache(*); end
@@ -21,10 +21,10 @@ module HTTPX
21
21
 
22
22
  def_delegator :@request, :uri
23
23
 
24
- def initialize(request, status, version, headers, options = {})
25
- @options = Options.new(options)
26
- @version = version
24
+ def initialize(request, status, version, headers)
27
25
  @request = request
26
+ @options = request.options
27
+ @version = version
28
28
  @status = Integer(status)
29
29
  @headers = @options.headers_class.new(headers)
30
30
  @body = @options.response_body_class.new(self, threshold_size: @options.body_threshold_size,
@@ -58,6 +58,7 @@ module HTTPX
58
58
 
59
59
  def raise_for_status
60
60
  return if @status < 400
61
+
61
62
  raise HTTPError, self
62
63
  end
63
64
 
@@ -70,6 +71,7 @@ module HTTPX
70
71
  @status == 304 || begin
71
72
  content_length = @headers["content-length"]
72
73
  return false if content_length.nil?
74
+
73
75
  content_length == "0"
74
76
  end
75
77
  end
@@ -94,6 +96,7 @@ module HTTPX
94
96
 
95
97
  def read(*args)
96
98
  return unless @buffer
99
+
97
100
  @buffer.read(*args)
98
101
  end
99
102
 
@@ -103,6 +106,7 @@ module HTTPX
103
106
 
104
107
  def each
105
108
  return enum_for(__method__) unless block_given?
109
+
106
110
  begin
107
111
  unless @state == :idle
108
112
  rewind
@@ -137,6 +141,7 @@ module HTTPX
137
141
 
138
142
  def copy_to(dest)
139
143
  return unless @buffer
144
+
140
145
  if dest.respond_to?(:path) && @buffer.respond_to?(:path)
141
146
  FileUtils.mv(@buffer.path, dest.path)
142
147
  else
@@ -148,6 +153,7 @@ module HTTPX
148
153
  # closes/cleans the buffer, resets everything
149
154
  def close
150
155
  return if @state == :idle
156
+
151
157
  @buffer.close
152
158
  @buffer.unlink if @buffer.respond_to?(:unlink)
153
159
  @buffer = nil
@@ -163,6 +169,7 @@ module HTTPX
163
169
 
164
170
  def rewind
165
171
  return if @state == :idle
172
+
166
173
  @buffer.rewind
167
174
  end
168
175
 
@@ -182,10 +189,9 @@ module HTTPX
182
189
  @buffer = Tempfile.new("httpx", encoding: @encoding, mode: File::RDWR)
183
190
  aux.rewind
184
191
  ::IO.copy_stream(aux, @buffer)
185
- # TODO: remove this if/when minor ruby is 2.3
186
- # (this looks like a bug from older versions)
187
- @buffer.pos = aux.pos #######################
188
- #############################################
192
+ # (this looks like a bug from Ruby < 2.3
193
+ @buffer.pos = aux.pos ##################
194
+ ########################################
189
195
  aux.close
190
196
  @state = :buffer
191
197
  end
@@ -197,8 +203,8 @@ module HTTPX
197
203
  end
198
204
 
199
205
  class ContentType
200
- MIME_TYPE_RE = %r{^([^/]+/[^;]+)(?:$|;)}
201
- CHARSET_RE = /;\s*charset=([^;]+)/i
206
+ MIME_TYPE_RE = %r{^([^/]+/[^;]+)(?:$|;)}.freeze
207
+ CHARSET_RE = /;\s*charset=([^;]+)/i.freeze
202
208
 
203
209
  attr_reader :mime_type, :charset
204
210
 
@@ -237,7 +243,7 @@ module HTTPX
237
243
  def initialize(error, options)
238
244
  @error = error
239
245
  @options = Options.new(options)
240
- log { "#{error.class}: #{error}" }
246
+ log_exception(@error)
241
247
  end
242
248
 
243
249
  def status
@@ -247,5 +253,13 @@ module HTTPX
247
253
  def raise_for_status
248
254
  raise @error
249
255
  end
256
+
257
+ # rubocop:disable Style/MissingRespondToMissing
258
+ def method_missing(meth, *, &block)
259
+ raise NoMethodError, "undefined response method `#{meth}' for error response" if Response.public_method_defined?(meth)
260
+
261
+ super
262
+ end
263
+ # rubocop:enable Style/MissingRespondToMissing
250
264
  end
251
265
  end
@@ -11,7 +11,7 @@ class HTTPX::Selector
11
11
  # I/O monitor
12
12
  #
13
13
  class Monitor
14
- attr_accessor :value, :interests, :readiness
14
+ attr_accessor :io, :interests, :readiness
15
15
 
16
16
  def initialize(io, interests, reactor)
17
17
  @io = io
@@ -31,6 +31,7 @@ class HTTPX::Selector
31
31
  # closes +@io+, deregisters from reactor (unless +deregister+ is false)
32
32
  def close(deregister = true)
33
33
  return if @closed
34
+
34
35
  @closed = true
35
36
  @reactor.deregister(@io) if deregister
36
37
  end
@@ -49,49 +50,44 @@ class HTTPX::Selector
49
50
  def initialize
50
51
  @readers = {}
51
52
  @writers = {}
52
- @lock = Mutex.new
53
53
  @__r__, @__w__ = IO.pipe
54
54
  @closed = false
55
55
  end
56
56
 
57
57
  # deregisters +io+ from selectables.
58
58
  def deregister(io)
59
- @lock.synchronize do
60
- rmonitor = @readers.delete(io)
61
- wmonitor = @writers.delete(io)
62
- monitor = rmonitor || wmonitor
63
- monitor.close(false) if monitor
64
- end
59
+ rmonitor = @readers.delete(io)
60
+ wmonitor = @writers.delete(io)
61
+ monitor = rmonitor || wmonitor
62
+ monitor.close(false) if monitor
65
63
  end
66
64
 
67
65
  # register +io+ for +interests+ events.
68
66
  def register(io, interests)
69
67
  readable = READABLE.include?(interests)
70
68
  writable = WRITABLE.include?(interests)
71
- @lock.synchronize do
72
- if readable
73
- monitor = @readers[io]
74
- if monitor
75
- monitor.interests = interests
76
- else
77
- monitor = Monitor.new(io, interests, self)
78
- end
79
- @readers[io] = monitor
80
- @writers.delete(io) unless writable
69
+ if readable
70
+ monitor = @readers[io]
71
+ if monitor
72
+ monitor.interests = interests
73
+ else
74
+ monitor = Monitor.new(io, interests, self)
81
75
  end
82
- if writable
83
- monitor = @writers[io]
84
- if monitor
85
- monitor.interests = interests
86
- else
87
- # reuse object
88
- monitor = readable ? @readers[io] : Monitor.new(io, interests, self)
89
- end
90
- @writers[io] = monitor
91
- @readers.delete(io) unless readable
76
+ @readers[io] = monitor
77
+ @writers.delete(io) unless writable
78
+ end
79
+ if writable
80
+ monitor = @writers[io]
81
+ if monitor
82
+ monitor.interests = interests
83
+ else
84
+ # reuse object
85
+ monitor = readable ? @readers[io] : Monitor.new(io, interests, self)
92
86
  end
93
- monitor
87
+ @writers[io] = monitor
88
+ @readers.delete(io) unless readable
94
89
  end
90
+ monitor
95
91
  end
96
92
 
97
93
  # waits for read/write events for +interval+. Yields for monitors of
@@ -99,22 +95,16 @@ class HTTPX::Selector
99
95
  #
100
96
  def select(interval)
101
97
  begin
102
- r = nil
103
- w = nil
104
- @lock.synchronize do
105
- r = @readers.keys
106
- w = @writers.keys
107
- end
98
+ r = @readers.keys
99
+ w = @writers.keys
108
100
  r.unshift(@__r__)
109
101
 
110
102
  readers, writers = IO.select(r, w, nil, interval)
111
103
 
112
104
  raise HTTPX::TimeoutError.new(interval, "timed out while waiting on select") if readers.nil? && writers.nil?
113
105
  rescue IOError, SystemCallError
114
- @lock.synchronize do
115
- @readers.reject! { |io, _| io.closed? }
116
- @writers.reject! { |io, _| io.closed? }
117
- end
106
+ @readers.reject! { |io, _| io.closed? }
107
+ @writers.reject! { |io, _| io.closed? }
118
108
  retry
119
109
  end
120
110
 
@@ -125,6 +115,7 @@ class HTTPX::Selector
125
115
  else
126
116
  monitor = io.closed? ? @readers.delete(io) : @readers[io]
127
117
  next unless monitor
118
+
128
119
  monitor.readiness = writers.delete(io) ? :rw : :r
129
120
  yield monitor
130
121
  end
@@ -133,6 +124,7 @@ class HTTPX::Selector
133
124
  writers.each do |io|
134
125
  monitor = io.closed? ? @writers.delete(io) : @writers[io]
135
126
  next unless monitor
127
+
136
128
  # don't double run this, the last iteration might have run this task already
137
129
  monitor.readiness = :w
138
130
  yield monitor
@@ -143,6 +135,7 @@ class HTTPX::Selector
143
135
  #
144
136
  def close
145
137
  return if @closed
138
+
146
139
  @__r__.close
147
140
  @__w__.close
148
141
  rescue IOError