withruntime 0.27.1 → 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. checksums.yaml +4 -4
  2. data/lib/withruntime/generated/apps.rb +192 -0
  3. data/lib/withruntime/generated/audit.rb +78 -0
  4. data/lib/withruntime/generated/browsers.rb +192 -0
  5. data/lib/withruntime/generated/builds.rb +192 -0
  6. data/lib/withruntime/generated/ci.rb +192 -0
  7. data/lib/withruntime/generated/containers.rb +192 -0
  8. data/lib/withruntime/generated/desktops.rb +192 -0
  9. data/lib/withruntime/generated/domains.rb +152 -0
  10. data/lib/withruntime/generated/egress_ips.rb +81 -0
  11. data/lib/withruntime/generated/events.rb +60 -0
  12. data/lib/withruntime/generated/feedback.rb +96 -0
  13. data/lib/withruntime/generated/functions.rb +192 -0
  14. data/lib/withruntime/generated/identities.rb +68 -0
  15. data/lib/withruntime/generated/images.rb +391 -0
  16. data/lib/withruntime/generated/job_runs.rb +110 -0
  17. data/lib/withruntime/generated/jobs.rb +333 -0
  18. data/lib/withruntime/generated/keys.rb +261 -0
  19. data/lib/withruntime/generated/kit.rb +111 -0
  20. data/lib/withruntime/generated/me.rb +69 -0
  21. data/lib/withruntime/generated/networks.rb +72 -0
  22. data/lib/withruntime/generated/notebooks.rb +192 -0
  23. data/lib/withruntime/generated/otel.rb +94 -0
  24. data/lib/withruntime/generated/ports.rb +118 -0
  25. data/lib/withruntime/generated/previews.rb +124 -0
  26. data/lib/withruntime/generated/projects.rb +108 -0
  27. data/lib/withruntime/generated/roles.rb +68 -0
  28. data/lib/withruntime/generated/routes.rb +11 -0
  29. data/lib/withruntime/generated/runtime.rb +188 -0
  30. data/lib/withruntime/generated/sandboxes.rb +1835 -0
  31. data/lib/withruntime/generated/secrets.rb +294 -0
  32. data/lib/withruntime/generated/sessions.rb +143 -0
  33. data/lib/withruntime/generated/sites.rb +192 -0
  34. data/lib/withruntime/generated/snapshots.rb +32 -0
  35. data/lib/withruntime/generated/tunnels.rb +165 -0
  36. data/lib/withruntime/generated/usage.rb +287 -0
  37. data/lib/withruntime/generated/volumes.rb +61 -0
  38. data/lib/withruntime/generated/webhooks.rb +91 -0
  39. data/lib/withruntime/generated/workspaces.rb +192 -0
  40. data/lib/withruntime/signing.rb +162 -0
  41. data/lib/withruntime/version.rb +2 -2
  42. data/lib/withruntime.rb +5 -43
  43. metadata +45 -35
  44. data/lib/withruntime/agents.rb +0 -877
  45. data/lib/withruntime/api_defaults.rb +0 -17
  46. data/lib/withruntime/apps.rb +0 -377
  47. data/lib/withruntime/browsers.rb +0 -104
  48. data/lib/withruntime/builds.rb +0 -218
  49. data/lib/withruntime/ci.rb +0 -164
  50. data/lib/withruntime/client.rb +0 -140
  51. data/lib/withruntime/containers.rb +0 -39
  52. data/lib/withruntime/credentials.rb +0 -111
  53. data/lib/withruntime/fields.rb +0 -40
  54. data/lib/withruntime/file_watch.rb +0 -73
  55. data/lib/withruntime/images.rb +0 -258
  56. data/lib/withruntime/network_products.rb +0 -49
  57. data/lib/withruntime/parity_products.rb +0 -51
  58. data/lib/withruntime/products.rb +0 -560
  59. data/lib/withruntime/record.rb +0 -99
  60. data/lib/withruntime/reservations.rb +0 -76
  61. data/lib/withruntime/sandbox.rb +0 -770
  62. data/lib/withruntime/tar.rb +0 -292
  63. data/lib/withruntime/transport.rb +0 -575
  64. data/lib/withruntime/tunnel.rb +0 -275
  65. data/lib/withruntime/wallet.rb +0 -68
  66. data/lib/withruntime/websocket.rb +0 -384
@@ -1,575 +0,0 @@
1
- # frozen_string_literal: true
2
-
3
- require "digest"
4
- require "json"
5
- require "net/http"
6
- require "securerandom"
7
- require "uri"
8
-
9
- module WithRuntime
10
- # One connection pool, one retry policy and one error shape for every
11
- # product: the same rules as the JavaScript, Python, Go and Java SDKs. Writes
12
- # carry an idempotency key, made per call and kept across the client's own
13
- # retries; transport failures, 429, 502, 503 and 504 are retried with backoff
14
- # and jitter; a create waits out full slots without credit, a full quota or a full region.
15
- class Transport
16
- class OpeningCancelled < IOError; end
17
- class Cancellation
18
- def initialize
19
- @lock = Mutex.new
20
- @cancelled = false
21
- @action = nil
22
- @changed = ConditionVariable.new
23
- end
24
- def cancelled? = @lock.synchronize { @cancelled }
25
- def register(&action)
26
- run = @lock.synchronize { @action = action; @cancelled }
27
- action.call if run
28
- end
29
- def wait(seconds)
30
- deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + seconds
31
- @lock.synchronize do
32
- until @cancelled
33
- left = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
34
- return true if left <= 0
35
- @changed.wait(@lock, left)
36
- end
37
- end
38
- false
39
- end
40
- def cancel
41
- action = @lock.synchronize { @cancelled = true; @changed.broadcast; @action }
42
- action&.call
43
- end
44
- end
45
- # How long one attempt waits for its answer to start. The API starts every
46
- # answer within 120 s: Caddy answers 503 itself past that, and a longer call
47
- # gets its 200 at 90 s and then its JSON (runtime-late-answer). An attempt
48
- # with no answer by then is talking to a server that is gone, such as a
49
- # controller that died with the call in flight while its address moved to
50
- # the new leader: a client that only waits sends nothing the new leader
51
- # could refuse (the leader drill on vin-5, 4 October 2026). It is sent
52
- # again under the same key.
53
- ANSWER_START = 125.0
54
- # The longest wait a server names that a call sleeps on (seconds). One
55
- # longer (a daily cap's hour) is raised at once, still retryable with its
56
- # retry_after, for the caller to schedule.
57
- LONGEST_WAIT = 60.0
58
- class << self
59
- attr_writer :answer_start
60
-
61
- def answer_start = @answer_start || ANSWER_START
62
- end
63
-
64
- NETWORK_ERRORS = [IOError, SystemCallError, SocketError, Timeout::Error, OpenSSL::SSL::SSLError,
65
- Net::HTTPBadResponse, Net::ProtocolError, EOFError].freeze
66
-
67
- attr_reader :base_url, :api_key, :timeout, :max_retries, :wait_for_capacity
68
-
69
- def initialize(api_key:, base_url:, timeout:, max_retries:, max_connections:, wait_for_capacity:)
70
- @api_key = api_key
71
- @base_url = base_url
72
- @uri = URI(base_url)
73
- @timeout = timeout
74
- @max_retries = max_retries
75
- @wait_for_capacity = wait_for_capacity
76
- @limit = [1, max_connections].max
77
- @idle = []
78
- @open = 0
79
- @lock = Mutex.new
80
- @freed = ConditionVariable.new
81
- @client = "sdk-ruby/#{VERSION}"
82
- end
83
-
84
- # Percent-encodes one path segment, as encodeURIComponent does.
85
- def self.segment(value)
86
- value.to_s.b.gsub(/[^A-Za-z0-9\-_.!~*'()]/) { |c| format("%%%02X", c.ord) }
87
- end
88
-
89
- # The value of a JSON call: a Hash, an Array or nil.
90
- def json(method, path, query: nil, body: nil, **options)
91
- with_connection(options, method) do |http, admitted|
92
- response = send_call(http, method, path, query: query, body: body, **admitted)
93
- text = response.body.to_s
94
- text.strip.empty? ? nil : JSON.parse(text)
95
- rescue JSON::ParserError => e
96
- raise Error.new("Unexpected answer from #{method} #{path}: #{e.message}", code: "unexpected_answer")
97
- end
98
- end
99
-
100
- # A GET's body to the block a chunk at a time as it arrives. A connection
101
- # lost part way is raised, never retried, so no chunk is given twice.
102
- def chunks(path, query: nil, accept: "application/octet-stream", &block)
103
- with_connection({}, "GET") do |http, admitted|
104
- send_call(http, "GET", path, query: query, accept: accept, no_retry: true, **admitted, &block)
105
- end
106
- nil
107
- end
108
-
109
- def bytes(method, path, query: nil, body: nil, **options)
110
- with_connection(options, method) do |http, admitted|
111
- send_call(http, method, path, query: query, body: body, **admitted).body.to_s.b
112
- end
113
- end
114
-
115
- # A file's bytes, checked against the length (x-content-length) and
116
- # SHA-256 (x-content-sha256) the API sent with them: a stream that broke
117
- # can arrive as one that ended, short. A short or different body is read
118
- # again twice, then download_incomplete.
119
- def file_bytes(path, query: nil, accept: "application/octet-stream", **options)
120
- attempt = 0
121
- loop do
122
- response = with_connection(options, "GET") do |http, admitted|
123
- send_call(http, "GET", path, query: query, accept: accept, **admitted)
124
- end
125
- body = response.body.to_s.b
126
- problem = Transport.check_body(response, body)
127
- return body unless problem
128
- raise problem if attempt >= 2
129
-
130
- sleep(backoff(attempt))
131
- attempt += 1
132
- end
133
- end
134
-
135
- # Why a whole file body cannot be trusted, or nil when it can.
136
- def self.check_body(response, body)
137
- expected = response["x-content-length"]
138
- if expected&.match?(/\A\d+\z/) && body.bytesize != expected.to_i
139
- return Error.new("The download ended at #{body.bytesize} of #{expected} bytes.",
140
- code: "download_incomplete",
141
- hint: "Read the file again; nothing was written for this read.",
142
- details: { "received" => body.bytesize, "expected" => expected.to_i })
143
- end
144
- sha256 = response["x-content-sha256"]
145
- return nil if sha256.nil? || sha256.empty? || Digest::SHA256.hexdigest(body).casecmp?(sha256)
146
-
147
- Error.new("The file's bytes did not match the SHA-256 the API sent with them.",
148
- code: "download_incomplete", hint: "Read the file again.")
149
- end
150
-
151
- # The status a late answer's body failed with, or 200 when it is an
152
- # answer. An API error is a body of `error` alone, with a code; an answer
153
- # may carry an `error` of its own (a code cell's exception) beside other
154
- # fields. A body that is not JSON was cut or garbled: 502.
155
- def self.late_failure(text)
156
- body = begin
157
- JSON.parse(text)
158
- rescue JSON::ParserError
159
- return 502
160
- end
161
- return 200 unless body.is_a?(Hash) && body.size == 1
162
- error = body["error"]
163
- return 200 unless error.is_a?(Hash) && error["code"].is_a?(String)
164
-
165
- status = error["status"]
166
- status.is_a?(Numeric) && status >= 400 ? status.to_i : 500
167
- end
168
-
169
- # Newline-delimited JSON events as they arrive, one Hash per event, on a
170
- # connection of their own. A connection lost after the first event is
171
- # raised, never retried, so no event is given twice; the caller resumes
172
- # from its cursor.
173
- def events(method, path, query: nil, body: nil, cancel: nil, **options, &block)
174
- return enum_for(:events, method, path, query: query, body: body, cancel: cancel, **options) unless block
175
-
176
- cancellation = Cancellation.new
177
- cancel&.register { cancellation.cancel }
178
- http = connect
179
- begin
180
- buffer = +""
181
- send_call(http, method, path, query: query, body: body, accept: "application/x-ndjson", cancel: cancellation, **options) do |chunk|
182
- buffer << chunk
183
- while (newline = buffer.index("\n"))
184
- line = buffer.slice!(0, newline + 1).strip
185
- yield JSON.parse(line) unless line.empty?
186
- end
187
- end
188
- yield JSON.parse(buffer) unless buffer.strip.empty?
189
- rescue JSON::ParserError => e
190
- raise Error.new("Unexpected stream line: #{e.message}", code: "unexpected_answer")
191
- ensure
192
- cancellation.cancel
193
- begin
194
- http.finish if http.started?
195
- rescue IOError
196
- nil
197
- end
198
- end
199
- end
200
-
201
- def connection_error(timed_out, write, key, cause = nil)
202
- if timed_out
203
- ConnectionError.new("The call was cancelled or ran past its deadline.", code: "timeout",
204
- hint: "Allow a longer timeout for a long call, or retry with the same idempotency key.",
205
- idempotency_key: key)
206
- else
207
- message = "No answer from Runtime at #{base_url}."
208
- message += " The change may have happened; retrying with the same idempotency key is safe." if write
209
- message += " (#{cause.class}: #{cause.message})" if cause
210
- ConnectionError.new(message, code: "connection_error",
211
- hint: "Check the network, HTTPS_PROXY, and RUNTIME_API_URL if you set it.",
212
- idempotency_key: key)
213
- end
214
- end
215
-
216
- private
217
-
218
- # HTTPS_PROXY (or HTTP_PROXY for an http origin) and NO_PROXY, read the
219
- # way curl reads them. Net::HTTP's own :ENV reads http_proxy even for HTTPS.
220
- def self.proxy_for(uri, env = ENV)
221
- names = uri.scheme == "https" ? %w[HTTPS_PROXY https_proxy] : %w[HTTP_PROXY http_proxy]
222
- value = names.map { |name| env[name] }.find { |found| found && !found.strip.empty? }
223
- return nil unless value
224
-
225
- # URI's matcher knows neither curl's "*" (no proxy at all) nor "*.example.com".
226
- exempt = (env["NO_PROXY"] || env["no_proxy"]).to_s.split(",").map(&:strip).reject(&:empty?)
227
- return nil if exempt.include?("*")
228
- exempt = exempt.map { |entry| entry.delete_prefix("*") }.join(",")
229
- return nil if !exempt.empty? && !URI::Generic.use_proxy?(uri.host, uri.host, uri.port, exempt)
230
-
231
- value = "http://#{value}" unless value.include?("://")
232
- URI(value.strip)
233
- end
234
-
235
- def connect
236
- proxy = Transport.proxy_for(@uri)
237
- http = if proxy
238
- Net::HTTP.new(@uri.host, @uri.port, proxy.host, proxy.port,
239
- proxy.user && URI.decode_www_form_component(proxy.user),
240
- proxy.password && URI.decode_www_form_component(proxy.password))
241
- else
242
- Net::HTTP.new(@uri.host, @uri.port, nil)
243
- end
244
- http.use_ssl = @uri.scheme == "https"
245
- http.max_retries = 0
246
- http.open_timeout = 30
247
- http.keep_alive_timeout = 30
248
- http
249
- end
250
-
251
- # Net::HTTP keeps its socket local until TCP, proxy CONNECT and TLS finish.
252
- # Keep that opening alone in an owned worker: raising an IOError there runs
253
- # Net::HTTP's connect rescue and closes its local socket. Never interrupt the
254
- # caller's event block, and join before returning so cancellation leaks no
255
- # opening thread or socket.
256
- def open_connection(http, cancel)
257
- return if http.started?
258
- return http.start unless cancel
259
-
260
- opened = false
261
- gate = Queue.new
262
- worker = Thread.new do
263
- Thread.current.report_on_exception = false
264
- gate.pop
265
- http.start
266
- end
267
- worker.report_on_exception = false
268
- cancel.register { worker.raise(OpeningCancelled, "Stream connection cancelled.") if worker.alive? }
269
- gate.push(true)
270
- worker.value
271
- opened = true
272
- ensure
273
- if worker&.alive?
274
- worker.raise(OpeningCancelled, "Stream connection cancelled.")
275
- begin
276
- worker.join
277
- rescue *NETWORK_ERRORS
278
- nil
279
- end
280
- end
281
- # connect can assign @socket immediately before do_start marks the
282
- # session started. An interruption between those lines skips connect's
283
- # rescue, so finish/started? alone cannot clean up this failed opening.
284
- if worker && !opened
285
- begin
286
- http.instance_variable_get(:@socket)&.close
287
- rescue IOError, SystemCallError
288
- nil
289
- end
290
- end
291
- end
292
-
293
- # A connection from the pool, waited for within the call's own deadline;
294
- # the wait is taken off it, so a queued call expires as an unqueued one
295
- # would, without sending anything.
296
- def with_connection(options, method)
297
- limit = options[:timeout] || @timeout
298
- deadline = now + limit
299
- http = @lock.synchronize do
300
- loop do
301
- break @idle.pop unless @idle.empty?
302
- if @open < @limit
303
- @open += 1
304
- break connect
305
- end
306
- left = deadline - now
307
- raise connection_error(true, method != "GET", options[:key]) if left <= 0
308
-
309
- @freed.wait(@lock, left)
310
- end
311
- end
312
- healthy = false
313
- begin
314
- left = deadline - now
315
- raise connection_error(true, method != "GET", options[:key]) if left <= 0
316
-
317
- result = yield http, options.merge(timeout: left)
318
- healthy = true
319
- result
320
- ensure
321
- @lock.synchronize do
322
- if healthy
323
- @idle.push(http)
324
- else
325
- http.finish if http.started?
326
- @open -= 1
327
- end
328
- @freed.signal
329
- end
330
- end
331
- end
332
-
333
- def deadline_watchdog(http, seconds)
334
- Thread.new do
335
- Thread.current.report_on_exception = false
336
- sleep(seconds)
337
- begin
338
- http.instance_variable_get(:@socket)&.io&.close
339
- rescue IOError, SystemCallError
340
- nil
341
- end
342
- end
343
- end
344
-
345
- # A connection that never opened: no byte of the request left this machine.
346
- NEVER_OPENED = [Errno::ECONNREFUSED, Errno::ENOENT, Errno::EHOSTUNREACH, Errno::ENETUNREACH,
347
- Errno::EADDRNOTAVAIL].freeze
348
-
349
- # Whether a failure, or one of its causes, is a connection that never
350
- # opened. A timeout or a reset never counts.
351
- def self.never_sent?(error)
352
- at = error
353
- 5.times do
354
- return false if at.nil?
355
- return true if NEVER_OPENED.any? { |kind| at.is_a?(kind) }
356
-
357
- at = at.cause
358
- end
359
- false
360
- end
361
-
362
- def backoff(attempt) = [8.0, 0.25 * (2**[attempt, 10].min)].min * (0.5 + rand)
363
-
364
- # 0.5 s, 1 s, 2 s, 4 s, then every 8 s, jittered, so a queue of CI jobs spreads out.
365
- def room_backoff(attempt) = [8.0, 0.5 * (2**[attempt, 10].min)].min * (0.75 + (rand * 0.5))
366
-
367
- def now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
368
-
369
- def target(path, query)
370
- pairs = []
371
- (query || {}).each do |key, value|
372
- next if value.nil?
373
-
374
- Array(value).each { |item| pairs << [key.to_s, item.to_s] }
375
- end
376
- pairs.empty? ? path : "#{path}?#{URI.encode_www_form(pairs)}"
377
- end
378
-
379
- # Sends a call, retrying as the rules say, and returns the response once it
380
- # succeeded. With a block, the body is streamed to it.
381
- # +bearer+ is sent instead of the API key: a webhook trigger's secret,
382
- # which is all its calls carry.
383
- def send_call(http, method, path, query: nil, body: nil, raw: nil, accept: "application/json", wait: nil,
384
- key: nil, no_retry: false, room: nil, timeout: nil, cancel: nil, bearer: nil, &stream)
385
- write = method != "GET"
386
- key = write ? (key || SecureRandom.uuid) : nil
387
- # room: nil for a call with no room wait; 0 for a create, wake, restart
388
- # or fork told at once.
389
- deadline = now + (timeout || @timeout) + (room || 0)
390
- room_until = now + (room || 0)
391
- room_attempt = 0
392
- payload = raw || (body.nil? ? nil : JSON.generate(body))
393
- attempt = 0
394
- # An answer that ran nothing is sent again until the call's deadline
395
- # (a Ruby call always has one: its timeout, plus any wait for room).
396
- idle_attempt = 0
397
- # The last try never left, and was sent again at once.
398
- resent_at_once = false
399
- loop do
400
- raise connection_error(true, write, key) if cancel&.cancelled?
401
- left = deadline - now
402
- raise connection_error(true, write, key) if left <= 0
403
-
404
- request = Net::HTTPGenericRequest.new(method, !payload.nil?, true, target(path, query))
405
- request["Authorization"] = "Bearer #{bearer || api_key}"
406
- request["Accept"] = accept
407
- request["X-Runtime-Client"] = @client
408
- request["User-Agent"] = "withruntime-ruby/#{VERSION}"
409
- request["Idempotency-Key"] = key if key
410
- request["Prefer"] = "wait=#{[120, wait].min}" if wait&.positive?
411
- if payload
412
- request["Content-Type"] = raw ? "application/octet-stream" : "application/json"
413
- request.body = payload
414
- end
415
- # Until its answer starts, a read waits at most answer_start; then the
416
- # call's whole deadline again.
417
- bounded = Transport.answer_start < left
418
- answered = false
419
- http.read_timeout = bounded ? Transport.answer_start : left
420
- http.write_timeout = left if http.respond_to?(:write_timeout=)
421
- http.open_timeout = [left, 30].min
422
- # Per-read limits do not bound a body that keeps trickling in: at the
423
- # deadline the socket is closed, and the read fails as a timeout.
424
- watchdog = deadline_watchdog(http, left)
425
- response = nil
426
- # A stream that gave its consumer anything is never sent again: that
427
- # would give the same events twice. The consumer's own errors are its
428
- # own, never a reason to retry.
429
- streamed = false
430
- consumer_error = nil
431
- begin
432
- open_connection(http, cancel)
433
- cancel&.register do
434
- http.read_timeout = 0.001
435
- http.write_timeout = 0.001
436
- http.open_timeout = 0.001
437
- begin
438
- http.instance_variable_get(:@socket)&.io&.close
439
- rescue IOError, SystemCallError
440
- nil
441
- end
442
- end
443
- raise connection_error(true, write, key) if cancel&.cancelled?
444
- if stream
445
- http.request(request) do |answer|
446
- answered = true
447
- http.read_timeout = [deadline - now, 0.001].max
448
- response = answer
449
- # A late answer is read whole first: it may be a failure.
450
- if answer.code.to_i < 300 && answer["runtime-late-answer"] != "true"
451
- answer.read_body do |chunk|
452
- streamed = true
453
- begin
454
- stream.call(chunk)
455
- rescue Exception => e
456
- consumer_error = e
457
- raise
458
- end
459
- end
460
- end
461
- end
462
- else
463
- response = http.request(request) do
464
- answered = true
465
- http.read_timeout = [deadline - now, 0.001].max
466
- end
467
- end
468
- status = response.code.to_i
469
- if status >= 300 && status < 400
470
- raise Net::HTTPBadResponse, "Runtime answered a redirect (#{status})"
471
- end
472
- rescue *NETWORK_ERRORS => e
473
- http.finish if http.started?
474
- raise if e.equal?(consumer_error)
475
-
476
- unanswered = bounded && !answered && e.is_a?(Net::ReadTimeout) && now < deadline
477
- late = (e.is_a?(Timeout::Error) && !unanswered) || now >= deadline || cancel&.cancelled?
478
- raise connection_error(late, write, key, e) if late || no_retry || streamed || attempt >= max_retries
479
-
480
- # A connection that never opened sent no byte of the request, so the
481
- # next try, which may reach another server, goes at once; once in a
482
- # row, so an API that is down is not hammered. A reset or a timeout
483
- # may have reached a server and keeps the backoff and its key's rules.
484
- at_once = Transport.never_sent?(e) && !resent_at_once
485
- resent_at_once = at_once
486
- pause(at_once ? rand * 0.05 : backoff(attempt), deadline, write, key, nil, cancel)
487
- attempt += 1
488
- next
489
- ensure
490
- watchdog.kill
491
- end
492
- # An answer that took longer than the API holds its headers (90 s)
493
- # comes as a 200 sent early, and then its JSON: the answer, or
494
- # {"error": ...} when the work failed after the status went
495
- # (runtime-late-answer). A failure is raised as the error it is,
496
- # never returned as a success.
497
- late = status < 300 && response["runtime-late-answer"] == "true"
498
- status = Transport.late_failure(response.body.to_s) if late
499
- if status < 300
500
- stream&.call(response.body.to_s) if late
501
- return response
502
- end
503
-
504
- text = response.body.to_s
505
- parsed = begin
506
- JSON.parse(text)
507
- rescue JSON::ParserError
508
- { "error" => { "message" => text.strip.empty? ? nil : text.strip[0, 500] }.compact }
509
- end
510
- failure = Error.from(status, parsed, key)
511
- # A quota on what is kept says it never clears by waiting.
512
- if !no_retry && room&.positive? && Error::WAITS_FOR_ROOM.include?(failure.code) && failure.details["field"] != "count" &&
513
- failure.sent_retryable != false
514
- room_left = room_until - now
515
- raise failure if room_left <= 0
516
-
517
- wait_for = failure.retry_after || room_backoff(room_attempt)
518
- room_attempt += 1
519
- pause([room_left, wait_for].min, deadline, write, key, failure, cancel)
520
- next
521
- end
522
- header = Float(response["Retry-After"], exception: false)
523
- header = nil unless header&.positive?
524
- # The wait the server names, when it names one.
525
- named = failure.retry_after || header
526
- # Nothing ran: safe to send again for any call, past max_retries,
527
- # with growing waits, until the deadline; then this refusal stands.
528
- # A late answer's failure came after the work began, so only a
529
- # guest's own refusal (guest_busy, before anything is journaled)
530
- # still says nothing ran.
531
- if [429, 503].include?(status) && Error::NOTHING_RAN.include?(failure.code) &&
532
- failure.sent_retryable != false && (!late || failure.code == "guest_busy")
533
- raise failure if named && named > LONGEST_WAIT
534
-
535
- pause_for = [30.0, (named || 0.25) * (2**[idle_attempt, 3].min)].min * (0.9 + (rand * 0.2))
536
- idle_attempt += 1
537
- raise failure if pause_for >= deadline - now
538
-
539
- pause(pause_for, deadline, write, key, failure, cancel)
540
- next
541
- end
542
- # The answer says whether the same call may work again; an older
543
- # server that does not say is retried on 429, 502, 503 and 504. A
544
- # refusal for room or the account's own limit is the room wait's on a
545
- # create, wake, restart or fork; any other call retries what it says.
546
- again = if failure.sent_retryable.nil?
547
- [429, 502, 503, 504].include?(status) && !Error::DELIBERATE.include?(failure.code)
548
- else
549
- failure.sent_retryable
550
- end
551
- raise failure if no_retry || !again || (room && Error::WAITS_FOR_ROOM.include?(failure.code)) ||
552
- attempt >= max_retries
553
- # A wait longer than a minute (a daily cap) is the caller's to
554
- # schedule: the error says when, in retry_after.
555
- raise failure if named && named > LONGEST_WAIT
556
-
557
- pause([30.0, named || backoff(attempt)].min * (0.9 + (rand * 0.2)), deadline, write, key, failure, cancel)
558
- attempt += 1
559
- end
560
- end
561
-
562
- def pause(seconds, deadline, write, key, failure = nil, cancel = nil)
563
- if now + seconds >= deadline
564
- raise failure if failure
565
-
566
- raise connection_error(true, write, key)
567
- end
568
- if cancel
569
- raise connection_error(true, write, key) unless cancel.wait(seconds)
570
- else
571
- sleep(seconds)
572
- end
573
- end
574
- end
575
- end