withruntime 0.27.1 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/withruntime/generated/apps.rb +192 -0
- data/lib/withruntime/generated/audit.rb +78 -0
- data/lib/withruntime/generated/browsers.rb +192 -0
- data/lib/withruntime/generated/builds.rb +192 -0
- data/lib/withruntime/generated/ci.rb +192 -0
- data/lib/withruntime/generated/containers.rb +192 -0
- data/lib/withruntime/generated/desktops.rb +192 -0
- data/lib/withruntime/generated/domains.rb +152 -0
- data/lib/withruntime/generated/egress_ips.rb +81 -0
- data/lib/withruntime/generated/events.rb +60 -0
- data/lib/withruntime/generated/feedback.rb +96 -0
- data/lib/withruntime/generated/functions.rb +192 -0
- data/lib/withruntime/generated/identities.rb +68 -0
- data/lib/withruntime/generated/images.rb +391 -0
- data/lib/withruntime/generated/job_runs.rb +110 -0
- data/lib/withruntime/generated/jobs.rb +333 -0
- data/lib/withruntime/generated/keys.rb +261 -0
- data/lib/withruntime/generated/kit.rb +111 -0
- data/lib/withruntime/generated/me.rb +69 -0
- data/lib/withruntime/generated/networks.rb +72 -0
- data/lib/withruntime/generated/notebooks.rb +192 -0
- data/lib/withruntime/generated/otel.rb +94 -0
- data/lib/withruntime/generated/ports.rb +118 -0
- data/lib/withruntime/generated/previews.rb +124 -0
- data/lib/withruntime/generated/projects.rb +108 -0
- data/lib/withruntime/generated/roles.rb +68 -0
- data/lib/withruntime/generated/routes.rb +11 -0
- data/lib/withruntime/generated/runtime.rb +188 -0
- data/lib/withruntime/generated/sandboxes.rb +1835 -0
- data/lib/withruntime/generated/secrets.rb +294 -0
- data/lib/withruntime/generated/sessions.rb +143 -0
- data/lib/withruntime/generated/sites.rb +192 -0
- data/lib/withruntime/generated/snapshots.rb +32 -0
- data/lib/withruntime/generated/tunnels.rb +165 -0
- data/lib/withruntime/generated/usage.rb +287 -0
- data/lib/withruntime/generated/volumes.rb +61 -0
- data/lib/withruntime/generated/webhooks.rb +91 -0
- data/lib/withruntime/generated/workspaces.rb +192 -0
- data/lib/withruntime/signing.rb +162 -0
- data/lib/withruntime/version.rb +2 -2
- data/lib/withruntime.rb +5 -43
- metadata +45 -35
- data/lib/withruntime/agents.rb +0 -877
- data/lib/withruntime/api_defaults.rb +0 -17
- data/lib/withruntime/apps.rb +0 -377
- data/lib/withruntime/browsers.rb +0 -104
- data/lib/withruntime/builds.rb +0 -218
- data/lib/withruntime/ci.rb +0 -164
- data/lib/withruntime/client.rb +0 -140
- data/lib/withruntime/containers.rb +0 -39
- data/lib/withruntime/credentials.rb +0 -111
- data/lib/withruntime/fields.rb +0 -40
- data/lib/withruntime/file_watch.rb +0 -73
- data/lib/withruntime/images.rb +0 -258
- data/lib/withruntime/network_products.rb +0 -49
- data/lib/withruntime/parity_products.rb +0 -51
- data/lib/withruntime/products.rb +0 -560
- data/lib/withruntime/record.rb +0 -99
- data/lib/withruntime/reservations.rb +0 -76
- data/lib/withruntime/sandbox.rb +0 -770
- data/lib/withruntime/tar.rb +0 -292
- data/lib/withruntime/transport.rb +0 -575
- data/lib/withruntime/tunnel.rb +0 -275
- data/lib/withruntime/wallet.rb +0 -68
- data/lib/withruntime/websocket.rb +0 -384
|
@@ -1,575 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require "digest"
|
|
4
|
-
require "json"
|
|
5
|
-
require "net/http"
|
|
6
|
-
require "securerandom"
|
|
7
|
-
require "uri"
|
|
8
|
-
|
|
9
|
-
module WithRuntime
|
|
10
|
-
# One connection pool, one retry policy and one error shape for every
|
|
11
|
-
# product: the same rules as the JavaScript, Python, Go and Java SDKs. Writes
|
|
12
|
-
# carry an idempotency key, made per call and kept across the client's own
|
|
13
|
-
# retries; transport failures, 429, 502, 503 and 504 are retried with backoff
|
|
14
|
-
# and jitter; a create waits out full slots without credit, a full quota or a full region.
|
|
15
|
-
class Transport
|
|
16
|
-
class OpeningCancelled < IOError; end
|
|
17
|
-
class Cancellation
|
|
18
|
-
def initialize
|
|
19
|
-
@lock = Mutex.new
|
|
20
|
-
@cancelled = false
|
|
21
|
-
@action = nil
|
|
22
|
-
@changed = ConditionVariable.new
|
|
23
|
-
end
|
|
24
|
-
def cancelled? = @lock.synchronize { @cancelled }
|
|
25
|
-
def register(&action)
|
|
26
|
-
run = @lock.synchronize { @action = action; @cancelled }
|
|
27
|
-
action.call if run
|
|
28
|
-
end
|
|
29
|
-
def wait(seconds)
|
|
30
|
-
deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + seconds
|
|
31
|
-
@lock.synchronize do
|
|
32
|
-
until @cancelled
|
|
33
|
-
left = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
34
|
-
return true if left <= 0
|
|
35
|
-
@changed.wait(@lock, left)
|
|
36
|
-
end
|
|
37
|
-
end
|
|
38
|
-
false
|
|
39
|
-
end
|
|
40
|
-
def cancel
|
|
41
|
-
action = @lock.synchronize { @cancelled = true; @changed.broadcast; @action }
|
|
42
|
-
action&.call
|
|
43
|
-
end
|
|
44
|
-
end
|
|
45
|
-
# How long one attempt waits for its answer to start. The API starts every
|
|
46
|
-
# answer within 120 s: Caddy answers 503 itself past that, and a longer call
|
|
47
|
-
# gets its 200 at 90 s and then its JSON (runtime-late-answer). An attempt
|
|
48
|
-
# with no answer by then is talking to a server that is gone, such as a
|
|
49
|
-
# controller that died with the call in flight while its address moved to
|
|
50
|
-
# the new leader: a client that only waits sends nothing the new leader
|
|
51
|
-
# could refuse (the leader drill on vin-5, 4 October 2026). It is sent
|
|
52
|
-
# again under the same key.
|
|
53
|
-
ANSWER_START = 125.0
|
|
54
|
-
# The longest wait a server names that a call sleeps on (seconds). One
|
|
55
|
-
# longer (a daily cap's hour) is raised at once, still retryable with its
|
|
56
|
-
# retry_after, for the caller to schedule.
|
|
57
|
-
LONGEST_WAIT = 60.0
|
|
58
|
-
class << self
|
|
59
|
-
attr_writer :answer_start
|
|
60
|
-
|
|
61
|
-
def answer_start = @answer_start || ANSWER_START
|
|
62
|
-
end
|
|
63
|
-
|
|
64
|
-
NETWORK_ERRORS = [IOError, SystemCallError, SocketError, Timeout::Error, OpenSSL::SSL::SSLError,
|
|
65
|
-
Net::HTTPBadResponse, Net::ProtocolError, EOFError].freeze
|
|
66
|
-
|
|
67
|
-
attr_reader :base_url, :api_key, :timeout, :max_retries, :wait_for_capacity
|
|
68
|
-
|
|
69
|
-
def initialize(api_key:, base_url:, timeout:, max_retries:, max_connections:, wait_for_capacity:)
|
|
70
|
-
@api_key = api_key
|
|
71
|
-
@base_url = base_url
|
|
72
|
-
@uri = URI(base_url)
|
|
73
|
-
@timeout = timeout
|
|
74
|
-
@max_retries = max_retries
|
|
75
|
-
@wait_for_capacity = wait_for_capacity
|
|
76
|
-
@limit = [1, max_connections].max
|
|
77
|
-
@idle = []
|
|
78
|
-
@open = 0
|
|
79
|
-
@lock = Mutex.new
|
|
80
|
-
@freed = ConditionVariable.new
|
|
81
|
-
@client = "sdk-ruby/#{VERSION}"
|
|
82
|
-
end
|
|
83
|
-
|
|
84
|
-
# Percent-encodes one path segment, as encodeURIComponent does.
|
|
85
|
-
def self.segment(value)
|
|
86
|
-
value.to_s.b.gsub(/[^A-Za-z0-9\-_.!~*'()]/) { |c| format("%%%02X", c.ord) }
|
|
87
|
-
end
|
|
88
|
-
|
|
89
|
-
# The value of a JSON call: a Hash, an Array or nil.
|
|
90
|
-
def json(method, path, query: nil, body: nil, **options)
|
|
91
|
-
with_connection(options, method) do |http, admitted|
|
|
92
|
-
response = send_call(http, method, path, query: query, body: body, **admitted)
|
|
93
|
-
text = response.body.to_s
|
|
94
|
-
text.strip.empty? ? nil : JSON.parse(text)
|
|
95
|
-
rescue JSON::ParserError => e
|
|
96
|
-
raise Error.new("Unexpected answer from #{method} #{path}: #{e.message}", code: "unexpected_answer")
|
|
97
|
-
end
|
|
98
|
-
end
|
|
99
|
-
|
|
100
|
-
# A GET's body to the block a chunk at a time as it arrives. A connection
|
|
101
|
-
# lost part way is raised, never retried, so no chunk is given twice.
|
|
102
|
-
def chunks(path, query: nil, accept: "application/octet-stream", &block)
|
|
103
|
-
with_connection({}, "GET") do |http, admitted|
|
|
104
|
-
send_call(http, "GET", path, query: query, accept: accept, no_retry: true, **admitted, &block)
|
|
105
|
-
end
|
|
106
|
-
nil
|
|
107
|
-
end
|
|
108
|
-
|
|
109
|
-
def bytes(method, path, query: nil, body: nil, **options)
|
|
110
|
-
with_connection(options, method) do |http, admitted|
|
|
111
|
-
send_call(http, method, path, query: query, body: body, **admitted).body.to_s.b
|
|
112
|
-
end
|
|
113
|
-
end
|
|
114
|
-
|
|
115
|
-
# A file's bytes, checked against the length (x-content-length) and
|
|
116
|
-
# SHA-256 (x-content-sha256) the API sent with them: a stream that broke
|
|
117
|
-
# can arrive as one that ended, short. A short or different body is read
|
|
118
|
-
# again twice, then download_incomplete.
|
|
119
|
-
def file_bytes(path, query: nil, accept: "application/octet-stream", **options)
|
|
120
|
-
attempt = 0
|
|
121
|
-
loop do
|
|
122
|
-
response = with_connection(options, "GET") do |http, admitted|
|
|
123
|
-
send_call(http, "GET", path, query: query, accept: accept, **admitted)
|
|
124
|
-
end
|
|
125
|
-
body = response.body.to_s.b
|
|
126
|
-
problem = Transport.check_body(response, body)
|
|
127
|
-
return body unless problem
|
|
128
|
-
raise problem if attempt >= 2
|
|
129
|
-
|
|
130
|
-
sleep(backoff(attempt))
|
|
131
|
-
attempt += 1
|
|
132
|
-
end
|
|
133
|
-
end
|
|
134
|
-
|
|
135
|
-
# Why a whole file body cannot be trusted, or nil when it can.
|
|
136
|
-
def self.check_body(response, body)
|
|
137
|
-
expected = response["x-content-length"]
|
|
138
|
-
if expected&.match?(/\A\d+\z/) && body.bytesize != expected.to_i
|
|
139
|
-
return Error.new("The download ended at #{body.bytesize} of #{expected} bytes.",
|
|
140
|
-
code: "download_incomplete",
|
|
141
|
-
hint: "Read the file again; nothing was written for this read.",
|
|
142
|
-
details: { "received" => body.bytesize, "expected" => expected.to_i })
|
|
143
|
-
end
|
|
144
|
-
sha256 = response["x-content-sha256"]
|
|
145
|
-
return nil if sha256.nil? || sha256.empty? || Digest::SHA256.hexdigest(body).casecmp?(sha256)
|
|
146
|
-
|
|
147
|
-
Error.new("The file's bytes did not match the SHA-256 the API sent with them.",
|
|
148
|
-
code: "download_incomplete", hint: "Read the file again.")
|
|
149
|
-
end
|
|
150
|
-
|
|
151
|
-
# The status a late answer's body failed with, or 200 when it is an
|
|
152
|
-
# answer. An API error is a body of `error` alone, with a code; an answer
|
|
153
|
-
# may carry an `error` of its own (a code cell's exception) beside other
|
|
154
|
-
# fields. A body that is not JSON was cut or garbled: 502.
|
|
155
|
-
def self.late_failure(text)
|
|
156
|
-
body = begin
|
|
157
|
-
JSON.parse(text)
|
|
158
|
-
rescue JSON::ParserError
|
|
159
|
-
return 502
|
|
160
|
-
end
|
|
161
|
-
return 200 unless body.is_a?(Hash) && body.size == 1
|
|
162
|
-
error = body["error"]
|
|
163
|
-
return 200 unless error.is_a?(Hash) && error["code"].is_a?(String)
|
|
164
|
-
|
|
165
|
-
status = error["status"]
|
|
166
|
-
status.is_a?(Numeric) && status >= 400 ? status.to_i : 500
|
|
167
|
-
end
|
|
168
|
-
|
|
169
|
-
# Newline-delimited JSON events as they arrive, one Hash per event, on a
|
|
170
|
-
# connection of their own. A connection lost after the first event is
|
|
171
|
-
# raised, never retried, so no event is given twice; the caller resumes
|
|
172
|
-
# from its cursor.
|
|
173
|
-
def events(method, path, query: nil, body: nil, cancel: nil, **options, &block)
|
|
174
|
-
return enum_for(:events, method, path, query: query, body: body, cancel: cancel, **options) unless block
|
|
175
|
-
|
|
176
|
-
cancellation = Cancellation.new
|
|
177
|
-
cancel&.register { cancellation.cancel }
|
|
178
|
-
http = connect
|
|
179
|
-
begin
|
|
180
|
-
buffer = +""
|
|
181
|
-
send_call(http, method, path, query: query, body: body, accept: "application/x-ndjson", cancel: cancellation, **options) do |chunk|
|
|
182
|
-
buffer << chunk
|
|
183
|
-
while (newline = buffer.index("\n"))
|
|
184
|
-
line = buffer.slice!(0, newline + 1).strip
|
|
185
|
-
yield JSON.parse(line) unless line.empty?
|
|
186
|
-
end
|
|
187
|
-
end
|
|
188
|
-
yield JSON.parse(buffer) unless buffer.strip.empty?
|
|
189
|
-
rescue JSON::ParserError => e
|
|
190
|
-
raise Error.new("Unexpected stream line: #{e.message}", code: "unexpected_answer")
|
|
191
|
-
ensure
|
|
192
|
-
cancellation.cancel
|
|
193
|
-
begin
|
|
194
|
-
http.finish if http.started?
|
|
195
|
-
rescue IOError
|
|
196
|
-
nil
|
|
197
|
-
end
|
|
198
|
-
end
|
|
199
|
-
end
|
|
200
|
-
|
|
201
|
-
def connection_error(timed_out, write, key, cause = nil)
|
|
202
|
-
if timed_out
|
|
203
|
-
ConnectionError.new("The call was cancelled or ran past its deadline.", code: "timeout",
|
|
204
|
-
hint: "Allow a longer timeout for a long call, or retry with the same idempotency key.",
|
|
205
|
-
idempotency_key: key)
|
|
206
|
-
else
|
|
207
|
-
message = "No answer from Runtime at #{base_url}."
|
|
208
|
-
message += " The change may have happened; retrying with the same idempotency key is safe." if write
|
|
209
|
-
message += " (#{cause.class}: #{cause.message})" if cause
|
|
210
|
-
ConnectionError.new(message, code: "connection_error",
|
|
211
|
-
hint: "Check the network, HTTPS_PROXY, and RUNTIME_API_URL if you set it.",
|
|
212
|
-
idempotency_key: key)
|
|
213
|
-
end
|
|
214
|
-
end
|
|
215
|
-
|
|
216
|
-
private
|
|
217
|
-
|
|
218
|
-
# HTTPS_PROXY (or HTTP_PROXY for an http origin) and NO_PROXY, read the
|
|
219
|
-
# way curl reads them. Net::HTTP's own :ENV reads http_proxy even for HTTPS.
|
|
220
|
-
def self.proxy_for(uri, env = ENV)
|
|
221
|
-
names = uri.scheme == "https" ? %w[HTTPS_PROXY https_proxy] : %w[HTTP_PROXY http_proxy]
|
|
222
|
-
value = names.map { |name| env[name] }.find { |found| found && !found.strip.empty? }
|
|
223
|
-
return nil unless value
|
|
224
|
-
|
|
225
|
-
# URI's matcher knows neither curl's "*" (no proxy at all) nor "*.example.com".
|
|
226
|
-
exempt = (env["NO_PROXY"] || env["no_proxy"]).to_s.split(",").map(&:strip).reject(&:empty?)
|
|
227
|
-
return nil if exempt.include?("*")
|
|
228
|
-
exempt = exempt.map { |entry| entry.delete_prefix("*") }.join(",")
|
|
229
|
-
return nil if !exempt.empty? && !URI::Generic.use_proxy?(uri.host, uri.host, uri.port, exempt)
|
|
230
|
-
|
|
231
|
-
value = "http://#{value}" unless value.include?("://")
|
|
232
|
-
URI(value.strip)
|
|
233
|
-
end
|
|
234
|
-
|
|
235
|
-
def connect
|
|
236
|
-
proxy = Transport.proxy_for(@uri)
|
|
237
|
-
http = if proxy
|
|
238
|
-
Net::HTTP.new(@uri.host, @uri.port, proxy.host, proxy.port,
|
|
239
|
-
proxy.user && URI.decode_www_form_component(proxy.user),
|
|
240
|
-
proxy.password && URI.decode_www_form_component(proxy.password))
|
|
241
|
-
else
|
|
242
|
-
Net::HTTP.new(@uri.host, @uri.port, nil)
|
|
243
|
-
end
|
|
244
|
-
http.use_ssl = @uri.scheme == "https"
|
|
245
|
-
http.max_retries = 0
|
|
246
|
-
http.open_timeout = 30
|
|
247
|
-
http.keep_alive_timeout = 30
|
|
248
|
-
http
|
|
249
|
-
end
|
|
250
|
-
|
|
251
|
-
# Net::HTTP keeps its socket local until TCP, proxy CONNECT and TLS finish.
|
|
252
|
-
# Keep that opening alone in an owned worker: raising an IOError there runs
|
|
253
|
-
# Net::HTTP's connect rescue and closes its local socket. Never interrupt the
|
|
254
|
-
# caller's event block, and join before returning so cancellation leaks no
|
|
255
|
-
# opening thread or socket.
|
|
256
|
-
def open_connection(http, cancel)
|
|
257
|
-
return if http.started?
|
|
258
|
-
return http.start unless cancel
|
|
259
|
-
|
|
260
|
-
opened = false
|
|
261
|
-
gate = Queue.new
|
|
262
|
-
worker = Thread.new do
|
|
263
|
-
Thread.current.report_on_exception = false
|
|
264
|
-
gate.pop
|
|
265
|
-
http.start
|
|
266
|
-
end
|
|
267
|
-
worker.report_on_exception = false
|
|
268
|
-
cancel.register { worker.raise(OpeningCancelled, "Stream connection cancelled.") if worker.alive? }
|
|
269
|
-
gate.push(true)
|
|
270
|
-
worker.value
|
|
271
|
-
opened = true
|
|
272
|
-
ensure
|
|
273
|
-
if worker&.alive?
|
|
274
|
-
worker.raise(OpeningCancelled, "Stream connection cancelled.")
|
|
275
|
-
begin
|
|
276
|
-
worker.join
|
|
277
|
-
rescue *NETWORK_ERRORS
|
|
278
|
-
nil
|
|
279
|
-
end
|
|
280
|
-
end
|
|
281
|
-
# connect can assign @socket immediately before do_start marks the
|
|
282
|
-
# session started. An interruption between those lines skips connect's
|
|
283
|
-
# rescue, so finish/started? alone cannot clean up this failed opening.
|
|
284
|
-
if worker && !opened
|
|
285
|
-
begin
|
|
286
|
-
http.instance_variable_get(:@socket)&.close
|
|
287
|
-
rescue IOError, SystemCallError
|
|
288
|
-
nil
|
|
289
|
-
end
|
|
290
|
-
end
|
|
291
|
-
end
|
|
292
|
-
|
|
293
|
-
# A connection from the pool, waited for within the call's own deadline;
|
|
294
|
-
# the wait is taken off it, so a queued call expires as an unqueued one
|
|
295
|
-
# would, without sending anything.
|
|
296
|
-
def with_connection(options, method)
|
|
297
|
-
limit = options[:timeout] || @timeout
|
|
298
|
-
deadline = now + limit
|
|
299
|
-
http = @lock.synchronize do
|
|
300
|
-
loop do
|
|
301
|
-
break @idle.pop unless @idle.empty?
|
|
302
|
-
if @open < @limit
|
|
303
|
-
@open += 1
|
|
304
|
-
break connect
|
|
305
|
-
end
|
|
306
|
-
left = deadline - now
|
|
307
|
-
raise connection_error(true, method != "GET", options[:key]) if left <= 0
|
|
308
|
-
|
|
309
|
-
@freed.wait(@lock, left)
|
|
310
|
-
end
|
|
311
|
-
end
|
|
312
|
-
healthy = false
|
|
313
|
-
begin
|
|
314
|
-
left = deadline - now
|
|
315
|
-
raise connection_error(true, method != "GET", options[:key]) if left <= 0
|
|
316
|
-
|
|
317
|
-
result = yield http, options.merge(timeout: left)
|
|
318
|
-
healthy = true
|
|
319
|
-
result
|
|
320
|
-
ensure
|
|
321
|
-
@lock.synchronize do
|
|
322
|
-
if healthy
|
|
323
|
-
@idle.push(http)
|
|
324
|
-
else
|
|
325
|
-
http.finish if http.started?
|
|
326
|
-
@open -= 1
|
|
327
|
-
end
|
|
328
|
-
@freed.signal
|
|
329
|
-
end
|
|
330
|
-
end
|
|
331
|
-
end
|
|
332
|
-
|
|
333
|
-
def deadline_watchdog(http, seconds)
|
|
334
|
-
Thread.new do
|
|
335
|
-
Thread.current.report_on_exception = false
|
|
336
|
-
sleep(seconds)
|
|
337
|
-
begin
|
|
338
|
-
http.instance_variable_get(:@socket)&.io&.close
|
|
339
|
-
rescue IOError, SystemCallError
|
|
340
|
-
nil
|
|
341
|
-
end
|
|
342
|
-
end
|
|
343
|
-
end
|
|
344
|
-
|
|
345
|
-
# A connection that never opened: no byte of the request left this machine.
|
|
346
|
-
NEVER_OPENED = [Errno::ECONNREFUSED, Errno::ENOENT, Errno::EHOSTUNREACH, Errno::ENETUNREACH,
|
|
347
|
-
Errno::EADDRNOTAVAIL].freeze
|
|
348
|
-
|
|
349
|
-
# Whether a failure, or one of its causes, is a connection that never
|
|
350
|
-
# opened. A timeout or a reset never counts.
|
|
351
|
-
def self.never_sent?(error)
|
|
352
|
-
at = error
|
|
353
|
-
5.times do
|
|
354
|
-
return false if at.nil?
|
|
355
|
-
return true if NEVER_OPENED.any? { |kind| at.is_a?(kind) }
|
|
356
|
-
|
|
357
|
-
at = at.cause
|
|
358
|
-
end
|
|
359
|
-
false
|
|
360
|
-
end
|
|
361
|
-
|
|
362
|
-
def backoff(attempt) = [8.0, 0.25 * (2**[attempt, 10].min)].min * (0.5 + rand)
|
|
363
|
-
|
|
364
|
-
# 0.5 s, 1 s, 2 s, 4 s, then every 8 s, jittered, so a queue of CI jobs spreads out.
|
|
365
|
-
def room_backoff(attempt) = [8.0, 0.5 * (2**[attempt, 10].min)].min * (0.75 + (rand * 0.5))
|
|
366
|
-
|
|
367
|
-
def now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
368
|
-
|
|
369
|
-
def target(path, query)
|
|
370
|
-
pairs = []
|
|
371
|
-
(query || {}).each do |key, value|
|
|
372
|
-
next if value.nil?
|
|
373
|
-
|
|
374
|
-
Array(value).each { |item| pairs << [key.to_s, item.to_s] }
|
|
375
|
-
end
|
|
376
|
-
pairs.empty? ? path : "#{path}?#{URI.encode_www_form(pairs)}"
|
|
377
|
-
end
|
|
378
|
-
|
|
379
|
-
# Sends a call, retrying as the rules say, and returns the response once it
|
|
380
|
-
# succeeded. With a block, the body is streamed to it.
|
|
381
|
-
# +bearer+ is sent instead of the API key: a webhook trigger's secret,
|
|
382
|
-
# which is all its calls carry.
|
|
383
|
-
def send_call(http, method, path, query: nil, body: nil, raw: nil, accept: "application/json", wait: nil,
|
|
384
|
-
key: nil, no_retry: false, room: nil, timeout: nil, cancel: nil, bearer: nil, &stream)
|
|
385
|
-
write = method != "GET"
|
|
386
|
-
key = write ? (key || SecureRandom.uuid) : nil
|
|
387
|
-
# room: nil for a call with no room wait; 0 for a create, wake, restart
|
|
388
|
-
# or fork told at once.
|
|
389
|
-
deadline = now + (timeout || @timeout) + (room || 0)
|
|
390
|
-
room_until = now + (room || 0)
|
|
391
|
-
room_attempt = 0
|
|
392
|
-
payload = raw || (body.nil? ? nil : JSON.generate(body))
|
|
393
|
-
attempt = 0
|
|
394
|
-
# An answer that ran nothing is sent again until the call's deadline
|
|
395
|
-
# (a Ruby call always has one: its timeout, plus any wait for room).
|
|
396
|
-
idle_attempt = 0
|
|
397
|
-
# The last try never left, and was sent again at once.
|
|
398
|
-
resent_at_once = false
|
|
399
|
-
loop do
|
|
400
|
-
raise connection_error(true, write, key) if cancel&.cancelled?
|
|
401
|
-
left = deadline - now
|
|
402
|
-
raise connection_error(true, write, key) if left <= 0
|
|
403
|
-
|
|
404
|
-
request = Net::HTTPGenericRequest.new(method, !payload.nil?, true, target(path, query))
|
|
405
|
-
request["Authorization"] = "Bearer #{bearer || api_key}"
|
|
406
|
-
request["Accept"] = accept
|
|
407
|
-
request["X-Runtime-Client"] = @client
|
|
408
|
-
request["User-Agent"] = "withruntime-ruby/#{VERSION}"
|
|
409
|
-
request["Idempotency-Key"] = key if key
|
|
410
|
-
request["Prefer"] = "wait=#{[120, wait].min}" if wait&.positive?
|
|
411
|
-
if payload
|
|
412
|
-
request["Content-Type"] = raw ? "application/octet-stream" : "application/json"
|
|
413
|
-
request.body = payload
|
|
414
|
-
end
|
|
415
|
-
# Until its answer starts, a read waits at most answer_start; then the
|
|
416
|
-
# call's whole deadline again.
|
|
417
|
-
bounded = Transport.answer_start < left
|
|
418
|
-
answered = false
|
|
419
|
-
http.read_timeout = bounded ? Transport.answer_start : left
|
|
420
|
-
http.write_timeout = left if http.respond_to?(:write_timeout=)
|
|
421
|
-
http.open_timeout = [left, 30].min
|
|
422
|
-
# Per-read limits do not bound a body that keeps trickling in: at the
|
|
423
|
-
# deadline the socket is closed, and the read fails as a timeout.
|
|
424
|
-
watchdog = deadline_watchdog(http, left)
|
|
425
|
-
response = nil
|
|
426
|
-
# A stream that gave its consumer anything is never sent again: that
|
|
427
|
-
# would give the same events twice. The consumer's own errors are its
|
|
428
|
-
# own, never a reason to retry.
|
|
429
|
-
streamed = false
|
|
430
|
-
consumer_error = nil
|
|
431
|
-
begin
|
|
432
|
-
open_connection(http, cancel)
|
|
433
|
-
cancel&.register do
|
|
434
|
-
http.read_timeout = 0.001
|
|
435
|
-
http.write_timeout = 0.001
|
|
436
|
-
http.open_timeout = 0.001
|
|
437
|
-
begin
|
|
438
|
-
http.instance_variable_get(:@socket)&.io&.close
|
|
439
|
-
rescue IOError, SystemCallError
|
|
440
|
-
nil
|
|
441
|
-
end
|
|
442
|
-
end
|
|
443
|
-
raise connection_error(true, write, key) if cancel&.cancelled?
|
|
444
|
-
if stream
|
|
445
|
-
http.request(request) do |answer|
|
|
446
|
-
answered = true
|
|
447
|
-
http.read_timeout = [deadline - now, 0.001].max
|
|
448
|
-
response = answer
|
|
449
|
-
# A late answer is read whole first: it may be a failure.
|
|
450
|
-
if answer.code.to_i < 300 && answer["runtime-late-answer"] != "true"
|
|
451
|
-
answer.read_body do |chunk|
|
|
452
|
-
streamed = true
|
|
453
|
-
begin
|
|
454
|
-
stream.call(chunk)
|
|
455
|
-
rescue Exception => e
|
|
456
|
-
consumer_error = e
|
|
457
|
-
raise
|
|
458
|
-
end
|
|
459
|
-
end
|
|
460
|
-
end
|
|
461
|
-
end
|
|
462
|
-
else
|
|
463
|
-
response = http.request(request) do
|
|
464
|
-
answered = true
|
|
465
|
-
http.read_timeout = [deadline - now, 0.001].max
|
|
466
|
-
end
|
|
467
|
-
end
|
|
468
|
-
status = response.code.to_i
|
|
469
|
-
if status >= 300 && status < 400
|
|
470
|
-
raise Net::HTTPBadResponse, "Runtime answered a redirect (#{status})"
|
|
471
|
-
end
|
|
472
|
-
rescue *NETWORK_ERRORS => e
|
|
473
|
-
http.finish if http.started?
|
|
474
|
-
raise if e.equal?(consumer_error)
|
|
475
|
-
|
|
476
|
-
unanswered = bounded && !answered && e.is_a?(Net::ReadTimeout) && now < deadline
|
|
477
|
-
late = (e.is_a?(Timeout::Error) && !unanswered) || now >= deadline || cancel&.cancelled?
|
|
478
|
-
raise connection_error(late, write, key, e) if late || no_retry || streamed || attempt >= max_retries
|
|
479
|
-
|
|
480
|
-
# A connection that never opened sent no byte of the request, so the
|
|
481
|
-
# next try, which may reach another server, goes at once; once in a
|
|
482
|
-
# row, so an API that is down is not hammered. A reset or a timeout
|
|
483
|
-
# may have reached a server and keeps the backoff and its key's rules.
|
|
484
|
-
at_once = Transport.never_sent?(e) && !resent_at_once
|
|
485
|
-
resent_at_once = at_once
|
|
486
|
-
pause(at_once ? rand * 0.05 : backoff(attempt), deadline, write, key, nil, cancel)
|
|
487
|
-
attempt += 1
|
|
488
|
-
next
|
|
489
|
-
ensure
|
|
490
|
-
watchdog.kill
|
|
491
|
-
end
|
|
492
|
-
# An answer that took longer than the API holds its headers (90 s)
|
|
493
|
-
# comes as a 200 sent early, and then its JSON: the answer, or
|
|
494
|
-
# {"error": ...} when the work failed after the status went
|
|
495
|
-
# (runtime-late-answer). A failure is raised as the error it is,
|
|
496
|
-
# never returned as a success.
|
|
497
|
-
late = status < 300 && response["runtime-late-answer"] == "true"
|
|
498
|
-
status = Transport.late_failure(response.body.to_s) if late
|
|
499
|
-
if status < 300
|
|
500
|
-
stream&.call(response.body.to_s) if late
|
|
501
|
-
return response
|
|
502
|
-
end
|
|
503
|
-
|
|
504
|
-
text = response.body.to_s
|
|
505
|
-
parsed = begin
|
|
506
|
-
JSON.parse(text)
|
|
507
|
-
rescue JSON::ParserError
|
|
508
|
-
{ "error" => { "message" => text.strip.empty? ? nil : text.strip[0, 500] }.compact }
|
|
509
|
-
end
|
|
510
|
-
failure = Error.from(status, parsed, key)
|
|
511
|
-
# A quota on what is kept says it never clears by waiting.
|
|
512
|
-
if !no_retry && room&.positive? && Error::WAITS_FOR_ROOM.include?(failure.code) && failure.details["field"] != "count" &&
|
|
513
|
-
failure.sent_retryable != false
|
|
514
|
-
room_left = room_until - now
|
|
515
|
-
raise failure if room_left <= 0
|
|
516
|
-
|
|
517
|
-
wait_for = failure.retry_after || room_backoff(room_attempt)
|
|
518
|
-
room_attempt += 1
|
|
519
|
-
pause([room_left, wait_for].min, deadline, write, key, failure, cancel)
|
|
520
|
-
next
|
|
521
|
-
end
|
|
522
|
-
header = Float(response["Retry-After"], exception: false)
|
|
523
|
-
header = nil unless header&.positive?
|
|
524
|
-
# The wait the server names, when it names one.
|
|
525
|
-
named = failure.retry_after || header
|
|
526
|
-
# Nothing ran: safe to send again for any call, past max_retries,
|
|
527
|
-
# with growing waits, until the deadline; then this refusal stands.
|
|
528
|
-
# A late answer's failure came after the work began, so only a
|
|
529
|
-
# guest's own refusal (guest_busy, before anything is journaled)
|
|
530
|
-
# still says nothing ran.
|
|
531
|
-
if [429, 503].include?(status) && Error::NOTHING_RAN.include?(failure.code) &&
|
|
532
|
-
failure.sent_retryable != false && (!late || failure.code == "guest_busy")
|
|
533
|
-
raise failure if named && named > LONGEST_WAIT
|
|
534
|
-
|
|
535
|
-
pause_for = [30.0, (named || 0.25) * (2**[idle_attempt, 3].min)].min * (0.9 + (rand * 0.2))
|
|
536
|
-
idle_attempt += 1
|
|
537
|
-
raise failure if pause_for >= deadline - now
|
|
538
|
-
|
|
539
|
-
pause(pause_for, deadline, write, key, failure, cancel)
|
|
540
|
-
next
|
|
541
|
-
end
|
|
542
|
-
# The answer says whether the same call may work again; an older
|
|
543
|
-
# server that does not say is retried on 429, 502, 503 and 504. A
|
|
544
|
-
# refusal for room or the account's own limit is the room wait's on a
|
|
545
|
-
# create, wake, restart or fork; any other call retries what it says.
|
|
546
|
-
again = if failure.sent_retryable.nil?
|
|
547
|
-
[429, 502, 503, 504].include?(status) && !Error::DELIBERATE.include?(failure.code)
|
|
548
|
-
else
|
|
549
|
-
failure.sent_retryable
|
|
550
|
-
end
|
|
551
|
-
raise failure if no_retry || !again || (room && Error::WAITS_FOR_ROOM.include?(failure.code)) ||
|
|
552
|
-
attempt >= max_retries
|
|
553
|
-
# A wait longer than a minute (a daily cap) is the caller's to
|
|
554
|
-
# schedule: the error says when, in retry_after.
|
|
555
|
-
raise failure if named && named > LONGEST_WAIT
|
|
556
|
-
|
|
557
|
-
pause([30.0, named || backoff(attempt)].min * (0.9 + (rand * 0.2)), deadline, write, key, failure, cancel)
|
|
558
|
-
attempt += 1
|
|
559
|
-
end
|
|
560
|
-
end
|
|
561
|
-
|
|
562
|
-
def pause(seconds, deadline, write, key, failure = nil, cancel = nil)
|
|
563
|
-
if now + seconds >= deadline
|
|
564
|
-
raise failure if failure
|
|
565
|
-
|
|
566
|
-
raise connection_error(true, write, key)
|
|
567
|
-
end
|
|
568
|
-
if cancel
|
|
569
|
-
raise connection_error(true, write, key) unless cancel.wait(seconds)
|
|
570
|
-
else
|
|
571
|
-
sleep(seconds)
|
|
572
|
-
end
|
|
573
|
-
end
|
|
574
|
-
end
|
|
575
|
-
end
|