@caelo-cms/provisioning 0.10.55 → 0.10.58
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-db.js +97 -24
- package/dist/cli-routing.js +1 -1
- package/dist/cli-routing.js.map +1 -1
- package/dist/cli.js +4 -1
- package/dist/cli.js.map +1 -1
- package/dist/compose.js +5 -0
- package/dist/compose.js.map +1 -1
- package/dist/converge-checkpoint.d.ts +66 -0
- package/dist/converge-checkpoint.d.ts.map +1 -0
- package/dist/converge-checkpoint.js +61 -0
- package/dist/converge-checkpoint.js.map +1 -0
- package/dist/converge-job-gcp.d.ts +46 -0
- package/dist/converge-job-gcp.d.ts.map +1 -0
- package/dist/converge-job-gcp.js +368 -0
- package/dist/converge-job-gcp.js.map +1 -0
- package/dist/converge-job-setup.d.ts +62 -0
- package/dist/converge-job-setup.d.ts.map +1 -0
- package/dist/converge-job-setup.js +228 -0
- package/dist/converge-job-setup.js.map +1 -0
- package/dist/converge-job.d.ts +141 -0
- package/dist/converge-job.d.ts.map +1 -0
- package/dist/converge-job.js +325 -0
- package/dist/converge-job.js.map +1 -0
- package/dist/gcp-names.d.ts +9 -0
- package/dist/gcp-names.d.ts.map +1 -1
- package/dist/gcp-names.js +13 -0
- package/dist/gcp-names.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/infra-contract.d.ts +1 -1
- package/dist/infra-contract.d.ts.map +1 -1
- package/dist/infra-contract.js +33 -1
- package/dist/infra-contract.js.map +1 -1
- package/dist/install-job.d.ts +11 -3
- package/dist/install-job.d.ts.map +1 -1
- package/dist/install-job.js +22 -12
- package/dist/install-job.js.map +1 -1
- package/dist/lifecycle.d.ts +13 -1
- package/dist/lifecycle.d.ts.map +1 -1
- package/dist/lifecycle.js +115 -58
- package/dist/lifecycle.js.map +1 -1
- package/dist/operator-access.d.ts +4 -0
- package/dist/operator-access.d.ts.map +1 -1
- package/dist/operator-access.js +43 -7
- package/dist/operator-access.js.map +1 -1
- package/dist/release-contract.d.ts +3 -2
- package/dist/release-contract.d.ts.map +1 -1
- package/dist/release-contract.js +5 -2
- package/dist/release-contract.js.map +1 -1
- package/dist/release-manifest.d.ts +2 -0
- package/dist/release-manifest.d.ts.map +1 -1
- package/dist/release-manifest.js.map +1 -1
- package/dist/remote-mcp-edge.d.ts +114 -0
- package/dist/remote-mcp-edge.d.ts.map +1 -0
- package/dist/remote-mcp-edge.js +116 -0
- package/dist/remote-mcp-edge.js.map +1 -0
- package/dist/remote-mcp.d.ts +139 -0
- package/dist/remote-mcp.d.ts.map +1 -0
- package/dist/remote-mcp.js +434 -0
- package/dist/remote-mcp.js.map +1 -0
- package/dist/runtime-secrets.d.ts +12 -0
- package/dist/runtime-secrets.d.ts.map +1 -1
- package/dist/runtime-secrets.js +32 -4
- package/dist/runtime-secrets.js.map +1 -1
- package/dist/seal-key-rotation.d.ts +38 -0
- package/dist/seal-key-rotation.d.ts.map +1 -0
- package/dist/seal-key-rotation.js +206 -0
- package/dist/seal-key-rotation.js.map +1 -0
- package/dist/stack-contract.d.ts +206 -3
- package/dist/stack-contract.d.ts.map +1 -1
- package/dist/stack-contract.js +219 -4
- package/dist/stack-contract.js.map +1 -1
- package/dist/stack-converge.d.ts +53 -1
- package/dist/stack-converge.d.ts.map +1 -1
- package/dist/stack-converge.js +124 -4
- package/dist/stack-converge.js.map +1 -1
- package/dist/state-lock-lease.d.ts +142 -0
- package/dist/state-lock-lease.d.ts.map +1 -0
- package/dist/state-lock-lease.js +347 -0
- package/dist/state-lock-lease.js.map +1 -0
- package/dist/state-store.d.ts +66 -9
- package/dist/state-store.d.ts.map +1 -1
- package/dist/state-store.js +249 -30
- package/dist/state-store.js.map +1 -1
- package/dist/upgrade/app-roll.d.ts +36 -7
- package/dist/upgrade/app-roll.d.ts.map +1 -1
- package/dist/upgrade/app-roll.js +53 -19
- package/dist/upgrade/app-roll.js.map +1 -1
- package/dist/upgrade/audit-job-roll.d.ts.map +1 -1
- package/dist/upgrade/audit-job-roll.js +3 -1
- package/dist/upgrade/audit-job-roll.js.map +1 -1
- package/dist/upgrade/converge.d.ts.map +1 -1
- package/dist/upgrade/converge.js +125 -12
- package/dist/upgrade/converge.js.map +1 -1
- package/dist/upgrade/gcp-lookup.d.ts +27 -10
- package/dist/upgrade/gcp-lookup.d.ts.map +1 -1
- package/dist/upgrade/gcp-lookup.js +46 -7
- package/dist/upgrade/gcp-lookup.js.map +1 -1
- package/dist/upgrade/record-rolled.d.ts +40 -0
- package/dist/upgrade/record-rolled.d.ts.map +1 -0
- package/dist/upgrade/record-rolled.js +46 -0
- package/dist/upgrade/record-rolled.js.map +1 -0
- package/dist/upgrade/upgrade.d.ts +45 -1
- package/dist/upgrade/upgrade.d.ts.map +1 -1
- package/dist/upgrade/upgrade.js +60 -17
- package/dist/upgrade/upgrade.js.map +1 -1
- package/dist/upgrader-assess.d.ts +15 -1
- package/dist/upgrader-assess.d.ts.map +1 -1
- package/dist/upgrader-assess.js +36 -6
- package/dist/upgrader-assess.js.map +1 -1
- package/dist/upgrader-gcp.d.ts +15 -0
- package/dist/upgrader-gcp.d.ts.map +1 -1
- package/dist/upgrader-gcp.js +42 -7
- package/dist/upgrader-gcp.js.map +1 -1
- package/dist/upgrader-iam-preflight.d.ts +123 -0
- package/dist/upgrader-iam-preflight.d.ts.map +1 -0
- package/dist/upgrader-iam-preflight.js +414 -0
- package/dist/upgrader-iam-preflight.js.map +1 -0
- package/dist/upgrader-job.d.ts +33 -9
- package/dist/upgrader-job.d.ts.map +1 -1
- package/dist/upgrader-job.js +169 -27
- package/dist/upgrader-job.js.map +1 -1
- package/dist/upgrader-records.d.ts +58 -0
- package/dist/upgrader-records.d.ts.map +1 -0
- package/dist/upgrader-records.js +87 -0
- package/dist/upgrader-records.js.map +1 -0
- package/dist/upgrader-reporter.d.ts +3 -2
- package/dist/upgrader-reporter.d.ts.map +1 -1
- package/dist/upgrader-reporter.js +10 -2
- package/dist/upgrader-reporter.js.map +1 -1
- package/dist/upgrader-setup.d.ts +25 -3
- package/dist/upgrader-setup.d.ts.map +1 -1
- package/dist/upgrader-setup.js +123 -11
- package/dist/upgrader-setup.js.map +1 -1
- package/dist/wizards/gcp-cost.d.ts.map +1 -1
- package/dist/wizards/gcp-cost.js +11 -3
- package/dist/wizards/gcp-cost.js.map +1 -1
- package/dist/wizards/gcp-pulumi.d.ts +6 -0
- package/dist/wizards/gcp-pulumi.d.ts.map +1 -1
- package/dist/wizards/gcp-pulumi.js +14 -2
- package/dist/wizards/gcp-pulumi.js.map +1 -1
- package/dist/wizards/gcp.d.ts.map +1 -1
- package/dist/wizards/gcp.js +106 -4
- package/dist/wizards/gcp.js.map +1 -1
- package/package.json +2 -2
- package/src/agent/run.test.ts +14 -1
- package/src/audit-job-setup.test.ts +7 -1
- package/src/cli-routing.ts +1 -1
- package/src/cli.ts +4 -1
- package/src/compose.ts +5 -0
- package/src/converge-checkpoint.ts +75 -0
- package/src/converge-entrypoint.test.ts +213 -0
- package/src/converge-job-gcp.test.ts +222 -0
- package/src/converge-job-gcp.ts +443 -0
- package/src/converge-job-setup.test.ts +152 -0
- package/src/converge-job-setup.ts +268 -0
- package/src/converge-job.test.ts +486 -0
- package/src/converge-job.ts +542 -0
- package/src/gateway-env-isolation.test.ts +8 -1
- package/src/gcp-names.ts +15 -0
- package/src/gcp-url-map.test.ts +9 -1
- package/src/index.ts +8 -0
- package/src/infra-contract.test.ts +8 -4
- package/src/infra-contract.ts +33 -1
- package/src/install-job.test.ts +3 -1
- package/src/install-job.ts +31 -14
- package/src/lifecycle.ts +137 -71
- package/src/operator-access.test.ts +24 -1
- package/src/operator-access.ts +44 -6
- package/src/release-contract.ts +5 -2
- package/src/release-manifest.test.ts +42 -0
- package/src/release-manifest.ts +2 -0
- package/src/remote-mcp-edge.ts +137 -0
- package/src/remote-mcp.test.ts +690 -0
- package/src/remote-mcp.ts +524 -0
- package/src/runtime-secrets.test.ts +31 -6
- package/src/runtime-secrets.ts +41 -3
- package/src/seal-key-rotation.test.ts +131 -0
- package/src/seal-key-rotation.ts +258 -0
- package/src/stack-contract-parity.test.ts +20 -3
- package/src/stack-contract.test.ts +32 -2
- package/src/stack-contract.ts +363 -4
- package/src/stack-converge-secrets.test.ts +4 -0
- package/src/stack-converge.test.ts +21 -5
- package/src/stack-converge.ts +162 -7
- package/src/state-attach.test.ts +7 -1
- package/src/state-lock-lease.test.ts +302 -0
- package/src/state-lock-lease.ts +423 -0
- package/src/state-lock.test.ts +568 -0
- package/src/state-store.ts +347 -27
- package/src/upgrade/app-roll.ts +102 -21
- package/src/upgrade/audit-job-roll.ts +3 -1
- package/src/upgrade/converge.ts +148 -10
- package/src/upgrade/gcp-lookup.test.ts +80 -0
- package/src/upgrade/gcp-lookup.ts +65 -8
- package/src/upgrade/record-rolled.test.ts +105 -0
- package/src/upgrade/record-rolled.ts +91 -0
- package/src/upgrade/upgrade.test.ts +203 -3
- package/src/upgrade/upgrade.ts +120 -24
- package/src/upgrader-assess.test.ts +92 -3
- package/src/upgrader-assess.ts +58 -9
- package/src/upgrader-gcp.ts +47 -8
- package/src/upgrader-iam-preflight.test.ts +393 -0
- package/src/upgrader-iam-preflight.ts +510 -0
- package/src/upgrader-job.test.ts +586 -15
- package/src/upgrader-job.ts +237 -53
- package/src/upgrader-records.ts +151 -0
- package/src/upgrader-reporter.test.ts +6 -0
- package/src/upgrader-reporter.ts +7 -2
- package/src/upgrader-setup.test.ts +3 -3
- package/src/upgrader-setup.ts +128 -11
- package/src/wizards/gcp-cost.ts +12 -3
- package/src/wizards/gcp-pulumi.test.ts +15 -0
- package/src/wizards/gcp-pulumi.ts +18 -2
- package/src/wizards/gcp.ts +134 -4
- package/stacks/aws/index.ts +10 -0
- package/stacks/azure/index.ts +9 -0
- package/stacks/gcp/index.ts +47 -7
- package/stacks/gcp-firebase/index.ts +38 -6
package/dist/state-store.js
CHANGED
|
@@ -20,12 +20,13 @@
|
|
|
20
20
|
* the bucket store shells out to `gcloud storage` synchronously.
|
|
21
21
|
*/
|
|
22
22
|
import { spawnSync } from "node:child_process";
|
|
23
|
+
import { randomUUID } from "node:crypto";
|
|
23
24
|
import { existsSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
24
|
-
import { hostname } from "node:os";
|
|
25
25
|
import { join } from "node:path";
|
|
26
26
|
import { assertNoCheckpointSecrets, assertNoSecrets } from "./checkpoint-secrets.js";
|
|
27
27
|
import { gcloudEnv } from "./gcloud.js";
|
|
28
28
|
import { OPS_INSTALL_PREFIX } from "./state-backend.js";
|
|
29
|
+
import { describeExecutionArgs, judgeHeldLock, lockBody, PROCESS_LEASE_MS, parseExecutionState, parseLockBody, renewedLockBody, } from "./state-lock-lease.js";
|
|
29
30
|
/** The real {@link SyncGcloud}: `gcloud` with prompts disabled. */
|
|
30
31
|
export const realSyncGcloud = (args, input) => {
|
|
31
32
|
const r = spawnSync("gcloud", args, {
|
|
@@ -150,51 +151,239 @@ export class StateLockHeldError extends Error {
|
|
|
150
151
|
export function bucketStateLockUrl(bucket) {
|
|
151
152
|
return `gs://${bucket}/${OPS_INSTALL_PREFIX}/state.lock`;
|
|
152
153
|
}
|
|
153
|
-
/**
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
}
|
|
154
|
+
/** gcloud's wording for a failed `--if-generation-match` precondition. */
|
|
155
|
+
const PRECONDITION_FAILED = /PreconditionFailed|412|At least one of the pre-conditions/i;
|
|
156
|
+
const defaultSchedule = (tick, everyMs) => {
|
|
157
|
+
const timer = setInterval(tick, everyMs);
|
|
158
|
+
// The heartbeat must never keep a finished CLI alive.
|
|
159
|
+
timer.unref?.();
|
|
160
|
+
return () => clearInterval(timer);
|
|
161
|
+
};
|
|
162
162
|
/**
|
|
163
163
|
* The install's state lock in its ops bucket, created only if absent
|
|
164
164
|
* (`--if-generation-match=0`). Held by every writer of a bucket-backed
|
|
165
|
-
* install's state — an installer run's `pulumi up`, `state migrate
|
|
166
|
-
*
|
|
167
|
-
*
|
|
168
|
-
* automatically.
|
|
165
|
+
* install's state — an installer run's `pulumi up`, `state migrate`, the
|
|
166
|
+
* CLI `upgrade`, the upgrader job — so none writes a backend another is
|
|
167
|
+
* switching away from.
|
|
169
168
|
*
|
|
170
|
-
*
|
|
169
|
+
* A lock that is held is judged (state-lock-lease.ts): one whose Cloud Run
|
|
170
|
+
* execution has finished, whose execution is gone after its lease, whose
|
|
171
|
+
* CLI process is gone on this host, or whose CLI lease ran out unrenewed,
|
|
172
|
+
* is broken with a generation-matched delete and taken once more. The
|
|
173
|
+
* generation is read BEFORE the body, so the delete can only remove the
|
|
174
|
+
* lock that was judged: if another run replaced it in between, the
|
|
175
|
+
* delete's precondition fails and nothing is removed. Two runs breaking the
|
|
176
|
+
* same dead lock cannot both win either — the slower one's create fails
|
|
177
|
+
* its own precondition and reports the lock as held.
|
|
178
|
+
*
|
|
179
|
+
* A CLI holder renews its lease every third of {@link PROCESS_LEASE_MS}
|
|
180
|
+
* with a generation-matched rewrite, so a run that is alive keeps its lock
|
|
181
|
+
* however long it takes, and a killed one loses it within one lease.
|
|
182
|
+
*
|
|
183
|
+
* @returns the release function; it stops the heartbeat and deletes only
|
|
184
|
+
* this lock's generation
|
|
185
|
+
* @throws StateLockHeldError when another run holds the lock
|
|
186
|
+
* @throws Error when the lock or its holder cannot be read (no access)
|
|
171
187
|
*/
|
|
172
|
-
export function acquireBucketStateLock(bucket, purpose, run = realSyncGcloud) {
|
|
188
|
+
export function acquireBucketStateLock(bucket, purpose, run = realSyncGcloud, opts = {}) {
|
|
173
189
|
const url = bucketStateLockUrl(bucket);
|
|
174
|
-
const
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
190
|
+
const now = opts.now ?? (() => new Date());
|
|
191
|
+
const holder = opts.holder ?? { leaseMs: PROCESS_LEASE_MS };
|
|
192
|
+
const id = randomUUID();
|
|
193
|
+
const create = () => run(["storage", "cp", "-", url, "--if-generation-match=0"], lockBody(purpose, now(), holder, id));
|
|
194
|
+
let r = create();
|
|
195
|
+
if (!r.ok && PRECONDITION_FAILED.test(r.stderr)) {
|
|
196
|
+
breakDeadLock(url, run, now(), opts.onBreak, opts.here);
|
|
197
|
+
r = create();
|
|
198
|
+
if (!r.ok && PRECONDITION_FAILED.test(r.stderr)) {
|
|
199
|
+
throw new StateLockHeldError(`Another run took this install's state lock at the same moment (${readLockText(url, run)}). Re-run once it has finished.`);
|
|
179
200
|
}
|
|
201
|
+
}
|
|
202
|
+
if (!r.ok)
|
|
180
203
|
throw new Error(`Could not take the state lock ${url}: ${r.stderr.trim()}`);
|
|
204
|
+
const owned = ownedGeneration(url, run, id);
|
|
205
|
+
if (owned.kind !== "owned") {
|
|
206
|
+
// Ours, just created — but without its generation a delete could hit a
|
|
207
|
+
// successor's lock. Give it back only once a second look proves it is
|
|
208
|
+
// still ours at a known generation; otherwise its lease ends it.
|
|
209
|
+
const retry = ownedGeneration(url, run, id);
|
|
210
|
+
if (retry.kind === "owned") {
|
|
211
|
+
run(["storage", "rm", url, `--if-generation-match=${retry.generation}`]);
|
|
212
|
+
}
|
|
213
|
+
throw new Error(`Could not read the generation of the state lock ${url} this run just took (${owned.why}). ${retry.kind === "owned"
|
|
214
|
+
? "It was given back."
|
|
215
|
+
: "It was left in place and expires with its lease; re-run after that."}`);
|
|
181
216
|
}
|
|
217
|
+
let generation = owned.generation;
|
|
218
|
+
let lost = null;
|
|
219
|
+
const stopHeartbeat = "execution" in holder
|
|
220
|
+
? () => { }
|
|
221
|
+
: (opts.schedule ?? defaultSchedule)(() => {
|
|
222
|
+
if (lost !== null)
|
|
223
|
+
return;
|
|
224
|
+
const renewed = renewLock(url, run, id, generation, now(), holder.leaseMs);
|
|
225
|
+
if (renewed.kind === "owned") {
|
|
226
|
+
generation = renewed.generation;
|
|
227
|
+
return;
|
|
228
|
+
}
|
|
229
|
+
if (renewed.kind === "lost") {
|
|
230
|
+
lost = renewed.why;
|
|
231
|
+
opts.onLost?.(`This run no longer holds the install's state lock (${renewed.why}). Stop it and re-run once the other run has finished.`);
|
|
232
|
+
}
|
|
233
|
+
// "error": a transient failure; the next beat tries again within the lease.
|
|
234
|
+
}, Math.floor(holder.leaseMs / 3));
|
|
182
235
|
return () => {
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
236
|
+
stopHeartbeat();
|
|
237
|
+
const rm = run(["storage", "rm", url, `--if-generation-match=${generation}`]);
|
|
238
|
+
if (rm.ok)
|
|
239
|
+
return;
|
|
240
|
+
if (PRECONDITION_FAILED.test(rm.stderr) || NOT_FOUND.test(rm.stderr)) {
|
|
241
|
+
throw new Error(`The state lock ${url} was no longer this run's when it finished (another run broke it); it was left alone.`);
|
|
242
|
+
}
|
|
243
|
+
throw new Error(`Could not release ${url}; remove it: gcloud storage rm ${url}`);
|
|
186
244
|
};
|
|
187
245
|
}
|
|
246
|
+
/**
|
|
247
|
+
* The lock's generation, provided the lock is still the one with `id`. The
|
|
248
|
+
* generation is read before the body: if another run replaced the lock in
|
|
249
|
+
* between, the body names its id and this returns "lost" — never a
|
|
250
|
+
* generation that is not ours.
|
|
251
|
+
*/
|
|
252
|
+
function ownedGeneration(url, run, id) {
|
|
253
|
+
const g = lockGeneration(url, run);
|
|
254
|
+
if (g.kind === "absent")
|
|
255
|
+
return { kind: "lost", why: "the lock is gone" };
|
|
256
|
+
if (g.kind === "error")
|
|
257
|
+
return { kind: "error", why: g.error };
|
|
258
|
+
const body = run(["storage", "cat", url]);
|
|
259
|
+
if (!body.ok) {
|
|
260
|
+
return NOT_FOUND.test(body.stderr)
|
|
261
|
+
? { kind: "lost", why: "the lock is gone" }
|
|
262
|
+
: { kind: "error", why: body.stderr.trim() || `gcloud exited ${body.exitCode}` };
|
|
263
|
+
}
|
|
264
|
+
const parsed = parseLockBody(body.stdout);
|
|
265
|
+
if (parsed?.id !== id) {
|
|
266
|
+
return { kind: "lost", why: `another run holds it now (${body.stdout.trim().slice(0, 200)})` };
|
|
267
|
+
}
|
|
268
|
+
return { kind: "owned", generation: g.generation };
|
|
269
|
+
}
|
|
270
|
+
/**
|
|
271
|
+
* One heartbeat: rewrite the lock with a later lease, only if it is still
|
|
272
|
+
* at `generation` (ours), and return its new generation.
|
|
273
|
+
*/
|
|
274
|
+
function renewLock(url, run, id, generation, now, leaseMs) {
|
|
275
|
+
const current = run(["storage", "cat", url]);
|
|
276
|
+
if (!current.ok) {
|
|
277
|
+
return NOT_FOUND.test(current.stderr)
|
|
278
|
+
? { kind: "lost", why: "the lock is gone" }
|
|
279
|
+
: { kind: "error", why: current.stderr.trim() };
|
|
280
|
+
}
|
|
281
|
+
const body = parseLockBody(current.stdout);
|
|
282
|
+
if (body?.id !== id) {
|
|
283
|
+
return {
|
|
284
|
+
kind: "lost",
|
|
285
|
+
why: `another run holds it now (${current.stdout.trim().slice(0, 200)})`,
|
|
286
|
+
};
|
|
287
|
+
}
|
|
288
|
+
const w = run(["storage", "cp", "-", url, `--if-generation-match=${generation}`], renewedLockBody(body, now, leaseMs));
|
|
289
|
+
if (!w.ok) {
|
|
290
|
+
return PRECONDITION_FAILED.test(w.stderr)
|
|
291
|
+
? { kind: "lost", why: "another run replaced it" }
|
|
292
|
+
: { kind: "error", why: w.stderr.trim() };
|
|
293
|
+
}
|
|
294
|
+
return ownedGeneration(url, run, id);
|
|
295
|
+
}
|
|
296
|
+
/** The lock object's current generation. */
|
|
297
|
+
function lockGeneration(url, run) {
|
|
298
|
+
const d = run(["storage", "objects", "describe", url, "--format=json"]);
|
|
299
|
+
if (!d.ok) {
|
|
300
|
+
return NOT_FOUND.test(d.stderr)
|
|
301
|
+
? { kind: "absent" }
|
|
302
|
+
: { kind: "error", error: d.stderr.trim() || `gcloud exited ${d.exitCode}` };
|
|
303
|
+
}
|
|
304
|
+
try {
|
|
305
|
+
// `gcloud storage objects describe` prints the generation as a string
|
|
306
|
+
// (`generation: '1706304784726341'`, cloud.google.com/storage/docs/
|
|
307
|
+
// viewing-editing-metadata); a JSON number would lose digits past 2^53,
|
|
308
|
+
// so only a string of digits is a generation the delete may match.
|
|
309
|
+
const g = JSON.parse(d.stdout).generation;
|
|
310
|
+
if (typeof g === "string" && /^[1-9][0-9]*$/.test(g)) {
|
|
311
|
+
return { kind: "found", generation: g };
|
|
312
|
+
}
|
|
313
|
+
}
|
|
314
|
+
catch {
|
|
315
|
+
// Reported below with the same message as a missing field.
|
|
316
|
+
}
|
|
317
|
+
return { kind: "error", error: "gcloud described the lock without a generation" };
|
|
318
|
+
}
|
|
319
|
+
function readLockText(url, run) {
|
|
320
|
+
const held = run(["storage", "cat", url]);
|
|
321
|
+
return held.ok ? held.stdout.trim() : url;
|
|
322
|
+
}
|
|
323
|
+
/**
|
|
324
|
+
* Judge the held lock at `url` and break it when its holder is dead.
|
|
325
|
+
* Returns normally when the lock was broken or vanished meanwhile, so the
|
|
326
|
+
* caller creates it once more; throws when it is live or cannot be judged.
|
|
327
|
+
*/
|
|
328
|
+
function breakDeadLock(url, run, now, onBreak, here) {
|
|
329
|
+
const found = lockGeneration(url, run);
|
|
330
|
+
if (found.kind === "absent")
|
|
331
|
+
return;
|
|
332
|
+
if (found.kind === "error") {
|
|
333
|
+
throw new Error(`Could not read the state lock ${url} another run holds: ${found.error}`);
|
|
334
|
+
}
|
|
335
|
+
const held = run(["storage", "cat", url]);
|
|
336
|
+
if (!held.ok) {
|
|
337
|
+
if (NOT_FOUND.test(held.stderr))
|
|
338
|
+
return;
|
|
339
|
+
throw new Error(`Could not read the state lock ${url}: ${held.stderr.trim()}`);
|
|
340
|
+
}
|
|
341
|
+
const verdict = judgeHeldLock(held.stdout, (ref) => parseExecutionState(run(describeExecutionArgs(ref))), now, here);
|
|
342
|
+
if (verdict.kind === "unknown") {
|
|
343
|
+
// Not contention: retrying will not help, and guessing "dead" could let
|
|
344
|
+
// two runs write the same state.
|
|
345
|
+
throw new Error(`The state lock ${url} is held and ${verdict.why}`);
|
|
346
|
+
}
|
|
347
|
+
if (verdict.kind === "held") {
|
|
348
|
+
throw new StateLockHeldError(verdict.manualRemoval
|
|
349
|
+
? `Another run is changing this install's state (${verdict.why}). Re-run once it has finished; if none is running, remove the stale lock: gcloud storage rm ${url}`
|
|
350
|
+
: `Another run is changing this install's state: ${verdict.why}. Re-run once it has finished.`);
|
|
351
|
+
}
|
|
352
|
+
const rm = run(["storage", "rm", url, `--if-generation-match=${found.generation}`]);
|
|
353
|
+
if (!rm.ok && !NOT_FOUND.test(rm.stderr)) {
|
|
354
|
+
if (PRECONDITION_FAILED.test(rm.stderr)) {
|
|
355
|
+
throw new StateLockHeldError(`Another run took this install's state lock while this one removed a stale one (${readLockText(url, run)}). Re-run once it has finished.`);
|
|
356
|
+
}
|
|
357
|
+
throw new Error(`Could not remove the stale state lock ${url}: ${rm.stderr.trim()}`);
|
|
358
|
+
}
|
|
359
|
+
onBreak?.(`Removed a stale state lock: ${verdict.why}.`);
|
|
360
|
+
}
|
|
188
361
|
/**
|
|
189
362
|
* The same lock for an install whose state is on this machine:
|
|
190
|
-
* `~/.caelo-<id>/state.lock`, created exclusively.
|
|
363
|
+
* `~/.caelo-<id>/state.lock`, created exclusively. A lock left by a
|
|
364
|
+
* process that is gone on this machine is taken over (#815).
|
|
191
365
|
*
|
|
366
|
+
* @param here this host and its pid check; tests pass a fake
|
|
192
367
|
* @returns the release function
|
|
193
368
|
*/
|
|
194
|
-
export function acquireLocalStateLock(root, purpose) {
|
|
369
|
+
export function acquireLocalStateLock(root, purpose, here) {
|
|
195
370
|
const path = join(root, "state.lock");
|
|
371
|
+
const take = () => writeFileSync(path, lockBody(purpose, new Date()), { flag: "wx", mode: 0o600 });
|
|
196
372
|
try {
|
|
197
|
-
|
|
373
|
+
try {
|
|
374
|
+
take();
|
|
375
|
+
}
|
|
376
|
+
catch (e) {
|
|
377
|
+
if (e.code !== "EEXIST")
|
|
378
|
+
throw e;
|
|
379
|
+
// #815 — the file is on this machine, so a holder whose process has
|
|
380
|
+
// ended is known dead: take its lock over instead of asking for `rm`.
|
|
381
|
+
const verdict = judgeHeldLock(readFileSync(path, "utf8"), () => ({ kind: "unknown", error: "a machine-local lock names no Cloud Run execution" }), new Date(), here);
|
|
382
|
+
if (verdict.kind !== "break")
|
|
383
|
+
throw e;
|
|
384
|
+
rmSync(path, { force: true });
|
|
385
|
+
take();
|
|
386
|
+
}
|
|
198
387
|
}
|
|
199
388
|
catch (e) {
|
|
200
389
|
if (e.code === "EEXIST") {
|
|
@@ -205,9 +394,39 @@ export function acquireLocalStateLock(root, purpose) {
|
|
|
205
394
|
return () => rmSync(path, { force: true });
|
|
206
395
|
}
|
|
207
396
|
/** The state lock for whichever backend `bucket` (gcs) or `root` (local) names. */
|
|
208
|
-
export function acquireStateLock(backend, root, purpose, run = realSyncGcloud) {
|
|
397
|
+
export function acquireStateLock(backend, root, purpose, run = realSyncGcloud, opts = {}) {
|
|
209
398
|
return backend.kind === "gcs"
|
|
210
|
-
? acquireBucketStateLock(backend.bucket, purpose, run)
|
|
211
|
-
: acquireLocalStateLock(root, purpose);
|
|
399
|
+
? acquireBucketStateLock(backend.bucket, purpose, run, opts)
|
|
400
|
+
: acquireLocalStateLock(root, purpose, opts.here);
|
|
401
|
+
}
|
|
402
|
+
/**
|
|
403
|
+
* Give the state lock back when the CLI is stopped (Ctrl-C, a closed Cloud
|
|
404
|
+
* Shell tab's SIGTERM), then exit with the signal's conventional code
|
|
405
|
+
* (#815). Without this a stopped run leaves its lock until its lease ends.
|
|
406
|
+
*
|
|
407
|
+
* @param report told what happened to the lock
|
|
408
|
+
* @returns removes the handlers; call it once the run released the lock itself
|
|
409
|
+
*/
|
|
410
|
+
export function releaseOnSignal(release, report, proc = process) {
|
|
411
|
+
const dispose = () => {
|
|
412
|
+
proc.removeListener("SIGINT", onInt);
|
|
413
|
+
proc.removeListener("SIGTERM", onTerm);
|
|
414
|
+
};
|
|
415
|
+
const stop = (signal, code) => {
|
|
416
|
+
dispose();
|
|
417
|
+
try {
|
|
418
|
+
release();
|
|
419
|
+
report(`Stopped (${signal}); the install's state lock was released.`);
|
|
420
|
+
}
|
|
421
|
+
catch (e) {
|
|
422
|
+
report(`Stopped (${signal}); ${e instanceof Error ? e.message : String(e)}`);
|
|
423
|
+
}
|
|
424
|
+
return proc.exit(code);
|
|
425
|
+
};
|
|
426
|
+
const onInt = () => stop("SIGINT", 130);
|
|
427
|
+
const onTerm = () => stop("SIGTERM", 143);
|
|
428
|
+
proc.once("SIGINT", onInt);
|
|
429
|
+
proc.once("SIGTERM", onTerm);
|
|
430
|
+
return dispose;
|
|
212
431
|
}
|
|
213
432
|
//# sourceMappingURL=state-store.js.map
|
package/dist/state-store.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"state-store.js","sourceRoot":"","sources":["../src/state-store.ts"],"names":[],"mappings":"AAAA,mCAAmC;AAEnC;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACtF,OAAO,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AACnC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,yBAAyB,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AACrF,OAAO,EAAqB,SAAS,EAAE,MAAM,aAAa,CAAC;AAC3D,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAiBxD,mEAAmE;AACnE,MAAM,CAAC,MAAM,cAAc,GAAe,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;IACxD,MAAM,CAAC,GAAG,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE;QAClC,GAAG,EAAE,SAAS,EAAE;QAChB,QAAQ,EAAE,MAAM;QAChB,SAAS,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;QAC3B,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;KAC7E,CAAC,CAAC;IACH,IAAI,CAAC,CAAC,KAAK;QAAE,MAAM,CAAC,CAAC,KAAK,CAAC;IAC3B,OAAO;QACL,EAAE,EAAE,CAAC,CAAC,MAAM,KAAK,CAAC;QAClB,MAAM,EAAE,CAAC,CAAC,MAAM,IAAI,EAAE;QACtB,MAAM,EAAE,CAAC,CAAC,MAAM,IAAI,EAAE;QACtB,QAAQ,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC;KACxB,CAAC;AACJ,CAAC,CAAC;AAEF;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAe,EAAE,KAAc;IAC9D,IAAI,IAAI,KAAK,cAAc;QAAE,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;;QACrD,yBAAyB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC5C,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC;AAC/C,CAAC;AAED,yEAAyE;AACzE,SAAS,eAAe,CAAC,IAAY,EAAE,IAAY;IACjD,MAAM,GAAG,GAAG,GAAG,IAAI,QAAQ,OAAO,CAAC,GAAG,EAAE,CAAC;IACzC,IAAI,CAAC;QACH,aAAa,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAC1C,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IACxB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7B,MAAM,CAAC,CAAC;IACV,CAAC;AACH,CAAC;AAED,wDAAwD;AACxD,MAAM,OAAO,eAAe;IAEG;IADpB,IAAI,GAAG,OAAgB,CAAC;IACjC,YAA6B,IAAY;QAAZ,SAAI,GAAJ,IAAI,CAAQ;IAAG,CAAC;IAE7C,IAAI,CAAC,IAAe;QAClB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACnC,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,IAAe,EAAE,KAAc;QACnC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,gBAAgB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;IACxE,CAAC;CACF;AAED,wDAAwD;AACxD,MAAM,SAAS,GAAG,iEAAiE,CAAC;AAEpF;;;;;;GAMG;AACH,MAAM,OAAO,aAAa;IAKL;IACA;IACA;IANV,IAAI,GAAG,KAAc,CAAC;IACd,KAAK,GAAG,IAAI,GAAG,EAA4B,CAAC;IAE7D,YACmB,MAAc,EACd,IAAY,EACZ,MAAkB,cAAc;QAFhC,WAAM,GAAN,MAAM,CAAQ;QACd,SAAI,GAAJ,IAAI,CAAQ;QACZ,QAAG,GAAH,GAAG,CAA6B;IAChD,CAAC;IAEJ,sCAAsC;IACtC,SAAS,CAAC,IAAe;QACvB,OAAO,QAAQ,IAAI,CAAC,MAAM,IAAI,kBAAkB,IAAI,IAAI,EAAE,CAAC;IAC7D,CAAC;IAED;;;;OAIG;IACH,IAAI,CAAC,IAAe;QAClB,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC7D,IAAI,CAAC,CAAC,EAAE;YAAE,OAAO,CAAC,CAAC,MAAM,CAAC;QAC1B,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;YAAE,OAAO,IAAI,CAAC;QAC1C,MAAM,IAAI,KAAK,CACb,kBAAkB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,iBAAiB,CAAC,CAAC,QAAQ,EAAE,2GAA2G,CACvM,CAAC;IACJ,CAAC;IAED,IAAI,CAAC,IAAe;QAClB,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC;QAC9D,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC3B,IAAI,IAAI,KAAK,IAAI;YAAE,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;QAChE,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,IAAe,EAAE,KAAc;QACnC,MAAM,IAAI,GAAG,gBAAgB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC3C,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAChB,CAAC,SAAS,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,iCAAiC,CAAC,EAC/E,IAAI,CACL,CAAC;QACF,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CACb,mBAAmB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,iBAAiB,CAAC,CAAC,QAAQ,EAAE,yEAAyE,CACtK,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC3B,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;IAC/C,CAAC;CACF;AAED;;;;GAIG;AACH,MAAM,OAAO,kBAAmB,SAAQ,KAAK;IACzB,IAAI,GAAG,oBAAoB,CAAC;CAC/C;AAED,+EAA+E;AAC/E,MAAM,UAAU,kBAAkB,CAAC,MAAc;IAC/C,OAAO,QAAQ,MAAM,IAAI,kBAAkB,aAAa,CAAC;AAC3D,CAAC;AAED,iDAAiD;AACjD,SAAS,QAAQ,CAAC,OAAe;IAC/B,OAAO,IAAI,CAAC,SAAS,CAAC;QACpB,OAAO;QACP,IAAI,EAAE,QAAQ,EAAE;QAChB,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACpC,CAAC,CAAC;AACL,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,sBAAsB,CACpC,MAAc,EACd,OAAe,EACf,MAAkB,cAAc;IAEhC,MAAM,GAAG,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC;IACvC,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,yBAAyB,CAAC,EAAE,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;IACzF,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QACV,IAAI,4DAA4D,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;YAChF,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;YAC1C,MAAM,IAAI,kBAAkB,CAC1B,iDAAiD,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,gGAAgG,GAAG,EAAE,CACzL,CAAC;QACJ,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,iCAAiC,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IAC9E,CAAC;IACD,OAAO,GAAG,EAAE;QACV,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;QACvC,IAAI,CAAC,EAAE,CAAC,EAAE;YAAE,MAAM,IAAI,KAAK,CAAC,qBAAqB,GAAG,kCAAkC,GAAG,EAAE,CAAC,CAAC;IAC/F,CAAC,CAAC;AACJ,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAY,EAAE,OAAe;IACjE,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;IACtC,IAAI,CAAC;QACH,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC,OAAO,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IACtE,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAK,CAA2B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACnD,MAAM,IAAI,kBAAkB,CAC1B,iDAAiD,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,iFAAiF,IAAI,EAAE,CAC1K,CAAC;QACJ,CAAC;QACD,MAAM,CAAC,CAAC;IACV,CAAC;IACD,OAAO,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AAC7C,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,gBAAgB,CAC9B,OAAuF,EACvF,IAAY,EACZ,OAAe,EACf,MAAkB,cAAc;IAEhC,OAAO,OAAO,CAAC,IAAI,KAAK,KAAK;QAC3B,CAAC,CAAC,sBAAsB,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,GAAG,CAAC;QACtD,CAAC,CAAC,qBAAqB,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;AAC3C,CAAC"}
|
|
1
|
+
{"version":3,"file":"state-store.js","sourceRoot":"","sources":["../src/state-store.ts"],"names":[],"mappings":"AAAA,mCAAmC;AAEnC;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AAC/C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACtF,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,yBAAyB,EAAE,eAAe,EAAE,MAAM,yBAAyB,CAAC;AACrF,OAAO,EAAqB,SAAS,EAAE,MAAM,aAAa,CAAC;AAC3D,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EACL,qBAAqB,EAErB,aAAa,EAGb,QAAQ,EACR,gBAAgB,EAChB,mBAAmB,EACnB,aAAa,EACb,eAAe,GAChB,MAAM,uBAAuB,CAAC;AAiB/B,mEAAmE;AACnE,MAAM,CAAC,MAAM,cAAc,GAAe,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;IACxD,MAAM,CAAC,GAAG,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE;QAClC,GAAG,EAAE,SAAS,EAAE;QAChB,QAAQ,EAAE,MAAM;QAChB,SAAS,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI;QAC3B,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;KAC7E,CAAC,CAAC;IACH,IAAI,CAAC,CAAC,KAAK;QAAE,MAAM,CAAC,CAAC,KAAK,CAAC;IAC3B,OAAO;QACL,EAAE,EAAE,CAAC,CAAC,MAAM,KAAK,CAAC;QAClB,MAAM,EAAE,CAAC,CAAC,MAAM,IAAI,EAAE;QACtB,MAAM,EAAE,CAAC,CAAC,MAAM,IAAI,EAAE;QACtB,QAAQ,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC;KACxB,CAAC;AACJ,CAAC,CAAC;AAEF;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,IAAe,EAAE,KAAc;IAC9D,IAAI,IAAI,KAAK,cAAc;QAAE,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;;QACrD,yBAAyB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IAC5C,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC;AAC/C,CAAC;AAED,yEAAyE;AACzE,SAAS,eAAe,CAAC,IAAY,EAAE,IAAY;IACjD,MAAM,GAAG,GAAG,GAAG,IAAI,QAAQ,OAAO,CAAC,GAAG,EAAE,CAAC;IACzC,IAAI,CAAC;QACH,aAAa,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAC1C,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;IACxB,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,MAAM,CAAC,GAAG,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC7B,MAAM,CAAC,CAAC;IACV,CAAC;AACH,CAAC;AAED,wDAAwD;AACxD,MAAM,OAAO,eAAe;IAEG;IADpB,IAAI,GAAG,OAAgB,CAAC;IACjC,YAA6B,IAAY;QAAZ,SAAI,GAAJ,IAAI,CAAQ;IAAG,CAAC;IAE7C,IAAI,CAAC,IAAe;QAClB,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACnC,OAAO,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,IAAe,EAAE,KAAc;QACnC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,gBAAgB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;IACxE,CAAC;CACF;AAED,wDAAwD;AACxD,MAAM,SAAS,GAAG,iEAAiE,CAAC;AAEpF;;;;;;GAMG;AACH,MAAM,OAAO,aAAa;IAKL;IACA;IACA;IANV,IAAI,GAAG,KAAc,CAAC;IACd,KAAK,GAAG,IAAI,GAAG,EAA4B,CAAC;IAE7D,YACmB,MAAc,EACd,IAAY,EACZ,MAAkB,cAAc;QAFhC,WAAM,GAAN,MAAM,CAAQ;QACd,SAAI,GAAJ,IAAI,CAAQ;QACZ,QAAG,GAAH,GAAG,CAA6B;IAChD,CAAC;IAEJ,sCAAsC;IACtC,SAAS,CAAC,IAAe;QACvB,OAAO,QAAQ,IAAI,CAAC,MAAM,IAAI,kBAAkB,IAAI,IAAI,EAAE,CAAC;IAC7D,CAAC;IAED;;;;OAIG;IACH,IAAI,CAAC,IAAe;QAClB,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC7D,IAAI,CAAC,CAAC,EAAE;YAAE,OAAO,CAAC,CAAC,MAAM,CAAC;QAC1B,IAAI,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;YAAE,OAAO,IAAI,CAAC;QAC1C,MAAM,IAAI,KAAK,CACb,kBAAkB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,iBAAiB,CAAC,CAAC,QAAQ,EAAE,2GAA2G,CACvM,CAAC;IACJ,CAAC;IAED,IAAI,CAAC,IAAe;QAClB,IAAI,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC;QAC9D,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7B,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC3B,IAAI,IAAI,KAAK,IAAI;YAAE,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;QAChE,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,IAAe,EAAE,KAAc;QACnC,MAAM,IAAI,GAAG,gBAAgB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC3C,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAChB,CAAC,SAAS,EAAE,IAAI,EAAE,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,iCAAiC,CAAC,EAC/E,IAAI,CACL,CAAC;QACF,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;YACV,MAAM,IAAI,KAAK,CACb,mBAAmB,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,iBAAiB,CAAC,CAAC,QAAQ,EAAE,yEAAyE,CACtK,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC3B,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;IAC/C,CAAC;CACF;AAED;;;;GAIG;AACH,MAAM,OAAO,kBAAmB,SAAQ,KAAK;IACzB,IAAI,GAAG,oBAAoB,CAAC;CAC/C;AAED,+EAA+E;AAC/E,MAAM,UAAU,kBAAkB,CAAC,MAAc;IAC/C,OAAO,QAAQ,MAAM,IAAI,kBAAkB,aAAa,CAAC;AAC3D,CAAC;AAED,0EAA0E;AAC1E,MAAM,mBAAmB,GAAG,4DAA4D,CAAC;AA0BzF,MAAM,eAAe,GAAG,CAAC,IAAgB,EAAE,OAAe,EAAgB,EAAE;IAC1E,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACzC,sDAAsD;IACtD,KAAK,CAAC,KAAK,EAAE,EAAE,CAAC;IAChB,OAAO,GAAG,EAAE,CAAC,aAAa,CAAC,KAAK,CAAC,CAAC;AACpC,CAAC,CAAC;AAEF;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,MAAM,UAAU,sBAAsB,CACpC,MAAc,EACd,OAAe,EACf,MAAkB,cAAc,EAChC,OAA0B,EAAE;IAE5B,MAAM,GAAG,GAAG,kBAAkB,CAAC,MAAM,CAAC,CAAC;IACvC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAe,IAAI,CAAC,MAAM,IAAI,EAAE,OAAO,EAAE,gBAAgB,EAAE,CAAC;IACxE,MAAM,EAAE,GAAG,UAAU,EAAE,CAAC;IACxB,MAAM,MAAM,GAAG,GAAG,EAAE,CAClB,GAAG,CACD,CAAC,SAAS,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,yBAAyB,CAAC,EACtD,QAAQ,CAAC,OAAO,EAAE,GAAG,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC,CACrC,CAAC;IACJ,IAAI,CAAC,GAAG,MAAM,EAAE,CAAC;IACjB,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;QAChD,aAAa,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QACxD,CAAC,GAAG,MAAM,EAAE,CAAC;QACb,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC;YAChD,MAAM,IAAI,kBAAkB,CAC1B,kEAAkE,YAAY,CAAC,GAAG,EAAE,GAAG,CAAC,iCAAiC,CAC1H,CAAC;QACJ,CAAC;IACH,CAAC;IACD,IAAI,CAAC,CAAC,CAAC,EAAE;QAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,GAAG,KAAK,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IACvF,MAAM,KAAK,GAAG,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;IAC5C,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC3B,uEAAuE;QACvE,sEAAsE;QACtE,iEAAiE;QACjE,MAAM,KAAK,GAAG,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;QAC5C,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC3B,GAAG,CAAC,CAAC,SAAS,EAAE,IAAI,EAAE,GAAG,EAAE,yBAAyB,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;QAC3E,CAAC;QACD,MAAM,IAAI,KAAK,CACb,mDAAmD,GAAG,wBAAwB,KAAK,CAAC,GAAG,MACrF,KAAK,CAAC,IAAI,KAAK,OAAO;YACpB,CAAC,CAAC,oBAAoB;YACtB,CAAC,CAAC,qEACN,EAAE,CACH,CAAC;IACJ,CAAC;IACD,IAAI,UAAU,GAAG,KAAK,CAAC,UAAU,CAAC;IAClC,IAAI,IAAI,GAAkB,IAAI,CAAC;IAC/B,MAAM,aAAa,GACjB,WAAW,IAAI,MAAM;QACnB,CAAC,CAAC,GAAG,EAAE,GAAE,CAAC;QACV,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,IAAI,eAAe,CAAC,CAChC,GAAG,EAAE;YACH,IAAI,IAAI,KAAK,IAAI;gBAAE,OAAO;YAC1B,MAAM,OAAO,GAAG,SAAS,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,UAAU,EAAE,GAAG,EAAE,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC;YAC3E,IAAI,OAAO,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;gBAC7B,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;gBAChC,OAAO;YACT,CAAC;YACD,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;gBAC5B,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC;gBACnB,IAAI,CAAC,MAAM,EAAE,CACX,sDAAsD,OAAO,CAAC,GAAG,wDAAwD,CAC1H,CAAC;YACJ,CAAC;YACD,4EAA4E;QAC9E,CAAC,EACD,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,GAAG,CAAC,CAAC,CAC/B,CAAC;IACR,OAAO,GAAG,EAAE;QACV,aAAa,EAAE,CAAC;QAChB,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,IAAI,EAAE,GAAG,EAAE,yBAAyB,UAAU,EAAE,CAAC,CAAC,CAAC;QAC9E,IAAI,EAAE,CAAC,EAAE;YAAE,OAAO;QAClB,IAAI,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC;YACrE,MAAM,IAAI,KAAK,CACb,kBAAkB,GAAG,uFAAuF,CAC7G,CAAC;QACJ,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,qBAAqB,GAAG,kCAAkC,GAAG,EAAE,CAAC,CAAC;IACnF,CAAC,CAAC;AACJ,CAAC;AAOD;;;;;GAKG;AACH,SAAS,eAAe,CAAC,GAAW,EAAE,GAAe,EAAE,EAAU;IAC/D,MAAM,CAAC,GAAG,cAAc,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACnC,IAAI,CAAC,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,kBAAkB,EAAE,CAAC;IAC1E,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO;QAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;IAC/D,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;IAC1C,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;QACb,OAAO,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;YAChC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,kBAAkB,EAAE;YAC3C,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,iBAAiB,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC;IACrF,CAAC;IACD,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC1C,IAAI,MAAM,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC;QACtB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,6BAA6B,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG,EAAE,CAAC;IACjG,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC;AACrD,CAAC;AAED;;;GAGG;AACH,SAAS,SAAS,CAChB,GAAW,EACX,GAAe,EACf,EAAU,EACV,UAAkB,EAClB,GAAS,EACT,OAAe;IAEf,MAAM,OAAO,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;IAC7C,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC;QAChB,OAAO,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YACnC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,kBAAkB,EAAE;YAC3C,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;IACpD,CAAC;IACD,MAAM,IAAI,GAAG,aAAa,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC3C,IAAI,IAAI,EAAE,EAAE,KAAK,EAAE,EAAE,CAAC;QACpB,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,GAAG,EAAE,6BAA6B,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,GAAG;SACzE,CAAC;IACJ,CAAC;IACD,MAAM,CAAC,GAAG,GAAG,CACX,CAAC,SAAS,EAAE,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,yBAAyB,UAAU,EAAE,CAAC,EAClE,eAAe,CAAC,IAAI,EAAE,GAAG,EAAE,OAAO,CAAC,CACpC,CAAC;IACF,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QACV,OAAO,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;YACvC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,yBAAyB,EAAE;YAClD,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;IAC9C,CAAC;IACD,OAAO,eAAe,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;AACvC,CAAC;AAOD,4CAA4C;AAC5C,SAAS,cAAc,CAAC,GAAW,EAAE,GAAe;IAClD,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,GAAG,EAAE,eAAe,CAAC,CAAC,CAAC;IACxE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC;QACV,OAAO,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;YAC7B,CAAC,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE;YACpB,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,iBAAiB,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC;IACjF,CAAC;IACD,IAAI,CAAC;QACH,sEAAsE;QACtE,oEAAoE;QACpE,wEAAwE;QACxE,mEAAmE;QACnE,MAAM,CAAC,GAAI,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAA8B,CAAC,UAAU,CAAC;QACxE,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;YACrD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC;QAC1C,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,2DAA2D;IAC7D,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,gDAAgD,EAAE,CAAC;AACpF,CAAC;AAED,SAAS,YAAY,CAAC,GAAW,EAAE,GAAe;IAChD,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;IAC1C,OAAO,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC;AAC5C,CAAC;AAED;;;;GAIG;AACH,SAAS,aAAa,CACpB,GAAW,EACX,GAAe,EACf,GAAS,EACT,OAAgD,EAChD,IAA2B;IAE3B,MAAM,KAAK,GAAG,cAAc,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;IACvC,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO;IACpC,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC3B,MAAM,IAAI,KAAK,CAAC,iCAAiC,GAAG,uBAAuB,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;IAC5F,CAAC;IACD,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;IAC1C,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;QACb,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;YAAE,OAAO;QACxC,MAAM,IAAI,KAAK,CAAC,iCAAiC,GAAG,KAAK,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IACjF,CAAC;IACD,MAAM,OAAO,GAAG,aAAa,CAC3B,IAAI,CAAC,MAAM,EACX,CAAC,GAAG,EAAE,EAAE,CAAC,mBAAmB,CAAC,GAAG,CAAC,qBAAqB,CAAC,GAAG,CAAC,CAAC,CAAC,EAC7D,GAAG,EACH,IAAI,CACL,CAAC;IACF,IAAI,OAAO,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC/B,wEAAwE;QACxE,iCAAiC;QACjC,MAAM,IAAI,KAAK,CAAC,kBAAkB,GAAG,gBAAgB,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IACtE,CAAC;IACD,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC5B,MAAM,IAAI,kBAAkB,CAC1B,OAAO,CAAC,aAAa;YACnB,CAAC,CAAC,iDAAiD,OAAO,CAAC,GAAG,gGAAgG,GAAG,EAAE;YACnK,CAAC,CAAC,iDAAiD,OAAO,CAAC,GAAG,gCAAgC,CACjG,CAAC;IACJ,CAAC;IACD,MAAM,EAAE,GAAG,GAAG,CAAC,CAAC,SAAS,EAAE,IAAI,EAAE,GAAG,EAAE,yBAAyB,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;IACpF,IAAI,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC;QACzC,IAAI,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC;YACxC,MAAM,IAAI,kBAAkB,CAC1B,kFAAkF,YAAY,CAAC,GAAG,EAAE,GAAG,CAAC,iCAAiC,CAC1I,CAAC;QACJ,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,yCAAyC,GAAG,KAAK,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IACvF,CAAC;IACD,OAAO,EAAE,CAAC,+BAA+B,OAAO,CAAC,GAAG,GAAG,CAAC,CAAC;AAC3D,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAY,EAAE,OAAe,EAAE,IAAgB;IACnF,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,EAAE,YAAY,CAAC,CAAC;IACtC,MAAM,IAAI,GAAG,GAAG,EAAE,CAChB,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC,OAAO,EAAE,IAAI,IAAI,EAAE,CAAC,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;IAClF,IAAI,CAAC;QACH,IAAI,CAAC;YACH,IAAI,EAAE,CAAC;QACT,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,IAAK,CAA2B,CAAC,IAAI,KAAK,QAAQ;gBAAE,MAAM,CAAC,CAAC;YAC5D,oEAAoE;YACpE,sEAAsE;YACtE,MAAM,OAAO,GAAG,aAAa,CAC3B,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,EAC1B,GAAG,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,mDAAmD,EAAE,CAAC,EACvF,IAAI,IAAI,EAAE,EACV,IAAI,CACL,CAAC;YACF,IAAI,OAAO,CAAC,IAAI,KAAK,OAAO;gBAAE,MAAM,CAAC,CAAC;YACtC,MAAM,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAC9B,IAAI,EAAE,CAAC;QACT,CAAC;IACH,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,IAAK,CAA2B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YACnD,MAAM,IAAI,kBAAkB,CAC1B,iDAAiD,YAAY,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,iFAAiF,IAAI,EAAE,CAC1K,CAAC;QACJ,CAAC;QACD,MAAM,CAAC,CAAC;IACV,CAAC;IACD,OAAO,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AAC7C,CAAC;AAED,mFAAmF;AACnF,MAAM,UAAU,gBAAgB,CAC9B,OAAuF,EACvF,IAAY,EACZ,OAAe,EACf,MAAkB,cAAc,EAChC,OAA0B,EAAE;IAE5B,OAAO,OAAO,CAAC,IAAI,KAAK,KAAK;QAC3B,CAAC,CAAC,sBAAsB,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC;QAC5D,CAAC,CAAC,qBAAqB,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;AACtD,CAAC;AASD;;;;;;;GAOG;AACH,MAAM,UAAU,eAAe,CAC7B,OAAmB,EACnB,MAAiC,EACjC,OAAqB,OAAO;IAE5B,MAAM,OAAO,GAAG,GAAG,EAAE;QACnB,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QACrC,IAAI,CAAC,cAAc,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IACzC,CAAC,CAAC;IACF,MAAM,IAAI,GAAG,CAAC,MAA4B,EAAE,IAAY,EAAS,EAAE;QACjE,OAAO,EAAE,CAAC;QACV,IAAI,CAAC;YACH,OAAO,EAAE,CAAC;YACV,MAAM,CAAC,YAAY,MAAM,2CAA2C,CAAC,CAAC;QACxE,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,CAAC,YAAY,MAAM,MAAM,CAAC,YAAY,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC/E,CAAC;QACD,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACzB,CAAC,CAAC;IACF,MAAM,KAAK,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,GAAG,EAAE,CAAC,IAAI,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;IAC1C,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;IAC3B,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAC7B,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
|
@@ -23,6 +23,7 @@
|
|
|
23
23
|
*/
|
|
24
24
|
import type { GcloudRunner, Sleep } from "../gcloud-retry.js";
|
|
25
25
|
import type { ImageDigests } from "../install-state.js";
|
|
26
|
+
import { type CloudRunEnvVar } from "../stack-contract.js";
|
|
26
27
|
import { type EnvChange, type LiveEnvValue, type LiveVolumes } from "../stack-converge.js";
|
|
27
28
|
import { type SqlBackup } from "./backup.js";
|
|
28
29
|
import type { HealthProbe } from "./health.js";
|
|
@@ -58,14 +59,24 @@ export interface ServicePlan {
|
|
|
58
59
|
/** Container memory limit now (null = Cloud Run's default). */
|
|
59
60
|
readonly liveMemory: string | null;
|
|
60
61
|
readonly liveVolumes: LiveVolumes;
|
|
62
|
+
/** Request timeout now, in seconds (null = Cloud Run's default). */
|
|
63
|
+
readonly liveTimeout: number | null;
|
|
61
64
|
}
|
|
62
65
|
/** A service plan plus the changes its roll applies. */
|
|
63
66
|
export interface RollPlan extends ServicePlan {
|
|
64
67
|
readonly envFlags: readonly string[];
|
|
65
68
|
readonly envChanges: readonly EnvChange[];
|
|
66
|
-
/**
|
|
69
|
+
/**
|
|
70
|
+
* #553 — e.g. `--memory=2Gi` when the admin runs below the stack default;
|
|
71
|
+
* #806 — `--timeout=3600` when the gateway runs below its contract timeout.
|
|
72
|
+
*/
|
|
67
73
|
readonly resourceFlags: readonly string[];
|
|
68
|
-
|
|
74
|
+
readonly resourceChanges: readonly string[];
|
|
75
|
+
/**
|
|
76
|
+
* The admin's media bucket volume when it is (partly) missing, the
|
|
77
|
+
* service's secret files (#806, the gateway's OAuth client) when not
|
|
78
|
+
* mounted; empty otherwise.
|
|
79
|
+
*/
|
|
69
80
|
readonly volumeFlags: readonly string[];
|
|
70
81
|
readonly volumeChanges: readonly string[];
|
|
71
82
|
}
|
|
@@ -90,11 +101,22 @@ export type PlanResult = {
|
|
|
90
101
|
* no change; fails when anything a roll needs is missing.
|
|
91
102
|
*/
|
|
92
103
|
export declare function planAppRoll(install: AppRollInstall, images: Readonly<Record<RollSlug, ReleaseImage>>, run: GcloudRunner): Promise<PlanResult>;
|
|
104
|
+
/** Env the converge half adds to, or keeps out of, the rolls ({@link AppRollHooks.afterMigrations}). */
|
|
105
|
+
export interface EnvAdjustments {
|
|
106
|
+
/** Contract vars to leave exactly as the services have them. */
|
|
107
|
+
readonly leaveUntouched?: readonly string[];
|
|
108
|
+
/**
|
|
109
|
+
* Vars outside the env contract the rolls set too — #806's remote-MCP
|
|
110
|
+
* identity env, which only the converge (operator identity) can resolve.
|
|
111
|
+
*/
|
|
112
|
+
readonly extraEnv?: Readonly<Partial<Record<RollSlug, readonly CloudRunEnvVar[]>>>;
|
|
113
|
+
}
|
|
93
114
|
/**
|
|
94
|
-
* The plan with
|
|
95
|
-
*
|
|
115
|
+
* The plan with the converge's adjustments: vars left exactly as the
|
|
116
|
+
* services have them (they point at something a converge step could not set
|
|
117
|
+
* up) and vars outside the contract the rolls set too.
|
|
96
118
|
*/
|
|
97
|
-
export declare function
|
|
119
|
+
export declare function adjustingEnv(plan: AppRollPlan, adjust: EnvAdjustments): AppRollPlan;
|
|
98
120
|
/** Print what the plan will change, one line per change. */
|
|
99
121
|
export declare function reportPlan(plan: AppRollPlan, reporter: UpgradeReporter): void;
|
|
100
122
|
/** Converge steps that must sit between app-roll steps (none for the upgrader job). */
|
|
@@ -104,10 +126,10 @@ export interface AppRollHooks {
|
|
|
104
126
|
* vars the rolls must leave untouched because what they point at could
|
|
105
127
|
* not be set up; `ok: false` aborts the upgrade (nothing rolled yet).
|
|
106
128
|
*/
|
|
107
|
-
afterMigrations?(): Promise<{
|
|
129
|
+
afterMigrations?(): Promise<({
|
|
108
130
|
readonly ok: true;
|
|
109
131
|
readonly leaveUntouched: readonly string[];
|
|
110
|
-
} | {
|
|
132
|
+
} & Pick<EnvAdjustments, "extraEnv">) | {
|
|
111
133
|
readonly ok: false;
|
|
112
134
|
}>;
|
|
113
135
|
/** Right before `slug` rolls; `ok: false` aborts and rolls back what rolled. */
|
|
@@ -158,6 +180,13 @@ export type AppRollResult = {
|
|
|
158
180
|
readonly phase: AppRollPhase;
|
|
159
181
|
/** Whether migrations ran before the failure (the backup matters then). */
|
|
160
182
|
readonly migrated: boolean;
|
|
183
|
+
/**
|
|
184
|
+
* Whether services had rolled and EVERY one of them was pinned back to
|
|
185
|
+
* the revision that served before — i.e. the previous release serves
|
|
186
|
+
* again. False when nothing had rolled, or when a pin failed (then a
|
|
187
|
+
* person must pin it; the reporter named which).
|
|
188
|
+
*/
|
|
189
|
+
readonly rolledBack: boolean;
|
|
161
190
|
};
|
|
162
191
|
/** Run the app-roll. See the file comment for the sequence. */
|
|
163
192
|
export declare function executeAppRoll(input: AppRollInput, hooks: AppRollHooks, deps: AppRollDeps): Promise<AppRollResult>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"app-roll.d.ts","sourceRoot":"","sources":["../../src/upgrade/app-roll.ts"],"names":[],"mappings":"AAEA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAE9D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"app-roll.d.ts","sourceRoot":"","sources":["../../src/upgrade/app-roll.ts"],"names":[],"mappings":"AAEA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAE9D,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAEL,KAAK,cAAc,EAIpB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAEL,KAAK,SAAS,EACd,KAAK,YAAY,EACjB,KAAK,WAAW,EAYjB,MAAM,sBAAsB,CAAC;AAE9B,OAAO,EAAkB,KAAK,SAAS,EAAE,MAAM,aAAa,CAAC;AAQ7D,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAErD,MAAM,MAAM,QAAQ,GAAG,OAAO,GAAG,SAAS,CAAC;AAE3C,gFAAgF;AAChF,eAAO,MAAM,UAAU,EAAE,SAAS,QAAQ,EAAyB,CAAC;AAEpE,uCAAuC;AACvC,MAAM,WAAW,cAAc;IAC7B,QAAQ,CAAC,QAAQ,EAAE,KAAK,GAAG,cAAc,CAAC;IAC1C,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,wCAAwC;IACxC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,iEAAiE;IACjE,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;CAC9B;AAED,8CAA8C;AAC9C,MAAM,WAAW,YAAY;IAC3B,qEAAqE;IACrE,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED,4DAA4D;AAC5D,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC;IACxB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,OAAO,EAAE,WAAW,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;IACpD,+DAA+D;IAC/D,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;IAClC,oEAAoE;IACpE,QAAQ,CAAC,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;CACrC;AAED,wDAAwD;AACxD,MAAM,WAAW,QAAS,SAAQ,WAAW;IAC3C,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;IACrC,QAAQ,CAAC,UAAU,EAAE,SAAS,SAAS,EAAE,CAAC;IAC1C;;;OAGG;IACH,QAAQ,CAAC,aAAa,EAAE,SAAS,MAAM,EAAE,CAAC;IAC1C,QAAQ,CAAC,eAAe,EAAE,SAAS,MAAM,EAAE,CAAC;IAC5C;;;;OAIG;IACH,QAAQ,CAAC,WAAW,EAAE,SAAS,MAAM,EAAE,CAAC;IACxC,QAAQ,CAAC,aAAa,EAAE,SAAS,MAAM,EAAE,CAAC;CAC3C;AAED,kFAAkF;AAClF,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,OAAO,EAAE,cAAc,CAAC;IACjC,6BAA6B;IAC7B,QAAQ,CAAC,QAAQ,EAAE,SAAS,QAAQ,EAAE,CAAC;IACvC,+DAA+D;IAC/D,QAAQ,CAAC,QAAQ,EAAE,SAAS,MAAM,EAAE,CAAC;CACtC;AAED,MAAM,MAAM,UAAU,GAClB;IAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAA;CAAE,GACjD;IAAE,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAEnD;;;;GAIG;AACH,wBAAsB,WAAW,CAC/B,OAAO,EAAE,cAAc,EACvB,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC,EAChD,GAAG,EAAE,YAAY,GAChB,OAAO,CAAC,UAAU,CAAC,CAiGrB;AAED,wGAAwG;AACxG,MAAM,WAAW,cAAc;IAC7B,gEAAgE;IAChE,QAAQ,CAAC,cAAc,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAC5C;;;OAGG;IACH,QAAQ,CAAC,QAAQ,CAAC,EAAE,QAAQ,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,EAAE,SAAS,cAAc,EAAE,CAAC,CAAC,CAAC,CAAC;CACpF;AA2BD;;;;GAIG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,WAAW,EAAE,MAAM,EAAE,cAAc,GAAG,WAAW,CAanF;AAED,4DAA4D;AAC5D,wBAAgB,UAAU,CAAC,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE,eAAe,GAAG,IAAI,CAW7E;AAED,uFAAuF;AACvF,MAAM,WAAW,YAAY;IAC3B;;;;OAIG;IACH,eAAe,CAAC,IAAI,OAAO,CACvB,CAAC;QAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;QAAC,QAAQ,CAAC,cAAc,EAAE,SAAS,MAAM,EAAE,CAAA;KAAE,GAAG,IAAI,CACvE,cAAc,EACd,UAAU,CACX,CAAC,GACF;QAAE,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAA;KAAE,CACzB,CAAC;IACF,gFAAgF;IAChF,UAAU,CAAC,CACT,IAAI,EAAE,QAAQ,GACb,OAAO,CAAC;QAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAA;KAAE,GAAG;QAAE,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC;QAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACnF,yEAAyE;IACzE,aAAa,CAAC,CAAC,IAAI,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC/C;AAED,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,GAAG,EAAE,YAAY,CAAC;IAC3B,QAAQ,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC;IACvB,QAAQ,CAAC,QAAQ,EAAE,eAAe,CAAC;IACnC,yEAAyE;IACzE,QAAQ,CAAC,OAAO,EAAE,CAChB,QAAQ,EAAE,MAAM,KACb,OAAO,CAAC;QAAE,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAChE,8EAA8E;IAC9E,QAAQ,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI,CAAC;IACpC,uEAAuE;IACvE,QAAQ,CAAC,aAAa,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,OAAO,CAAC,IAAI,CAAC,CAAC;CAClE;AAED,MAAM,WAAW,YAAY;IAC3B,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAC;IAC3B;;;OAGG;IACH,QAAQ,CAAC,MAAM,EAAE,SAAS,CAAC;IAC3B,oFAAoF;IACpF,QAAQ,CAAC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;IACtC;;;OAGG;IACH,QAAQ,CAAC,kBAAkB,EAAE,OAAO,GAAG,IAAI,CAAC;CAC7C;AAED,MAAM,MAAM,YAAY,GAAG,iBAAiB,GAAG,YAAY,GAAG,UAAU,GAAG,MAAM,GAAG,QAAQ,CAAC;AAE7F,MAAM,MAAM,aAAa,GACrB;IAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,CAAC;IAAC,QAAQ,CAAC,OAAO,EAAE,YAAY,CAAA;CAAE,GACrD;IACE,QAAQ,CAAC,EAAE,EAAE,KAAK,CAAC;IACnB,QAAQ,CAAC,KAAK,EAAE,YAAY,CAAC;IAC7B,2EAA2E;IAC3E,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC;IAC3B;;;;;OAKG;IACH,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC;CAC9B,CAAC;AAEN,+DAA+D;AAC/D,wBAAsB,cAAc,CAClC,KAAK,EAAE,YAAY,EACnB,KAAK,EAAE,YAAY,EACnB,IAAI,EAAE,WAAW,GAChB,OAAO,CAAC,aAAa,CAAC,CA2MxB"}
|
package/dist/upgrade/app-roll.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// SPDX-License-Identifier: MPL-2.0
|
|
2
2
|
import { gatewayServiceAccountEmail, runServiceAccountEmail } from "../gcp-names.js";
|
|
3
|
-
import { adminMediaVolume } from "../stack-contract.js";
|
|
4
|
-
import { liveContainerEnv, liveContainerMemory, liveVolumes, planAdminMemory, planContractEnv, planMediaVolume, rollService, serviceRollArgs, } from "../stack-converge.js";
|
|
3
|
+
import { adminMediaVolume, GATEWAY_TIMEOUT_SECONDS, RETIRED_SECRET_FILES, SERVICE_SECRET_FILES, } from "../stack-contract.js";
|
|
4
|
+
import { liveContainerEnv, liveContainerMemory, liveTimeoutSeconds, liveVolumes, planAdminMemory, planContractEnv, planMediaVolume, planRequestTimeout, planSecretFiles, rollService, serviceRollArgs, } from "../stack-converge.js";
|
|
5
5
|
import { describeAuditJobImage, setAuditJobImage } from "./audit-job-roll.js";
|
|
6
6
|
import { restoreCommand } from "./backup.js";
|
|
7
7
|
import { describeServiceJson, findCurrentRevision, findServiceUrl, pinTraffic, resolveGcpResourceName, } from "./gcp-lookup.js";
|
|
@@ -41,9 +41,10 @@ export async function planAppRoll(install, images, run) {
|
|
|
41
41
|
liveEnv: liveContainerEnv(json),
|
|
42
42
|
liveMemory: liveContainerMemory(json),
|
|
43
43
|
liveVolumes: liveVolumes(json),
|
|
44
|
+
liveTimeout: liveTimeoutSeconds(json),
|
|
44
45
|
});
|
|
45
46
|
}
|
|
46
|
-
const env = planEnv(install, services,
|
|
47
|
+
const env = planEnv(install, services, {});
|
|
47
48
|
if (!env.ok)
|
|
48
49
|
return { ok: false, error: `the install's env contract can't be applied: ${env.error}` };
|
|
49
50
|
// Without the media volume every upload is lost on the next revision or
|
|
@@ -52,6 +53,17 @@ export async function planAppRoll(install, images, run) {
|
|
|
52
53
|
const media = planMediaVolume(admin.liveVolumes, adminMediaVolume(install.projectId, install.env));
|
|
53
54
|
if (!media.ok)
|
|
54
55
|
return { ok: false, error: `the admin's media volume can't be set up: ${media.error}` };
|
|
56
|
+
// #806/#914 — each service's secret files (the gateway's Google sign-in
|
|
57
|
+
// client). A file the roll cannot mount fails the plan: the new gateway
|
|
58
|
+
// reads its path from the env contract.
|
|
59
|
+
const secretFiles = {};
|
|
60
|
+
for (const s of services) {
|
|
61
|
+
const files = planSecretFiles(s.liveVolumes, SERVICE_SECRET_FILES[s.slug], install.env, RETIRED_SECRET_FILES[s.slug]);
|
|
62
|
+
if (!files.ok) {
|
|
63
|
+
return { ok: false, error: `the ${s.slug}'s secret files can't be mounted: ${files.error}` };
|
|
64
|
+
}
|
|
65
|
+
secretFiles[s.slug] = files;
|
|
66
|
+
}
|
|
55
67
|
const warnings = [];
|
|
56
68
|
const rolls = services.map((s) => {
|
|
57
69
|
// #553 — the admin runs the Lighthouse quality audit; raise it to the
|
|
@@ -59,18 +71,27 @@ export async function planAppRoll(install, images, run) {
|
|
|
59
71
|
const memory = s.slug === "admin" ? planAdminMemory(s.liveMemory) : null;
|
|
60
72
|
if (memory && !memory.ok)
|
|
61
73
|
warnings.push(`admin memory left as is: ${memory.error}`);
|
|
74
|
+
// #806 — the gateway streams remote-MCP chat turns (stack-contract.ts
|
|
75
|
+
// GATEWAY_TIMEOUT_SECONDS). The admin's timeout is the stacks' alone.
|
|
76
|
+
const timeout = s.slug === "gateway" ? planRequestTimeout(s.liveTimeout, GATEWAY_TIMEOUT_SECONDS) : null;
|
|
77
|
+
const memoryFlags = memory?.ok ? memory.flags : [];
|
|
78
|
+
const timeoutFlags = timeout?.ok ? timeout.flags : [];
|
|
62
79
|
return {
|
|
63
80
|
...s,
|
|
64
81
|
envFlags: env.services[s.slug].flags,
|
|
65
82
|
envChanges: env.services[s.slug].changes,
|
|
66
|
-
resourceFlags:
|
|
67
|
-
|
|
68
|
-
|
|
83
|
+
resourceFlags: [...memoryFlags, ...timeoutFlags],
|
|
84
|
+
resourceChanges: [
|
|
85
|
+
...memoryFlags.map((f) => `${s.liveMemory ?? "(default)"} → ${f}`),
|
|
86
|
+
...(timeout?.ok ? timeout.changes : []),
|
|
87
|
+
],
|
|
88
|
+
volumeFlags: [...(s.slug === "admin" ? media.flags : []), ...secretFiles[s.slug].flags],
|
|
89
|
+
volumeChanges: [...(s.slug === "admin" ? media.changes : []), ...secretFiles[s.slug].changes],
|
|
69
90
|
};
|
|
70
91
|
});
|
|
71
92
|
return { ok: true, plan: { install, services: rolls, warnings } };
|
|
72
93
|
}
|
|
73
|
-
function planEnv(install, services,
|
|
94
|
+
function planEnv(install, services, adjust) {
|
|
74
95
|
const deployed = Object.fromEntries(services.map((s) => [s.slug, { serviceName: s.serviceName, liveEnv: s.liveEnv }]));
|
|
75
96
|
return planContractEnv({
|
|
76
97
|
provider: install.provider,
|
|
@@ -79,16 +100,21 @@ function planEnv(install, services, leaveUntouched) {
|
|
|
79
100
|
domain: install.domain,
|
|
80
101
|
region: install.region,
|
|
81
102
|
stagingHost: install.stagingHost,
|
|
82
|
-
}, deployed, {
|
|
103
|
+
}, deployed, {
|
|
104
|
+
leaveUntouched: adjust.leaveUntouched ?? [],
|
|
105
|
+
...(adjust.extraEnv ? { extraEnv: adjust.extraEnv } : {}),
|
|
106
|
+
});
|
|
83
107
|
}
|
|
84
108
|
/**
|
|
85
|
-
* The plan with
|
|
86
|
-
*
|
|
109
|
+
* The plan with the converge's adjustments: vars left exactly as the
|
|
110
|
+
* services have them (they point at something a converge step could not set
|
|
111
|
+
* up) and vars outside the contract the rolls set too.
|
|
87
112
|
*/
|
|
88
|
-
export function
|
|
89
|
-
|
|
113
|
+
export function adjustingEnv(plan, adjust) {
|
|
114
|
+
const extra = Object.values(adjust.extraEnv ?? {}).some((v) => (v?.length ?? 0) > 0);
|
|
115
|
+
if ((adjust.leaveUntouched?.length ?? 0) === 0 && !extra)
|
|
90
116
|
return plan;
|
|
91
|
-
const env = planEnv(plan.install, plan.services,
|
|
117
|
+
const env = planEnv(plan.install, plan.services, adjust);
|
|
92
118
|
if (!env.ok)
|
|
93
119
|
return plan;
|
|
94
120
|
return {
|
|
@@ -106,10 +132,9 @@ export function reportPlan(plan, reporter) {
|
|
|
106
132
|
reporter.warn(w);
|
|
107
133
|
for (const s of plan.services) {
|
|
108
134
|
for (const c of s.volumeChanges)
|
|
109
|
-
reporter.info(`${s.slug}
|
|
110
|
-
for (const
|
|
111
|
-
reporter.info(`${s.slug} resources: ${
|
|
112
|
-
}
|
|
135
|
+
reporter.info(`${s.slug} volumes: ${c}`);
|
|
136
|
+
for (const c of s.resourceChanges)
|
|
137
|
+
reporter.info(`${s.slug} resources: ${c}`);
|
|
113
138
|
for (const c of s.envChanges) {
|
|
114
139
|
reporter.info(`${s.slug} env ${c.name}: ${c.from === undefined ? "(unset)" : c.from} → ${c.to === undefined ? "(removed)" : c.to}`);
|
|
115
140
|
}
|
|
@@ -120,10 +145,13 @@ export async function executeAppRoll(input, hooks, deps) {
|
|
|
120
145
|
const { reporter: r, run } = deps;
|
|
121
146
|
const { install } = input.plan;
|
|
122
147
|
let plan = input.plan;
|
|
148
|
+
/** Set by {@link rollBack}: null until one ran with services to restore. */
|
|
149
|
+
let rollbackOk = null;
|
|
123
150
|
const fail = (phase, migrated) => ({
|
|
124
151
|
ok: false,
|
|
125
152
|
phase,
|
|
126
153
|
migrated,
|
|
154
|
+
rolledBack: rollbackOk === true,
|
|
127
155
|
});
|
|
128
156
|
// A probe that cannot reach the install now would only fail after the
|
|
129
157
|
// migrations, and roll a healthy release back.
|
|
@@ -158,7 +186,10 @@ export async function executeAppRoll(input, hooks, deps) {
|
|
|
158
186
|
const converged = await hooks.afterMigrations();
|
|
159
187
|
if (!converged.ok)
|
|
160
188
|
return fail("converge", true);
|
|
161
|
-
plan =
|
|
189
|
+
plan = adjustingEnv(plan, {
|
|
190
|
+
leaveUntouched: converged.leaveUntouched,
|
|
191
|
+
...(converged.extraEnv ? { extraEnv: converged.extraEnv } : {}),
|
|
192
|
+
});
|
|
162
193
|
}
|
|
163
194
|
const rolled = [];
|
|
164
195
|
/** The audit job's image before this roll moved it (#875); null = not moved. */
|
|
@@ -175,12 +206,15 @@ export async function executeAppRoll(input, hooks, deps) {
|
|
|
175
206
|
if (services.length === 0)
|
|
176
207
|
return;
|
|
177
208
|
r.warn(`Rolling back ${services.length} service(s) to the revisions that served before...`);
|
|
209
|
+
rollbackOk = true;
|
|
178
210
|
for (const p of [...services].reverse()) {
|
|
179
211
|
const ok = await pinTraffic(run, install.projectId, install.region, p.serviceName, p.priorRevision);
|
|
180
212
|
if (ok)
|
|
181
213
|
r.info(` ${p.slug} traffic restored to ${p.priorRevision}`);
|
|
182
|
-
else
|
|
214
|
+
else {
|
|
215
|
+
rollbackOk = false;
|
|
183
216
|
r.error(` ${p.slug} rollback FAILED — pin ${p.priorRevision} by hand`);
|
|
217
|
+
}
|
|
184
218
|
await hooks.afterRollback?.(p.slug);
|
|
185
219
|
}
|
|
186
220
|
};
|