@caelo-cms/provisioning 0.10.53 → 0.10.57
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent-db.js +272 -128
- package/dist/audit-job-setup.d.ts +69 -0
- package/dist/audit-job-setup.d.ts.map +1 -0
- package/dist/audit-job-setup.js +317 -0
- package/dist/audit-job-setup.js.map +1 -0
- package/dist/caddy.d.ts +9 -0
- package/dist/caddy.d.ts.map +1 -1
- package/dist/caddy.js +38 -0
- package/dist/caddy.js.map +1 -1
- package/dist/cli-routing.js +1 -1
- package/dist/cli-routing.js.map +1 -1
- package/dist/cli.js +36 -9
- package/dist/cli.js.map +1 -1
- package/dist/compose.js +5 -0
- package/dist/compose.js.map +1 -1
- package/dist/converge-checkpoint.d.ts +66 -0
- package/dist/converge-checkpoint.d.ts.map +1 -0
- package/dist/converge-checkpoint.js +61 -0
- package/dist/converge-checkpoint.js.map +1 -0
- package/dist/converge-job-gcp.d.ts +46 -0
- package/dist/converge-job-gcp.d.ts.map +1 -0
- package/dist/converge-job-gcp.js +368 -0
- package/dist/converge-job-gcp.js.map +1 -0
- package/dist/converge-job-setup.d.ts +62 -0
- package/dist/converge-job-setup.d.ts.map +1 -0
- package/dist/converge-job-setup.js +228 -0
- package/dist/converge-job-setup.js.map +1 -0
- package/dist/converge-job.d.ts +141 -0
- package/dist/converge-job.d.ts.map +1 -0
- package/dist/converge-job.js +325 -0
- package/dist/converge-job.js.map +1 -0
- package/dist/gcp-names.d.ts +25 -1
- package/dist/gcp-names.d.ts.map +1 -1
- package/dist/gcp-names.js +36 -0
- package/dist/gcp-names.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/infra-contract.d.ts +1 -1
- package/dist/infra-contract.d.ts.map +1 -1
- package/dist/infra-contract.js +28 -1
- package/dist/infra-contract.js.map +1 -1
- package/dist/install-job.d.ts +11 -3
- package/dist/install-job.d.ts.map +1 -1
- package/dist/install-job.js +22 -12
- package/dist/install-job.js.map +1 -1
- package/dist/lifecycle.d.ts +13 -1
- package/dist/lifecycle.d.ts.map +1 -1
- package/dist/lifecycle.js +115 -58
- package/dist/lifecycle.js.map +1 -1
- package/dist/operator-access.d.ts.map +1 -1
- package/dist/operator-access.js +13 -3
- package/dist/operator-access.js.map +1 -1
- package/dist/private-file.d.ts +26 -0
- package/dist/private-file.d.ts.map +1 -0
- package/dist/private-file.js +49 -0
- package/dist/private-file.js.map +1 -0
- package/dist/release-contract.d.ts +3 -2
- package/dist/release-contract.d.ts.map +1 -1
- package/dist/release-contract.js +8 -2
- package/dist/release-contract.js.map +1 -1
- package/dist/release-manifest.d.ts +2 -0
- package/dist/release-manifest.d.ts.map +1 -1
- package/dist/release-manifest.js.map +1 -1
- package/dist/remote-mcp-edge.d.ts +114 -0
- package/dist/remote-mcp-edge.d.ts.map +1 -0
- package/dist/remote-mcp-edge.js +116 -0
- package/dist/remote-mcp-edge.js.map +1 -0
- package/dist/remote-mcp.d.ts +117 -0
- package/dist/remote-mcp.d.ts.map +1 -0
- package/dist/remote-mcp.js +379 -0
- package/dist/remote-mcp.js.map +1 -0
- package/dist/runtime-secrets.d.ts +12 -0
- package/dist/runtime-secrets.d.ts.map +1 -1
- package/dist/runtime-secrets.js +32 -4
- package/dist/runtime-secrets.js.map +1 -1
- package/dist/seal-key-rotation.d.ts +38 -0
- package/dist/seal-key-rotation.d.ts.map +1 -0
- package/dist/seal-key-rotation.js +206 -0
- package/dist/seal-key-rotation.js.map +1 -0
- package/dist/stack-contract.d.ts +175 -3
- package/dist/stack-contract.d.ts.map +1 -1
- package/dist/stack-contract.js +183 -3
- package/dist/stack-contract.js.map +1 -1
- package/dist/stack-converge.d.ts +38 -1
- package/dist/stack-converge.d.ts.map +1 -1
- package/dist/stack-converge.js +74 -4
- package/dist/stack-converge.js.map +1 -1
- package/dist/state-lock-lease.d.ts +142 -0
- package/dist/state-lock-lease.d.ts.map +1 -0
- package/dist/state-lock-lease.js +347 -0
- package/dist/state-lock-lease.js.map +1 -0
- package/dist/state-store.d.ts +66 -9
- package/dist/state-store.d.ts.map +1 -1
- package/dist/state-store.js +249 -30
- package/dist/state-store.js.map +1 -1
- package/dist/tar-entries.d.ts.map +1 -1
- package/dist/tar-entries.js +84 -9
- package/dist/tar-entries.js.map +1 -1
- package/dist/upgrade/app-roll.d.ts +36 -7
- package/dist/upgrade/app-roll.d.ts.map +1 -1
- package/dist/upgrade/app-roll.js +88 -19
- package/dist/upgrade/app-roll.js.map +1 -1
- package/dist/upgrade/audit-job-roll.d.ts +38 -0
- package/dist/upgrade/audit-job-roll.d.ts.map +1 -0
- package/dist/upgrade/audit-job-roll.js +43 -0
- package/dist/upgrade/audit-job-roll.js.map +1 -0
- package/dist/upgrade/converge.d.ts.map +1 -1
- package/dist/upgrade/converge.js +141 -8
- package/dist/upgrade/converge.js.map +1 -1
- package/dist/upgrade/gcp-lookup.d.ts +27 -10
- package/dist/upgrade/gcp-lookup.d.ts.map +1 -1
- package/dist/upgrade/gcp-lookup.js +46 -7
- package/dist/upgrade/gcp-lookup.js.map +1 -1
- package/dist/upgrade/record-rolled.d.ts +40 -0
- package/dist/upgrade/record-rolled.d.ts.map +1 -0
- package/dist/upgrade/record-rolled.js +46 -0
- package/dist/upgrade/record-rolled.js.map +1 -0
- package/dist/upgrade/upgrade.d.ts +45 -1
- package/dist/upgrade/upgrade.d.ts.map +1 -1
- package/dist/upgrade/upgrade.js +60 -17
- package/dist/upgrade/upgrade.js.map +1 -1
- package/dist/upgrader-assess.d.ts +15 -1
- package/dist/upgrader-assess.d.ts.map +1 -1
- package/dist/upgrader-assess.js +36 -6
- package/dist/upgrader-assess.js.map +1 -1
- package/dist/upgrader-gcp.d.ts +15 -0
- package/dist/upgrader-gcp.d.ts.map +1 -1
- package/dist/upgrader-gcp.js +42 -7
- package/dist/upgrader-gcp.js.map +1 -1
- package/dist/upgrader-iam-preflight.d.ts +123 -0
- package/dist/upgrader-iam-preflight.d.ts.map +1 -0
- package/dist/upgrader-iam-preflight.js +414 -0
- package/dist/upgrader-iam-preflight.js.map +1 -0
- package/dist/upgrader-job.d.ts +33 -9
- package/dist/upgrader-job.d.ts.map +1 -1
- package/dist/upgrader-job.js +169 -27
- package/dist/upgrader-job.js.map +1 -1
- package/dist/upgrader-records.d.ts +58 -0
- package/dist/upgrader-records.d.ts.map +1 -0
- package/dist/upgrader-records.js +87 -0
- package/dist/upgrader-records.js.map +1 -0
- package/dist/upgrader-reporter.d.ts +3 -2
- package/dist/upgrader-reporter.d.ts.map +1 -1
- package/dist/upgrader-reporter.js +10 -2
- package/dist/upgrader-reporter.js.map +1 -1
- package/dist/upgrader-setup.d.ts +25 -3
- package/dist/upgrader-setup.d.ts.map +1 -1
- package/dist/upgrader-setup.js +132 -14
- package/dist/upgrader-setup.js.map +1 -1
- package/dist/wizards/gcp-cost.d.ts.map +1 -1
- package/dist/wizards/gcp-cost.js +14 -6
- package/dist/wizards/gcp-cost.js.map +1 -1
- package/dist/wizards/gcp-pulumi.d.ts +6 -0
- package/dist/wizards/gcp-pulumi.d.ts.map +1 -1
- package/dist/wizards/gcp-pulumi.js +7 -0
- package/dist/wizards/gcp-pulumi.js.map +1 -1
- package/dist/wizards/gcp.d.ts.map +1 -1
- package/dist/wizards/gcp.js +130 -4
- package/dist/wizards/gcp.js.map +1 -1
- package/package.json +4 -4
- package/src/agent/run.test.ts +15 -1
- package/src/audit-job-setup.test.ts +234 -0
- package/src/audit-job-setup.ts +360 -0
- package/src/caddy-redirects.docker.test.ts +19 -6
- package/src/caddy.test.ts +30 -0
- package/src/caddy.ts +43 -0
- package/src/cli-init-validation.test.ts +44 -0
- package/src/cli-routing.ts +1 -1
- package/src/cli.ts +40 -8
- package/src/compose.ts +5 -0
- package/src/converge-checkpoint.ts +75 -0
- package/src/converge-entrypoint.test.ts +213 -0
- package/src/converge-job-gcp.test.ts +222 -0
- package/src/converge-job-gcp.ts +443 -0
- package/src/converge-job-setup.test.ts +152 -0
- package/src/converge-job-setup.ts +268 -0
- package/src/converge-job.test.ts +486 -0
- package/src/converge-job.ts +542 -0
- package/src/gateway-env-isolation.test.ts +8 -1
- package/src/gcp-names.ts +43 -1
- package/src/gcp-url-map.test.ts +9 -1
- package/src/index.ts +8 -0
- package/src/infra-contract.test.ts +7 -4
- package/src/infra-contract.ts +28 -1
- package/src/install-job.test.ts +3 -1
- package/src/install-job.ts +31 -14
- package/src/lifecycle.ts +137 -71
- package/src/operator-access.test.ts +14 -0
- package/src/operator-access.ts +13 -3
- package/src/private-file.test.ts +80 -0
- package/src/private-file.ts +50 -0
- package/src/release-contract.ts +8 -2
- package/src/release-manifest.test.ts +42 -0
- package/src/release-manifest.ts +2 -0
- package/src/remote-mcp-edge.ts +137 -0
- package/src/remote-mcp.test.ts +517 -0
- package/src/remote-mcp.ts +460 -0
- package/src/runtime-secrets.test.ts +31 -6
- package/src/runtime-secrets.ts +41 -3
- package/src/seal-key-rotation.test.ts +131 -0
- package/src/seal-key-rotation.ts +258 -0
- package/src/stack-contract-parity.test.ts +20 -3
- package/src/stack-contract.test.ts +34 -2
- package/src/stack-contract.ts +305 -3
- package/src/stack-converge-secrets.test.ts +3 -0
- package/src/stack-converge.test.ts +21 -5
- package/src/stack-converge.ts +101 -7
- package/src/state-attach.test.ts +7 -1
- package/src/state-lock-lease.test.ts +302 -0
- package/src/state-lock-lease.ts +423 -0
- package/src/state-lock.test.ts +568 -0
- package/src/state-store.ts +347 -27
- package/src/tar-entries.test.ts +149 -7
- package/src/tar-entries.ts +90 -9
- package/src/upgrade/app-roll.ts +135 -21
- package/src/upgrade/audit-job-roll.ts +68 -0
- package/src/upgrade/converge.ts +170 -7
- package/src/upgrade/gcp-lookup.test.ts +80 -0
- package/src/upgrade/gcp-lookup.ts +65 -8
- package/src/upgrade/record-rolled.test.ts +105 -0
- package/src/upgrade/record-rolled.ts +91 -0
- package/src/upgrade/upgrade.test.ts +279 -5
- package/src/upgrade/upgrade.ts +120 -24
- package/src/upgrader-assess.test.ts +93 -3
- package/src/upgrader-assess.ts +58 -9
- package/src/upgrader-gcp.ts +47 -8
- package/src/upgrader-iam-preflight.test.ts +393 -0
- package/src/upgrader-iam-preflight.ts +510 -0
- package/src/upgrader-job.test.ts +587 -15
- package/src/upgrader-job.ts +237 -53
- package/src/upgrader-records.ts +151 -0
- package/src/upgrader-reporter.test.ts +6 -0
- package/src/upgrader-reporter.ts +7 -2
- package/src/upgrader-setup.test.ts +4 -3
- package/src/upgrader-setup.ts +137 -13
- package/src/wizards/gcp-cost.ts +15 -6
- package/src/wizards/gcp-pulumi.ts +7 -0
- package/src/wizards/gcp.ts +162 -4
- package/stacks/aws/index.ts +18 -0
- package/stacks/azure/index.ts +18 -0
- package/stacks/gcp/index.ts +59 -7
- package/stacks/gcp-firebase/index.ts +50 -6
package/src/state-store.ts
CHANGED
|
@@ -22,12 +22,24 @@
|
|
|
22
22
|
*/
|
|
23
23
|
|
|
24
24
|
import { spawnSync } from "node:child_process";
|
|
25
|
+
import { randomUUID } from "node:crypto";
|
|
25
26
|
import { existsSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
26
|
-
import { hostname } from "node:os";
|
|
27
27
|
import { join } from "node:path";
|
|
28
28
|
import { assertNoCheckpointSecrets, assertNoSecrets } from "./checkpoint-secrets.js";
|
|
29
29
|
import { type GcloudResult, gcloudEnv } from "./gcloud.js";
|
|
30
30
|
import { OPS_INSTALL_PREFIX } from "./state-backend.js";
|
|
31
|
+
import {
|
|
32
|
+
describeExecutionArgs,
|
|
33
|
+
type ExecutionLockHolder,
|
|
34
|
+
judgeHeldLock,
|
|
35
|
+
type LocalHost,
|
|
36
|
+
type LockHolder,
|
|
37
|
+
lockBody,
|
|
38
|
+
PROCESS_LEASE_MS,
|
|
39
|
+
parseExecutionState,
|
|
40
|
+
parseLockBody,
|
|
41
|
+
renewedLockBody,
|
|
42
|
+
} from "./state-lock-lease.js";
|
|
31
43
|
|
|
32
44
|
/** The two documents a store holds. */
|
|
33
45
|
export type StateFile = "install.json" | "progress.json";
|
|
@@ -176,58 +188,324 @@ export function bucketStateLockUrl(bucket: string): string {
|
|
|
176
188
|
return `gs://${bucket}/${OPS_INSTALL_PREFIX}/state.lock`;
|
|
177
189
|
}
|
|
178
190
|
|
|
179
|
-
/**
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
191
|
+
/** gcloud's wording for a failed `--if-generation-match` precondition. */
|
|
192
|
+
const PRECONDITION_FAILED = /PreconditionFailed|412|At least one of the pre-conditions/i;
|
|
193
|
+
|
|
194
|
+
/** How a bucket lock is taken: who holds it, and the clock to judge leases by. */
|
|
195
|
+
export interface BucketLockOptions {
|
|
196
|
+
/**
|
|
197
|
+
* The Cloud Run execution taking the lock (#815): the lock's lease is the
|
|
198
|
+
* execution's task timeout, and a later run may break it once Cloud Run
|
|
199
|
+
* says this execution has finished. Without it the holder is this
|
|
200
|
+
* process (a CLI run): the lock names host + pid, carries a lease of
|
|
201
|
+
* {@link PROCESS_LEASE_MS} and is renewed by a heartbeat until released.
|
|
202
|
+
*/
|
|
203
|
+
readonly holder?: ExecutionLockHolder;
|
|
204
|
+
/** Told when a dead holder's lock was broken, with why. */
|
|
205
|
+
readonly onBreak?: (message: string) => void;
|
|
206
|
+
/**
|
|
207
|
+
* Told when the heartbeat found the lock no longer this run's (another
|
|
208
|
+
* run broke it after the lease ran out, e.g. while this machine slept).
|
|
209
|
+
*/
|
|
210
|
+
readonly onLost?: (message: string) => void;
|
|
211
|
+
readonly now?: () => Date;
|
|
212
|
+
/** Starts the heartbeat; returns its stop. Defaults to an unref'd setInterval. */
|
|
213
|
+
readonly schedule?: (tick: () => void, everyMs: number) => () => void;
|
|
214
|
+
/** This host and its pid check (state-lock-lease.ts LocalHost); tests pass a fake. */
|
|
215
|
+
readonly here?: LocalHost;
|
|
187
216
|
}
|
|
188
217
|
|
|
218
|
+
const defaultSchedule = (tick: () => void, everyMs: number): (() => void) => {
|
|
219
|
+
const timer = setInterval(tick, everyMs);
|
|
220
|
+
// The heartbeat must never keep a finished CLI alive.
|
|
221
|
+
timer.unref?.();
|
|
222
|
+
return () => clearInterval(timer);
|
|
223
|
+
};
|
|
224
|
+
|
|
189
225
|
/**
|
|
190
226
|
* The install's state lock in its ops bucket, created only if absent
|
|
191
227
|
* (`--if-generation-match=0`). Held by every writer of a bucket-backed
|
|
192
|
-
* install's state — an installer run's `pulumi up`, `state migrate
|
|
193
|
-
*
|
|
194
|
-
*
|
|
195
|
-
* automatically.
|
|
228
|
+
* install's state — an installer run's `pulumi up`, `state migrate`, the
|
|
229
|
+
* CLI `upgrade`, the upgrader job — so none writes a backend another is
|
|
230
|
+
* switching away from.
|
|
196
231
|
*
|
|
197
|
-
*
|
|
232
|
+
* A lock that is held is judged (state-lock-lease.ts): one whose Cloud Run
|
|
233
|
+
* execution has finished, whose execution is gone after its lease, whose
|
|
234
|
+
* CLI process is gone on this host, or whose CLI lease ran out unrenewed,
|
|
235
|
+
* is broken with a generation-matched delete and taken once more. The
|
|
236
|
+
* generation is read BEFORE the body, so the delete can only remove the
|
|
237
|
+
* lock that was judged: if another run replaced it in between, the
|
|
238
|
+
* delete's precondition fails and nothing is removed. Two runs breaking the
|
|
239
|
+
* same dead lock cannot both win either — the slower one's create fails
|
|
240
|
+
* its own precondition and reports the lock as held.
|
|
241
|
+
*
|
|
242
|
+
* A CLI holder renews its lease every third of {@link PROCESS_LEASE_MS}
|
|
243
|
+
* with a generation-matched rewrite, so a run that is alive keeps its lock
|
|
244
|
+
* however long it takes, and a killed one loses it within one lease.
|
|
245
|
+
*
|
|
246
|
+
* @returns the release function; it stops the heartbeat and deletes only
|
|
247
|
+
* this lock's generation
|
|
248
|
+
* @throws StateLockHeldError when another run holds the lock
|
|
249
|
+
* @throws Error when the lock or its holder cannot be read (no access)
|
|
198
250
|
*/
|
|
199
251
|
export function acquireBucketStateLock(
|
|
200
252
|
bucket: string,
|
|
201
253
|
purpose: string,
|
|
202
254
|
run: SyncGcloud = realSyncGcloud,
|
|
255
|
+
opts: BucketLockOptions = {},
|
|
203
256
|
): () => void {
|
|
204
257
|
const url = bucketStateLockUrl(bucket);
|
|
205
|
-
const
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
258
|
+
const now = opts.now ?? (() => new Date());
|
|
259
|
+
const holder: LockHolder = opts.holder ?? { leaseMs: PROCESS_LEASE_MS };
|
|
260
|
+
const id = randomUUID();
|
|
261
|
+
const create = () =>
|
|
262
|
+
run(
|
|
263
|
+
["storage", "cp", "-", url, "--if-generation-match=0"],
|
|
264
|
+
lockBody(purpose, now(), holder, id),
|
|
265
|
+
);
|
|
266
|
+
let r = create();
|
|
267
|
+
if (!r.ok && PRECONDITION_FAILED.test(r.stderr)) {
|
|
268
|
+
breakDeadLock(url, run, now(), opts.onBreak, opts.here);
|
|
269
|
+
r = create();
|
|
270
|
+
if (!r.ok && PRECONDITION_FAILED.test(r.stderr)) {
|
|
209
271
|
throw new StateLockHeldError(
|
|
210
|
-
`Another run
|
|
272
|
+
`Another run took this install's state lock at the same moment (${readLockText(url, run)}). Re-run once it has finished.`,
|
|
211
273
|
);
|
|
212
274
|
}
|
|
213
|
-
throw new Error(`Could not take the state lock ${url}: ${r.stderr.trim()}`);
|
|
214
275
|
}
|
|
276
|
+
if (!r.ok) throw new Error(`Could not take the state lock ${url}: ${r.stderr.trim()}`);
|
|
277
|
+
const owned = ownedGeneration(url, run, id);
|
|
278
|
+
if (owned.kind !== "owned") {
|
|
279
|
+
// Ours, just created — but without its generation a delete could hit a
|
|
280
|
+
// successor's lock. Give it back only once a second look proves it is
|
|
281
|
+
// still ours at a known generation; otherwise its lease ends it.
|
|
282
|
+
const retry = ownedGeneration(url, run, id);
|
|
283
|
+
if (retry.kind === "owned") {
|
|
284
|
+
run(["storage", "rm", url, `--if-generation-match=${retry.generation}`]);
|
|
285
|
+
}
|
|
286
|
+
throw new Error(
|
|
287
|
+
`Could not read the generation of the state lock ${url} this run just took (${owned.why}). ${
|
|
288
|
+
retry.kind === "owned"
|
|
289
|
+
? "It was given back."
|
|
290
|
+
: "It was left in place and expires with its lease; re-run after that."
|
|
291
|
+
}`,
|
|
292
|
+
);
|
|
293
|
+
}
|
|
294
|
+
let generation = owned.generation;
|
|
295
|
+
let lost: string | null = null;
|
|
296
|
+
const stopHeartbeat =
|
|
297
|
+
"execution" in holder
|
|
298
|
+
? () => {}
|
|
299
|
+
: (opts.schedule ?? defaultSchedule)(
|
|
300
|
+
() => {
|
|
301
|
+
if (lost !== null) return;
|
|
302
|
+
const renewed = renewLock(url, run, id, generation, now(), holder.leaseMs);
|
|
303
|
+
if (renewed.kind === "owned") {
|
|
304
|
+
generation = renewed.generation;
|
|
305
|
+
return;
|
|
306
|
+
}
|
|
307
|
+
if (renewed.kind === "lost") {
|
|
308
|
+
lost = renewed.why;
|
|
309
|
+
opts.onLost?.(
|
|
310
|
+
`This run no longer holds the install's state lock (${renewed.why}). Stop it and re-run once the other run has finished.`,
|
|
311
|
+
);
|
|
312
|
+
}
|
|
313
|
+
// "error": a transient failure; the next beat tries again within the lease.
|
|
314
|
+
},
|
|
315
|
+
Math.floor(holder.leaseMs / 3),
|
|
316
|
+
);
|
|
215
317
|
return () => {
|
|
216
|
-
|
|
217
|
-
|
|
318
|
+
stopHeartbeat();
|
|
319
|
+
const rm = run(["storage", "rm", url, `--if-generation-match=${generation}`]);
|
|
320
|
+
if (rm.ok) return;
|
|
321
|
+
if (PRECONDITION_FAILED.test(rm.stderr) || NOT_FOUND.test(rm.stderr)) {
|
|
322
|
+
throw new Error(
|
|
323
|
+
`The state lock ${url} was no longer this run's when it finished (another run broke it); it was left alone.`,
|
|
324
|
+
);
|
|
325
|
+
}
|
|
326
|
+
throw new Error(`Could not release ${url}; remove it: gcloud storage rm ${url}`);
|
|
218
327
|
};
|
|
219
328
|
}
|
|
220
329
|
|
|
330
|
+
type Ownership =
|
|
331
|
+
| { readonly kind: "owned"; readonly generation: string }
|
|
332
|
+
| { readonly kind: "lost"; readonly why: string }
|
|
333
|
+
| { readonly kind: "error"; readonly why: string };
|
|
334
|
+
|
|
335
|
+
/**
|
|
336
|
+
* The lock's generation, provided the lock is still the one with `id`. The
|
|
337
|
+
* generation is read before the body: if another run replaced the lock in
|
|
338
|
+
* between, the body names its id and this returns "lost" — never a
|
|
339
|
+
* generation that is not ours.
|
|
340
|
+
*/
|
|
341
|
+
function ownedGeneration(url: string, run: SyncGcloud, id: string): Ownership {
|
|
342
|
+
const g = lockGeneration(url, run);
|
|
343
|
+
if (g.kind === "absent") return { kind: "lost", why: "the lock is gone" };
|
|
344
|
+
if (g.kind === "error") return { kind: "error", why: g.error };
|
|
345
|
+
const body = run(["storage", "cat", url]);
|
|
346
|
+
if (!body.ok) {
|
|
347
|
+
return NOT_FOUND.test(body.stderr)
|
|
348
|
+
? { kind: "lost", why: "the lock is gone" }
|
|
349
|
+
: { kind: "error", why: body.stderr.trim() || `gcloud exited ${body.exitCode}` };
|
|
350
|
+
}
|
|
351
|
+
const parsed = parseLockBody(body.stdout);
|
|
352
|
+
if (parsed?.id !== id) {
|
|
353
|
+
return { kind: "lost", why: `another run holds it now (${body.stdout.trim().slice(0, 200)})` };
|
|
354
|
+
}
|
|
355
|
+
return { kind: "owned", generation: g.generation };
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
/**
|
|
359
|
+
* One heartbeat: rewrite the lock with a later lease, only if it is still
|
|
360
|
+
* at `generation` (ours), and return its new generation.
|
|
361
|
+
*/
|
|
362
|
+
function renewLock(
|
|
363
|
+
url: string,
|
|
364
|
+
run: SyncGcloud,
|
|
365
|
+
id: string,
|
|
366
|
+
generation: string,
|
|
367
|
+
now: Date,
|
|
368
|
+
leaseMs: number,
|
|
369
|
+
): Ownership {
|
|
370
|
+
const current = run(["storage", "cat", url]);
|
|
371
|
+
if (!current.ok) {
|
|
372
|
+
return NOT_FOUND.test(current.stderr)
|
|
373
|
+
? { kind: "lost", why: "the lock is gone" }
|
|
374
|
+
: { kind: "error", why: current.stderr.trim() };
|
|
375
|
+
}
|
|
376
|
+
const body = parseLockBody(current.stdout);
|
|
377
|
+
if (body?.id !== id) {
|
|
378
|
+
return {
|
|
379
|
+
kind: "lost",
|
|
380
|
+
why: `another run holds it now (${current.stdout.trim().slice(0, 200)})`,
|
|
381
|
+
};
|
|
382
|
+
}
|
|
383
|
+
const w = run(
|
|
384
|
+
["storage", "cp", "-", url, `--if-generation-match=${generation}`],
|
|
385
|
+
renewedLockBody(body, now, leaseMs),
|
|
386
|
+
);
|
|
387
|
+
if (!w.ok) {
|
|
388
|
+
return PRECONDITION_FAILED.test(w.stderr)
|
|
389
|
+
? { kind: "lost", why: "another run replaced it" }
|
|
390
|
+
: { kind: "error", why: w.stderr.trim() };
|
|
391
|
+
}
|
|
392
|
+
return ownedGeneration(url, run, id);
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
type LockGeneration =
|
|
396
|
+
| { readonly kind: "found"; readonly generation: string }
|
|
397
|
+
| { readonly kind: "absent" }
|
|
398
|
+
| { readonly kind: "error"; readonly error: string };
|
|
399
|
+
|
|
400
|
+
/** The lock object's current generation. */
|
|
401
|
+
function lockGeneration(url: string, run: SyncGcloud): LockGeneration {
|
|
402
|
+
const d = run(["storage", "objects", "describe", url, "--format=json"]);
|
|
403
|
+
if (!d.ok) {
|
|
404
|
+
return NOT_FOUND.test(d.stderr)
|
|
405
|
+
? { kind: "absent" }
|
|
406
|
+
: { kind: "error", error: d.stderr.trim() || `gcloud exited ${d.exitCode}` };
|
|
407
|
+
}
|
|
408
|
+
try {
|
|
409
|
+
// `gcloud storage objects describe` prints the generation as a string
|
|
410
|
+
// (`generation: '1706304784726341'`, cloud.google.com/storage/docs/
|
|
411
|
+
// viewing-editing-metadata); a JSON number would lose digits past 2^53,
|
|
412
|
+
// so only a string of digits is a generation the delete may match.
|
|
413
|
+
const g = (JSON.parse(d.stdout) as { generation?: unknown }).generation;
|
|
414
|
+
if (typeof g === "string" && /^[1-9][0-9]*$/.test(g)) {
|
|
415
|
+
return { kind: "found", generation: g };
|
|
416
|
+
}
|
|
417
|
+
} catch {
|
|
418
|
+
// Reported below with the same message as a missing field.
|
|
419
|
+
}
|
|
420
|
+
return { kind: "error", error: "gcloud described the lock without a generation" };
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
function readLockText(url: string, run: SyncGcloud): string {
|
|
424
|
+
const held = run(["storage", "cat", url]);
|
|
425
|
+
return held.ok ? held.stdout.trim() : url;
|
|
426
|
+
}
|
|
427
|
+
|
|
428
|
+
/**
|
|
429
|
+
* Judge the held lock at `url` and break it when its holder is dead.
|
|
430
|
+
* Returns normally when the lock was broken or vanished meanwhile, so the
|
|
431
|
+
* caller creates it once more; throws when it is live or cannot be judged.
|
|
432
|
+
*/
|
|
433
|
+
function breakDeadLock(
|
|
434
|
+
url: string,
|
|
435
|
+
run: SyncGcloud,
|
|
436
|
+
now: Date,
|
|
437
|
+
onBreak: ((message: string) => void) | undefined,
|
|
438
|
+
here: LocalHost | undefined,
|
|
439
|
+
): void {
|
|
440
|
+
const found = lockGeneration(url, run);
|
|
441
|
+
if (found.kind === "absent") return;
|
|
442
|
+
if (found.kind === "error") {
|
|
443
|
+
throw new Error(`Could not read the state lock ${url} another run holds: ${found.error}`);
|
|
444
|
+
}
|
|
445
|
+
const held = run(["storage", "cat", url]);
|
|
446
|
+
if (!held.ok) {
|
|
447
|
+
if (NOT_FOUND.test(held.stderr)) return;
|
|
448
|
+
throw new Error(`Could not read the state lock ${url}: ${held.stderr.trim()}`);
|
|
449
|
+
}
|
|
450
|
+
const verdict = judgeHeldLock(
|
|
451
|
+
held.stdout,
|
|
452
|
+
(ref) => parseExecutionState(run(describeExecutionArgs(ref))),
|
|
453
|
+
now,
|
|
454
|
+
here,
|
|
455
|
+
);
|
|
456
|
+
if (verdict.kind === "unknown") {
|
|
457
|
+
// Not contention: retrying will not help, and guessing "dead" could let
|
|
458
|
+
// two runs write the same state.
|
|
459
|
+
throw new Error(`The state lock ${url} is held and ${verdict.why}`);
|
|
460
|
+
}
|
|
461
|
+
if (verdict.kind === "held") {
|
|
462
|
+
throw new StateLockHeldError(
|
|
463
|
+
verdict.manualRemoval
|
|
464
|
+
? `Another run is changing this install's state (${verdict.why}). Re-run once it has finished; if none is running, remove the stale lock: gcloud storage rm ${url}`
|
|
465
|
+
: `Another run is changing this install's state: ${verdict.why}. Re-run once it has finished.`,
|
|
466
|
+
);
|
|
467
|
+
}
|
|
468
|
+
const rm = run(["storage", "rm", url, `--if-generation-match=${found.generation}`]);
|
|
469
|
+
if (!rm.ok && !NOT_FOUND.test(rm.stderr)) {
|
|
470
|
+
if (PRECONDITION_FAILED.test(rm.stderr)) {
|
|
471
|
+
throw new StateLockHeldError(
|
|
472
|
+
`Another run took this install's state lock while this one removed a stale one (${readLockText(url, run)}). Re-run once it has finished.`,
|
|
473
|
+
);
|
|
474
|
+
}
|
|
475
|
+
throw new Error(`Could not remove the stale state lock ${url}: ${rm.stderr.trim()}`);
|
|
476
|
+
}
|
|
477
|
+
onBreak?.(`Removed a stale state lock: ${verdict.why}.`);
|
|
478
|
+
}
|
|
479
|
+
|
|
221
480
|
/**
|
|
222
481
|
* The same lock for an install whose state is on this machine:
|
|
223
|
-
* `~/.caelo-<id>/state.lock`, created exclusively.
|
|
482
|
+
* `~/.caelo-<id>/state.lock`, created exclusively. A lock left by a
|
|
483
|
+
* process that is gone on this machine is taken over (#815).
|
|
224
484
|
*
|
|
485
|
+
* @param here this host and its pid check; tests pass a fake
|
|
225
486
|
* @returns the release function
|
|
226
487
|
*/
|
|
227
|
-
export function acquireLocalStateLock(root: string, purpose: string): () => void {
|
|
488
|
+
export function acquireLocalStateLock(root: string, purpose: string, here?: LocalHost): () => void {
|
|
228
489
|
const path = join(root, "state.lock");
|
|
490
|
+
const take = () =>
|
|
491
|
+
writeFileSync(path, lockBody(purpose, new Date()), { flag: "wx", mode: 0o600 });
|
|
229
492
|
try {
|
|
230
|
-
|
|
493
|
+
try {
|
|
494
|
+
take();
|
|
495
|
+
} catch (e) {
|
|
496
|
+
if ((e as NodeJS.ErrnoException).code !== "EEXIST") throw e;
|
|
497
|
+
// #815 — the file is on this machine, so a holder whose process has
|
|
498
|
+
// ended is known dead: take its lock over instead of asking for `rm`.
|
|
499
|
+
const verdict = judgeHeldLock(
|
|
500
|
+
readFileSync(path, "utf8"),
|
|
501
|
+
() => ({ kind: "unknown", error: "a machine-local lock names no Cloud Run execution" }),
|
|
502
|
+
new Date(),
|
|
503
|
+
here,
|
|
504
|
+
);
|
|
505
|
+
if (verdict.kind !== "break") throw e;
|
|
506
|
+
rmSync(path, { force: true });
|
|
507
|
+
take();
|
|
508
|
+
}
|
|
231
509
|
} catch (e) {
|
|
232
510
|
if ((e as NodeJS.ErrnoException).code === "EEXIST") {
|
|
233
511
|
throw new StateLockHeldError(
|
|
@@ -245,8 +523,50 @@ export function acquireStateLock(
|
|
|
245
523
|
root: string,
|
|
246
524
|
purpose: string,
|
|
247
525
|
run: SyncGcloud = realSyncGcloud,
|
|
526
|
+
opts: BucketLockOptions = {},
|
|
248
527
|
): () => void {
|
|
249
528
|
return backend.kind === "gcs"
|
|
250
|
-
? acquireBucketStateLock(backend.bucket, purpose, run)
|
|
251
|
-
: acquireLocalStateLock(root, purpose);
|
|
529
|
+
? acquireBucketStateLock(backend.bucket, purpose, run, opts)
|
|
530
|
+
: acquireLocalStateLock(root, purpose, opts.here);
|
|
531
|
+
}
|
|
532
|
+
|
|
533
|
+
/** The part of `process` {@link releaseOnSignal} needs; tests pass a fake. */
|
|
534
|
+
export interface SignalTarget {
|
|
535
|
+
once(signal: "SIGINT" | "SIGTERM", listener: () => void): unknown;
|
|
536
|
+
removeListener(signal: "SIGINT" | "SIGTERM", listener: () => void): unknown;
|
|
537
|
+
exit(code: number): never;
|
|
538
|
+
}
|
|
539
|
+
|
|
540
|
+
/**
|
|
541
|
+
* Give the state lock back when the CLI is stopped (Ctrl-C, a closed Cloud
|
|
542
|
+
* Shell tab's SIGTERM), then exit with the signal's conventional code
|
|
543
|
+
* (#815). Without this a stopped run leaves its lock until its lease ends.
|
|
544
|
+
*
|
|
545
|
+
* @param report told what happened to the lock
|
|
546
|
+
* @returns removes the handlers; call it once the run released the lock itself
|
|
547
|
+
*/
|
|
548
|
+
export function releaseOnSignal(
|
|
549
|
+
release: () => void,
|
|
550
|
+
report: (message: string) => void,
|
|
551
|
+
proc: SignalTarget = process,
|
|
552
|
+
): () => void {
|
|
553
|
+
const dispose = () => {
|
|
554
|
+
proc.removeListener("SIGINT", onInt);
|
|
555
|
+
proc.removeListener("SIGTERM", onTerm);
|
|
556
|
+
};
|
|
557
|
+
const stop = (signal: "SIGINT" | "SIGTERM", code: number): never => {
|
|
558
|
+
dispose();
|
|
559
|
+
try {
|
|
560
|
+
release();
|
|
561
|
+
report(`Stopped (${signal}); the install's state lock was released.`);
|
|
562
|
+
} catch (e) {
|
|
563
|
+
report(`Stopped (${signal}); ${e instanceof Error ? e.message : String(e)}`);
|
|
564
|
+
}
|
|
565
|
+
return proc.exit(code);
|
|
566
|
+
};
|
|
567
|
+
const onInt = () => stop("SIGINT", 130);
|
|
568
|
+
const onTerm = () => stop("SIGTERM", 143);
|
|
569
|
+
proc.once("SIGINT", onInt);
|
|
570
|
+
proc.once("SIGTERM", onTerm);
|
|
571
|
+
return dispose;
|
|
252
572
|
}
|
package/src/tar-entries.test.ts
CHANGED
|
@@ -35,18 +35,160 @@ describe("listTarEntries", () => {
|
|
|
35
35
|
});
|
|
36
36
|
|
|
37
37
|
it("refuses a truncated archive instead of skipping members", () => {
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
38
|
+
expect(() => listTarEntries(header("a.txt", "0", { sizeField: 512 }))).toThrow(/truncated/);
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
it("rejects PAX global extended headers ('g')", () => {
|
|
42
|
+
const g = member("PaxHeader", "g", paxLine("path=/etc/passwd"));
|
|
43
|
+
expect(() => listTarEntries(concat(g, member("safe.txt", "0"), END))).toThrow(
|
|
44
|
+
/pax global extended header/,
|
|
45
|
+
);
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
it("refuses a member named by two pax headers — GNU tar keeps the last, bsdtar drops the second", () => {
|
|
49
|
+
// Both tars extract `../../evil` here (the second header carries no
|
|
50
|
+
// path); reading the first header's path would have checked `safe.txt`.
|
|
51
|
+
const archive = concat(
|
|
52
|
+
member("PaxHeader", "x", paxLine("path=srv/caelo/safe.txt")),
|
|
53
|
+
member("PaxHeader", "x", paxLine("mtime=1")),
|
|
54
|
+
member("../../evil", "0"),
|
|
55
|
+
END,
|
|
56
|
+
);
|
|
57
|
+
expect(() => listTarEntries(archive)).toThrow(/several extension headers \(xx\)/);
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
it("refuses a GNU long name next to a pax path, and two long names", () => {
|
|
61
|
+
// bsdtar extracts the `L` name (`../../evil`), GNU tar the pax path.
|
|
62
|
+
const mixed = concat(
|
|
63
|
+
member("././@LongLink", "L", "../../evil\0"),
|
|
64
|
+
member("PaxHeader", "x", paxLine("path=srv/caelo/safe.txt")),
|
|
65
|
+
member("x", "0"),
|
|
66
|
+
END,
|
|
67
|
+
);
|
|
68
|
+
expect(() => listTarEntries(mixed)).toThrow(/several extension headers \(Lx\)/);
|
|
69
|
+
const twice = concat(
|
|
70
|
+
member("././@LongLink", "L", "../../evil\0"),
|
|
71
|
+
member("././@LongLink", "L", "safe.txt\0"),
|
|
72
|
+
member("x", "0"),
|
|
73
|
+
END,
|
|
74
|
+
);
|
|
75
|
+
expect(() => listTarEntries(twice)).toThrow(/several extension headers \(LL\)/);
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
it("still reads a GNU long name together with a long link", () => {
|
|
79
|
+
const archive = concat(
|
|
80
|
+
member("././@LongLink", "K", "target\0"),
|
|
81
|
+
member("././@LongLink", "L", `${"a".repeat(120)}\0`),
|
|
82
|
+
member("short", "2", "", { link: "ignored" }),
|
|
83
|
+
END,
|
|
84
|
+
);
|
|
85
|
+
expect(listTarEntries(archive)).toEqual([
|
|
86
|
+
{ name: "a".repeat(120), type: "symlink", linkTarget: "target" },
|
|
87
|
+
]);
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
it("refuses a header whose checksum does not match — GNU tar would skip it and read its body as headers", () => {
|
|
91
|
+
const archive = concat(
|
|
92
|
+
header("srv/caelo/safe.txt", "0", { sizeField: 512, badChecksum: true }),
|
|
93
|
+
header("../../evil", "0"),
|
|
94
|
+
END,
|
|
95
|
+
);
|
|
96
|
+
expect(() => listTarEntries(archive)).toThrow(/checksum mismatch/);
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
it("refuses a directory or symlink with a body — bsdtar reads the next block as a header", () => {
|
|
100
|
+
for (const [name, type] of [
|
|
101
|
+
["srv/caelo/link", "2"],
|
|
102
|
+
["srv/caelo/dir/", "5"],
|
|
103
|
+
] as const) {
|
|
104
|
+
const archive = concat(
|
|
105
|
+
header(name, type, { sizeField: 512, link: "a" }),
|
|
106
|
+
header("../../evil", "0"),
|
|
107
|
+
END,
|
|
108
|
+
);
|
|
109
|
+
expect(() => listTarEntries(archive)).toThrow(/with a body/);
|
|
110
|
+
}
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
it("walks a member's body by its pax size, so a member hidden behind the header size is seen", () => {
|
|
114
|
+
// tar -x reads `safe.txt` as empty (pax size=0) and then extracts the
|
|
115
|
+
// `../../evil` header that the ustar size field claims is file data.
|
|
116
|
+
const archive = concat(
|
|
117
|
+
member("PaxHeader", "x", paxLine("size=0")),
|
|
118
|
+
header("safe.txt", "0", { sizeField: 512 }),
|
|
119
|
+
member("../../evil", "0"),
|
|
120
|
+
END,
|
|
121
|
+
);
|
|
122
|
+
expect(listTarEntries(archive).map((e) => e.name)).toEqual(["safe.txt", "../../evil"]);
|
|
123
|
+
});
|
|
124
|
+
|
|
125
|
+
it("rejects GNU sparse pax records and an unreadable pax size", () => {
|
|
126
|
+
const sparse = member("PaxHeader", "x", paxLine("GNU.sparse.name=../../evil"));
|
|
127
|
+
expect(() => listTarEntries(concat(sparse, member("safe.txt", "0")))).toThrow(/GNU sparse/);
|
|
128
|
+
const badSize = member("PaxHeader", "x", paxLine("size=-1"));
|
|
129
|
+
expect(() => listTarEntries(concat(badSize, member("safe.txt", "0")))).toThrow(
|
|
130
|
+
/unreadable pax size/,
|
|
131
|
+
);
|
|
43
132
|
});
|
|
44
133
|
});
|
|
45
134
|
|
|
46
|
-
|
|
47
|
-
|
|
135
|
+
const END = new Uint8Array(1024);
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* One ustar header block with a valid checksum (unless `badChecksum`).
|
|
139
|
+
* `sizeField` is what the header claims, independent of any body after it.
|
|
140
|
+
*/
|
|
141
|
+
function header(
|
|
142
|
+
name: string,
|
|
143
|
+
typeflag: string,
|
|
144
|
+
opts: { sizeField?: number; link?: string; badChecksum?: boolean } = {},
|
|
145
|
+
): Uint8Array {
|
|
146
|
+
const enc = new TextEncoder();
|
|
147
|
+
const h = new Uint8Array(512);
|
|
148
|
+
h.set(enc.encode(name), 0);
|
|
149
|
+
h.set(enc.encode(`${(opts.sizeField ?? 0).toString(8).padStart(11, "0")}\0`), 124);
|
|
150
|
+
h[156] = typeflag.charCodeAt(0);
|
|
151
|
+
if (opts.link) h.set(enc.encode(opts.link), 157);
|
|
152
|
+
h.set(enc.encode("ustar\0"), 257);
|
|
153
|
+
h.fill(0x20, 148, 156);
|
|
154
|
+
const sum = h.reduce((n, b) => n + b, 0) + (opts.badChecksum ? 1 : 0);
|
|
155
|
+
h.set(enc.encode(`${sum.toString(8).padStart(6, "0")}\0 `), 148);
|
|
156
|
+
return h;
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/** A header plus its padded body. */
|
|
160
|
+
function member(name: string, typeflag: string, bodyText = "", opts: { link?: string } = {}) {
|
|
161
|
+
const body = new TextEncoder().encode(bodyText);
|
|
162
|
+
const out = new Uint8Array(512 + Math.ceil(body.length / 512) * 512);
|
|
163
|
+
out.set(header(name, typeflag, { sizeField: body.length, link: opts.link }), 0);
|
|
164
|
+
out.set(body, 512);
|
|
165
|
+
return out;
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/** One pax record; its leading length counts its own digits. */
|
|
169
|
+
function paxLine(kv: string): string {
|
|
170
|
+
const base = ` ${kv}\n`;
|
|
171
|
+
const len = String(base.length + 2).length + base.length;
|
|
172
|
+
return `${len}${base}`;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
function concat(...parts: Uint8Array[]): Uint8Array {
|
|
176
|
+
const out = new Uint8Array(parts.reduce((n, p) => n + p.length, 0));
|
|
177
|
+
let at = 0;
|
|
178
|
+
for (const p of parts) {
|
|
179
|
+
out.set(p, at);
|
|
180
|
+
at += p.length;
|
|
181
|
+
}
|
|
182
|
+
return out;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
describe("escapesRoot & normalizeMemberName", () => {
|
|
186
|
+
it("flags absolute, parent-relative, and dot-segment member names", () => {
|
|
48
187
|
expect(escapesRoot("/etc/passwd")).toBe(true);
|
|
49
188
|
expect(escapesRoot("srv/caelo/../../app")).toBe(true);
|
|
189
|
+
expect(escapesRoot(".//../etc/passwd")).toBe(true);
|
|
190
|
+
expect(escapesRoot("a/./b")).toBe(true);
|
|
50
191
|
expect(escapesRoot("./srv/caelo/a..b")).toBe(false);
|
|
192
|
+
expect(normalizeMemberName(".//srv//caelo/")).toBe("srv/caelo");
|
|
51
193
|
});
|
|
52
194
|
});
|