@caelo-cms/provisioning 0.10.53 → 0.10.57

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (244) hide show
  1. package/dist/agent-db.js +272 -128
  2. package/dist/audit-job-setup.d.ts +69 -0
  3. package/dist/audit-job-setup.d.ts.map +1 -0
  4. package/dist/audit-job-setup.js +317 -0
  5. package/dist/audit-job-setup.js.map +1 -0
  6. package/dist/caddy.d.ts +9 -0
  7. package/dist/caddy.d.ts.map +1 -1
  8. package/dist/caddy.js +38 -0
  9. package/dist/caddy.js.map +1 -1
  10. package/dist/cli-routing.js +1 -1
  11. package/dist/cli-routing.js.map +1 -1
  12. package/dist/cli.js +36 -9
  13. package/dist/cli.js.map +1 -1
  14. package/dist/compose.js +5 -0
  15. package/dist/compose.js.map +1 -1
  16. package/dist/converge-checkpoint.d.ts +66 -0
  17. package/dist/converge-checkpoint.d.ts.map +1 -0
  18. package/dist/converge-checkpoint.js +61 -0
  19. package/dist/converge-checkpoint.js.map +1 -0
  20. package/dist/converge-job-gcp.d.ts +46 -0
  21. package/dist/converge-job-gcp.d.ts.map +1 -0
  22. package/dist/converge-job-gcp.js +368 -0
  23. package/dist/converge-job-gcp.js.map +1 -0
  24. package/dist/converge-job-setup.d.ts +62 -0
  25. package/dist/converge-job-setup.d.ts.map +1 -0
  26. package/dist/converge-job-setup.js +228 -0
  27. package/dist/converge-job-setup.js.map +1 -0
  28. package/dist/converge-job.d.ts +141 -0
  29. package/dist/converge-job.d.ts.map +1 -0
  30. package/dist/converge-job.js +325 -0
  31. package/dist/converge-job.js.map +1 -0
  32. package/dist/gcp-names.d.ts +25 -1
  33. package/dist/gcp-names.d.ts.map +1 -1
  34. package/dist/gcp-names.js +36 -0
  35. package/dist/gcp-names.js.map +1 -1
  36. package/dist/index.d.ts +2 -0
  37. package/dist/index.d.ts.map +1 -1
  38. package/dist/index.js +2 -0
  39. package/dist/index.js.map +1 -1
  40. package/dist/infra-contract.d.ts +1 -1
  41. package/dist/infra-contract.d.ts.map +1 -1
  42. package/dist/infra-contract.js +28 -1
  43. package/dist/infra-contract.js.map +1 -1
  44. package/dist/install-job.d.ts +11 -3
  45. package/dist/install-job.d.ts.map +1 -1
  46. package/dist/install-job.js +22 -12
  47. package/dist/install-job.js.map +1 -1
  48. package/dist/lifecycle.d.ts +13 -1
  49. package/dist/lifecycle.d.ts.map +1 -1
  50. package/dist/lifecycle.js +115 -58
  51. package/dist/lifecycle.js.map +1 -1
  52. package/dist/operator-access.d.ts.map +1 -1
  53. package/dist/operator-access.js +13 -3
  54. package/dist/operator-access.js.map +1 -1
  55. package/dist/private-file.d.ts +26 -0
  56. package/dist/private-file.d.ts.map +1 -0
  57. package/dist/private-file.js +49 -0
  58. package/dist/private-file.js.map +1 -0
  59. package/dist/release-contract.d.ts +3 -2
  60. package/dist/release-contract.d.ts.map +1 -1
  61. package/dist/release-contract.js +8 -2
  62. package/dist/release-contract.js.map +1 -1
  63. package/dist/release-manifest.d.ts +2 -0
  64. package/dist/release-manifest.d.ts.map +1 -1
  65. package/dist/release-manifest.js.map +1 -1
  66. package/dist/remote-mcp-edge.d.ts +114 -0
  67. package/dist/remote-mcp-edge.d.ts.map +1 -0
  68. package/dist/remote-mcp-edge.js +116 -0
  69. package/dist/remote-mcp-edge.js.map +1 -0
  70. package/dist/remote-mcp.d.ts +117 -0
  71. package/dist/remote-mcp.d.ts.map +1 -0
  72. package/dist/remote-mcp.js +379 -0
  73. package/dist/remote-mcp.js.map +1 -0
  74. package/dist/runtime-secrets.d.ts +12 -0
  75. package/dist/runtime-secrets.d.ts.map +1 -1
  76. package/dist/runtime-secrets.js +32 -4
  77. package/dist/runtime-secrets.js.map +1 -1
  78. package/dist/seal-key-rotation.d.ts +38 -0
  79. package/dist/seal-key-rotation.d.ts.map +1 -0
  80. package/dist/seal-key-rotation.js +206 -0
  81. package/dist/seal-key-rotation.js.map +1 -0
  82. package/dist/stack-contract.d.ts +175 -3
  83. package/dist/stack-contract.d.ts.map +1 -1
  84. package/dist/stack-contract.js +183 -3
  85. package/dist/stack-contract.js.map +1 -1
  86. package/dist/stack-converge.d.ts +38 -1
  87. package/dist/stack-converge.d.ts.map +1 -1
  88. package/dist/stack-converge.js +74 -4
  89. package/dist/stack-converge.js.map +1 -1
  90. package/dist/state-lock-lease.d.ts +142 -0
  91. package/dist/state-lock-lease.d.ts.map +1 -0
  92. package/dist/state-lock-lease.js +347 -0
  93. package/dist/state-lock-lease.js.map +1 -0
  94. package/dist/state-store.d.ts +66 -9
  95. package/dist/state-store.d.ts.map +1 -1
  96. package/dist/state-store.js +249 -30
  97. package/dist/state-store.js.map +1 -1
  98. package/dist/tar-entries.d.ts.map +1 -1
  99. package/dist/tar-entries.js +84 -9
  100. package/dist/tar-entries.js.map +1 -1
  101. package/dist/upgrade/app-roll.d.ts +36 -7
  102. package/dist/upgrade/app-roll.d.ts.map +1 -1
  103. package/dist/upgrade/app-roll.js +88 -19
  104. package/dist/upgrade/app-roll.js.map +1 -1
  105. package/dist/upgrade/audit-job-roll.d.ts +38 -0
  106. package/dist/upgrade/audit-job-roll.d.ts.map +1 -0
  107. package/dist/upgrade/audit-job-roll.js +43 -0
  108. package/dist/upgrade/audit-job-roll.js.map +1 -0
  109. package/dist/upgrade/converge.d.ts.map +1 -1
  110. package/dist/upgrade/converge.js +141 -8
  111. package/dist/upgrade/converge.js.map +1 -1
  112. package/dist/upgrade/gcp-lookup.d.ts +27 -10
  113. package/dist/upgrade/gcp-lookup.d.ts.map +1 -1
  114. package/dist/upgrade/gcp-lookup.js +46 -7
  115. package/dist/upgrade/gcp-lookup.js.map +1 -1
  116. package/dist/upgrade/record-rolled.d.ts +40 -0
  117. package/dist/upgrade/record-rolled.d.ts.map +1 -0
  118. package/dist/upgrade/record-rolled.js +46 -0
  119. package/dist/upgrade/record-rolled.js.map +1 -0
  120. package/dist/upgrade/upgrade.d.ts +45 -1
  121. package/dist/upgrade/upgrade.d.ts.map +1 -1
  122. package/dist/upgrade/upgrade.js +60 -17
  123. package/dist/upgrade/upgrade.js.map +1 -1
  124. package/dist/upgrader-assess.d.ts +15 -1
  125. package/dist/upgrader-assess.d.ts.map +1 -1
  126. package/dist/upgrader-assess.js +36 -6
  127. package/dist/upgrader-assess.js.map +1 -1
  128. package/dist/upgrader-gcp.d.ts +15 -0
  129. package/dist/upgrader-gcp.d.ts.map +1 -1
  130. package/dist/upgrader-gcp.js +42 -7
  131. package/dist/upgrader-gcp.js.map +1 -1
  132. package/dist/upgrader-iam-preflight.d.ts +123 -0
  133. package/dist/upgrader-iam-preflight.d.ts.map +1 -0
  134. package/dist/upgrader-iam-preflight.js +414 -0
  135. package/dist/upgrader-iam-preflight.js.map +1 -0
  136. package/dist/upgrader-job.d.ts +33 -9
  137. package/dist/upgrader-job.d.ts.map +1 -1
  138. package/dist/upgrader-job.js +169 -27
  139. package/dist/upgrader-job.js.map +1 -1
  140. package/dist/upgrader-records.d.ts +58 -0
  141. package/dist/upgrader-records.d.ts.map +1 -0
  142. package/dist/upgrader-records.js +87 -0
  143. package/dist/upgrader-records.js.map +1 -0
  144. package/dist/upgrader-reporter.d.ts +3 -2
  145. package/dist/upgrader-reporter.d.ts.map +1 -1
  146. package/dist/upgrader-reporter.js +10 -2
  147. package/dist/upgrader-reporter.js.map +1 -1
  148. package/dist/upgrader-setup.d.ts +25 -3
  149. package/dist/upgrader-setup.d.ts.map +1 -1
  150. package/dist/upgrader-setup.js +132 -14
  151. package/dist/upgrader-setup.js.map +1 -1
  152. package/dist/wizards/gcp-cost.d.ts.map +1 -1
  153. package/dist/wizards/gcp-cost.js +14 -6
  154. package/dist/wizards/gcp-cost.js.map +1 -1
  155. package/dist/wizards/gcp-pulumi.d.ts +6 -0
  156. package/dist/wizards/gcp-pulumi.d.ts.map +1 -1
  157. package/dist/wizards/gcp-pulumi.js +7 -0
  158. package/dist/wizards/gcp-pulumi.js.map +1 -1
  159. package/dist/wizards/gcp.d.ts.map +1 -1
  160. package/dist/wizards/gcp.js +130 -4
  161. package/dist/wizards/gcp.js.map +1 -1
  162. package/package.json +4 -4
  163. package/src/agent/run.test.ts +15 -1
  164. package/src/audit-job-setup.test.ts +234 -0
  165. package/src/audit-job-setup.ts +360 -0
  166. package/src/caddy-redirects.docker.test.ts +19 -6
  167. package/src/caddy.test.ts +30 -0
  168. package/src/caddy.ts +43 -0
  169. package/src/cli-init-validation.test.ts +44 -0
  170. package/src/cli-routing.ts +1 -1
  171. package/src/cli.ts +40 -8
  172. package/src/compose.ts +5 -0
  173. package/src/converge-checkpoint.ts +75 -0
  174. package/src/converge-entrypoint.test.ts +213 -0
  175. package/src/converge-job-gcp.test.ts +222 -0
  176. package/src/converge-job-gcp.ts +443 -0
  177. package/src/converge-job-setup.test.ts +152 -0
  178. package/src/converge-job-setup.ts +268 -0
  179. package/src/converge-job.test.ts +486 -0
  180. package/src/converge-job.ts +542 -0
  181. package/src/gateway-env-isolation.test.ts +8 -1
  182. package/src/gcp-names.ts +43 -1
  183. package/src/gcp-url-map.test.ts +9 -1
  184. package/src/index.ts +8 -0
  185. package/src/infra-contract.test.ts +7 -4
  186. package/src/infra-contract.ts +28 -1
  187. package/src/install-job.test.ts +3 -1
  188. package/src/install-job.ts +31 -14
  189. package/src/lifecycle.ts +137 -71
  190. package/src/operator-access.test.ts +14 -0
  191. package/src/operator-access.ts +13 -3
  192. package/src/private-file.test.ts +80 -0
  193. package/src/private-file.ts +50 -0
  194. package/src/release-contract.ts +8 -2
  195. package/src/release-manifest.test.ts +42 -0
  196. package/src/release-manifest.ts +2 -0
  197. package/src/remote-mcp-edge.ts +137 -0
  198. package/src/remote-mcp.test.ts +517 -0
  199. package/src/remote-mcp.ts +460 -0
  200. package/src/runtime-secrets.test.ts +31 -6
  201. package/src/runtime-secrets.ts +41 -3
  202. package/src/seal-key-rotation.test.ts +131 -0
  203. package/src/seal-key-rotation.ts +258 -0
  204. package/src/stack-contract-parity.test.ts +20 -3
  205. package/src/stack-contract.test.ts +34 -2
  206. package/src/stack-contract.ts +305 -3
  207. package/src/stack-converge-secrets.test.ts +3 -0
  208. package/src/stack-converge.test.ts +21 -5
  209. package/src/stack-converge.ts +101 -7
  210. package/src/state-attach.test.ts +7 -1
  211. package/src/state-lock-lease.test.ts +302 -0
  212. package/src/state-lock-lease.ts +423 -0
  213. package/src/state-lock.test.ts +568 -0
  214. package/src/state-store.ts +347 -27
  215. package/src/tar-entries.test.ts +149 -7
  216. package/src/tar-entries.ts +90 -9
  217. package/src/upgrade/app-roll.ts +135 -21
  218. package/src/upgrade/audit-job-roll.ts +68 -0
  219. package/src/upgrade/converge.ts +170 -7
  220. package/src/upgrade/gcp-lookup.test.ts +80 -0
  221. package/src/upgrade/gcp-lookup.ts +65 -8
  222. package/src/upgrade/record-rolled.test.ts +105 -0
  223. package/src/upgrade/record-rolled.ts +91 -0
  224. package/src/upgrade/upgrade.test.ts +279 -5
  225. package/src/upgrade/upgrade.ts +120 -24
  226. package/src/upgrader-assess.test.ts +93 -3
  227. package/src/upgrader-assess.ts +58 -9
  228. package/src/upgrader-gcp.ts +47 -8
  229. package/src/upgrader-iam-preflight.test.ts +393 -0
  230. package/src/upgrader-iam-preflight.ts +510 -0
  231. package/src/upgrader-job.test.ts +587 -15
  232. package/src/upgrader-job.ts +237 -53
  233. package/src/upgrader-records.ts +151 -0
  234. package/src/upgrader-reporter.test.ts +6 -0
  235. package/src/upgrader-reporter.ts +7 -2
  236. package/src/upgrader-setup.test.ts +4 -3
  237. package/src/upgrader-setup.ts +137 -13
  238. package/src/wizards/gcp-cost.ts +15 -6
  239. package/src/wizards/gcp-pulumi.ts +7 -0
  240. package/src/wizards/gcp.ts +162 -4
  241. package/stacks/aws/index.ts +18 -0
  242. package/stacks/azure/index.ts +18 -0
  243. package/stacks/gcp/index.ts +59 -7
  244. package/stacks/gcp-firebase/index.ts +50 -6
@@ -22,12 +22,24 @@
22
22
  */
23
23
 
24
24
  import { spawnSync } from "node:child_process";
25
+ import { randomUUID } from "node:crypto";
25
26
  import { existsSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
26
- import { hostname } from "node:os";
27
27
  import { join } from "node:path";
28
28
  import { assertNoCheckpointSecrets, assertNoSecrets } from "./checkpoint-secrets.js";
29
29
  import { type GcloudResult, gcloudEnv } from "./gcloud.js";
30
30
  import { OPS_INSTALL_PREFIX } from "./state-backend.js";
31
+ import {
32
+ describeExecutionArgs,
33
+ type ExecutionLockHolder,
34
+ judgeHeldLock,
35
+ type LocalHost,
36
+ type LockHolder,
37
+ lockBody,
38
+ PROCESS_LEASE_MS,
39
+ parseExecutionState,
40
+ parseLockBody,
41
+ renewedLockBody,
42
+ } from "./state-lock-lease.js";
31
43
 
32
44
  /** The two documents a store holds. */
33
45
  export type StateFile = "install.json" | "progress.json";
@@ -176,58 +188,324 @@ export function bucketStateLockUrl(bucket: string): string {
176
188
  return `gs://${bucket}/${OPS_INSTALL_PREFIX}/state.lock`;
177
189
  }
178
190
 
179
- /** Who holds a lock, for the refusal message. */
180
- function lockBody(purpose: string): string {
181
- return JSON.stringify({
182
- purpose,
183
- host: hostname(),
184
- pid: process.pid,
185
- startedAt: new Date().toISOString(),
186
- });
191
+ /** gcloud's wording for a failed `--if-generation-match` precondition. */
192
+ const PRECONDITION_FAILED = /PreconditionFailed|412|At least one of the pre-conditions/i;
193
+
194
+ /** How a bucket lock is taken: who holds it, and the clock to judge leases by. */
195
+ export interface BucketLockOptions {
196
+ /**
197
+ * The Cloud Run execution taking the lock (#815): the lock's lease is the
198
+ * execution's task timeout, and a later run may break it once Cloud Run
199
+ * says this execution has finished. Without it the holder is this
200
+ * process (a CLI run): the lock names host + pid, carries a lease of
201
+ * {@link PROCESS_LEASE_MS} and is renewed by a heartbeat until released.
202
+ */
203
+ readonly holder?: ExecutionLockHolder;
204
+ /** Told when a dead holder's lock was broken, with why. */
205
+ readonly onBreak?: (message: string) => void;
206
+ /**
207
+ * Told when the heartbeat found the lock no longer this run's (another
208
+ * run broke it after the lease ran out, e.g. while this machine slept).
209
+ */
210
+ readonly onLost?: (message: string) => void;
211
+ readonly now?: () => Date;
212
+ /** Starts the heartbeat; returns its stop. Defaults to an unref'd setInterval. */
213
+ readonly schedule?: (tick: () => void, everyMs: number) => () => void;
214
+ /** This host and its pid check (state-lock-lease.ts LocalHost); tests pass a fake. */
215
+ readonly here?: LocalHost;
187
216
  }
188
217
 
218
+ const defaultSchedule = (tick: () => void, everyMs: number): (() => void) => {
219
+ const timer = setInterval(tick, everyMs);
220
+ // The heartbeat must never keep a finished CLI alive.
221
+ timer.unref?.();
222
+ return () => clearInterval(timer);
223
+ };
224
+
189
225
  /**
190
226
  * The install's state lock in its ops bucket, created only if absent
191
227
  * (`--if-generation-match=0`). Held by every writer of a bucket-backed
192
- * install's state — an installer run's `pulumi up`, `state migrate` — so
193
- * none writes a backend another is switching away from. A lock left by a
194
- * crashed run is reported with the command to remove it, never broken
195
- * automatically.
228
+ * install's state — an installer run's `pulumi up`, `state migrate`, the
229
+ * CLI `upgrade`, the upgrader job — so none writes a backend another is
230
+ * switching away from.
196
231
  *
197
- * @returns the release function
232
+ * A lock that is held is judged (state-lock-lease.ts): one whose Cloud Run
233
+ * execution has finished, whose execution is gone after its lease, whose
234
+ * CLI process is gone on this host, or whose CLI lease ran out unrenewed,
235
+ * is broken with a generation-matched delete and taken once more. The
236
+ * generation is read BEFORE the body, so the delete can only remove the
237
+ * lock that was judged: if another run replaced it in between, the
238
+ * delete's precondition fails and nothing is removed. Two runs breaking the
239
+ * same dead lock cannot both win either — the slower one's create fails
240
+ * its own precondition and reports the lock as held.
241
+ *
242
+ * A CLI holder renews its lease every third of {@link PROCESS_LEASE_MS}
243
+ * with a generation-matched rewrite, so a run that is alive keeps its lock
244
+ * however long it takes, and a killed one loses it within one lease.
245
+ *
246
+ * @returns the release function; it stops the heartbeat and deletes only
247
+ * this lock's generation
248
+ * @throws StateLockHeldError when another run holds the lock
249
+ * @throws Error when the lock or its holder cannot be read (no access)
198
250
  */
199
251
  export function acquireBucketStateLock(
200
252
  bucket: string,
201
253
  purpose: string,
202
254
  run: SyncGcloud = realSyncGcloud,
255
+ opts: BucketLockOptions = {},
203
256
  ): () => void {
204
257
  const url = bucketStateLockUrl(bucket);
205
- const r = run(["storage", "cp", "-", url, "--if-generation-match=0"], lockBody(purpose));
206
- if (!r.ok) {
207
- if (/PreconditionFailed|412|At least one of the pre-conditions/i.test(r.stderr)) {
208
- const held = run(["storage", "cat", url]);
258
+ const now = opts.now ?? (() => new Date());
259
+ const holder: LockHolder = opts.holder ?? { leaseMs: PROCESS_LEASE_MS };
260
+ const id = randomUUID();
261
+ const create = () =>
262
+ run(
263
+ ["storage", "cp", "-", url, "--if-generation-match=0"],
264
+ lockBody(purpose, now(), holder, id),
265
+ );
266
+ let r = create();
267
+ if (!r.ok && PRECONDITION_FAILED.test(r.stderr)) {
268
+ breakDeadLock(url, run, now(), opts.onBreak, opts.here);
269
+ r = create();
270
+ if (!r.ok && PRECONDITION_FAILED.test(r.stderr)) {
209
271
  throw new StateLockHeldError(
210
- `Another run is changing this install's state (${held.ok ? held.stdout.trim() : url}). Re-run once it has finished; if none is running, remove the stale lock: gcloud storage rm ${url}`,
272
+ `Another run took this install's state lock at the same moment (${readLockText(url, run)}). Re-run once it has finished.`,
211
273
  );
212
274
  }
213
- throw new Error(`Could not take the state lock ${url}: ${r.stderr.trim()}`);
214
275
  }
276
+ if (!r.ok) throw new Error(`Could not take the state lock ${url}: ${r.stderr.trim()}`);
277
+ const owned = ownedGeneration(url, run, id);
278
+ if (owned.kind !== "owned") {
279
+ // Ours, just created — but without its generation a delete could hit a
280
+ // successor's lock. Give it back only once a second look proves it is
281
+ // still ours at a known generation; otherwise its lease ends it.
282
+ const retry = ownedGeneration(url, run, id);
283
+ if (retry.kind === "owned") {
284
+ run(["storage", "rm", url, `--if-generation-match=${retry.generation}`]);
285
+ }
286
+ throw new Error(
287
+ `Could not read the generation of the state lock ${url} this run just took (${owned.why}). ${
288
+ retry.kind === "owned"
289
+ ? "It was given back."
290
+ : "It was left in place and expires with its lease; re-run after that."
291
+ }`,
292
+ );
293
+ }
294
+ let generation = owned.generation;
295
+ let lost: string | null = null;
296
+ const stopHeartbeat =
297
+ "execution" in holder
298
+ ? () => {}
299
+ : (opts.schedule ?? defaultSchedule)(
300
+ () => {
301
+ if (lost !== null) return;
302
+ const renewed = renewLock(url, run, id, generation, now(), holder.leaseMs);
303
+ if (renewed.kind === "owned") {
304
+ generation = renewed.generation;
305
+ return;
306
+ }
307
+ if (renewed.kind === "lost") {
308
+ lost = renewed.why;
309
+ opts.onLost?.(
310
+ `This run no longer holds the install's state lock (${renewed.why}). Stop it and re-run once the other run has finished.`,
311
+ );
312
+ }
313
+ // "error": a transient failure; the next beat tries again within the lease.
314
+ },
315
+ Math.floor(holder.leaseMs / 3),
316
+ );
215
317
  return () => {
216
- const rm = run(["storage", "rm", url]);
217
- if (!rm.ok) throw new Error(`Could not release ${url}; remove it: gcloud storage rm ${url}`);
318
+ stopHeartbeat();
319
+ const rm = run(["storage", "rm", url, `--if-generation-match=${generation}`]);
320
+ if (rm.ok) return;
321
+ if (PRECONDITION_FAILED.test(rm.stderr) || NOT_FOUND.test(rm.stderr)) {
322
+ throw new Error(
323
+ `The state lock ${url} was no longer this run's when it finished (another run broke it); it was left alone.`,
324
+ );
325
+ }
326
+ throw new Error(`Could not release ${url}; remove it: gcloud storage rm ${url}`);
218
327
  };
219
328
  }
220
329
 
330
+ type Ownership =
331
+ | { readonly kind: "owned"; readonly generation: string }
332
+ | { readonly kind: "lost"; readonly why: string }
333
+ | { readonly kind: "error"; readonly why: string };
334
+
335
+ /**
336
+ * The lock's generation, provided the lock is still the one with `id`. The
337
+ * generation is read before the body: if another run replaced the lock in
338
+ * between, the body names its id and this returns "lost" — never a
339
+ * generation that is not ours.
340
+ */
341
+ function ownedGeneration(url: string, run: SyncGcloud, id: string): Ownership {
342
+ const g = lockGeneration(url, run);
343
+ if (g.kind === "absent") return { kind: "lost", why: "the lock is gone" };
344
+ if (g.kind === "error") return { kind: "error", why: g.error };
345
+ const body = run(["storage", "cat", url]);
346
+ if (!body.ok) {
347
+ return NOT_FOUND.test(body.stderr)
348
+ ? { kind: "lost", why: "the lock is gone" }
349
+ : { kind: "error", why: body.stderr.trim() || `gcloud exited ${body.exitCode}` };
350
+ }
351
+ const parsed = parseLockBody(body.stdout);
352
+ if (parsed?.id !== id) {
353
+ return { kind: "lost", why: `another run holds it now (${body.stdout.trim().slice(0, 200)})` };
354
+ }
355
+ return { kind: "owned", generation: g.generation };
356
+ }
357
+
358
+ /**
359
+ * One heartbeat: rewrite the lock with a later lease, only if it is still
360
+ * at `generation` (ours), and return its new generation.
361
+ */
362
+ function renewLock(
363
+ url: string,
364
+ run: SyncGcloud,
365
+ id: string,
366
+ generation: string,
367
+ now: Date,
368
+ leaseMs: number,
369
+ ): Ownership {
370
+ const current = run(["storage", "cat", url]);
371
+ if (!current.ok) {
372
+ return NOT_FOUND.test(current.stderr)
373
+ ? { kind: "lost", why: "the lock is gone" }
374
+ : { kind: "error", why: current.stderr.trim() };
375
+ }
376
+ const body = parseLockBody(current.stdout);
377
+ if (body?.id !== id) {
378
+ return {
379
+ kind: "lost",
380
+ why: `another run holds it now (${current.stdout.trim().slice(0, 200)})`,
381
+ };
382
+ }
383
+ const w = run(
384
+ ["storage", "cp", "-", url, `--if-generation-match=${generation}`],
385
+ renewedLockBody(body, now, leaseMs),
386
+ );
387
+ if (!w.ok) {
388
+ return PRECONDITION_FAILED.test(w.stderr)
389
+ ? { kind: "lost", why: "another run replaced it" }
390
+ : { kind: "error", why: w.stderr.trim() };
391
+ }
392
+ return ownedGeneration(url, run, id);
393
+ }
394
+
395
+ type LockGeneration =
396
+ | { readonly kind: "found"; readonly generation: string }
397
+ | { readonly kind: "absent" }
398
+ | { readonly kind: "error"; readonly error: string };
399
+
400
+ /** The lock object's current generation. */
401
+ function lockGeneration(url: string, run: SyncGcloud): LockGeneration {
402
+ const d = run(["storage", "objects", "describe", url, "--format=json"]);
403
+ if (!d.ok) {
404
+ return NOT_FOUND.test(d.stderr)
405
+ ? { kind: "absent" }
406
+ : { kind: "error", error: d.stderr.trim() || `gcloud exited ${d.exitCode}` };
407
+ }
408
+ try {
409
+ // `gcloud storage objects describe` prints the generation as a string
410
+ // (`generation: '1706304784726341'`, cloud.google.com/storage/docs/
411
+ // viewing-editing-metadata); a JSON number would lose digits past 2^53,
412
+ // so only a string of digits is a generation the delete may match.
413
+ const g = (JSON.parse(d.stdout) as { generation?: unknown }).generation;
414
+ if (typeof g === "string" && /^[1-9][0-9]*$/.test(g)) {
415
+ return { kind: "found", generation: g };
416
+ }
417
+ } catch {
418
+ // Reported below with the same message as a missing field.
419
+ }
420
+ return { kind: "error", error: "gcloud described the lock without a generation" };
421
+ }
422
+
423
+ function readLockText(url: string, run: SyncGcloud): string {
424
+ const held = run(["storage", "cat", url]);
425
+ return held.ok ? held.stdout.trim() : url;
426
+ }
427
+
428
+ /**
429
+ * Judge the held lock at `url` and break it when its holder is dead.
430
+ * Returns normally when the lock was broken or vanished meanwhile, so the
431
+ * caller creates it once more; throws when it is live or cannot be judged.
432
+ */
433
+ function breakDeadLock(
434
+ url: string,
435
+ run: SyncGcloud,
436
+ now: Date,
437
+ onBreak: ((message: string) => void) | undefined,
438
+ here: LocalHost | undefined,
439
+ ): void {
440
+ const found = lockGeneration(url, run);
441
+ if (found.kind === "absent") return;
442
+ if (found.kind === "error") {
443
+ throw new Error(`Could not read the state lock ${url} another run holds: ${found.error}`);
444
+ }
445
+ const held = run(["storage", "cat", url]);
446
+ if (!held.ok) {
447
+ if (NOT_FOUND.test(held.stderr)) return;
448
+ throw new Error(`Could not read the state lock ${url}: ${held.stderr.trim()}`);
449
+ }
450
+ const verdict = judgeHeldLock(
451
+ held.stdout,
452
+ (ref) => parseExecutionState(run(describeExecutionArgs(ref))),
453
+ now,
454
+ here,
455
+ );
456
+ if (verdict.kind === "unknown") {
457
+ // Not contention: retrying will not help, and guessing "dead" could let
458
+ // two runs write the same state.
459
+ throw new Error(`The state lock ${url} is held and ${verdict.why}`);
460
+ }
461
+ if (verdict.kind === "held") {
462
+ throw new StateLockHeldError(
463
+ verdict.manualRemoval
464
+ ? `Another run is changing this install's state (${verdict.why}). Re-run once it has finished; if none is running, remove the stale lock: gcloud storage rm ${url}`
465
+ : `Another run is changing this install's state: ${verdict.why}. Re-run once it has finished.`,
466
+ );
467
+ }
468
+ const rm = run(["storage", "rm", url, `--if-generation-match=${found.generation}`]);
469
+ if (!rm.ok && !NOT_FOUND.test(rm.stderr)) {
470
+ if (PRECONDITION_FAILED.test(rm.stderr)) {
471
+ throw new StateLockHeldError(
472
+ `Another run took this install's state lock while this one removed a stale one (${readLockText(url, run)}). Re-run once it has finished.`,
473
+ );
474
+ }
475
+ throw new Error(`Could not remove the stale state lock ${url}: ${rm.stderr.trim()}`);
476
+ }
477
+ onBreak?.(`Removed a stale state lock: ${verdict.why}.`);
478
+ }
479
+
221
480
  /**
222
481
  * The same lock for an install whose state is on this machine:
223
- * `~/.caelo-<id>/state.lock`, created exclusively.
482
+ * `~/.caelo-<id>/state.lock`, created exclusively. A lock left by a
483
+ * process that is gone on this machine is taken over (#815).
224
484
  *
485
+ * @param here this host and its pid check; tests pass a fake
225
486
  * @returns the release function
226
487
  */
227
- export function acquireLocalStateLock(root: string, purpose: string): () => void {
488
+ export function acquireLocalStateLock(root: string, purpose: string, here?: LocalHost): () => void {
228
489
  const path = join(root, "state.lock");
490
+ const take = () =>
491
+ writeFileSync(path, lockBody(purpose, new Date()), { flag: "wx", mode: 0o600 });
229
492
  try {
230
- writeFileSync(path, lockBody(purpose), { flag: "wx", mode: 0o600 });
493
+ try {
494
+ take();
495
+ } catch (e) {
496
+ if ((e as NodeJS.ErrnoException).code !== "EEXIST") throw e;
497
+ // #815 — the file is on this machine, so a holder whose process has
498
+ // ended is known dead: take its lock over instead of asking for `rm`.
499
+ const verdict = judgeHeldLock(
500
+ readFileSync(path, "utf8"),
501
+ () => ({ kind: "unknown", error: "a machine-local lock names no Cloud Run execution" }),
502
+ new Date(),
503
+ here,
504
+ );
505
+ if (verdict.kind !== "break") throw e;
506
+ rmSync(path, { force: true });
507
+ take();
508
+ }
231
509
  } catch (e) {
232
510
  if ((e as NodeJS.ErrnoException).code === "EEXIST") {
233
511
  throw new StateLockHeldError(
@@ -245,8 +523,50 @@ export function acquireStateLock(
245
523
  root: string,
246
524
  purpose: string,
247
525
  run: SyncGcloud = realSyncGcloud,
526
+ opts: BucketLockOptions = {},
248
527
  ): () => void {
249
528
  return backend.kind === "gcs"
250
- ? acquireBucketStateLock(backend.bucket, purpose, run)
251
- : acquireLocalStateLock(root, purpose);
529
+ ? acquireBucketStateLock(backend.bucket, purpose, run, opts)
530
+ : acquireLocalStateLock(root, purpose, opts.here);
531
+ }
532
+
533
+ /** The part of `process` {@link releaseOnSignal} needs; tests pass a fake. */
534
+ export interface SignalTarget {
535
+ once(signal: "SIGINT" | "SIGTERM", listener: () => void): unknown;
536
+ removeListener(signal: "SIGINT" | "SIGTERM", listener: () => void): unknown;
537
+ exit(code: number): never;
538
+ }
539
+
540
+ /**
541
+ * Give the state lock back when the CLI is stopped (Ctrl-C, a closed Cloud
542
+ * Shell tab's SIGTERM), then exit with the signal's conventional code
543
+ * (#815). Without this a stopped run leaves its lock until its lease ends.
544
+ *
545
+ * @param report told what happened to the lock
546
+ * @returns removes the handlers; call it once the run released the lock itself
547
+ */
548
+ export function releaseOnSignal(
549
+ release: () => void,
550
+ report: (message: string) => void,
551
+ proc: SignalTarget = process,
552
+ ): () => void {
553
+ const dispose = () => {
554
+ proc.removeListener("SIGINT", onInt);
555
+ proc.removeListener("SIGTERM", onTerm);
556
+ };
557
+ const stop = (signal: "SIGINT" | "SIGTERM", code: number): never => {
558
+ dispose();
559
+ try {
560
+ release();
561
+ report(`Stopped (${signal}); the install's state lock was released.`);
562
+ } catch (e) {
563
+ report(`Stopped (${signal}); ${e instanceof Error ? e.message : String(e)}`);
564
+ }
565
+ return proc.exit(code);
566
+ };
567
+ const onInt = () => stop("SIGINT", 130);
568
+ const onTerm = () => stop("SIGTERM", 143);
569
+ proc.once("SIGINT", onInt);
570
+ proc.once("SIGTERM", onTerm);
571
+ return dispose;
252
572
  }
@@ -35,18 +35,160 @@ describe("listTarEntries", () => {
35
35
  });
36
36
 
37
37
  it("refuses a truncated archive instead of skipping members", () => {
38
- const header = new Uint8Array(512);
39
- header.set(new TextEncoder().encode("a.txt"), 0);
40
- header.set(new TextEncoder().encode("00000001000\0"), 124); // 512 bytes of data, none present
41
- header[156] = "0".charCodeAt(0);
42
- expect(() => listTarEntries(header)).toThrow(/truncated/);
38
+ expect(() => listTarEntries(header("a.txt", "0", { sizeField: 512 }))).toThrow(/truncated/);
39
+ });
40
+
41
+ it("rejects PAX global extended headers ('g')", () => {
42
+ const g = member("PaxHeader", "g", paxLine("path=/etc/passwd"));
43
+ expect(() => listTarEntries(concat(g, member("safe.txt", "0"), END))).toThrow(
44
+ /pax global extended header/,
45
+ );
46
+ });
47
+
48
+ it("refuses a member named by two pax headers — GNU tar keeps the last, bsdtar drops the second", () => {
49
+ // Both tars extract `../../evil` here (the second header carries no
50
+ // path); reading the first header's path would have checked `safe.txt`.
51
+ const archive = concat(
52
+ member("PaxHeader", "x", paxLine("path=srv/caelo/safe.txt")),
53
+ member("PaxHeader", "x", paxLine("mtime=1")),
54
+ member("../../evil", "0"),
55
+ END,
56
+ );
57
+ expect(() => listTarEntries(archive)).toThrow(/several extension headers \(xx\)/);
58
+ });
59
+
60
+ it("refuses a GNU long name next to a pax path, and two long names", () => {
61
+ // bsdtar extracts the `L` name (`../../evil`), GNU tar the pax path.
62
+ const mixed = concat(
63
+ member("././@LongLink", "L", "../../evil\0"),
64
+ member("PaxHeader", "x", paxLine("path=srv/caelo/safe.txt")),
65
+ member("x", "0"),
66
+ END,
67
+ );
68
+ expect(() => listTarEntries(mixed)).toThrow(/several extension headers \(Lx\)/);
69
+ const twice = concat(
70
+ member("././@LongLink", "L", "../../evil\0"),
71
+ member("././@LongLink", "L", "safe.txt\0"),
72
+ member("x", "0"),
73
+ END,
74
+ );
75
+ expect(() => listTarEntries(twice)).toThrow(/several extension headers \(LL\)/);
76
+ });
77
+
78
+ it("still reads a GNU long name together with a long link", () => {
79
+ const archive = concat(
80
+ member("././@LongLink", "K", "target\0"),
81
+ member("././@LongLink", "L", `${"a".repeat(120)}\0`),
82
+ member("short", "2", "", { link: "ignored" }),
83
+ END,
84
+ );
85
+ expect(listTarEntries(archive)).toEqual([
86
+ { name: "a".repeat(120), type: "symlink", linkTarget: "target" },
87
+ ]);
88
+ });
89
+
90
+ it("refuses a header whose checksum does not match — GNU tar would skip it and read its body as headers", () => {
91
+ const archive = concat(
92
+ header("srv/caelo/safe.txt", "0", { sizeField: 512, badChecksum: true }),
93
+ header("../../evil", "0"),
94
+ END,
95
+ );
96
+ expect(() => listTarEntries(archive)).toThrow(/checksum mismatch/);
97
+ });
98
+
99
+ it("refuses a directory or symlink with a body — bsdtar reads the next block as a header", () => {
100
+ for (const [name, type] of [
101
+ ["srv/caelo/link", "2"],
102
+ ["srv/caelo/dir/", "5"],
103
+ ] as const) {
104
+ const archive = concat(
105
+ header(name, type, { sizeField: 512, link: "a" }),
106
+ header("../../evil", "0"),
107
+ END,
108
+ );
109
+ expect(() => listTarEntries(archive)).toThrow(/with a body/);
110
+ }
111
+ });
112
+
113
+ it("walks a member's body by its pax size, so a member hidden behind the header size is seen", () => {
114
+ // tar -x reads `safe.txt` as empty (pax size=0) and then extracts the
115
+ // `../../evil` header that the ustar size field claims is file data.
116
+ const archive = concat(
117
+ member("PaxHeader", "x", paxLine("size=0")),
118
+ header("safe.txt", "0", { sizeField: 512 }),
119
+ member("../../evil", "0"),
120
+ END,
121
+ );
122
+ expect(listTarEntries(archive).map((e) => e.name)).toEqual(["safe.txt", "../../evil"]);
123
+ });
124
+
125
+ it("rejects GNU sparse pax records and an unreadable pax size", () => {
126
+ const sparse = member("PaxHeader", "x", paxLine("GNU.sparse.name=../../evil"));
127
+ expect(() => listTarEntries(concat(sparse, member("safe.txt", "0")))).toThrow(/GNU sparse/);
128
+ const badSize = member("PaxHeader", "x", paxLine("size=-1"));
129
+ expect(() => listTarEntries(concat(badSize, member("safe.txt", "0")))).toThrow(
130
+ /unreadable pax size/,
131
+ );
43
132
  });
44
133
  });
45
134
 
46
- describe("escapesRoot", () => {
47
- it("flags absolute and parent-relative member names", () => {
135
+ const END = new Uint8Array(1024);
136
+
137
+ /**
138
+ * One ustar header block with a valid checksum (unless `badChecksum`).
139
+ * `sizeField` is what the header claims, independent of any body after it.
140
+ */
141
+ function header(
142
+ name: string,
143
+ typeflag: string,
144
+ opts: { sizeField?: number; link?: string; badChecksum?: boolean } = {},
145
+ ): Uint8Array {
146
+ const enc = new TextEncoder();
147
+ const h = new Uint8Array(512);
148
+ h.set(enc.encode(name), 0);
149
+ h.set(enc.encode(`${(opts.sizeField ?? 0).toString(8).padStart(11, "0")}\0`), 124);
150
+ h[156] = typeflag.charCodeAt(0);
151
+ if (opts.link) h.set(enc.encode(opts.link), 157);
152
+ h.set(enc.encode("ustar\0"), 257);
153
+ h.fill(0x20, 148, 156);
154
+ const sum = h.reduce((n, b) => n + b, 0) + (opts.badChecksum ? 1 : 0);
155
+ h.set(enc.encode(`${sum.toString(8).padStart(6, "0")}\0 `), 148);
156
+ return h;
157
+ }
158
+
159
+ /** A header plus its padded body. */
160
+ function member(name: string, typeflag: string, bodyText = "", opts: { link?: string } = {}) {
161
+ const body = new TextEncoder().encode(bodyText);
162
+ const out = new Uint8Array(512 + Math.ceil(body.length / 512) * 512);
163
+ out.set(header(name, typeflag, { sizeField: body.length, link: opts.link }), 0);
164
+ out.set(body, 512);
165
+ return out;
166
+ }
167
+
168
+ /** One pax record; its leading length counts its own digits. */
169
+ function paxLine(kv: string): string {
170
+ const base = ` ${kv}\n`;
171
+ const len = String(base.length + 2).length + base.length;
172
+ return `${len}${base}`;
173
+ }
174
+
175
+ function concat(...parts: Uint8Array[]): Uint8Array {
176
+ const out = new Uint8Array(parts.reduce((n, p) => n + p.length, 0));
177
+ let at = 0;
178
+ for (const p of parts) {
179
+ out.set(p, at);
180
+ at += p.length;
181
+ }
182
+ return out;
183
+ }
184
+
185
+ describe("escapesRoot & normalizeMemberName", () => {
186
+ it("flags absolute, parent-relative, and dot-segment member names", () => {
48
187
  expect(escapesRoot("/etc/passwd")).toBe(true);
49
188
  expect(escapesRoot("srv/caelo/../../app")).toBe(true);
189
+ expect(escapesRoot(".//../etc/passwd")).toBe(true);
190
+ expect(escapesRoot("a/./b")).toBe(true);
50
191
  expect(escapesRoot("./srv/caelo/a..b")).toBe(false);
192
+ expect(normalizeMemberName(".//srv//caelo/")).toBe("srv/caelo");
51
193
  });
52
194
  });