@caelo-cms/provisioning 0.10.53 → 0.10.57

package/dist/state-lock-lease.d.ts

@@ -0,0 +1,142 @@
+import { z } from "zod";
+import type { GcloudResult } from "./gcloud.js";
+/** A Cloud Run job execution, fully addressed. */
+export interface CloudRunExecutionRef {
+    readonly projectId: string;
+    readonly region: string;
+    /** The execution's short name (`CLOUD_RUN_EXECUTION`). */
+    readonly name: string;
+}
+/**
+ * A lock holder that Cloud Run can vouch for: the execution and the
+ * longest it can run (its task timeout).
+ */
+export interface ExecutionLockHolder {
+    readonly execution: CloudRunExecutionRef;
+    readonly maxRuntimeMs: number;
+}
+/**
+ * A lock holder outside Cloud Run (a CLI process): its lease is short and
+ * renewed by a heartbeat while it runs, so a holder that dies stops
+ * renewing and its lock expires on its own (#815).
+ */
+export interface ProcessLockHolder {
+    readonly leaseMs: number;
+}
+/** Who takes a lock: a Cloud Run execution, or a renewing process. */
+export type LockHolder = ExecutionLockHolder | ProcessLockHolder;
+/** The lease of a CLI process's lock; renewed every third of it. */
+export declare const PROCESS_LEASE_MS: number;
+/**
+ * Margin added to a lease for clock skew between the holder and the run
+ * judging it (both read their own clock; Cloud Run's are NTP-synced, a
+ * developer laptop's may not be).
+ */
+export declare const LEASE_SKEW_MS: number;
+/** The lock object's body. `execution` + `leaseUntil` appear together (#815). */
+declare const lockBodySchema: z.ZodObject<{
+    purpose: z.ZodString;
+    id: z.ZodOptional<z.ZodString>;
+    host: z.ZodOptional<z.ZodString>;
+    machine: z.ZodOptional<z.ZodNullable<z.ZodString>>;
+    pid: z.ZodOptional<z.ZodNumber>;
+    startedAt: z.ZodOptional<z.ZodString>;
+    execution: z.ZodOptional<z.ZodObject<{
+        projectId: z.ZodString;
+        region: z.ZodString;
+        name: z.ZodString;
+    }, z.core.$strip>>;
+    leaseUntil: z.ZodOptional<z.ZodString>;
+}, z.core.$strip>;
+/** The lock body as written by {@link lockBody}. */
+export type LockBody = z.infer<typeof lockBodySchema>;
+/**
+ * The body a new lock is written with.
+ *
+ * @param holder a Cloud Run execution (lease: `now + maxRuntimeMs +
+ *   LEASE_SKEW_MS`) or a renewing process (lease: `now + leaseMs +
+ *   LEASE_SKEW_MS`); none only for the machine-local lock file, which is
+ *   judged by its pid alone
+ * @param id the acquisition's id; a fresh one when omitted
+ */
+export declare function lockBody(purpose: string, now: Date, holder?: LockHolder, id?: string): string;
+/**
+ * The same lock body with its lease moved to `now + leaseMs + skew` — the
+ * heartbeat of a process holder. Everything else (id, start) is kept.
+ */
+export declare function renewedLockBody(body: LockBody, now: Date, leaseMs: number): string;
+/** Whether process `pid` exists on this host (signal 0 checks without signalling). */
+export declare function processAlive(pid: number): boolean;
+/**
+ * An id for this machine AND pid namespace, so a pid is only ever checked
+ * where it was taken: the host name, the network cards' MAC addresses and
+ * (on Linux) the pid namespace. Null inside Cloud Run, where every
+ * container is called `localhost`, has its own pid space and no stable
+ * hardware to tell it apart — a lock from there is judged by its lease
+ * alone. Null too when the machine shows no network card. An id that
+ * changes while a lock is held (a new network card) only makes a later
+ * judge wait for the lease: a mismatch is always the safe direction.
+ */
+export declare function machineId(env?: NodeJS.ProcessEnv): string | null;
+/** The judging run's own host and how it checks a local pid. */
+export interface LocalHost {
+    readonly host: string;
+    /**
+     * {@link machineId}; a lock that names another (or none) is never judged
+     * by its pid. Left out only by a lock from before the machine id, which
+     * falls back to comparing host names (state-lock-lease tests).
+     */
+    readonly machine?: string | null;
+    readonly alive: (pid: number) => boolean;
+}
+/** Parse a lock body; null when it is not one this code wrote. */
+export declare function parseLockBody(text: string): LockBody | null;
+/** What Cloud Run says about the holder's execution. */
+export type ExecutionState = {
+    readonly kind: "running";
+} | {
+    readonly kind: "finished";
+    readonly how: string;
+} | {
+    readonly kind: "not-found";
+}
+/** Cloud Run could not be asked; `error` says why. */
+ | {
+    readonly kind: "unknown";
+    readonly error: string;
+};
+/**
+ * Read an execution's state from `gcloud run jobs executions describe
+ * --format=json`. An execution has finished once Cloud Run stamped its
+ * completion time or decided its `Completed` condition (True: succeeded;
+ * False: failed or cancelled). Anything else is still running — including
+ * an execution Cloud Run has not started yet.
+ */
+export declare function parseExecutionState(result: GcloudResult): ExecutionState;
+/** `gcloud run jobs executions describe`, for {@link judgeHeldLock}. */
+export declare function describeExecutionArgs(ref: CloudRunExecutionRef): string[];
+/** Whether a held lock may be broken, and the words for either outcome. */
+export type LockVerdict = {
+    readonly kind: "break";
+    readonly why: string;
+} | {
+    readonly kind: "held";
+    readonly why: string;
+    readonly manualRemoval: boolean;
+}
+/** The holder could not be checked; never treated as "dead". */
+ | {
+    readonly kind: "unknown";
+    readonly why: string;
+};
+/**
+ * Decide whether the lock with body `text` may be broken.
+ *
+ * @param describe looks the holder's execution up in Cloud Run; called only
+ *   when the body names one
+ * @param now the judging run's clock, compared against the lease
+ * @param here this host's name and pid check, for a process holder's lock
+ */
+export declare function judgeHeldLock(text: string, describe: (ref: CloudRunExecutionRef) => ExecutionState, now: Date, here?: LocalHost): LockVerdict;
+export {};
+//# sourceMappingURL=state-lock-lease.d.ts.map

package/dist/state-lock-lease.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"state-lock-lease.d.ts","sourceRoot":"","sources":["../src/state-lock-lease.ts"],"names":[],"mappings":"AAgDA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,kDAAkD;AAClD,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,0DAA0D;IAC1D,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAED;;;GAGG;AACH,MAAM,WAAW,mBAAmB;IAClC,QAAQ,CAAC,SAAS,EAAE,oBAAoB,CAAC;IACzC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;CAC/B;AAED;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;CAC1B;AAED,sEAAsE;AACtE,MAAM,MAAM,UAAU,GAAG,mBAAmB,GAAG,iBAAiB,CAAC;AAEjE,oEAAoE;AACpE,eAAO,MAAM,gBAAgB,QAAc,CAAC;AAE5C;;;;GAIG;AACH,eAAO,MAAM,aAAa,QAAa,CAAC;AAQxC,iFAAiF;AACjF,QAAA,MAAM,cAAc;;;;;;;;;;;;;iBAWlB,CAAC;AAEH,oDAAoD;AACpD,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,cAAc,CAAC,CAAC;AAEtD;;;;;;;;GAQG;AACH,wBAAgB,QAAQ,CACtB,OAAO,EAAE,MAAM,EACf,GAAG,EAAE,IAAI,EACT,MAAM,CAAC,EAAE,UAAU,EACnB,EAAE,GAAE,MAAqB,GACxB,MAAM,CAeR;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,QAAQ,EAAE,GAAG,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,CAKlF;AAED,sFAAsF;AACtF,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAQjD;AAED;;;;;;;;;GASG;AACH,wBAAgB,SAAS,CAAC,GAAG,GAAE,MAAM,CAAC,UAAwB,GAAG,MAAM,GAAG,IAAI,CAmB7E;AAED,gEAAgE;AAChE,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB;;;;OAIG;IACH,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,QAAQ,CAAC,KAAK,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC;CAC1C;AAID,kEAAkE;AAClE,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,QAAQ,GAAG,IAAI,CAS3D;AAED,wDAAwD;AACxD,MAAM,MAAM,cAAc,GACtB;IAAE,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAA;CAAE,GAC5B;IAAE,QAAQ,CAAC,IAAI,EAAE,UAAU,CAAC;IAAC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GACnD;IAAE,QAAQ,CAAC,IAAI,EAAE,WAAW,CAAA;CAAE;AAChC,sDAAsD;GACpD;IAAE,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IAAC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAqBzD;;;;;;GAMG;AACH,wBAAgB,mBAAmB,CAAC,MAAM,EAAE,YAAY,GAAG,cAAc,CA2BxE;AAED,wEAAwE;AACxE,wBAAgB,qBAAqB,CAAC,GAAG,EAAE,oBAAoB,GAAG,MAAM,EAAE,CAWzE;AAED,2EAA2E;AAC3E,MAAM,MAAM,WAAW,GACnB;IAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IAAC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;CAAE,GAChD;IAAE,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,aAAa,EAAE,OAAO,CAAA;CAAE;AAClF,gEAAgE;GAC9D;IAAE,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IAAC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAA;CAAE,CAAC;AAEvD;;;;;;;GAOG;AACH,wBAAgB,aAAa,CAC3B,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,CAAC,GAAG,EAAE,oBAAoB,KAAK,cAAc,EACvD,GAAG,EAAE,IAAI,EACT,IAAI,GAAE,SAAqB,GAC1B,WAAW,CA8Cb"}

package/dist/state-lock-lease.js

@@ -0,0 +1,347 @@
+// SPDX-License-Identifier: MPL-2.0
+/**
+ * The lease on the install's bucket state lock (#815): who holds it, and
+ * whether a later run may break it.
+ *
+ * A lock written by a Cloud Run job execution (the upgrader job, later the
+ * converge job #819) names that execution and a lease: the execution's
+ * start plus its task timeout, after which Cloud Run has killed it for
+ * sure. A run that finds the lock held asks Cloud Run about the holder:
+ *
+ *   - the execution has finished (succeeded, failed, cancelled): it can no
+ *     longer write, so its lock is dead and may be broken;
+ *   - it is still running: the lock is live and is NEVER broken — not even
+ *     past its lease (a lease is our estimate; Cloud Run's state is fact);
+ *   - Cloud Run does not know it (deleted, or a name we cannot resolve):
+ *     broken only once the lease has expired, when no execution of that
+ *     job can still be alive;
+ *   - Cloud Run cannot be asked (no access, gcloud error): not decided
+ *     here — the caller reports the error; guessing "dead" could let two
+ *     runs write the same state.
+ *
+ * A lock taken by a process outside Cloud Run (a CLI run on someone's
+ * machine or in Cloud Shell) names its host and pid and carries a short
+ * lease that the holder renews while it runs (the heartbeat in
+ * state-store.ts). Such a lock is broken when:
+ *
+ *   - it was taken on THIS machine and its pid is no longer alive (a
+ *     killed CLI on the same machine — no need to wait for the lease). The
+ *     machine is told apart by {@link machineId}, not by its host name
+ *     alone: every Cloud Run container calls itself the same, and two
+ *     laptops may too, so a pid that is dead HERE says nothing about a
+ *     process THERE. Inside Cloud Run there is no machine id at all, so a
+ *     lock from there is only ever judged by its lease; or
+ *   - its lease has expired: the holder stopped renewing it (killed, or a
+ *     closed Cloud Shell), so it can no longer be writing.
+ *
+ * Only a lock written before #815 (no lease, no execution) is never broken
+ * automatically; its holder message names the command to remove it.
+ *
+ * Everything here is pure except {@link describeExecution}, which takes a
+ * {@link SyncGcloud}; the lock itself (create, generation-matched break,
+ * release) lives in state-store.ts.
+ */
+import { createHash, randomUUID } from "node:crypto";
+import { readlinkSync } from "node:fs";
+import { hostname, networkInterfaces } from "node:os";
+import { z } from "zod";
+/** The lease of a CLI process's lock; renewed every third of it. */
+export const PROCESS_LEASE_MS = 10 * 60_000;
+/**
+ * Margin added to a lease for clock skew between the holder and the run
+ * judging it (both read their own clock; Cloud Run's are NTP-synced, a
+ * developer laptop's may not be).
+ */
+export const LEASE_SKEW_MS = 5 * 60_000;
+const executionRefSchema = z.object({
+    projectId: z.string().min(1),
+    region: z.string().min(1),
+    name: z.string().min(1),
+});
+/** The lock object's body. `execution` + `leaseUntil` appear together (#815). */
+const lockBodySchema = z.object({
+    purpose: z.string(),
+    /** Unique per acquisition: proves a lock is still the one this run took. */
+    id: z.string().optional(),
+    host: z.string().optional(),
+    /** {@link machineId} of the writer; null when it has none (Cloud Run). */
+    machine: z.string().nullable().optional(),
+    pid: z.number().optional(),
+    startedAt: z.string().optional(),
+    execution: executionRefSchema.optional(),
+    leaseUntil: z.string().optional(),
+});
+/**
+ * The body a new lock is written with.
+ *
+ * @param holder a Cloud Run execution (lease: `now + maxRuntimeMs +
+ *   LEASE_SKEW_MS`) or a renewing process (lease: `now + leaseMs +
+ *   LEASE_SKEW_MS`); none only for the machine-local lock file, which is
+ *   judged by its pid alone
+ * @param id the acquisition's id; a fresh one when omitted
+ */
+export function lockBody(purpose, now, holder, id = randomUUID()) {
+    const leaseMs = holder ? ("execution" in holder ? holder.maxRuntimeMs : holder.leaseMs) : null;
+    const body = {
+        purpose,
+        id,
+        host: THIS_HOST.host,
+        machine: THIS_HOST.machine ?? null,
+        pid: process.pid,
+        startedAt: now.toISOString(),
+        ...(holder && "execution" in holder ? { execution: { ...holder.execution } } : {}),
+        ...(leaseMs !== null
+            ? { leaseUntil: new Date(now.getTime() + leaseMs + LEASE_SKEW_MS).toISOString() }
+            : {}),
+    };
+    return JSON.stringify(body);
+}
+/**
+ * The same lock body with its lease moved to `now + leaseMs + skew` — the
+ * heartbeat of a process holder. Everything else (id, start) is kept.
+ */
+export function renewedLockBody(body, now, leaseMs) {
+    return JSON.stringify({
+        ...body,
+        leaseUntil: new Date(now.getTime() + leaseMs + LEASE_SKEW_MS).toISOString(),
+    });
+}
+/** Whether process `pid` exists on this host (signal 0 checks without signalling). */
+export function processAlive(pid) {
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (e) {
+        // EPERM: it exists, it is just someone else's.
+        return e.code === "EPERM";
+    }
+}
+/**
+ * An id for this machine AND pid namespace, so a pid is only ever checked
+ * where it was taken: the host name, the network cards' MAC addresses and
+ * (on Linux) the pid namespace. Null inside Cloud Run, where every
+ * container is called `localhost`, has its own pid space and no stable
+ * hardware to tell it apart — a lock from there is judged by its lease
+ * alone. Null too when the machine shows no network card. An id that
+ * changes while a lock is held (a new network card) only makes a later
+ * judge wait for the lease: a mismatch is always the safe direction.
+ */
+export function machineId(env = process.env) {
+    if (env.CLOUD_RUN_EXECUTION || env.K_SERVICE)
+        return null;
+    const macs = new Set();
+    for (const cards of Object.values(networkInterfaces())) {
+        for (const card of cards ?? []) {
+            if (!card.internal && card.mac && card.mac !== "00:00:00:00:00:00")
+                macs.add(card.mac);
+        }
+    }
+    if (macs.size === 0)
+        return null;
+    let pidSpace = "";
+    try {
+        pidSpace = readlinkSync("/proc/self/ns/pid");
+    }
+    catch {
+        // Not Linux: the machine has one pid space, which the MACs already name.
+    }
+    return createHash("sha256")
+        .update([hostname(), pidSpace, ...[...macs].sort()].join("\n"))
+        .digest("hex")
+        .slice(0, 32);
+}
+const THIS_HOST = { host: hostname(), machine: machineId(), alive: processAlive };
+/** Parse a lock body; null when it is not one this code wrote. */
+export function parseLockBody(text) {
+    let raw;
+    try {
+        raw = JSON.parse(text);
+    }
+    catch {
+        return null;
+    }
+    const parsed = lockBodySchema.safeParse(raw);
+    return parsed.success ? parsed.data : null;
+}
+const NOT_FOUND = /NOT_FOUND|not found|could not be found|does not exist/i;
+const executionStatusSchema = z.object({
+    status: z
+        .object({
+        completionTime: z.string().optional(),
+        conditions: z
+            .array(z.object({
+            type: z.string(),
+            status: z.string(),
+            reason: z.string().optional(),
+        }))
+            .optional(),
+    })
+        .optional(),
+});
+/**
+ * Read an execution's state from `gcloud run jobs executions describe
+ * --format=json`. An execution has finished once Cloud Run stamped its
+ * completion time or decided its `Completed` condition (True: succeeded;
+ * False: failed or cancelled). Anything else is still running — including
+ * an execution Cloud Run has not started yet.
+ */
+export function parseExecutionState(result) {
+    if (!result.ok) {
+        if (NOT_FOUND.test(result.stderr))
+            return { kind: "not-found" };
+        return {
+            kind: "unknown",
+            error: result.stderr.trim() || `gcloud exited ${result.exitCode}`,
+        };
+    }
+    let raw;
+    try {
+        raw = JSON.parse(result.stdout);
+    }
+    catch {
+        return { kind: "unknown", error: "gcloud returned no JSON for the execution" };
+    }
+    const parsed = executionStatusSchema.safeParse(raw);
+    if (!parsed.success) {
+        return { kind: "unknown", error: "gcloud returned an execution without a readable status" };
+    }
+    const status = parsed.data.status;
+    const completed = status?.conditions?.find((c) => c.type === "Completed");
+    if (completed && (completed.status === "True" || completed.status === "False")) {
+        const how = completed.status === "True" ? "succeeded" : (completed.reason ?? "failed").toLowerCase();
+        return { kind: "finished", how };
+    }
+    if (status?.completionTime)
+        return { kind: "finished", how: "completed" };
+    return { kind: "running" };
+}
+/** `gcloud run jobs executions describe`, for {@link judgeHeldLock}. */
+export function describeExecutionArgs(ref) {
+    return [
+        "run",
+        "jobs",
+        "executions",
+        "describe",
+        ref.name,
+        `--region=${ref.region}`,
+        `--project=${ref.projectId}`,
+        "--format=json",
+    ];
+}
+/**
+ * Decide whether the lock with body `text` may be broken.
+ *
+ * @param describe looks the holder's execution up in Cloud Run; called only
+ *   when the body names one
+ * @param now the judging run's clock, compared against the lease
+ * @param here this host's name and pid check, for a process holder's lock
+ */
+export function judgeHeldLock(text, describe, now, here = THIS_HOST) {
+    const body = parseLockBody(text);
+    if (!body) {
+        return {
+            kind: "held",
+            why: `the lock's holder is not readable (${text.trim().slice(0, 200) || "empty"})`,
+            manualRemoval: true,
+        };
+    }
+    const who = holderText(body);
+    if (!body.execution)
+        return judgeProcessLock(body, who, now, here);
+    const leaseUntil = body.leaseUntil ? Date.parse(body.leaseUntil) : Number.NaN;
+    const state = describe(body.execution);
+    switch (state.kind) {
+        case "finished":
+            return {
+                kind: "break",
+                why: `${who} has ${state.how}, but its lock was left behind`,
+            };
+        case "running":
+            return { kind: "held", why: `${who} is still running`, manualRemoval: false };
+        case "not-found":
+            if (Number.isNaN(leaseUntil)) {
+                return {
+                    kind: "held",
+                    why: `${who} is unknown to Cloud Run and its lock carries no readable lease`,
+                    manualRemoval: true,
+                };
+            }
+            if (now.getTime() > leaseUntil) {
+                return {
+                    kind: "break",
+                    why: `${who} is unknown to Cloud Run and its lease ended at ${body.leaseUntil}`,
+                };
+            }
+            return {
+                kind: "held",
+                why: `${who} is unknown to Cloud Run, but its lease runs until ${body.leaseUntil}`,
+                manualRemoval: false,
+            };
+        case "unknown":
+            return {
+                kind: "unknown",
+                why: `could not check whether ${who} is still running: ${state.error}`,
+            };
+    }
+}
+/**
+ * A process holder's lock (a CLI run): broken when its pid is dead on this
+ * very host, or once its lease — renewed while the holder runs — expired.
+ */
+function judgeProcessLock(body, who, now, here) {
+    if (body.pid !== undefined && sameMachine(body, here) && !here.alive(body.pid)) {
+        return { kind: "break", why: `${who} has ended (its process is gone on this machine)` };
+    }
+    if (body.leaseUntil === undefined) {
+        // Written before #815: no lease to wait out, and its host may be
+        // another machine — only a person may remove it.
+        return { kind: "held", why: `held by ${who}`, manualRemoval: true };
+    }
+    const leaseUntil = Date.parse(body.leaseUntil);
+    if (Number.isNaN(leaseUntil)) {
+        return {
+            kind: "held",
+            why: `held by ${who}, whose lock carries no readable lease`,
+            manualRemoval: true,
+        };
+    }
+    if (now.getTime() > leaseUntil) {
+        return {
+            kind: "break",
+            why: `${who} stopped renewing its lock; its lease ended at ${body.leaseUntil}`,
+        };
+    }
+    return {
+        kind: "held",
+        why: `held by ${who}; its lease runs until ${body.leaseUntil} unless renewed`,
+        manualRemoval: false,
+    };
+}
+/**
+ * Whether the lock was taken where `here` can check its pid. A lock naming
+ * a machine id needs this run's to match; one naming none (Cloud Run)
+ * never matches; a lock from before machine ids compares host names.
+ */
+function sameMachine(body, here) {
+    if (body.host !== here.host)
+        return false;
+    // A judge in Cloud Run shares its host name with every other container.
+    if (here.machine === null)
+        return false;
+    if (body.machine === undefined)
+        return true;
+    return body.machine !== null && here.machine != null && body.machine === here.machine;
+}
+/** "caelo upgrader job (run X, started T)" — for messages. */
+function holderText(body) {
+    const parts = [];
+    if (body.execution)
+        parts.push(`run ${body.execution.name}`);
+    else if (body.host)
+        parts.push(`on ${body.host}${body.pid ? `, pid ${body.pid}` : ""}`);
+    if (body.startedAt)
+        parts.push(`started ${body.startedAt}`);
+    return parts.length > 0 ? `${body.purpose} (${parts.join(", ")})` : body.purpose;
+}
+//# sourceMappingURL=state-lock-lease.js.map

package/dist/state-lock-lease.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"state-lock-lease.js","sourceRoot":"","sources":["../src/state-lock-lease.ts"],"names":[],"mappings":"AAAA,mCAAmC;AAEnC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AAEH,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,QAAQ,EAAE,iBAAiB,EAAE,MAAM,SAAS,CAAC;AACtD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAgCxB,oEAAoE;AACpE,MAAM,CAAC,MAAM,gBAAgB,GAAG,EAAE,GAAG,MAAM,CAAC;AAE5C;;;;GAIG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,GAAG,MAAM,CAAC;AAExC,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC5B,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACzB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;CACxB,CAAC,CAAC;AAEH,iFAAiF;AACjF,MAAM,cAAc,GAAG,CAAC,CAAC,MAAM,CAAC;IAC9B,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;IACnB,4EAA4E;IAC5E,EAAE,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IACzB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC3B,0EAA0E;IAC1E,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE;IACzC,GAAG,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC1B,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAChC,SAAS,EAAE,kBAAkB,CAAC,QAAQ,EAAE;IACxC,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;CAClC,CAAC,CAAC;AAKH;;;;;;;;GAQG;AACH,MAAM,UAAU,QAAQ,CACtB,OAAe,EACf,GAAS,EACT,MAAmB,EACnB,KAAa,UAAU,EAAE;IAEzB,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,IAAI,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC/F,MAAM,IAAI,GAAa;QACrB,OAAO;QACP,EAAE;QACF,IAAI,EAAE,SAAS,CAAC,IAAI;QACpB,OAAO,EAAE,SAAS,CAAC,OAAO,IAAI,IAAI;QAClC,GAAG,EAAE,OAAO,CAAC,GAAG;QAChB,SAAS,EAAE,GAAG,CAAC,WAAW,EAAE;QAC5B,GAAG,CAAC,MAAM,IAAI,WAAW,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,EAAE,GAAG,MAAM,CAAC,SAAS,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAClF,GAAG,CAAC,OAAO,KAAK,IAAI;YAClB,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,OAAO,GAAG,aAAa,CAAC,CAAC,WAAW,EAAE,EAAE;YACjF,CAAC,CAAC,EAAE,CAAC;KACR,CAAC;IACF,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;AAC9B,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,IAAc,EAAE,GAAS,EAAE,OAAe;IACxE,OAAO,IAAI,CAAC,SAAS,CAAC;QACpB,GAAG,IAAI;QACP,UAAU,EAAE,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,OAAO,GAAG,aAAa,CAAC,CAAC,WAAW,EAAE;KAC5E,CAAC,CAAC;AACL,CAAC;AAED,sFAAsF;AACtF,MAAM,UAAU,YAAY,CAAC,GAAW;IACtC,IAAI,CAAC;QACH,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC;QACX,+CAA+C;QAC/C,OAAQ,CAA2B,CAAC,IAAI,KAAK,OAAO,CAAC;IACvD,CAAC;AACH,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,SAAS,CAAC,MAAyB,OAAO,CAAC,GAAG;IAC5D,IAAI,GAAG,CAAC,mBAAmB,IAAI,GAAG,CAAC,SAAS;QAAE,OAAO,IAAI,CAAC;IAC1D,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,CAAC,iBAAiB,EAAE,CAAC,EAAE,CAAC;QACvD,KAAK,MAAM,IAAI,IAAI,KAAK,IAAI,EAAE,EAAE,CAAC;YAC/B,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,KAAK,mBAAmB;gBAAE,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACzF,CAAC;IACH,CAAC;IACD,IAAI,IAAI,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACjC,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,IAAI,CAAC;QACH,QAAQ,GAAG,YAAY,CAAC,mBAAmB,CAAC,CAAC;IAC/C,CAAC;IAAC,MAAM,CAAC;QACP,yEAAyE;IAC3E,CAAC;IACD,OAAO,UAAU,CAAC,QAAQ,CAAC;SACxB,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,QAAQ,EAAE,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SAC9D,MAAM,CAAC,KAAK,CAAC;SACb,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAClB,CAAC;AAcD,MAAM,SAAS,GAAc,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,OAAO,EAAE,SAAS,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC;AAE7F,kEAAkE;AAClE,MAAM,UAAU,aAAa,CAAC,IAAY;IACxC,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACzB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,MAAM,GAAG,cAAc,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IAC7C,OAAO,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7C,CAAC;AAUD,MAAM,SAAS,GAAG,wDAAwD,CAAC;AAE3E,MAAM,qBAAqB,GAAG,CAAC,CAAC,MAAM,CAAC;IACrC,MAAM,EAAE,CAAC;SACN,MAAM,CAAC;QACN,cAAc,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QACrC,UAAU,EAAE,CAAC;aACV,KAAK,CACJ,CAAC,CAAC,MAAM,CAAC;YACP,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;YAChB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE;YAClB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;SAC9B,CAAC,CACH;aACA,QAAQ,EAAE;KACd,CAAC;SACD,QAAQ,EAAE;CACd,CAAC,CAAC;AAEH;;;;;;GAMG;AACH,MAAM,UAAU,mBAAmB,CAAC,MAAoB;IACtD,IAAI,CAAC,MAAM,CAAC,EAAE,EAAE,CAAC;QACf,IAAI,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;YAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC;QAChE,OAAO;YACL,IAAI,EAAE,SAAS;YACf,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,IAAI,iBAAiB,MAAM,CAAC,QAAQ,EAAE;SAClE,CAAC;IACJ,CAAC;IACD,IAAI,GAAY,CAAC;IACjB,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAClC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,2CAA2C,EAAE,CAAC;IACjF,CAAC;IACD,MAAM,MAAM,GAAG,qBAAqB,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;IACpD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,wDAAwD,EAAE,CAAC;IAC9F,CAAC;IACD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC;IAClC,MAAM,SAAS,GAAG,MAAM,EAAE,UAAU,EAAE,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,WAAW,CAAC,CAAC;IAC1E,IAAI,SAAS,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,MAAM,IAAI,SAAS,CAAC,MAAM,KAAK,OAAO,CAAC,EAAE,CAAC;QAC/E,MAAM,GAAG,GACP,SAAS,CAAC,MAAM,KAAK,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,IAAI,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;QAC3F,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,CAAC;IACnC,CAAC;IACD,IAAI,MAAM,EAAE,cAAc;QAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,GAAG,EAAE,WAAW,EAAE,CAAC;IAC1E,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;AAC7B,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,qBAAqB,CAAC,GAAyB;IAC7D,OAAO;QACL,KAAK;QACL,MAAM;QACN,YAAY;QACZ,UAAU;QACV,GAAG,CAAC,IAAI;QACR,YAAY,GAAG,CAAC,MAAM,EAAE;QACxB,aAAa,GAAG,CAAC,SAAS,EAAE;QAC5B,eAAe;KAChB,CAAC;AACJ,CAAC;AASD;;;;;;;GAOG;AACH,MAAM,UAAU,aAAa,CAC3B,IAAY,EACZ,QAAuD,EACvD,GAAS,EACT,OAAkB,SAAS;IAE3B,MAAM,IAAI,GAAG,aAAa,CAAC,IAAI,CAAC,CAAC;IACjC,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,GAAG,EAAE,sCAAsC,IAAI,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,OAAO,GAAG;YAClF,aAAa,EAAE,IAAI;SACpB,CAAC;IACJ,CAAC;IACD,MAAM,GAAG,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAC7B,IAAI,CAAC,IAAI,CAAC,SAAS;QAAE,OAAO,gBAAgB,CAAC,IAAI,EAAE,GAAG,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;IACnE,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;IAC9E,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IACvC,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,KAAK,UAAU;YACb,OAAO;gBACL,IAAI,EAAE,OAAO;gBACb,GAAG,EAAE,GAAG,GAAG,QAAQ,KAAK,CAAC,GAAG,gCAAgC;aAC7D,CAAC;QACJ,KAAK,SAAS;YACZ,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,GAAG,mBAAmB,EAAE,aAAa,EAAE,KAAK,EAAE,CAAC;QAChF,KAAK,WAAW;YACd,IAAI,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC;gBAC7B,OAAO;oBACL,IAAI,EAAE,MAAM;oBACZ,GAAG,EAAE,GAAG,GAAG,iEAAiE;oBAC5E,aAAa,EAAE,IAAI;iBACpB,CAAC;YACJ,CAAC;YACD,IAAI,GAAG,CAAC,OAAO,EAAE,GAAG,UAAU,EAAE,CAAC;gBAC/B,OAAO;oBACL,IAAI,EAAE,OAAO;oBACb,GAAG,EAAE,GAAG,GAAG,mDAAmD,IAAI,CAAC,UAAU,EAAE;iBAChF,CAAC;YACJ,CAAC;YACD,OAAO;gBACL,IAAI,EAAE,MAAM;gBACZ,GAAG,EAAE,GAAG,GAAG,sDAAsD,IAAI,CAAC,UAAU,EAAE;gBAClF,aAAa,EAAE,KAAK;aACrB,CAAC;QACJ,KAAK,SAAS;YACZ,OAAO;gBACL,IAAI,EAAE,SAAS;gBACf,GAAG,EAAE,2BAA2B,GAAG,sBAAsB,KAAK,CAAC,KAAK,EAAE;aACvE,CAAC;IACN,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,SAAS,gBAAgB,CAAC,IAAc,EAAE,GAAW,EAAE,GAAS,EAAE,IAAe;IAC/E,IAAI,IAAI,CAAC,GAAG,KAAK,SAAS,IAAI,WAAW,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QAC/E,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,GAAG,kDAAkD,EAAE,CAAC;IAC1F,CAAC;IACD,IAAI,IAAI,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;QAClC,iEAAiE;QACjE,iDAAiD;QACjD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,WAAW,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC;IACtE,CAAC;IACD,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC/C,IAAI,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC;QAC7B,OAAO;YACL,IAAI,EAAE,MAAM;YACZ,GAAG,EAAE,WAAW,GAAG,wCAAwC;YAC3D,aAAa,EAAE,IAAI;SACpB,CAAC;IACJ,CAAC;IACD,IAAI,GAAG,CAAC,OAAO,EAAE,GAAG,UAAU,EAAE,CAAC;QAC/B,OAAO;YACL,IAAI,EAAE,OAAO;YACb,GAAG,EAAE,GAAG,GAAG,kDAAkD,IAAI,CAAC,UAAU,EAAE;SAC/E,CAAC;IACJ,CAAC;IACD,OAAO;QACL,IAAI,EAAE,MAAM;QACZ,GAAG,EAAE,WAAW,GAAG,0BAA0B,IAAI,CAAC,UAAU,iBAAiB;QAC7E,aAAa,EAAE,KAAK;KACrB,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,SAAS,WAAW,CAAC,IAAc,EAAE,IAAe;IAClD,IAAI,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI;QAAE,OAAO,KAAK,CAAC;IAC1C,wEAAwE;IACxE,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI;QAAE,OAAO,KAAK,CAAC;IACxC,IAAI,IAAI,CAAC,OAAO,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IAC5C,OAAO,IAAI,CAAC,OAAO,KAAK,IAAI,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,CAAC,OAAO,CAAC;AACxF,CAAC;AAED,8DAA8D;AAC9D,SAAS,UAAU,CAAC,IAAc;IAChC,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,IAAI,CAAC,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;SACxD,IAAI,IAAI,CAAC,IAAI;QAAE,KAAK,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IACxF,IAAI,IAAI,CAAC,SAAS;QAAE,KAAK,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC;IAC5D,OAAO,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,OAAO,KAAK,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC;AACnF,CAAC"}

package/dist/state-store.d.ts

@@ -1,4 +1,5 @@
 import { type GcloudResult } from "./gcloud.js";
+import { type ExecutionLockHolder, type LocalHost } from "./state-lock-lease.js";
 /** The two documents a store holds. */
 export type StateFile = "install.json" | "progress.json";
 /** Reads and writes an install's metadata documents. */
@@ -62,29 +63,85 @@ export declare class StateLockHeldError extends Error {
 }
 /** `gs://<bucket>/install/state.lock` — see {@link acquireBucketStateLock}. */
 export declare function bucketStateLockUrl(bucket: string): string;
+/** How a bucket lock is taken: who holds it, and the clock to judge leases by. */
+export interface BucketLockOptions {
+    /**
+     * The Cloud Run execution taking the lock (#815): the lock's lease is the
+     * execution's task timeout, and a later run may break it once Cloud Run
+     * says this execution has finished. Without it the holder is this
+     * process (a CLI run): the lock names host + pid, carries a lease of
+     * {@link PROCESS_LEASE_MS} and is renewed by a heartbeat until released.
+     */
+    readonly holder?: ExecutionLockHolder;
+    /** Told when a dead holder's lock was broken, with why. */
+    readonly onBreak?: (message: string) => void;
+    /**
+     * Told when the heartbeat found the lock no longer this run's (another
+     * run broke it after the lease ran out, e.g. while this machine slept).
+     */
+    readonly onLost?: (message: string) => void;
+    readonly now?: () => Date;
+    /** Starts the heartbeat; returns its stop. Defaults to an unref'd setInterval. */
+    readonly schedule?: (tick: () => void, everyMs: number) => () => void;
+    /** This host and its pid check (state-lock-lease.ts LocalHost); tests pass a fake. */
+    readonly here?: LocalHost;
+}
 /**
  * The install's state lock in its ops bucket, created only if absent
  * (`--if-generation-match=0`). Held by every writer of a bucket-backed
- * install's state — an installer run's `pulumi up`, `state migrate` — so
- * none writes a backend another is switching away from. A lock left by a
- * crashed run is reported with the command to remove it, never broken
- * automatically.
+ * install's state — an installer run's `pulumi up`, `state migrate`, the
+ * CLI `upgrade`, the upgrader job — so none writes a backend another is
+ * switching away from.
  *
- * @returns the release function
+ * A lock that is held is judged (state-lock-lease.ts): one whose Cloud Run
+ * execution has finished, whose execution is gone after its lease, whose
+ * CLI process is gone on this host, or whose CLI lease ran out unrenewed,
+ * is broken with a generation-matched delete and taken once more. The
+ * generation is read BEFORE the body, so the delete can only remove the
+ * lock that was judged: if another run replaced it in between, the
+ * delete's precondition fails and nothing is removed. Two runs breaking the
+ * same dead lock cannot both win either — the slower one's create fails
+ * its own precondition and reports the lock as held.
+ *
+ * A CLI holder renews its lease every third of {@link PROCESS_LEASE_MS}
+ * with a generation-matched rewrite, so a run that is alive keeps its lock
+ * however long it takes, and a killed one loses it within one lease.
+ *
+ * @returns the release function; it stops the heartbeat and deletes only
+ *   this lock's generation
+ * @throws StateLockHeldError when another run holds the lock
+ * @throws Error when the lock or its holder cannot be read (no access)
  */
-export declare function acquireBucketStateLock(bucket: string, purpose: string, run?: SyncGcloud): () => void;
+export declare function acquireBucketStateLock(bucket: string, purpose: string, run?: SyncGcloud, opts?: BucketLockOptions): () => void;
 /**
  * The same lock for an install whose state is on this machine:
- * `~/.caelo-<id>/state.lock`, created exclusively.
+ * `~/.caelo-<id>/state.lock`, created exclusively. A lock left by a
+ * process that is gone on this machine is taken over (#815).
  *
+ * @param here this host and its pid check; tests pass a fake
  * @returns the release function
  */
-export declare function acquireLocalStateLock(root: string, purpose: string): () => void;
+export declare function acquireLocalStateLock(root: string, purpose: string, here?: LocalHost): () => void;
 /** The state lock for whichever backend `bucket` (gcs) or `root` (local) names. */
 export declare function acquireStateLock(backend: {
     readonly kind: "local";
 } | {
     readonly kind: "gcs";
     readonly bucket: string;
-}, root: string, purpose: string, run?: SyncGcloud): () => void;
+}, root: string, purpose: string, run?: SyncGcloud, opts?: BucketLockOptions): () => void;
+/** The part of `process` {@link releaseOnSignal} needs; tests pass a fake. */
+export interface SignalTarget {
+    once(signal: "SIGINT" | "SIGTERM", listener: () => void): unknown;
+    removeListener(signal: "SIGINT" | "SIGTERM", listener: () => void): unknown;
+    exit(code: number): never;
+}
+/**
+ * Give the state lock back when the CLI is stopped (Ctrl-C, a closed Cloud
+ * Shell tab's SIGTERM), then exit with the signal's conventional code
+ * (#815). Without this a stopped run leaves its lock until its lease ends.
+ *
+ * @param report told what happened to the lock
+ * @returns removes the handlers; call it once the run released the lock itself
+ */
+export declare function releaseOnSignal(release: () => void, report: (message: string) => void, proc?: SignalTarget): () => void;
 //# sourceMappingURL=state-store.d.ts.map

package/dist/state-store.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"state-store.d.ts","sourceRoot":"","sources":["../src/state-store.ts"],"names":[],"mappings":"AA4BA,OAAO,EAAE,KAAK,YAAY,EAAa,MAAM,aAAa,CAAC;AAG3D,uCAAuC;AACvC,MAAM,MAAM,SAAS,GAAG,cAAc,GAAG,eAAe,CAAC;AAEzD,wDAAwD;AACxD,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,OAAO,GAAG,KAAK,CAAC;IAC/B,2DAA2D;IAC3D,IAAI,CAAC,IAAI,EAAE,SAAS,GAAG,MAAM,GAAG,IAAI,CAAC;IACrC,qDAAqD;IACrD,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,GAAG,IAAI,CAAC;CAC9C;AAED,mFAAmF;AACnF,MAAM,MAAM,UAAU,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,KAAK,CAAC,EAAE,MAAM,KAAK,YAAY,CAAC;AAE1E,mEAAmE;AACnE,eAAO,MAAM,cAAc,EAAE,UAc5B,CAAC;AAEF;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,GAAG,MAAM,CAIxE;AAcD,wDAAwD;AACxD,qBAAa,eAAgB,YAAW,UAAU;IAEpC,OAAO,CAAC,QAAQ,CAAC,IAAI;IADjC,QAAQ,CAAC,IAAI,EAAG,OAAO,CAAU;gBACJ,IAAI,EAAE,MAAM;IAEzC,IAAI,CAAC,IAAI,EAAE,SAAS,GAAG,MAAM,GAAG,IAAI;IAKpC,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,GAAG,IAAI;CAG7C;AAKD;;;;;;GAMG;AACH,qBAAa,aAAc,YAAW,UAAU;IAK5C,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,GAAG;IANtB,QAAQ,CAAC,IAAI,EAAG,KAAK,CAAU;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAuC;gBAG1C,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EACZ,GAAG,GAAE,UAA2B;IAGnD,sCAAsC;IACtC,SAAS,CAAC,IAAI,EAAE,SAAS,GAAG,MAAM;IAIlC;;;;OAIG;IACH,IAAI,CAAC,IAAI,EAAE,SAAS,GAAG,MAAM,GAAG,IAAI;IASpC,IAAI,CAAC,IAAI,EAAE,SAAS,GAAG,MAAM,GAAG,IAAI;IAQpC,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,GAAG,IAAI;CAc7C;AAED;;;;GAIG;AACH,qBAAa,kBAAmB,SAAQ,KAAK;IAC3C,SAAkB,IAAI,wBAAwB;CAC/C;AAED,+EAA+E;AAC/E,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAEzD;AAYD;;;;;;;;;GASG;AACH,wBAAgB,sBAAsB,CACpC,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,EACf,GAAG,GAAE,UAA2B,GAC/B,MAAM,IAAI,CAgBZ;AAED;;;;;GAKG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,MAAM,IAAI,CAa/E;AAED,mFAAmF;AACnF,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE;IAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;CAAE,GAAG;IAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CAAE,EACvF,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,GAAG,GAAE,UAA2B,GAC/B,MAAM,IAAI,CAIZ"}
+{"version":3,"file":"state-store.d.ts","sourceRoot":"","sources":["../src/state-store.ts"],"names":[],"mappings":"AA4BA,OAAO,EAAE,KAAK,YAAY,EAAa,MAAM,aAAa,CAAC;AAE3D,OAAO,EAEL,KAAK,mBAAmB,EAExB,KAAK,SAAS,EAOf,MAAM,uBAAuB,CAAC;AAE/B,uCAAuC;AACvC,MAAM,MAAM,SAAS,GAAG,cAAc,GAAG,eAAe,CAAC;AAEzD,wDAAwD;AACxD,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,OAAO,GAAG,KAAK,CAAC;IAC/B,2DAA2D;IAC3D,IAAI,CAAC,IAAI,EAAE,SAAS,GAAG,MAAM,GAAG,IAAI,CAAC;IACrC,qDAAqD;IACrD,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,GAAG,IAAI,CAAC;CAC9C;AAED,mFAAmF;AACnF,MAAM,MAAM,UAAU,GAAG,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,KAAK,CAAC,EAAE,MAAM,KAAK,YAAY,CAAC;AAE1E,mEAAmE;AACnE,eAAO,MAAM,cAAc,EAAE,UAc5B,CAAC;AAEF;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,GAAG,MAAM,CAIxE;AAcD,wDAAwD;AACxD,qBAAa,eAAgB,YAAW,UAAU;IAEpC,OAAO,CAAC,QAAQ,CAAC,IAAI;IADjC,QAAQ,CAAC,IAAI,EAAG,OAAO,CAAU;gBACJ,IAAI,EAAE,MAAM;IAEzC,IAAI,CAAC,IAAI,EAAE,SAAS,GAAG,MAAM,GAAG,IAAI;IAKpC,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,GAAG,IAAI;CAG7C;AAKD;;;;;;GAMG;AACH,qBAAa,aAAc,YAAW,UAAU;IAK5C,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,GAAG;IANtB,QAAQ,CAAC,IAAI,EAAG,KAAK,CAAU;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAuC;gBAG1C,MAAM,EAAE,MAAM,EACd,IAAI,EAAE,MAAM,EACZ,GAAG,GAAE,UAA2B;IAGnD,sCAAsC;IACtC,SAAS,CAAC,IAAI,EAAE,SAAS,GAAG,MAAM;IAIlC;;;;OAIG;IACH,IAAI,CAAC,IAAI,EAAE,SAAS,GAAG,MAAM,GAAG,IAAI;IASpC,IAAI,CAAC,IAAI,EAAE,SAAS,GAAG,MAAM,GAAG,IAAI;IAQpC,KAAK,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,GAAG,IAAI;CAc7C;AAED;;;;GAIG;AACH,qBAAa,kBAAmB,SAAQ,KAAK;IAC3C,SAAkB,IAAI,wBAAwB;CAC/C;AAED,+EAA+E;AAC/E,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAEzD;AAKD,kFAAkF;AAClF,MAAM,WAAW,iBAAiB;IAChC;;;;;;OAMG;IACH,QAAQ,CAAC,MAAM,CAAC,EAAE,mBAAmB,CAAC;IACtC,2DAA2D;IAC3D,QAAQ,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAC7C;;;OAGG;IACH,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IAC5C,QAAQ,CAAC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IAC1B,kFAAkF;IAClF,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,IAAI,EAAE,OAAO,EAAE,MAAM,KAAK,MAAM,IAAI,CAAC;IACtE,sFAAsF;IACtF,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,CAAC;CAC3B;AASD;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AACH,wBAAgB,sBAAsB,CACpC,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,EACf,GAAG,GAAE,UAA2B,EAChC,IAAI,GAAE,iBAAsB,GAC3B,MAAM,IAAI,CAwEZ;AAwJD;;;;;;;GAOG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,SAAS,GAAG,MAAM,IAAI,CA8BjG;AAED,mFAAmF;AACnF,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE;IAAE,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAA;CAAE,GAAG;IAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC;IAAC,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAA;CAAE,EACvF,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,MAAM,EACf,GAAG,GAAE,UAA2B,EAChC,IAAI,GAAE,iBAAsB,GAC3B,MAAM,IAAI,CAIZ;AAED,8EAA8E;AAC9E,MAAM,WAAW,YAAY;IAC3B,IAAI,CAAC,MAAM,EAAE,QAAQ,GAAG,SAAS,EAAE,QAAQ,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC;IAClE,cAAc,CAAC,MAAM,EAAE,QAAQ,GAAG,SAAS,EAAE,QAAQ,EAAE,MAAM,IAAI,GAAG,OAAO,CAAC;IAC5E,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,KAAK,CAAC;CAC3B;AAED;;;;;;;GAOG;AACH,wBAAgB,eAAe,CAC7B,OAAO,EAAE,MAAM,IAAI,EACnB,MAAM,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,EACjC,IAAI,GAAE,YAAsB,GAC3B,MAAM,IAAI,CAoBZ"}