@indigoai-us/hq-cloud 6.18.44-alpha.48 → 6.18.44-alpha.49

package/dist/cli/rescue-breaker.test.js

@@ -0,0 +1,812 @@
+/**
+ * Writability preflight, recovery-snapshot circuit breaker recovery, and
+ * `.hq/rescue-blocked.json` visibility for `hq rescue`.
+ *
+ * Production (2026-09-16): six agent boxes froze on hq-core releases from
+ * 15 Aug–7 Sep because `core/scripts` was root-owned. Drift-rescue
+ * `renameSync(core/scripts/qmd-reindex-bg.sh → personal/scripts/...)` failed
+ * EACCES twice, the safety snapshot was already allocated, and the breaker
+ * then held forever with no heartbeat signal.
+ */
+import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
+import { execFileSync } from "child_process";
+import * as fs from "fs";
+import * as os from "os";
+import * as path from "path";
+import { canonicalRootHash } from "../backup-prune.js";
+import { collectUnwritableRescueMutationPaths, parseRescueArgs, requireBackupCapacity, runRescue, } from "./rescue-core.js";
+function has(bin, ...args) {
+    try {
+        execFileSync(bin, args, { stdio: "ignore" });
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+const toolsAvailable = has("git", "--version") && has("rsync", "--version");
+const canChmodDenyWrite = process.platform !== "win32" &&
+    (typeof process.getuid !== "function" || process.getuid() !== 0);
+function runRescueCapture(argv, env, opts = {}) {
+    let stdout = "";
+    let stderr = "";
+    const origOut = process.stdout.write.bind(process.stdout);
+    const origErr = process.stderr.write.bind(process.stderr);
+    process.stdout.write = ((chunk) => {
+        stdout += String(chunk);
+        return true;
+    });
+    process.stderr.write = ((chunk) => {
+        stderr += String(chunk);
+        return true;
+    });
+    let status;
+    let threw;
+    try {
+        status =
+            runRescue(argv, {
+                env,
+                backupCapacityStatfsSync: (() => ({
+                    bavail: 1_000_000_000_000,
+                    bsize: 1,
+                    blocks: 1_000_000_000_000,
+                })),
+                ...opts,
+            }).status;
+    }
+    catch (e) {
+        threw = e;
+    }
+    finally {
+        process.stdout.write = origOut;
+        process.stderr.write = origErr;
+    }
+    return { status, stdout, stderr, threw };
+}
+describe("parseRescueArgs --recover-snapshots", () => {
+    it("defaults off and accepts the flag", () => {
+        expect(parseRescueArgs([]).recoverSnapshots).toBe(false);
+        expect(parseRescueArgs(["--recover-snapshots"]).recoverSnapshots).toBe(true);
+    });
+});
+describe("requireBackupCapacity", () => {
+    const gib = 1024 ** 3;
+    const gb = 1000 ** 3;
+    function checkCapacity(estimate, capacity, free, preserveBytes = 0) {
+        const root = fs.mkdtempSync(path.join(os.tmpdir(), "hq-rescue-capacity-"));
+        const hqRoot = path.join(root, "hq");
+        fs.mkdirSync(path.join(hqRoot, "core"), { recursive: true });
+        const estimateFile = path.join(hqRoot, "core", "estimate.bin");
+        fs.writeFileSync(estimateFile, "");
+        fs.truncateSync(estimateFile, estimate - preserveBytes);
+        const preserveSubpaths = preserveBytes > 0 ? ["core/packages"] : [];
+        if (preserveBytes > 0) {
+            const preserveFile = path.join(hqRoot, "core", "packages", "kept.bin");
+            fs.mkdirSync(path.dirname(preserveFile), { recursive: true });
+            fs.writeFileSync(preserveFile, "");
+            fs.truncateSync(preserveFile, preserveBytes);
+        }
+        let output = "";
+        const statfs = (() => ({
+            bavail: free,
+            bsize: 1,
+            blocks: capacity,
+        }));
+        try {
+            requireBackupCapacity({
+                backupRoot: path.join(root, "backups"),
+                preserveSubpaths,
+            }, hqRoot, ["core"], hqRoot, {}, (line) => {
+                output += line;
+            }, statfs, () => false);
+            return { output, refused: false };
+        }
+        catch (error) {
+            if (!(error instanceof Error) || error.message !== "exit 1")
+                throw error;
+            return { output, refused: true };
+        }
+        finally {
+            fs.rmSync(root, { recursive: true, force: true });
+        }
+    }
+    it("allows a 3 GiB estimate on a 500 GB disk with 15.7 GB free", () => {
+        const result = checkCapacity(3 * gib, 500 * gb, 15.7 * gb);
+        expect(result.refused).toBe(false);
+        expect(result.output).toBe("");
+    });
+    it("refuses a 1 GiB estimate with only 4 GB free and keeps disk_full", () => {
+        const result = checkCapacity(1 * gib, 64 * gb, 4 * gb);
+        expect(result.refused).toBe(true);
+        expect(result.output).toContain("error: insufficient free space for safety snapshot (need ");
+        expect(result.output).toContain("HQ_RESCUE_FAILURE_KIND=disk_full");
+    });
+    it("keeps at least the 5 GiB margin on a 64 GB disk", () => {
+        const result = checkCapacity(1 * gib, 64 * gb, 7 * gb);
+        expect(result.refused).toBe(true);
+        expect(result.output).toContain("HQ_RESCUE_FAILURE_KIND=disk_full");
+    });
+    it("uses the 5 GiB floor when 10% of capacity is lower", () => {
+        const result = checkCapacity(1 * gib, 32 * gb, 6 * gib - 1);
+        expect(result.refused).toBe(true);
+        expect(result.output).toContain(`need ${1 * gib + 5 * gib} bytes`);
+    });
+    it("refuses when the estimate alone exceeds free space", () => {
+        const result = checkCapacity(8 * gib, 64 * gb, 7 * gib);
+        expect(result.refused).toBe(true);
+        expect(result.output).toContain("HQ_RESCUE_FAILURE_KIND=disk_full");
+    });
+    it("includes a preserved subtree copied to a same-filesystem shuttle", () => {
+        const result = checkCapacity(30 * gib, 500 * gib, 45 * gib, 20 * gib);
+        expect(result.refused).toBe(true);
+        expect(result.output).toContain("need 64424509440 bytes");
+        expect(result.output).toContain("HQ_RESCUE_FAILURE_KIND=disk_full");
+    });
+});
+describe("collectUnwritableRescueMutationPaths", () => {
+    it.skipIf(!canChmodDenyWrite)("fails closed on a read-only parent dir", () => {
+        const root = fs.mkdtempSync(path.join(os.tmpdir(), "hq-rescue-preflight-ro-"));
+        const parent = path.join(root, "core", "scripts");
+        const file = path.join(parent, "qmd-reindex-bg.sh");
+        fs.mkdirSync(parent, { recursive: true });
+        fs.mkdirSync(path.join(root, "personal"), { recursive: true });
+        fs.writeFileSync(file, "local-edit\n");
+        fs.chmodSync(parent, 0o555);
+        try {
+            const failed = collectUnwritableRescueMutationPaths(root, [
+                {
+                    affectedRels: ["core/scripts/qmd-reindex-bg.sh"],
+                    destParentRels: ["personal/scripts"],
+                },
+            ]);
+            expect(failed).toContain("core/scripts");
+            expect(failed.join("\n")).not.toMatch(/personal/);
+        }
+        finally {
+            fs.chmodSync(parent, 0o755);
+            fs.rmSync(root, { recursive: true, force: true });
+        }
+    });
+    it("passes on a writable tree", () => {
+        const root = fs.mkdtempSync(path.join(os.tmpdir(), "hq-rescue-preflight-rw-"));
+        try {
+            const parent = path.join(root, "core", "scripts");
+            fs.mkdirSync(parent, { recursive: true });
+            fs.mkdirSync(path.join(root, "personal"), { recursive: true });
+            fs.writeFileSync(path.join(parent, "qmd-reindex-bg.sh"), "local-edit\n");
+            expect(collectUnwritableRescueMutationPaths(root, [
+                {
+                    affectedRels: ["core/scripts/qmd-reindex-bg.sh"],
+                    destParentRels: ["personal/scripts"],
+                },
+            ])).toEqual([]);
+        }
+        finally {
+            fs.rmSync(root, { recursive: true, force: true });
+        }
+    });
+});
+describe.skipIf(!toolsAvailable)("rescue recovery-snapshot breaker + mutation preflight", () => {
+    let workDir;
+    let upstream;
+    let shimDir;
+    let floorSha;
+    let baseEnv;
+    let caseSeq = 0;
+    const git = (cwd, ...args) => execFileSync("git", args, {
+        cwd,
+        stdio: ["ignore", "pipe", "pipe"],
+        env: {
+            ...process.env,
+            GIT_AUTHOR_NAME: "t",
+            GIT_AUTHOR_EMAIL: "t@t",
+            GIT_COMMITTER_NAME: "t",
+            GIT_COMMITTER_EMAIL: "t@t",
+        },
+    })
+        .toString()
+        .trim();
+    beforeAll(() => {
+        workDir = fs.mkdtempSync(path.join(os.tmpdir(), "hq-rescue-breaker-"));
+        upstream = path.join(workDir, "upstream");
+        fs.mkdirSync(path.join(upstream, "core", "scripts"), { recursive: true });
+        git(workDir, "init", "-b", "main", "upstream");
+        fs.writeFileSync(path.join(upstream, "core/a.md"), "v1\n");
+        fs.writeFileSync(path.join(upstream, "core/scripts/qmd-reindex-bg.sh"), "release-v1\n");
+        git(upstream, "add", "-A");
+        git(upstream, "commit", "-m", "floor");
+        floorSha = git(upstream, "rev-parse", "HEAD");
+        fs.writeFileSync(path.join(upstream, "core/a.md"), "v2\n");
+        fs.writeFileSync(path.join(upstream, "core/scripts/qmd-reindex-bg.sh"), "release-v2\n");
+        git(upstream, "add", "-A");
+        git(upstream, "commit", "-m", "head");
+        const realGit = execFileSync("bash", ["-lc", "command -v git"]).toString().trim() ||
+            "/usr/bin/git";
+        shimDir = path.join(workDir, "shim");
+        fs.mkdirSync(shimDir, { recursive: true });
+        const shim = `#!/usr/bin/env bash
+if [ "$1" = "clone" ] || [ "$1" = "ls-remote" ]; then
+  args=()
+  for a in "$@"; do
+    case "$a" in
+      https://github.com/*) a=${JSON.stringify(upstream)} ;;
+    esac
+    args+=("$a")
+  done
+  exec ${JSON.stringify(realGit)} "\${args[@]}"
+fi
+exec ${JSON.stringify(realGit)} "$@"
+`;
+        fs.writeFileSync(path.join(shimDir, "git"), shim, { mode: 0o755 });
+        baseEnv = { ...process.env, PATH: `${shimDir}:${process.env.PATH ?? ""}` };
+    }, 60_000);
+    afterAll(() => {
+        if (workDir)
+            fs.rmSync(workDir, { recursive: true, force: true });
+    });
+    function makeHqRoot() {
+        caseSeq += 1;
+        const hqRoot = path.join(workDir, `hq-${caseSeq}`);
+        fs.mkdirSync(path.join(hqRoot, "core", "scripts"), { recursive: true });
+        fs.mkdirSync(path.join(hqRoot, "companies"), { recursive: true });
+        fs.mkdirSync(path.join(hqRoot, "personal"), { recursive: true });
+        fs.writeFileSync(path.join(hqRoot, "core/a.md"), "v1\n");
+        fs.writeFileSync(path.join(hqRoot, "core/scripts/qmd-reindex-bg.sh"), "local-edit\n");
+        return hqRoot;
+    }
+    function liveArgs(hqRoot, backupRoot, extra = []) {
+        return [
+            "--hq-root",
+            hqRoot,
+            "--source",
+            "test/repo",
+            "--ref",
+            "main",
+            "--floor-sha",
+            floorSha,
+            "--paths",
+            "core",
+            "--backup-dir",
+            backupRoot,
+            "--yes",
+            ...extra,
+        ];
+    }
+    function plantRecoverySnapshots(hqRoot, backupRoot) {
+        const canonical = fs.realpathSync.native(hqRoot);
+        const hash = canonicalRootHash(canonical);
+        const names = [
+            `pre-update-2026-08-15T12-00-00Z-${hash}-one`,
+            `pre-update-2026-09-07T12-00-00Z-${hash}-two`,
+        ];
+        fs.mkdirSync(backupRoot, { recursive: true });
+        for (const name of names) {
+            const dir = path.join(backupRoot, name);
+            fs.mkdirSync(dir, { recursive: true });
+            const at = "2026-08-15T12:00:00.000Z";
+            const runId = name.slice(name.lastIndexOf("-") + 1);
+            fs.writeFileSync(path.join(dir, "SNAPSHOT.json"), JSON.stringify({
+                schemaVersion: 1,
+                status: "recovery-required",
+                canonicalHqRoot: canonical,
+                rootHash: hash,
+                runId,
+                creatorPid: 1,
+                creatorStartedAt: at,
+                source: "test/repo",
+                ref: "main",
+                sourceSha: "abc",
+                createdAt: at,
+                completedAt: at,
+                includedRoots: ["core"],
+                fileCount: 1,
+                byteCount: 1,
+            }) + "\n");
+            fs.writeFileSync(path.join(dir, "keep.txt"), "recovery evidence\n");
+        }
+        return names;
+    }
+    function plantProofSnapshots(hqRoot, backupRoot, sourceSha) {
+        const names = plantRecoverySnapshots(hqRoot, backupRoot);
+        for (const name of names) {
+            const dir = path.join(backupRoot, name);
+            const manifest = JSON.parse(fs.readFileSync(path.join(dir, "SNAPSHOT.json"), "utf8"));
+            manifest.sourceSha = sourceSha;
+            fs.cpSync(path.join(hqRoot, "core"), path.join(dir, "core"), {
+                recursive: true,
+            });
+            fs.writeFileSync(path.join(dir, "SNAPSHOT.json"), JSON.stringify(manifest) + "\n");
+        }
+        return names;
+    }
+    function makeProofHqRoot() {
+        const hqRoot = makeHqRoot();
+        fs.writeFileSync(path.join(hqRoot, "core/a.md"), "v2\n");
+        fs.writeFileSync(path.join(hqRoot, "core/scripts/qmd-reindex-bg.sh"), "release-v2\n");
+        fs.writeFileSync(path.join(hqRoot, "core/old.conflict-2026-test"), "discarded artifact\n");
+        return hqRoot;
+    }
+    function plantCompletedGoodSnapshot(hqRoot, backupRoot) {
+        const canonical = fs.realpathSync.native(hqRoot);
+        const hash = canonicalRootHash(canonical);
+        const name = `pre-update-2026-09-08T12-00-00Z-${hash}-good`;
+        const dir = path.join(backupRoot, name);
+        const at = "2026-09-08T12:00:00.000Z";
+        fs.mkdirSync(dir, { recursive: true });
+        fs.writeFileSync(path.join(dir, "SNAPSHOT.json"), JSON.stringify({
+            schemaVersion: 1,
+            status: "completed-good",
+            canonicalHqRoot: canonical,
+            rootHash: hash,
+            runId: "good",
+            creatorPid: 1,
+            creatorStartedAt: at,
+            source: "test/repo",
+            ref: "main",
+            sourceSha: "abc",
+            createdAt: at,
+            completedAt: at,
+            includedRoots: ["core"],
+            fileCount: 1,
+            byteCount: 1,
+        }) + "\n");
+    }
+    function plantRetentionSnapshots(hqRoot, backupRoot) {
+        const canonical = fs.realpathSync.native(hqRoot);
+        const hash = canonicalRootHash(canonical);
+        const make = (date, suffix, status) => {
+            const createdAt = new Date(date).toISOString();
+            const name = `pre-update-${createdAt.replaceAll(":", "-").replace(".000", "")}-${hash}-${suffix}`;
+            const dir = path.join(backupRoot, name);
+            fs.mkdirSync(dir, { recursive: true });
+            fs.mkdirSync(path.join(dir, "core"), { recursive: true });
+            fs.writeFileSync(path.join(dir, "core", "a.md"), `${suffix}\n`);
+            fs.writeFileSync(path.join(dir, "SNAPSHOT.json"), JSON.stringify({
+                schemaVersion: 1,
+                status,
+                canonicalHqRoot: canonical,
+                rootHash: hash,
+                runId: suffix,
+                creatorPid: 1,
+                creatorStartedAt: createdAt,
+                source: "test/repo",
+                ref: "main",
+                sourceSha: "abc",
+                createdAt,
+                completedAt: createdAt,
+                includedRoots: ["core"],
+                ...(status === "incomplete"
+                    ? {
+                        skippedEntries: [
+                            {
+                                relativePath: "core/missing.txt",
+                                operation: "read",
+                                code: "EACCES",
+                                message: "test fixture",
+                                occurredAt: createdAt,
+                            },
+                        ],
+                    }
+                    : {}),
+                fileCount: 1,
+                byteCount: 1,
+            }) + "\n");
+            return name;
+        };
+        fs.mkdirSync(backupRoot, { recursive: true });
+        return {
+            failedOld: make("2026-09-01T12:00:00.000Z", "failed-old", "failed-pre-mutation"),
+            failedNew: make("2026-09-02T12:00:00.000Z", "failed-new", "failed-pre-mutation"),
+            good: make("2026-09-03T12:00:00.000Z", "good", "completed-good"),
+            incompleteOld: make("2026-09-04T12:00:00.000Z", "incomplete-old", "incomplete"),
+            incompleteNew: make("2026-09-05T12:00:00.000Z", "incomplete-new", "incomplete"),
+        };
+    }
+    function snapshotNames(backupRoot) {
+        try {
+            return fs.readdirSync(backupRoot).filter((name) => name.startsWith("pre-update-"));
+        }
+        catch {
+            return [];
+        }
+    }
+    it.skipIf(!canChmodDenyWrite)("preflight fails closed on a read-only parent dir and does not create a snapshot dir", () => {
+        const hqRoot = makeHqRoot();
+        const backupRoot = path.join(workDir, `backups-ro-${caseSeq}`);
+        fs.mkdirSync(backupRoot, { recursive: true });
+        const parent = path.join(hqRoot, "core", "scripts");
+        fs.chmodSync(parent, 0o555);
+        try {
+            const r = runRescueCapture(liveArgs(hqRoot, backupRoot), baseEnv);
+            const out = `${r.stdout}\n${r.stderr}\n${String(r.threw ?? "")}`;
+            expect(r.threw, out).toBeUndefined();
+            expect(r.status, out).toBe(1);
+            expect(out).toContain("cannot write the paths rescue would rename or delete");
+            expect(out).toContain("chown -R");
+            expect(out).toContain(path.join(hqRoot, "core"));
+            expect(snapshotNames(backupRoot)).toEqual([]);
+        }
+        finally {
+            fs.chmodSync(parent, 0o755);
+        }
+    });
+    it("preflight passes on a writable tree", () => {
+        const hqRoot = makeHqRoot();
+        const backupRoot = path.join(workDir, `backups-rw-${caseSeq}`);
+        const r = runRescueCapture(liveArgs(hqRoot, backupRoot), baseEnv);
+        const out = `${r.stdout}\n${r.stderr}\n${String(r.threw ?? "")}`;
+        expect(r.threw, out).toBeUndefined();
+        expect(r.status, out).toBe(0);
+        expect(out).not.toContain("cannot write the paths rescue would rename or delete");
+        expect(snapshotNames(backupRoot)).toHaveLength(1);
+    });
+    it("prunes only retention-eligible failed and partial snapshots before a disk-full preflight", () => {
+        const hqRoot = makeHqRoot();
+        const backupRoot = path.join(workDir, `backups-retention-preflight-${caseSeq}`);
+        const planted = plantRetentionSnapshots(hqRoot, backupRoot);
+        const statfs = () => ({ bavail: 0, bsize: 1, blocks: 1 });
+        const opts = { backupCapacityStatfsSync: statfs };
+        const result = runRescueCapture(liveArgs(hqRoot, backupRoot, ["--cloud-update"]), baseEnv, opts);
+        const output = `${result.stdout}\n${result.stderr}\n${String(result.threw ?? "")}`;
+        expect(result.threw, output).toBeUndefined();
+        expect(result.status, output).toBe(1);
+        expect(snapshotNames(backupRoot).sort()).toEqual([planted.failedNew, planted.good, planted.incompleteNew].sort());
+        expect(fs.existsSync(path.join(backupRoot, planted.failedOld))).toBe(false);
+        expect(fs.existsSync(path.join(backupRoot, planted.incompleteOld))).toBe(false);
+        expect(fs.readFileSync(path.join(backupRoot, planted.failedNew, "core", "a.md"), "utf8")).toBe("failed-new\n");
+        expect(fs.readFileSync(path.join(backupRoot, planted.incompleteNew, "core", "a.md"), "utf8")).toBe("incomplete-new\n");
+    });
+    it("classifies insufficient backup capacity as disk_full before allocating a snapshot", () => {
+        const hqRoot = makeHqRoot();
+        const backupRoot = path.join(workDir, `backups-disk-full-${caseSeq}`);
+        const statfs = () => ({ bavail: 0, bsize: 1, blocks: 1 });
+        const opts = { backupCapacityStatfsSync: statfs };
+        const result = runRescueCapture(liveArgs(hqRoot, backupRoot, ["--cloud-update"]), baseEnv, opts);
+        const output = `${result.stdout}\n${result.stderr}\n${String(result.threw ?? "")}`;
+        expect(result.threw, output).toBeUndefined();
+        expect(result.status, output).toBe(1);
+        expect(result.stderr).toContain("HQ_RESCUE_FAILURE_KIND=disk_full");
+        expect(snapshotNames(backupRoot)).toEqual([]);
+    });
+    it("uses a same-filesystem clone probe to admit a snapshot with its metadata allowance", () => {
+        const hqRoot = makeHqRoot();
+        const backupRoot = path.join(workDir, `backups-clone-capacity-${caseSeq}`);
+        const cloneProbe = vi.fn((_hqRoot, _destRoot, _roots) => true);
+        const statfs = () => ({
+            bavail: 64 * 1024 * 1024,
+            bsize: 1,
+            blocks: 500 * 1024 * 1024 * 1024,
+        });
+        const opts = {
+            backupCapacityStatfsSync: statfs,
+            backupCapacityCloneProbe: cloneProbe,
+        };
+        const result = runRescueCapture(liveArgs(hqRoot, backupRoot, ["--cloud-update"]), baseEnv, opts);
+        const output = `${result.stdout}\n${result.stderr}\n${String(result.threw ?? "")}`;
+        expect(result.threw, output).toBeUndefined();
+        expect(result.status, output).toBe(0);
+        expect(cloneProbe).toHaveBeenCalledOnce();
+        expect(snapshotNames(backupRoot)).toHaveLength(1);
+    });
+    it("keeps the full snapshot estimate when cloning is unsupported or the roots differ by filesystem", () => {
+        for (const scenario of ["unsupported", "different-filesystem"]) {
+            const hqRoot = makeHqRoot();
+            const backupRoot = path.join(workDir, `backups-${scenario}-${caseSeq}`);
+            const cloneProbe = vi.fn(() => true);
+            const unsupportedProbe = vi.fn(() => false);
+            const opts = {
+                backupCapacityStatfsSync: () => ({
+                    bavail: 64 * 1024 * 1024,
+                    bsize: 1,
+                    blocks: 500 * 1024 * 1024 * 1024,
+                }),
+                backupCapacityCloneProbe: scenario === "unsupported" ? unsupportedProbe : cloneProbe,
+                ...(scenario === "different-filesystem"
+                    ? {
+                        backupCapacityStatSync: (target) => ({
+                            ...fs.statSync(target),
+                            dev: fs.realpathSync(hqRoot) === String(target) ? 1 : 2,
+                        }),
+                    }
+                    : {}),
+            };
+            const result = runRescueCapture(liveArgs(hqRoot, backupRoot, ["--cloud-update"]), baseEnv, opts);
+            const output = `${result.stdout}\n${result.stderr}\n${String(result.threw ?? "")}`;
+            expect(result.threw, scenario + output).toBeUndefined();
+            expect(result.status, scenario + output).toBe(0);
+            expect(result.stderr).toContain("HQ_RESCUE_SNAPSHOT=skipped-low-space");
+            expect(result.stderr).not.toContain("HQ_RESCUE_FAILURE_KIND=disk_full");
+            expect(snapshotNames(backupRoot)).toEqual([]);
+            if (scenario === "unsupported") {
+                expect(unsupportedProbe).toHaveBeenCalledOnce();
+            }
+            else {
+                expect(cloneProbe).not.toHaveBeenCalled();
+            }
+        }
+    });
+    it("continues cloud updates without a snapshot when clone metadata will not fit, preserving an older marker", () => {
+        const hqRoot = makeHqRoot();
+        const backupRoot = path.join(workDir, `backups-low-space-${caseSeq}`);
+        const marker = path.join(hqRoot, ".hq", "rescue-blocked.json");
+        fs.mkdirSync(path.dirname(marker), { recursive: true });
+        fs.writeFileSync(marker, "older recovery evidence\n");
+        const cloneProbe = vi.fn(() => true);
+        const statfs = () => ({
+            bavail: 32 * 1024 * 1024,
+            bsize: 1,
+            blocks: 500 * 1024 * 1024 * 1024,
+        });
+        const opts = {
+            backupCapacityStatfsSync: statfs,
+            backupCapacityCloneProbe: cloneProbe,
+        };
+        const result = runRescueCapture(liveArgs(hqRoot, backupRoot, ["--cloud-update"]), baseEnv, opts);
+        const output = `${result.stdout}\n${result.stderr}\n${String(result.threw ?? "")}`;
+        expect(result.threw, output).toBeUndefined();
+        expect(result.status, output).toBe(0);
+        expect(result.stderr).toContain("HQ_RESCUE_SNAPSHOT=skipped-low-space");
+        expect(result.stdout).toMatch(/warning:.*snapshot skipped.*cloud update continues/i);
+        expect(fs.readFileSync(marker, "utf8")).toBe("older recovery evidence\n");
+        expect(snapshotNames(backupRoot)).toEqual([]);
+        expect(result.stderr).not.toContain("HQ_RESCUE_FAILURE_KIND=snapshot-recovery-circuit-breaker");
+    });
+    it("accounts for a cloned preserve shuttle before deciding a cloud snapshot fits", () => {
+        const hqRoot = makeHqRoot();
+        const backupRoot = path.join(workDir, `backups-shuttle-capacity-${caseSeq}`);
+        const keptFile = path.join(hqRoot, "core", "packages", "kept.bin");
+        fs.mkdirSync(path.dirname(keptFile), { recursive: true });
+        fs.writeFileSync(keptFile, "preserved data\n");
+        const cloneProbe = vi.fn(() => true);
+        const opts = {
+            backupCapacityStatfsSync: () => ({
+                bavail: 96 * 1024 * 1024,
+                bsize: 1,
+                blocks: 500 * 1024 ** 3,
+            }),
+            backupCapacityCloneProbe: cloneProbe,
+        };
+        const result = runRescueCapture(liveArgs(hqRoot, backupRoot, ["--cloud-update", "--preserve-subpath", "core/packages"]), baseEnv, opts);
+        const output = `${result.stdout}\n${result.stderr}\n${String(result.threw ?? "")}`;
+        expect(result.threw, output).toBeUndefined();
+        expect(result.status, output).toBe(0);
+        expect(result.stderr).toContain("HQ_RESCUE_SNAPSHOT=skipped-low-space");
+        expect(cloneProbe).toHaveBeenCalledOnce();
+        expect(snapshotNames(backupRoot)).toEqual([]);
+        expect(fs.readFileSync(keptFile, "utf8")).toBe("preserved data\n");
+    });
+    it("checks clone capacity for a preserve shuttle on its separate temp filesystem", () => {
+        const hqRoot = makeHqRoot();
+        const backupRoot = path.join(workDir, `backups-separate-shuttle-${caseSeq}`);
+        const keptFile = path.join(hqRoot, "core", "packages", "kept.bin");
+        fs.mkdirSync(path.dirname(keptFile), { recursive: true });
+        fs.writeFileSync(keptFile, "preserved data\n");
+        const cloneProbe = vi.fn((_hqRoot, _destRoot, _roots) => true);
+        const tempRoot = fs.realpathSync(os.tmpdir());
+        const hqReal = fs.realpathSync(hqRoot);
+        const opts = {
+            backupCapacityStatfsSync: () => ({
+                bavail: 96 * 1024 * 1024,
+                bsize: 1,
+                blocks: 500 * 1024 ** 3,
+            }),
+            backupCapacityCloneProbe: cloneProbe,
+            backupCapacityStatSync: (target) => ({
+                ...fs.statSync(target),
+                dev: [hqReal, tempRoot].includes(fs.realpathSync(String(target))) ? 1 : 2,
+            }),
+        };
+        const result = runRescueCapture(liveArgs(hqRoot, backupRoot, ["--cloud-update", "--preserve-subpath", "core/packages"]), baseEnv, opts);
+        const output = `${result.stdout}\n${result.stderr}\n${String(result.threw ?? "")}`;
+        expect(result.threw, output).toBeUndefined();
+        expect(result.status, output).toBe(0);
+        expect(result.stderr).toContain("HQ_RESCUE_SNAPSHOT=skipped-low-space");
+        expect(cloneProbe).toHaveBeenCalledOnce();
+        expect(fs.realpathSync(String(cloneProbe.mock.calls[0]?.[1]))).toBe(tempRoot);
+        expect(snapshotNames(backupRoot)).toEqual([]);
+        expect(fs.readFileSync(keptFile, "utf8")).toBe("preserved data\n");
+    });
+    it("keeps the disk_full refusal when free space cannot cover the cloud overlay", () => {
+        const hqRoot = makeHqRoot();
+        const backupRoot = path.join(workDir, `backups-overlay-space-${caseSeq}`);
+        const marker = path.join(hqRoot, ".hq", "rescue-blocked.json");
+        const opts = {
+            backupCapacityStatfsSync: () => ({ bavail: 2 * 1024 * 1024, bsize: 1, blocks: 0 }),
+            backupCapacityCloneProbe: () => true,
+        };
+        const result = runRescueCapture(liveArgs(hqRoot, backupRoot, ["--cloud-update"]), baseEnv, opts);
+        const output = `${result.stdout}\n${result.stderr}\n${String(result.threw ?? "")}`;
+        expect(result.threw, output).toBeUndefined();
+        expect(result.status, output).toBe(1);
+        expect(result.stderr).toMatch(/error: insufficient free space for safety snapshot \(need \d+ bytes, have 2097152\)\./);
+        expect(result.stderr).toContain("HQ_RESCUE_FAILURE_KIND=disk_full");
+        expect(result.stderr).not.toContain("HQ_RESCUE_SNAPSHOT=skipped-low-space");
+        expect(fs.existsSync(marker)).toBe(false);
+        expect(snapshotNames(backupRoot)).toEqual([]);
+    });
+    it("does not create recovery-required evidence for a completed update without a snapshot", () => {
+        const hqRoot = makeHqRoot();
+        const backupRoot = path.join(workDir, `backups-no-marker-${caseSeq}`);
+        const marker = path.join(hqRoot, ".hq", "rescue-blocked.json");
+        const opts = {
+            backupCapacityStatfsSync: () => ({
+                bavail: 32 * 1024 * 1024,
+                bsize: 1,
+                blocks: 500 * 1024 ** 3,
+            }),
+            backupCapacityCloneProbe: () => true,
+        };
+        const result = runRescueCapture(liveArgs(hqRoot, backupRoot, ["--cloud-update"]), baseEnv, opts);
+        const output = `${result.stdout}\n${result.stderr}\n${String(result.threw ?? "")}`;
+        expect(result.threw, output).toBeUndefined();
+        expect(result.status, output).toBe(0);
+        expect(result.stderr).toContain("HQ_RESCUE_SNAPSHOT=skipped-low-space");
+        expect(fs.existsSync(marker)).toBe(false);
+        expect(snapshotNames(backupRoot)).toEqual([]);
+    });
+    it("does not skip an interactive rescue when snapshot capacity is low", () => {
+        const hqRoot = makeHqRoot();
+        const backupRoot = path.join(workDir, `backups-interactive-space-${caseSeq}`);
+        const opts = {
+            backupCapacityStatfsSync: () => ({ bavail: 32 * 1024 * 1024, bsize: 1, blocks: 500 * 1024 ** 3 }),
+            backupCapacityCloneProbe: () => true,
+        };
+        const result = runRescueCapture(liveArgs(hqRoot, backupRoot), baseEnv, opts);
+        const output = `${result.stdout}\n${result.stderr}\n${String(result.threw ?? "")}`;
+        expect(result.threw, output).toBeUndefined();
+        expect(result.status, output).toBe(1);
+        expect(result.stderr).toContain("HQ_RESCUE_FAILURE_KIND=disk_full");
+        expect(result.stderr).not.toContain("HQ_RESCUE_SNAPSHOT=skipped-low-space");
+        expect(snapshotNames(backupRoot)).toEqual([]);
+    });
+    it("probes the nearest existing ancestor for a new nested backup root", () => {
+        const hqRoot = makeHqRoot();
+        const backupRoot = path.join(workDir, `missing-parent-${caseSeq}`, "nested", "backups");
+        let probedPath;
+        const statfs = (target) => {
+            probedPath = String(target);
+            return { bavail: 1_000_000_000_000, bsize: 1, blocks: 1_000_000_000_000 };
+        };
+        const opts = { backupCapacityStatfsSync: statfs };
+        const result = runRescueCapture(liveArgs(hqRoot, backupRoot, ["--cloud-update"]), baseEnv, opts);
+        const output = `${result.stdout}\n${result.stderr}\n${String(result.threw ?? "")}`;
+        expect(result.threw, output).toBeUndefined();
+        expect(result.status, output).toBe(0);
+        expect(probedPath).toBe(workDir);
+        expect(snapshotNames(backupRoot)).toHaveLength(1);
+    });
+    it("with --recover-snapshots, unverified recovery snapshots remain and rescue refuses", () => {
+        const hqRoot = makeHqRoot();
+        const backupRoot = path.join(workDir, `backups-resolve-${caseSeq}`);
+        const planted = plantRecoverySnapshots(hqRoot, backupRoot);
+        const r = runRescueCapture(liveArgs(hqRoot, backupRoot, ["--recover-snapshots"]), baseEnv);
+        const out = `${r.stdout}\n${r.stderr}\n${String(r.threw ?? "")}`;
+        expect(r.threw, out).toBeUndefined();
+        expect(r.status, out).toBe(1);
+        expect(out).toContain("snapshot recovery refused");
+        for (const name of planted) {
+            expect(fs.existsSync(path.join(backupRoot, name)), name).toBe(true);
+        }
+        expect(snapshotNames(backupRoot).sort()).toEqual([...planted].sort());
+    });
+    it("writes rescue-blocked.json when the breaker trips and removes it after a successful run", () => {
+        const hqRoot = makeHqRoot();
+        const backupRoot = path.join(workDir, `backups-marker-${caseSeq}`);
+        const planted = plantRecoverySnapshots(hqRoot, backupRoot);
+        const marker = path.join(hqRoot, ".hq", "rescue-blocked.json");
+        const tripped = runRescueCapture(liveArgs(hqRoot, backupRoot), baseEnv);
+        const tripOut = `${tripped.stdout}\n${tripped.stderr}\n${String(tripped.threw ?? "")}`;
+        expect(tripped.threw, tripOut).toBeUndefined();
+        expect(tripped.status, tripOut).toBe(1);
+        expect(tripOut).toContain("safety snapshot circuit breaker is open");
+        expect(fs.existsSync(marker)).toBe(true);
+        const body = JSON.parse(fs.readFileSync(marker, "utf8"));
+        expect(body.reason).toBe("recovery-snapshot-breaker");
+        expect(body.snapshots.sort()).toEqual([...planted].sort());
+        expect(body.hint).toContain("--recover-snapshots");
+        expect(Number.isNaN(Date.parse(body.at))).toBe(false);
+        expect(snapshotNames(backupRoot).sort()).toEqual([...planted].sort());
+        expect(tripOut).toContain("HQ_RESCUE_FAILURE_KIND=snapshot-recovery-circuit-breaker");
+        plantCompletedGoodSnapshot(hqRoot, backupRoot);
+        const inspected = runRescueCapture(liveArgs(hqRoot, backupRoot, ["--recover-snapshots"]), baseEnv);
+        const inspectedOut = `${inspected.stdout}\n${inspected.stderr}\n${String(inspected.threw ?? "")}`;
+        expect(inspected.threw, inspectedOut).toBeUndefined();
+        expect(inspected.status, inspectedOut).toBe(0);
+        expect(snapshotNames(backupRoot)).toHaveLength(1);
+        expect(fs.existsSync(marker)).toBe(false);
+        const recovered = runRescueCapture(liveArgs(hqRoot, backupRoot), baseEnv);
+        const recoveredOut = `${recovered.stdout}\n${recovered.stderr}\n${String(recovered.threw ?? "")}`;
+        expect(recovered.threw, recoveredOut).toBeUndefined();
+        expect(recovered.status, recoveredOut).toBe(0);
+        expect(fs.existsSync(marker)).toBe(false);
+    });
+    it("keeps the rescue-blocked marker when an update succeeds without a completed-good safety snapshot", () => {
+        const hqRoot = makeHqRoot();
+        const backupRoot = path.join(workDir, `backups-marker-no-backup-${caseSeq}`);
+        plantRecoverySnapshots(hqRoot, backupRoot);
+        const marker = path.join(hqRoot, ".hq", "rescue-blocked.json");
+        const tripped = runRescueCapture(liveArgs(hqRoot, backupRoot), baseEnv);
+        const tripOut = `${tripped.stdout}\n${tripped.stderr}\n${String(tripped.threw ?? "")}`;
+        expect(tripped.status, tripOut).toBe(1);
+        expect(fs.existsSync(marker)).toBe(true);
+        const unbacked = runRescueCapture(liveArgs(hqRoot, backupRoot, ["--no-backup"]), baseEnv);
+        const unbackedOut = `${unbacked.stdout}\n${unbacked.stderr}\n${String(unbacked.threw ?? "")}`;
+        expect(unbacked.threw, unbackedOut).toBeUndefined();
+        expect(unbacked.status, unbackedOut).toBe(0);
+        expect(fs.existsSync(marker)).toBe(true);
+    });
+    it("self-heals the breaker when the release and every carve-out have proof", () => {
+        const hqRoot = makeProofHqRoot();
+        const backupRoot = path.join(workDir, `backups-proof-${caseSeq}`);
+        const sourceSha = git(upstream, "rev-parse", "HEAD");
+        const planted = plantProofSnapshots(hqRoot, backupRoot, sourceSha);
+        const recovered = runRescueCapture(liveArgs(hqRoot, backupRoot), baseEnv);
+        const output = `${recovered.stdout}\n${recovered.stderr}\n${String(recovered.threw ?? "")}`;
+        expect(recovered.threw, output).toBeUndefined();
+        expect(recovered.status, output).toBe(0);
+        expect(recovered.stdout).toContain("==> Recovery proof passed: interrupted update matches the release and preserved paths; continuing.");
+        expect(recovered.stderr).toContain("HQ_RESCUE_NOTE=recovery-proof-passed");
+        const snapshots = snapshotNames(backupRoot);
+        expect(snapshots).toHaveLength(1);
+        const manifest = JSON.parse(fs.readFileSync(path.join(backupRoot, snapshots[0], "SNAPSHOT.json"), "utf8"));
+        expect(manifest.status).toBe("completed-good");
+        expect(fs.existsSync(path.join(backupRoot, planted[0]))).toBe(false);
+    });
+    it("keeps the breaker closed when a carve-out differs from the newest snapshot", () => {
+        const hqRoot = makeProofHqRoot();
+        const backupRoot = path.join(workDir, `backups-carveout-${caseSeq}`);
+        const sourceSha = git(upstream, "rev-parse", "HEAD");
+        const planted = plantProofSnapshots(hqRoot, backupRoot, sourceSha);
+        fs.mkdirSync(path.join(hqRoot, "core/packages"), { recursive: true });
+        fs.writeFileSync(path.join(hqRoot, "core/packages/local.txt"), "changed carve-out\n");
+        const blocked = runRescueCapture(liveArgs(hqRoot, backupRoot), baseEnv);
+        const output = `${blocked.stdout}\n${blocked.stderr}\n${String(blocked.threw ?? "")}`;
+        expect(blocked.threw, output).toBeUndefined();
+        expect(blocked.status, output).toBe(1);
+        expect(blocked.stderr).toContain("recovery proof failed: carve-out:core/packages");
+        expect(blocked.stderr).toContain("HQ_RESCUE_FAILURE_KIND=snapshot-recovery-circuit-breaker");
+        expect(snapshotNames(backupRoot).sort()).toEqual([...planted].sort());
+    });
+    it("keeps the breaker closed when the newest snapshot source SHA differs", () => {
+        const hqRoot = makeProofHqRoot();
+        const backupRoot = path.join(workDir, `backups-sha-${caseSeq}`);
+        const sourceSha = git(upstream, "rev-parse", "HEAD");
+        const planted = plantProofSnapshots(hqRoot, backupRoot, "b".repeat(40));
+        const blocked = runRescueCapture(liveArgs(hqRoot, backupRoot), baseEnv);
+        const output = `${blocked.stdout}\n${blocked.stderr}\n${String(blocked.threw ?? "")}`;
+        expect(sourceSha).toHaveLength(40);
+        expect(blocked.threw, output).toBeUndefined();
+        expect(blocked.status, output).toBe(1);
+        expect(blocked.stderr).toContain("recovery proof failed: sha");
+        expect(blocked.stderr).toContain("HQ_RESCUE_FAILURE_KIND=snapshot-recovery-circuit-breaker");
+        expect(snapshotNames(backupRoot).sort()).toEqual([...planted].sort());
+    });
+    it("still refuses --restore-snapshot for an incomplete recovery snapshot", () => {
+        const hqRoot = makeProofHqRoot();
+        const backupRoot = path.join(workDir, `backups-incomplete-${caseSeq}`);
+        const planted = plantRecoverySnapshots(hqRoot, backupRoot);
+        const manifestPath = path.join(backupRoot, planted[1], "SNAPSHOT.json");
+        const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8"));
+        manifest.skippedEntries = [
+            {
+                relativePath: "core/missing.txt",
+                operation: "read",
+                code: "EDEADLK",
+                message: "cloud placeholder",
+                occurredAt: "2026-09-07T12:00:00.000Z",
+            },
+        ];
+        fs.writeFileSync(manifestPath, JSON.stringify(manifest) + "\n");
+        const restored = runRescueCapture(liveArgs(hqRoot, backupRoot, [
+            "--recover-snapshots",
+            "--restore-snapshot",
+            planted[1],
+        ]), baseEnv);
+        const output = `${restored.stdout}\n${restored.stderr}\n${String(restored.threw ?? "")}`;
+        expect(restored.threw, output).toBeUndefined();
+        expect(restored.status, output).toBe(1);
+        expect(restored.stderr).toContain("is incomplete");
+        expect(restored.stderr).toContain("cannot restore every captured root");
+    });
+});
+//# sourceMappingURL=rescue-breaker.test.js.map