@indigoai-us/hq-cloud 6.18.44-alpha.9 → 6.18.44

package/dist/cli/rescue-breaker.test.js

@@ -1,812 +0,0 @@
-/**
- * Writability preflight, recovery-snapshot circuit breaker recovery, and
- * `.hq/rescue-blocked.json` visibility for `hq rescue`.
- *
- * Production (2026-09-16): six agent boxes froze on hq-core releases from
- * 15 Aug–7 Sep because `core/scripts` was root-owned. Drift-rescue
- * `renameSync(core/scripts/qmd-reindex-bg.sh → personal/scripts/...)` failed
- * EACCES twice, the safety snapshot was already allocated, and the breaker
- * then held forever with no heartbeat signal.
- */
-import { afterAll, beforeAll, describe, expect, it, vi } from "vitest";
-import { execFileSync } from "child_process";
-import * as fs from "fs";
-import * as os from "os";
-import * as path from "path";
-import { canonicalRootHash } from "../backup-prune.js";
-import { collectUnwritableRescueMutationPaths, parseRescueArgs, requireBackupCapacity, runRescue, } from "./rescue-core.js";
-function has(bin, ...args) {
-    try {
-        execFileSync(bin, args, { stdio: "ignore" });
-        return true;
-    }
-    catch {
-        return false;
-    }
-}
-const toolsAvailable = has("git", "--version") && has("rsync", "--version");
-const canChmodDenyWrite = process.platform !== "win32" &&
-    (typeof process.getuid !== "function" || process.getuid() !== 0);
-function runRescueCapture(argv, env, opts = {}) {
-    let stdout = "";
-    let stderr = "";
-    const origOut = process.stdout.write.bind(process.stdout);
-    const origErr = process.stderr.write.bind(process.stderr);
-    process.stdout.write = ((chunk) => {
-        stdout += String(chunk);
-        return true;
-    });
-    process.stderr.write = ((chunk) => {
-        stderr += String(chunk);
-        return true;
-    });
-    let status;
-    let threw;
-    try {
-        status =
-            runRescue(argv, {
-                env,
-                backupCapacityStatfsSync: (() => ({
-                    bavail: 1_000_000_000_000,
-                    bsize: 1,
-                    blocks: 1_000_000_000_000,
-                })),
-                ...opts,
-            }).status;
-    }
-    catch (e) {
-        threw = e;
-    }
-    finally {
-        process.stdout.write = origOut;
-        process.stderr.write = origErr;
-    }
-    return { status, stdout, stderr, threw };
-}
-describe("parseRescueArgs --recover-snapshots", () => {
-    it("defaults off and accepts the flag", () => {
-        expect(parseRescueArgs([]).recoverSnapshots).toBe(false);
-        expect(parseRescueArgs(["--recover-snapshots"]).recoverSnapshots).toBe(true);
-    });
-});
-describe("requireBackupCapacity", () => {
-    const gib = 1024 ** 3;
-    const gb = 1000 ** 3;
-    function checkCapacity(estimate, capacity, free, preserveBytes = 0) {
-        const root = fs.mkdtempSync(path.join(os.tmpdir(), "hq-rescue-capacity-"));
-        const hqRoot = path.join(root, "hq");
-        fs.mkdirSync(path.join(hqRoot, "core"), { recursive: true });
-        const estimateFile = path.join(hqRoot, "core", "estimate.bin");
-        fs.writeFileSync(estimateFile, "");
-        fs.truncateSync(estimateFile, estimate - preserveBytes);
-        const preserveSubpaths = preserveBytes > 0 ? ["core/packages"] : [];
-        if (preserveBytes > 0) {
-            const preserveFile = path.join(hqRoot, "core", "packages", "kept.bin");
-            fs.mkdirSync(path.dirname(preserveFile), { recursive: true });
-            fs.writeFileSync(preserveFile, "");
-            fs.truncateSync(preserveFile, preserveBytes);
-        }
-        let output = "";
-        const statfs = (() => ({
-            bavail: free,
-            bsize: 1,
-            blocks: capacity,
-        }));
-        try {
-            requireBackupCapacity({
-                backupRoot: path.join(root, "backups"),
-                preserveSubpaths,
-            }, hqRoot, ["core"], hqRoot, {}, (line) => {
-                output += line;
-            }, statfs, () => false);
-            return { output, refused: false };
-        }
-        catch (error) {
-            if (!(error instanceof Error) || error.message !== "exit 1")
-                throw error;
-            return { output, refused: true };
-        }
-        finally {
-            fs.rmSync(root, { recursive: true, force: true });
-        }
-    }
-    it("allows a 3 GiB estimate on a 500 GB disk with 15.7 GB free", () => {
-        const result = checkCapacity(3 * gib, 500 * gb, 15.7 * gb);
-        expect(result.refused).toBe(false);
-        expect(result.output).toBe("");
-    });
-    it("refuses a 1 GiB estimate with only 4 GB free and keeps disk_full", () => {
-        const result = checkCapacity(1 * gib, 64 * gb, 4 * gb);
-        expect(result.refused).toBe(true);
-        expect(result.output).toContain("error: insufficient free space for safety snapshot (need ");
-        expect(result.output).toContain("HQ_RESCUE_FAILURE_KIND=disk_full");
-    });
-    it("keeps at least the 5 GiB margin on a 64 GB disk", () => {
-        const result = checkCapacity(1 * gib, 64 * gb, 7 * gb);
-        expect(result.refused).toBe(true);
-        expect(result.output).toContain("HQ_RESCUE_FAILURE_KIND=disk_full");
-    });
-    it("uses the 5 GiB floor when 10% of capacity is lower", () => {
-        const result = checkCapacity(1 * gib, 32 * gb, 6 * gib - 1);
-        expect(result.refused).toBe(true);
-        expect(result.output).toContain(`need ${1 * gib + 5 * gib} bytes`);
-    });
-    it("refuses when the estimate alone exceeds free space", () => {
-        const result = checkCapacity(8 * gib, 64 * gb, 7 * gib);
-        expect(result.refused).toBe(true);
-        expect(result.output).toContain("HQ_RESCUE_FAILURE_KIND=disk_full");
-    });
-    it("includes a preserved subtree copied to a same-filesystem shuttle", () => {
-        const result = checkCapacity(30 * gib, 500 * gib, 45 * gib, 20 * gib);
-        expect(result.refused).toBe(true);
-        expect(result.output).toContain("need 64424509440 bytes");
-        expect(result.output).toContain("HQ_RESCUE_FAILURE_KIND=disk_full");
-    });
-});
-describe("collectUnwritableRescueMutationPaths", () => {
-    it.skipIf(!canChmodDenyWrite)("fails closed on a read-only parent dir", () => {
-        const root = fs.mkdtempSync(path.join(os.tmpdir(), "hq-rescue-preflight-ro-"));
-        const parent = path.join(root, "core", "scripts");
-        const file = path.join(parent, "qmd-reindex-bg.sh");
-        fs.mkdirSync(parent, { recursive: true });
-        fs.mkdirSync(path.join(root, "personal"), { recursive: true });
-        fs.writeFileSync(file, "local-edit\n");
-        fs.chmodSync(parent, 0o555);
-        try {
-            const failed = collectUnwritableRescueMutationPaths(root, [
-                {
-                    affectedRels: ["core/scripts/qmd-reindex-bg.sh"],
-                    destParentRels: ["personal/scripts"],
-                },
-            ]);
-            expect(failed).toContain("core/scripts");
-            expect(failed.join("\n")).not.toMatch(/personal/);
-        }
-        finally {
-            fs.chmodSync(parent, 0o755);
-            fs.rmSync(root, { recursive: true, force: true });
-        }
-    });
-    it("passes on a writable tree", () => {
-        const root = fs.mkdtempSync(path.join(os.tmpdir(), "hq-rescue-preflight-rw-"));
-        try {
-            const parent = path.join(root, "core", "scripts");
-            fs.mkdirSync(parent, { recursive: true });
-            fs.mkdirSync(path.join(root, "personal"), { recursive: true });
-            fs.writeFileSync(path.join(parent, "qmd-reindex-bg.sh"), "local-edit\n");
-            expect(collectUnwritableRescueMutationPaths(root, [
-                {
-                    affectedRels: ["core/scripts/qmd-reindex-bg.sh"],
-                    destParentRels: ["personal/scripts"],
-                },
-            ])).toEqual([]);
-        }
-        finally {
-            fs.rmSync(root, { recursive: true, force: true });
-        }
-    });
-});
-describe.skipIf(!toolsAvailable)("rescue recovery-snapshot breaker + mutation preflight", () => {
-    let workDir;
-    let upstream;
-    let shimDir;
-    let floorSha;
-    let baseEnv;
-    let caseSeq = 0;
-    const git = (cwd, ...args) => execFileSync("git", args, {
-        cwd,
-        stdio: ["ignore", "pipe", "pipe"],
-        env: {
-            ...process.env,
-            GIT_AUTHOR_NAME: "t",
-            GIT_AUTHOR_EMAIL: "t@t",
-            GIT_COMMITTER_NAME: "t",
-            GIT_COMMITTER_EMAIL: "t@t",
-        },
-    })
-        .toString()
-        .trim();
-    beforeAll(() => {
-        workDir = fs.mkdtempSync(path.join(os.tmpdir(), "hq-rescue-breaker-"));
-        upstream = path.join(workDir, "upstream");
-        fs.mkdirSync(path.join(upstream, "core", "scripts"), { recursive: true });
-        git(workDir, "init", "-b", "main", "upstream");
-        fs.writeFileSync(path.join(upstream, "core/a.md"), "v1\n");
-        fs.writeFileSync(path.join(upstream, "core/scripts/qmd-reindex-bg.sh"), "release-v1\n");
-        git(upstream, "add", "-A");
-        git(upstream, "commit", "-m", "floor");
-        floorSha = git(upstream, "rev-parse", "HEAD");
-        fs.writeFileSync(path.join(upstream, "core/a.md"), "v2\n");
-        fs.writeFileSync(path.join(upstream, "core/scripts/qmd-reindex-bg.sh"), "release-v2\n");
-        git(upstream, "add", "-A");
-        git(upstream, "commit", "-m", "head");
-        const realGit = execFileSync("bash", ["-lc", "command -v git"]).toString().trim() ||
-            "/usr/bin/git";
-        shimDir = path.join(workDir, "shim");
-        fs.mkdirSync(shimDir, { recursive: true });
-        const shim = `#!/usr/bin/env bash
-if [ "$1" = "clone" ] || [ "$1" = "ls-remote" ]; then
-  args=()
-  for a in "$@"; do
-    case "$a" in
-      https://github.com/*) a=${JSON.stringify(upstream)} ;;
-    esac
-    args+=("$a")
-  done
-  exec ${JSON.stringify(realGit)} "\${args[@]}"
-fi
-exec ${JSON.stringify(realGit)} "$@"
-`;
-        fs.writeFileSync(path.join(shimDir, "git"), shim, { mode: 0o755 });
-        baseEnv = { ...process.env, PATH: `${shimDir}:${process.env.PATH ?? ""}` };
-    }, 60_000);
-    afterAll(() => {
-        if (workDir)
-            fs.rmSync(workDir, { recursive: true, force: true });
-    });
-    function makeHqRoot() {
-        caseSeq += 1;
-        const hqRoot = path.join(workDir, `hq-${caseSeq}`);
-        fs.mkdirSync(path.join(hqRoot, "core", "scripts"), { recursive: true });
-        fs.mkdirSync(path.join(hqRoot, "companies"), { recursive: true });
-        fs.mkdirSync(path.join(hqRoot, "personal"), { recursive: true });
-        fs.writeFileSync(path.join(hqRoot, "core/a.md"), "v1\n");
-        fs.writeFileSync(path.join(hqRoot, "core/scripts/qmd-reindex-bg.sh"), "local-edit\n");
-        return hqRoot;
-    }
-    function liveArgs(hqRoot, backupRoot, extra = []) {
-        return [
-            "--hq-root",
-            hqRoot,
-            "--source",
-            "test/repo",
-            "--ref",
-            "main",
-            "--floor-sha",
-            floorSha,
-            "--paths",
-            "core",
-            "--backup-dir",
-            backupRoot,
-            "--yes",
-            ...extra,
-        ];
-    }
-    function plantRecoverySnapshots(hqRoot, backupRoot) {
-        const canonical = fs.realpathSync.native(hqRoot);
-        const hash = canonicalRootHash(canonical);
-        const names = [
-            `pre-update-2026-08-15T12-00-00Z-${hash}-one`,
-            `pre-update-2026-09-07T12-00-00Z-${hash}-two`,
-        ];
-        fs.mkdirSync(backupRoot, { recursive: true });
-        for (const name of names) {
-            const dir = path.join(backupRoot, name);
-            fs.mkdirSync(dir, { recursive: true });
-            const at = "2026-08-15T12:00:00.000Z";
-            const runId = name.slice(name.lastIndexOf("-") + 1);
-            fs.writeFileSync(path.join(dir, "SNAPSHOT.json"), JSON.stringify({
-                schemaVersion: 1,
-                status: "recovery-required",
-                canonicalHqRoot: canonical,
-                rootHash: hash,
-                runId,
-                creatorPid: 1,
-                creatorStartedAt: at,
-                source: "test/repo",
-                ref: "main",
-                sourceSha: "abc",
-                createdAt: at,
-                completedAt: at,
-                includedRoots: ["core"],
-                fileCount: 1,
-                byteCount: 1,
-            }) + "\n");
-            fs.writeFileSync(path.join(dir, "keep.txt"), "recovery evidence\n");
-        }
-        return names;
-    }
-    function plantProofSnapshots(hqRoot, backupRoot, sourceSha) {
-        const names = plantRecoverySnapshots(hqRoot, backupRoot);
-        for (const name of names) {
-            const dir = path.join(backupRoot, name);
-            const manifest = JSON.parse(fs.readFileSync(path.join(dir, "SNAPSHOT.json"), "utf8"));
-            manifest.sourceSha = sourceSha;
-            fs.cpSync(path.join(hqRoot, "core"), path.join(dir, "core"), {
-                recursive: true,
-            });
-            fs.writeFileSync(path.join(dir, "SNAPSHOT.json"), JSON.stringify(manifest) + "\n");
-        }
-        return names;
-    }
-    function makeProofHqRoot() {
-        const hqRoot = makeHqRoot();
-        fs.writeFileSync(path.join(hqRoot, "core/a.md"), "v2\n");
-        fs.writeFileSync(path.join(hqRoot, "core/scripts/qmd-reindex-bg.sh"), "release-v2\n");
-        fs.writeFileSync(path.join(hqRoot, "core/old.conflict-2026-test"), "discarded artifact\n");
-        return hqRoot;
-    }
-    function plantCompletedGoodSnapshot(hqRoot, backupRoot) {
-        const canonical = fs.realpathSync.native(hqRoot);
-        const hash = canonicalRootHash(canonical);
-        const name = `pre-update-2026-09-08T12-00-00Z-${hash}-good`;
-        const dir = path.join(backupRoot, name);
-        const at = "2026-09-08T12:00:00.000Z";
-        fs.mkdirSync(dir, { recursive: true });
-        fs.writeFileSync(path.join(dir, "SNAPSHOT.json"), JSON.stringify({
-            schemaVersion: 1,
-            status: "completed-good",
-            canonicalHqRoot: canonical,
-            rootHash: hash,
-            runId: "good",
-            creatorPid: 1,
-            creatorStartedAt: at,
-            source: "test/repo",
-            ref: "main",
-            sourceSha: "abc",
-            createdAt: at,
-            completedAt: at,
-            includedRoots: ["core"],
-            fileCount: 1,
-            byteCount: 1,
-        }) + "\n");
-    }
-    function plantRetentionSnapshots(hqRoot, backupRoot) {
-        const canonical = fs.realpathSync.native(hqRoot);
-        const hash = canonicalRootHash(canonical);
-        const make = (date, suffix, status) => {
-            const createdAt = new Date(date).toISOString();
-            const name = `pre-update-${createdAt.replaceAll(":", "-").replace(".000", "")}-${hash}-${suffix}`;
-            const dir = path.join(backupRoot, name);
-            fs.mkdirSync(dir, { recursive: true });
-            fs.mkdirSync(path.join(dir, "core"), { recursive: true });
-            fs.writeFileSync(path.join(dir, "core", "a.md"), `${suffix}\n`);
-            fs.writeFileSync(path.join(dir, "SNAPSHOT.json"), JSON.stringify({
-                schemaVersion: 1,
-                status,
-                canonicalHqRoot: canonical,
-                rootHash: hash,
-                runId: suffix,
-                creatorPid: 1,
-                creatorStartedAt: createdAt,
-                source: "test/repo",
-                ref: "main",
-                sourceSha: "abc",
-                createdAt,
-                completedAt: createdAt,
-                includedRoots: ["core"],
-                ...(status === "incomplete"
-                    ? {
-                        skippedEntries: [
-                            {
-                                relativePath: "core/missing.txt",
-                                operation: "read",
-                                code: "EACCES",
-                                message: "test fixture",
-                                occurredAt: createdAt,
-                            },
-                        ],
-                    }
-                    : {}),
-                fileCount: 1,
-                byteCount: 1,
-            }) + "\n");
-            return name;
-        };
-        fs.mkdirSync(backupRoot, { recursive: true });
-        return {
-            failedOld: make("2026-09-01T12:00:00.000Z", "failed-old", "failed-pre-mutation"),
-            failedNew: make("2026-09-02T12:00:00.000Z", "failed-new", "failed-pre-mutation"),
-            good: make("2026-09-03T12:00:00.000Z", "good", "completed-good"),
-            incompleteOld: make("2026-09-04T12:00:00.000Z", "incomplete-old", "incomplete"),
-            incompleteNew: make("2026-09-05T12:00:00.000Z", "incomplete-new", "incomplete"),
-        };
-    }
-    function snapshotNames(backupRoot) {
-        try {
-            return fs.readdirSync(backupRoot).filter((name) => name.startsWith("pre-update-"));
-        }
-        catch {
-            return [];
-        }
-    }
-    it.skipIf(!canChmodDenyWrite)("preflight fails closed on a read-only parent dir and does not create a snapshot dir", () => {
-        const hqRoot = makeHqRoot();
-        const backupRoot = path.join(workDir, `backups-ro-${caseSeq}`);
-        fs.mkdirSync(backupRoot, { recursive: true });
-        const parent = path.join(hqRoot, "core", "scripts");
-        fs.chmodSync(parent, 0o555);
-        try {
-            const r = runRescueCapture(liveArgs(hqRoot, backupRoot), baseEnv);
-            const out = `${r.stdout}\n${r.stderr}\n${String(r.threw ?? "")}`;
-            expect(r.threw, out).toBeUndefined();
-            expect(r.status, out).toBe(1);
-            expect(out).toContain("cannot write the paths rescue would rename or delete");
-            expect(out).toContain("chown -R");
-            expect(out).toContain(path.join(hqRoot, "core"));
-            expect(snapshotNames(backupRoot)).toEqual([]);
-        }
-        finally {
-            fs.chmodSync(parent, 0o755);
-        }
-    });
-    it("preflight passes on a writable tree", () => {
-        const hqRoot = makeHqRoot();
-        const backupRoot = path.join(workDir, `backups-rw-${caseSeq}`);
-        const r = runRescueCapture(liveArgs(hqRoot, backupRoot), baseEnv);
-        const out = `${r.stdout}\n${r.stderr}\n${String(r.threw ?? "")}`;
-        expect(r.threw, out).toBeUndefined();
-        expect(r.status, out).toBe(0);
-        expect(out).not.toContain("cannot write the paths rescue would rename or delete");
-        expect(snapshotNames(backupRoot)).toHaveLength(1);
-    });
-    it("prunes only retention-eligible failed and partial snapshots before a disk-full preflight", () => {
-        const hqRoot = makeHqRoot();
-        const backupRoot = path.join(workDir, `backups-retention-preflight-${caseSeq}`);
-        const planted = plantRetentionSnapshots(hqRoot, backupRoot);
-        const statfs = () => ({ bavail: 0, bsize: 1, blocks: 1 });
-        const opts = { backupCapacityStatfsSync: statfs };
-        const result = runRescueCapture(liveArgs(hqRoot, backupRoot, ["--cloud-update"]), baseEnv, opts);
-        const output = `${result.stdout}\n${result.stderr}\n${String(result.threw ?? "")}`;
-        expect(result.threw, output).toBeUndefined();
-        expect(result.status, output).toBe(1);
-        expect(snapshotNames(backupRoot).sort()).toEqual([planted.failedNew, planted.good, planted.incompleteNew].sort());
-        expect(fs.existsSync(path.join(backupRoot, planted.failedOld))).toBe(false);
-        expect(fs.existsSync(path.join(backupRoot, planted.incompleteOld))).toBe(false);
-        expect(fs.readFileSync(path.join(backupRoot, planted.failedNew, "core", "a.md"), "utf8")).toBe("failed-new\n");
-        expect(fs.readFileSync(path.join(backupRoot, planted.incompleteNew, "core", "a.md"), "utf8")).toBe("incomplete-new\n");
-    });
-    it("classifies insufficient backup capacity as disk_full before allocating a snapshot", () => {
-        const hqRoot = makeHqRoot();
-        const backupRoot = path.join(workDir, `backups-disk-full-${caseSeq}`);
-        const statfs = () => ({ bavail: 0, bsize: 1, blocks: 1 });
-        const opts = { backupCapacityStatfsSync: statfs };
-        const result = runRescueCapture(liveArgs(hqRoot, backupRoot, ["--cloud-update"]), baseEnv, opts);
-        const output = `${result.stdout}\n${result.stderr}\n${String(result.threw ?? "")}`;
-        expect(result.threw, output).toBeUndefined();
-        expect(result.status, output).toBe(1);
-        expect(result.stderr).toContain("HQ_RESCUE_FAILURE_KIND=disk_full");
-        expect(snapshotNames(backupRoot)).toEqual([]);
-    });
-    it("uses a same-filesystem clone probe to admit a snapshot with its metadata allowance", () => {
-        const hqRoot = makeHqRoot();
-        const backupRoot = path.join(workDir, `backups-clone-capacity-${caseSeq}`);
-        const cloneProbe = vi.fn((_hqRoot, _destRoot, _roots) => true);
-        const statfs = () => ({
-            bavail: 64 * 1024 * 1024,
-            bsize: 1,
-            blocks: 500 * 1024 * 1024 * 1024,
-        });
-        const opts = {
-            backupCapacityStatfsSync: statfs,
-            backupCapacityCloneProbe: cloneProbe,
-        };
-        const result = runRescueCapture(liveArgs(hqRoot, backupRoot, ["--cloud-update"]), baseEnv, opts);
-        const output = `${result.stdout}\n${result.stderr}\n${String(result.threw ?? "")}`;
-        expect(result.threw, output).toBeUndefined();
-        expect(result.status, output).toBe(0);
-        expect(cloneProbe).toHaveBeenCalledOnce();
-        expect(snapshotNames(backupRoot)).toHaveLength(1);
-    });
-    it("keeps the full snapshot estimate when cloning is unsupported or the roots differ by filesystem", () => {
-        for (const scenario of ["unsupported", "different-filesystem"]) {
-            const hqRoot = makeHqRoot();
-            const backupRoot = path.join(workDir, `backups-${scenario}-${caseSeq}`);
-            const cloneProbe = vi.fn(() => true);
-            const unsupportedProbe = vi.fn(() => false);
-            const opts = {
-                backupCapacityStatfsSync: () => ({
-                    bavail: 64 * 1024 * 1024,
-                    bsize: 1,
-                    blocks: 500 * 1024 * 1024 * 1024,
-                }),
-                backupCapacityCloneProbe: scenario === "unsupported" ? unsupportedProbe : cloneProbe,
-                ...(scenario === "different-filesystem"
-                    ? {
-                        backupCapacityStatSync: (target) => ({
-                            ...fs.statSync(target),
-                            dev: fs.realpathSync(hqRoot) === String(target) ? 1 : 2,
-                        }),
-                    }
-                    : {}),
-            };
-            const result = runRescueCapture(liveArgs(hqRoot, backupRoot, ["--cloud-update"]), baseEnv, opts);
-            const output = `${result.stdout}\n${result.stderr}\n${String(result.threw ?? "")}`;
-            expect(result.threw, scenario + output).toBeUndefined();
-            expect(result.status, scenario + output).toBe(0);
-            expect(result.stderr).toContain("HQ_RESCUE_SNAPSHOT=skipped-low-space");
-            expect(result.stderr).not.toContain("HQ_RESCUE_FAILURE_KIND=disk_full");
-            expect(snapshotNames(backupRoot)).toEqual([]);
-            if (scenario === "unsupported") {
-                expect(unsupportedProbe).toHaveBeenCalledOnce();
-            }
-            else {
-                expect(cloneProbe).not.toHaveBeenCalled();
-            }
-        }
-    });
-    it("continues cloud updates without a snapshot when clone metadata will not fit, preserving an older marker", () => {
-        const hqRoot = makeHqRoot();
-        const backupRoot = path.join(workDir, `backups-low-space-${caseSeq}`);
-        const marker = path.join(hqRoot, ".hq", "rescue-blocked.json");
-        fs.mkdirSync(path.dirname(marker), { recursive: true });
-        fs.writeFileSync(marker, "older recovery evidence\n");
-        const cloneProbe = vi.fn(() => true);
-        const statfs = () => ({
-            bavail: 32 * 1024 * 1024,
-            bsize: 1,
-            blocks: 500 * 1024 * 1024 * 1024,
-        });
-        const opts = {
-            backupCapacityStatfsSync: statfs,
-            backupCapacityCloneProbe: cloneProbe,
-        };
-        const result = runRescueCapture(liveArgs(hqRoot, backupRoot, ["--cloud-update"]), baseEnv, opts);
-        const output = `${result.stdout}\n${result.stderr}\n${String(result.threw ?? "")}`;
-        expect(result.threw, output).toBeUndefined();
-        expect(result.status, output).toBe(0);
-        expect(result.stderr).toContain("HQ_RESCUE_SNAPSHOT=skipped-low-space");
-        expect(result.stdout).toMatch(/warning:.*snapshot skipped.*cloud update continues/i);
-        expect(fs.readFileSync(marker, "utf8")).toBe("older recovery evidence\n");
-        expect(snapshotNames(backupRoot)).toEqual([]);
-        expect(result.stderr).not.toContain("HQ_RESCUE_FAILURE_KIND=snapshot-recovery-circuit-breaker");
-    });
-    it("accounts for a cloned preserve shuttle before deciding a cloud snapshot fits", () => {
-        const hqRoot = makeHqRoot();
-        const backupRoot = path.join(workDir, `backups-shuttle-capacity-${caseSeq}`);
-        const keptFile = path.join(hqRoot, "core", "packages", "kept.bin");
-        fs.mkdirSync(path.dirname(keptFile), { recursive: true });
-        fs.writeFileSync(keptFile, "preserved data\n");
-        const cloneProbe = vi.fn(() => true);
-        const opts = {
-            backupCapacityStatfsSync: () => ({
-                bavail: 96 * 1024 * 1024,
-                bsize: 1,
-                blocks: 500 * 1024 ** 3,
-            }),
-            backupCapacityCloneProbe: cloneProbe,
-        };
-        const result = runRescueCapture(liveArgs(hqRoot, backupRoot, ["--cloud-update", "--preserve-subpath", "core/packages"]), baseEnv, opts);
-        const output = `${result.stdout}\n${result.stderr}\n${String(result.threw ?? "")}`;
-        expect(result.threw, output).toBeUndefined();
-        expect(result.status, output).toBe(0);
-        expect(result.stderr).toContain("HQ_RESCUE_SNAPSHOT=skipped-low-space");
-        expect(cloneProbe).toHaveBeenCalledOnce();
-        expect(snapshotNames(backupRoot)).toEqual([]);
-        expect(fs.readFileSync(keptFile, "utf8")).toBe("preserved data\n");
-    });
-    it("checks clone capacity for a preserve shuttle on its separate temp filesystem", () => {
-        const hqRoot = makeHqRoot();
-        const backupRoot = path.join(workDir, `backups-separate-shuttle-${caseSeq}`);
-        const keptFile = path.join(hqRoot, "core", "packages", "kept.bin");
-        fs.mkdirSync(path.dirname(keptFile), { recursive: true });
-        fs.writeFileSync(keptFile, "preserved data\n");
-        const cloneProbe = vi.fn((_hqRoot, _destRoot, _roots) => true);
-        const tempRoot = fs.realpathSync(os.tmpdir());
-        const hqReal = fs.realpathSync(hqRoot);
-        const opts = {
-            backupCapacityStatfsSync: () => ({
-                bavail: 96 * 1024 * 1024,
-                bsize: 1,
-                blocks: 500 * 1024 ** 3,
-            }),
-            backupCapacityCloneProbe: cloneProbe,
-            backupCapacityStatSync: (target) => ({
-                ...fs.statSync(target),
-                dev: [hqReal, tempRoot].includes(fs.realpathSync(String(target))) ? 1 : 2,
-            }),
-        };
-        const result = runRescueCapture(liveArgs(hqRoot, backupRoot, ["--cloud-update", "--preserve-subpath", "core/packages"]), baseEnv, opts);
-        const output = `${result.stdout}\n${result.stderr}\n${String(result.threw ?? "")}`;
-        expect(result.threw, output).toBeUndefined();
-        expect(result.status, output).toBe(0);
-        expect(result.stderr).toContain("HQ_RESCUE_SNAPSHOT=skipped-low-space");
-        expect(cloneProbe).toHaveBeenCalledOnce();
-        expect(fs.realpathSync(String(cloneProbe.mock.calls[0]?.[1]))).toBe(tempRoot);
-        expect(snapshotNames(backupRoot)).toEqual([]);
-        expect(fs.readFileSync(keptFile, "utf8")).toBe("preserved data\n");
-    });
-    it("keeps the disk_full refusal when free space cannot cover the cloud overlay", () => {
-        const hqRoot = makeHqRoot();
-        const backupRoot = path.join(workDir, `backups-overlay-space-${caseSeq}`);
-        const marker = path.join(hqRoot, ".hq", "rescue-blocked.json");
-        const opts = {
-            backupCapacityStatfsSync: () => ({ bavail: 2 * 1024 * 1024, bsize: 1, blocks: 0 }),
-            backupCapacityCloneProbe: () => true,
-        };
-        const result = runRescueCapture(liveArgs(hqRoot, backupRoot, ["--cloud-update"]), baseEnv, opts);
-        const output = `${result.stdout}\n${result.stderr}\n${String(result.threw ?? "")}`;
-        expect(result.threw, output).toBeUndefined();
-        expect(result.status, output).toBe(1);
-        expect(result.stderr).toMatch(/error: insufficient free space for safety snapshot \(need \d+ bytes, have 2097152\)\./);
-        expect(result.stderr).toContain("HQ_RESCUE_FAILURE_KIND=disk_full");
-        expect(result.stderr).not.toContain("HQ_RESCUE_SNAPSHOT=skipped-low-space");
-        expect(fs.existsSync(marker)).toBe(false);
-        expect(snapshotNames(backupRoot)).toEqual([]);
-    });
-    it("does not create recovery-required evidence for a completed update without a snapshot", () => {
-        const hqRoot = makeHqRoot();
-        const backupRoot = path.join(workDir, `backups-no-marker-${caseSeq}`);
-        const marker = path.join(hqRoot, ".hq", "rescue-blocked.json");
-        const opts = {
-            backupCapacityStatfsSync: () => ({
-                bavail: 32 * 1024 * 1024,
-                bsize: 1,
-                blocks: 500 * 1024 ** 3,
-            }),
-            backupCapacityCloneProbe: () => true,
-        };
-        const result = runRescueCapture(liveArgs(hqRoot, backupRoot, ["--cloud-update"]), baseEnv, opts);
-        const output = `${result.stdout}\n${result.stderr}\n${String(result.threw ?? "")}`;
-        expect(result.threw, output).toBeUndefined();
-        expect(result.status, output).toBe(0);
-        expect(result.stderr).toContain("HQ_RESCUE_SNAPSHOT=skipped-low-space");
-        expect(fs.existsSync(marker)).toBe(false);
-        expect(snapshotNames(backupRoot)).toEqual([]);
-    });
-    it("does not skip an interactive rescue when snapshot capacity is low", () => {
-        const hqRoot = makeHqRoot();
-        const backupRoot = path.join(workDir, `backups-interactive-space-${caseSeq}`);
-        const opts = {
-            backupCapacityStatfsSync: () => ({ bavail: 32 * 1024 * 1024, bsize: 1, blocks: 500 * 1024 ** 3 }),
-            backupCapacityCloneProbe: () => true,
-        };
-        const result = runRescueCapture(liveArgs(hqRoot, backupRoot), baseEnv, opts);
-        const output = `${result.stdout}\n${result.stderr}\n${String(result.threw ?? "")}`;
-        expect(result.threw, output).toBeUndefined();
-        expect(result.status, output).toBe(1);
-        expect(result.stderr).toContain("HQ_RESCUE_FAILURE_KIND=disk_full");
-        expect(result.stderr).not.toContain("HQ_RESCUE_SNAPSHOT=skipped-low-space");
-        expect(snapshotNames(backupRoot)).toEqual([]);
-    });
-    it("probes the nearest existing ancestor for a new nested backup root", () => {
-        const hqRoot = makeHqRoot();
-        const backupRoot = path.join(workDir, `missing-parent-${caseSeq}`, "nested", "backups");
-        let probedPath;
-        const statfs = (target) => {
-            probedPath = String(target);
-            return { bavail: 1_000_000_000_000, bsize: 1, blocks: 1_000_000_000_000 };
-        };
-        const opts = { backupCapacityStatfsSync: statfs };
-        const result = runRescueCapture(liveArgs(hqRoot, backupRoot, ["--cloud-update"]), baseEnv, opts);
-        const output = `${result.stdout}\n${result.stderr}\n${String(result.threw ?? "")}`;
-        expect(result.threw, output).toBeUndefined();
-        expect(result.status, output).toBe(0);
-        expect(probedPath).toBe(workDir);
-        expect(snapshotNames(backupRoot)).toHaveLength(1);
-    });
-    it("with --recover-snapshots, unverified recovery snapshots remain and rescue refuses", () => {
-        const hqRoot = makeHqRoot();
-        const backupRoot = path.join(workDir, `backups-resolve-${caseSeq}`);
-        const planted = plantRecoverySnapshots(hqRoot, backupRoot);
-        const r = runRescueCapture(liveArgs(hqRoot, backupRoot, ["--recover-snapshots"]), baseEnv);
-        const out = `${r.stdout}\n${r.stderr}\n${String(r.threw ?? "")}`;
-        expect(r.threw, out).toBeUndefined();
-        expect(r.status, out).toBe(1);
-        expect(out).toContain("snapshot recovery refused");
-        for (const name of planted) {
-            expect(fs.existsSync(path.join(backupRoot, name)), name).toBe(true);
-        }
-        expect(snapshotNames(backupRoot).sort()).toEqual([...planted].sort());
-    });
-    it("writes rescue-blocked.json when the breaker trips and removes it after a successful run", () => {
-        const hqRoot = makeHqRoot();
-        const backupRoot = path.join(workDir, `backups-marker-${caseSeq}`);
-        const planted = plantRecoverySnapshots(hqRoot, backupRoot);
-        const marker = path.join(hqRoot, ".hq", "rescue-blocked.json");
-        const tripped = runRescueCapture(liveArgs(hqRoot, backupRoot), baseEnv);
-        const tripOut = `${tripped.stdout}\n${tripped.stderr}\n${String(tripped.threw ?? "")}`;
-        expect(tripped.threw, tripOut).toBeUndefined();
-        expect(tripped.status, tripOut).toBe(1);
-        expect(tripOut).toContain("safety snapshot circuit breaker is open");
-        expect(fs.existsSync(marker)).toBe(true);
-        const body = JSON.parse(fs.readFileSync(marker, "utf8"));
-        expect(body.reason).toBe("recovery-snapshot-breaker");
-        expect(body.snapshots.sort()).toEqual([...planted].sort());
-        expect(body.hint).toContain("--recover-snapshots");
-        expect(Number.isNaN(Date.parse(body.at))).toBe(false);
-        expect(snapshotNames(backupRoot).sort()).toEqual([...planted].sort());
-        expect(tripOut).toContain("HQ_RESCUE_FAILURE_KIND=snapshot-recovery-circuit-breaker");
-        plantCompletedGoodSnapshot(hqRoot, backupRoot);
-        const inspected = runRescueCapture(liveArgs(hqRoot, backupRoot, ["--recover-snapshots"]), baseEnv);
-        const inspectedOut = `${inspected.stdout}\n${inspected.stderr}\n${String(inspected.threw ?? "")}`;
-        expect(inspected.threw, inspectedOut).toBeUndefined();
-        expect(inspected.status, inspectedOut).toBe(0);
-        expect(snapshotNames(backupRoot)).toHaveLength(1);
-        expect(fs.existsSync(marker)).toBe(false);
-        const recovered = runRescueCapture(liveArgs(hqRoot, backupRoot), baseEnv);
-        const recoveredOut = `${recovered.stdout}\n${recovered.stderr}\n${String(recovered.threw ?? "")}`;
-        expect(recovered.threw, recoveredOut).toBeUndefined();
-        expect(recovered.status, recoveredOut).toBe(0);
-        expect(fs.existsSync(marker)).toBe(false);
-    });
-    it("keeps the rescue-blocked marker when an update succeeds without a completed-good safety snapshot", () => {
-        const hqRoot = makeHqRoot();
-        const backupRoot = path.join(workDir, `backups-marker-no-backup-${caseSeq}`);
-        plantRecoverySnapshots(hqRoot, backupRoot);
-        const marker = path.join(hqRoot, ".hq", "rescue-blocked.json");
-        const tripped = runRescueCapture(liveArgs(hqRoot, backupRoot), baseEnv);
-        const tripOut = `${tripped.stdout}\n${tripped.stderr}\n${String(tripped.threw ?? "")}`;
-        expect(tripped.status, tripOut).toBe(1);
-        expect(fs.existsSync(marker)).toBe(true);
-        const unbacked = runRescueCapture(liveArgs(hqRoot, backupRoot, ["--no-backup"]), baseEnv);
-        const unbackedOut = `${unbacked.stdout}\n${unbacked.stderr}\n${String(unbacked.threw ?? "")}`;
-        expect(unbacked.threw, unbackedOut).toBeUndefined();
-        expect(unbacked.status, unbackedOut).toBe(0);
-        expect(fs.existsSync(marker)).toBe(true);
-    });
-    it("self-heals the breaker when the release and every carve-out have proof", () => {
-        const hqRoot = makeProofHqRoot();
-        const backupRoot = path.join(workDir, `backups-proof-${caseSeq}`);
-        const sourceSha = git(upstream, "rev-parse", "HEAD");
-        const planted = plantProofSnapshots(hqRoot, backupRoot, sourceSha);
-        const recovered = runRescueCapture(liveArgs(hqRoot, backupRoot), baseEnv);
-        const output = `${recovered.stdout}\n${recovered.stderr}\n${String(recovered.threw ?? "")}`;
-        expect(recovered.threw, output).toBeUndefined();
-        expect(recovered.status, output).toBe(0);
-        expect(recovered.stdout).toContain("==> Recovery proof passed: interrupted update matches the release and preserved paths; continuing.");
-        expect(recovered.stderr).toContain("HQ_RESCUE_NOTE=recovery-proof-passed");
-        const snapshots = snapshotNames(backupRoot);
-        expect(snapshots).toHaveLength(1);
-        const manifest = JSON.parse(fs.readFileSync(path.join(backupRoot, snapshots[0], "SNAPSHOT.json"), "utf8"));
-        expect(manifest.status).toBe("completed-good");
-        expect(fs.existsSync(path.join(backupRoot, planted[0]))).toBe(false);
-    });
-    it("keeps the breaker closed when a carve-out differs from the newest snapshot", () => {
-        const hqRoot = makeProofHqRoot();
-        const backupRoot = path.join(workDir, `backups-carveout-${caseSeq}`);
-        const sourceSha = git(upstream, "rev-parse", "HEAD");
-        const planted = plantProofSnapshots(hqRoot, backupRoot, sourceSha);
-        fs.mkdirSync(path.join(hqRoot, "core/packages"), { recursive: true });
-        fs.writeFileSync(path.join(hqRoot, "core/packages/local.txt"), "changed carve-out\n");
-        const blocked = runRescueCapture(liveArgs(hqRoot, backupRoot), baseEnv);
-        const output = `${blocked.stdout}\n${blocked.stderr}\n${String(blocked.threw ?? "")}`;
-        expect(blocked.threw, output).toBeUndefined();
-        expect(blocked.status, output).toBe(1);
-        expect(blocked.stderr).toContain("recovery proof failed: carve-out:core/packages");
-        expect(blocked.stderr).toContain("HQ_RESCUE_FAILURE_KIND=snapshot-recovery-circuit-breaker");
-        expect(snapshotNames(backupRoot).sort()).toEqual([...planted].sort());
-    });
-    it("keeps the breaker closed when the newest snapshot source SHA differs", () => {
-        const hqRoot = makeProofHqRoot();
-        const backupRoot = path.join(workDir, `backups-sha-${caseSeq}`);
-        const sourceSha = git(upstream, "rev-parse", "HEAD");
-        const planted = plantProofSnapshots(hqRoot, backupRoot, "b".repeat(40));
-        const blocked = runRescueCapture(liveArgs(hqRoot, backupRoot), baseEnv);
-        const output = `${blocked.stdout}\n${blocked.stderr}\n${String(blocked.threw ?? "")}`;
-        expect(sourceSha).toHaveLength(40);
-        expect(blocked.threw, output).toBeUndefined();
-        expect(blocked.status, output).toBe(1);
-        expect(blocked.stderr).toContain("recovery proof failed: sha");
-        expect(blocked.stderr).toContain("HQ_RESCUE_FAILURE_KIND=snapshot-recovery-circuit-breaker");
-        expect(snapshotNames(backupRoot).sort()).toEqual([...planted].sort());
-    });
-    it("still refuses --restore-snapshot for an incomplete recovery snapshot", () => {
-        const hqRoot = makeProofHqRoot();
-        const backupRoot = path.join(workDir, `backups-incomplete-${caseSeq}`);
-        const planted = plantRecoverySnapshots(hqRoot, backupRoot);
-        const manifestPath = path.join(backupRoot, planted[1], "SNAPSHOT.json");
-        const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8"));
-        manifest.skippedEntries = [
-            {
-                relativePath: "core/missing.txt",
-                operation: "read",
-                code: "EDEADLK",
-                message: "cloud placeholder",
-                occurredAt: "2026-09-07T12:00:00.000Z",
-            },
-        ];
-        fs.writeFileSync(manifestPath, JSON.stringify(manifest) + "\n");
-        const restored = runRescueCapture(liveArgs(hqRoot, backupRoot, [
-            "--recover-snapshots",
-            "--restore-snapshot",
-            planted[1],
-        ]), baseEnv);
-        const output = `${restored.stdout}\n${restored.stderr}\n${String(restored.threw ?? "")}`;
-        expect(restored.threw, output).toBeUndefined();
-        expect(restored.status, output).toBe(1);
-        expect(restored.stderr).toContain("is incomplete");
-        expect(restored.stderr).toContain("cannot restore every captured root");
-    });
-});
-//# sourceMappingURL=rescue-breaker.test.js.map