@indigoai-us/hq-cloud 6.18.44-alpha.48 → 6.18.44-alpha.49

package/dist/operation-lock.test.js

@@ -0,0 +1,796 @@
+/**
+ * Unit tests for the per-HQ-root operation mutex.
+ */
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import { spawn, spawnSync } from "child_process";
+import { once } from "node:events";
+import * as fs from "fs";
+import * as os from "os";
+import * as path from "path";
+import { acquireOperationLock, acquireOperationLockAsync, withOperationLock, withOperationLockSync, lockPathFor, rethrowLockCreateError, OperationLockedError, OperationLockUnwritableError, OPERATION_LOCKED_EXIT, OPERATION_LOCK_UNWRITABLE_EXIT, DEFAULT_LOCK_POLL_MS, } from "./operation-lock.js";
+/** A PID that is guaranteed dead: spawn a node that exits immediately, reuse its pid. */
+function deadPid() {
+    const r = spawnSync(process.execPath, ["-e", ""], { stdio: "ignore" });
+    if (!r.pid)
+        throw new Error("could not spawn to obtain a dead pid");
+    return r.pid;
+}
+function spawnLongLivedProcess() {
+    const child = spawn(process.execPath, ["-e", "setInterval(() => {}, 1000)"], {
+        stdio: "ignore",
+        windowsHide: true,
+    });
+    if (child.pid === undefined)
+        throw new Error("could not start a live lock-holder process");
+    return { child, pid: child.pid };
+}
+async function stopLongLivedProcess(child) {
+    if (child.exitCode !== null || child.signalCode !== null)
+        return;
+    const exited = once(child, "exit");
+    child.kill();
+    await exited;
+}
+function writeLock(p, info) {
+    fs.mkdirSync(path.dirname(p), { recursive: true });
+    const full = {
+        pid: process.pid,
+        command: "sync",
+        startedAt: new Date(0).toISOString(),
+        hqRoot: "/x",
+        ...info,
+    };
+    fs.writeFileSync(p, JSON.stringify(full));
+}
+function spawnGateWorker(root, stateDir, command) {
+    const moduleUrl = new URL("./operation-lock.ts", import.meta.url).href;
+    const program = `
+    import { acquireOperationLock } from ${JSON.stringify(moduleUrl)};
+    const handle = acquireOperationLock(${JSON.stringify(root)}, ${JSON.stringify(command)}, {
+      pollIntervalMs: 10,
+      onWaitStart: () => process.send?.({ kind: "waiting" }),
+    });
+    process.send?.({ kind: "acquired", pid: process.pid });
+    process.on("message", (message) => {
+      if (message === "release") {
+        handle.release();
+        process.exit(0);
+      }
+    });
+  `;
+    return spawn(process.execPath, ["--experimental-strip-types", "--input-type=module", "-e", program], {
+        env: { ...process.env, HQ_STATE_DIR: stateDir },
+        stdio: ["ignore", "ignore", "ignore", "ipc"],
+    });
+}
+function nextGateMessage(child, kind) {
+    return new Promise((resolve, reject) => {
+        const timeout = setTimeout(() => reject(new Error(`worker did not report ${kind}`)), 5_000);
+        const onMessage = (message) => {
+            if (message?.kind !== kind)
+                return;
+            clearTimeout(timeout);
+            child.off("message", onMessage);
+            child.off("error", onError);
+            child.off("exit", onExit);
+            resolve(message);
+        };
+        const onError = (error) => {
+            clearTimeout(timeout);
+            child.off("message", onMessage);
+            reject(error);
+        };
+        const onExit = (code, signal) => {
+            clearTimeout(timeout);
+            child.off("message", onMessage);
+            reject(new Error(`worker exited before ${kind}: code=${code} signal=${signal}`));
+        };
+        child.on("message", onMessage);
+        child.once("error", onError);
+        child.once("exit", onExit);
+    });
+}
+function releaseGateWorker(child) {
+    return new Promise((resolve, reject) => {
+        child.once("error", reject);
+        child.once("exit", (code) => (code === 0 ? resolve() : reject(new Error(`worker exit ${code}`))));
+        child.send("release");
+    });
+}
+describe("operation-lock", () => {
+    let stateDir;
+    let rootA;
+    let rootB;
+    beforeEach(() => {
+        stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "hq-oplock-state-"));
+        process.env.HQ_STATE_DIR = stateDir;
+        delete process.env.HQ_DISABLE_OP_LOCK;
+        delete process.env.HQ_OP_LOCK_TIMEOUT;
+        rootA = fs.mkdtempSync(path.join(os.tmpdir(), "hq-rootA-"));
+        rootB = fs.mkdtempSync(path.join(os.tmpdir(), "hq-rootB-"));
+    });
+    afterEach(() => {
+        fs.rmSync(stateDir, { recursive: true, force: true });
+        fs.rmSync(rootA, { recursive: true, force: true });
+        fs.rmSync(rootB, { recursive: true, force: true });
+        delete process.env.HQ_STATE_DIR;
+        delete process.env.HQ_DISABLE_OP_LOCK;
+        delete process.env.HQ_OP_LOCK_TIMEOUT;
+    });
+    it("the lock path is under the state dir, keyed per canonical root", () => {
+        const a = lockPathFor(rootA);
+        const b = lockPathFor(rootB);
+        expect(a.startsWith(path.join(stateDir, "locks"))).toBe(true);
+        expect(a).not.toBe(b); // different roots → different lock files
+        // canonical: a trailing-slash variant maps to the same lock
+        expect(lockPathFor(rootA + path.sep)).toBe(a);
+    });
+    it("scopes partition the lock: same root, different scope → different file, no mutual block", () => {
+        // feedback_ed98d810: reindex must not be starved by a long-lived sync/watch
+        // holder. Distinct scopes hash to distinct lock files and never block each
+        // other, while the same scope stays a real cross-process mutex.
+        const opPath = lockPathFor(rootA); // default "operation" scope
+        const reindexPath = lockPathFor(rootA, "reindex");
+        expect(reindexPath).not.toBe(opPath);
+        expect(path.basename(reindexPath).startsWith("reindex-")).toBe(true);
+        expect(path.basename(opPath).startsWith("operation-")).toBe(true);
+        let opHandle;
+        let reindexHandle;
+        try {
+            // A live "operation" holder (as a watch-mode runner keeps) does NOT block
+            // a "reindex"-scope acquire, even with no wait.
+            opHandle = acquireOperationLock(rootA, "sync");
+            expect(() => {
+                reindexHandle = acquireOperationLock(rootA, "reindex", {
+                    scope: "reindex",
+                    wait: false,
+                });
+            }).not.toThrow();
+            expect(reindexHandle && fs.existsSync(reindexHandle.path)).toBe(true);
+            // ...but a second acquire on the SAME "reindex" scope is still refused.
+            expect(() => acquireOperationLock(rootA, "reindex", { scope: "reindex", wait: false })).toThrowError(OperationLockedError);
+        }
+        finally {
+            reindexHandle?.release();
+            opHandle?.release();
+        }
+    });
+    it("fast exclusive acquisition excludes regular callers through the same marker", () => {
+        const fast = acquireOperationLock(rootA, "area-ledger-writer", {
+            mode: "exclusive",
+            scope: "area-ledger-writer-test",
+            timeoutSec: 0,
+            required: true,
+            fsync: false,
+            fastExclusive: true,
+        });
+        try {
+            expect(() => acquireOperationLock(rootA, "area-ledger-writer", {
+                mode: "exclusive",
+                scope: "area-ledger-writer-test",
+                timeoutSec: 0,
+                required: true,
+            })).toThrow(OperationLockedError);
+        }
+        finally {
+            fast.release();
+        }
+        const regular = acquireOperationLock(rootA, "area-ledger-writer", {
+            mode: "exclusive",
+            scope: "area-ledger-writer-test",
+            timeoutSec: 0,
+            required: true,
+        });
+        try {
+            expect(() => acquireOperationLock(rootA, "area-ledger-writer", {
+                mode: "exclusive",
+                scope: "area-ledger-writer-test",
+                timeoutSec: 0,
+                required: true,
+                fsync: false,
+                fastExclusive: true,
+            })).toThrow(OperationLockedError);
+        }
+        finally {
+            regular.release();
+        }
+    });
+    it("fast exclusive marker excludes a foreign process until normal release", async () => {
+        const scope = "area-ledger-writer-fast-process-test";
+        const moduleUrl = new URL("./operation-lock.ts", import.meta.url).href;
+        const program = `
+      import { acquireOperationLock } from ${JSON.stringify(moduleUrl)};
+      const handle = acquireOperationLock(${JSON.stringify(rootA)}, "area-ledger-writer", {
+        mode: "exclusive", scope: ${JSON.stringify(scope)}, timeoutSec: 0,
+        fsync: false, fastExclusive: true, required: true,
+      });
+      process.send?.({ kind: "acquired", pid: process.pid });
+      process.on("message", (message) => {
+        if (message === "release") { handle.release(); process.exit(0); }
+      });
+    `;
+        const child = spawn(process.execPath, ["--experimental-strip-types", "--input-type=module", "-e", program], { env: { ...process.env, HQ_STATE_DIR: stateDir }, stdio: ["ignore", "ignore", "ignore", "ipc"] });
+        try {
+            await nextGateMessage(child, "acquired");
+            expect(() => acquireOperationLock(rootA, "area-ledger-writer", {
+                mode: "exclusive",
+                scope,
+                timeoutSec: 0,
+                required: true,
+            })).toThrow(OperationLockedError);
+            await releaseGateWorker(child);
+            const afterRelease = acquireOperationLock(rootA, "area-ledger-writer", {
+                mode: "exclusive",
+                scope,
+                timeoutSec: 0,
+                required: true,
+            });
+            afterRelease.release();
+        }
+        finally {
+            if (child.exitCode === null && child.signalCode === null)
+                await releaseGateWorker(child);
+        }
+    });
+    it("F14: symlink aliases share the same root lock", () => {
+        const aliasParent = fs.mkdtempSync(path.join(os.tmpdir(), "hq-rootA-link-"));
+        const aliasRoot = path.join(aliasParent, "alias");
+        fs.symlinkSync(rootA, aliasRoot, "dir");
+        let realHandle;
+        let aliasHandle;
+        try {
+            expect(lockPathFor(aliasRoot)).toBe(lockPathFor(rootA));
+            realHandle = acquireOperationLock(rootA, "sync");
+            expect(() => {
+                aliasHandle = acquireOperationLock(aliasRoot, "rescue", { wait: false });
+            }).toThrowError(OperationLockedError);
+        }
+        finally {
+            aliasHandle?.release();
+            realHandle?.release();
+            fs.rmSync(aliasParent, { recursive: true, force: true });
+        }
+    });
+    it("acquires, writes holder info, and releases (file gone after release)", () => {
+        const h = acquireOperationLock(rootA, "sync");
+        expect(fs.existsSync(h.path)).toBe(true);
+        const info = JSON.parse(fs.readFileSync(h.path, "utf8"));
+        expect(info.pid).toBe(process.pid);
+        expect(info.command).toBe("sync");
+        h.release();
+        expect(fs.existsSync(h.path)).toBe(false);
+    });
+    it("admits independent sync-worker PIDs together, then hands off to queued rescue", async () => {
+        const first = spawnGateWorker(rootA, stateDir, "sync");
+        const second = spawnGateWorker(rootA, stateDir, "sync");
+        const [firstReady, secondReady] = await Promise.all([
+            nextGateMessage(first, "acquired"),
+            nextGateMessage(second, "acquired"),
+        ]);
+        expect(firstReady.pid).not.toBe(secondReady.pid);
+        const p = lockPathFor(rootA);
+        const memberPrefix = `${path.basename(p)}.shared-`;
+        expect(fs.readdirSync(path.dirname(p)).filter((entry) => entry.startsWith(memberPrefix))).toHaveLength(2);
+        const rescue = spawnGateWorker(rootA, stateDir, "rescue");
+        await nextGateMessage(rescue, "waiting");
+        // Writer preference closes admission as soon as rescue is queued: a new
+        // chore cannot sneak in behind the two already admitted worker PIDs.
+        expect(() => acquireOperationLock(rootA, "sync", { wait: false })).toThrowError(OperationLockedError);
+        await Promise.all([releaseGateWorker(first), releaseGateWorker(second)]);
+        await nextGateMessage(rescue, "acquired");
+        expect(() => acquireOperationLock(rootA, "sync", { wait: false })).toThrowError(OperationLockedError);
+        await releaseGateWorker(rescue);
+    }, 20_000);
+    it("reclaims a killed worker membership without creating a false-free window", async () => {
+        const worker = spawnGateWorker(rootA, stateDir, "sync");
+        await nextGateMessage(worker, "acquired");
+        const exited = new Promise((resolve) => worker.once("exit", () => resolve()));
+        worker.kill("SIGKILL");
+        await exited;
+        // The member file remains until a gate contender observes its dead PID.
+        // Rescue reclaims that stale lease before it declares exclusivity.
+        const rescue = acquireOperationLock(rootA, "rescue", { wait: false });
+        expect(rescue.info.command).toBe("rescue");
+        rescue.release();
+    }, 20_000);
+    it("async maintenance waits without blocking the parent event loop", async () => {
+        const worker = acquireOperationLock(rootA, "sync");
+        let timerRan = false;
+        const timer = new Promise((resolve) => setTimeout(() => {
+            timerRan = true;
+            resolve();
+        }, 20));
+        const rescue = acquireOperationLockAsync(rootA, "rescue", {
+            pollIntervalMs: 5,
+            onWaitStart: () => undefined,
+        });
+        await timer;
+        expect(timerRan).toBe(true);
+        worker.release();
+        (await rescue).release();
+    });
+    // Regression (HQ-CLI-2): when the lock's state dir is not writable, acquiring
+    // must fail with an actionable OperationLockUnwritableError, NOT a raw
+    // `EPERM: operation not permitted, open` (macOS) / `EACCES` (Linux) crash.
+    describe("unwritable lock dir (HQ-CLI-2)", () => {
+        // The classification decision (which fs error codes mean "unwritable lock"
+        // vs. a genuinely different fault) is verified directly against the exported
+        // `rethrowLockCreateError` helper. ESM module namespaces can't be spied, so
+        // we can't mock `fs.openSync`; testing the classifier is both deterministic
+        // (uid/OS-independent) and a tighter unit of the actual decision.
+        const permErr = (code, msg) => Object.assign(new Error(`${code}: ${msg}, open`), { code, syscall: "open" });
+        it.each(["EPERM", "EACCES", "EROFS"])("classifies %s as OperationLockUnwritableError with actionable guidance", (code) => {
+            const dir = path.join(stateDir, "locks");
+            let thrown;
+            try {
+                rethrowLockCreateError(permErr(code, "not permitted"), dir);
+            }
+            catch (e) {
+                thrown = e;
+            }
+            expect(thrown).toBeInstanceOf(OperationLockUnwritableError);
+            const err = thrown;
+            expect(err.lockDir).toBe(dir);
+            expect(err.cause.code).toBe(code);
+            // Actionable: names the recovery levers so the user can self-serve.
+            expect(err.message).toContain("HQ_DISABLE_OP_LOCK=1");
+            expect(err.message).toContain("HQ_STATE_DIR");
+        });
+        it("POSITIVE CONTROL: a NON-permission fs error propagates unchanged (not masked)", () => {
+            // A disk-full error is a genuine, different fault — it must NOT be
+            // reclassified as an unwritable-lock error.
+            const dir = path.join(stateDir, "locks");
+            let thrown;
+            try {
+                rethrowLockCreateError(permErr("ENOSPC", "no space left on device"), dir);
+            }
+            catch (e) {
+                thrown = e;
+            }
+            expect(thrown).not.toBeInstanceOf(OperationLockUnwritableError);
+            expect(thrown.code).toBe("ENOSPC");
+        });
+        it("real unwritable locks dir → OperationLockUnwritableError, distinct exit (skipped as root)", () => {
+            // End-to-end proof through the real acquire path: a locks dir with no write
+            // bit makes the temp-file create fail (EACCES on Linux / EPERM on macOS),
+            // and acquire must surface the typed error rather than crash. CI runners
+            // run non-root, so this exercises the full path there.
+            // Windows does not enforce these POSIX chmod mode bits on directories.
+            if (process.platform === "win32")
+                return;
+            if (typeof process.getuid === "function" && process.getuid() === 0) {
+                // root bypasses DAC permission bits, so a chmod-based unwritable dir
+                // cannot be exercised — the classifier cases above cover the logic.
+                return;
+            }
+            const locksDir = path.join(stateDir, "locks");
+            fs.mkdirSync(locksDir, { recursive: true });
+            fs.chmodSync(locksDir, 0o500); // r-x: cannot create files inside
+            try {
+                expect(() => acquireOperationLock(rootA, "reindex", { wait: false })).toThrowError(OperationLockUnwritableError);
+            }
+            finally {
+                fs.chmodSync(locksDir, 0o700); // restore so afterEach cleanup can rm it
+            }
+        });
+        it("async acquire also surfaces the typed error on an unwritable locks dir (skipped as root)", async () => {
+            // Windows does not enforce these POSIX chmod mode bits on directories.
+            if (process.platform === "win32")
+                return;
+            if (typeof process.getuid === "function" && process.getuid() === 0) {
+                return;
+            }
+            const locksDir = path.join(stateDir, "locks");
+            fs.mkdirSync(locksDir, { recursive: true });
+            fs.chmodSync(locksDir, 0o500);
+            try {
+                await expect(acquireOperationLockAsync(rootB, "reindex", { wait: false })).rejects.toBeInstanceOf(OperationLockUnwritableError);
+            }
+            finally {
+                fs.chmodSync(locksDir, 0o700);
+            }
+        });
+        it("exposes a distinct exit code from the busy case", () => {
+            expect(OPERATION_LOCK_UNWRITABLE_EXIT).not.toBe(OPERATION_LOCKED_EXIT);
+            expect(OPERATION_LOCK_UNWRITABLE_EXIT).toBe(18);
+        });
+    });
+    it("refuses immediately (wait:false) with the holder's command + pid when a LIVE process holds it", async () => {
+        const holder = spawnLongLivedProcess();
+        try {
+            // A spawned child represents a distinct live holder and stays alive through
+            // the assertions. The same-process reclaim path does not apply; wait:false
+            // preserves the refuse-immediately behavior while the default waits.
+            writeLock(lockPathFor(rootA), { pid: holder.pid, command: "rescue" });
+            expect(() => acquireOperationLock(rootA, "sync", { wait: false })).toThrowError(OperationLockedError);
+            try {
+                acquireOperationLock(rootA, "sync", { wait: false });
+            }
+            catch (e) {
+                const err = e;
+                expect(err.holder.command).toBe("rescue");
+                expect(err.holder.pid).toBe(holder.pid);
+                expect(err.message).toContain("rescue");
+                expect(err.message).toContain("pid " + holder.pid);
+            }
+        }
+        finally {
+            await stopLongLivedProcess(holder.child);
+        }
+    });
+    it("timeoutSec:0 refuses immediately (no wait) — equivalent to wait:false", async () => {
+        const holder = spawnLongLivedProcess();
+        try {
+            writeLock(lockPathFor(rootA), { pid: holder.pid, command: "sync" });
+            const start = Date.now();
+            expect(() => acquireOperationLock(rootA, "reindex", { timeoutSec: 0 })).toThrowError(OperationLockedError);
+            // Did not actually sleep.
+            expect(Date.now() - start).toBeLessThan(DEFAULT_LOCK_POLL_MS);
+        }
+        finally {
+            await stopLongLivedProcess(holder.child);
+        }
+    });
+    it("a bounded timeoutSec waits, then refuses with the old message + exit code", async () => {
+        const holder = spawnLongLivedProcess();
+        try {
+            writeLock(lockPathFor(rootA), { pid: holder.pid, command: "rescue" });
+            const start = Date.now();
+            let thrown;
+            try {
+                // 150ms bound, 40ms poll → waits ~150ms then gives up. Suppress the
+                // stderr status line with a no-op onWaitStart.
+                acquireOperationLock(rootA, "sync", {
+                    timeoutSec: 0.15,
+                    pollIntervalMs: 40,
+                    onWaitStart: () => { },
+                });
+            }
+            catch (e) {
+                thrown = e;
+            }
+            const elapsed = Date.now() - start;
+            expect(thrown).toBeInstanceOf(OperationLockedError);
+            expect(thrown.message).toContain("rescue");
+            expect(OPERATION_LOCKED_EXIT).toBe(17);
+            // It actually waited (didn't refuse instantly) but didn't hang forever.
+            expect(elapsed).toBeGreaterThanOrEqual(120);
+            expect(elapsed).toBeLessThan(3000);
+            const waiterPrefix = path.basename(lockPathFor(rootA)) + ".wait-";
+            expect(fs.readdirSync(path.dirname(lockPathFor(rootA))).filter((entry) => entry.startsWith(waiterPrefix))).toEqual([]);
+        }
+        finally {
+            await stopLongLivedProcess(holder.child);
+        }
+    });
+    it("HQ_OP_LOCK_TIMEOUT env bounds the wait when no explicit option is given", async () => {
+        const holder = spawnLongLivedProcess();
+        try {
+            process.env.HQ_OP_LOCK_TIMEOUT = "0"; // 0 → refuse immediately
+            writeLock(lockPathFor(rootA), { pid: holder.pid, command: "sync" });
+            const start = Date.now();
+            expect(() => acquireOperationLock(rootA, "rescue", { onWaitStart: () => { } })).toThrowError(OperationLockedError);
+            expect(Date.now() - start).toBeLessThan(DEFAULT_LOCK_POLL_MS);
+        }
+        finally {
+            await stopLongLivedProcess(holder.child);
+        }
+    });
+    it("an explicit timeoutSec overrides the HQ_OP_LOCK_TIMEOUT env", async () => {
+        const holder = spawnLongLivedProcess();
+        try {
+            process.env.HQ_OP_LOCK_TIMEOUT = "9999"; // would be a near-infinite wait
+            writeLock(lockPathFor(rootA), { pid: holder.pid, command: "sync" });
+            // The explicit 0 wins → refuse immediately rather than honoring the env.
+            expect(() => acquireOperationLock(rootA, "rescue", { timeoutSec: 0 })).toThrowError(OperationLockedError);
+        }
+        finally {
+            await stopLongLivedProcess(holder.child);
+        }
+    });
+    it("onWaitStart fires exactly once, naming the holder, even across many polls", async () => {
+        const holder = spawnLongLivedProcess();
+        try {
+            writeLock(lockPathFor(rootA), { pid: holder.pid, command: "rescue" });
+            const calls = [];
+            expect(() => acquireOperationLock(rootA, "sync", {
+                timeoutSec: 0.16,
+                pollIntervalMs: 30, // ~5 polls within the window
+                onWaitStart: (lockHolder, attempted) => calls.push({ cmd: lockHolder.command, attempted }),
+            })).toThrowError(OperationLockedError);
+            expect(calls).toHaveLength(1);
+            expect(calls[0]).toEqual({ cmd: "rescue", attempted: "sync" });
+        }
+        finally {
+            await stopLongLivedProcess(holder.child);
+        }
+    });
+    it("a waiter acquires the lock the moment the holder releases (async poll path)", async () => {
+        const holder = spawnLongLivedProcess();
+        try {
+            const p = lockPathFor(rootA);
+            // A foreign LIVE holder (a spawned process) initially owns the lock.
+            writeLock(p, { pid: holder.pid, command: "rescue" });
+            // Simulate the holder finishing ~80ms in by removing its lock file.
+            const release = setTimeout(() => fs.rmSync(p, { force: true }), 80);
+            let releaseLock;
+            try {
+                const start = Date.now();
+                const h = await acquireOperationLockAsync(rootA, "sync", {
+                    pollIntervalMs: 20,
+                    onWaitStart: () => { },
+                });
+                releaseLock = h.release;
+                const elapsed = Date.now() - start;
+                // We waited for the release, then took it over.
+                expect(elapsed).toBeGreaterThanOrEqual(60);
+                const info = JSON.parse(fs.readFileSync(h.path, "utf8"));
+                expect(info.pid).toBe(process.pid);
+                expect(info.command).toBe("sync");
+            }
+            finally {
+                clearTimeout(release);
+                releaseLock?.();
+            }
+        }
+        finally {
+            await stopLongLivedProcess(holder.child);
+        }
+    });
+    it("a waiting rescue gets the next release before continuous watch passes", async () => {
+        // Model an active watch pass, then keep scheduling another short watch pass
+        // synchronously at every release. Without waiter intent, that reacquire loop
+        // leaves no observable free interval for rescue's poll and can starve it
+        // forever. With deferral, the already-waiting rescue owns the next handoff.
+        const activePass = acquireOperationLock(rootA, "sync", { wait: false });
+        let announceWait = () => { };
+        const waitAnnounced = new Promise((resolve) => {
+            announceWait = resolve;
+        });
+        const rescuePending = acquireOperationLockAsync(rootA, "rescue", {
+            pollIntervalMs: 5,
+            onWaitStart: announceWait,
+        });
+        await waitAnnounced;
+        let stopped = false;
+        let watchPasses = 0;
+        let resolveWatchStopped = () => { };
+        const watchStopped = new Promise((resolve) => {
+            resolveWatchStopped = resolve;
+        });
+        const runNextWatchPass = () => {
+            if (stopped) {
+                resolveWatchStopped();
+                return;
+            }
+            try {
+                const handle = acquireOperationLock(rootA, "sync", {
+                    timeoutSec: 0,
+                    deferToWaiters: true,
+                    onWaitStart: () => { },
+                });
+                watchPasses++;
+                setTimeout(() => {
+                    handle.release();
+                    runNextWatchPass();
+                }, 2);
+            }
+            catch (err) {
+                if (!(err instanceof OperationLockedError))
+                    throw err;
+                setTimeout(runNextWatchPass, 2);
+            }
+        };
+        runNextWatchPass();
+        activePass.release();
+        let timeout;
+        const rescue = await Promise.race([
+            rescuePending,
+            new Promise((_resolve, reject) => {
+                timeout = setTimeout(() => reject(new Error("rescue handoff timed out")), 250);
+            }),
+        ]).finally(() => clearTimeout(timeout));
+        // Continuous watch attempts remained excluded while rescue acquired and
+        // held the same operation lock: priority never weakens mutual exclusion.
+        await new Promise((resolve) => setTimeout(resolve, 20));
+        expect(rescue.info.command).toBe("rescue");
+        expect(watchPasses).toBe(0);
+        rescue.release();
+        await new Promise((resolve, reject) => {
+            const deadline = setTimeout(() => reject(new Error("watch did not resume after rescue")), 250);
+            const check = () => {
+                if (watchPasses > 0) {
+                    clearTimeout(deadline);
+                    resolve();
+                }
+                else {
+                    setTimeout(check, 2);
+                }
+            };
+            check();
+        });
+        stopped = true;
+        await watchStopped;
+    });
+    it("a dead waiter's intent is pruned instead of blocking watch mode", () => {
+        const p = lockPathFor(rootA);
+        const stalePid = deadPid();
+        const waiterPath = `${p}.wait-${stalePid}-stale`;
+        writeLock(waiterPath, { pid: stalePid, command: "rescue" });
+        const handle = acquireOperationLock(rootA, "sync", {
+            timeoutSec: 0,
+            deferToWaiters: true,
+        });
+        expect(fs.existsSync(waiterPath)).toBe(false);
+        expect(handle.info.command).toBe("sync");
+        handle.release();
+    });
+    it("multiple foreign holders in a row: each release lets the next waiter in (no FIFO guarantee)", async () => {
+        const holder = spawnLongLivedProcess();
+        try {
+            // The mutex is CROSS-PROCESS: it keys liveness on the holder's PID. Two
+            // waiters in the SAME process share a pid, so the same-process reclaim path
+            // would let them stomp each other — that scenario is unsupported by design.
+            // Here we model the real case: a sequence of FOREIGN holders (a spawned process) that
+            // each release, with a single waiter acquiring the instant the lock frees.
+            // Order among multiple distinct-process waiters is whoever wins the next
+            // O_EXCL race after a free — best-effort, NOT FIFO (documented).
+            const p = lockPathFor(rootA);
+            writeLock(p, { pid: holder.pid, command: "sync" });
+            // Free it shortly; the waiter should grab it right after.
+            const release = setTimeout(() => fs.rmSync(p, { force: true }), 50);
+            let releaseLock;
+            try {
+                const h = await acquireOperationLockAsync(rootA, "reindex", {
+                    pollIntervalMs: 15,
+                    onWaitStart: () => { },
+                });
+                releaseLock = h.release;
+                const info = JSON.parse(fs.readFileSync(h.path, "utf8"));
+                expect(info.command).toBe("reindex");
+                expect(info.pid).toBe(process.pid);
+            }
+            finally {
+                clearTimeout(release);
+                releaseLock?.();
+            }
+        }
+        finally {
+            await stopLongLivedProcess(holder.child);
+        }
+    });
+    it("reclaims a stale lock whose holder PID is dead (takeover, never waits)", () => {
+        const stale = deadPid();
+        writeLock(lockPathFor(rootA), { pid: stale, command: "sync" });
+        const start = Date.now();
+        // The dead holder must not block us — even with an infinite default wait,
+        // takeover is immediate.
+        const h = acquireOperationLock(rootA, "rescue");
+        expect(Date.now() - start).toBeLessThan(DEFAULT_LOCK_POLL_MS);
+        const info = JSON.parse(fs.readFileSync(h.path, "utf8"));
+        expect(info.pid).toBe(process.pid); // we took it over
+        expect(info.command).toBe("rescue");
+        h.release();
+    });
+    it("treats a torn/unreadable lock file as busy", () => {
+        const p = lockPathFor(rootA);
+        fs.mkdirSync(path.dirname(p), { recursive: true });
+        fs.writeFileSync(p, "{ this is not valid json");
+        expect(() => acquireOperationLock(rootA, "reindex", { wait: false })).toThrowError(OperationLockedError);
+        expect(fs.readFileSync(p, "utf8")).toBe("{ this is not valid json");
+    });
+    it("F34: fresh empty lock files are not reclaimed as stale", () => {
+        const p = lockPathFor(rootA);
+        fs.mkdirSync(path.dirname(p), { recursive: true });
+        fs.closeSync(fs.openSync(p, "wx"));
+        let handle;
+        try {
+            expect(() => {
+                handle = acquireOperationLock(rootA, "rescue", { wait: false });
+            }).toThrow();
+            expect(fs.existsSync(p)).toBe(true);
+            expect(fs.readFileSync(p, "utf8")).toBe("");
+        }
+        finally {
+            handle?.release();
+            fs.rmSync(p, { force: true });
+        }
+    });
+    it("R-F34: initializing empty and torn locks are busy, not reclaimed", () => {
+        const p = lockPathFor(rootA);
+        fs.mkdirSync(path.dirname(p), { recursive: true });
+        fs.closeSync(fs.openSync(p, "wx"));
+        const old = new Date(Date.now() - 10_000);
+        fs.utimesSync(p, old, old);
+        expect(() => acquireOperationLock(rootA, "sync", { wait: false })).toThrowError(OperationLockedError);
+        expect(fs.readFileSync(p, "utf8")).toBe("");
+        fs.writeFileSync(p, "{ torn");
+        expect(() => acquireOperationLock(rootA, "sync", { wait: false })).toThrowError(OperationLockedError);
+        expect(fs.readFileSync(p, "utf8")).toBe("{ torn");
+    });
+    it("fast exclusive writers preserve fresh initializing markers and reclaim expired ones", () => {
+        const scope = "area-ledger-writer-fast-metadata-test";
+        const p = lockPathFor(rootA, scope);
+        fs.mkdirSync(path.dirname(p), { recursive: true });
+        fs.closeSync(fs.openSync(p, "wx"));
+        const options = {
+            mode: "exclusive",
+            scope,
+            timeoutSec: 0,
+            required: true,
+            fsync: false,
+            fastExclusive: true,
+        };
+        expect(() => acquireOperationLock(rootA, "area-ledger-writer", options)).toThrowError(OperationLockedError);
+        expect(fs.readFileSync(p, "utf8")).toBe("");
+        const old = new Date(Date.now() - 10_000);
+        fs.utimesSync(p, old, old);
+        const handle = acquireOperationLock(rootA, "area-ledger-writer", options);
+        try {
+            expect(JSON.parse(fs.readFileSync(p, "utf8"))).toMatchObject({
+                pid: process.pid,
+                command: "area-ledger-writer",
+            });
+        }
+        finally {
+            handle.release();
+        }
+    });
+    it("different HQ roots are independent — both may hold concurrently", () => {
+        const a = acquireOperationLock(rootA, "sync");
+        const b = acquireOperationLock(rootB, "rescue"); // must NOT block
+        expect(fs.existsSync(a.path)).toBe(true);
+        expect(fs.existsSync(b.path)).toBe(true);
+        expect(a.path).not.toBe(b.path);
+        a.release();
+        b.release();
+    });
+    it("the same root is mutually exclusive across different commands", async () => {
+        const holder = spawnLongLivedProcess();
+        try {
+            // A live sync in ANOTHER process holds the root (a spawned process holds it).
+            const p = lockPathFor(rootA);
+            writeLock(p, { pid: holder.pid, command: "sync" });
+            // Neither rescue nor reindex may acquire while that sync holds it
+            // (wait:false → assert the refusal without hanging on the new wait default).
+            expect(() => acquireOperationLock(rootA, "rescue", { wait: false })).toThrowError(OperationLockedError);
+            expect(() => acquireOperationLock(rootA, "reindex", { wait: false })).toThrowError(OperationLockedError);
+            // Once that sync finishes (its lock is gone), the next command acquires.
+            fs.unlinkSync(p);
+            const h2 = acquireOperationLock(rootA, "reindex");
+            expect(fs.existsSync(h2.path)).toBe(true);
+            h2.release();
+        }
+        finally {
+            await stopLongLivedProcess(holder.child);
+        }
+    });
+    it("withOperationLockSync releases even when the body throws", () => {
+        const p = lockPathFor(rootA);
+        expect(() => withOperationLockSync(rootA, "reindex", () => {
+            expect(fs.existsSync(p)).toBe(true); // held during the body
+            throw new Error("boom");
+        })).toThrow("boom");
+        expect(fs.existsSync(p)).toBe(false); // released on the way out
+    });
+    it("withOperationLock (async) releases even when the body throws", async () => {
+        const p = lockPathFor(rootA);
+        await expect(withOperationLock(rootA, "sync", async () => {
+            expect(fs.existsSync(p)).toBe(true);
+            throw new Error("boom");
+        })).rejects.toThrow("boom");
+        expect(fs.existsSync(p)).toBe(false);
+    });
+    it("HQ_DISABLE_OP_LOCK=1 makes acquisition a no-op", () => {
+        process.env.HQ_DISABLE_OP_LOCK = "1";
+        // Even with a live holder on record, the escape hatch acquires without error.
+        writeLock(lockPathFor(rootA), { pid: process.pid, command: "sync" });
+        const h = acquireOperationLock(rootA, "rescue");
+        expect(h.path).toBe(""); // no real lock file written
+        h.release(); // no-op, no throw
+    });
+    it("OPERATION_LOCKED_EXIT is a stable non-zero code", () => {
+        expect(OPERATION_LOCKED_EXIT).toBe(17);
+    });
+});
+//# sourceMappingURL=operation-lock.test.js.map