@indigoai-us/hq-cloud 6.14.50 → 6.15.1

package/src/operation-lock.test.ts

@@ -1,484 +0,0 @@
-/**
- * Unit tests for the per-HQ-root operation mutex.
- */
-
-import { describe, it, expect, beforeEach, afterEach } from "vitest";
-import { spawnSync } from "child_process";
-import * as fs from "fs";
-import * as os from "os";
-import * as path from "path";
-import {
-  acquireOperationLock,
-  acquireOperationLockAsync,
-  withOperationLock,
-  withOperationLockSync,
-  lockPathFor,
-  rethrowLockCreateError,
-  OperationLockedError,
-  OperationLockUnwritableError,
-  OPERATION_LOCKED_EXIT,
-  OPERATION_LOCK_UNWRITABLE_EXIT,
-  DEFAULT_LOCK_POLL_MS,
-  type LockInfo,
-} from "./operation-lock.js";
-
-/** A PID that is guaranteed dead: spawn a node that exits immediately, reuse its pid. */
-function deadPid(): number {
-  const r = spawnSync(process.execPath, ["-e", ""], { stdio: "ignore" });
-  if (!r.pid) throw new Error("could not spawn to obtain a dead pid");
-  return r.pid;
-}
-
-function writeLock(p: string, info: Partial<LockInfo>): void {
-  fs.mkdirSync(path.dirname(p), { recursive: true });
-  const full: LockInfo = {
-    pid: 1,
-    command: "sync",
-    startedAt: new Date(0).toISOString(),
-    hqRoot: "/x",
-    ...info,
-  };
-  fs.writeFileSync(p, JSON.stringify(full));
-}
-
-describe("operation-lock", () => {
-  let stateDir: string;
-  let rootA: string;
-  let rootB: string;
-
-  beforeEach(() => {
-    stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "hq-oplock-state-"));
-    process.env.HQ_STATE_DIR = stateDir;
-    delete process.env.HQ_DISABLE_OP_LOCK;
-    delete process.env.HQ_OP_LOCK_TIMEOUT;
-    rootA = fs.mkdtempSync(path.join(os.tmpdir(), "hq-rootA-"));
-    rootB = fs.mkdtempSync(path.join(os.tmpdir(), "hq-rootB-"));
-  });
-
-  afterEach(() => {
-    fs.rmSync(stateDir, { recursive: true, force: true });
-    fs.rmSync(rootA, { recursive: true, force: true });
-    fs.rmSync(rootB, { recursive: true, force: true });
-    delete process.env.HQ_STATE_DIR;
-    delete process.env.HQ_DISABLE_OP_LOCK;
-    delete process.env.HQ_OP_LOCK_TIMEOUT;
-  });
-
-  it("the lock path is under the state dir, keyed per canonical root", () => {
-    const a = lockPathFor(rootA);
-    const b = lockPathFor(rootB);
-    expect(a.startsWith(path.join(stateDir, "locks"))).toBe(true);
-    expect(a).not.toBe(b); // different roots → different lock files
-    // canonical: a trailing-slash variant maps to the same lock
-    expect(lockPathFor(rootA + path.sep)).toBe(a);
-  });
-
-  it("scopes partition the lock: same root, different scope → different file, no mutual block", () => {
-    // feedback_ed98d810: reindex must not be starved by a long-lived sync/watch
-    // holder. Distinct scopes hash to distinct lock files and never block each
-    // other, while the same scope stays a real cross-process mutex.
-    const opPath = lockPathFor(rootA); // default "operation" scope
-    const reindexPath = lockPathFor(rootA, "reindex");
-    expect(reindexPath).not.toBe(opPath);
-    expect(path.basename(reindexPath).startsWith("reindex-")).toBe(true);
-    expect(path.basename(opPath).startsWith("operation-")).toBe(true);
-
-    let opHandle: ReturnType<typeof acquireOperationLock> | undefined;
-    let reindexHandle: ReturnType<typeof acquireOperationLock> | undefined;
-    try {
-      // A live "operation" holder (as a watch-mode runner keeps) does NOT block
-      // a "reindex"-scope acquire, even with no wait.
-      opHandle = acquireOperationLock(rootA, "sync");
-      expect(() => {
-        reindexHandle = acquireOperationLock(rootA, "reindex", {
-          scope: "reindex",
-          wait: false,
-        });
-      }).not.toThrow();
-      expect(reindexHandle && fs.existsSync(reindexHandle.path)).toBe(true);
-      // ...but a second acquire on the SAME "reindex" scope is still refused.
-      expect(() =>
-        acquireOperationLock(rootA, "reindex", { scope: "reindex", wait: false }),
-      ).toThrowError(OperationLockedError);
-    } finally {
-      reindexHandle?.release();
-      opHandle?.release();
-    }
-  });
-
-  it("F14: symlink aliases share the same root lock", () => {
-    const aliasParent = fs.mkdtempSync(path.join(os.tmpdir(), "hq-rootA-link-"));
-    const aliasRoot = path.join(aliasParent, "alias");
-    fs.symlinkSync(rootA, aliasRoot, "dir");
-
-    let realHandle: ReturnType<typeof acquireOperationLock> | undefined;
-    let aliasHandle: ReturnType<typeof acquireOperationLock> | undefined;
-    try {
-      expect(lockPathFor(aliasRoot)).toBe(lockPathFor(rootA));
-
-      realHandle = acquireOperationLock(rootA, "sync");
-      expect(() => {
-        aliasHandle = acquireOperationLock(aliasRoot, "rescue", { wait: false });
-      }).toThrowError(OperationLockedError);
-    } finally {
-      aliasHandle?.release();
-      realHandle?.release();
-      fs.rmSync(aliasParent, { recursive: true, force: true });
-    }
-  });
-
-  it("acquires, writes holder info, and releases (file gone after release)", () => {
-    const h = acquireOperationLock(rootA, "sync");
-    expect(fs.existsSync(h.path)).toBe(true);
-    const info = JSON.parse(fs.readFileSync(h.path, "utf8")) as LockInfo;
-    expect(info.pid).toBe(process.pid);
-    expect(info.command).toBe("sync");
-    h.release();
-    expect(fs.existsSync(h.path)).toBe(false);
-  });
-
-  // Regression (HQ-CLI-2): when the lock's state dir is not writable, acquiring
-  // must fail with an actionable OperationLockUnwritableError, NOT a raw
-  // `EPERM: operation not permitted, open` (macOS) / `EACCES` (Linux) crash.
-  describe("unwritable lock dir (HQ-CLI-2)", () => {
-    // The classification decision (which fs error codes mean "unwritable lock"
-    // vs. a genuinely different fault) is verified directly against the exported
-    // `rethrowLockCreateError` helper. ESM module namespaces can't be spied, so
-    // we can't mock `fs.openSync`; testing the classifier is both deterministic
-    // (uid/OS-independent) and a tighter unit of the actual decision.
-    const permErr = (code: string, msg: string): NodeJS.ErrnoException =>
-      Object.assign(new Error(`${code}: ${msg}, open`), { code, syscall: "open" });
-
-    it.each(["EPERM", "EACCES", "EROFS"])(
-      "classifies %s as OperationLockUnwritableError with actionable guidance",
-      (code) => {
-        const dir = path.join(stateDir, "locks");
-        let thrown: unknown;
-        try {
-          rethrowLockCreateError(permErr(code, "not permitted"), dir);
-        } catch (e) {
-          thrown = e;
-        }
-        expect(thrown).toBeInstanceOf(OperationLockUnwritableError);
-        const err = thrown as OperationLockUnwritableError;
-        expect(err.lockDir).toBe(dir);
-        expect(err.cause.code).toBe(code);
-        // Actionable: names the recovery levers so the user can self-serve.
-        expect(err.message).toContain("HQ_DISABLE_OP_LOCK=1");
-        expect(err.message).toContain("HQ_STATE_DIR");
-      },
-    );
-
-    it("POSITIVE CONTROL: a NON-permission fs error propagates unchanged (not masked)", () => {
-      // A disk-full error is a genuine, different fault — it must NOT be
-      // reclassified as an unwritable-lock error.
-      const dir = path.join(stateDir, "locks");
-      let thrown: unknown;
-      try {
-        rethrowLockCreateError(permErr("ENOSPC", "no space left on device"), dir);
-      } catch (e) {
-        thrown = e;
-      }
-      expect(thrown).not.toBeInstanceOf(OperationLockUnwritableError);
-      expect((thrown as NodeJS.ErrnoException).code).toBe("ENOSPC");
-    });
-
-    it("real unwritable locks dir → OperationLockUnwritableError, distinct exit (skipped as root)", () => {
-      // End-to-end proof through the real acquire path: a locks dir with no write
-      // bit makes the temp-file create fail (EACCES on Linux / EPERM on macOS),
-      // and acquire must surface the typed error rather than crash. CI runners
-      // run non-root, so this exercises the full path there.
-      if (typeof process.getuid === "function" && process.getuid() === 0) {
-        // root bypasses DAC permission bits, so a chmod-based unwritable dir
-        // cannot be exercised — the classifier cases above cover the logic.
-        return;
-      }
-      const locksDir = path.join(stateDir, "locks");
-      fs.mkdirSync(locksDir, { recursive: true });
-      fs.chmodSync(locksDir, 0o500); // r-x: cannot create files inside
-      try {
-        expect(() => acquireOperationLock(rootA, "reindex", { wait: false })).toThrowError(
-          OperationLockUnwritableError,
-        );
-      } finally {
-        fs.chmodSync(locksDir, 0o700); // restore so afterEach cleanup can rm it
-      }
-    });
-
-    it("async acquire also surfaces the typed error on an unwritable locks dir (skipped as root)", async () => {
-      if (typeof process.getuid === "function" && process.getuid() === 0) {
-        return;
-      }
-      const locksDir = path.join(stateDir, "locks");
-      fs.mkdirSync(locksDir, { recursive: true });
-      fs.chmodSync(locksDir, 0o500);
-      try {
-        await expect(
-          acquireOperationLockAsync(rootB, "reindex", { wait: false }),
-        ).rejects.toBeInstanceOf(OperationLockUnwritableError);
-      } finally {
-        fs.chmodSync(locksDir, 0o700);
-      }
-    });
-
-    it("exposes a distinct exit code from the busy case", () => {
-      expect(OPERATION_LOCK_UNWRITABLE_EXIT).not.toBe(OPERATION_LOCKED_EXIT);
-      expect(OPERATION_LOCK_UNWRITABLE_EXIT).toBe(18);
-    });
-  });
-
-  it("refuses immediately (wait:false) with the holder's command + pid when a LIVE process holds it", () => {
-    // Simulate a DIFFERENT live process holding the lock. PID 1 (init/systemd)
-    // is always alive and is never our own pid, so kill(1,0) reports alive and
-    // the same-process reclaim path does not apply. `wait:false` keeps the old
-    // refuse-immediately behavior (the default is now to WAIT).
-    writeLock(lockPathFor(rootA), { pid: 1, command: "rescue" });
-    expect(() => acquireOperationLock(rootA, "sync", { wait: false })).toThrowError(
-      OperationLockedError,
-    );
-    try {
-      acquireOperationLock(rootA, "sync", { wait: false });
-    } catch (e) {
-      const err = e as OperationLockedError;
-      expect(err.holder.command).toBe("rescue");
-      expect(err.holder.pid).toBe(1);
-      expect(err.message).toContain("rescue");
-      expect(err.message).toContain("pid 1");
-    }
-  });
-
-  it("timeoutSec:0 refuses immediately (no wait) — equivalent to wait:false", () => {
-    writeLock(lockPathFor(rootA), { pid: 1, command: "sync" });
-    const start = Date.now();
-    expect(() => acquireOperationLock(rootA, "reindex", { timeoutSec: 0 })).toThrowError(
-      OperationLockedError,
-    );
-    // Did not actually sleep.
-    expect(Date.now() - start).toBeLessThan(DEFAULT_LOCK_POLL_MS);
-  });
-
-  it("a bounded timeoutSec waits, then refuses with the old message + exit code", () => {
-    writeLock(lockPathFor(rootA), { pid: 1, command: "rescue" });
-    const start = Date.now();
-    let thrown: unknown;
-    try {
-      // 150ms bound, 40ms poll → waits ~150ms then gives up. Suppress the
-      // stderr status line with a no-op onWaitStart.
-      acquireOperationLock(rootA, "sync", {
-        timeoutSec: 0.15,
-        pollIntervalMs: 40,
-        onWaitStart: () => {},
-      });
-    } catch (e) {
-      thrown = e;
-    }
-    const elapsed = Date.now() - start;
-    expect(thrown).toBeInstanceOf(OperationLockedError);
-    expect((thrown as OperationLockedError).message).toContain("rescue");
-    expect(OPERATION_LOCKED_EXIT).toBe(17);
-    // It actually waited (didn't refuse instantly) but didn't hang forever.
-    expect(elapsed).toBeGreaterThanOrEqual(120);
-    expect(elapsed).toBeLessThan(3000);
-  });
-
-  it("HQ_OP_LOCK_TIMEOUT env bounds the wait when no explicit option is given", () => {
-    process.env.HQ_OP_LOCK_TIMEOUT = "0"; // 0 → refuse immediately
-    writeLock(lockPathFor(rootA), { pid: 1, command: "sync" });
-    const start = Date.now();
-    expect(() =>
-      acquireOperationLock(rootA, "rescue", { onWaitStart: () => {} }),
-    ).toThrowError(OperationLockedError);
-    expect(Date.now() - start).toBeLessThan(DEFAULT_LOCK_POLL_MS);
-  });
-
-  it("an explicit timeoutSec overrides the HQ_OP_LOCK_TIMEOUT env", () => {
-    process.env.HQ_OP_LOCK_TIMEOUT = "9999"; // would be a near-infinite wait
-    writeLock(lockPathFor(rootA), { pid: 1, command: "sync" });
-    // The explicit 0 wins → refuse immediately rather than honoring the env.
-    expect(() =>
-      acquireOperationLock(rootA, "rescue", { timeoutSec: 0 }),
-    ).toThrowError(OperationLockedError);
-  });
-
-  it("onWaitStart fires exactly once, naming the holder, even across many polls", () => {
-    writeLock(lockPathFor(rootA), { pid: 1, command: "rescue" });
-    const calls: Array<{ cmd: string; attempted: string }> = [];
-    expect(() =>
-      acquireOperationLock(rootA, "sync", {
-        timeoutSec: 0.16,
-        pollIntervalMs: 30, // ~5 polls within the window
-        onWaitStart: (holder, attempted) => calls.push({ cmd: holder.command, attempted }),
-      }),
-    ).toThrowError(OperationLockedError);
-    expect(calls).toHaveLength(1);
-    expect(calls[0]).toEqual({ cmd: "rescue", attempted: "sync" });
-  });
-
-  it("a waiter acquires the lock the moment the holder releases (async poll path)", async () => {
-    const p = lockPathFor(rootA);
-    // A foreign LIVE holder (pid 1) initially owns the lock.
-    writeLock(p, { pid: 1, command: "rescue" });
-    // Simulate the holder finishing ~80ms in by removing its lock file.
-    const release = setTimeout(() => fs.rmSync(p, { force: true }), 80);
-    const start = Date.now();
-    const h = await acquireOperationLockAsync(rootA, "sync", {
-      pollIntervalMs: 20,
-      onWaitStart: () => {},
-    });
-    clearTimeout(release);
-    const elapsed = Date.now() - start;
-    // We waited for the release, then took it over.
-    expect(elapsed).toBeGreaterThanOrEqual(60);
-    const info = JSON.parse(fs.readFileSync(h.path, "utf8")) as LockInfo;
-    expect(info.pid).toBe(process.pid);
-    expect(info.command).toBe("sync");
-    h.release();
-  });
-
-  it("multiple foreign holders in a row: each release lets the next waiter in (no FIFO guarantee)", async () => {
-    // The mutex is CROSS-PROCESS: it keys liveness on the holder's PID. Two
-    // waiters in the SAME process share a pid, so the same-process reclaim path
-    // would let them stomp each other — that scenario is unsupported by design.
-    // Here we model the real case: a sequence of FOREIGN holders (pid 1) that
-    // each release, with a single waiter acquiring the instant the lock frees.
-    // Order among multiple distinct-process waiters is whoever wins the next
-    // O_EXCL race after a free — best-effort, NOT FIFO (documented).
-    const p = lockPathFor(rootA);
-    writeLock(p, { pid: 1, command: "sync" });
-    // Free it shortly; the waiter should grab it right after.
-    setTimeout(() => fs.rmSync(p, { force: true }), 50);
-    const h = await acquireOperationLockAsync(rootA, "reindex", {
-      pollIntervalMs: 15,
-      onWaitStart: () => {},
-    });
-    const info = JSON.parse(fs.readFileSync(h.path, "utf8")) as LockInfo;
-    expect(info.command).toBe("reindex");
-    expect(info.pid).toBe(process.pid);
-    h.release();
-  });
-
-  it("reclaims a stale lock whose holder PID is dead (takeover, never waits)", () => {
-    const stale = deadPid();
-    writeLock(lockPathFor(rootA), { pid: stale, command: "sync" });
-    const start = Date.now();
-    // The dead holder must not block us — even with an infinite default wait,
-    // takeover is immediate.
-    const h = acquireOperationLock(rootA, "rescue");
-    expect(Date.now() - start).toBeLessThan(DEFAULT_LOCK_POLL_MS);
-    const info = JSON.parse(fs.readFileSync(h.path, "utf8")) as LockInfo;
-    expect(info.pid).toBe(process.pid); // we took it over
-    expect(info.command).toBe("rescue");
-    h.release();
-  });
-
-  it("treats a torn/unreadable lock file as busy", () => {
-    const p = lockPathFor(rootA);
-    fs.mkdirSync(path.dirname(p), { recursive: true });
-    fs.writeFileSync(p, "{ this is not valid json");
-    expect(() => acquireOperationLock(rootA, "reindex", { wait: false })).toThrowError(
-      OperationLockedError,
-    );
-    expect(fs.readFileSync(p, "utf8")).toBe("{ this is not valid json");
-  });
-
-  it("F34: fresh empty lock files are not reclaimed as stale", () => {
-    const p = lockPathFor(rootA);
-    fs.mkdirSync(path.dirname(p), { recursive: true });
-    fs.closeSync(fs.openSync(p, "wx"));
-
-    let handle: ReturnType<typeof acquireOperationLock> | undefined;
-    try {
-      expect(() => {
-        handle = acquireOperationLock(rootA, "rescue", { wait: false });
-      }).toThrow();
-      expect(fs.existsSync(p)).toBe(true);
-      expect(fs.readFileSync(p, "utf8")).toBe("");
-    } finally {
-      handle?.release();
-      fs.rmSync(p, { force: true });
-    }
-  });
-
-  it("R-F34: initializing empty and torn locks are busy, not reclaimed", () => {
-    const p = lockPathFor(rootA);
-    fs.mkdirSync(path.dirname(p), { recursive: true });
-
-    fs.closeSync(fs.openSync(p, "wx"));
-    const old = new Date(Date.now() - 10_000);
-    fs.utimesSync(p, old, old);
-    expect(() => acquireOperationLock(rootA, "sync", { wait: false })).toThrowError(
-      OperationLockedError,
-    );
-    expect(fs.readFileSync(p, "utf8")).toBe("");
-
-    fs.writeFileSync(p, "{ torn");
-    expect(() => acquireOperationLock(rootA, "sync", { wait: false })).toThrowError(
-      OperationLockedError,
-    );
-    expect(fs.readFileSync(p, "utf8")).toBe("{ torn");
-  });
-
-  it("different HQ roots are independent — both may hold concurrently", () => {
-    const a = acquireOperationLock(rootA, "sync");
-    const b = acquireOperationLock(rootB, "rescue"); // must NOT block
-    expect(fs.existsSync(a.path)).toBe(true);
-    expect(fs.existsSync(b.path)).toBe(true);
-    expect(a.path).not.toBe(b.path);
-    a.release();
-    b.release();
-  });
-
-  it("the same root is mutually exclusive across different commands", () => {
-    // A live sync in ANOTHER process holds the root (pid 1 stands in for it).
-    const p = lockPathFor(rootA);
-    writeLock(p, { pid: 1, command: "sync" });
-    // Neither rescue nor reindex may acquire while that sync holds it
-    // (wait:false → assert the refusal without hanging on the new wait default).
-    expect(() => acquireOperationLock(rootA, "rescue", { wait: false })).toThrowError(
-      OperationLockedError,
-    );
-    expect(() => acquireOperationLock(rootA, "reindex", { wait: false })).toThrowError(
-      OperationLockedError,
-    );
-    // Once that sync finishes (its lock is gone), the next command acquires.
-    fs.unlinkSync(p);
-    const h2 = acquireOperationLock(rootA, "reindex");
-    expect(fs.existsSync(h2.path)).toBe(true);
-    h2.release();
-  });
-
-  it("withOperationLockSync releases even when the body throws", () => {
-    const p = lockPathFor(rootA);
-    expect(() =>
-      withOperationLockSync(rootA, "reindex", () => {
-        expect(fs.existsSync(p)).toBe(true); // held during the body
-        throw new Error("boom");
-      }),
-    ).toThrow("boom");
-    expect(fs.existsSync(p)).toBe(false); // released on the way out
-  });
-
-  it("withOperationLock (async) releases even when the body throws", async () => {
-    const p = lockPathFor(rootA);
-    await expect(
-      withOperationLock(rootA, "sync", async () => {
-        expect(fs.existsSync(p)).toBe(true);
-        throw new Error("boom");
-      }),
-    ).rejects.toThrow("boom");
-    expect(fs.existsSync(p)).toBe(false);
-  });
-
-  it("HQ_DISABLE_OP_LOCK=1 makes acquisition a no-op", () => {
-    process.env.HQ_DISABLE_OP_LOCK = "1";
-    // Even with a live holder on record, the escape hatch acquires without error.
-    writeLock(lockPathFor(rootA), { pid: process.pid, command: "sync" });
-    const h = acquireOperationLock(rootA, "rescue");
-    expect(h.path).toBe(""); // no real lock file written
-    h.release(); // no-op, no throw
-  });
-
-  it("OPERATION_LOCKED_EXIT is a stable non-zero code", () => {
-    expect(OPERATION_LOCKED_EXIT).toBe(17);
-  });
-});