@indigoai-us/hq-cloud 6.2.7 → 6.3.0

package/src/cli/reindex.ts

@@ -17,10 +17,24 @@
 import { spawnSync } from "child_process";
 import * as fs from "fs";
 import * as path from "path";
+import {
+  acquireOperationLock,
+  OperationLockedError,
+  OPERATION_LOCKED_EXIT,
+  type LockHandle,
+} from "../operation-lock.js";
 
 export interface ReindexOptions {
   /** HQ root to operate on. Defaults to process.cwd(). */
   repoRoot?: string;
+  /**
+   * Skip the per-root operation lock. Internal callers (`sync()` / `rescue()`)
+   * already hold the lock for this root and pass `true` so reindex doesn't try
+   * to re-acquire it and refuse against their own live PID. Standalone callers
+   * (`hq reindex`, the reindex hook) leave it falsy so reindex is mutually
+   * exclusive with a running sync/rescue.
+   */
+  skipLock?: boolean;
 }
 
 export interface ReindexResult {
@@ -129,6 +143,25 @@ export function reindex(opts: ReindexOptions = {}): ReindexResult {
   }
   const root = path.resolve(rawRoot);
 
+  // Acquire the per-root operation lock unless an internal caller (sync/rescue,
+  // which already hold it) opted out. A live holder → refuse fast with the
+  // holder's command + PID. The whole body runs inside the try so the lock is
+  // released on every exit path (the process-level signal/exit hooks are the
+  // crash backstop).
+  let opLock: LockHandle | null = null;
+  if (!opts.skipLock) {
+    try {
+      opLock = acquireOperationLock(root, "reindex");
+    } catch (err) {
+      if (err instanceof OperationLockedError) {
+        warn(err.message);
+        return { status: OPERATION_LOCKED_EXIT };
+      }
+      throw err;
+    }
+  }
+  try {
+
   fs.mkdirSync(path.join(root, ".claude", "skills"), { recursive: true });
 
   // --- Build (namespace, src_rel) pairs -------------------------------------
@@ -363,4 +396,7 @@ export function reindex(opts: ReindexOptions = {}): ReindexResult {
   }
 
   return { status: 0 };
+  } finally {
+    opLock?.release();
+  }
 }

package/src/cli/rescue.reindex.test.ts

@@ -7,7 +7,10 @@
  * runs, and ./reindex.js is mocked to a spy so we assert the call without
  * touching disk.
  */
-import { describe, it, expect, vi, beforeEach } from "vitest";
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import * as fs from "fs";
+import * as os from "os";
+import * as path from "path";
 
 vi.mock("./rescue-core.js", () => ({
   runRescue: vi.fn(() => ({ status: 0 })),
@@ -21,16 +24,28 @@ import { reindex } from "./reindex.js";
 import { rescue } from "./rescue.js";
 
 describe("rescue → reindex", () => {
+  let stateDir: string;
+
   beforeEach(() => {
     vi.clearAllMocks();
     (runRescue as unknown as ReturnType<typeof vi.fn>).mockReturnValue({ status: 0 });
+    // rescue() now takes the per-root operation lock; redirect it to a tmp dir.
+    stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "rescue-reindex-state-"));
+    process.env.HQ_STATE_DIR = stateDir;
+  });
+
+  afterEach(() => {
+    fs.rmSync(stateDir, { recursive: true, force: true });
+    delete process.env.HQ_STATE_DIR;
   });
 
   it("refreshes via reindex after a successful rescue", () => {
     const r = rescue({ hqRoot: "/tmp/hq", assumeYes: true });
     expect(r.status).toBe(0);
     expect(reindex).toHaveBeenCalledTimes(1);
-    expect(reindex).toHaveBeenCalledWith({ repoRoot: "/tmp/hq" });
+    // skipLock: rescue already holds the per-root operation lock, so the
+    // internal reindex must not try to re-acquire it.
+    expect(reindex).toHaveBeenCalledWith({ repoRoot: "/tmp/hq", skipLock: true });
   });
 
   it("does NOT run reindex on a dry-run", () => {

package/src/cli/rescue.ts

@@ -14,6 +14,11 @@
  */
 import { reindex } from "./reindex.js";
 import { runRescue } from "./rescue-core.js";
+import {
+  acquireOperationLock,
+  OperationLockedError,
+  OPERATION_LOCKED_EXIT,
+} from "../operation-lock.js";
 
 export interface RescueOptions {
   /** HQ root to operate on. Passed as `--hq-root`. Defaults to the script's
@@ -91,21 +96,41 @@ export function buildRescueArgs(opts: RescueOptions = {}): string[] {
  * confirmation prompt (unless `assumeYes` is set).
  */
 export function rescue(opts: RescueOptions = {}): RescueResult {
-  const args = buildRescueArgs(opts);
-  const env: NodeJS.ProcessEnv = { ...process.env };
-  if (opts.ghToken) env.GH_TOKEN = opts.ghToken;
-  const { status } = runRescue(args, { env });
-  // A successful, non-dry-run rescue re-lays-down core/, so refresh the
-  // generated skill wrappers, personal-overlay mirrors, and workers registry.
-  // Best-effort + idempotent — never overrides the rescue's own exit status.
-  // repoRoot falls back to process.cwd() (reindex's default) when hqRoot is
-  // omitted, matching the rescue script's own cwd-based default.
-  if (status === 0 && !opts.dryRun) {
-    try {
-      reindex({ repoRoot: opts.hqRoot });
-    } catch {
-      // best-effort
+  // Rescue is mutually exclusive with sync/reindex on this HQ root. Key the
+  // lock on the same root the rescue script resolves (cwd when --hq-root is
+  // omitted). Rescue is never the exempt push watcher, so it always locks.
+  const lockRoot = opts.hqRoot ?? process.cwd();
+  let handle;
+  try {
+    handle = acquireOperationLock(lockRoot, "rescue");
+  } catch (err) {
+    if (err instanceof OperationLockedError) {
+      process.stderr.write(err.message + "\n");
+      return { status: OPERATION_LOCKED_EXIT };
     }
+    throw err;
+  }
+  try {
+    const args = buildRescueArgs(opts);
+    const env: NodeJS.ProcessEnv = { ...process.env };
+    if (opts.ghToken) env.GH_TOKEN = opts.ghToken;
+    const { status } = runRescue(args, { env });
+    // A successful, non-dry-run rescue re-lays-down core/, so refresh the
+    // generated skill wrappers, personal-overlay mirrors, and workers registry.
+    // Best-effort + idempotent — never overrides the rescue's own exit status.
+    // repoRoot falls back to process.cwd() (reindex's default) when hqRoot is
+    // omitted, matching the rescue script's own cwd-based default. `skipLock`
+    // because we already hold the per-root lock — reindex's own acquire would
+    // otherwise see our live PID and refuse.
+    if (status === 0 && !opts.dryRun) {
+      try {
+        reindex({ repoRoot: opts.hqRoot, skipLock: true });
+      } catch {
+        // best-effort
+      }
+    }
+    return { status };
+  } finally {
+    handle.release();
   }
-  return { status };
 }

package/src/cli/sync.test.ts

@@ -123,7 +123,8 @@ describe("sync", () => {
 
   it("runs reindex against hqRoot after a sync that downloaded files", async () => {
     await sync({ company: "acme", vaultConfig: mockConfig, hqRoot: tmpDir });
-    expect(reindex).toHaveBeenCalledWith({ repoRoot: tmpDir });
+    // skipLock: the surrounding sync run already holds the per-root lock.
+    expect(reindex).toHaveBeenCalledWith({ repoRoot: tmpDir, skipLock: true });
   });
 
   it("skips reindex when skipReindex is set", async () => {

package/src/cli/sync.ts

@@ -1338,7 +1338,9 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
     shrinkResult.cleanRemoved > 0;
   if (!options.skipReindex && changedOnDisk) {
     try {
-      reindex({ repoRoot: hqRoot });
+      // skipLock: the surrounding sync run already holds this root's operation
+      // lock; reindex re-acquiring would refuse against our own live PID.
+      reindex({ repoRoot: hqRoot, skipLock: true });
     } catch {
       // best-effort: a post-sync refresh failure never fails the sync
     }

package/src/operation-lock.test.ts

@@ -0,0 +1,162 @@
+/**
+ * Unit tests for the per-HQ-root operation mutex.
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import { spawnSync } from "child_process";
+import * as fs from "fs";
+import * as os from "os";
+import * as path from "path";
+import {
+  acquireOperationLock,
+  withOperationLockSync,
+  lockPathFor,
+  OperationLockedError,
+  OPERATION_LOCKED_EXIT,
+  type LockInfo,
+} from "./operation-lock.js";
+
+/** A PID that is guaranteed dead: spawn a node that exits immediately, reuse its pid. */
+function deadPid(): number {
+  const r = spawnSync(process.execPath, ["-e", ""], { stdio: "ignore" });
+  if (!r.pid) throw new Error("could not spawn to obtain a dead pid");
+  return r.pid;
+}
+
+function writeLock(p: string, info: Partial<LockInfo>): void {
+  fs.mkdirSync(path.dirname(p), { recursive: true });
+  const full: LockInfo = {
+    pid: 1,
+    command: "sync",
+    startedAt: new Date(0).toISOString(),
+    hqRoot: "/x",
+    ...info,
+  };
+  fs.writeFileSync(p, JSON.stringify(full));
+}
+
+describe("operation-lock", () => {
+  let stateDir: string;
+  let rootA: string;
+  let rootB: string;
+
+  beforeEach(() => {
+    stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "hq-oplock-state-"));
+    process.env.HQ_STATE_DIR = stateDir;
+    delete process.env.HQ_DISABLE_OP_LOCK;
+    rootA = fs.mkdtempSync(path.join(os.tmpdir(), "hq-rootA-"));
+    rootB = fs.mkdtempSync(path.join(os.tmpdir(), "hq-rootB-"));
+  });
+
+  afterEach(() => {
+    fs.rmSync(stateDir, { recursive: true, force: true });
+    fs.rmSync(rootA, { recursive: true, force: true });
+    fs.rmSync(rootB, { recursive: true, force: true });
+    delete process.env.HQ_STATE_DIR;
+    delete process.env.HQ_DISABLE_OP_LOCK;
+  });
+
+  it("the lock path is under the state dir, keyed per canonical root", () => {
+    const a = lockPathFor(rootA);
+    const b = lockPathFor(rootB);
+    expect(a.startsWith(path.join(stateDir, "locks"))).toBe(true);
+    expect(a).not.toBe(b); // different roots → different lock files
+    // canonical: a trailing-slash variant maps to the same lock
+    expect(lockPathFor(rootA + path.sep)).toBe(a);
+  });
+
+  it("acquires, writes holder info, and releases (file gone after release)", () => {
+    const h = acquireOperationLock(rootA, "sync");
+    expect(fs.existsSync(h.path)).toBe(true);
+    const info = JSON.parse(fs.readFileSync(h.path, "utf8")) as LockInfo;
+    expect(info.pid).toBe(process.pid);
+    expect(info.command).toBe("sync");
+    h.release();
+    expect(fs.existsSync(h.path)).toBe(false);
+  });
+
+  it("refuses fast with the holder's command + pid when a LIVE process holds it", () => {
+    // Simulate a DIFFERENT live process holding the lock. PID 1 (init/systemd)
+    // is always alive and is never our own pid, so kill(1,0) reports alive and
+    // the same-process reclaim path does not apply.
+    writeLock(lockPathFor(rootA), { pid: 1, command: "rescue" });
+    expect(() => acquireOperationLock(rootA, "sync")).toThrowError(OperationLockedError);
+    try {
+      acquireOperationLock(rootA, "sync");
+    } catch (e) {
+      const err = e as OperationLockedError;
+      expect(err.holder.command).toBe("rescue");
+      expect(err.holder.pid).toBe(1);
+      expect(err.message).toContain("rescue");
+      expect(err.message).toContain("pid 1");
+    }
+  });
+
+  it("reclaims a stale lock whose holder PID is dead (takeover)", () => {
+    const stale = deadPid();
+    writeLock(lockPathFor(rootA), { pid: stale, command: "sync" });
+    // The dead holder must not block us.
+    const h = acquireOperationLock(rootA, "rescue");
+    const info = JSON.parse(fs.readFileSync(h.path, "utf8")) as LockInfo;
+    expect(info.pid).toBe(process.pid); // we took it over
+    expect(info.command).toBe("rescue");
+    h.release();
+  });
+
+  it("reclaims a torn/unreadable lock file", () => {
+    const p = lockPathFor(rootA);
+    fs.mkdirSync(path.dirname(p), { recursive: true });
+    fs.writeFileSync(p, "{ this is not valid json");
+    const h = acquireOperationLock(rootA, "reindex");
+    expect(fs.existsSync(h.path)).toBe(true);
+    h.release();
+  });
+
+  it("different HQ roots are independent — both may hold concurrently", () => {
+    const a = acquireOperationLock(rootA, "sync");
+    const b = acquireOperationLock(rootB, "rescue"); // must NOT refuse
+    expect(fs.existsSync(a.path)).toBe(true);
+    expect(fs.existsSync(b.path)).toBe(true);
+    expect(a.path).not.toBe(b.path);
+    a.release();
+    b.release();
+  });
+
+  it("the same root is mutually exclusive across different commands", () => {
+    // A live sync in ANOTHER process holds the root (pid 1 stands in for it).
+    const p = lockPathFor(rootA);
+    writeLock(p, { pid: 1, command: "sync" });
+    // Neither rescue nor reindex may acquire while that sync holds it.
+    expect(() => acquireOperationLock(rootA, "rescue")).toThrowError(OperationLockedError);
+    expect(() => acquireOperationLock(rootA, "reindex")).toThrowError(OperationLockedError);
+    // Once that sync finishes (its lock is gone), the next command acquires.
+    fs.unlinkSync(p);
+    const h2 = acquireOperationLock(rootA, "reindex");
+    expect(fs.existsSync(h2.path)).toBe(true);
+    h2.release();
+  });
+
+  it("withOperationLockSync releases even when the body throws", () => {
+    const p = lockPathFor(rootA);
+    expect(() =>
+      withOperationLockSync(rootA, "reindex", () => {
+        expect(fs.existsSync(p)).toBe(true); // held during the body
+        throw new Error("boom");
+      }),
+    ).toThrow("boom");
+    expect(fs.existsSync(p)).toBe(false); // released on the way out
+  });
+
+  it("HQ_DISABLE_OP_LOCK=1 makes acquisition a no-op", () => {
+    process.env.HQ_DISABLE_OP_LOCK = "1";
+    // Even with a live holder on record, the escape hatch acquires without error.
+    writeLock(lockPathFor(rootA), { pid: process.pid, command: "sync" });
+    const h = acquireOperationLock(rootA, "rescue");
+    expect(h.path).toBe(""); // no real lock file written
+    h.release(); // no-op, no throw
+  });
+
+  it("OPERATION_LOCKED_EXIT is a stable non-zero code", () => {
+    expect(OPERATION_LOCKED_EXIT).toBe(17);
+  });
+});

package/src/operation-lock.ts

@@ -0,0 +1,293 @@
+/**
+ * Per-HQ-root mutual exclusion for the long-running operations
+ * (`sync`, `rescue`, `reindex`).
+ *
+ * Contract:
+ *   - At most ONE of sync / rescue / reindex runs at a time **per HQ root**.
+ *     The lock is shared across all three (keyed only by the root, not the
+ *     command), so e.g. a rescue refuses while a sync holds it. Different HQ
+ *     roots are fully independent — they hash to different lock files.
+ *   - The push watcher / watch+event-push runner is EXEMPT: it never calls in
+ *     here, so it neither takes the lock nor is blocked by it (its targeted
+ *     in-process push passes are likewise lock-free).
+ *
+ * ## Where the lock lives — and why
+ *
+ * `<stateDir>/locks/operation-<hash(canonicalRoot)>.lock`, where
+ * `stateDir = $HQ_STATE_DIR || ~/.hq`. This is deliberately NOT inside the HQ
+ * root:
+ *   - It must never round-trip to the cloud. A lock is machine-local, per-run
+ *     state; syncing it to S3 (and thence to other machines/roots) would be a
+ *     correctness bug. `~/.hq` is the established machine-local state dir
+ *     (journals already live there) and is never synced.
+ *   - `rescue` repairs a possibly-broken HQ root; a lock that depends on the
+ *     root being healthy is exactly backwards. `~/.hq` is independent of the
+ *     root's health.
+ *   - Keying the filename by a hash of the *canonical* root path makes the
+ *     lock per-root and prevents leakage across roots, while keeping the path
+ *     short and filesystem-safe.
+ *
+ * ## Atomicity, liveness, takeover
+ *
+ *   - Acquisition uses `open(…, "wx")` (O_CREAT | O_EXCL) — an atomic
+ *     create-if-absent. Exactly one racer can create the file; the loser sees
+ *     EEXIST and re-evaluates.
+ *   - The lock records the holder's `{ pid, command, startedAt, hqRoot }`. On
+ *     EEXIST we test the recorded PID with `process.kill(pid, 0)`:
+ *       * ESRCH  → the holder is gone (crashed / killed -9 / stale file) →
+ *                  reclaim the lock.
+ *       * EPERM  → the PID exists but is owned by another user → treat as ALIVE
+ *                  (conservative: refuse rather than risk two concurrent ops).
+ *       * success → alive → refuse fast with {@link OperationLockedError}
+ *                  naming the holding command + PID.
+ *   - PID reuse is an inherent, un-eliminable race for any PID-based scheme: if
+ *     the original holder crashed and the OS later handed its PID to an
+ *     unrelated process, we conservatively read that as "still held" and
+ *     refuse. We accept that false-busy over the far worse false-free, and
+ *     record `startedAt`/`command` so an operator can diagnose a wedged lock.
+ *
+ * ## Release
+ *
+ *   - Normal exit: the `with*` wrappers release in a `finally`.
+ *   - Signals (SIGINT/SIGTERM): a one-time handler releases every held lock,
+ *     then re-raises the default disposition so exit status is unchanged.
+ *   - Hard crash (SIGKILL / power loss): nothing runs, but the stale-PID
+ *     takeover above reclaims the lock on the next attempt.
+ *   - `process.on("exit")`: a final best-effort synchronous unlink.
+ *
+ * ## Escape hatch
+ *
+ *   `HQ_DISABLE_OP_LOCK=1` makes acquisition a no-op (returns a handle whose
+ *   release does nothing). For emergencies and for callers that manage
+ *   exclusion themselves; documented, off by default.
+ */
+
+import * as crypto from "crypto";
+import * as fs from "fs";
+import * as os from "os";
+import * as path from "path";
+
+/** Process exit code used when an operation is refused because the lock is held. */
+export const OPERATION_LOCKED_EXIT = 17;
+
+export interface LockInfo {
+  pid: number;
+  command: string;
+  /** ISO-8601 acquisition time. */
+  startedAt: string;
+  /** Canonical HQ root the lock guards (diagnostic only). */
+  hqRoot: string;
+}
+
+/** Thrown by `acquireOperationLock` when a LIVE holder owns the lock. */
+export class OperationLockedError extends Error {
+  constructor(
+    public readonly holder: LockInfo,
+    public readonly attempted: string,
+  ) {
+    super(
+      `Refusing to start "${attempted}": another HQ operation is already ` +
+        `running for this HQ root — "${holder.command}" (pid ${holder.pid}, ` +
+        `started ${holder.startedAt}). Wait for it to finish, or stop that ` +
+        `process, then retry.`,
+    );
+    this.name = "OperationLockedError";
+  }
+}
+
+export interface LockHandle {
+  /** Absolute path of the lock file. */
+  readonly path: string;
+  /** The info written for this holder. */
+  readonly info: LockInfo;
+  /** Idempotently release the lock iff this process still owns it. */
+  release(): void;
+}
+
+function stateDir(): string {
+  return process.env.HQ_STATE_DIR || path.join(os.homedir(), ".hq");
+}
+
+/** Absolute lock path for a given HQ root. Exported for tests. */
+export function lockPathFor(hqRoot: string): string {
+  const canon = path.resolve(hqRoot);
+  const key = crypto.createHash("sha1").update(canon).digest("hex").slice(0, 16);
+  return path.join(stateDir(), "locks", `operation-${key}.lock`);
+}
+
+/**
+ * Is `pid` a live process? `kill(pid, 0)` sends no signal; it only probes.
+ * ESRCH → no such process (dead/stale). EPERM → exists but not ours → ALIVE
+ * (conservative). Anything else → assume alive rather than risk a double-run.
+ */
+function pidAlive(pid: number): boolean {
+  if (!Number.isInteger(pid) || pid <= 0) return false;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (err) {
+    const code = (err as NodeJS.ErrnoException)?.code;
+    if (code === "ESRCH") return false;
+    return true; // EPERM (exists) or unknown → treat as alive
+  }
+}
+
+function readLockInfo(p: string): LockInfo | null {
+  try {
+    const parsed = JSON.parse(fs.readFileSync(p, "utf8")) as LockInfo;
+    if (parsed && typeof parsed.pid === "number" && typeof parsed.command === "string") {
+      return parsed;
+    }
+    return null;
+  } catch {
+    return null;
+  }
+}
+
+// ── Process-wide release plumbing ──────────────────────────────────────────
+// Track every lock this process currently holds so the signal/exit hooks can
+// release all of them. The hooks are installed exactly once.
+
+const heldLocks = new Set<LockHandle>();
+let hooksInstalled = false;
+
+function unlinkIfOwned(p: string): void {
+  // Only remove a lock whose recorded pid is THIS process — never clobber a
+  // lock another process took over after a (hypothetical) reclaim race.
+  const info = readLockInfo(p);
+  if (info && info.pid === process.pid) {
+    try {
+      fs.unlinkSync(p);
+    } catch {
+      /* already gone — fine */
+    }
+  }
+}
+
+function installHooksOnce(): void {
+  if (hooksInstalled) return;
+  hooksInstalled = true;
+
+  process.on("exit", () => {
+    for (const h of heldLocks) unlinkIfOwned(h.path);
+  });
+
+  for (const sig of ["SIGINT", "SIGTERM", "SIGHUP"] as const) {
+    process.on(sig, () => {
+      for (const h of heldLocks) unlinkIfOwned(h.path);
+      // Re-raise with the default disposition so the exit status is the normal
+      // signal status (and a second Ctrl-C still works). Removing our listener
+      // first avoids recursing back into this handler.
+      process.removeAllListeners(sig);
+      process.kill(process.pid, sig);
+    });
+  }
+}
+
+function makeHandle(p: string, info: LockInfo): LockHandle {
+  const handle: LockHandle = {
+    path: p,
+    info,
+    release() {
+      heldLocks.delete(handle);
+      unlinkIfOwned(p);
+    },
+  };
+  heldLocks.add(handle);
+  installHooksOnce();
+  return handle;
+}
+
+const NOOP_HANDLE_BASE = { release() {} };
+
+/**
+ * Acquire the per-root operation lock for `command`. Returns a {@link LockHandle}
+ * on success; throws {@link OperationLockedError} when a live holder owns it.
+ * Reclaims a stale lock (dead holder) transparently.
+ */
+export function acquireOperationLock(hqRoot: string, command: string): LockHandle {
+  if (process.env.HQ_DISABLE_OP_LOCK === "1") {
+    const info: LockInfo = {
+      pid: process.pid,
+      command,
+      startedAt: new Date().toISOString(),
+      hqRoot: path.resolve(hqRoot),
+    };
+    return { ...NOOP_HANDLE_BASE, path: "", info };
+  }
+
+  const p = lockPathFor(hqRoot);
+  fs.mkdirSync(path.dirname(p), { recursive: true });
+
+  const info: LockInfo = {
+    pid: process.pid,
+    command,
+    startedAt: new Date().toISOString(),
+    hqRoot: path.resolve(hqRoot),
+  };
+  const payload = JSON.stringify(info, null, 2);
+
+  // Bounded retry: each iteration is one atomic create attempt. EEXIST against
+  // a stale holder reclaims and retries; EEXIST against a live holder refuses.
+  const MAX_ATTEMPTS = 5;
+  for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
+    let fd: number;
+    try {
+      fd = fs.openSync(p, "wx"); // O_CREAT | O_EXCL — atomic
+    } catch (err) {
+      if ((err as NodeJS.ErrnoException)?.code !== "EEXIST") throw err;
+
+      const holder = readLockInfo(p);
+      if (holder && holder.pid !== process.pid && pidAlive(holder.pid)) {
+        throw new OperationLockedError(holder, command);
+      }
+      // Stale (dead holder), unreadable/torn, or our own leftover → reclaim.
+      try {
+        fs.unlinkSync(p);
+      } catch {
+        /* someone else reclaimed it first; the next openSync re-evaluates */
+      }
+      continue;
+    }
+    try {
+      fs.writeSync(fd, payload);
+    } finally {
+      fs.closeSync(fd);
+    }
+    return makeHandle(p, info);
+  }
+
+  // Pathological churn (another process reclaiming in lockstep). Surface it
+  // rather than spin forever.
+  throw new Error(
+    `Could not acquire HQ operation lock at ${p} after ${MAX_ATTEMPTS} attempts`,
+  );
+}
+
+/** Run `fn` while holding the per-root lock for `command` (async). */
+export async function withOperationLock<T>(
+  hqRoot: string,
+  command: string,
+  fn: () => Promise<T>,
+): Promise<T> {
+  const handle = acquireOperationLock(hqRoot, command);
+  try {
+    return await fn();
+  } finally {
+    handle.release();
+  }
+}
+
+/** Run `fn` while holding the per-root lock for `command` (synchronous). */
+export function withOperationLockSync<T>(
+  hqRoot: string,
+  command: string,
+  fn: () => T,
+): T {
+  const handle = acquireOperationLock(hqRoot, command);
+  try {
+    return fn();
+  } finally {
+    handle.release();
+  }
+}