@indigoai-us/hq-cloud 6.2.6 → 6.3.0

package/src/cli/reindex.test.ts

@@ -13,16 +13,24 @@ import * as fs from "fs";
 import * as path from "path";
 import * as os from "os";
 import { reindex } from "./reindex.js";
+import { lockPathFor, OPERATION_LOCKED_EXIT } from "../operation-lock.js";
 
 describe("reindex", () => {
   let root: string;
+  let stateDir: string;
 
   beforeEach(() => {
     root = fs.mkdtempSync(path.join(os.tmpdir(), "ms-test-"));
+    // Redirect the operation lock into a throwaway dir so reindex's default
+    // locking never touches the real ~/.hq during tests.
+    stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "ms-state-"));
+    process.env.HQ_STATE_DIR = stateDir;
   });
 
   afterEach(() => {
     fs.rmSync(root, { recursive: true, force: true });
+    fs.rmSync(stateDir, { recursive: true, force: true });
+    delete process.env.HQ_STATE_DIR;
   });
 
   function writeSkill(rel: string): void {
@@ -124,4 +132,41 @@ describe("reindex", () => {
     expect(fs.existsSync(wrapper)).toBe(true);
     expect(fs.lstatSync(path.join(wrapper, "SKILL.md")).isSymbolicLink()).toBe(true);
   });
+
+  // ── operation lock (mutual exclusion with sync/rescue) ──────────────────
+
+  it("refuses (OPERATION_LOCKED_EXIT) when another op holds this root's lock", () => {
+    // A live holder in another process (pid 1) → reindex must refuse fast and
+    // do no work.
+    const lp = lockPathFor(root);
+    fs.mkdirSync(path.dirname(lp), { recursive: true });
+    fs.writeFileSync(
+      lp,
+      JSON.stringify({ pid: 1, command: "sync", startedAt: new Date(0).toISOString(), hqRoot: root }),
+    );
+    const before = fs.existsSync(path.join(root, ".claude/skills"));
+    const { status } = reindex({ repoRoot: root });
+    expect(status).toBe(OPERATION_LOCKED_EXIT);
+    // It refused before doing any work (didn't create .claude/skills).
+    expect(fs.existsSync(path.join(root, ".claude/skills"))).toBe(before);
+  });
+
+  it("skipLock bypasses the lock (internal sync/rescue caller path)", () => {
+    const lp = lockPathFor(root);
+    fs.mkdirSync(path.dirname(lp), { recursive: true });
+    fs.writeFileSync(
+      lp,
+      JSON.stringify({ pid: 1, command: "sync", startedAt: new Date(0).toISOString(), hqRoot: root }),
+    );
+    // Even with a live holder on record, the internal caller (which already
+    // holds the lock) runs to completion.
+    const { status } = reindex({ repoRoot: root, skipLock: true });
+    expect(status).toBe(0);
+    expect(fs.existsSync(path.join(root, ".claude/skills"))).toBe(true);
+  });
+
+  it("releases the lock after a normal run (no leftover lock file)", () => {
+    reindex({ repoRoot: root });
+    expect(fs.existsSync(lockPathFor(root))).toBe(false);
+  });
 });

package/src/cli/reindex.ts

@@ -17,10 +17,24 @@
 import { spawnSync } from "child_process";
 import * as fs from "fs";
 import * as path from "path";
+import {
+  acquireOperationLock,
+  OperationLockedError,
+  OPERATION_LOCKED_EXIT,
+  type LockHandle,
+} from "../operation-lock.js";
 
 export interface ReindexOptions {
   /** HQ root to operate on. Defaults to process.cwd(). */
   repoRoot?: string;
+  /**
+   * Skip the per-root operation lock. Internal callers (`sync()` / `rescue()`)
+   * already hold the lock for this root and pass `true` so reindex doesn't try
+   * to re-acquire it and refuse against their own live PID. Standalone callers
+   * (`hq reindex`, the reindex hook) leave it falsy so reindex is mutually
+   * exclusive with a running sync/rescue.
+   */
+  skipLock?: boolean;
 }
 
 export interface ReindexResult {
@@ -129,6 +143,25 @@ export function reindex(opts: ReindexOptions = {}): ReindexResult {
   }
   const root = path.resolve(rawRoot);
 
+  // Acquire the per-root operation lock unless an internal caller (sync/rescue,
+  // which already hold it) opted out. A live holder → refuse fast with the
+  // holder's command + PID. The whole body runs inside the try so the lock is
+  // released on every exit path (the process-level signal/exit hooks are the
+  // crash backstop).
+  let opLock: LockHandle | null = null;
+  if (!opts.skipLock) {
+    try {
+      opLock = acquireOperationLock(root, "reindex");
+    } catch (err) {
+      if (err instanceof OperationLockedError) {
+        warn(err.message);
+        return { status: OPERATION_LOCKED_EXIT };
+      }
+      throw err;
+    }
+  }
+  try {
+
   fs.mkdirSync(path.join(root, ".claude", "skills"), { recursive: true });
 
   // --- Build (namespace, src_rel) pairs -------------------------------------
@@ -363,4 +396,7 @@ export function reindex(opts: ReindexOptions = {}): ReindexResult {
   }
 
   return { status: 0 };
+  } finally {
+    opLock?.release();
+  }
 }

package/src/cli/rescue.reindex.test.ts

@@ -7,7 +7,10 @@
  * runs, and ./reindex.js is mocked to a spy so we assert the call without
  * touching disk.
  */
-import { describe, it, expect, vi, beforeEach } from "vitest";
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import * as fs from "fs";
+import * as os from "os";
+import * as path from "path";
 
 vi.mock("./rescue-core.js", () => ({
   runRescue: vi.fn(() => ({ status: 0 })),
@@ -21,16 +24,28 @@ import { reindex } from "./reindex.js";
 import { rescue } from "./rescue.js";
 
 describe("rescue → reindex", () => {
+  let stateDir: string;
+
   beforeEach(() => {
     vi.clearAllMocks();
     (runRescue as unknown as ReturnType<typeof vi.fn>).mockReturnValue({ status: 0 });
+    // rescue() now takes the per-root operation lock; redirect it to a tmp dir.
+    stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "rescue-reindex-state-"));
+    process.env.HQ_STATE_DIR = stateDir;
+  });
+
+  afterEach(() => {
+    fs.rmSync(stateDir, { recursive: true, force: true });
+    delete process.env.HQ_STATE_DIR;
   });
 
   it("refreshes via reindex after a successful rescue", () => {
     const r = rescue({ hqRoot: "/tmp/hq", assumeYes: true });
     expect(r.status).toBe(0);
     expect(reindex).toHaveBeenCalledTimes(1);
-    expect(reindex).toHaveBeenCalledWith({ repoRoot: "/tmp/hq" });
+    // skipLock: rescue already holds the per-root operation lock, so the
+    // internal reindex must not try to re-acquire it.
+    expect(reindex).toHaveBeenCalledWith({ repoRoot: "/tmp/hq", skipLock: true });
   });
 
   it("does NOT run reindex on a dry-run", () => {

package/src/cli/rescue.ts

@@ -14,6 +14,11 @@
  */
 import { reindex } from "./reindex.js";
 import { runRescue } from "./rescue-core.js";
+import {
+  acquireOperationLock,
+  OperationLockedError,
+  OPERATION_LOCKED_EXIT,
+} from "../operation-lock.js";
 
 export interface RescueOptions {
   /** HQ root to operate on. Passed as `--hq-root`. Defaults to the script's
@@ -91,21 +96,41 @@ export function buildRescueArgs(opts: RescueOptions = {}): string[] {
  * confirmation prompt (unless `assumeYes` is set).
  */
 export function rescue(opts: RescueOptions = {}): RescueResult {
-  const args = buildRescueArgs(opts);
-  const env: NodeJS.ProcessEnv = { ...process.env };
-  if (opts.ghToken) env.GH_TOKEN = opts.ghToken;
-  const { status } = runRescue(args, { env });
-  // A successful, non-dry-run rescue re-lays-down core/, so refresh the
-  // generated skill wrappers, personal-overlay mirrors, and workers registry.
-  // Best-effort + idempotent — never overrides the rescue's own exit status.
-  // repoRoot falls back to process.cwd() (reindex's default) when hqRoot is
-  // omitted, matching the rescue script's own cwd-based default.
-  if (status === 0 && !opts.dryRun) {
-    try {
-      reindex({ repoRoot: opts.hqRoot });
-    } catch {
-      // best-effort
+  // Rescue is mutually exclusive with sync/reindex on this HQ root. Key the
+  // lock on the same root the rescue script resolves (cwd when --hq-root is
+  // omitted). Rescue is never the exempt push watcher, so it always locks.
+  const lockRoot = opts.hqRoot ?? process.cwd();
+  let handle;
+  try {
+    handle = acquireOperationLock(lockRoot, "rescue");
+  } catch (err) {
+    if (err instanceof OperationLockedError) {
+      process.stderr.write(err.message + "\n");
+      return { status: OPERATION_LOCKED_EXIT };
     }
+    throw err;
+  }
+  try {
+    const args = buildRescueArgs(opts);
+    const env: NodeJS.ProcessEnv = { ...process.env };
+    if (opts.ghToken) env.GH_TOKEN = opts.ghToken;
+    const { status } = runRescue(args, { env });
+    // A successful, non-dry-run rescue re-lays-down core/, so refresh the
+    // generated skill wrappers, personal-overlay mirrors, and workers registry.
+    // Best-effort + idempotent — never overrides the rescue's own exit status.
+    // repoRoot falls back to process.cwd() (reindex's default) when hqRoot is
+    // omitted, matching the rescue script's own cwd-based default. `skipLock`
+    // because we already hold the per-root lock — reindex's own acquire would
+    // otherwise see our live PID and refuse.
+    if (status === 0 && !opts.dryRun) {
+      try {
+        reindex({ repoRoot: opts.hqRoot, skipLock: true });
+      } catch {
+        // best-effort
+      }
+    }
+    return { status };
+  } finally {
+    handle.release();
   }
-  return { status };
 }

package/src/cli/sync.test.ts

@@ -123,7 +123,8 @@ describe("sync", () => {
 
   it("runs reindex against hqRoot after a sync that downloaded files", async () => {
     await sync({ company: "acme", vaultConfig: mockConfig, hqRoot: tmpDir });
-    expect(reindex).toHaveBeenCalledWith({ repoRoot: tmpDir });
+    // skipLock: the surrounding sync run already holds the per-root lock.
+    expect(reindex).toHaveBeenCalledWith({ repoRoot: tmpDir, skipLock: true });
   });
 
   it("skips reindex when skipReindex is set", async () => {

package/src/cli/sync.ts

@@ -1338,7 +1338,9 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
     shrinkResult.cleanRemoved > 0;
   if (!options.skipReindex && changedOnDisk) {
     try {
-      reindex({ repoRoot: hqRoot });
+      // skipLock: the surrounding sync run already holds this root's operation
+      // lock; reindex re-acquiring would refuse against our own live PID.
+      reindex({ repoRoot: hqRoot, skipLock: true });
     } catch {
       // best-effort: a post-sync refresh failure never fails the sync
     }

package/src/operation-lock.test.ts

@@ -0,0 +1,162 @@
+/**
+ * Unit tests for the per-HQ-root operation mutex.
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import { spawnSync } from "child_process";
+import * as fs from "fs";
+import * as os from "os";
+import * as path from "path";
+import {
+  acquireOperationLock,
+  withOperationLockSync,
+  lockPathFor,
+  OperationLockedError,
+  OPERATION_LOCKED_EXIT,
+  type LockInfo,
+} from "./operation-lock.js";
+
+/** A PID that is guaranteed dead: spawn a node that exits immediately, reuse its pid. */
+function deadPid(): number {
+  const r = spawnSync(process.execPath, ["-e", ""], { stdio: "ignore" });
+  if (!r.pid) throw new Error("could not spawn to obtain a dead pid");
+  return r.pid;
+}
+
+function writeLock(p: string, info: Partial<LockInfo>): void {
+  fs.mkdirSync(path.dirname(p), { recursive: true });
+  const full: LockInfo = {
+    pid: 1,
+    command: "sync",
+    startedAt: new Date(0).toISOString(),
+    hqRoot: "/x",
+    ...info,
+  };
+  fs.writeFileSync(p, JSON.stringify(full));
+}
+
+describe("operation-lock", () => {
+  let stateDir: string;
+  let rootA: string;
+  let rootB: string;
+
+  beforeEach(() => {
+    stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "hq-oplock-state-"));
+    process.env.HQ_STATE_DIR = stateDir;
+    delete process.env.HQ_DISABLE_OP_LOCK;
+    rootA = fs.mkdtempSync(path.join(os.tmpdir(), "hq-rootA-"));
+    rootB = fs.mkdtempSync(path.join(os.tmpdir(), "hq-rootB-"));
+  });
+
+  afterEach(() => {
+    fs.rmSync(stateDir, { recursive: true, force: true });
+    fs.rmSync(rootA, { recursive: true, force: true });
+    fs.rmSync(rootB, { recursive: true, force: true });
+    delete process.env.HQ_STATE_DIR;
+    delete process.env.HQ_DISABLE_OP_LOCK;
+  });
+
+  it("the lock path is under the state dir, keyed per canonical root", () => {
+    const a = lockPathFor(rootA);
+    const b = lockPathFor(rootB);
+    expect(a.startsWith(path.join(stateDir, "locks"))).toBe(true);
+    expect(a).not.toBe(b); // different roots → different lock files
+    // canonical: a trailing-slash variant maps to the same lock
+    expect(lockPathFor(rootA + path.sep)).toBe(a);
+  });
+
+  it("acquires, writes holder info, and releases (file gone after release)", () => {
+    const h = acquireOperationLock(rootA, "sync");
+    expect(fs.existsSync(h.path)).toBe(true);
+    const info = JSON.parse(fs.readFileSync(h.path, "utf8")) as LockInfo;
+    expect(info.pid).toBe(process.pid);
+    expect(info.command).toBe("sync");
+    h.release();
+    expect(fs.existsSync(h.path)).toBe(false);
+  });
+
+  it("refuses fast with the holder's command + pid when a LIVE process holds it", () => {
+    // Simulate a DIFFERENT live process holding the lock. PID 1 (init/systemd)
+    // is always alive and is never our own pid, so kill(1,0) reports alive and
+    // the same-process reclaim path does not apply.
+    writeLock(lockPathFor(rootA), { pid: 1, command: "rescue" });
+    expect(() => acquireOperationLock(rootA, "sync")).toThrowError(OperationLockedError);
+    try {
+      acquireOperationLock(rootA, "sync");
+    } catch (e) {
+      const err = e as OperationLockedError;
+      expect(err.holder.command).toBe("rescue");
+      expect(err.holder.pid).toBe(1);
+      expect(err.message).toContain("rescue");
+      expect(err.message).toContain("pid 1");
+    }
+  });
+
+  it("reclaims a stale lock whose holder PID is dead (takeover)", () => {
+    const stale = deadPid();
+    writeLock(lockPathFor(rootA), { pid: stale, command: "sync" });
+    // The dead holder must not block us.
+    const h = acquireOperationLock(rootA, "rescue");
+    const info = JSON.parse(fs.readFileSync(h.path, "utf8")) as LockInfo;
+    expect(info.pid).toBe(process.pid); // we took it over
+    expect(info.command).toBe("rescue");
+    h.release();
+  });
+
+  it("reclaims a torn/unreadable lock file", () => {
+    const p = lockPathFor(rootA);
+    fs.mkdirSync(path.dirname(p), { recursive: true });
+    fs.writeFileSync(p, "{ this is not valid json");
+    const h = acquireOperationLock(rootA, "reindex");
+    expect(fs.existsSync(h.path)).toBe(true);
+    h.release();
+  });
+
+  it("different HQ roots are independent — both may hold concurrently", () => {
+    const a = acquireOperationLock(rootA, "sync");
+    const b = acquireOperationLock(rootB, "rescue"); // must NOT refuse
+    expect(fs.existsSync(a.path)).toBe(true);
+    expect(fs.existsSync(b.path)).toBe(true);
+    expect(a.path).not.toBe(b.path);
+    a.release();
+    b.release();
+  });
+
+  it("the same root is mutually exclusive across different commands", () => {
+    // A live sync in ANOTHER process holds the root (pid 1 stands in for it).
+    const p = lockPathFor(rootA);
+    writeLock(p, { pid: 1, command: "sync" });
+    // Neither rescue nor reindex may acquire while that sync holds it.
+    expect(() => acquireOperationLock(rootA, "rescue")).toThrowError(OperationLockedError);
+    expect(() => acquireOperationLock(rootA, "reindex")).toThrowError(OperationLockedError);
+    // Once that sync finishes (its lock is gone), the next command acquires.
+    fs.unlinkSync(p);
+    const h2 = acquireOperationLock(rootA, "reindex");
+    expect(fs.existsSync(h2.path)).toBe(true);
+    h2.release();
+  });
+
+  it("withOperationLockSync releases even when the body throws", () => {
+    const p = lockPathFor(rootA);
+    expect(() =>
+      withOperationLockSync(rootA, "reindex", () => {
+        expect(fs.existsSync(p)).toBe(true); // held during the body
+        throw new Error("boom");
+      }),
+    ).toThrow("boom");
+    expect(fs.existsSync(p)).toBe(false); // released on the way out
+  });
+
+  it("HQ_DISABLE_OP_LOCK=1 makes acquisition a no-op", () => {
+    process.env.HQ_DISABLE_OP_LOCK = "1";
+    // Even with a live holder on record, the escape hatch acquires without error.
+    writeLock(lockPathFor(rootA), { pid: process.pid, command: "sync" });
+    const h = acquireOperationLock(rootA, "rescue");
+    expect(h.path).toBe(""); // no real lock file written
+    h.release(); // no-op, no throw
+  });
+
+  it("OPERATION_LOCKED_EXIT is a stable non-zero code", () => {
+    expect(OPERATION_LOCKED_EXIT).toBe(17);
+  });
+});