@indigoai-us/hq-cloud 6.2.7 → 6.3.1

package/src/cli/rescue-core.ts

@@ -384,9 +384,22 @@ function doRescue(
     hqRoot = fs.realpathSync(process.cwd());
   }
 
-  if (!isDir(path.join(hqRoot, "companies")) || !isDir(path.join(hqRoot, "personal"))) {
+  // HQ-root sanity gate — guards against wiping a non-HQ directory. `companies/`
+  // is the stable anchor (present and preserved on every HQ install), confirmed
+  // by at least one core scaffold marker. We accept ANY of `.claude/`, `core/`,
+  // or `personal/` because the layout has drifted across releases: v14.0.0 ships
+  // NEITHER `personal/` (added v15) NOR `core/` (the v15 scaffold home) — it only
+  // has `.claude/`. Requiring `personal/` here left v14.0.0 users with no upgrade
+  // path (rescue aborted; DEV-1741). `.claude/` exists on every version v14→v15,
+  // so it admits the full upgrade range while still rejecting a bare directory.
+  const hasHqMarker =
+    isDir(path.join(hqRoot, ".claude")) ||
+    isDir(path.join(hqRoot, "core")) ||
+    isDir(path.join(hqRoot, "personal"));
+  if (!isDir(path.join(hqRoot, "companies")) || !hasHqMarker) {
     err(
-      `error: ${hqRoot} does not look like an HQ root (missing companies/ or personal/). Aborting.\n`,
+      `error: ${hqRoot} does not look like an HQ root ` +
+        `(need companies/ plus one of .claude/, core/, or personal/). Aborting.\n`,
     );
     err("       pass --hq-root <dir> if the script is not at personal/skills/<skill>/.\n");
     throw new ExitError(3);

package/src/cli/rescue-hq-root-guard.test.ts

@@ -0,0 +1,193 @@
+/**
+ * Regression test for the rescue HQ-root sanity gate in src/cli/rescue-core.ts.
+ *
+ * Bug (DEV-1741): the gate required BOTH `companies/` AND `personal/`. The
+ * `personal/` directory was only introduced in v15 — a faithful v14.0.0 install
+ * ships NEITHER `personal/` NOR `core/` (its scaffold lived in top-level dirs
+ * like `scripts/`, `workers/`, `knowledge/`, plus `.claude/`). So every v14.0.0
+ * user hit `error: ... missing companies/ or personal/. Aborting.` (exit 3) and
+ * had no supported upgrade path to v15.
+ *
+ * The gate now anchors on `companies/` plus ANY ONE of `.claude/`, `core/`, or
+ * `personal/`. `.claude/` exists on every release from v14 through v15, so the
+ * relaxed gate admits the full upgrade range while still rejecting a directory
+ * that is not an HQ root at all.
+ *
+ * Like the sibling drift test, the rescue clones its source from GitHub, so we
+ * shim `git clone` to a local fixture repo. All runs are `--dry-run` (no
+ * destructive ops, no backup) and we assert on the gate's behavior (exit code +
+ * message), not the full overlay.
+ */
+import { describe, it, expect, beforeAll, afterAll } from "vitest";
+import { execFileSync } from "child_process";
+import * as fs from "fs";
+import * as os from "os";
+import * as path from "path";
+import { runRescue } from "./rescue-core.js";
+
+function hasGit(): boolean {
+  try {
+    execFileSync("git", ["--version"], { stdio: "ignore" });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+const gitAvailable = hasGit();
+
+/** Run the rescue in-process, capturing its stdout/stderr. */
+function runRescueCapture(argv: string[], env: NodeJS.ProcessEnv) {
+  let stdout = "";
+  let stderr = "";
+  const origOut = process.stdout.write.bind(process.stdout);
+  const origErr = process.stderr.write.bind(process.stderr);
+  process.stdout.write = ((chunk: unknown) => {
+    stdout += String(chunk);
+    return true;
+  }) as typeof process.stdout.write;
+  process.stderr.write = ((chunk: unknown) => {
+    stderr += String(chunk);
+    return true;
+  }) as typeof process.stderr.write;
+  let status: number;
+  try {
+    status = runRescue(argv, { env }).status;
+  } finally {
+    process.stdout.write = origOut;
+    process.stderr.write = origErr;
+  }
+  return { status, stdout, stderr };
+}
+
+describe.skipIf(!gitAvailable)("rescue HQ-root sanity gate", () => {
+  let workDir: string;
+  let upstream: string;
+  let shimDir: string;
+  let floorSha: string;
+  let env: NodeJS.ProcessEnv;
+
+  const git = (cwd: string, ...args: string[]) =>
+    execFileSync("git", args, {
+      cwd,
+      stdio: ["ignore", "pipe", "pipe"],
+      env: {
+        ...process.env,
+        GIT_AUTHOR_NAME: "t",
+        GIT_AUTHOR_EMAIL: "t@t",
+        GIT_COMMITTER_NAME: "t",
+        GIT_COMMITTER_EMAIL: "t@t",
+      },
+    })
+      .toString()
+      .trim();
+
+  beforeAll(() => {
+    workDir = fs.mkdtempSync(path.join(os.tmpdir(), "hq-rescue-gate-"));
+
+    // --- Minimal local "upstream" repo (one core file, floor + HEAD). ---
+    upstream = path.join(workDir, "upstream");
+    fs.mkdirSync(path.join(upstream, "core"), { recursive: true });
+    git(workDir, "init", "-b", "main", "upstream");
+    fs.writeFileSync(path.join(upstream, "core/x.md"), "v1\n");
+    git(upstream, "add", "-A");
+    git(upstream, "commit", "-m", "floor");
+    floorSha = git(upstream, "rev-parse", "HEAD");
+    fs.writeFileSync(path.join(upstream, "core/x.md"), "v2\n");
+    git(upstream, "add", "-A");
+    git(upstream, "commit", "-m", "head");
+
+    // --- git shim: rewrite `git clone <github-url> dest` to the fixture. ---
+    const realGit =
+      execFileSync("bash", ["-lc", "command -v git"]).toString().trim() || "/usr/bin/git";
+    shimDir = path.join(workDir, "shim");
+    fs.mkdirSync(shimDir, { recursive: true });
+    const shim = `#!/usr/bin/env bash
+if [ "$1" = "clone" ]; then
+  args=()
+  for a in "$@"; do
+    case "$a" in
+      https://github.com/*) a=${JSON.stringify(upstream)} ;;
+    esac
+    args+=("$a")
+  done
+  exec ${JSON.stringify(realGit)} "\${args[@]}"
+fi
+exec ${JSON.stringify(realGit)} "$@"
+`;
+    fs.writeFileSync(path.join(shimDir, "git"), shim, { mode: 0o755 });
+    env = { ...process.env, PATH: `${shimDir}:${process.env.PATH ?? ""}` };
+  }, 60_000); // git init + commits can exceed vitest's 10s default under parallel load
+
+  afterAll(() => {
+    if (workDir) fs.rmSync(workDir, { recursive: true, force: true });
+  });
+
+  function makeRoot(name: string, dirs: string[]): string {
+    const root = path.join(workDir, name);
+    for (const d of dirs) fs.mkdirSync(path.join(root, d), { recursive: true });
+    return root;
+  }
+
+  function rescueDry(hqRoot: string) {
+    return runRescueCapture(
+      [
+        "--hq-root", hqRoot,
+        "--source", "test/repo",
+        "--ref", "main",
+        "--floor-sha", floorSha,
+        "--dry-run",
+        "--yes",
+        "--no-backup",
+      ],
+      env,
+    );
+  }
+
+  it("admits a faithful v14.0.0 root (companies/ + .claude/, NO personal/, NO core/)", () => {
+    // Exactly the shape that aborted before the fix.
+    const root = makeRoot("v14", ["companies", ".claude", "repos", "workspace"]);
+    expect(fs.existsSync(path.join(root, "personal"))).toBe(false);
+    expect(fs.existsSync(path.join(root, "core"))).toBe(false);
+
+    const r = rescueDry(root);
+    const out = `${r.stdout}\n${r.stderr}`;
+    // Must get PAST the gate — not the exit-3 abort.
+    expect(r.status, out).not.toBe(3);
+    expect(out).not.toContain("does not look like an HQ root");
+    expect(r.status, out).toBe(0);
+  });
+
+  it("admits a v15 root (companies/ + core/ + personal/)", () => {
+    const root = makeRoot("v15", ["companies", ".claude", "core", "personal", "repos", "workspace"]);
+    const r = rescueDry(root);
+    const out = `${r.stdout}\n${r.stderr}`;
+    expect(r.status, out).toBe(0);
+    expect(out).not.toContain("does not look like an HQ root");
+  });
+
+  it("admits a core-only root (companies/ + core/, no .claude/ or personal/)", () => {
+    const root = makeRoot("coreonly", ["companies", "core"]);
+    const r = rescueDry(root);
+    const out = `${r.stdout}\n${r.stderr}`;
+    expect(r.status, out).not.toBe(3);
+    expect(out).not.toContain("does not look like an HQ root");
+  });
+
+  it("still rejects a non-HQ directory (companies/ only — no scaffold marker)", () => {
+    const root = makeRoot("bare", ["companies"]);
+    const r = rescueDry(root);
+    const out = `${r.stdout}\n${r.stderr}`;
+    expect(r.status, out).toBe(3);
+    expect(out).toContain("does not look like an HQ root");
+    expect(out).toContain("need companies/ plus one of .claude/, core/, or personal/");
+  });
+
+  it("still rejects a directory with a marker but no companies/", () => {
+    const root = makeRoot("nocompanies", [".claude", "core", "personal"]);
+    const r = rescueDry(root);
+    const out = `${r.stdout}\n${r.stderr}`;
+    expect(r.status, out).toBe(3);
+    expect(out).toContain("does not look like an HQ root");
+  });
+});

package/src/cli/rescue.reindex.test.ts

@@ -7,7 +7,10 @@
  * runs, and ./reindex.js is mocked to a spy so we assert the call without
  * touching disk.
  */
-import { describe, it, expect, vi, beforeEach } from "vitest";
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import * as fs from "fs";
+import * as os from "os";
+import * as path from "path";
 
 vi.mock("./rescue-core.js", () => ({
   runRescue: vi.fn(() => ({ status: 0 })),
@@ -21,16 +24,28 @@ import { reindex } from "./reindex.js";
 import { rescue } from "./rescue.js";
 
 describe("rescue → reindex", () => {
+  let stateDir: string;
+
   beforeEach(() => {
     vi.clearAllMocks();
     (runRescue as unknown as ReturnType<typeof vi.fn>).mockReturnValue({ status: 0 });
+    // rescue() now takes the per-root operation lock; redirect it to a tmp dir.
+    stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "rescue-reindex-state-"));
+    process.env.HQ_STATE_DIR = stateDir;
+  });
+
+  afterEach(() => {
+    fs.rmSync(stateDir, { recursive: true, force: true });
+    delete process.env.HQ_STATE_DIR;
   });
 
   it("refreshes via reindex after a successful rescue", () => {
     const r = rescue({ hqRoot: "/tmp/hq", assumeYes: true });
     expect(r.status).toBe(0);
     expect(reindex).toHaveBeenCalledTimes(1);
-    expect(reindex).toHaveBeenCalledWith({ repoRoot: "/tmp/hq" });
+    // skipLock: rescue already holds the per-root operation lock, so the
+    // internal reindex must not try to re-acquire it.
+    expect(reindex).toHaveBeenCalledWith({ repoRoot: "/tmp/hq", skipLock: true });
   });
 
   it("does NOT run reindex on a dry-run", () => {

package/src/cli/rescue.ts

@@ -14,6 +14,11 @@
  */
 import { reindex } from "./reindex.js";
 import { runRescue } from "./rescue-core.js";
+import {
+  acquireOperationLock,
+  OperationLockedError,
+  OPERATION_LOCKED_EXIT,
+} from "../operation-lock.js";
 
 export interface RescueOptions {
   /** HQ root to operate on. Passed as `--hq-root`. Defaults to the script's
@@ -91,21 +96,41 @@ export function buildRescueArgs(opts: RescueOptions = {}): string[] {
  * confirmation prompt (unless `assumeYes` is set).
  */
 export function rescue(opts: RescueOptions = {}): RescueResult {
-  const args = buildRescueArgs(opts);
-  const env: NodeJS.ProcessEnv = { ...process.env };
-  if (opts.ghToken) env.GH_TOKEN = opts.ghToken;
-  const { status } = runRescue(args, { env });
-  // A successful, non-dry-run rescue re-lays-down core/, so refresh the
-  // generated skill wrappers, personal-overlay mirrors, and workers registry.
-  // Best-effort + idempotent — never overrides the rescue's own exit status.
-  // repoRoot falls back to process.cwd() (reindex's default) when hqRoot is
-  // omitted, matching the rescue script's own cwd-based default.
-  if (status === 0 && !opts.dryRun) {
-    try {
-      reindex({ repoRoot: opts.hqRoot });
-    } catch {
-      // best-effort
+  // Rescue is mutually exclusive with sync/reindex on this HQ root. Key the
+  // lock on the same root the rescue script resolves (cwd when --hq-root is
+  // omitted). Rescue is never the exempt push watcher, so it always locks.
+  const lockRoot = opts.hqRoot ?? process.cwd();
+  let handle;
+  try {
+    handle = acquireOperationLock(lockRoot, "rescue");
+  } catch (err) {
+    if (err instanceof OperationLockedError) {
+      process.stderr.write(err.message + "\n");
+      return { status: OPERATION_LOCKED_EXIT };
     }
+    throw err;
+  }
+  try {
+    const args = buildRescueArgs(opts);
+    const env: NodeJS.ProcessEnv = { ...process.env };
+    if (opts.ghToken) env.GH_TOKEN = opts.ghToken;
+    const { status } = runRescue(args, { env });
+    // A successful, non-dry-run rescue re-lays-down core/, so refresh the
+    // generated skill wrappers, personal-overlay mirrors, and workers registry.
+    // Best-effort + idempotent — never overrides the rescue's own exit status.
+    // repoRoot falls back to process.cwd() (reindex's default) when hqRoot is
+    // omitted, matching the rescue script's own cwd-based default. `skipLock`
+    // because we already hold the per-root lock — reindex's own acquire would
+    // otherwise see our live PID and refuse.
+    if (status === 0 && !opts.dryRun) {
+      try {
+        reindex({ repoRoot: opts.hqRoot, skipLock: true });
+      } catch {
+        // best-effort
+      }
+    }
+    return { status };
+  } finally {
+    handle.release();
   }
-  return { status };
 }

package/src/cli/sync.test.ts

@@ -123,7 +123,8 @@ describe("sync", () => {
 
   it("runs reindex against hqRoot after a sync that downloaded files", async () => {
     await sync({ company: "acme", vaultConfig: mockConfig, hqRoot: tmpDir });
-    expect(reindex).toHaveBeenCalledWith({ repoRoot: tmpDir });
+    // skipLock: the surrounding sync run already holds the per-root lock.
+    expect(reindex).toHaveBeenCalledWith({ repoRoot: tmpDir, skipLock: true });
   });
 
   it("skips reindex when skipReindex is set", async () => {

package/src/cli/sync.ts

@@ -1338,7 +1338,9 @@ export async function sync(options: SyncOptions): Promise<SyncResult> {
     shrinkResult.cleanRemoved > 0;
   if (!options.skipReindex && changedOnDisk) {
     try {
-      reindex({ repoRoot: hqRoot });
+      // skipLock: the surrounding sync run already holds this root's operation
+      // lock; reindex re-acquiring would refuse against our own live PID.
+      reindex({ repoRoot: hqRoot, skipLock: true });
     } catch {
       // best-effort: a post-sync refresh failure never fails the sync
     }

package/src/operation-lock.test.ts

@@ -0,0 +1,162 @@
+/**
+ * Unit tests for the per-HQ-root operation mutex.
+ */
+
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import { spawnSync } from "child_process";
+import * as fs from "fs";
+import * as os from "os";
+import * as path from "path";
+import {
+  acquireOperationLock,
+  withOperationLockSync,
+  lockPathFor,
+  OperationLockedError,
+  OPERATION_LOCKED_EXIT,
+  type LockInfo,
+} from "./operation-lock.js";
+
+/** A PID that is guaranteed dead: spawn a node that exits immediately, reuse its pid. */
+function deadPid(): number {
+  const r = spawnSync(process.execPath, ["-e", ""], { stdio: "ignore" });
+  if (!r.pid) throw new Error("could not spawn to obtain a dead pid");
+  return r.pid;
+}
+
+function writeLock(p: string, info: Partial<LockInfo>): void {
+  fs.mkdirSync(path.dirname(p), { recursive: true });
+  const full: LockInfo = {
+    pid: 1,
+    command: "sync",
+    startedAt: new Date(0).toISOString(),
+    hqRoot: "/x",
+    ...info,
+  };
+  fs.writeFileSync(p, JSON.stringify(full));
+}
+
+describe("operation-lock", () => {
+  let stateDir: string;
+  let rootA: string;
+  let rootB: string;
+
+  beforeEach(() => {
+    stateDir = fs.mkdtempSync(path.join(os.tmpdir(), "hq-oplock-state-"));
+    process.env.HQ_STATE_DIR = stateDir;
+    delete process.env.HQ_DISABLE_OP_LOCK;
+    rootA = fs.mkdtempSync(path.join(os.tmpdir(), "hq-rootA-"));
+    rootB = fs.mkdtempSync(path.join(os.tmpdir(), "hq-rootB-"));
+  });
+
+  afterEach(() => {
+    fs.rmSync(stateDir, { recursive: true, force: true });
+    fs.rmSync(rootA, { recursive: true, force: true });
+    fs.rmSync(rootB, { recursive: true, force: true });
+    delete process.env.HQ_STATE_DIR;
+    delete process.env.HQ_DISABLE_OP_LOCK;
+  });
+
+  it("the lock path is under the state dir, keyed per canonical root", () => {
+    const a = lockPathFor(rootA);
+    const b = lockPathFor(rootB);
+    expect(a.startsWith(path.join(stateDir, "locks"))).toBe(true);
+    expect(a).not.toBe(b); // different roots → different lock files
+    // canonical: a trailing-slash variant maps to the same lock
+    expect(lockPathFor(rootA + path.sep)).toBe(a);
+  });
+
+  it("acquires, writes holder info, and releases (file gone after release)", () => {
+    const h = acquireOperationLock(rootA, "sync");
+    expect(fs.existsSync(h.path)).toBe(true);
+    const info = JSON.parse(fs.readFileSync(h.path, "utf8")) as LockInfo;
+    expect(info.pid).toBe(process.pid);
+    expect(info.command).toBe("sync");
+    h.release();
+    expect(fs.existsSync(h.path)).toBe(false);
+  });
+
+  it("refuses fast with the holder's command + pid when a LIVE process holds it", () => {
+    // Simulate a DIFFERENT live process holding the lock. PID 1 (init/systemd)
+    // is always alive and is never our own pid, so kill(1,0) reports alive and
+    // the same-process reclaim path does not apply.
+    writeLock(lockPathFor(rootA), { pid: 1, command: "rescue" });
+    expect(() => acquireOperationLock(rootA, "sync")).toThrowError(OperationLockedError);
+    try {
+      acquireOperationLock(rootA, "sync");
+    } catch (e) {
+      const err = e as OperationLockedError;
+      expect(err.holder.command).toBe("rescue");
+      expect(err.holder.pid).toBe(1);
+      expect(err.message).toContain("rescue");
+      expect(err.message).toContain("pid 1");
+    }
+  });
+
+  it("reclaims a stale lock whose holder PID is dead (takeover)", () => {
+    const stale = deadPid();
+    writeLock(lockPathFor(rootA), { pid: stale, command: "sync" });
+    // The dead holder must not block us.
+    const h = acquireOperationLock(rootA, "rescue");
+    const info = JSON.parse(fs.readFileSync(h.path, "utf8")) as LockInfo;
+    expect(info.pid).toBe(process.pid); // we took it over
+    expect(info.command).toBe("rescue");
+    h.release();
+  });
+
+  it("reclaims a torn/unreadable lock file", () => {
+    const p = lockPathFor(rootA);
+    fs.mkdirSync(path.dirname(p), { recursive: true });
+    fs.writeFileSync(p, "{ this is not valid json");
+    const h = acquireOperationLock(rootA, "reindex");
+    expect(fs.existsSync(h.path)).toBe(true);
+    h.release();
+  });
+
+  it("different HQ roots are independent — both may hold concurrently", () => {
+    const a = acquireOperationLock(rootA, "sync");
+    const b = acquireOperationLock(rootB, "rescue"); // must NOT refuse
+    expect(fs.existsSync(a.path)).toBe(true);
+    expect(fs.existsSync(b.path)).toBe(true);
+    expect(a.path).not.toBe(b.path);
+    a.release();
+    b.release();
+  });
+
+  it("the same root is mutually exclusive across different commands", () => {
+    // A live sync in ANOTHER process holds the root (pid 1 stands in for it).
+    const p = lockPathFor(rootA);
+    writeLock(p, { pid: 1, command: "sync" });
+    // Neither rescue nor reindex may acquire while that sync holds it.
+    expect(() => acquireOperationLock(rootA, "rescue")).toThrowError(OperationLockedError);
+    expect(() => acquireOperationLock(rootA, "reindex")).toThrowError(OperationLockedError);
+    // Once that sync finishes (its lock is gone), the next command acquires.
+    fs.unlinkSync(p);
+    const h2 = acquireOperationLock(rootA, "reindex");
+    expect(fs.existsSync(h2.path)).toBe(true);
+    h2.release();
+  });
+
+  it("withOperationLockSync releases even when the body throws", () => {
+    const p = lockPathFor(rootA);
+    expect(() =>
+      withOperationLockSync(rootA, "reindex", () => {
+        expect(fs.existsSync(p)).toBe(true); // held during the body
+        throw new Error("boom");
+      }),
+    ).toThrow("boom");
+    expect(fs.existsSync(p)).toBe(false); // released on the way out
+  });
+
+  it("HQ_DISABLE_OP_LOCK=1 makes acquisition a no-op", () => {
+    process.env.HQ_DISABLE_OP_LOCK = "1";
+    // Even with a live holder on record, the escape hatch acquires without error.
+    writeLock(lockPathFor(rootA), { pid: process.pid, command: "sync" });
+    const h = acquireOperationLock(rootA, "rescue");
+    expect(h.path).toBe(""); // no real lock file written
+    h.release(); // no-op, no throw
+  });
+
+  it("OPERATION_LOCKED_EXIT is a stable non-zero code", () => {
+    expect(OPERATION_LOCKED_EXIT).toBe(17);
+  });
+});