@indigoai-us/hq-cloud 6.2.7 → 6.3.1

package/src/operation-lock.ts

@@ -0,0 +1,293 @@
+/**
+ * Per-HQ-root mutual exclusion for the long-running operations
+ * (`sync`, `rescue`, `reindex`).
+ *
+ * Contract:
+ *   - At most ONE of sync / rescue / reindex runs at a time **per HQ root**.
+ *     The lock is shared across all three (keyed only by the root, not the
+ *     command), so e.g. a rescue refuses while a sync holds it. Different HQ
+ *     roots are fully independent — they hash to different lock files.
+ *   - The push watcher / watch+event-push runner is EXEMPT: it never calls in
+ *     here, so it neither takes the lock nor is blocked by it (its targeted
+ *     in-process push passes are likewise lock-free).
+ *
+ * ## Where the lock lives — and why
+ *
+ * `<stateDir>/locks/operation-<hash(canonicalRoot)>.lock`, where
+ * `stateDir = $HQ_STATE_DIR || ~/.hq`. This is deliberately NOT inside the HQ
+ * root:
+ *   - It must never round-trip to the cloud. A lock is machine-local, per-run
+ *     state; syncing it to S3 (and thence to other machines/roots) would be a
+ *     correctness bug. `~/.hq` is the established machine-local state dir
+ *     (journals already live there) and is never synced.
+ *   - `rescue` repairs a possibly-broken HQ root; a lock that depends on the
+ *     root being healthy is exactly backwards. `~/.hq` is independent of the
+ *     root's health.
+ *   - Keying the filename by a hash of the *canonical* root path makes the
+ *     lock per-root and prevents leakage across roots, while keeping the path
+ *     short and filesystem-safe.
+ *
+ * ## Atomicity, liveness, takeover
+ *
+ *   - Acquisition uses `open(…, "wx")` (O_CREAT | O_EXCL) — an atomic
+ *     create-if-absent. Exactly one racer can create the file; the loser sees
+ *     EEXIST and re-evaluates.
+ *   - The lock records the holder's `{ pid, command, startedAt, hqRoot }`. On
+ *     EEXIST we test the recorded PID with `process.kill(pid, 0)`:
+ *       * ESRCH  → the holder is gone (crashed / killed -9 / stale file) →
+ *                  reclaim the lock.
+ *       * EPERM  → the PID exists but is owned by another user → treat as ALIVE
+ *                  (conservative: refuse rather than risk two concurrent ops).
+ *       * success → alive → refuse fast with {@link OperationLockedError}
+ *                  naming the holding command + PID.
+ *   - PID reuse is an inherent, un-eliminable race for any PID-based scheme: if
+ *     the original holder crashed and the OS later handed its PID to an
+ *     unrelated process, we conservatively read that as "still held" and
+ *     refuse. We accept that false-busy over the far worse false-free, and
+ *     record `startedAt`/`command` so an operator can diagnose a wedged lock.
+ *
+ * ## Release
+ *
+ *   - Normal exit: the `with*` wrappers release in a `finally`.
+ *   - Signals (SIGINT/SIGTERM): a one-time handler releases every held lock,
+ *     then re-raises the default disposition so exit status is unchanged.
+ *   - Hard crash (SIGKILL / power loss): nothing runs, but the stale-PID
+ *     takeover above reclaims the lock on the next attempt.
+ *   - `process.on("exit")`: a final best-effort synchronous unlink.
+ *
+ * ## Escape hatch
+ *
+ *   `HQ_DISABLE_OP_LOCK=1` makes acquisition a no-op (returns a handle whose
+ *   release does nothing). For emergencies and for callers that manage
+ *   exclusion themselves; documented, off by default.
+ */
+
+import * as crypto from "crypto";
+import * as fs from "fs";
+import * as os from "os";
+import * as path from "path";
+
+/** Process exit code used when an operation is refused because the lock is held. */
+export const OPERATION_LOCKED_EXIT = 17;
+
+export interface LockInfo {
+  pid: number;
+  command: string;
+  /** ISO-8601 acquisition time. */
+  startedAt: string;
+  /** Canonical HQ root the lock guards (diagnostic only). */
+  hqRoot: string;
+}
+
+/** Thrown by `acquireOperationLock` when a LIVE holder owns the lock. */
+export class OperationLockedError extends Error {
+  constructor(
+    public readonly holder: LockInfo,
+    public readonly attempted: string,
+  ) {
+    super(
+      `Refusing to start "${attempted}": another HQ operation is already ` +
+        `running for this HQ root — "${holder.command}" (pid ${holder.pid}, ` +
+        `started ${holder.startedAt}). Wait for it to finish, or stop that ` +
+        `process, then retry.`,
+    );
+    this.name = "OperationLockedError";
+  }
+}
+
+export interface LockHandle {
+  /** Absolute path of the lock file. */
+  readonly path: string;
+  /** The info written for this holder. */
+  readonly info: LockInfo;
+  /** Idempotently release the lock iff this process still owns it. */
+  release(): void;
+}
+
+function stateDir(): string {
+  return process.env.HQ_STATE_DIR || path.join(os.homedir(), ".hq");
+}
+
+/** Absolute lock path for a given HQ root. Exported for tests. */
+export function lockPathFor(hqRoot: string): string {
+  const canon = path.resolve(hqRoot);
+  const key = crypto.createHash("sha1").update(canon).digest("hex").slice(0, 16);
+  return path.join(stateDir(), "locks", `operation-${key}.lock`);
+}
+
+/**
+ * Is `pid` a live process? `kill(pid, 0)` sends no signal; it only probes.
+ * ESRCH → no such process (dead/stale). EPERM → exists but not ours → ALIVE
+ * (conservative). Anything else → assume alive rather than risk a double-run.
+ */
+function pidAlive(pid: number): boolean {
+  if (!Number.isInteger(pid) || pid <= 0) return false;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (err) {
+    const code = (err as NodeJS.ErrnoException)?.code;
+    if (code === "ESRCH") return false;
+    return true; // EPERM (exists) or unknown → treat as alive
+  }
+}
+
+function readLockInfo(p: string): LockInfo | null {
+  try {
+    const parsed = JSON.parse(fs.readFileSync(p, "utf8")) as LockInfo;
+    if (parsed && typeof parsed.pid === "number" && typeof parsed.command === "string") {
+      return parsed;
+    }
+    return null;
+  } catch {
+    return null;
+  }
+}
+
+// ── Process-wide release plumbing ──────────────────────────────────────────
+// Track every lock this process currently holds so the signal/exit hooks can
+// release all of them. The hooks are installed exactly once.
+
+const heldLocks = new Set<LockHandle>();
+let hooksInstalled = false;
+
+function unlinkIfOwned(p: string): void {
+  // Only remove a lock whose recorded pid is THIS process — never clobber a
+  // lock another process took over after a (hypothetical) reclaim race.
+  const info = readLockInfo(p);
+  if (info && info.pid === process.pid) {
+    try {
+      fs.unlinkSync(p);
+    } catch {
+      /* already gone — fine */
+    }
+  }
+}
+
+function installHooksOnce(): void {
+  if (hooksInstalled) return;
+  hooksInstalled = true;
+
+  process.on("exit", () => {
+    for (const h of heldLocks) unlinkIfOwned(h.path);
+  });
+
+  for (const sig of ["SIGINT", "SIGTERM", "SIGHUP"] as const) {
+    process.on(sig, () => {
+      for (const h of heldLocks) unlinkIfOwned(h.path);
+      // Re-raise with the default disposition so the exit status is the normal
+      // signal status (and a second Ctrl-C still works). Removing our listener
+      // first avoids recursing back into this handler.
+      process.removeAllListeners(sig);
+      process.kill(process.pid, sig);
+    });
+  }
+}
+
+function makeHandle(p: string, info: LockInfo): LockHandle {
+  const handle: LockHandle = {
+    path: p,
+    info,
+    release() {
+      heldLocks.delete(handle);
+      unlinkIfOwned(p);
+    },
+  };
+  heldLocks.add(handle);
+  installHooksOnce();
+  return handle;
+}
+
+const NOOP_HANDLE_BASE = { release() {} };
+
+/**
+ * Acquire the per-root operation lock for `command`. Returns a {@link LockHandle}
+ * on success; throws {@link OperationLockedError} when a live holder owns it.
+ * Reclaims a stale lock (dead holder) transparently.
+ */
+export function acquireOperationLock(hqRoot: string, command: string): LockHandle {
+  if (process.env.HQ_DISABLE_OP_LOCK === "1") {
+    const info: LockInfo = {
+      pid: process.pid,
+      command,
+      startedAt: new Date().toISOString(),
+      hqRoot: path.resolve(hqRoot),
+    };
+    return { ...NOOP_HANDLE_BASE, path: "", info };
+  }
+
+  const p = lockPathFor(hqRoot);
+  fs.mkdirSync(path.dirname(p), { recursive: true });
+
+  const info: LockInfo = {
+    pid: process.pid,
+    command,
+    startedAt: new Date().toISOString(),
+    hqRoot: path.resolve(hqRoot),
+  };
+  const payload = JSON.stringify(info, null, 2);
+
+  // Bounded retry: each iteration is one atomic create attempt. EEXIST against
+  // a stale holder reclaims and retries; EEXIST against a live holder refuses.
+  const MAX_ATTEMPTS = 5;
+  for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
+    let fd: number;
+    try {
+      fd = fs.openSync(p, "wx"); // O_CREAT | O_EXCL — atomic
+    } catch (err) {
+      if ((err as NodeJS.ErrnoException)?.code !== "EEXIST") throw err;
+
+      const holder = readLockInfo(p);
+      if (holder && holder.pid !== process.pid && pidAlive(holder.pid)) {
+        throw new OperationLockedError(holder, command);
+      }
+      // Stale (dead holder), unreadable/torn, or our own leftover → reclaim.
+      try {
+        fs.unlinkSync(p);
+      } catch {
+        /* someone else reclaimed it first; the next openSync re-evaluates */
+      }
+      continue;
+    }
+    try {
+      fs.writeSync(fd, payload);
+    } finally {
+      fs.closeSync(fd);
+    }
+    return makeHandle(p, info);
+  }
+
+  // Pathological churn (another process reclaiming in lockstep). Surface it
+  // rather than spin forever.
+  throw new Error(
+    `Could not acquire HQ operation lock at ${p} after ${MAX_ATTEMPTS} attempts`,
+  );
+}
+
+/** Run `fn` while holding the per-root lock for `command` (async). */
+export async function withOperationLock<T>(
+  hqRoot: string,
+  command: string,
+  fn: () => Promise<T>,
+): Promise<T> {
+  const handle = acquireOperationLock(hqRoot, command);
+  try {
+    return await fn();
+  } finally {
+    handle.release();
+  }
+}
+
+/** Run `fn` while holding the per-root lock for `command` (synchronous). */
+export function withOperationLockSync<T>(
+  hqRoot: string,
+  command: string,
+  fn: () => T,
+): T {
+  const handle = acquireOperationLock(hqRoot, command);
+  try {
+    return fn();
+  } finally {
+    handle.release();
+  }
+}