@indigoai-us/hq-cloud 6.7.1 → 6.8.0

package/src/operation-lock.ts

@@ -35,11 +35,44 @@
  *   - The lock records the holder's `{ pid, command, startedAt, hqRoot }`. On
  *     EEXIST we test the recorded PID with `process.kill(pid, 0)`:
  *       * ESRCH  → the holder is gone (crashed / killed -9 / stale file) →
- *                  reclaim the lock.
+ *                  reclaim the lock IMMEDIATELY (a dead holder never makes us
+ *                  wait).
  *       * EPERM  → the PID exists but is owned by another user → treat as ALIVE
- *                  (conservative: refuse rather than risk two concurrent ops).
- *       * success → alive → refuse fast with {@link OperationLockedError}
- *                  naming the holding command + PID.
+ *                  (conservative: wait rather than risk two concurrent ops).
+ *       * success → alive → WAIT for the holder to release, then acquire (see
+ *                  "Waiting" below). The fast-refusal path is still reachable
+ *                  via an explicit timeout / `wait: false`.
+ *
+ * ## Waiting for a live holder (default behavior)
+ *
+ *   When a LIVE holder owns the lock, acquisition WAITS by default: it polls
+ *   (~2s) and acquires the instant the holder releases, rather than refusing
+ *   fast. A single status line is written to stderr the first time we start
+ *   waiting ("Waiting for <command> (pid N) to finish…"), never per-poll.
+ *   This is what an interactive `sync` / `rescue` / `reindex` invocation wants —
+ *   queue behind the running op instead of erroring out.
+ *
+ *   A bounded escape exists for scripts that must not block forever:
+ *     - `timeoutSec` option, or the `HQ_OP_LOCK_TIMEOUT` env var (seconds).
+ *       The option wins over the env. After the bound elapses we throw
+ *       {@link OperationLockedError} (exit 17) with the same clear refusal
+ *       message as before.
+ *     - `timeoutSec === 0` (or `HQ_OP_LOCK_TIMEOUT=0`, or `wait: false`) → do
+ *       not wait at all; refuse immediately. This is the pre-wait behavior.
+ *     - absent / negative / unparseable → INFINITE wait (the documented
+ *       default).
+ *   Stale-PID takeover is unconditional and happens BEFORE any wait — a dead
+ *   holder is reclaimed at once regardless of the wait config.
+ *
+ *   Ordering / scope caveats:
+ *     - This is a CROSS-PROCESS mutex keyed on the holder's PID. Two concurrent
+ *       acquisitions inside the SAME process share a PID, so the same-process
+ *       reclaim path lets them stomp each other — in-process concurrent acquire
+ *       is unsupported (the real consumers — sync / rescue / reindex — are
+ *       separate processes).
+ *     - When several distinct processes wait on the same lock, the next one to
+ *       win the O_EXCL race after a free acquires. Order is best-effort, NOT
+ *       FIFO — do not depend on arrival order.
  *   - PID reuse is an inherent, un-eliminable race for any PID-based scheme: if
  *     the original holder crashed and the OS later handed its PID to an
  *     unrelated process, we conservatively read that as "still held" and
@@ -104,6 +137,100 @@ export interface LockHandle {
   release(): void;
 }
 
+/** Default poll interval while waiting on a live holder. */
+export const DEFAULT_LOCK_POLL_MS = 2000;
+
+/** Options controlling how `acquireOperationLock*` behaves against a LIVE holder. */
+export interface AcquireOptions {
+  /**
+   * When a LIVE holder owns the lock: `true` (default) → WAIT-poll until it
+   * frees, then acquire; `false` → refuse immediately with
+   * {@link OperationLockedError}. A `timeoutSec` of 0 is equivalent to
+   * `wait: false`.
+   */
+  wait?: boolean;
+  /**
+   * Bounded wait, in seconds, before giving up and throwing
+   * {@link OperationLockedError} (exit 17). Precedence: this option > the
+   * `HQ_OP_LOCK_TIMEOUT` env var > infinite. `0` → do not wait at all (refuse
+   * immediately). Negative / non-finite → treated as absent (infinite wait).
+   * Fractional values are honored (used by tests); the CLI flags accept whole
+   * seconds.
+   */
+  timeoutSec?: number;
+  /** Poll interval in ms while waiting. Defaults to {@link DEFAULT_LOCK_POLL_MS}. */
+  pollIntervalMs?: number;
+  /**
+   * Invoked exactly ONCE, the first time we begin waiting on a live holder.
+   * Defaults to a single "Waiting for …" line on stderr. Pass a custom hook
+   * (or a no-op) to redirect/silence the status line.
+   */
+  onWaitStart?: (holder: LockInfo, attempted: string) => void;
+}
+
+interface ResolvedWaitConfig {
+  /** null → wait forever; >= 0 → wait at most this many ms (0 = no wait). */
+  timeoutMs: number | null;
+  pollMs: number;
+  onWaitStart: (holder: LockInfo, attempted: string) => void;
+}
+
+/** Default status line: a single stderr message naming the holder. */
+function defaultOnWaitStart(holder: LockInfo, attempted: string): void {
+  process.stderr.write(
+    `Waiting for "${holder.command}" (pid ${holder.pid}) to finish before ` +
+      `starting "${attempted}"… (set HQ_OP_LOCK_TIMEOUT=<secs> to bound the wait)\n`,
+  );
+}
+
+/**
+ * Resolve the effective wait config from explicit options + the
+ * `HQ_OP_LOCK_TIMEOUT` env var. Option timeout wins over the env; `wait: false`
+ * forces a zero (no-wait) timeout.
+ */
+function resolveWaitConfig(opts: AcquireOptions): ResolvedWaitConfig {
+  // Parse a seconds value into ms, or null for "absent/infinite". Only a
+  // finite, non-negative number counts; everything else (NaN, Infinity, <0)
+  // means "no explicit bound".
+  const toMs = (sec: number | undefined): number | null => {
+    if (sec === undefined) return null;
+    if (!Number.isFinite(sec) || sec < 0) return null;
+    return Math.round(sec * 1000);
+  };
+
+  let timeoutMs: number | null;
+  if (opts.timeoutSec !== undefined) {
+    timeoutMs = toMs(opts.timeoutSec);
+  } else {
+    const envRaw = process.env.HQ_OP_LOCK_TIMEOUT;
+    timeoutMs = envRaw !== undefined && envRaw !== "" ? toMs(Number(envRaw)) : null;
+  }
+
+  // `wait: false` is shorthand for a zero-length wait (refuse immediately).
+  if (opts.wait === false) timeoutMs = 0;
+
+  const pollMs =
+    opts.pollIntervalMs && opts.pollIntervalMs > 0
+      ? opts.pollIntervalMs
+      : DEFAULT_LOCK_POLL_MS;
+
+  return { timeoutMs, pollMs, onWaitStart: opts.onWaitStart ?? defaultOnWaitStart };
+}
+
+/** Block the current thread for `ms` without busy-spinning (sync consumers). */
+function sleepSync(ms: number): void {
+  if (ms <= 0) return;
+  // Atomics.wait on a private buffer is a clean, CPU-free sleep. The value at
+  // index 0 is 0 and nothing ever notifies it, so this always sleeps the full
+  // timeout (or less if interrupted) and returns "timed-out".
+  Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
+}
+
+/** Non-blocking sleep for the async consumer. */
+function sleepAsync(ms: number): Promise<void> {
+  return new Promise((resolve) => setTimeout(resolve, Math.max(0, ms)));
+}
+
 function stateDir(): string {
   return process.env.HQ_STATE_DIR || path.join(os.homedir(), ".hq");
 }
@@ -200,35 +327,45 @@ function makeHandle(p: string, info: LockInfo): LockHandle {
 
 const NOOP_HANDLE_BASE = { release() {} };
 
-/**
- * Acquire the per-root operation lock for `command`. Returns a {@link LockHandle}
- * on success; throws {@link OperationLockedError} when a live holder owns it.
- * Reclaims a stale lock (dead holder) transparently.
- */
-export function acquireOperationLock(hqRoot: string, command: string): LockHandle {
-  if (process.env.HQ_DISABLE_OP_LOCK === "1") {
-    const info: LockInfo = {
-      pid: process.pid,
-      command,
-      startedAt: new Date().toISOString(),
-      hqRoot: path.resolve(hqRoot),
-    };
-    return { ...NOOP_HANDLE_BASE, path: "", info };
-  }
+/** No-op handle for the `HQ_DISABLE_OP_LOCK=1` escape hatch. */
+function disabledHandle(hqRoot: string, command: string): LockHandle {
+  const info: LockInfo = {
+    pid: process.pid,
+    command,
+    startedAt: new Date().toISOString(),
+    hqRoot: path.resolve(hqRoot),
+  };
+  return { ...NOOP_HANDLE_BASE, path: "", info };
+}
 
+/** Build the lock payload + ensure the locks dir exists. */
+function prepareLock(hqRoot: string, command: string): { p: string; info: LockInfo; payload: string } {
   const p = lockPathFor(hqRoot);
   fs.mkdirSync(path.dirname(p), { recursive: true });
-
   const info: LockInfo = {
     pid: process.pid,
     command,
     startedAt: new Date().toISOString(),
     hqRoot: path.resolve(hqRoot),
   };
-  const payload = JSON.stringify(info, null, 2);
+  return { p, info, payload: JSON.stringify(info, null, 2) };
+}
 
+/**
+ * One acquisition pass. Returns the {@link LockHandle} on success, or
+ * `{ busy }` naming the LIVE holder that blocked us (so the caller can decide
+ * to wait or refuse). A stale/torn/own-leftover lock is reclaimed in-pass and
+ * never reported as busy. Throws only on genuinely pathological churn or a
+ * non-EEXIST fs error.
+ */
+function tryAcquireOnce(
+  p: string,
+  info: LockInfo,
+  payload: string,
+): { handle: LockHandle } | { busy: LockInfo } {
   // Bounded retry: each iteration is one atomic create attempt. EEXIST against
-  // a stale holder reclaims and retries; EEXIST against a live holder refuses.
+  // a stale holder reclaims and retries; EEXIST against a live holder reports
+  // it as busy.
   const MAX_ATTEMPTS = 5;
   for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
     let fd: number;
@@ -239,7 +376,7 @@ export function acquireOperationLock(hqRoot: string, command: string): LockHandl
 
       const holder = readLockInfo(p);
       if (holder && holder.pid !== process.pid && pidAlive(holder.pid)) {
-        throw new OperationLockedError(holder, command);
+        return { busy: holder };
       }
       // Stale (dead holder), unreadable/torn, or our own leftover → reclaim.
       try {
@@ -254,7 +391,7 @@ export function acquireOperationLock(hqRoot: string, command: string): LockHandl
     } finally {
       fs.closeSync(fd);
     }
-    return makeHandle(p, info);
+    return { handle: makeHandle(p, info) };
   }
 
   // Pathological churn (another process reclaiming in lockstep). Surface it
@@ -264,13 +401,83 @@ export function acquireOperationLock(hqRoot: string, command: string): LockHandl
   );
 }
 
+/** ms left until `deadline` (null deadline → never expires). */
+function remainingMs(deadline: number | null): number {
+  return deadline === null ? Infinity : deadline - Date.now();
+}
+
+/**
+ * Acquire the per-root operation lock for `command` (synchronous). Returns a
+ * {@link LockHandle} on success. Against a LIVE holder it WAITS by default
+ * (polling, blocking the thread) and acquires the moment the holder releases;
+ * pass `timeoutSec`/`wait` (or set `HQ_OP_LOCK_TIMEOUT`) to bound or disable the
+ * wait — on expiry it throws {@link OperationLockedError}. A stale lock (dead
+ * holder) is reclaimed immediately, never waited on.
+ */
+export function acquireOperationLock(
+  hqRoot: string,
+  command: string,
+  opts: AcquireOptions = {},
+): LockHandle {
+  if (process.env.HQ_DISABLE_OP_LOCK === "1") return disabledHandle(hqRoot, command);
+
+  const { p, info, payload } = prepareLock(hqRoot, command);
+  const cfg = resolveWaitConfig(opts);
+  const deadline = cfg.timeoutMs === null ? null : Date.now() + cfg.timeoutMs;
+  let announced = false;
+
+  for (;;) {
+    const res = tryAcquireOnce(p, info, payload);
+    if ("handle" in res) return res.handle;
+
+    // A live holder blocked us. Decide: refuse now, or wait and retry.
+    if (remainingMs(deadline) <= 0) throw new OperationLockedError(res.busy, command);
+    if (!announced) {
+      announced = true;
+      cfg.onWaitStart(res.busy, command);
+    }
+    sleepSync(Math.min(cfg.pollMs, remainingMs(deadline)));
+  }
+}
+
+/**
+ * Async counterpart to {@link acquireOperationLock}. Identical semantics, but
+ * the wait yields the event loop (via `setTimeout`) instead of blocking the
+ * thread — required for the async `sync` runner.
+ */
+export async function acquireOperationLockAsync(
+  hqRoot: string,
+  command: string,
+  opts: AcquireOptions = {},
+): Promise<LockHandle> {
+  if (process.env.HQ_DISABLE_OP_LOCK === "1") return disabledHandle(hqRoot, command);
+
+  const { p, info, payload } = prepareLock(hqRoot, command);
+  const cfg = resolveWaitConfig(opts);
+  const deadline = cfg.timeoutMs === null ? null : Date.now() + cfg.timeoutMs;
+  let announced = false;
+
+  for (;;) {
+    const res = tryAcquireOnce(p, info, payload);
+    if ("handle" in res) return res.handle;
+
+    if (remainingMs(deadline) <= 0) throw new OperationLockedError(res.busy, command);
+    if (!announced) {
+      announced = true;
+      cfg.onWaitStart(res.busy, command);
+    }
+    await sleepAsync(Math.min(cfg.pollMs, remainingMs(deadline)));
+  }
+}
+
 /** Run `fn` while holding the per-root lock for `command` (async). */
 export async function withOperationLock<T>(
   hqRoot: string,
   command: string,
   fn: () => Promise<T>,
+  opts: AcquireOptions = {},
 ): Promise<T> {
-  const handle = acquireOperationLock(hqRoot, command);
+  const handle = await acquireOperationLockAsync(hqRoot, command, opts);
   try {
     return await fn();
   } finally {
@@ -283,8 +490,9 @@ export function withOperationLockSync<T>(
   hqRoot: string,
   command: string,
   fn: () => T,
+  opts: AcquireOptions = {},
 ): T {
-  const handle = acquireOperationLock(hqRoot, command);
+  const handle = acquireOperationLock(hqRoot, command, opts);
   try {
     return fn();
   } finally {