@botbuddy/cli 1.29.2 → 1.29.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@botbuddy/cli",
-  "version": "1.29.2",
+  "version": "1.29.4",
   "description": "BotBuddy — Swarm coordination CLI for multi-agent workflows",
   "type": "module",
   "bin": {

package/src/wait-checkpoint.mjs

@@ -0,0 +1,410 @@
+// BOT-1656 — crash-recovery state for `bb wait`.
+//
+// Checkpoints are deliberately small, worktree-local data.  They never contain a
+// credential, command string, or unbounded spine payload; they only preserve the
+// information required to re-arm the *same* logical observation after a harness
+// restarts.
+
+import { chmod, link, lstat, mkdir, open, readFile, readdir, rename, rmdir, unlink } from "node:fs/promises";
+import { readFileSync, unlinkSync } from "node:fs";
+import { execFile as execFileCallback } from "node:child_process";
+import { dirname, join, resolve, relative, sep } from "node:path";
+import { promisify } from "node:util";
+
+export const CHECKPOINT_SCHEMA_VERSION = 1;
+export const CHECKPOINT_MAX_BYTES = 256 * 1024;
+// `bb wait` caps a public receipt at 64 KiB. Keep another 8 KiB for the
+// terminal status, ownership, cloud-delivery, and timestamp fields added after
+// initial creation so every accepted checkpoint can durably save its result.
+export const CHECKPOINT_TERMINAL_RESERVE_BYTES = 72 * 1024;
+export const INVENTORY_MAX_ITEMS = 20;
+const ID_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/;
+const STATUSES = new Set(["preparing", "armed", "interrupted", "terminal", "cancelled", "acknowledged"]);
+const LOCK_MAX_BYTES = 4 * 1024;
+const execFile = promisify(execFileCallback);
+const heldFenceLocks = new Map();
+let installedFenceExitHook = false;
+
+export class WaitCheckpointError extends Error {
+  constructor(code, message = code) {
+    super(message);
+    this.name = "WaitCheckpointError";
+    this.code = code;
+  }
+}
+
+function safeId(id) {
+  if (typeof id !== "string" || !ID_RE.test(id)) {
+    throw new WaitCheckpointError("wait_checkpoint_invalid", "local_wait_id is invalid");
+  }
+  return id;
+}
+
+function assertUnderRoot(root, path) {
+  const rel = relative(root, path);
+  if (rel === "" || rel.startsWith(`..${sep}`) || rel === ".." || resolve(root, rel) !== path) {
+    throw new WaitCheckpointError("wait_checkpoint_invalid", "checkpoint is outside its worktree state directory");
+  }
+}
+
+function validateCheckpoint(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value) || value.schema_version !== CHECKPOINT_SCHEMA_VERSION) {
+    throw new WaitCheckpointError("wait_checkpoint_invalid", "checkpoint schema is unsupported or malformed");
+  }
+  safeId(value.local_wait_id);
+  if (!STATUSES.has(value.status) || typeof value.deadline_at !== "string" || !Number.isFinite(Date.parse(value.deadline_at))) {
+    throw new WaitCheckpointError("wait_checkpoint_invalid", "checkpoint has invalid status or deadline");
+  }
+  if (!Array.isArray(value.conditions)) {
+    throw new WaitCheckpointError("wait_checkpoint_invalid", "checkpoint conditions are invalid");
+  }
+  return value;
+}
+
+async function fsyncDirectory(directory) {
+  let handle;
+  try { handle = await open(directory, "r"); await handle.sync(); }
+  catch { /* Windows/filesystems that cannot fsync a directory still get atomic rename. */ }
+  finally { await handle?.close(); }
+}
+
+async function durableJson(path, value) {
+  const text = `${JSON.stringify(value)}\n`;
+  if (Buffer.byteLength(text, "utf8") > CHECKPOINT_MAX_BYTES) {
+    throw new WaitCheckpointError("wait_checkpoint_write_failed", "checkpoint exceeds 256 KiB");
+  }
+  const temp = `${path}.${process.pid}.${crypto.randomUUID()}.tmp`;
+  let handle;
+  try {
+    handle = await open(temp, "wx", 0o600);
+    await handle.writeFile(text);
+    await handle.sync();
+    await handle.close();
+    handle = null;
+    await chmod(temp, 0o600);
+    await rename(temp, path);
+    await chmod(path, 0o600);
+    await fsyncDirectory(dirname(path));
+  } catch (error) {
+    await handle?.close().catch(() => {});
+    await unlink(temp).catch(() => {});
+    if (error instanceof WaitCheckpointError) throw error;
+    throw new WaitCheckpointError("wait_checkpoint_write_failed", `could not persist wait checkpoint: ${error?.message ?? error}`);
+  }
+}
+
+async function processStartedAt(pid) {
+  if (!Number.isSafeInteger(pid) || pid <= 0) return null;
+  try {
+    const { stdout } = await execFile("ps", ["-o", "lstart=", "-p", String(pid)], {
+      timeout: 1_000,
+      maxBuffer: LOCK_MAX_BYTES,
+    });
+    const value = String(stdout).trim();
+    return value || null;
+  } catch {
+    return null;
+  }
+}
+
+function installFenceExitHook() {
+  if (installedFenceExitHook) return;
+  installedFenceExitHook = true;
+  process.once("exit", () => {
+    for (const [path, token] of heldFenceLocks) {
+      try {
+        const lock = JSON.parse(readFileSync(path, "utf8"));
+        if (lock?.token === token) unlinkSync(path);
+      } catch { /* A SIGKILL or a replaced lock is intentionally left alone. */ }
+    }
+  });
+}
+
+export function createWaitCheckpointStore({ root = process.cwd(), now = () => new Date() } = {}) {
+  const worktree = resolve(root);
+  const directory = join(worktree, ".botbuddy", "waits");
+  const pathFor = (id) => {
+    const path = join(directory, `${safeId(id)}.json`);
+    assertUnderRoot(directory, path);
+    return path;
+  };
+  const lockPathFor = (id) => {
+    const path = join(directory, `${safeId(id)}.lock`);
+    assertUnderRoot(directory, path);
+    return path;
+  };
+  const recoveryPathFor = (id) => {
+    const path = join(directory, `${safeId(id)}.recovery`);
+    assertUnderRoot(directory, path);
+    return path;
+  };
+  const ensureDirectory = async () => {
+    try { await mkdir(directory, { recursive: true, mode: 0o700 }); await chmod(directory, 0o700); }
+    catch (error) { throw new WaitCheckpointError("wait_checkpoint_write_failed", `could not create wait state directory: ${error?.message ?? error}`); }
+  };
+  const read = async (id) => {
+    const path = pathFor(id);
+    let stat;
+    try { stat = await lstat(path); }
+    catch (error) {
+      if (error?.code === "ENOENT") throw new WaitCheckpointError("wait_checkpoint_missing", `saved wait ${id} was not found`);
+      throw new WaitCheckpointError("wait_checkpoint_invalid", `could not inspect saved wait: ${error?.message ?? error}`);
+    }
+    if (!stat.isFile() || stat.isSymbolicLink() || stat.size > CHECKPOINT_MAX_BYTES) {
+      throw new WaitCheckpointError("wait_checkpoint_invalid", "checkpoint is not a regular bounded JSON file");
+    }
+    let parsed;
+    try { parsed = JSON.parse(await readFile(path, "utf8")); }
+    catch { throw new WaitCheckpointError("wait_checkpoint_invalid", "checkpoint is not valid JSON"); }
+    return validateCheckpoint(parsed);
+  };
+  const write = async (value) => {
+    validateCheckpoint(value);
+    await ensureDirectory();
+    await durableJson(pathFor(value.local_wait_id), value);
+    return value;
+  };
+  const readLock = async (path) => {
+    let stat;
+    try { stat = await lstat(path); }
+    catch (error) {
+      if (error?.code === "ENOENT") return null;
+      throw new WaitCheckpointError("wait_owner_unknown", `could not inspect wait owner: ${error?.message ?? error}`);
+    }
+    if (!stat.isFile() || stat.isSymbolicLink() || stat.size > LOCK_MAX_BYTES) {
+      throw new WaitCheckpointError("wait_owner_unknown", "wait ownership fence is invalid");
+    }
+    let lock;
+    try { lock = JSON.parse(await readFile(path, "utf8")); }
+    catch { throw new WaitCheckpointError("wait_owner_unknown", "wait ownership fence is malformed"); }
+    if (!lock || typeof lock.token !== "string" || !Number.isSafeInteger(lock.pid) ||
+      typeof lock.process_started_at !== "string" || !lock.process_started_at) {
+      throw new WaitCheckpointError("wait_owner_unknown", "wait ownership fence is malformed");
+    }
+    return lock;
+  };
+  const lockIsLive = async (lock) => {
+    try { process.kill(lock.pid, 0); }
+    catch (error) {
+      if (error?.code === "ESRCH") return false;
+      return true; // EPERM is deliberately conservative.
+    }
+    const startedAt = await processStartedAt(lock.pid);
+    // A live PID without a matching process-instance identity is never ownership proof.
+    return startedAt != null && startedAt === lock.process_started_at;
+  };
+  const removeRecoveryDirectory = async (path) => {
+    await unlink(join(path, "owner.json")).catch(() => {});
+    await rmdir(path).catch(() => {});
+  };
+  const acquireRecoveryMutex = async (id, owner) => {
+    const path = recoveryPathFor(id);
+    const temp = `${path}.${process.pid}.${crypto.randomUUID()}.tmp`;
+    await mkdir(temp, { mode: 0o700 });
+    const token = crypto.randomUUID();
+    await durableJson(join(temp, "owner.json"), { ...owner, token });
+    const retired = [];
+    for (let attempt = 0; attempt < 3; attempt += 1) {
+      try {
+        await rename(temp, path);
+        await fsyncDirectory(directory);
+        let released = false;
+        return async () => {
+          if (released) return;
+          released = true;
+          // Retire ownership with one atomic rename. A crash while deleting the
+          // retired directory leaves the fixed mutex path free for recovery.
+          const releasedPath = `${path}.${token}.${crypto.randomUUID()}.released`;
+          await rename(path, releasedPath).catch((error) => {
+            if (error?.code !== "ENOENT") throw error;
+          });
+          await removeRecoveryDirectory(releasedPath);
+          for (const stale of retired) await removeRecoveryDirectory(stale);
+          await fsyncDirectory(directory);
+        };
+      } catch (error) {
+        if (!["EEXIST", "ENOTEMPTY"].includes(error?.code)) {
+          await removeRecoveryDirectory(temp);
+          throw error;
+        }
+      }
+      let current;
+      try { current = await readLock(join(path, "owner.json")); }
+      catch (error) {
+        if (error?.code !== "wait_owner_unknown") throw error;
+        current = null;
+      }
+      if (current && await lockIsLive(current)) {
+        await removeRecoveryDirectory(temp);
+        throw new WaitCheckpointError("wait_owner_active", `saved wait ${id} recovery is already in progress`);
+      }
+      // A missing/malformed owner is also stale. The deterministic target keeps
+      // two contenders that observed the same generation from retiring a fresh
+      // mutex published after the first contender wins.
+      const stale = `${path}.${current?.token ?? "invalid"}.stale`;
+      assertUnderRoot(directory, stale);
+      try {
+        await rename(path, stale);
+        retired.push(stale);
+      } catch (error) {
+        if (!["ENOENT", "EEXIST", "ENOTEMPTY"].includes(error?.code)) throw error;
+      }
+    }
+    await removeRecoveryDirectory(temp);
+    throw new WaitCheckpointError("wait_owner_active", `saved wait ${id} recovery ownership changed`);
+  };
+  return {
+    directory,
+    async create(input) {
+      const id = safeId(input.local_wait_id);
+      const timestamp = now().toISOString();
+      const checkpoint = validateCheckpoint({
+        schema_version: CHECKPOINT_SCHEMA_VERSION,
+        local_wait_id: id,
+        status: "preparing",
+        created_at: timestamp,
+        updated_at: timestamp,
+        deadline_at: input.deadline_at,
+        conditions: input.conditions,
+        mode: input.mode ?? "any",
+        ...input,
+        local_wait_id: id,
+        // Scope comes from the store, never caller-supplied checkpoint metadata.
+        worktree,
+      });
+      if (Buffer.byteLength(JSON.stringify(checkpoint), "utf8") >
+          CHECKPOINT_MAX_BYTES - CHECKPOINT_TERMINAL_RESERVE_BYTES) {
+        throw new WaitCheckpointError(
+          "wait_checkpoint_write_failed",
+          "checkpoint leaves insufficient space for its terminal receipt",
+        );
+      }
+      return write(checkpoint);
+    },
+    read,
+    async update(id, patch) {
+      const current = await read(id);
+      const next = validateCheckpoint({ ...current, ...patch, local_wait_id: current.local_wait_id, updated_at: now().toISOString() });
+      return write(next);
+    },
+    async acknowledge(id, { cloudAckPending = false } = {}) {
+      const current = await read(id);
+      if (current.status === "acknowledged") {
+        return cloudAckPending && current.cloud_ack_pending !== true
+          ? this.update(id, { cloud_ack_pending: true })
+          : current;
+      }
+      if (current.status !== "terminal") {
+        throw new WaitCheckpointError("wait_not_resumable", "only a saved terminal wait can be acknowledged");
+      }
+      // The local acknowledgement and its cloud-delivery outbox are one durable
+      // transition. A crash after hiding the checkpoint must still leave bare
+      // `bb wait` a retryable acknowledgement entry.
+      return this.update(id, {
+        status: "acknowledged",
+        acknowledged_at: now().toISOString(),
+        cloud_ack_pending: cloudAckPending,
+      });
+    },
+    async acquireFence(id) {
+      const localWaitId = safeId(id);
+      await ensureDirectory();
+      const path = lockPathFor(localWaitId);
+      const processStartedAtValue = await processStartedAt(process.pid);
+      if (!processStartedAtValue) {
+        throw new WaitCheckpointError("wait_owner_unknown", "could not establish this process identity for wait recovery");
+      }
+      const owner = {
+        pid: process.pid,
+        process_started_at: processStartedAtValue,
+        acquired_at: now().toISOString(),
+      };
+      let releaseRecovery;
+      try {
+        releaseRecovery = await acquireRecoveryMutex(localWaitId, owner);
+        for (let attempt = 0; attempt < 2; attempt += 1) {
+          const token = crypto.randomUUID();
+          const candidate = `${path}.${process.pid}.${token}.candidate`;
+          try {
+            // Write and fsync a private candidate before publishing it with an
+            // atomic no-clobber hard link. SIGKILL can leave a candidate behind,
+            // but never a partially written fence at the shared path.
+            await durableJson(candidate, { ...owner, token });
+            await link(candidate, path);
+            await unlink(candidate).catch(() => {});
+            await fsyncDirectory(directory);
+            heldFenceLocks.set(path, token);
+            installFenceExitHook();
+            let released = false;
+            return {
+              owner,
+              async release() {
+                if (released) return;
+                released = true;
+                heldFenceLocks.delete(path);
+                const current = await readLock(path);
+                if (current?.token === token) {
+                  await unlink(path);
+                  await fsyncDirectory(directory);
+                }
+              },
+            };
+          } catch (error) {
+            await unlink(candidate).catch(() => {});
+            if (error?.code !== "EEXIST") {
+              throw new WaitCheckpointError("wait_checkpoint_write_failed", `could not acquire wait fence: ${error?.message ?? error}`);
+            }
+          }
+          let current;
+          try { current = await readLock(path); }
+          catch (error) {
+            if (error?.code !== "wait_owner_unknown") throw error;
+            // The recovery mutex prevents a new generation from appearing here.
+            // A malformed/partial shared fence therefore belongs to a crashed
+            // publisher and is safe to retire before the next atomic attempt.
+            await unlink(path).catch((unlinkError) => {
+              if (unlinkError?.code !== "ENOENT") throw unlinkError;
+            });
+            await fsyncDirectory(directory);
+            continue;
+          }
+          if (!current) continue;
+          if (await lockIsLive(current)) {
+            throw new WaitCheckpointError("wait_owner_active", `saved wait ${localWaitId} already has a live owner`);
+          }
+          // The crash-reclaimable recovery mutex serializes this replacement, so
+          // no second resumer can unlink a fresh generation at the shared path.
+          await unlink(path).catch((error) => {
+            if (error?.code !== "ENOENT") throw error;
+          });
+          await fsyncDirectory(directory);
+        }
+        throw new WaitCheckpointError("wait_owner_active", `saved wait ${localWaitId} ownership changed while recovering`);
+      } finally {
+        await releaseRecovery?.();
+      }
+    },
+    async list({ includeInactive = false, offset = 0, limit = INVENTORY_MAX_ITEMS } = {}) {
+      let names = [];
+      try { names = (await readdir(directory)).filter((name) => name.endsWith(".json")).sort(); }
+      catch (error) { if (error?.code === "ENOENT") return { items: [], has_more: false, next_offset: null }; throw error; }
+      const entries = [];
+      for (const name of names) {
+        const id = name.slice(0, -5);
+        try {
+          const value = await read(id);
+          const pendingCancellation = value.status === "cancelled" && value.cloud_cancel_pending === true;
+          const pendingAcknowledgement = value.status === "acknowledged" && value.cloud_ack_pending === true;
+          if (includeInactive || pendingCancellation || pendingAcknowledgement ||
+              !["acknowledged", "cancelled"].includes(value.status)) entries.push(value);
+        } catch (error) {
+          entries.push({ local_wait_id: id, status: "invalid", error: error.code ?? "wait_checkpoint_invalid" });
+        }
+      }
+      entries.sort((a, b) => String(b.updated_at ?? "").localeCompare(String(a.updated_at ?? "")));
+      const page = entries.slice(offset, offset + Math.min(Math.max(1, limit), INVENTORY_MAX_ITEMS));
+      const next = offset + page.length;
+      return { items: page, has_more: next < entries.length, next_offset: next < entries.length ? next : null };
+    },
+  };
+}