@botbuddy/cli 1.59.9 → 1.60.0

package/src/run-delivery.mjs

@@ -0,0 +1,496 @@
+// BOT-2600 — bounded terminal delivery for durable `bb run` commands.
+//
+// A detached `bb run` worker owns its workload and writes exactly one receipt.
+// Once that receipt is durable, publishing it (update_command_run) is only a
+// delivery problem, and it must not keep a resident Node process per run alive:
+// on 2026-10-20 one Mac had 142 finished workers (6 GB) retrying rejections
+// that could never succeed. So:
+//
+//   - the worker retries transient failures within a bounded monotonic budget
+//     and parks permanent rejections immediately, recording
+//     `terminal_update.parked` in its private context, then exits;
+//   - a single per-machine delivery process (`bb run --deliver`, guarded by a
+//     PID lock under ~/.botbuddy/) retries what is still retryable and exits
+//     when nothing retryable remains. Any `bb` invocation starts it on demand;
+//   - workers left by older CLI versions (which retry forever) are adopted:
+//     their delivery is taken over and the worker is stopped only when it is
+//     provably finished (exact command line, same user, terminal receipt, no
+//     child processes, already in its terminal retry loop).
+//
+// The receipt and context files stay authoritative; nothing is deleted except a
+// context whose delivery was accepted (exactly what the worker always did).
+// The identity model is unchanged (BOT-2052): a delivery is only ever sent with
+// the session token sealed for the run's owner, never an ambient credential.
+
+import { createHash, randomUUID } from "node:crypto";
+import { chmod, link, mkdir, readdir, readFile, rename, stat, unlink, writeFile } from "node:fs/promises";
+import { execFile, spawn } from "node:child_process";
+import { dirname, join } from "node:path";
+import { homedir } from "node:os";
+import { fileURLToPath } from "node:url";
+import { performance } from "node:perf_hooks";
+import { AGENT_KEY_RE } from "./agent-key.mjs";
+import { DEFAULT_SERVER_URL } from "./config.mjs";
+
+/** The worker's own retry budget for transient failures (AC-1: ≤ 10 minutes). */
+export const TERMINAL_RETRY_BUDGET_MS = 10 * 60_000;
+/** One update_command_run attempt never waits longer than this. */
+export const DELIVERY_CALL_TIMEOUT_MS = 30_000;
+/** Transient attempts the delivery process makes before parking for good. */
+export const DELIVERY_MAX_ATTEMPTS = 50;
+/** The delivery process exits after this long even with retryable work left. */
+export const DELIVERY_LIFETIME_MS = 30 * 60_000;
+/** A non-forced `bb` invocation re-scans at most this often. */
+export const DELIVERY_RESCAN_COOLDOWN_MS = 60_000;
+const DELIVERY_FIRST_BACKOFF_MS = 30_000;
+const DELIVERY_MAX_BACKOFF_MS = 5 * 60_000;
+const STOP_GRACE_MS = 3_000;
+
+export const TERMINAL_RECEIPT_STATUSES = new Set(["success", "failure", "cancelled", "timed_out", "owner_lost"]);
+const API_STATUS = { success: "succeeded", failure: "failed", cancelled: "cancelled", timed_out: "timed_out", owner_lost: "owner_lost" };
+const MAX_CAPTURE_BYTES = 8_192;
+// The update_command_run arguments a persisted terminal_update may carry.
+const UPDATE_FIELDS = ["run_id", "run_credential", "status", "exit_code", "invalidated_reason", "receipt_path", "receipt_content_type", "receipt_byte_size", "receipt_sha256", "output_characters", "truncated"];
+
+// Never retried: decided locally without a request reaching the server (no
+// usable session token), or a definitive server state (a different terminal
+// status is already recorded; the payload itself is refused).
+const PERMANENT_ERRORS = new Set([
+  "session_token_required", "session_token_malformed", "not_authenticated", "token_expired",
+  "command_terminal_immutable", "artifact_required",
+]);
+// Trust boundary for server rejections. The server can answer with an auth or
+// credential rejection when its own lookup failed transiently (it discards the
+// read error: BOT-2611), so none of these is proof that a retry cannot
+// succeed. The worker parks every one at once and exits (it never spends its
+// retry budget on them), and the delivery process gives each a few spaced
+// attempts before parking it for good, so a recovered server recovers the
+// terminal result while a dead credential costs at most a handful of calls.
+const AMBIGUOUS_ERRORS = new Set([
+  "agent_identity_required", "authentication_required", "unauthorized", "run_credential_invalid",
+  "session_token_revoked", "credential_unknown",
+]);
+/** Delivery-process attempts for an ambiguous rejection before parking for good. */
+export const DELIVERY_AMBIGUOUS_MAX_ATTEMPTS = 5;
+const LOCAL_ENVIRONMENT_RE =/uv_os_get_passwd|\bENOENT\b|\bEACCES\b|\bEPERM\b/;
+
+export function commandRunsDir(home = homedir()) { return join(home, ".botbuddy", "command-runs"); }
+export function deliveryLockPath(home = homedir()) { return join(home, ".botbuddy", "run-delivery.lock"); }
+export function deliveryStatePath(home = homedir()) { return join(home, ".botbuddy", "run-delivery.json"); }
+
+function errorText(result) {
+  const value = result?.data?.error ?? result?.error ?? result?.code ?? null;
+  return typeof value === "string" && value ? value : "terminal_update_rejected";
+}
+
+export function terminalUpdateAccepted(result) {
+  return Boolean(result?.ok && !result.isError && result.data?.error == null && result.data?.accepted !== false);
+}
+
+/**
+ * Classify a failed update_command_run attempt.
+ * `parkNow` marks a retryable failure the worker must not spend its own
+ * budget on: it parks at once and only the delivery process retries it.
+ * @returns {{ reason: string, retryable: boolean, kind: string, parkNow?: boolean }}
+ */
+export function classifyTerminalUpdateFailure(result, thrown = null) {
+  if (thrown) {
+    const message = thrown instanceof Error ? thrown.message : String(thrown);
+    if (thrown?.code === "ERR_SYSTEM_ERROR" || LOCAL_ENVIRONMENT_RE.test(message)) {
+      return { reason: message, retryable: false, kind: "local_environment" };
+    }
+    return { reason: message, retryable: true, kind: "transient" };
+  }
+  const reason = errorText(result);
+  const codes = [result?.data?.error, result?.error, result?.code, result?.error_code].filter((v) => typeof v === "string");
+  if (codes.some((code) => PERMANENT_ERRORS.has(code))) return { reason, retryable: false, kind: "permanent_rejection" };
+  if (result?.auth === true || codes.some((code) => AMBIGUOUS_ERRORS.has(code))) return { reason, retryable: true, kind: "ambiguous_rejection", parkNow: true };
+  if (codes.some((code) => code.startsWith("usage:"))) return { reason, retryable: false, kind: "local_environment" };
+  // Transport failures, timeouts, 5xx/429 and anything unrecognised stay
+  // retryable; the budget (worker) or attempt cap (delivery process) bounds them.
+  return { reason, retryable: true, kind: "transient" };
+}
+
+/** The sealed owner's identity, never an ambient or machine-level credential. */
+export function callWithContextIdentity(context, call) {
+  const token = sealedToken(context);
+  // Old contexts cannot safely fall back to the machine-level owner key. Ask
+  // the API layer to require a session instead; the retained receipt/context
+  // makes recovery possible without attributing a callback to the wrong agent.
+  return (name, args, extra = {}) => call(name, args, token
+    ? { ...extra, auth: { "x-agent-api-key": token } }
+    : { ...extra, requireSessionToken: true });
+}
+
+function sealedToken(context) {
+  return typeof context?.callback_session_token === "string" && AGENT_KEY_RE.test(context.callback_session_token)
+    ? context.callback_session_token : null;
+}
+
+export async function writeJsonAtomic(path, value) {
+  await mkdir(dirname(path), { recursive: true });
+  // Write-then-rename so a concurrent reader never observes a torn file.
+  const tmp = `${path}.${process.pid}.${randomUUID()}.tmp`;
+  await writeFile(tmp, JSON.stringify(value), { mode: 0o600 });
+  await chmod(tmp, 0o600);
+  await rename(tmp, path);
+}
+
+async function readJson(path) {
+  try { return JSON.parse(await readFile(path, "utf8")); } catch (error) {
+    if (error?.code === "ENOENT") return null;
+    if (error instanceof SyntaxError) return undefined;
+    throw error;
+  }
+}
+
+function withTimeout(extra = {}) {
+  return { ...extra, signal: AbortSignal.timeout(DELIVERY_CALL_TIMEOUT_MS) };
+}
+
+/**
+ * The endpoint a run was registered with. A sealed token is only ever sent
+ * back there, whatever endpoint the delivery process itself was started with.
+ * Contexts written before BOT-2600 did not record it; the CLI's default
+ * (production) endpoint is the only one a context without an override could
+ * have been registered with by default, so it is used for those.
+ */
+export function contextServerUrl(context) {
+  return typeof context?.server_url === "string" && /^https?:\/\//.test(context.server_url) ? context.server_url : DEFAULT_SERVER_URL;
+}
+
+/** One update_command_run attempt with the sealed identity and a bounded wait. */
+export async function attemptTerminalUpdate(context, update, call) {
+  const identified = callWithContextIdentity(context, call);
+  try {
+    const result = await identified("update_command_run", update, withTimeout({ serverUrl: contextServerUrl(context) }));
+    if (terminalUpdateAccepted(result)) return { accepted: true, result };
+    return { accepted: false, result, failure: classifyTerminalUpdateFailure(result) };
+  } catch (error) {
+    return { accepted: false, result: null, failure: classifyTerminalUpdateFailure(null, error) };
+  }
+}
+
+export function parkedRecord(failure, by) {
+  return { reason: failure.reason, retryable: failure.retryable, kind: failure.kind, by, parked_at: new Date().toISOString() };
+}
+
+/** Rebuild the terminal update from a durable receipt (a worker that crashed first). */
+export async function terminalUpdateFromReceipt(context, receipt) {
+  const bytes = await readFile(context.receipt_path);
+  const outputCharacters = Number.isInteger(receipt.output_characters) ? receipt.output_characters : 0;
+  const reason = receipt.reason ?? (receipt.status === "timed_out" ? "timeout" : receipt.status === "owner_lost" ? "owner_lost" : undefined);
+  return {
+    run_id: context.run_id, run_credential: context.run_credential, status: API_STATUS[receipt.status],
+    exit_code: receipt.exit_code ?? undefined, invalidated_reason: reason ?? undefined,
+    receipt_path: context.receipt_upload, receipt_content_type: "application/json",
+    receipt_byte_size: bytes.length, receipt_sha256: createHash("sha256").update(bytes).digest("hex"),
+    output_characters: outputCharacters, truncated: outputCharacters > MAX_CAPTURE_BYTES,
+  };
+}
+
+function updateFromPersisted(terminalUpdate) {
+  const update = {};
+  for (const key of UPDATE_FIELDS) if (terminalUpdate[key] !== undefined) update[key] = terminalUpdate[key];
+  return update;
+}
+
+// ── processes ────────────────────────────────────────────────────────────────
+
+export function parseProcessList(text) {
+  const rows = [];
+  for (const line of String(text).split("\n")) {
+    const match = /^\s*(\d+)\s+(\d+)\s+(\d+)\s(.*)$/.exec(line);
+    if (match) rows.push({ pid: Number(match[1]), ppid: Number(match[2]), uid: Number(match[3]), args: match[4].trim() });
+  }
+  return rows;
+}
+
+export function listProcessesWithPs() {
+  return new Promise((resolve, reject) => {
+    execFile("ps", ["-ww", "-A", "-o", "pid=,ppid=,uid=,args="], { maxBuffer: 32 * 1024 * 1024 }, (error, stdout) => {
+      if (error) reject(error); else resolve(parseProcessList(stdout));
+    });
+  });
+}
+
+export function pidAlive(pid) {
+  if (!Number.isInteger(pid) || pid <= 0) return false;
+  try { process.kill(pid, 0); return true; } catch (error) { return error?.code === "EPERM"; }
+}
+
+// Every process whose argument string names this context as its worker.
+function workersFor(processes, contextFile) {
+  const suffix = ` run --worker ${contextFile}`;
+  return processes.filter((proc) => proc.args.endsWith(suffix) || proc.args.includes(`${suffix} `));
+}
+
+// AC-4: exactly `<node> <…>botbuddy.mjs run --worker <this context>`, owned by
+// this user, and with no child process. The receipt/terminal checks are the
+// caller's, made on the same context read.
+export function isStoppableLegacyWorker(proc, { contextFile, uid, processes }) {
+  const suffix = ` run --worker ${contextFile}`;
+  if (!proc.args.endsWith(suffix)) return false;
+  if (!/^\S+\s+\S*botbuddy\.mjs$/.test(proc.args.slice(0, -suffix.length))) return false;
+  if (proc.uid !== uid) return false;
+  return !processes.some((other) => other.ppid === proc.pid);
+}
+
+async function stopLegacyWorker(pid, { contextFile, uid, listProcesses, killProcess, isAlive, sleep }) {
+  // Re-verify on a fresh listing right before signalling (PID reuse, a new child).
+  const fresh = await listProcesses();
+  const proc = fresh.find((candidate) => candidate.pid === pid);
+  if (!proc || !isStoppableLegacyWorker(proc, { contextFile, uid, processes: fresh })) return !proc;
+  killProcess(pid, "SIGTERM");
+  for (let waited = 0; waited < STOP_GRACE_MS; waited += 100) {
+    if (!isAlive(pid)) return true;
+    await sleep(100);
+  }
+  return !isAlive(pid);
+}
+
+// ── one drain pass ────────────────────────────────────────────────────────────
+
+/**
+ * Deliver every pending terminal update once.
+ * @returns {Promise<{ delivered: number, parked: number, pending: number, stopped: number, owned: number, wrote: boolean }>}
+ */
+export async function drainOnce({
+  home = homedir(), call, listProcesses = listProcessesWithPs, killProcess = (pid, signal) => process.kill(pid, signal),
+  isAlive = pidAlive, uid = process.getuid?.() ?? -1, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
+  listContexts = async () => (await readdir(commandRunsDir(home)).catch((error) => { if (error?.code === "ENOENT") return []; throw error; }))
+    .filter((name) => name.endsWith(".context.json")).sort(),
+  maxAttempts = DELIVERY_MAX_ATTEMPTS,
+} = {}) {
+  const totals = { delivered: 0, parked: 0, pending: 0, stopped: 0, owned: 0, wrote: false };
+  const names = await listContexts();
+  if (!names.length) return totals;
+  let processes = null;
+  for (const name of names) {
+    const contextFile = join(commandRunsDir(home), name);
+    const context = await readJson(contextFile);
+    // Deleted by a successful delivery mid-scan, or unreadable: not ours.
+    if (!context || typeof context.run_id !== "string" || typeof context.receipt_path !== "string") continue;
+    // A pre-workload cancellation is owned by the cancel handoff path. A
+    // marker written by a cancel that raced workload start leaves the worker
+    // owning a normal terminal update, which stays eligible here.
+    if (!context.terminal_update && await readFile(`${contextFile}.cancelled`).then(() => true, () => false)) continue;
+    const receipt = await readJson(context.receipt_path);
+    // Still executing (no terminal receipt yet): never touched.
+    if (!receipt || receipt.run_id !== context.run_id || !TERMINAL_RECEIPT_STATUSES.has(receipt.status)) continue;
+    const persisted = context.terminal_update ?? null;
+    if (persisted?.parked && persisted.parked.retryable === false) continue;
+    processes ??= await listProcesses();
+    const workers = workersFor(processes, contextFile).filter((proc) => isAlive(proc.pid));
+    if (workers.length && !persisted?.parked) {
+      // A bounded (current) worker owns its delivery until it parks it. A
+      // legacy worker is adopted only once it is provably in its terminal
+      // retry loop and stopping it cannot touch running work.
+      const legacyLoop = persisted && persisted.retry_budget_ms === undefined;
+      const stoppable = legacyLoop && workers.every((proc) => isStoppableLegacyWorker(proc, { contextFile, uid, processes }));
+      if (!stoppable) { totals.owned += 1; continue; }
+      let allStopped = true;
+      for (const proc of workers) {
+        if (await stopLegacyWorker(proc.pid, { contextFile, uid, listProcesses, killProcess, isAlive, sleep })) totals.stopped += 1;
+        else allStopped = false;
+      }
+      processes = null;
+      if (!allStopped) { totals.owned += 1; continue; }
+    }
+    // Re-read: a stopped legacy worker may have rewritten the context last.
+    const current = await readJson(contextFile);
+    if (!current) continue;
+    const update = current.terminal_update ? updateFromPersisted(current.terminal_update) : await terminalUpdateFromReceipt(current, receipt);
+    const previous = current.terminal_update ?? {};
+    const attempts = (Number.isInteger(previous.attempts) ? previous.attempts : 0) + 1;
+    const deliveryAttempts = (Number.isInteger(previous.delivery_attempts) ? previous.delivery_attempts : 0) + 1;
+    if (!sealedToken(current)) {
+      // No sealed owner: a delivery from this process could only use an
+      // ambient identity, which BOT-2052 forbids. Park it visibly instead.
+      const failure = { reason: "callback_session_token_missing", retryable: false, kind: "callback_session_token_missing" };
+      await writeJsonAtomic(contextFile, { ...current, terminal_update: { ...previous, ...update, parked: parkedRecord(failure, "delivery") } });
+      totals.parked += 1; totals.wrote = true;
+      continue;
+    }
+    const outcome = await attemptTerminalUpdate(current, update, call);
+    if (outcome.accepted) {
+      try { await unlink(contextFile); } catch { /* already removed */ }
+      totals.delivered += 1; totals.wrote = true;
+      continue;
+    }
+    let failure = outcome.failure;
+    const cap = failure.kind === "ambiguous_rejection" ? Math.min(maxAttempts, DELIVERY_AMBIGUOUS_MAX_ATTEMPTS) : maxAttempts;
+    if (failure.retryable && deliveryAttempts >= cap) failure = { ...failure, retryable: false, kind: "delivery_attempts_exhausted" };
+    await writeJsonAtomic(contextFile, { ...current, terminal_update: {
+      ...previous, ...update, attempts, delivery_attempts: deliveryAttempts, last_error: failure.reason,
+      parked: failure.retryable ? { ...(previous.parked ?? parkedRecord(failure, "delivery")), reason: failure.reason, retryable: true } : parkedRecord(failure, "delivery"),
+    } });
+    totals.wrote = true;
+    if (failure.retryable) totals.pending += 1; else totals.parked += 1;
+  }
+  return totals;
+}
+
+// ── the per-machine lock ──────────────────────────────────────────────────────
+
+/**
+ * Take the per-machine delivery lock. A lock whose holder PID is gone is stale
+ * and is taken over.
+ */
+export async function acquireDeliveryLock({ home = homedir(), pid = process.pid, isAlive = pidAlive } = {}) {
+  const lockFile = deliveryLockPath(home);
+  await mkdir(dirname(lockFile), { recursive: true });
+  const candidate = `${lockFile}.${pid}.${randomUUID()}.tmp`;
+  await writeFile(candidate, JSON.stringify({ pid, started_at: new Date().toISOString() }), { mode: 0o600 });
+  try {
+    for (let round = 0; round < 3; round++) {
+      try {
+        await link(candidate, lockFile); // atomic create-if-absent
+        return { acquired: true, holderPid: pid, release: async () => {
+          const holder = await readJson(lockFile);
+          if (holder?.pid === pid) await unlink(lockFile).catch(() => {});
+        } };
+      } catch (error) {
+        if (error?.code !== "EEXIST") throw error;
+      }
+      const holder = await readJson(lockFile);
+      if (holder === null) continue; // released between link and read
+      const holderPid = holder?.pid;
+      if (Number.isInteger(holderPid) && isAlive(holderPid)) return { acquired: false, holderPid, release: async () => {} };
+      // Stale. Only the holder of the reclaim mutex may remove the lock file,
+      // so it can never move aside a lock another contender just acquired:
+      // while the lock file exists, nobody else can replace it (link fails),
+      // and no other reclaimer can run, so what we re-read is what we remove.
+      const reclaim = await acquireReclaimMutex(lockFile, candidate, isAlive);
+      if (!reclaim) return { acquired: false, holderPid: null, release: async () => {} };
+      try {
+        const current = await readJson(lockFile);
+        if (current && !(Number.isInteger(current.pid) && !isAlive(current.pid))) {
+          return { acquired: false, holderPid: current?.pid ?? null, release: async () => {} };
+        }
+        if (current) await unlink(lockFile).catch((error) => { if (error?.code !== "ENOENT") throw error; });
+      } finally {
+        await reclaim.release();
+      }
+    }
+    const holder = await readJson(lockFile);
+    return { acquired: false, holderPid: holder?.pid ?? null, release: async () => {} };
+  } finally {
+    await unlink(candidate).catch(() => {});
+  }
+}
+
+// The reclaim mutex serialises stale-lock removal. Its critical section is a
+// read and an unlink, so a reclaim file left by a crashed contender is taken
+// over only once its holder is dead AND it is older than RECLAIM_STALE_MS.
+// Trust boundary: two contenders racing to clear a crashed reclaimer's mutex
+// in the same instant is not defended further (it needs a crash inside a
+// sub-millisecond section plus concurrent contention).
+const RECLAIM_STALE_MS = 10_000;
+async function acquireReclaimMutex(lockFile, candidate, isAlive) {
+  const mutex = `${lockFile}.reclaim`;
+  for (let attempt = 0; attempt < 2; attempt++) {
+    try {
+      await link(candidate, mutex);
+      return { release: () => unlink(mutex).catch(() => {}) };
+    } catch (error) {
+      if (error?.code !== "EEXIST") throw error;
+    }
+    const holder = await readJson(mutex);
+    if (holder === null) continue;
+    let ageMs = 0;
+    try { ageMs = Date.now() - (await stat(mutex)).mtimeMs; } catch { continue; }
+    const dead = !(Number.isInteger(holder?.pid) && isAlive(holder.pid));
+    if (!dead || ageMs < RECLAIM_STALE_MS) return null; // another contender is reclaiming
+    await unlink(mutex).catch(() => {});
+  }
+  return null;
+}
+
+async function dirMtimeMs(home) {
+  try { return (await stat(commandRunsDir(home))).mtimeMs; } catch (error) {
+    if (error?.code === "ENOENT") return null;
+    throw error;
+  }
+}
+
+/**
+ * The delivery process: hold the lock, drain until nothing retryable remains
+ * (or the lifetime cap), record the settled directory state, release, exit.
+ */
+export async function runDelivery({
+  home = homedir(), call, sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)), now = () => performance.now(),
+  lifetimeMs = DELIVERY_LIFETIME_MS, ...drainOptions
+} = {}) {
+  const lock = await acquireDeliveryLock({ home });
+  if (!lock.acquired) return { outcome: "held", holder_pid: lock.holderPid, delivered: 0, parked: 0, pending: 0, stopped: 0 };
+  const totals = { delivered: 0, parked: 0, pending: 0, stopped: 0, owned: 0 };
+  const started = now();
+  let backoff = DELIVERY_FIRST_BACKOFF_MS;
+  let settlePasses = 0;
+  try {
+    for (;;) {
+      const mtime = await dirMtimeMs(home);
+      const pass = await drainOnce({ home, call, sleep, ...drainOptions });
+      for (const key of ["delivered", "parked", "stopped"]) totals[key] += pass[key];
+      totals.pending = pass.pending;
+      totals.owned = pass.owned;
+      if (pass.pending === 0) {
+        // Our own writes move the directory mtime; one more pass confirms the
+        // directory is settled so `bb` invocations stop re-starting us.
+        if (pass.wrote && settlePasses++ < 3) continue;
+        // A delivery still owned by a live worker is not settled: if that
+        // worker dies before it writes again, the directory never changes, so
+        // leave the state open for a re-scan after the cooldown.
+        await writeJsonAtomic(deliveryStatePath(home), { dir_mtime_ms: pass.owned > 0 ? null : mtime, settled_at: new Date().toISOString(), owned: pass.owned });
+        return { outcome: "drained", ...totals };
+      }
+      settlePasses = 0;
+      if (now() - started + backoff > lifetimeMs) {
+        // Retryable work remains: leave the state unsettled so the next `bb`
+        // invocation starts a fresh delivery process.
+        await writeJsonAtomic(deliveryStatePath(home), { dir_mtime_ms: null, settled_at: null, pending: pass.pending });
+        return { outcome: "lifetime_exhausted", ...totals };
+      }
+      await sleep(backoff);
+      backoff = Math.min(backoff * 2, DELIVERY_MAX_BACKOFF_MS);
+    }
+  } finally {
+    await lock.release();
+  }
+}
+
+/**
+ * Called by every `bb` invocation (and by a worker that parked a retryable
+ * delivery): start the delivery process when the run directory changed since
+ * it last settled and no live process holds the lock. Cheap: a stat and two
+ * small reads. Never throws into the caller's command.
+ */
+export async function ensureRunDelivery({
+  home = homedir(), env = process.env, spawnImpl = spawn, isAlive = pidAlive, force = false, cooldownMs = DELIVERY_RESCAN_COOLDOWN_MS,
+} = {}) {
+  try {
+    if (env.BOTBUDDY_RUN_DELIVERY === "off") return { started: false, reason: "disabled" };
+    const mtime = await dirMtimeMs(home);
+    if (mtime === null) return { started: false, reason: "no_runs" };
+    const state = await readJson(deliveryStatePath(home));
+    if (!force && state && state.dir_mtime_ms === mtime) return { started: false, reason: "up_to_date" };
+    // A worker still inside its own retry budget rewrites its context on every
+    // attempt. Re-scanning on each of those changes is wasted work: the worker
+    // starts us itself (force) when it parks, so otherwise scan at most once
+    // per cooldown. Both times are this machine's filesystem clock.
+    if (!force && state) {
+      const stateAge = Date.now() - (await stat(deliveryStatePath(home))).mtimeMs;
+      if (stateAge >= 0 && stateAge < cooldownMs) return { started: false, reason: "cooldown" };
+    }
+    const holder = await readJson(deliveryLockPath(home));
+    if (Number.isInteger(holder?.pid) && isAlive(holder.pid)) return { started: false, reason: "held" };
+    const bin = join(dirname(fileURLToPath(import.meta.url)), "..", "bin", "botbuddy.mjs");
+    const child = spawnImpl(process.execPath, [bin, "run", "--deliver"], { detached: true, stdio: "ignore", cwd: home, windowsHide: true });
+    child.on?.("error", () => { /* best effort: the next invocation retries */ });
+    child.unref?.();
+    return { started: true, reason: "pending" };
+  } catch (error) {
+    return { started: false, reason: `error:${error instanceof Error ? error.message : String(error)}` };
+  }
+}