@botbuddy/cli 1.59.9 → 1.60.0

package/src/run.mjs

@@ -13,7 +13,9 @@ import { dirname, join } from "node:path";
 import { homedir } from "node:os";
 import { spawn } from "node:child_process";
 import { fileURLToPath } from "node:url";
+import { performance } from "node:perf_hooks";
 import { callToolJson } from "./api.mjs";
+import { SERVER_URL } from "./config.mjs";
 import { parseLaneEvents, laneCaseCounts, flushLaneCases, buildLaneSummary, laneSummaryFilename } from "./test-lane-events.mjs";
 import { AGENT_KEY_RE, readAgentSessionTokenEnv, legacyAgentAliasHint } from "./agent-key.mjs";
 import { hasAgentSessionRuntime, isRejectedCachedMcpSession, resolveAgentSessionCredential, selfHealAgentSession } from "./agent-session.mjs";
@@ -23,6 +25,10 @@ import { createTelemetryOutbox } from "./telemetry-outbox.mjs";
 import { deliverExecutionEvent } from "./telemetry-delivery.mjs";
 import { attestTelemetryCredential, loadTelemetryIdentity } from "./telemetry-config.mjs";
 import { VERSION } from "./version.mjs";
+import {
+  TERMINAL_RETRY_BUDGET_MS, DELIVERY_CALL_TIMEOUT_MS, callWithContextIdentity, classifyTerminalUpdateFailure,
+  ensureRunDelivery, parkedRecord, runDelivery, terminalUpdateAccepted,
+} from "./run-delivery.mjs";
 
 export const RUN_SCHEMA_VERSION = 1;
 export const EXIT = Object.freeze({ OK: 0, AUTH: 3, INVALID: 4, BACKEND: 5, INTERNAL: 7 });
@@ -405,7 +411,7 @@ export async function launchRun(argv, {
   // every callback is attributable to the same agent. It deliberately stays
   // outside child_env, receipts, and the workload's process environment.
   const knownSession = opts.sessionId ?? managedCredential?.identity?.sessionId ?? managedCredential?.sessionId;
-  await writeJson(context, { schema_version: RUN_SCHEMA_VERSION, context_path: context, run_id: runId, run_credential: registration.data.run_credential, receipt_upload: registration.data.receipt_upload, command, cwd, timeout_seconds: opts.timeout, receipt_path: receipt, launched_at: new Date().toISOString(), ...(opts.sessionToken ? { callback_session_token: opts.sessionToken } : {}), ...(knownSession ? { callback_session_id: knownSession } : {}), ...(childEnv && Object.keys(childEnv).length ? { child_env: childEnv } : {}), ...(testRun ? { test_run: testRun } : {}) });
+  await writeJson(context, { schema_version: RUN_SCHEMA_VERSION, context_path: context, run_id: runId, run_credential: registration.data.run_credential, receipt_upload: registration.data.receipt_upload, command, cwd, timeout_seconds: opts.timeout, receipt_path: receipt, launched_at: new Date().toISOString(), server_url: SERVER_URL,...(opts.sessionToken ? { callback_session_token: opts.sessionToken } : {}), ...(knownSession ? { callback_session_id: knownSession } : {}), ...(childEnv && Object.keys(childEnv).length ? { child_env: childEnv } : {}), ...(testRun ? { test_run: testRun } : {}) });
   const [file, args] = workerInvocation(context);
   try {
     const worker = spawnImpl(file, args, { detached: true, stdio: "ignore", windowsHide: true });
@@ -426,7 +432,10 @@ export async function runWorker(contextFile, {
   call = callToolJson,
   signalEmitter = process,
   terminate = terminateProcessGroup,
+  // BOT-2600: the bounded terminal delivery (see retryTerminalUpdate).
+  sleep, now, terminalBudgetMs, startDelivery,
 } = {}) {
+  const retry = { sleep, now, budgetMs: terminalBudgetMs, startDelivery };
   const context = JSON.parse(await readFile(contextFile, "utf8"));
   const callbackCall = callWithContextIdentity(context, call);
   const ownerHeartbeat = startOwnerHeartbeat(context, callbackCall);
@@ -492,7 +501,7 @@ export async function runWorker(contextFile, {
       child = spawnImpl(context.command[0], context.command.slice(1), { cwd: context.cwd, detached: true, stdio: ["ignore", "pipe", "pipe"], windowsHide: true, ...(context.child_env ? { env: { ...process.env, ...context.child_env } } : {}) });
     } catch (error) {
       const endedAt = new Date().toISOString();
-      const receipt = await finalize(context, { status: "owner_lost", startedAt, endedAt, exitCode: null, signal: null, reason: `workload_spawn_failed:${error instanceof Error ? error.message : String(error)}` }, callbackCall);
+      const receipt = await finalize(context, { status: "owner_lost", startedAt, endedAt, exitCode: null, signal: null, reason: `workload_spawn_failed:${error instanceof Error ? error.message : String(error)}` }, callbackCall, retry);
       return receipt;
     }
   // A failed spawn can report through the child `error` event on the next turn.
@@ -523,11 +532,11 @@ export async function runWorker(contextFile, {
     if (lane) await lane.stop();
     const endedAt = new Date().toISOString();
     if (result.error) {
-      const receipt = await finalize(context, { status: "owner_lost", startedAt, endedAt, exitCode: null, signal: null, reason: `workload_spawn_failed:${result.error.message}`, output: capture.output, outputCharacters: capture.characters }, callbackCall);
+      const receipt = await finalize(context, { status: "owner_lost", startedAt, endedAt, exitCode: null, signal: null, reason: `workload_spawn_failed:${result.error.message}`, output: capture.output, outputCharacters: capture.characters }, callbackCall, retry);
       return receipt;
     }
     const status = terminalStatus({ ...result, timedOut });
-    const receipt = await finalize(context, { status, startedAt, endedAt, exitCode: result.exitCode, signal: result.signal, output: capture.output, outputCharacters: capture.characters }, callbackCall);
+    const receipt = await finalize(context, { status, startedAt, endedAt, exitCode: result.exitCode, signal: result.signal, output: capture.output, outputCharacters: capture.characters }, callbackCall, retry);
     return receipt;
   } finally {
     if (ownerHeartbeat) await ownerHeartbeat.stop();
@@ -556,7 +565,14 @@ function startOwnerHeartbeat(context, call, { intervalMs = 5 * 60_000, setInterv
   tick();
   const timer = setIntervalFn(tick, intervalMs);
   timer.unref?.();
-  return { async stop() { stopped = true; clearInterval(timer); await chain; } };
+  // BOT-2600: an in-flight heartbeat must never hold a finished worker open.
+  return { async stop() {
+    stopped = true;
+    clearInterval(timer);
+    let bound;
+    await Promise.race([chain, new Promise((resolve) => { bound = setTimeout(resolve, 2_000); bound.unref?.(); })]);
+    clearTimeout(bound);
+  } };
 }
 
 // Periodic lane flusher: reduce the NDJSON events file to the current case state
@@ -785,7 +801,7 @@ export async function cancelRun(runId, {
   return { exitCode: EXIT.OK, receipt: { schema_version: RUN_SCHEMA_VERSION, outcome: "cancellation_requested", run_id: runId, exit_code: EXIT.OK } };
 }
 
-async function finalize(context, state, call) {
+async function finalize(context, state, call, retry = {}) {
   const boundedLog = String(state.output ?? "").slice(-8192);
   const outputCharacters = state.outputCharacters ?? String(state.output ?? "").length;
   const receipt = { schema_version: RUN_SCHEMA_VERSION, run_id: context.run_id, status: state.status, started_at: state.startedAt, ended_at: state.endedAt, exit_code: state.exitCode, signal: state.signal, recovery_command: state.status === "failure" ? `botbuddy run --session-id <new-session-id> --environment local -- ${context.command.map((x) => JSON.stringify(x)).join(" ")}` : null, bounded_log: boundedLog, output_characters: outputCharacters, reason: state.reason ?? null };
@@ -795,43 +811,77 @@ async function finalize(context, state, call) {
   const apiStatus = ({ success: "succeeded", failure: "failed", cancelled: "cancelled", timed_out: "timed_out", owner_lost: "owner_lost" })[state.status];
   const update = { run_id: context.run_id, run_credential: context.run_credential, status: apiStatus, exit_code: state.exitCode ?? undefined, invalidated_reason: state.reason ?? (state.status === "timed_out" ? "timeout" : state.status === "owner_lost" ? "owner_lost" : undefined), receipt_path: context.receipt_upload, receipt_content_type: "application/json", receipt_byte_size: bytes.size, receipt_sha256: sha256, output_characters: outputCharacters, truncated: outputCharacters > MAX_CAPTURE_BYTES };
   // A locally durable receipt is enough to publish the linked test run. Do it
-  // before the command callback's intentionally unbounded retry: otherwise an
-  // auth outage (for example agent_identity_required) strands a 0-case run in
-  // active state forever even though its command has already failed locally.
+  // before the command callback's retry: otherwise an auth outage (for example
+  // agent_identity_required) strands a 0-case run in active state even though
+  // its command has already failed locally.
   if (context.test_run) await completeTestRun(context, { status: state.status, exitCode: state.exitCode, signal: state.signal, startedAt: state.startedAt, endedAt: state.endedAt, call });
-  await retryTerminalUpdate(context, update, call);
-  try { await unlink(context.context_path); } catch { /* receipt is authoritative; context cleanup is best effort */ }
+  const delivery = await retryTerminalUpdate(context, update, call, retry);
+  // A parked delivery keeps its context: it is the durable record the
+  // per-machine delivery process (and an operator) reads.
+  if (delivery.delivered) {
+    try { await unlink(context.context_path); } catch { /* receipt is authoritative; context cleanup is best effort */ }
+  }
   return receipt;
 }
 
-function callWithContextIdentity(context, call) {
-  const token = typeof context.callback_session_token === "string" && SESSION_TOKEN_RE.test(context.callback_session_token)
-    ? context.callback_session_token
-    : null;
-  // Old contexts cannot safely fall back to the machine-level owner key. Ask
-  // the API layer to require a session instead; the retained receipt/context
-  // makes recovery possible without attributing a callback to the wrong agent.
-  return (name, args) => call(name, args, token
-    ? { auth: { "x-agent-api-key": token } }
-    : { requireSessionToken: true });
-}
-
-export async function retryTerminalUpdate(context, update, call, { sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)) } = {}) {
+// BOT-2600: the terminal callback is bounded. The worker that owns the receipt
+// retries transient failures (transport, timeout, 5xx) within a monotonic
+// budget of at most ten minutes and parks a permanent rejection (no owning
+// identity, an invalid run credential, a local environment error) at once.
+// Either way it records `terminal_update.parked` in the private context and
+// returns so the worker process exits; a retryable parked delivery is handed to
+// the single per-machine delivery process (`bb run --deliver`).
+export async function retryTerminalUpdate(context, update, call, {
+  sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
+  now = () => performance.now(),
+  budgetMs = TERMINAL_RETRY_BUDGET_MS,
+  startDelivery = () => ensureRunDelivery({ force: true }),
+} = {}) {
+  const started = now();
   let attempts = 0;
   let delay = 1_000;
   for (;;) {
-    let result;
-    try { result = await call("update_command_run", update); } catch (error) { result = { ok: false, error: error instanceof Error ? error.message : String(error) }; }
-    if (result?.ok && !result.isError && result.data?.error == null && result.data?.accepted !== false) return result;
+    let result = null;
+    let failure;
+    try {
+      result = await call("update_command_run", update, {
+        signal: AbortSignal.timeout(DELIVERY_CALL_TIMEOUT_MS),
+        // The endpoint this run was registered with (BOT-2600; sealed at launch).
+        ...(typeof context.server_url === "string" && context.server_url ? { serverUrl: context.server_url } : {}),
+      });
+      if (terminalUpdateAccepted(result)) return { delivered: true, attempts: attempts + 1, result };
+      failure = classifyTerminalUpdateFailure(result);
+    } catch (error) {
+      failure = classifyTerminalUpdateFailure(null, error);
+    }
     attempts += 1;
-    await writeJson(context.context_path, { ...context, terminal_update: { ...update, attempts, last_error: result?.error ?? result?.data?.error ?? "terminal_update_rejected", next_retry_at: new Date(Date.now() + delay).toISOString() } });
+    const exhausted = failure.retryable && !failure.parkNow && now() - started + delay > budgetMs;
+    if (!failure.retryable || failure.parkNow || exhausted) {
+      const parked = parkedRecord(exhausted ? { ...failure, kind: "retry_budget_exhausted" } : failure, "worker");
+      await writeJson(context.context_path, { ...context, terminal_update: { ...update, attempts, last_error: failure.reason, retry_budget_ms: budgetMs, parked } });
+      if (parked.retryable) { try { await startDelivery(); } catch { /* the next bb invocation starts it */ } }
+      return { delivered: false, attempts, parked, result };
+    }
+    await writeJson(context.context_path, { ...context, terminal_update: { ...update, attempts, last_error: failure.reason, retry_budget_ms: budgetMs, next_retry_at: new Date(Date.now() + delay).toISOString() } });
     await sleep(delay);
     delay = Math.min(delay * 2, 60_000);
   }
 }
 
 export async function cmdRun(args) {
-  if (args[0] === "--worker") { await runWorker(args[1]); return; }
+  if (args[0] === "--worker") {
+    await runWorker(args[1]);
+    // BOT-2600: the receipt is durable and its delivery is accepted or parked.
+    // Nothing may keep a finished worker resident; an unref'd guard exits even
+    // if a stray handle (a keep-alive socket) would otherwise hold the loop.
+    setTimeout(() => process.exit(process.exitCode ?? 0), 1_000).unref();
+    return;
+  }
+  if (args[0] === "--deliver") {
+    const result = await runDelivery({ call: callToolJson });
+    if (args.includes("--json")) oneLine({ schema_version: RUN_SCHEMA_VERSION, ...result }, false);
+    return;
+  }
   if (args[0] === "cancel") {
     const result = await cancelRun(args[1] ?? "");
     oneLine(result.receipt, false);