@agent-native/core 0.112.0 → 0.114.0

package/src/agent/run-manager.ts

@@ -0,0 +1,1825 @@
+import { captureError } from "../server/capture-error.js";
+import {
+  isLlmCredentialError,
+  LLM_MISSING_CREDENTIALS_ERROR_CODE,
+  LLM_MISSING_CREDENTIALS_MESSAGE,
+} from "./engine/credential-errors.js";
+import { EngineError } from "./engine/types.js";
+import {
+  insertRun,
+  insertRunEvent,
+  updateRunStatusIfRunning,
+  markRunAborted,
+  getRunAbortState,
+  getRunStatus,
+  getRunEventsSince,
+  getRunById,
+  getRunByThread,
+  cleanupOldRuns,
+  updateRunHeartbeat,
+  bumpRunProgress,
+  setRunInFlightMarker,
+  reapIfStale,
+  reapUnclaimedBackgroundRun,
+  shouldRedispatchUnclaimedBackgroundRun,
+  reconcileTerminalRunFromEvents,
+  ensureTerminalRunEvent,
+  getLastTerminalRunEvent,
+  resolveErroredRunTerminalEvent,
+  setRunError,
+  setRunTerminalReason,
+} from "./run-store.js";
+import type { AgentChatEvent, RunEvent, RunStatus } from "./types.js";
+
+export interface ActiveRun {
+  runId: string;
+  threadId: string;
+  /** Logical-turn identity (see StartRunOptions.turnId). Defaults to runId. */
+  turnId: string;
+  events: RunEvent[];
+  status: RunStatus;
+  subscribers: Set<(event: RunEvent) => void>;
+  abort: AbortController;
+  abortReason?: string;
+  startedAt: number;
+}
+
+const activeRuns = new Map<string, ActiveRun>();
+const threadToRun = new Map<string, string>();
+
+/** How long to keep completed runs in memory before cleanup (5 min) */
+const CLEANUP_DELAY_MS = 5 * 60 * 1000;
+
+/**
+ * Default run chunk budget for hosted/serverless deploys.
+ *
+ * This MUST fire before the two upstream hard walls that otherwise kill a run
+ * mid-turn with no chance to hand off:
+ *   1. The Builder model gateway keeps a 45s cap only for hosted foreground
+ *      runs; local and proven background-function runs use longer caps.
+ *   2. Serverless functions are hard-killed around 60-65s (the heartbeat then
+ *      reaps the row as a stale_run).
+ * Production data showed every cutoff landing in the 44-70s window with ZERO
+ * auto_continue events ever emitted — i.e. the old 45s default raced the 45s
+ * gateway and lost, and per-template overrides (e.g. 240_000) pushed it past
+ * BOTH walls so it could never fire. 40s leaves ~5s of headroom under the
+ * gateway wall to abort, persist the partial turn, write the terminal event,
+ * and emit a clean auto_continue so the client resumes seamlessly.
+ */
+export const DEFAULT_HOSTED_RUN_SOFT_TIMEOUT_MS = 40_000;
+
+/**
+ * Hard ceiling for the hosted soft timeout. On a hosted runtime the
+ * foreground auto_continue soft timeout can never usefully exceed this — the
+ * synchronous function (~60s) wall kills the run first, so a larger configured
+ * or env value just guarantees the cutoff is a hard error instead of a graceful
+ * hand-off. Any resolved value above this is clamped down for hosted foreground
+ * runs. Local dev (non-hosted) is left alone so long-running local turns aren't
+ * chunked.
+ *
+ * IMPORTANT: this clamp is for the INTERACTIVE / foreground path and must NOT
+ * be raised. The foreground POST still rides a synchronous serverless function
+ * (~60-65s wall), so 40s remains correct there. The only sanctioned exception
+ * is the opt-in `backgroundFunction` mode (see
+ * `BACKGROUND_SOFT_TIMEOUT_CEILING_MS`), which runs inside a Netlify background
+ * function (no ~60s wall, 15-min budget) and therefore can safely outlast 40s.
+ */
+export const HOSTED_SOFT_TIMEOUT_CEILING_MS = 40_000;
+
+/**
+ * Hard ceiling for the soft timeout when a run executes inside a Netlify
+ * background function (any deployed function whose name ends in `-background`).
+ * Background functions return 202 immediately and run detached for up to 15
+ * minutes, so the ~60s synchronous function wall that 40s defends against does
+ * NOT apply. 13 minutes leaves ~2 min of headroom under Netlify's 15-min hard
+ * kill to abort, persist the partial turn, write the terminal event, and (for
+ * the rare >13-min turn) self-fire another background continuation.
+ *
+ * This ceiling is used ONLY when a caller explicitly opts in with
+ * `backgroundFunction: true`. It does not change the foreground/interactive
+ * ceiling and does not fire unless the durable-background path dispatched the
+ * run into a background function. Per the design doc Guardrail, the 40s
+ * interactive clamp stays correct for every non-background run.
+ */
+export const BACKGROUND_SOFT_TIMEOUT_CEILING_MS = 13 * 60_000; // 780_000
+
+/**
+ * Default soft-timeout budget for a background-function run when the caller
+ * does not pass an explicit `softTimeoutMs`. Same value as the ceiling — we
+ * want a background turn to use nearly its whole 15-min budget before handing
+ * off to a chained background continuation.
+ */
+export const DEFAULT_BACKGROUND_RUN_SOFT_TIMEOUT_MS =
+  BACKGROUND_SOFT_TIMEOUT_CEILING_MS;
+
+/**
+ * Default no-progress window for a run executing inside a proven durable
+ * background function. Keep this below the 13-minute soft timeout so a truly
+ * wedged background turn can still checkpoint, persist, and continue before
+ * the function budget expires, but far above the foreground 150s window so
+ * large Design/Plan/Assets generations are not chopped up while the model is
+ * legitimately planning a big tool payload.
+ */
+export const DEFAULT_BACKGROUND_NO_PROGRESS_TIMEOUT_MS =
+  BACKGROUND_SOFT_TIMEOUT_CEILING_MS - 60_000;
+
+/**
+ * AUTHORITATIVE no-progress backstop for a run, enforced by the run manager
+ * itself (timer-driven, independent of any layer below).
+ *
+ * The finer-grained watchdogs inside the agent loop (model-stream and
+ * action-preparation no-progress, both 90s) only guard the model event stream
+ * — a stall in any segment OUTSIDE that guarded loop (engine-call
+ * establishment, worker setup between continuation chunks, a wedged transport
+ * that emits keepalives while the loop never runs) previously hung forever
+ * with the client watching keepalives. This backstop covers every segment by
+ * construction: if no REAL progress event (see `shouldBumpProgressForEvent`;
+ * keepalives and zero-byte prep activity don't count) lands for this long —
+ * and no tool call is in flight (tool execution legitimately emits nothing
+ * for minutes and has its own 12-min timeout) — the run manager emits
+ * `auto_continue { reason: "no_progress" }` and aborts the chunk, exactly
+ * like the soft timeout, so the normal continuation machinery recovers it.
+ *
+ * Sits above the 90s in-loop watchdogs (they get first chance to recover with
+ * better context). Foreground hosted chunks keep this short so the user sees
+ * recovery promptly; proven durable-background chunks use
+ * `DEFAULT_BACKGROUND_NO_PROGRESS_TIMEOUT_MS` so large outputs can use the
+ * background budget. Only armed when a soft-timeout regime is active (hosted
+ * runs); local dev stays unbounded.
+ */
+export const RUN_NO_PROGRESS_HARD_TIMEOUT_MS = 150_000;
+
+/** Default SQL retention for completed run event logs (24 hours). */
+export const DEFAULT_COMPLETED_RUN_RETENTION_MS = 24 * 60 * 60 * 1000;
+
+/**
+ * Default SQL retention for errored/aborted run event logs (7 days). Kept
+ * longer than completed runs so cut-off / failed chats survive for pattern
+ * analysis (listErroredRuns) — these are rare and small, and they are exactly
+ * the runs we need to study to keep hardening reliability.
+ */
+export const DEFAULT_ERRORED_RUN_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
+
+/**
+ * How recently a terminal run must have started for `/runs/active` to surface
+ * it. Reconnect after this window won't replay the run — typical real-world
+ * disconnects resolve in seconds, so 10 minutes is generous while keeping us
+ * from resurrecting ancient turns when the user reopens an old thread.
+ */
+export const TERMINAL_RUN_RECONNECT_WINDOW_MS = 10 * 60 * 1000;
+
+/** Fast poll cadence while a SQL-backed SSE subscription is actively receiving rows. */
+export const SQL_SUBSCRIPTION_ACTIVE_POLL_MS = 125;
+
+/** Baseline SQL-backed SSE poll cadence when the run is idle. */
+export const SQL_SUBSCRIPTION_IDLE_POLL_MS = 500;
+
+/**
+ * Keep briefly polling quickly after rows arrive so token streams stay smooth,
+ * then back off to the idle cadence if the producer goes quiet.
+ */
+export const SQL_SUBSCRIPTION_ACTIVE_GRACE_MS = 2_000;
+
+/** Keep terminal/status probes at the historical cadence to bound DB work. */
+export const SQL_SUBSCRIPTION_STATUS_POLL_MS = 500;
+
+export function resolveSqlSubscriptionPollMs(
+  now: number,
+  activePollUntil: number,
+): number {
+  return now < activePollUntil
+    ? SQL_SUBSCRIPTION_ACTIVE_POLL_MS
+    : SQL_SUBSCRIPTION_IDLE_POLL_MS;
+}
+
+const PROVIDER_RATE_LIMITED_ERROR_CODE = "provider_rate_limited";
+
+function isPreparingActionActivityEvent(event: AgentChatEvent): boolean {
+  if (event.type !== "activity") return false;
+  const label = event.label.trim().toLowerCase();
+  return label.startsWith("preparing ") && label.includes(" action");
+}
+
+function getRunErrorMessage(err: unknown): string {
+  if (
+    typeof err === "object" &&
+    err !== null &&
+    "message" in err &&
+    typeof err.message === "string" &&
+    err.message.trim().length > 0
+  ) {
+    return err.message;
+  }
+  return "Unknown error";
+}
+
+function getEngineRunErrorCode(err: EngineError): string | undefined {
+  if (err.errorCode) return err.errorCode;
+  if (err.statusCode === 429) return PROVIDER_RATE_LIMITED_ERROR_CODE;
+  return undefined;
+}
+
+function getEngineRunErrorDetails(err: EngineError): string | undefined {
+  if (err.statusCode === 429) return err.message;
+  return undefined;
+}
+
+function shouldCaptureRunError(err: unknown): boolean {
+  if (!(err instanceof EngineError)) return true;
+  const errorCode = getEngineRunErrorCode(err);
+  if (isLlmCredentialError(err, errorCode)) return false;
+  if (err.statusCode === 401 || err.statusCode === 403) return false;
+  if (/^40[13] status code\b/i.test(err.message)) return false;
+  if (err.message.trim().toLowerCase() === "connection error.") return false;
+  if (!errorCode) return true;
+  const normalizedCode = errorCode.toLowerCase();
+  return (
+    !normalizedCode.startsWith("credits-limit") &&
+    normalizedCode !== "builder_gateway_network_error" &&
+    normalizedCode !== "provider_rate_limited" &&
+    normalizedCode !== "rate_limit_exceeded"
+  );
+}
+
+export interface StartRunOptions {
+  /** Optional internal run chunk budget. When reached, the framework emits an
+   * auto-continuation signal instead of a user-facing timeout. Leave unset for
+   * no framework-imposed run timeout. */
+  softTimeoutMs?: number;
+  /** Opt into the hosted/serverless default chunk budget. Only callers with
+   * automatic continuation support should enable this. */
+  useHostedSoftTimeoutDefault?: boolean;
+  /** Stable identity for the logical assistant turn this run belongs to. A
+   * turn may span several continuation runs (each chunk is its own run); they
+   * share one `turnId` so the durable assistant message can be folded across
+   * them instead of dropped per-run. Defaults to the runId (turn == run). */
+  turnId?: string;
+  /**
+   * Opt into the durable-background-function soft-timeout regime for THIS run
+   * only. When true, `resolveRunSoftTimeoutMs` lifts the hosted ceiling from
+   * 40s to ~13min (`BACKGROUND_SOFT_TIMEOUT_CEILING_MS`) because the run is
+   * executing inside a Netlify background function (no ~60s wall). Off by
+   * default — the foreground/interactive path never sets this, so its 40s
+   * clamp is unchanged. See the design doc + the durable-background dispatch
+   * decision in production-agent.ts.
+   */
+  backgroundFunction?: boolean;
+  /**
+   * Override the run-manager-level no-progress backstop
+   * (`RUN_NO_PROGRESS_HARD_TIMEOUT_MS`). `0` disables it. Defaults to the
+   * backstop constant whenever a soft-timeout regime is active (hosted runs)
+   * and to disabled otherwise (local dev stays unbounded).
+   */
+  noProgressTimeoutMs?: number;
+  /**
+   * Lifecycle metadata persisted to `agent_runs.dispatch_mode` and surfaced to
+   * clients through `/runs/active`. This does not change run-manager behavior;
+   * callers use it to describe who owns continuation at hosted chunk boundaries.
+   */
+  dispatchMode?: "foreground" | "foreground-self-chain";
+}
+
+export interface ResolveRunSoftTimeoutOptions {
+  useHostedDefault?: boolean;
+  /**
+   * Resolve the soft timeout for a run executing inside a Netlify background
+   * function. Lifts the hosted clamp to `BACKGROUND_SOFT_TIMEOUT_CEILING_MS`
+   * (~13min) for this invocation only and, when no override/env is supplied,
+   * defaults to `DEFAULT_BACKGROUND_RUN_SOFT_TIMEOUT_MS`. Does NOT change the
+   * foreground ceiling. Off by default.
+   */
+  backgroundFunction?: boolean;
+}
+
+/**
+ * True on hosted/serverless runtimes where the soft-timeout regime applies
+ * (see `resolveRunSoftTimeoutMs`, which resolves to 0 — disabled — off these
+ * runtimes). Exported so production-agent.ts can gate its foreground
+ * first-model-event cap (`FOREGROUND_FIRST_MODEL_EVENT_TIMEOUT_MS`) on the
+ * SAME predicate that selects the 40s clamp: the cap only makes sense where
+ * that clamp (and the platform wall behind it) exists.
+ */
+export function isHostedRuntime(): boolean {
+  if (
+    process.env.NETLIFY &&
+    process.env.NETLIFY !== "false" &&
+    process.env.NETLIFY_LOCAL !== "true"
+  ) {
+    return true;
+  }
+  if (
+    process.env.AWS_LAMBDA_FUNCTION_NAME &&
+    process.env.NETLIFY_LOCAL !== "true"
+  ) {
+    return true;
+  }
+  return Boolean(
+    process.env.CF_PAGES ||
+    process.env.VERCEL ||
+    process.env.VERCEL_ENV ||
+    process.env.RENDER ||
+    process.env.FLY_APP_NAME ||
+    process.env.K_SERVICE,
+  );
+}
+
+export function resolveRunSoftTimeoutMs(
+  overrideMs?: number,
+  options?: ResolveRunSoftTimeoutOptions,
+): number {
+  const hosted = isHostedRuntime();
+  const background = options?.backgroundFunction === true;
+  // The interactive/foreground ceiling is 40s — the synchronous serverless
+  // function wall. A background-function run (opt-in only) has no ~60s wall, so
+  // it is allowed to outlast that and is clamped to the larger 13-min ceiling
+  // instead. The 40s clamp for non-background hosted runs is unchanged.
+  const ceiling = background
+    ? BACKGROUND_SOFT_TIMEOUT_CEILING_MS
+    : HOSTED_SOFT_TIMEOUT_CEILING_MS;
+  // A configured/env soft timeout that exceeds the upstream walls can never
+  // actually fire (the gateway/function kills the run first), so clamp it down
+  // on hosted runtimes. This is what makes auto_continue reach the client
+  // instead of the run dying as builder_gateway_timeout / stale_run. `0` means
+  // "disabled" and is never clamped up.
+  const clampHosted = (ms: number): number =>
+    hosted && ms > ceiling ? ceiling : ms;
+
+  if (typeof overrideMs === "number" && Number.isFinite(overrideMs)) {
+    return clampHosted(Math.max(0, overrideMs));
+  }
+  const envValue = process.env.AGENT_RUN_SOFT_TIMEOUT_MS;
+  if (envValue !== undefined) {
+    const raw = Number(envValue);
+    if (Number.isFinite(raw) && raw >= 0) return clampHosted(raw);
+  }
+  // A background-function run uses the full background budget by default; the
+  // foreground default (40s) is unchanged.
+  if (background) {
+    return hosted ? DEFAULT_BACKGROUND_RUN_SOFT_TIMEOUT_MS : 0;
+  }
+  return options?.useHostedDefault && hosted
+    ? DEFAULT_HOSTED_RUN_SOFT_TIMEOUT_MS
+    : 0;
+}
+
+export function resolveCompletedRunRetentionMs(): number {
+  const envValue = process.env.AGENT_RUN_RETENTION_MS;
+  if (envValue !== undefined) {
+    const raw = Number(envValue);
+    if (Number.isFinite(raw) && raw >= 0) return raw;
+  }
+  return DEFAULT_COMPLETED_RUN_RETENTION_MS;
+}
+
+export function resolveErroredRunRetentionMs(): number {
+  const envValue = process.env.AGENT_ERRORED_RUN_RETENTION_MS;
+  if (envValue !== undefined) {
+    const raw = Number(envValue);
+    if (Number.isFinite(raw) && raw >= 0) return raw;
+  }
+  return DEFAULT_ERRORED_RUN_RETENTION_MS;
+}
+
+function isTerminalRunEvent(event: AgentChatEvent): boolean {
+  return (
+    event.type === "done" ||
+    event.type === "error" ||
+    event.type === "missing_api_key" ||
+    event.type === "loop_limit" ||
+    event.type === "auto_continue"
+  );
+}
+
+function terminalEventForcesErroredStatus(event: AgentChatEvent | null) {
+  return event?.type === "error" || event?.type === "missing_api_key";
+}
+
+function terminalReasonForRun(
+  finalStatus: "completed" | "errored" | "aborted",
+  terminalEvent: AgentChatEvent | null,
+  abortReason: string | undefined,
+  completionError: unknown,
+): string {
+  if (terminalEvent?.type === "auto_continue") {
+    return terminalEvent.reason || "auto_continue";
+  }
+  if (terminalEvent?.type === "loop_limit") return "loop_limit";
+  if (terminalEvent?.type === "missing_api_key") return "missing_api_key";
+  if (terminalEvent?.type === "error") {
+    return `error:${terminalEvent.errorCode || "unknown"}`;
+  }
+  if (finalStatus === "aborted") return `aborted:${abortReason ?? "user"}`;
+  if (completionError) return "completion_error";
+  if (finalStatus === "errored") return "error:unknown";
+  return "done";
+}
+
+function abortInMemoryRun(run: ActiveRun, reason: string = "user") {
+  run.abortReason = reason;
+  run.status = "aborted";
+  if (threadToRun.get(run.threadId) === run.runId) {
+    threadToRun.delete(run.threadId);
+  }
+  run.abort.abort(reason);
+  for (const subscriber of run.subscribers) {
+    try {
+      subscriber({ seq: run.events.length, event: { type: "done" } });
+    } catch {
+      // ignore — subscriber is being removed below
+    }
+  }
+  run.subscribers.clear();
+}
+
+/**
+ * Start a new agent run in the background.
+ * `runFn` receives a `send` callback and an `AbortSignal`.
+ * The run continues even if all SSE subscribers disconnect.
+ *
+ * Events are persisted to SQL for cross-isolate access (Cloudflare Workers).
+ */
+export function startRun(
+  runId: string,
+  threadId: string,
+  runFn: (
+    send: (event: AgentChatEvent) => void,
+    signal: AbortSignal,
+  ) => Promise<void>,
+  onComplete?: (run: ActiveRun) => void | Promise<void>,
+  options?: StartRunOptions,
+): ActiveRun {
+  // If there's already a run for this thread, abort it
+  const existingRunId = threadToRun.get(threadId);
+  if (existingRunId) {
+    abortRun(existingRunId);
+  }
+
+  const abort = new AbortController();
+  let softTimedOut = false;
+  const run: ActiveRun = {
+    runId,
+    threadId,
+    turnId: options?.turnId ?? runId,
+    events: [],
+    status: "running",
+    subscribers: new Set(),
+    abort,
+    startedAt: Date.now(),
+  };
+
+  activeRuns.set(runId, run);
+  threadToRun.set(threadId, runId);
+
+  const captureRunPersistenceError = (
+    error: unknown,
+    phase: "insert-run" | "insert-event",
+    extra: Record<string, unknown> = {},
+  ) => {
+    captureError(error, {
+      route: "/_agent-native/agent-chat",
+      tags: {
+        source: "agent-run-manager",
+        phase,
+        runStatus: run.status,
+      },
+      extra: {
+        runId,
+        threadId,
+        eventCount: run.events.length,
+        startedAt: run.startedAt,
+        ...extra,
+      },
+      contexts: {
+        agentRun: {
+          runId,
+          threadId,
+          status: run.status,
+          phase,
+          eventCount: run.events.length,
+          startedAt: run.startedAt,
+        },
+      },
+    });
+  };
+
+  // Persist run to SQL without blocking the response. Keep the promise so
+  // final status cannot race ahead of a slow initial INSERT and then get
+  // overwritten by a late row stuck at status='running'.
+  const insertOptions = options?.dispatchMode
+    ? { dispatchMode: options.dispatchMode }
+    : undefined;
+  const insertRunPromise = (
+    insertOptions
+      ? insertRun(runId, threadId, options?.turnId, insertOptions)
+      : insertRun(runId, threadId, options?.turnId)
+  ).catch((error) => {
+    captureRunPersistenceError(error, "insert-run");
+  });
+
+  // Per-run event persistence chain: events are chained so SQL inserts commit
+  // in seq order. Without this, a fast seq=5 commit before a slow seq=4 means
+  // the SQL poller advances its cursor past seq=4 (lastSeq = 5+1 = 6) and
+  // reconnecting clients permanently miss that event (silent gap). Chaining
+  // per run ensures order without blocking the fast in-memory SSE path.
+  let persistenceChain: Promise<void> = Promise.resolve();
+
+  // Throttle the durable progress timestamp to at most once per second so
+  // a chatty token-by-token stream doesn't translate into one DB write per
+  // chunk. The stuck-detector threshold is on the order of tens of seconds,
+  // so 1s resolution is plenty.
+  let lastProgressBumpAt = 0;
+  const preparingActivityBytes = new Map<string, number>();
+  const preparingActivityTools = new Map<string, string>();
+  const preparingActivityRestartHighWater = new Map<string, number>();
+  let eventPersistenceErrorCaptured = false;
+  const bumpProgressIfDue = () => {
+    const now = Date.now();
+    if (now - lastProgressBumpAt < 1000) return;
+    lastProgressBumpAt = now;
+    bumpRunProgress(runId).catch(() => {});
+  };
+  const shouldBumpProgressForEvent = (event: AgentChatEvent): boolean => {
+    if (event.type === "stream_keepalive") return false;
+    if (event.type === "clear") {
+      for (const [key, bytes] of preparingActivityBytes) {
+        const toolKey = preparingActivityTools.get(key) ?? key;
+        preparingActivityRestartHighWater.set(
+          toolKey,
+          Math.max(preparingActivityRestartHighWater.get(toolKey) ?? 0, bytes),
+        );
+      }
+      preparingActivityBytes.clear();
+      preparingActivityTools.clear();
+      return false;
+    }
+    if (event.type === "activity" && isPreparingActionActivityEvent(event)) {
+      const toolKey = event.tool?.trim() || event.label.trim();
+      const activityKey = `${toolKey}:${event.id?.trim() || "no-id"}`;
+      const progressBytes =
+        typeof event.progressBytes === "number" &&
+        Number.isFinite(event.progressBytes) &&
+        event.progressBytes >= 0
+          ? Math.floor(event.progressBytes)
+          : undefined;
+      if (progressBytes === undefined) return false;
+      const restartHighWater =
+        preparingActivityRestartHighWater.get(toolKey) ?? 0;
+      if (!event.id?.trim()) {
+        if (progressBytes <= restartHighWater) return false;
+        preparingActivityTools.set(activityKey, toolKey);
+        preparingActivityBytes.set(
+          activityKey,
+          Math.max(preparingActivityBytes.get(activityKey) ?? 0, progressBytes),
+        );
+        if (preparingActivityRestartHighWater.has(toolKey)) {
+          preparingActivityRestartHighWater.set(
+            toolKey,
+            Math.max(restartHighWater, progressBytes),
+          );
+        }
+        return progressBytes > 0;
+      }
+      const previousBytes = Math.max(
+        preparingActivityBytes.get(activityKey) ?? 0,
+        restartHighWater,
+      );
+      if (
+        !preparingActivityBytes.has(activityKey) &&
+        progressBytes === 0 &&
+        !preparingActivityRestartHighWater.has(toolKey)
+      ) {
+        preparingActivityTools.set(activityKey, toolKey);
+        preparingActivityBytes.set(activityKey, 0);
+        preparingActivityRestartHighWater.set(toolKey, 0);
+        return true;
+      }
+      if (progressBytes <= previousBytes) {
+        preparingActivityTools.set(activityKey, toolKey);
+        preparingActivityBytes.set(
+          activityKey,
+          Math.max(previousBytes, progressBytes),
+        );
+        return false;
+      }
+      preparingActivityTools.set(activityKey, toolKey);
+      preparingActivityBytes.set(activityKey, progressBytes);
+      if (preparingActivityRestartHighWater.has(toolKey)) {
+        preparingActivityRestartHighWater.set(
+          toolKey,
+          Math.max(
+            preparingActivityRestartHighWater.get(toolKey) ?? 0,
+            progressBytes,
+          ),
+        );
+      }
+      return true;
+    }
+    if (event.type === "tool_start" || event.type === "tool_done") {
+      preparingActivityBytes.clear();
+      preparingActivityTools.clear();
+      preparingActivityRestartHighWater.clear();
+    }
+    if (event.type === "done" || event.type === "error") {
+      preparingActivityBytes.clear();
+      preparingActivityTools.clear();
+      preparingActivityRestartHighWater.clear();
+    }
+    return true;
+  };
+
+  // ── No-progress backstop (see RUN_NO_PROGRESS_HARD_TIMEOUT_MS) ──────────
+  // Timer-driven and independent of the agent loop, so it fires even when the
+  // stall is in a segment the in-loop watchdogs never see (engine-call
+  // establishment, setup, a wedged transport emitting keepalives). Tool calls
+  // and sub-agent calls in flight suspend it — tool execution legitimately
+  // emits nothing for minutes and carries its own 12-min timeout.
+  let lastRealProgressAt = Date.now();
+  let inFlightWorkCount = 0;
+  const trackInFlightWork = (event: AgentChatEvent) => {
+    const wasIdle = inFlightWorkCount === 0;
+    if (event.type === "tool_start") {
+      inFlightWorkCount += 1;
+    } else if (event.type === "tool_done") {
+      inFlightWorkCount = Math.max(0, inFlightWorkCount - 1);
+    } else if (event.type === "agent_call") {
+      if (event.status === "start") {
+        inFlightWorkCount += 1;
+      } else {
+        inFlightWorkCount = Math.max(0, inFlightWorkCount - 1);
+      }
+    } else {
+      return; // Not a work-tracking event — no transition possible.
+    }
+    // Mirror the 0<->N transition into SQL so a stale reaper running in a
+    // DIFFERENT isolate (a client's SQL-subscription poll, a sibling
+    // isolate's opportunistic cleanup, a fresh boot's startup sweep) can
+    // grant this demonstrably-alive run a bounded grace even when THIS
+    // isolate's own heartbeat write is failing (e.g. Neon pooler saturation)
+    // — `inFlightWorkCount` itself is in-memory and invisible to those other
+    // isolates. Fire-and-forget: never block event emission on this write,
+    // and a write failure here is no worse than today's behavior (the row
+    // just gets no grace). See `setRunInFlightMarker` / `IN_FLIGHT_RUN_STALE_GRACE_MS`
+    // in run-store.ts for the full reasoning and the bounded-grace derivation.
+    if (wasIdle && inFlightWorkCount > 0) {
+      setRunInFlightMarker(runId, true).catch(() => {});
+    } else if (!wasIdle && inFlightWorkCount === 0) {
+      setRunInFlightMarker(runId, false).catch(() => {});
+    }
+  };
+  const checkNoProgressBackstop = () => {
+    if (noProgressTimeoutMs <= 0) return;
+    if (run.status !== "running" || abort.signal.aborted) return;
+    if (inFlightWorkCount > 0) return;
+    if (Date.now() - lastRealProgressAt < noProgressTimeoutMs) return;
+    console.error(
+      `[run-manager] no real progress for ${noProgressTimeoutMs}ms with no tool in flight — ` +
+        `checkpointing run for continuation`,
+      runId,
+    );
+    // Mirror the soft-timeout semantics exactly: the chunk completes (not
+    // aborts) at an auto_continue boundary, so the continuation machinery —
+    // server-chained for background workers, client-driven for foreground —
+    // recovers the turn.
+    softTimedOut = true;
+    send({ type: "auto_continue", reason: "no_progress" });
+    abort.abort("no_progress");
+  };
+
+  // Periodic SQL abort check interval (for cross-isolate abort on Workers).
+  // Also self-aborts when our row is no longer status='running' — catches the
+  // false-stale-reap zombie scenario where the reaper flipped the row while
+  // this isolate was briefly unable to heartbeat (DB latency / GC pause).
+  let lastAbortCheck = Date.now() - 3000;
+  const checkSqlAbort = () => {
+    const now = Date.now();
+    if (now - lastAbortCheck < 3000) return;
+    lastAbortCheck = now;
+    getRunAbortState(runId)
+      .then(async (state) => {
+        if (state.aborted && !abort.signal.aborted) {
+          abortInMemoryRun(run, state.reason ?? "user");
+          return;
+        }
+        // If the row is no longer 'running' (reaped / replaced) and we're
+        // still executing, self-abort so we stop executing and don't overwrite
+        // the newer state with our terminal write.
+        if (!abort.signal.aborted) {
+          const status = await getRunStatus(runId);
+          if (status !== null && status !== "running") {
+            abortInMemoryRun(run, "displaced");
+          }
+        }
+      })
+      .catch(() => {});
+  };
+
+  // Heartbeat: bump heartbeat_at every 1.5s so watchers can detect a dead
+  // producer (process crash, HMR restart, isolate eviction) quickly and
+  // reap the row. Paired with RUN_STALE_MS (15s) — 10x the interval to
+  // tolerate transient DB slowness without false positives.
+  let consecutiveHeartbeatFailures = 0;
+  // Single-flight the heartbeat write. The timer fires every 1.5s but a write
+  // can take up to the DB op timeout (~8s) when the Neon pooler is saturated.
+  // Firing a fresh write each tick regardless piled up ~5 concurrent writes
+  // under contention, each holding a pooler connection — ADDING to the exact
+  // connection-cap exhaustion that starves the heartbeat and false-reaps the
+  // run as stale. Skip a tick's write while one is still outstanding so a run
+  // holds at most one heartbeat connection. The abort/backstop checks below
+  // still run every tick (they don't touch the DB on the hot path).
+  let heartbeatInFlight = false;
+  const heartbeatTimer: ReturnType<typeof setInterval> = setInterval(() => {
+    if (!heartbeatInFlight) {
+      heartbeatInFlight = true;
+      updateRunHeartbeat(runId)
+        .then(() => {
+          consecutiveHeartbeatFailures = 0;
+        })
+        .catch((error) => {
+          consecutiveHeartbeatFailures += 1;
+          // Swallow routine single-tick blips; escalate once failures approach
+          // the stale window so false-positive stale_run from silent write
+          // failures is diagnosable.
+          if (consecutiveHeartbeatFailures >= 3) {
+            captureError(error, {
+              route: "/_agent-native/agent-chat",
+              tags: {
+                source: "agent-run-manager",
+                phase: "heartbeat",
+                consecutiveFailures: String(consecutiveHeartbeatFailures),
+              },
+              extra: { runId, threadId },
+            });
+          }
+        })
+        .finally(() => {
+          heartbeatInFlight = false;
+        });
+    }
+    checkSqlAbort();
+    checkNoProgressBackstop();
+  }, 1500);
+  const softTimeoutMs = resolveRunSoftTimeoutMs(options?.softTimeoutMs, {
+    useHostedDefault: options?.useHostedSoftTimeoutDefault === true,
+    backgroundFunction: options?.backgroundFunction === true,
+  });
+  // Armed only when a soft-timeout regime is active (hosted): local dev keeps
+  // unbounded runs. For 40s foreground chunks the soft timeout always fires
+  // first, so in practice this guards the long background chunks.
+  const noProgressTimeoutMs =
+    options?.noProgressTimeoutMs ??
+    (softTimeoutMs > 0
+      ? options?.backgroundFunction === true
+        ? DEFAULT_BACKGROUND_NO_PROGRESS_TIMEOUT_MS
+        : RUN_NO_PROGRESS_HARD_TIMEOUT_MS
+      : 0);
+  const softTimeoutTimer =
+    softTimeoutMs > 0
+      ? setTimeout(() => {
+          if (run.status !== "running" || abort.signal.aborted) return;
+          softTimedOut = true;
+          send({
+            type: "auto_continue",
+            reason: "run_timeout",
+          });
+          abort.abort("run_timeout");
+        }, softTimeoutMs)
+      : null;
+  let pendingTerminalEvent: RunEvent | null = null;
+
+  const captureRunError = (error: unknown, phase: "run" | "completion") => {
+    const errorCode =
+      error instanceof EngineError ? getEngineRunErrorCode(error) : undefined;
+    captureError(error, {
+      route: "/_agent-native/agent-chat",
+      tags: {
+        source: "agent-run-manager",
+        phase,
+        runStatus: run.status,
+        softTimedOut: softTimedOut ? "true" : "false",
+        abortReason: run.abortReason,
+        errorCode,
+      },
+      extra: {
+        runId,
+        threadId,
+        eventCount: run.events.length,
+        startedAt: run.startedAt,
+        softTimeoutMs,
+      },
+      contexts: {
+        agentRun: {
+          runId,
+          threadId,
+          status: run.status,
+          phase,
+          eventCount: run.events.length,
+          startedAt: run.startedAt,
+          softTimeoutMs,
+          softTimedOut,
+          abortReason: run.abortReason,
+        },
+      },
+    });
+  };
+
+  const emitRunEvent = (
+    runEvent: RunEvent,
+    options?: { surfacePersistenceError?: boolean },
+  ): Promise<void> => {
+    run.events.push(runEvent);
+
+    // Notify in-memory subscribers (same isolate, fast path)
+    for (const subscriber of run.subscribers) {
+      try {
+        subscriber(runEvent);
+      } catch {
+        run.subscribers.delete(subscriber);
+      }
+    }
+
+    // Bump the durable progress timestamp. Distinct from the heartbeat:
+    // heartbeat = "process is up", progress = "real work is happening." The
+    // gap between them is what the client-side stuck-detector reads to tell
+    // a hung run from a healthy one. Keepalive and zero-byte action prep are
+    // liveness only; streamed input bytes, text, and tool lifecycle events are
+    // real progress.
+    trackInFlightWork(runEvent.event);
+    if (shouldBumpProgressForEvent(runEvent.event)) {
+      lastRealProgressAt = Date.now();
+      bumpProgressIfDue();
+    }
+
+    // Persist event to SQL. Events are chained through persistenceChain so
+    // inserts commit in seq order — an out-of-order commit would advance the
+    // SQL poller's cursor past the slow row, permanently dropping it for
+    // reconnecting clients. Terminal events surface persistence errors so the
+    // caller can decide how to handle a failed final write.
+    const thisInsert = persistenceChain.then(() =>
+      insertRunEvent(runId, runEvent.seq, JSON.stringify(runEvent.event)),
+    );
+    persistenceChain = thisInsert.catch((error) => {
+      if (!eventPersistenceErrorCaptured) {
+        eventPersistenceErrorCaptured = true;
+        captureRunPersistenceError(error, "insert-event", {
+          seq: runEvent.seq,
+          eventType: runEvent.event.type,
+        });
+      }
+    });
+    const persistence = thisInsert;
+    if (!options?.surfacePersistenceError) {
+      persistence.catch(() => {});
+    }
+
+    checkSqlAbort();
+    return persistence;
+  };
+
+  const send = (event: AgentChatEvent) => {
+    if (run.status === "aborted" && abort.signal.aborted) return;
+
+    const runEvent: RunEvent = { seq: run.events.length, event };
+    if (isTerminalRunEvent(event)) {
+      pendingTerminalEvent = runEvent;
+      return;
+    }
+
+    emitRunEvent(runEvent);
+  };
+
+  // Run in background — intentionally detached from any HTTP connection
+  const runPromise = runFn(send, abort.signal)
+    .then(() => {
+      if (abort.signal.aborted) {
+        run.status = softTimedOut ? "completed" : "aborted";
+        return;
+      }
+      run.status = "completed";
+    })
+    .catch((err) => {
+      // Don't surface abort errors — the run was intentionally stopped
+      if (abort.signal.aborted) {
+        run.status = softTimedOut ? "completed" : "aborted";
+        return;
+      }
+      run.status = "errored";
+      if (shouldCaptureRunError(err)) {
+        captureRunError(err, "run");
+      }
+      const errorMessage = getRunErrorMessage(err);
+      const errorCode =
+        err instanceof EngineError ? getEngineRunErrorCode(err) : undefined;
+      const details =
+        err instanceof EngineError ? getEngineRunErrorDetails(err) : undefined;
+      send({
+        type: "error",
+        error: errorMessage,
+        ...(errorCode ? { errorCode } : {}),
+        ...(details ? { details } : {}),
+        ...(err instanceof EngineError && err.upgradeUrl
+          ? { upgradeUrl: err.upgradeUrl }
+          : {}),
+      });
+    })
+    .finally(async () => {
+      // Ordering matters here — this is the atomic-complete boundary.
+      // Invariant: once agent_runs.status flips to "completed"/"errored"
+      // in SQL, thread_data for this turn is already durable. This lets
+      // reconnecting clients trust the simple rule "status != running →
+      // fetch thread_data" without polling/retrying for a race window
+      // where onComplete was still pending.
+
+      // 1. Await the completion callback (thread_data save). Heartbeat is
+      //    still ticking so the run doesn't look stale to any concurrent
+      //    /runs/active check while we wait for SQL writes to land.
+      let completionError: unknown = null;
+      let terminalPersistenceError: unknown = null;
+      const terminalEvent = pendingTerminalEvent?.event ?? null;
+      if (
+        onComplete &&
+        !(run.status === "aborted" && run.abortReason === "no_progress")
+      ) {
+        try {
+          const completionStatus =
+            run.status !== "aborted" &&
+            terminalEventForcesErroredStatus(terminalEvent)
+              ? "errored"
+              : run.status;
+          const completionRun: ActiveRun =
+            pendingTerminalEvent || completionStatus !== run.status
+              ? {
+                  ...run,
+                  status: completionStatus,
+                  events: pendingTerminalEvent
+                    ? [...run.events, pendingTerminalEvent]
+                    : run.events,
+                }
+              : run;
+          await onComplete(completionRun);
+        } catch (err) {
+          completionError = err;
+          captureRunError(err, "completion");
+          console.error(
+            "[run-manager] onComplete callback error:",
+            err instanceof Error ? err.message : err,
+          );
+        }
+      }
+
+      // 2. Compute final status. If the completion callback threw, we'd
+      //    rather mark the run errored than claim success with incomplete
+      //    thread_data.
+      const finalStatus =
+        run.status === "aborted"
+          ? "aborted"
+          : run.status === "errored" ||
+              completionError ||
+              terminalEventForcesErroredStatus(terminalEvent)
+            ? "errored"
+            : "completed";
+      const terminalReason = terminalReasonForRun(
+        finalStatus,
+        terminalEvent,
+        run.abortReason,
+        completionError,
+      );
+
+      // 3. Emit the terminal event only after thread_data is durable. Live
+      //    SSE clients close on this event and usually fetch thread_data
+      //    immediately, so emitting it earlier recreates the final-message
+      //    race this manager is meant to avoid.
+      if (finalStatus === "completed" || finalStatus === "errored") {
+        // Choose the terminal event payload (done / the stashed terminal /
+        // a synthesized error). NOTE: the `seq` carried by
+        // `pendingTerminalEvent` was captured by `send()` at stash time as
+        // `run.events.length` and is NOT authoritative — if the runFn emitted
+        // any more events before it actually stopped on the abort signal,
+        // those events were pushed and reused that same seq. Persisting the
+        // terminal event with the stale seq would collide with an
+        // already-persisted streaming event and get silently dropped by
+        // insertRunEvent's `ON CONFLICT (run_id, seq) DO NOTHING`, so the
+        // client would never see the terminal/continuation signal. We always
+        // re-stamp the seq at emit time (max-seq+1) just below.
+        const terminalEvent: AgentChatEvent =
+          finalStatus === "completed"
+            ? (pendingTerminalEvent?.event ?? { type: "done" })
+            : pendingTerminalEvent?.event.type === "error" ||
+                pendingTerminalEvent?.event.type === "missing_api_key"
+              ? pendingTerminalEvent.event
+              : pendingTerminalEvent?.event.type === "auto_continue"
+                ? // The run was checkpointed at a soft-timeout/loop boundary and
+                  // is recoverable: the partial turn is in agent_run_events and
+                  // the continuation run will re-attempt the thread_data save.
+                  // Even though the completion save failed (finalStatus stays
+                  // "errored" for SQL/diagnostics), re-emit the auto_continue so
+                  // the client resumes instead of seeing a dead chat.
+                  pendingTerminalEvent.event
+                : {
+                    type: "error",
+                    error: completionError
+                      ? "Agent response could not be saved."
+                      : "Agent run ended unexpectedly",
+                  };
+        const last = run.events[run.events.length - 1];
+        if (!last || !isTerminalRunEvent(last.event)) {
+          // Assign the seq at EMIT time, not at stash time. `run.events` is a
+          // contiguous 0-based log, so `run.events.length` is the next free
+          // seq and can never collide with an event that was pushed after the
+          // terminal event was stashed.
+          const terminal: RunEvent = {
+            seq: run.events.length,
+            event: terminalEvent,
+          };
+          try {
+            await emitRunEvent(terminal, { surfacePersistenceError: true });
+          } catch (err) {
+            terminalPersistenceError = err;
+            captureRunError(err, "completion");
+            console.error(
+              "[run-manager] terminal event persistence error:",
+              err instanceof Error ? err.message : err,
+            );
+          }
+        }
+      }
+      for (const subscriber of run.subscribers) {
+        run.subscribers.delete(subscriber);
+      }
+
+      // 4. Stop the heartbeat — all liveness writes are done.
+      clearInterval(heartbeatTimer);
+      if (softTimeoutTimer) clearTimeout(softTimeoutTimer);
+
+      // 5. Persist final status to SQL. Use the conditional write so a zombie
+      //    run (reaped or displaced while executing) cannot clobber the newer
+      //    status written by the reaper or a replacement run.
+      try {
+        await insertRunPromise;
+        if (!terminalPersistenceError) {
+          let statusUpdated = false;
+          try {
+            statusUpdated = await updateRunStatusIfRunning(runId, finalStatus);
+          } catch {
+            statusUpdated = false;
+          }
+          if (statusUpdated) {
+            await setRunTerminalReason(runId, terminalReason);
+          } else {
+            await reconcileTerminalRunFromEvents(runId).catch(() => false);
+          }
+        }
+      } catch {
+        // Best-effort — reapIfStale will eventually clean this up via
+        // the heartbeat-stale path.
+      }
+
+      // 5b. Record terminal failure classification for errored runs so
+      //     cut-off / failed chats are queryable for pattern analysis. Read
+      //     the actual error event the run emitted (errorCode + message) so
+      //     diagnostics reflect the real cause (builder_gateway_timeout,
+      //     stale_run, context_length_exceeded, completion_error, …).
+      if (finalStatus === "errored") {
+        let errorCode: string | undefined;
+        let errorDetail: string | undefined;
+        const diagnosticEvents = pendingTerminalEvent
+          ? [...run.events, pendingTerminalEvent]
+          : run.events;
+        for (let i = diagnosticEvents.length - 1; i >= 0; i--) {
+          const ev = diagnosticEvents[i].event as {
+            type: string;
+            error?: string;
+            errorCode?: string;
+            details?: string;
+          };
+          if (ev.type === "missing_api_key") {
+            errorCode = LLM_MISSING_CREDENTIALS_ERROR_CODE;
+            errorDetail = LLM_MISSING_CREDENTIALS_MESSAGE;
+            break;
+          } else if (ev.type === "error") {
+            errorCode = ev.errorCode;
+            errorDetail = ev.error ?? ev.details;
+            break;
+          }
+        }
+        if (completionError && !errorCode) {
+          errorCode = "completion_error";
+          errorDetail =
+            errorDetail ??
+            (completionError instanceof Error
+              ? completionError.message
+              : String(completionError));
+        }
+        await setRunError(runId, errorCode ?? "unknown", errorDetail);
+      }
+
+      // 6. Schedule in-memory cleanup + opportunistic old-run pruning.
+      setTimeout(() => {
+        activeRuns.delete(runId);
+        if (threadToRun.get(threadId) === runId) {
+          threadToRun.delete(threadId);
+        }
+      }, CLEANUP_DELAY_MS);
+      cleanupOldRuns(
+        resolveCompletedRunRetentionMs(),
+        resolveErroredRunRetentionMs(),
+      ).catch(() => {});
+    });
+
+  // On Cloudflare Workers, keep the isolate alive for this run
+  try {
+    const cfCtx = (
+      globalThis as typeof globalThis & {
+        __cf_ctx?: { waitUntil(promise: Promise<unknown>): void };
+      }
+    ).__cf_ctx;
+    if (cfCtx?.waitUntil) {
+      cfCtx.waitUntil(runPromise);
+    }
+  } catch {
+    // Not on Workers — ignore
+  }
+
+  return run;
+}
+
+/**
+ * Subscribe to a run's events starting from `fromSeq`.
+ * Returns a ReadableStream that replays buffered events then live-tails.
+ * Cancelling the stream only unsubscribes — does NOT abort the agent.
+ *
+ * Falls back to SQL polling when the run is not in local memory
+ * (cross-isolate reconnection on Workers).
+ */
+export function subscribeToRun(
+  runId: string,
+  fromSeq: number,
+): ReadableStream<Uint8Array> | null {
+  const run = activeRuns.get(runId);
+  if (run) {
+    return subscribeInMemory(run, fromSeq);
+  }
+  // Not in local memory — try SQL (cross-isolate path)
+  return subscribeFromSQL(runId, fromSeq);
+}
+
+/** In-memory subscription (same isolate, fast path) */
+function subscribeInMemory(
+  run: ActiveRun,
+  fromSeq: number,
+): ReadableStream<Uint8Array> {
+  const encoder = new TextEncoder();
+  let subscriberRef: ((event: RunEvent) => void) | null = null;
+  let pingTimer: ReturnType<typeof setInterval> | null = null;
+
+  return new ReadableStream({
+    start(controller) {
+      const ping = () => {
+        try {
+          controller.enqueue(encoder.encode(`: ping ${Date.now()}\n\n`));
+        } catch {
+          if (subscriberRef) run.subscribers.delete(subscriberRef);
+          if (pingTimer) clearInterval(pingTimer);
+        }
+      };
+      ping();
+      pingTimer = setInterval(ping, 10_000);
+
+      // Replay buffered events from fromSeq
+      for (let i = fromSeq; i < run.events.length; i++) {
+        try {
+          controller.enqueue(
+            encoder.encode(
+              `data: ${JSON.stringify({ ...run.events[i].event, seq: run.events[i].seq })}\n\n`,
+            ),
+          );
+        } catch {
+          return;
+        }
+      }
+
+      // If run is already done, close immediately
+      if (run.status !== "running") {
+        if (pingTimer) clearInterval(pingTimer);
+        controller.close();
+        return;
+      }
+
+      // Subscribe to live events
+      subscriberRef = (event: RunEvent) => {
+        try {
+          controller.enqueue(
+            encoder.encode(
+              `data: ${JSON.stringify({ ...event.event, seq: event.seq })}\n\n`,
+            ),
+          );
+          // Close stream after terminal events
+          if (isTerminalRunEvent(event.event)) {
+            run.subscribers.delete(subscriberRef!);
+            if (pingTimer) clearInterval(pingTimer);
+            controller.close();
+          }
+        } catch {
+          run.subscribers.delete(subscriberRef!);
+        }
+      };
+
+      run.subscribers.add(subscriberRef);
+    },
+    cancel() {
+      // Only unsubscribe — do NOT abort the agent run
+      if (subscriberRef) run.subscribers.delete(subscriberRef);
+      if (pingTimer) clearInterval(pingTimer);
+    },
+  });
+}
+
+/** SQL-based subscription (cross-isolate, polling) */
+function subscribeFromSQL(
+  runId: string,
+  fromSeq: number,
+): ReadableStream<Uint8Array> | null {
+  const encoder = new TextEncoder();
+  let cancelled = false;
+  let pollTimer: ReturnType<typeof setTimeout> | null = null;
+  let pingTimer: ReturnType<typeof setInterval> | null = null;
+
+  return new ReadableStream({
+    async start(controller) {
+      let lastSeq = fromSeq;
+      let activePollUntil = 0;
+      let lastStatusCheckAt = 0;
+      const ping = () => {
+        try {
+          controller.enqueue(encoder.encode(`: ping ${Date.now()}\n\n`));
+        } catch {
+          cancelled = true;
+          if (pingTimer) clearInterval(pingTimer);
+        }
+      };
+      ping();
+      pingTimer = setInterval(ping, 10_000);
+
+      const poll = async () => {
+        if (cancelled) return;
+        try {
+          // Read new events from SQL
+          const events = await getRunEventsSince(runId, lastSeq);
+          if (events.length > 0) {
+            activePollUntil = Date.now() + SQL_SUBSCRIPTION_ACTIVE_GRACE_MS;
+          }
+          for (const { seq, eventData } of events) {
+            // Advance the cursor first, before any parse/enqueue branch can
+            // `continue`/`return`. Otherwise a single corrupt (unparseable)
+            // event row is re-fetched on every poll tick forever, wedging the
+            // SSE stream open and never delivering a terminal event.
+            lastSeq = seq + 1;
+            let parsed: any;
+            try {
+              parsed = JSON.parse(eventData);
+            } catch {
+              continue;
+            }
+            try {
+              controller.enqueue(
+                encoder.encode(
+                  `data: ${JSON.stringify({ ...parsed, seq })}\n\n`,
+                ),
+              );
+            } catch {
+              cancelled = true;
+              return;
+            }
+
+            // Close on terminal events
+            if (isTerminalRunEvent(parsed)) {
+              if (pingTimer) clearInterval(pingTimer);
+              controller.close();
+              return;
+            }
+          }
+
+          // Check if run completed (no terminal event but status changed)
+          if (events.length === 0) {
+            const now = Date.now();
+            if (now - lastStatusCheckAt < SQL_SUBSCRIPTION_STATUS_POLL_MS) {
+              if (!cancelled) {
+                const pollMs = resolveSqlSubscriptionPollMs(
+                  now,
+                  activePollUntil,
+                );
+                pollTimer = setTimeout(poll, pollMs);
+              }
+              return;
+            }
+            lastStatusCheckAt = now;
+            // Opportunistically reap a stale producer before trusting SQL's
+            // "running" status — otherwise a crashed server leaves us polling
+            // forever.
+            await reapIfStale(runId).catch(() => {});
+            const run = await getRunById(runId);
+            if (!run || run.status !== "running") {
+              // Run ended — do one final event read, then close
+              const finalEvents = await getRunEventsSince(runId, lastSeq);
+              for (const { seq, eventData } of finalEvents) {
+                // Advance first — see the main poll loop above for why.
+                lastSeq = seq + 1;
+                let parsed: any;
+                try {
+                  parsed = JSON.parse(eventData);
+                } catch {
+                  continue;
+                }
+                try {
+                  controller.enqueue(
+                    encoder.encode(
+                      `data: ${JSON.stringify({ ...parsed, seq })}\n\n`,
+                    ),
+                  );
+                } catch {
+                  cancelled = true;
+                  return;
+                }
+                if (isTerminalRunEvent(parsed)) {
+                  if (pingTimer) clearInterval(pingTimer);
+                  controller.close();
+                  return;
+                }
+              }
+              if (run?.status === "aborted") {
+                try {
+                  controller.enqueue(
+                    encoder.encode(
+                      `data: ${JSON.stringify({ type: "done", seq: lastSeq })}\n\n`,
+                    ),
+                  );
+                } catch {
+                  cancelled = true;
+                  return;
+                }
+              } else if (run?.status === "completed") {
+                try {
+                  controller.enqueue(
+                    encoder.encode(
+                      `data: ${JSON.stringify({ type: "done", seq: lastSeq })}\n\n`,
+                    ),
+                  );
+                } catch {
+                  cancelled = true;
+                  return;
+                }
+              } else if (run?.status === "errored") {
+                // The run row is terminal but this subscriber's cursor is
+                // already past (or never saw) the terminal event. Prefer the
+                // REAL last terminal event / row error_detail over inventing
+                // a stale_run card — slides prod showed Connection error.
+                // rows being mislabeled as stale_run on reconnect because
+                // this path always synthesized STALE_RUN_ERROR_EVENT.
+                const existing = await getLastTerminalRunEvent(runId).catch(
+                  () => null,
+                );
+                const resolved = existing
+                  ? { event: existing.event, shouldPersist: false }
+                  : resolveErroredRunTerminalEvent(run);
+                if (resolved.shouldPersist) {
+                  await ensureTerminalRunEvent(runId, resolved.event).catch(
+                    () => {},
+                  );
+                }
+                try {
+                  controller.enqueue(
+                    encoder.encode(
+                      `data: ${JSON.stringify({
+                        ...resolved.event,
+                        seq: existing?.seq ?? lastSeq,
+                      })}\n\n`,
+                    ),
+                  );
+                } catch {
+                  cancelled = true;
+                  return;
+                }
+              }
+              if (pingTimer) clearInterval(pingTimer);
+              controller.close();
+              return;
+            }
+          }
+
+          // Schedule next poll
+          if (!cancelled) {
+            const pollMs = resolveSqlSubscriptionPollMs(
+              Date.now(),
+              activePollUntil,
+            );
+            pollTimer = setTimeout(poll, pollMs);
+          }
+        } catch {
+          // SQL error — close stream
+          try {
+            if (pingTimer) clearInterval(pingTimer);
+            controller.close();
+          } catch {}
+        }
+      };
+
+      // Verify run exists before starting poll
+      try {
+        const run = await getRunById(runId);
+        if (!run) {
+          if (pingTimer) clearInterval(pingTimer);
+          controller.close();
+          return;
+        }
+      } catch {
+        controller.close();
+        return;
+      }
+
+      await poll();
+    },
+    cancel() {
+      cancelled = true;
+      if (pollTimer) clearTimeout(pollTimer);
+      if (pingTimer) clearInterval(pingTimer);
+    },
+  });
+}
+
+/** Get the active run for a thread (if any) — checks memory then SQL */
+export function getActiveRunForThread(threadId: string): ActiveRun | null {
+  const runId = threadToRun.get(threadId);
+  if (runId) {
+    const run = activeRuns.get(runId);
+    if (run) return run;
+  }
+  return null;
+}
+
+/**
+ * Async version that also checks SQL — for cross-isolate access.
+ * Used by the /runs/active endpoint.
+ *
+ * Returns `heartbeatAt` so the client can independently decide a run is
+ * dead even before the server-side stale reap has fired. Returns
+ * `lastProgressAt` so the client-side stuck-detector can show a
+ * user-visible "this chat looks stuck" affordance when a run is alive
+ * (heartbeating) but not actually emitting events. Returns
+ * `awaitingRedispatch` so the client's background follow loop can tell a
+ * legitimately-deferred `chainServerDrivenContinuation` successor (recovery
+ * in progress server-side) apart from a genuinely dead run — see this
+ * field's own doc comment below.
+ */
+export async function getActiveRunForThreadAsync(threadId: string): Promise<{
+  runId: string;
+  threadId: string;
+  turnId: string;
+  status: string;
+  heartbeatAt: number;
+  lastProgressAt: number | null;
+  /** How the run was dispatched/continued (foreground, foreground-self-chain, background...). */
+  dispatchMode?: string | null;
+  /** Compact terminal classification, e.g. done, run_timeout, stale_run. */
+  terminalReason?: string | null;
+  /**
+   * Last reached `_process-run` worker stage as a JSON string
+   * `{stage,detail?,at}`. Surfaced so a silent background-worker death is
+   * diagnosable from the client WITHOUT the unreadable bg-fn logs.
+   */
+  diagStage?: string | null;
+  /**
+   * True exactly when this run is a `chainServerDrivenContinuation` deferral
+   * (dispatch_mode === 'background', never claimed) still inside
+   * `UNCLAIMED_BACKGROUND_RUN_REDISPATCH_BOUND_MS` — the same condition this
+   * function already uses below to skip its own `reapUnclaimedBackgroundRun`.
+   * Surfaced on `/runs/active` (agent-chat-plugin.ts) so
+   * `agent-chat-adapter.ts`'s follow loop can tell "silently deferred,
+   * server-side recovery in progress" apart from "dead" and stop counting the
+   * quiet gap against its idle timeout — see the THREE-SITE INVARIANT comment
+   * below and in agent-chat-plugin.ts / production-agent.ts. Always false for
+   * an in-memory run (that isolate IS the live producer) and for any run that
+   * isn't an unclaimed background dispatch.
+   */
+  awaitingRedispatch: boolean;
+  /**
+   * True exactly when this run's `in_flight_since` marker is set — a tool
+   * call or A2A `agent_call` delegation is open and has not yet resolved
+   * (see `setRunInFlightMarker` / `IN_FLIGHT_RUN_STALE_GRACE_MS` in
+   * run-store.ts). This is the SAME signal `reapIfStale` reads to grant its
+   * bounded stale-reap grace — computed here from the identical
+   * `in_flight_since` column via `getRunByThread`, never re-derived, so the
+   * client and the reaper cannot disagree about what "in flight" means.
+   *
+   * Surfaced on `/runs/active` (agent-chat-plugin.ts) as the
+   * server-authoritative alternative to the client-side proxy
+   * `RunStuckBanner` currently infers from unresolved `tool-call` content
+   * parts in the local message list — that proxy can go stale after a
+   * reconnect or reader-mode replay; this cannot, because it is read fresh
+   * from SQL on every poll.
+   */
+  hasInFlightWork: boolean;
+} | null> {
+  // Check memory first — return both running AND recently-completed runs
+  // that still have events in memory. This allows sub-agent tabs to replay
+  // the full conversation from completed runs via SSE.
+  const memRun = getActiveRunForThread(threadId);
+  if (memRun && (memRun.status === "running" || memRun.events.length > 0)) {
+    const sqlSnapshot = await fetchRunThreadSnapshot(memRun.runId, threadId);
+
+    // FIX 1 (durable-background incident): a terminal in-memory run (chunk
+    // completed at a soft-timeout/no-progress/loop-limit boundary, or any
+    // other terminal outcome) never clears `threadToRun` — it stays this
+    // thread's resident in-memory candidate for up to CLEANUP_DELAY_MS
+    // (5 min), and `fetchRunThreadSnapshot` above returns null the instant
+    // SQL's newest row for the thread is no longer THIS run (i.e. a
+    // successor already exists). Left alone, every poll that lands on this
+    // warm isolate would keep falling back to `memRun.status` below and
+    // never discover that a newer, still-running successor already exists
+    // in SQL — exactly the "stale terminal run masks a live successor" bug
+    // that produced the mid-sentence dead turn. Only pay for the extra SQL
+    // read here, in the terminal-candidate branch; the common "still
+    // running" poll (the vast majority) never reaches it.
+    if (!sqlSnapshot && memRun.status !== "running") {
+      const successor = await fetchNewerNonTerminalRunForSameTurn(
+        threadId,
+        memRun,
+      );
+      if (successor) {
+        return {
+          runId: successor.id,
+          threadId: successor.threadId,
+          turnId: successor.turnId ?? successor.id,
+          status: successor.status,
+          heartbeatAt: successor.heartbeatAt ?? successor.startedAt,
+          lastProgressAt: successor.lastProgressAt,
+          dispatchMode: successor.dispatchMode,
+          terminalReason: successor.terminalReason,
+          diagStage: successor.diagStage,
+          // Definitionally non-terminal and freshly read from SQL above —
+          // never the stale in-memory candidate's own state.
+          awaitingRedispatch: false,
+          hasInFlightWork: successor.inFlightSince != null,
+        };
+      }
+    }
+
+    const status = sqlSnapshot?.status ?? memRun.status;
+    const heartbeatAt =
+      status === "running"
+        ? Date.now()
+        : (sqlSnapshot?.heartbeatAt ?? memRun.startedAt);
+    return {
+      runId: memRun.runId,
+      threadId: memRun.threadId,
+      turnId: memRun.turnId,
+      status,
+      // In-memory means this isolate is the producer. By definition, the
+      // heartbeat is fresh as of "now" while the run is still running. Once
+      // SQL has terminal truth, prefer that timestamp so a stale in-memory
+      // buffer cannot keep the browser believing a finished background run is
+      // still alive.
+      heartbeatAt,
+      // For an in-memory run we don't have a separate "last event emit"
+      // timestamp tracked in JS — the SQL bump is throttled per-second.
+      // Read it back from SQL on demand. For the common case the SQL row
+      // is well under 1s old; if it isn't, the stuck-detector will pick
+      // it up on the next poll cycle.
+      lastProgressAt: sqlSnapshot?.lastProgressAt ?? null,
+      dispatchMode: sqlSnapshot?.dispatchMode ?? null,
+      terminalReason: sqlSnapshot?.terminalReason ?? null,
+      diagStage: sqlSnapshot?.diagStage ?? null,
+      // In-memory means this isolate is the live producer — never the
+      // "deferred, nobody producing" state this flag identifies.
+      awaitingRedispatch: false,
+      // Read from the SAME SQL snapshot the other fields above already read
+      // (`fetchRunThreadSnapshot` -> `getRunByThread`) rather than the
+      // producer's own in-memory `inFlightWorkCount` — this isolate IS the
+      // live producer, but there is no separate in-memory channel wired for
+      // that counter today, and the SQL marker is written on every 0<->N
+      // transition (see run-manager's `trackInFlightWork`), so it is at most
+      // one event behind here — the same tolerance `lastProgressAt` above
+      // already accepts.
+      hasInFlightWork: sqlSnapshot?.inFlightSince != null,
+    };
+  }
+  // Fall back to SQL — also surface recently terminated runs so the client
+  // can reconnect and replay synthesized done/error events instead of
+  // retrying the original POST. Without this, a POST that fails after the
+  // server already accepted (and finished) the run would re-execute the
+  // turn and double-apply mutations: the in-memory branch above already
+  // returns terminal runs whose events are still buffered, but the SQL
+  // path is the only authority once memory has been evicted.
+  try {
+    const sqlRun = await getRunByThread(threadId, { includeTerminal: true });
+    if (!sqlRun) return null;
+    if (sqlRun.status === "running") {
+      // FALLBACK HARDENING: a background-dispatched run that is still UNCLAIMED
+      // (dispatch_mode === 'background', never flipped to 'background-processing')
+      // past the tight grace means the bg-fn worker never started — a silent
+      // async-worker death that the 202-ack inline fallback can't catch. Reap it
+      // early and recoverably (background_worker_never_started) so the run no
+      // longer hangs for the full 90s window. Only fires when there is provably
+      // no live worker; a claimed/heartbeating run is left alone by the
+      // conditional SQL.
+      //
+      // REDISPATCH-BOUND GUARD (must be kept in lockstep with the "Unclaimed
+      // background-run sweep" in agent-chat-plugin.ts and with
+      // chainServerDrivenContinuation's deferral in production-agent.ts — do NOT
+      // remove this guard without reading those two sites):
+      // `chainServerDrivenContinuation` now DEFERS a dispatch-failed successor
+      // instead of erroring it — it leaves the row status='running',
+      // dispatch_mode='background' with its dispatch_payload intact so the sweep
+      // can silently redispatch it. This client poll runs every ~1s while a
+      // client is connected, so without this guard it would reap that deferred
+      // successor at the 25s unclaimed grace — long before the ~2-min sweep —
+      // converting the intended SILENT server-side recovery into a user-visible
+      // `background_worker_never_started` manual-retry error (that terminal
+      // reason does NOT auto-continue in the client follow loop; only `stale_run`
+      // does). While the successor is still inside its redispatch bound we skip
+      // this reap and leave it for the sweep. The outer backstops still bound it:
+      // `reapIfStale` below reaps a heartbeat-stale background row at 90s
+      // (BACKGROUND_RUN_STALE_MS) to the recoverable `stale_run` — which the
+      // follow loop AUTO-continues — and once the redispatch bound is exceeded
+      // this reap fires loudly as before. So recovery stays automatic in the
+      // common case and loud failure is only moved later, never removed.
+      //
+      // `isUnclaimedBackgroundDispatch` also becomes the `awaitingRedispatch`
+      // wire field below once the still-inside-the-bound check passes — see
+      // this function's doc comment and the THREE-SITE INVARIANT comment in
+      // agent-chat-plugin.ts / production-agent.ts.
+      const isUnclaimedBackgroundDispatch =
+        sqlRun.dispatchMode === "background";
+      const stillInsideRedispatchBound = shouldRedispatchUnclaimedBackgroundRun(
+        { startedAt: sqlRun.startedAt },
+      );
+      if (isUnclaimedBackgroundDispatch && !stillInsideRedispatchBound) {
+        const recovered = await reapUnclaimedBackgroundRun(sqlRun.id).catch(
+          () => false,
+        );
+        if (recovered) return null;
+      }
+      // If the producer is dead (no recent heartbeat), reap before the
+      // client can see a stale "running" status and enter a reconnect
+      // loop it can never exit.
+      const reaped = await reapIfStale(sqlRun.id).catch(() => false);
+      if (reaped) return null;
+      return {
+        runId: sqlRun.id,
+        threadId: sqlRun.threadId,
+        turnId: sqlRun.turnId ?? sqlRun.id,
+        status: sqlRun.status,
+        heartbeatAt: sqlRun.heartbeatAt ?? sqlRun.startedAt,
+        lastProgressAt: sqlRun.lastProgressAt,
+        dispatchMode: sqlRun.dispatchMode,
+        terminalReason: sqlRun.terminalReason,
+        diagStage: sqlRun.diagStage,
+        awaitingRedispatch:
+          isUnclaimedBackgroundDispatch && stillInsideRedispatchBound,
+        // Same `in_flight_since` column `reapIfStale` (just called above,
+        // and it did NOT reap this row) reads for its own grace decision —
+        // one source of truth, not a second re-derived notion of "in flight".
+        hasInFlightWork: sqlRun.inFlightSince != null,
+      };
+    }
+    if (sqlRun.status === "completed" || sqlRun.status === "errored") {
+      // Cap how far back we'll surface terminal runs as "active". The goal
+      // is to catch the recently-completed-but-reconnecting case, not to
+      // resurrect ancient turns when the user reopens an old thread.
+      //
+      // Measure age from the run's terminal timestamp, not its start. A
+      // long-running task that ran 11 minutes and completed five seconds
+      // ago should still be reachable — the client's disconnect happened
+      // around completion, so completion time is what matters for the
+      // "is the user still here waiting?" question. Fall back to the last
+      // heartbeat (older deployments may have unset completed_at) and
+      // finally to startedAt for ancient rows.
+      const referenceAt =
+        sqlRun.completedAt ?? sqlRun.heartbeatAt ?? sqlRun.startedAt;
+      const terminalAge = Date.now() - referenceAt;
+      if (terminalAge > TERMINAL_RUN_RECONNECT_WINDOW_MS) return null;
+      return {
+        runId: sqlRun.id,
+        threadId: sqlRun.threadId,
+        turnId: sqlRun.turnId ?? sqlRun.id,
+        status: sqlRun.status,
+        heartbeatAt: sqlRun.heartbeatAt ?? sqlRun.startedAt,
+        lastProgressAt: sqlRun.lastProgressAt,
+        dispatchMode: sqlRun.dispatchMode,
+        terminalReason: sqlRun.terminalReason,
+        diagStage: sqlRun.diagStage,
+        // Terminal already — never the "still deferred, running" state.
+        awaitingRedispatch: false,
+        // Terminal already — no live work can still be in flight.
+        hasInFlightWork: false,
+      };
+    }
+  } catch {
+    // SQL error — fall through
+  }
+  return null;
+}
+
+async function fetchRunThreadSnapshot(runId: string, threadId: string) {
+  try {
+    // `getRunById` returns a narrow projection today; ask for the row via
+    // the thread lookup which carries dispatch/terminal/progress fields.
+    const byThread = await getRunByThread(threadId, {
+      includeTerminal: true,
+    });
+    if (byThread && byThread.id === runId) return byThread;
+    return null;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * FIX 1 (durable-background incident): find a genuinely newer, still-running
+ * SQL row for the SAME turn as a terminal in-memory `ActiveRun` — used only
+ * when `fetchRunThreadSnapshot` found no SQL row matching the in-memory
+ * run's own id (i.e. SQL's newest row for the thread is a different run).
+ * `getRunByThread` always returns the thread's newest row by `started_at`,
+ * so this is the same read `fetchRunThreadSnapshot` already made; we just
+ * don't throw its result away when the id doesn't match.
+ *
+ * Scoped to the SAME `turnId` (not just the same thread) so an unrelated,
+ * later user turn on the same thread is never mistaken for a continuation
+ * successor of this one.
+ */
+async function fetchNewerNonTerminalRunForSameTurn(
+  threadId: string,
+  memRun: ActiveRun,
+): Promise<Awaited<ReturnType<typeof getRunByThread>> | null> {
+  try {
+    const latest = await getRunByThread(threadId, { includeTerminal: true });
+    if (
+      latest &&
+      latest.id !== memRun.runId &&
+      latest.status === "running" &&
+      latest.startedAt > memRun.startedAt &&
+      (latest.turnId ?? latest.id) === memRun.turnId
+    ) {
+      return latest;
+    }
+    return null;
+  } catch {
+    return null;
+  }
+}
+
+/** Get a run by ID */
+export function getRun(runId: string): ActiveRun | null {
+  return activeRuns.get(runId) ?? null;
+}
+
+function abortRunInMemory(runId: string, reason: string): boolean {
+  const run = activeRuns.get(runId);
+  if (run) {
+    abortInMemoryRun(run, reason);
+  }
+  return !!run;
+}
+
+/** Explicitly abort a run (e.g. Stop button). */
+export function abortRun(runId: string, reason: string = "user"): boolean {
+  const abortedInMemory = abortRunInMemory(runId, reason);
+  // Also mark as aborted in SQL (for cross-isolate abort on Workers)
+  markRunAborted(runId, reason).catch(() => {});
+  return abortedInMemory;
+}
+
+/**
+ * Abort a run and wait until the cross-isolate SQL state and terminal event
+ * are durable. Request handlers that start recovery immediately after aborting
+ * must use this path; otherwise the recovery POST can race the old row while
+ * it is still marked running.
+ */
+export async function abortRunDurably(
+  runId: string,
+  reason: string = "user",
+): Promise<boolean> {
+  const abortedInMemory = abortRunInMemory(runId, reason);
+  try {
+    await markRunAborted(runId, reason);
+  } catch (error) {
+    // The local run is already stopped. A transient durable cleanup failure
+    // must not turn the user's Stop/Retry request into a 500 after that
+    // irreversible in-memory outcome. Capture it for repair/reaping and let
+    // the request report the abort it did complete.
+    captureError(error, {
+      route: "/_agent-native/agent-chat/runs/:id/abort",
+      tags: {
+        source: "agent-run-manager",
+        phase: "abort-run",
+      },
+      extra: { runId, reason, abortedInMemory },
+    });
+    console.error(
+      "[run-manager] durable abort persistence failed:",
+      error instanceof Error ? error.message : error,
+    );
+  }
+  return abortedInMemory;
+}
+
+// Re-export so callers can avoid importing from run-store directly.
+export { tryClaimRunSlot } from "./run-store.js";