@agent-native/core 0.92.2 → 0.92.4

package/corpus/core/src/agent/engine/builder-engine.ts

@@ -485,11 +485,14 @@ async function* parseJsonlStream(
   let pendingText = "";
   let pendingThinking: { text: string; signature?: string } | null = null;
 
-  const flushPending = () => {
+  const flushPendingText = () => {
     if (pendingText) {
       parts.push({ type: "text", text: pendingText });
       pendingText = "";
     }
+  };
+
+  const flushPendingThinking = () => {
     if (pendingThinking) {
       parts.push({
         type: "thinking",
@@ -502,6 +505,11 @@ async function* parseJsonlStream(
     }
   };
 
+  const flushPending = () => {
+    flushPendingText();
+    flushPendingThinking();
+  };
+
   try {
     for await (const line of readJsonlLines(
       reader,
@@ -527,13 +535,16 @@ async function* parseJsonlStream(
       switch (event.type) {
         case "text-delta": {
           const text = event.text ?? "";
+          flushPendingThinking();
           pendingText += text;
           yield { type: "text-delta", text };
           break;
         }
 
-        case "thinking-delta": {
+        case "thinking-delta":
+        case "reasoning-delta": {
           const text = event.text ?? "";
+          flushPendingText();
           if (!pendingThinking) pendingThinking = { text: "" };
           pendingThinking.text += text;
           if (event.signature) pendingThinking.signature = event.signature;
@@ -645,10 +656,18 @@ async function* parseJsonlStream(
             const isCredentialAuthError =
               Boolean(explicitErrMsg) &&
               isBuilderCredentialAuthError(String(errMsg));
+            // Anthropic's bare "Connection error." often arrives here with no
+            // gateway code. Tag it as a network error so in-run retries and
+            // run-level resume treat it as transient instead of terminal.
+            const isProviderConnectionError =
+              typeof explicitErrMsg === "string" &&
+              isProviderConnectionErrorMessage(String(explicitErrMsg));
             const errCode = isCredentialAuthError
               ? "builder_auth_error"
-              : (gatewayErrCode ??
-                (!explicitErrMsg ? "builder_gateway_error" : undefined));
+              : isProviderConnectionError
+                ? BUILDER_GATEWAY_NETWORK_ERROR_CODE
+                : (gatewayErrCode ??
+                  (!explicitErrMsg ? "builder_gateway_error" : undefined));
             console.error(
               `[builder-engine] stop reason=error model=${model} code=${errCode ?? "(none)"} error=${errMsg}`,
             );
@@ -987,6 +1006,9 @@ function isBuilderGatewayNetworkError(err: unknown): boolean {
     text.includes("econnaborted") ||
     text.includes("fetch failed") ||
     text.includes("network error") ||
+    // Anthropic SDK's APIConnectionError default ("Connection error.") is
+    // often forwarded by the Builder gateway as a stop event with no code.
+    text.includes("connection error") ||
     text.includes("connection reset") ||
     text.includes("connection closed") ||
     text.includes("stream closed") ||
@@ -994,6 +1016,10 @@ function isBuilderGatewayNetworkError(err: unknown): boolean {
   );
 }
 
+function isProviderConnectionErrorMessage(message: string): boolean {
+  return message.trim().toLowerCase() === "connection error.";
+}
+
 function captureBuilderGatewayTransportError(
   err: unknown,
   context: {

package/corpus/core/src/agent/engine/registry.ts

@@ -121,6 +121,64 @@ export function isAgentEnginePackageInstalled(
   return packageNames.every(canResolvePackage);
 }
 
+interface ParsedVersionedModelId {
+  family: string;
+  version: number[];
+  suffix: string;
+}
+
+function parseVersionedModelId(model: string): ParsedVersionedModelId | null {
+  const match =
+    /^(?<family>.+?)[-.](?<version>\d+(?:[-.]\d+)*)(?<suffix>(?:[-.][a-z][a-z0-9]*)*)$/i.exec(
+      model.trim().toLowerCase(),
+    );
+  const groups = match?.groups;
+  if (!groups?.family || !groups.version) return null;
+
+  const version = groups.version.split(/[-.]/).map((part) => Number(part));
+  if (version.some((part) => !Number.isSafeInteger(part))) return null;
+
+  return {
+    family: groups.family,
+    version,
+    suffix: groups.suffix ?? "",
+  };
+}
+
+function compareModelVersions(left: number[], right: number[]): number {
+  const length = Math.max(left.length, right.length);
+  for (let index = 0; index < length; index += 1) {
+    const delta = (left[index] ?? 0) - (right[index] ?? 0);
+    if (delta !== 0) return delta;
+  }
+  return 0;
+}
+
+function findLatestSupportedVersionMatch(
+  candidate: string,
+  supportedModels: readonly string[],
+): string | undefined {
+  const parsedCandidate = parseVersionedModelId(candidate);
+  if (!parsedCandidate) return undefined;
+
+  let best: { model: string; version: number[] } | undefined;
+  for (const supportedModel of supportedModels) {
+    const parsedSupported = parseVersionedModelId(supportedModel);
+    if (!parsedSupported) continue;
+    if (parsedSupported.family !== parsedCandidate.family) continue;
+    if (parsedSupported.suffix !== parsedCandidate.suffix) continue;
+    if (
+      best &&
+      compareModelVersions(parsedSupported.version, best.version) <= 0
+    ) {
+      continue;
+    }
+    best = { model: supportedModel, version: parsedSupported.version };
+  }
+
+  return best?.model;
+}
+
 export function normalizeModelForEngine(
   engine: Pick<AgentEngine, "name" | "defaultModel" | "supportedModels">,
   model: string | null | undefined,
@@ -128,13 +186,16 @@ export function normalizeModelForEngine(
   const candidate = typeof model === "string" ? model.trim() : "";
   if (!candidate) return engine.defaultModel;
 
-  if (engine.name !== "builder") return candidate;
+  if (engine.supportedModels.length === 0) return candidate;
 
   if (candidate === "auto" || engine.supportedModels.includes(candidate)) {
     return candidate;
   }
 
-  return engine.supportedModels.includes("auto") ? "auto" : engine.defaultModel;
+  return (
+    findLatestSupportedVersionMatch(candidate, engine.supportedModels) ??
+    engine.defaultModel
+  );
 }
 
 function assertAgentEnginePackageInstalled(entry: AgentEngineEntry): void {

package/corpus/core/src/agent/model-config.ts

@@ -267,8 +267,6 @@ export const AGENT_MODEL_CONFIG = {
       "gemini-3-1-pro",
       "gemini-3-5-flash",
       "gemini-3-1-flash-lite",
-      "grok-code-fast",
-      "qwen3-coder",
     ],
   },
   anthropic: {

package/corpus/core/src/agent/production-agent.ts

@@ -1181,6 +1181,7 @@ export function isRetryableError(err: unknown): boolean {
   return (
     code === "builder_gateway_error" ||
     code === "builder_gateway_network_error" ||
+    code === "provider_network_error" ||
     code === "http_429" ||
     code === "http_500" ||
     code === "http_502" ||
@@ -1206,6 +1207,11 @@ export function isRetryableError(err: unknown): boolean {
     msg.includes("gateway error") ||
     msg.includes("socket hang up") ||
     msg.includes("connection reset") ||
+    // Anthropic SDK APIConnectionError default message is exactly
+    // "Connection error." — Builder gateway often forwards it as a stop
+    // event with no structured code. Without this match the run fails in
+    // ~3s and the client recovery loop storms POSTs.
+    msg.includes("connection error") ||
     msg.includes("too many requests") ||
     msg.includes("timeout") ||
     msg.includes("gateway timeout") ||
@@ -1853,7 +1859,8 @@ export function isResumableEngineError(err: unknown): boolean {
     err instanceof EngineError ? (err.errorCode ?? "").toLowerCase() : "";
   if (
     code === "builder_gateway_timeout" ||
-    code === "builder_gateway_network_error"
+    code === "builder_gateway_network_error" ||
+    code === "provider_network_error"
   ) {
     return true;
   }
@@ -1873,6 +1880,7 @@ export function isResumableEngineError(err: unknown): boolean {
     text.includes("econnaborted") ||
     text.includes("fetch failed") ||
     text.includes("network error") ||
+    text.includes("connection error") ||
     text.includes("connection reset") ||
     text.includes("connection closed") ||
     text.includes("stream closed") ||
@@ -4455,6 +4463,8 @@ function isRecoverableContinuationError(event: {
   return (
     event.recoverable === true ||
     code === "builder_gateway_timeout" ||
+    code === "builder_gateway_network_error" ||
+    code === "provider_network_error" ||
     code === "stale_run" ||
     code === "timeout" ||
     code === "timeout_error" ||
@@ -4463,6 +4473,7 @@ function isRecoverableContinuationError(event: {
     code === "http_529" ||
     code === "run_timeout" ||
     message.includes("timeout") ||
+    message.includes("connection error") ||
     message.includes("temporarily unavailable")
   );
 }

package/corpus/core/src/agent/run-manager.ts

@@ -18,9 +18,10 @@ import {
   reapUnclaimedBackgroundRun,
   reconcileTerminalRunFromEvents,
   ensureTerminalRunEvent,
+  getLastTerminalRunEvent,
+  resolveErroredRunTerminalEvent,
   setRunError,
   setRunTerminalReason,
-  STALE_RUN_ERROR_EVENT,
 } from "./run-store.js";
 import type { AgentChatEvent, RunEvent, RunStatus } from "./types.js";
 
@@ -681,28 +682,43 @@ export function startRun(
   // reap the row. Paired with RUN_STALE_MS (15s) — 10x the interval to
   // tolerate transient DB slowness without false positives.
   let consecutiveHeartbeatFailures = 0;
+  // Single-flight the heartbeat write. The timer fires every 1.5s but a write
+  // can take up to the DB op timeout (~8s) when the Neon pooler is saturated.
+  // Firing a fresh write each tick regardless piled up ~5 concurrent writes
+  // under contention, each holding a pooler connection — ADDING to the exact
+  // connection-cap exhaustion that starves the heartbeat and false-reaps the
+  // run as stale. Skip a tick's write while one is still outstanding so a run
+  // holds at most one heartbeat connection. The abort/backstop checks below
+  // still run every tick (they don't touch the DB on the hot path).
+  let heartbeatInFlight = false;
   const heartbeatTimer: ReturnType<typeof setInterval> = setInterval(() => {
-    updateRunHeartbeat(runId)
-      .then(() => {
-        consecutiveHeartbeatFailures = 0;
-      })
-      .catch((error) => {
-        consecutiveHeartbeatFailures += 1;
-        // Swallow routine single-tick blips; escalate once failures approach
-        // the stale window so false-positive stale_run from silent write
-        // failures is diagnosable.
-        if (consecutiveHeartbeatFailures >= 3) {
-          captureError(error, {
-            route: "/_agent-native/agent-chat",
-            tags: {
-              source: "agent-run-manager",
-              phase: "heartbeat",
-              consecutiveFailures: String(consecutiveHeartbeatFailures),
-            },
-            extra: { runId, threadId },
-          });
-        }
-      });
+    if (!heartbeatInFlight) {
+      heartbeatInFlight = true;
+      updateRunHeartbeat(runId)
+        .then(() => {
+          consecutiveHeartbeatFailures = 0;
+        })
+        .catch((error) => {
+          consecutiveHeartbeatFailures += 1;
+          // Swallow routine single-tick blips; escalate once failures approach
+          // the stale window so false-positive stale_run from silent write
+          // failures is diagnosable.
+          if (consecutiveHeartbeatFailures >= 3) {
+            captureError(error, {
+              route: "/_agent-native/agent-chat",
+              tags: {
+                source: "agent-run-manager",
+                phase: "heartbeat",
+                consecutiveFailures: String(consecutiveHeartbeatFailures),
+              },
+              extra: { runId, threadId },
+            });
+          }
+        })
+        .finally(() => {
+          heartbeatInFlight = false;
+        });
+    }
     checkSqlAbort();
     checkNoProgressBackstop();
   }, 1500);
@@ -1289,24 +1305,29 @@ function subscribeFromSQL(
                   return;
                 }
               } else if (run?.status === "errored") {
-                // The run row was flipped to `errored` but no terminal event
-                // was ever persisted — almost always means a reaper's silent
-                // `appendTerminalRunEvent(...).catch(() => {})` swallowed a
-                // transient DB error, so the user-facing situation is the
-                // same as a stale-run reap. Send the friendly event AND try
-                // to persist it so future reconnects replay it from SQL
-                // rather than regenerating it (the user used to see a bare
-                // "run_terminal_event_missing" debug string here).
-                await ensureTerminalRunEvent(
-                  runId,
-                  STALE_RUN_ERROR_EVENT,
-                ).catch(() => {});
+                // The run row is terminal but this subscriber's cursor is
+                // already past (or never saw) the terminal event. Prefer the
+                // REAL last terminal event / row error_detail over inventing
+                // a stale_run card — slides prod showed Connection error.
+                // rows being mislabeled as stale_run on reconnect because
+                // this path always synthesized STALE_RUN_ERROR_EVENT.
+                const existing = await getLastTerminalRunEvent(runId).catch(
+                  () => null,
+                );
+                const resolved = existing
+                  ? { event: existing.event, shouldPersist: false }
+                  : resolveErroredRunTerminalEvent(run);
+                if (resolved.shouldPersist) {
+                  await ensureTerminalRunEvent(runId, resolved.event).catch(
+                    () => {},
+                  );
+                }
                 try {
                   controller.enqueue(
                     encoder.encode(
                       `data: ${JSON.stringify({
-                        ...STALE_RUN_ERROR_EVENT,
-                        seq: lastSeq,
+                        ...resolved.event,
+                        seq: existing?.seq ?? lastSeq,
                       })}\n\n`,
                     ),
                   );

package/corpus/core/src/agent/run-store.ts

@@ -994,8 +994,13 @@ export async function recordRunDiagnostic(
 export async function updateRunHeartbeat(runId: string): Promise<void> {
   await ensureRunTables();
   const client = getDbExec();
+  // Only bump liveness while the row is still running. Zombie producers that
+  // keep their setInterval after status flips to errored/completed used to
+  // rewrite heartbeat_at for minutes after the turn died (seen on slides
+  // prod: heartbeat continued ~400s past completed_at), which confuses
+  // triage and can keep /runs/active looking "fresh" after failure.
   await client.execute({
-    sql: `UPDATE agent_runs SET heartbeat_at = ? WHERE id = ?`,
+    sql: `UPDATE agent_runs SET heartbeat_at = ? WHERE id = ? AND status = 'running'`,
     args: [Date.now(), runId],
   });
 }
@@ -1259,11 +1264,13 @@ export async function getRunById(runId: string): Promise<{
   threadId: string;
   status: string;
   startedAt: number;
+  errorCode: string | null;
+  errorDetail: string | null;
 } | null> {
   await ensureRunTables();
   const client = getDbExec();
   const { rows } = await client.execute({
-    sql: `SELECT id, thread_id, status, started_at FROM agent_runs WHERE id = ?`,
+    sql: `SELECT id, thread_id, status, started_at, error_code, error_detail FROM agent_runs WHERE id = ?`,
     args: [runId],
   });
   if (rows.length === 0) return null;
@@ -1272,15 +1279,87 @@ export async function getRunById(runId: string): Promise<{
     thread_id: string;
     status: string;
     started_at: number | string;
+    error_code?: string | null;
+    error_detail?: string | null;
   };
   return {
     id: r.id,
     threadId: r.thread_id,
     status: r.status,
     startedAt: Number(r.started_at),
+    errorCode: r.error_code ?? null,
+    errorDetail: r.error_detail ?? null,
   };
 }
 
+/**
+ * Read the latest terminal event already persisted for a run, if any.
+ * Used by SSE reconnect when the client cursor is already past that event
+ * (so `getRunEventsSince` returns empty) but the row is terminal — we must
+ * replay the REAL error instead of inventing a stale_run card.
+ */
+export async function getLastTerminalRunEvent(
+  runId: string,
+): Promise<{ seq: number; event: Record<string, unknown> } | null> {
+  await ensureRunTables();
+  const client = getDbExec();
+  const { rows } = await client.execute({
+    sql: `SELECT seq, event_data FROM agent_run_events WHERE run_id = ? ORDER BY seq DESC LIMIT 1`,
+    args: [runId],
+  });
+  const last = rows[0] as
+    | { seq?: number | string; event_data?: string }
+    | undefined;
+  if (!last?.event_data) return null;
+  try {
+    const parsed = JSON.parse(last.event_data) as Record<string, unknown>;
+    if (
+      parsed?.type === "done" ||
+      parsed?.type === "error" ||
+      parsed?.type === "missing_api_key" ||
+      parsed?.type === "loop_limit" ||
+      parsed?.type === "auto_continue"
+    ) {
+      return { seq: Number(last.seq ?? 0), event: parsed };
+    }
+  } catch {
+    return null;
+  }
+  return null;
+}
+
+/**
+ * Build the terminal error payload to stream when an `errored` run has no
+ * in-cursor terminal event. Prefer the real last terminal event, then the
+ * row's error_code/error_detail, and only then the generic stale_run card.
+ */
+export function resolveErroredRunTerminalEvent(run: {
+  errorCode?: string | null;
+  errorDetail?: string | null;
+}): {
+  event: Record<string, unknown>;
+  shouldPersist: boolean;
+} {
+  const code = typeof run.errorCode === "string" ? run.errorCode.trim() : "";
+  const detail =
+    typeof run.errorDetail === "string" ? run.errorDetail.trim() : "";
+  if (code === STALE_RUN_ERROR_EVENT.errorCode) {
+    return { event: { ...STALE_RUN_ERROR_EVENT }, shouldPersist: true };
+  }
+  if (detail || (code && code !== "unknown")) {
+    return {
+      event: {
+        type: "error",
+        error: detail || "The agent run failed.",
+        ...(code && code !== "unknown" ? { errorCode: code } : {}),
+        recoverable: true,
+      },
+      shouldPersist: true,
+    };
+  }
+  return { event: { ...STALE_RUN_ERROR_EVENT }, shouldPersist: true };
+}
+
 export async function getRunByThread(
   threadId: string,
   options?: { includeTerminal?: boolean },

package/corpus/core/src/agent/thread-data-builder.ts

@@ -3,6 +3,7 @@ import {
   normalizeCodeAgentTranscript,
   type CodeAgentTranscriptEvent as CoreCodeAgentTranscriptEvent,
   type NormalizedCodeAgentStatusEvent,
+  type NormalizedCodeAgentThinkingEvent,
   type NormalizedCodeAgentToolEvent,
   type NormalizedCodeAgentTranscriptItem,
 } from "../code-agents/transcript-normalizer.js";
@@ -124,6 +125,15 @@ export function buildAssistantMessage(
     }
   };
 
+  const appendReasoning = (text: string) => {
+    const last = content[content.length - 1];
+    if (last && last.type === "reasoning") {
+      last.text = (last.text ?? "") + text;
+    } else {
+      content.push({ type: "reasoning", text });
+    }
+  };
+
   for (const { event } of events) {
     if (event.type === "clear") {
       clearAssistantDraftContent(content);
@@ -135,6 +145,11 @@ export function buildAssistantMessage(
       continue;
     }
 
+    if (event.type === "thinking") {
+      appendReasoning(event.text ?? "");
+      continue;
+    }
+
     if (event.type === "tool_start") {
       toolCallCounter += 1;
       const toolCallId = runId
@@ -255,12 +270,18 @@ function clearAssistantDraftContent(content: ContentPart[]): void {
   for (let index = content.length - 1; index >= 0; index--) {
     const part = content[index];
     if (!part) continue;
-    if (part.type === "text") {
+    if (part.type === "text" || part.type === "reasoning") {
       content.splice(index, 1);
       continue;
     }
     if (part.type === "tool-call" && part.result === undefined) {
-      content.splice(index, 1);
+      // Keep materialized in-flight tool cards across retry clears so persisted
+      // thread rebuilds match the live SSE processor and avoid hide→show flicker.
+      const isEphemeral =
+        (part as { activity?: boolean }).activity === true ||
+        part.argsText === "" ||
+        Object.keys(part.args ?? {}).length === 0;
+      if (isEphemeral) content.splice(index, 1);
     }
   }
 }
@@ -1062,6 +1083,9 @@ function contentPartForCodeAgentTranscriptItem(
   if (item.type === "tool") {
     return toolContentPartForCodeAgentTranscriptItem(item);
   }
+  if (item.type === "thinking") {
+    return thinkingContentPartForCodeAgentTranscriptItem(item);
+  }
   if (item.type === "status") {
     const text = statusTextForCodeAgentTranscriptItem(item, options);
     return text ? { type: "text", text } : null;
@@ -1069,6 +1093,13 @@ function contentPartForCodeAgentTranscriptItem(
   return null;
 }
 
+function thinkingContentPartForCodeAgentTranscriptItem(
+  item: NormalizedCodeAgentThinkingEvent,
+): ContentPart | null {
+  const text = item.text.trim();
+  return text ? { type: "reasoning", text } : null;
+}
+
 function toolContentPartForCodeAgentTranscriptItem(
   item: NormalizedCodeAgentToolEvent,
 ): ContentPart {