@ixo/oracle-runtime-workers 0.4.0 → 0.9.0

package/src/do/contracts.ts

@@ -1,4 +1,4 @@
-import type { BotStatus } from '@ixo/matrix-bot-workers-sdk';
+import type { BotStatus, EncryptedFileInfo } from '@ixo/matrix-bot-workers-sdk';
 import type { AttachmentInput } from '../attachments/types';
 import type { RealtimeStatus } from '../realtime/realtime-endpoint';
 /**
@@ -107,6 +107,8 @@ export interface OracleWorkerEnv {
   /** Display markup for `GET /models` prices (default 1.6). */
   MODEL_PRICE_MARKUP?: string;
   MAIN_REASONING_EFFORT?: string;
+  /** LangGraph steps one turn may take before `GraphRecursionError` (default 600; Node hard-codes 200). */
+  TURN_RECURSION_LIMIT?: string;
   /** Platform model provider. Default `openrouter`; `nebius` for self-hosted. */
   LLM_PROVIDER?: 'openrouter' | 'nebius';
   /** Nebius Token Factory API key — required when `LLM_PROVIDER=nebius`. */
@@ -205,6 +207,21 @@ export interface TurnResult {
   messageId?: string;
   /** Tool calls made during the turn (name + summarized status), for logging. */
   toolCalls: Array<{ name: string; status: 'done' | 'error' }>;
+  /** The stored reply of a turn that had already finished (a gateway asked again after a reset). */
+  replayed?: boolean;
+}
+
+/**
+ * Prefix of the error a user object raises when a gateway asks again about a
+ * room message whose turn the object itself lost mid-run (see
+ * `src/do/matrix-turn-ledger.ts`): not re-run, the user is told to retry.
+ */
+export const TURN_INTERRUPTED_MARKER =
+  'turn interrupted by a runtime reset before it finished; not re-run';
+
+export function isInterruptedTurnError(err: unknown): boolean {
+  const message = err instanceof Error ? err.message : String(err);
+  return message.includes(TURN_INTERRUPTED_MARKER);
 }
 
 /** Row shape served by `GET /sessions`. */
@@ -341,6 +358,10 @@ export interface UserOracleObject extends Rpc.DurableObjectBranded {
   resetWorkingCopy(): Promise<{ reloadedFromOwnerStore: boolean }>;
   /** Raw upstream schema of the memory search tool as fetched right now. */
   debugMemorySchema(userDid: string): Promise<MemorySchemaDebug>;
+  /** Forcibly reset the object like a platform host drain (debug routes only). The call itself rejects. */
+  debugAbortObject(): Promise<void>;
+  /** Forget when the last re-authorise prompt was posted (debug routes only). */
+  debugResetReauthThrottle(): Promise<void>;
   /** Raw session row for `GET /debug/sessions/:id` (null when unknown). */
   debugSession(
     userDid: string,
@@ -354,7 +375,8 @@ export interface UserOracleObject extends Rpc.DurableObjectBranded {
    * Diagnostics: the object's current alarm and the user's task records — the
    * pair that answers "is this task going to fire, and when?".
    */
-  tasksStatus(): Promise<{
+  /** Task records, open runs and the alarm; boots the object when cold (`userDid` = the caller). */
+  tasksStatus(userDid?: string): Promise<{
     now: number;
     alarm: number | null;
     schedulerActive: boolean;
@@ -367,6 +389,15 @@ export interface UserOracleObject extends Rpc.DurableObjectBranded {
       consecutiveFailures: number;
       deliveryRoomId: string | null;
     }>;
+    /** Runs no incarnation has finished: what an alarm will re-deliver or close. */
+    openRuns: Array<{
+      runId: string;
+      taskId: string;
+      startedAt: string;
+      state: 'running' | 'delivering';
+      attempts: number;
+      retryAt: number | null;
+    }>;
   }>;
 }
 
@@ -388,17 +419,21 @@ export interface CreateRoomOptions {
  * rest is the oracle's.
  */
 export interface MatrixGatewayObject extends Rpc.DurableObjectBranded {
-  /** Bytes of a media event (decrypted when it carries an encrypted `file`); null when unknown. */
-  downloadEventMedia(
+  /**
+   * A media event's content as a stream, AS STORED: in an E2EE room the
+   * ciphertext plus the `EncryptedFile` fields to pipe through the SDK's
+   * `createAttachmentDecryptor` on the caller's side (a stream that errors
+   * on the far side of an RPC only surfaces here as a disconnect, so the
+   * hash check belongs to the consumer). Null when unknown or redacted.
+   */
+  downloadEventMediaStream(
     roomId: string,
     eventId: string,
-  ): Promise<{
-    bytes: Uint8Array;
-    mimetype?: string;
-    filename?: string;
-  } | null>;
-  /** Authenticated media download of an `mxc://` URI. */
-  downloadMxcMedia(mxc: string): Promise<Uint8Array>;
+  ): Promise<MediaStream | null>;
+  /** Authenticated media download of an `mxc://` URI, as a stream. */
+  downloadMxcMediaStream(mxc: string): Promise<ReadableStream<Uint8Array>>;
+  /** Forcibly reset the gateway object like a platform host drain (debug routes only). The call itself rejects. */
+  debugAbortObject(): Promise<void>;
   /** Stop the bot (operator/debug; `ensureStarted`/cron brings it back). */
   stop(): Promise<void>;
   /** Idempotent — starts the bot + keep-alive alarm if not running. */
@@ -451,8 +486,18 @@ export interface MatrixGatewayObject extends Rpc.DurableObjectBranded {
       txnId?: string;
     },
   ): Promise<string>;
-  /** Send an arbitrary timeline event (`content` JSON-encoded). Returns the event id. */
-  sendEvent(roomId: string, type: string, content: JsonString): Promise<string>;
+  /**
+   * Send an arbitrary timeline event (`content` JSON-encoded). Returns the
+   * event id. `txnId` pins the transaction id, as for `sendText`: the
+   * homeserver deduplicates by it per device, so a send retried after a lost
+   * response returns the original event instead of posting a second one.
+   */
+  sendEvent(
+    roomId: string,
+    type: string,
+    content: JsonString,
+    opts?: { txnId?: string },
+  ): Promise<string>;
   /** Resolve the canonical user↔oracle room for a user DID (alias lookup, cached). */
   resolveUserRoom(
     userDid: string,
@@ -465,17 +510,23 @@ export interface MatrixGatewayObject extends Rpc.DurableObjectBranded {
    * Returns the new event id. Wire-compatible with the Node runtime so
    * existing users migrate transparently.
    */
-  uploadUserSnapshot(
+  uploadUserSnapshotStream(
     userDid: string,
     storageKey: string,
-    bytes: Uint8Array,
+    /** The gzipped file; encrypted chunk by chunk in an E2EE room, never buffered. */
+    body: ReadableStream<Uint8Array>,
     filename: string,
+    /** Exact byte length of `body` (the homeserver needs a Content-Length). */
+    size: number,
   ): Promise<{ eventId: string }>;
-  /** Download the latest snapshot for `storageKey`, or null when none exists. */
-  downloadUserSnapshot(
+  /**
+   * The latest snapshot for `storageKey` as a stream, as stored (see
+   * `downloadEventMediaStream`), or null when none exists.
+   */
+  downloadUserSnapshotStream(
     userDid: string,
     storageKey: string,
-  ): Promise<{ bytes: Uint8Array; eventId: string } | null>;
+  ): Promise<SnapshotStream | null>;
   /** Read a room state event's content, JSON-encoded (state is never E2EE). Null when absent. */
   getRoomStateEvent(
     roomId: string,
@@ -533,10 +584,32 @@ export interface OutboxRow {
 }
 
 /** `MatrixBotDO.status()` plus the gateway's own turn bookkeeping. */
+/**
+ * Media handed across the Durable Object boundary as a stream, as stored on
+ * the homeserver: `file` is present when the event carried an encrypted
+ * attachment — pipe `stream` through the SDK's `createAttachmentDecryptor`
+ * with it; without `file` the stream is the plain upload. `size` is the
+ * sender's `info.size` when it set one.
+ */
+export interface MediaStream {
+  stream: ReadableStream<Uint8Array>;
+  file?: EncryptedFileInfo;
+  size?: number;
+  mimetype?: string;
+  filename?: string;
+}
+
+/** `MediaStream` of a user's snapshot, plus the media event that carries it. */
+export interface SnapshotStream extends MediaStream {
+  eventId: string;
+}
+
 export interface GatewayStatus extends BotStatus {
   turns: { inFlight: number; waiting: number };
   /** Debounce buffers waiting to become turns. */
   ingestPending: number;
+  /** Room messages whose turn has not ended yet (durable; replayed after a reset). */
+  inbox: number;
 }
 
 /** Derive the durable-object name for a user's object. */

package/src/do/matrix-turn-ledger.ts

@@ -0,0 +1,126 @@
+/**
+ * What the user object remembers about each room message it was asked to
+ * answer, so the gateway can ask again after a reset without the turn ever
+ * running twice (`src/matrix/inbox-store.ts` is the asking side).
+ *
+ * One row per Matrix event id, in the user's own SQLite next to the
+ * checkpoints: `started_at` is written before the turn runs, `reply_text` +
+ * `answered_at` when it has finished. A second request for the same event:
+ *
+ * - answered → the stored text is returned; no model, no tools;
+ * - running in this instance → the caller attaches to the running turn
+ *   (kept in memory by the object, not here);
+ * - started, not answered, not running → the object itself reset mid-turn;
+ *   tools may have run, so the turn is NOT re-run: the request is refused
+ *   with `TURN_INTERRUPTED_MARKER` and the gateway tells the user to try
+ *   again — the same thing that happens today when the object dies while
+ *   the gateway is alive;
+ * - unknown → run once.
+ *
+ * Rows older than `RETENTION_MS` are pruned at setup; a replay that late
+ * cannot happen (the gateway inbox gives up after `MAX_TURN_REPLAYS`).
+ */
+import type { DoSqliteDatabase } from '../sqlite/database';
+
+export interface MatrixTurnRecord {
+  eventId: string;
+  sessionId: string;
+  requestId: string;
+  startedAt: string;
+  answeredAt: string | null;
+  replyText: string | null;
+}
+
+export type MatrixTurnDecision = 'run' | 'answered' | 'interrupted';
+
+/** Rows are kept for a week; the gateway never replays anything that old. */
+export const RETENTION_MS = 7 * 24 * 3600 * 1000;
+
+export function decideMatrixTurn(
+  existing: MatrixTurnRecord | undefined,
+): MatrixTurnDecision {
+  if (!existing) return 'run';
+  return existing.replyText !== null ? 'answered' : 'interrupted';
+}
+
+type LedgerRow = {
+  event_id: string;
+  session_id: string;
+  request_id: string;
+  started_at: string;
+  answered_at: string | null;
+  reply_text: string | null;
+} & Record<string, string | null>;
+
+export class MatrixTurnLedger {
+  private setupPromise: Promise<void> | undefined;
+
+  constructor(
+    readonly db: DoSqliteDatabase,
+    private readonly now: () => number = () => Date.now(),
+  ) {}
+
+  setup(): Promise<void> {
+    this.setupPromise ??= this.doSetup().catch((error: unknown) => {
+      this.setupPromise = undefined;
+      throw error;
+    });
+    return this.setupPromise;
+  }
+
+  private async doSetup(): Promise<void> {
+    await this.db.run(`
+      CREATE TABLE IF NOT EXISTS matrix_turns (
+        event_id TEXT PRIMARY KEY,
+        session_id TEXT NOT NULL,
+        request_id TEXT NOT NULL,
+        started_at TEXT NOT NULL,
+        answered_at TEXT,
+        reply_text TEXT
+      )`);
+    await this.db.run(`DELETE FROM matrix_turns WHERE started_at < ?`, [
+      new Date(this.now() - RETENTION_MS).toISOString(),
+    ]);
+  }
+
+  async get(eventId: string): Promise<MatrixTurnRecord | undefined> {
+    await this.setup();
+    const row = await this.db.get<LedgerRow>(
+      `SELECT event_id, session_id, request_id, started_at, answered_at, reply_text
+       FROM matrix_turns WHERE event_id = ?`,
+      [eventId],
+    );
+    if (!row) return undefined;
+    return {
+      eventId: row.event_id,
+      sessionId: row.session_id,
+      requestId: row.request_id,
+      startedAt: row.started_at,
+      answeredAt: row.answered_at,
+      replyText: row.reply_text,
+    };
+  }
+
+  /** Mark the turn as started. Durable before any model or tool runs. */
+  async start(
+    eventId: string,
+    sessionId: string,
+    requestId: string,
+  ): Promise<void> {
+    await this.setup();
+    await this.db.run(
+      `INSERT OR IGNORE INTO matrix_turns (event_id, session_id, request_id, started_at)
+       VALUES (?, ?, ?, ?)`,
+      [eventId, sessionId, requestId, new Date(this.now()).toISOString()],
+    );
+  }
+
+  /** Record the reply; from now on the event is answered, never re-run. */
+  async answer(eventId: string, replyText: string): Promise<void> {
+    await this.setup();
+    await this.db.run(
+      `UPDATE matrix_turns SET reply_text = ?, answered_at = ? WHERE event_id = ?`,
+      [replyText, new Date(this.now()).toISOString(), eventId],
+    );
+  }
+}

package/src/do/reauth-prompt.ts

@@ -0,0 +1,76 @@
+/**
+ * The `ixo.oracle.delegation_required` prompt: when a room message arrives
+ * and the user has no usable delegation, the object posts an event the web
+ * app listens for to open its "authorize for Matrix" modal in place. It is
+ * throttled per user (`UCAN_REAUTH_PROMPT_THROTTLE_SECONDS`) so the room is
+ * not spammed while the user is away.
+ *
+ * The throttle is stamped only AFTER the homeserver accepted the event. The
+ * earlier order — stamp, then send — turned a send lost to a gateway restart
+ * into six hours of silence: the user kept messaging an oracle that could
+ * not act, and nothing told them why. The send is retried across a restart
+ * under one transaction id, so a response lost after the homeserver accepted
+ * the event is deduplicated rather than posted twice; a second turn arriving
+ * while a prompt is in flight joins it instead of posting another.
+ */
+import { retryTxnId } from '../matrix/txn-id';
+import { retryGateway, type RetryGatewayOptions } from './gateway-retry';
+
+export interface ReauthPromptDeps {
+  throttleMs: number;
+  /** When the last prompt was accepted (undefined = never). */
+  getStamp(): Promise<number | undefined>;
+  setStamp(at: number): Promise<void>;
+  /** Post the event under the transaction id; resolves to its event id. */
+  send(roomId: string, txnId: string): Promise<string>;
+  /** Keeps the (un-awaited) prompt alive past the request that started it: `ctx.waitUntil`. */
+  keepAlive(work: Promise<unknown>): void;
+  log(message: string): void;
+  warn(message: string): void;
+  now?: () => number;
+  retry?: Pick<RetryGatewayOptions, 'delaysMs' | 'sleep' | 'isTransient'>;
+}
+
+const errorText = (err: unknown): string =>
+  err instanceof Error ? err.message : String(err);
+
+export class ReauthPrompter {
+  private inFlight: Promise<void> | null = null;
+
+  constructor(private readonly deps: ReauthPromptDeps) {}
+
+  /** Prompt unless throttled or already in flight. Never rejects. */
+  prompt(userDid: string, roomId: string): Promise<void> {
+    if (this.inFlight) return this.inFlight;
+    const run = this.run(userDid, roomId)
+      .catch((err: unknown) => {
+        this.deps.warn(
+          `[user-do] delegation-required event failed for ${userDid}: ${errorText(err)}`,
+        );
+      })
+      .finally(() => {
+        this.inFlight = null;
+      });
+    this.inFlight = run;
+    this.deps.keepAlive(run);
+    return run;
+  }
+
+  private async run(userDid: string, roomId: string): Promise<void> {
+    const now = this.deps.now?.() ?? Date.now();
+    const last = await this.deps.getStamp();
+    if (last !== undefined && now - last < this.deps.throttleMs) return;
+    const txnId = retryTxnId('reauth');
+    const eventId = await retryGateway(() => this.deps.send(roomId, txnId), {
+      ...this.deps.retry,
+      onRetry: (err, attempt, delayMs) =>
+        this.deps.warn(
+          `[user-do] delegation-required event send failed (attempt ${attempt}) — retrying in ${delayMs} ms: ${errorText(err)}`,
+        ),
+    });
+    await this.deps.setStamp(this.deps.now?.() ?? Date.now());
+    this.deps.log(
+      `[user-do] sent ixo.oracle.delegation_required (${eventId}) to ${roomId} for ${userDid}`,
+    );
+  }
+}

package/src/do/room-mirror.ts

@@ -0,0 +1,111 @@
+/**
+ * The room mirror of HTTP chats: every Portal turn is replayed into the
+ * user's oracle room as a thread under the session's marker event, the user
+ * message when the turn starts and the reply when it ends (Node's
+ * `MessagesService`). Node fires these and forgets them; on Workers the
+ * gateway object is replaced on every deploy and can be drained at any time,
+ * so a mirror sent at that instant fails with a transport error and, unless
+ * retried, is gone for good — the room transcript then disagrees with the
+ * Portal's for ever.
+ *
+ * Two rules make the retry safe:
+ * - each send carries a transaction id derived from the session, the request
+ *   and the author, so a send whose response was lost is deduplicated by the
+ *   homeserver rather than posted twice;
+ * - sends are serialised per session (parallel across sessions), so a reply
+ *   being retried can never overtake the message it answers, and a later
+ *   turn's mirrors queue behind an earlier turn's.
+ */
+import { matrixTxnId } from '../matrix/txn-id';
+import { retryGateway, type RetryGatewayOptions } from './gateway-retry';
+
+export type MirrorAuthor = 'user' | 'oracle';
+
+export function mirrorTxnId(
+  sessionId: string,
+  requestId: string,
+  who: MirrorAuthor,
+): string {
+  return matrixTxnId(
+    'replay',
+    sessionId,
+    requestId,
+    who === 'user' ? 'u' : 'o',
+  );
+}
+
+export interface MirrorSend {
+  roomId: string;
+  body: string;
+  formattedBody?: string;
+  /** The session's marker event — the thread the mirror is posted under. */
+  threadId: string;
+  txnId: string;
+}
+
+export interface RoomMirrorDeps {
+  sendText(send: MirrorSend): Promise<string>;
+  /** Keeps the (un-awaited) send alive past the request that started it: `ctx.waitUntil`. */
+  keepAlive(work: Promise<unknown>): void;
+  log(message: string): void;
+  warn(message: string): void;
+  /** Retry policy override (tests shrink the delays). */
+  retry?: Pick<RetryGatewayOptions, 'delaysMs' | 'sleep' | 'isTransient'>;
+}
+
+const errorText = (err: unknown): string =>
+  err instanceof Error ? err.message : String(err);
+
+export class RoomMirror {
+  /** Per session: the tail of its send chain. Removed once the chain drains. */
+  private readonly chains = new Map<string, Promise<void>>();
+
+  constructor(private readonly deps: RoomMirrorDeps) {}
+
+  /** Sessions with a mirror still in flight. */
+  get pending(): number {
+    return this.chains.size;
+  }
+
+  /**
+   * Queue one mirror behind the session's previous ones. `prepare` runs when
+   * the session's earlier mirrors have settled and resolves what to send
+   * (`null` = nothing to mirror, e.g. no room). Never rejects: a mirror that
+   * cannot be posted is logged and the chain moves on.
+   */
+  enqueue(
+    sessionId: string,
+    prepare: () => Promise<MirrorSend | null>,
+    label: string,
+  ): Promise<void> {
+    const previous = this.chains.get(sessionId) ?? Promise.resolve();
+    const tracked: Promise<void> = previous
+      .then(() => prepare())
+      .then((send) => (send ? this.send(send, label) : undefined))
+      .catch((err: unknown) => {
+        this.deps.warn(
+          `[user-do] Matrix replay (${label}) failed — session=${sessionId}: ${errorText(err)}`,
+        );
+      })
+      .finally(() => {
+        if (this.chains.get(sessionId) === tracked)
+          this.chains.delete(sessionId);
+      });
+    this.chains.set(sessionId, tracked);
+    this.deps.keepAlive(tracked);
+    return tracked;
+  }
+
+  private async send(send: MirrorSend, label: string): Promise<void> {
+    const eventId = await retryGateway(() => this.deps.sendText(send), {
+      ...this.deps.retry,
+      onRetry: (err, attempt, delayMs) =>
+        this.deps.warn(
+          `[user-do] Matrix replay (${label}) send failed (attempt ${attempt}) — retrying in ${delayMs} ms with the same transaction id: ${errorText(err)}`,
+        ),
+    });
+    this.deps.log(
+      `[user-do] Matrix replay (${label}) → ${eventId} in thread ${send.threadId}`,
+    );
+  }
+}

package/src/do/sse-stream.ts

@@ -15,6 +15,7 @@ import {
   redactOperatorFault,
 } from '../llm/provider-error';
 import type { AIMessageChunk, ToolMessage } from '@langchain/core/messages';
+import { isToolMessage } from '@langchain/core/messages';
 
 export interface SseTurnRunnerInput {
   /** `agent.streamEvents(stateInput, config)` async iterable. */
@@ -75,9 +76,30 @@ interface ToolCallPayload {
   requestId: string;
   toolName: string;
   args: Record<string, unknown>;
-  status: 'isRunning' | 'done';
+  status: 'isRunning' | 'done' | 'error';
   output?: string;
   eventId?: string;
+  /** Set with `status: 'error'`: what the tool reported (schema rejection, retries exhausted). */
+  error?: string;
+}
+
+/**
+ * Model calls the agent makes for itself — the summarization middleware
+ * condensing the thread, a sub-agent's inner turn — stream through the same
+ * `streamEvents` pipe as the user-facing reply. Only the outermost agent's
+ * model output belongs on the wire; everything else is bookkeeping the user
+ * never asked to read (the "Here is a summary of the conversation to date"
+ * leak). The summarizer tags its invoke with `lc_source: 'summarization'`;
+ * sub-agents run under their own run name below the main agent.
+ */
+export function isInternalModelEvent(evt: {
+  metadata?: Record<string, unknown>;
+  tags?: string[];
+}): boolean {
+  const source = evt.metadata?.['lc_source'];
+  if (source === 'summarization') return true;
+  const tags = evt.tags ?? [];
+  return tags.some((t) => t === 'summarization' || t === 'internal');
 }
 
 interface ActionCallPayload {
@@ -230,6 +252,32 @@ export function createSseTurnStream(
         toolCallMap.clear();
       };
 
+      // Tool calls the main agent's model produced, by call id, until their
+      // tool run starts. A call whose arguments fail the tool's schema never
+      // starts (LangChain validates before the tool's start callback), so
+      // `on_tool_start` / `on_tool_end` never fire for it; the error tool
+      // message the tools node returns is the only trace. Anything still
+      // recorded when that node ends is reported from that message.
+      const pendingCalls = new Map<
+        string,
+        { toolName: string; args: Record<string, unknown> }
+      >();
+      const sameArgs = (a: unknown, b: unknown) => {
+        try {
+          return JSON.stringify(a) === JSON.stringify(b);
+        } catch {
+          return false;
+        }
+      };
+      const forgetStartedCall = (toolName: string, args: unknown) => {
+        for (const [id, call] of pendingCalls) {
+          if (call.toolName === toolName && sameArgs(call.args, args)) {
+            pendingCalls.delete(id);
+            return;
+          }
+        }
+      };
+
       const run = async () => {
         let fullContent = '';
         try {
@@ -240,12 +288,59 @@ export function createSseTurnStream(
               run_id: string;
               name?: string;
               data: unknown;
+              metadata?: Record<string, unknown>;
+              tags?: string[];
             };
+            if (evt.event === 'on_chat_model_end') {
+              if (isInternalModelEvent(evt)) continue;
+              const output = (evt.data as { output?: AIMessageChunk })?.output;
+              for (const call of output?.tool_calls ?? []) {
+                if (call.id)
+                  pendingCalls.set(call.id, {
+                    toolName: call.name,
+                    args: (call.args ?? {}) as Record<string, unknown>,
+                  });
+              }
+              continue;
+            }
+            if (evt.event === 'on_chain_end' && evt.name === 'tools') {
+              if (pendingCalls.size === 0) continue;
+              const output = (evt.data as { output?: unknown })?.output;
+              const messages: unknown[] = Array.isArray(output)
+                ? output
+                : Array.isArray((output as { messages?: unknown[] })?.messages)
+                  ? ((output as { messages: unknown[] }).messages ?? [])
+                  : [];
+              for (const message of messages) {
+                if (!isToolMessage(message)) continue;
+                const call = pendingCalls.get(message.tool_call_id);
+                if (!call) continue;
+                pendingCalls.delete(message.tool_call_id);
+                const text = toText(message.content);
+                const failed = message.status === 'error';
+                write('tool_call', {
+                  sessionId,
+                  requestId,
+                  toolName: message.name ?? call.toolName,
+                  args: {
+                    ...call.args,
+                    toolName: message.name ?? call.toolName,
+                  },
+                  status: failed ? 'error' : 'done',
+                  output: text,
+                  ...(failed && { error: text || 'Tool call rejected' }),
+                  eventId: message.tool_call_id,
+                });
+              }
+              pendingCalls.clear();
+              continue;
+            }
             if (evt.event === 'on_tool_start') {
               const toolName = evt.name ?? 'tool';
               const args = extractToolArgs(
                 (evt.data as { input?: unknown })?.input,
               );
+              forgetStartedCall(toolName, args);
               if (input.agActionNames?.has(toolName)) {
                 const payload: ActionCallPayload = {
                   requestId,
@@ -295,10 +390,16 @@ export function createSseTurnStream(
               }
               const tool = toolCallMap.get(evt.run_id);
               if (tool) {
+                // A tool whose call was rejected (schema mismatch) or whose
+                // retries ran out comes back as a ToolMessage with
+                // `status: 'error'`, not as `on_tool_error`; the client must
+                // see it as failed, not as a finished call with odd output.
+                const failed = output?.status === 'error';
                 write('tool_call', {
                   ...tool,
-                  status: 'done',
+                  status: failed ? 'error' : 'done',
                   output: text,
+                  ...(failed && { error: text || 'Tool failed' }),
                   eventId: evt.run_id,
                   args: {
                     ...tool.args,
@@ -342,6 +443,7 @@ export function createSseTurnStream(
               continue;
             }
             if (evt.event === 'on_chat_model_stream') {
+              if (isInternalModelEvent(evt)) continue;
               const chunk = (evt.data as { chunk?: AIMessageChunk })?.chunk;
               if (!chunk) continue;
               const raw = chunk.additional_kwargs?.__raw_response as