@anteros/core 0.0.44 → 0.0.47

package/lib/agent.ts

@@ -1,10 +1,13 @@
 import { AppError, fn } from "./error";
+import { DEFAULT_REDACT_KEYS, isSensitiveKey, redactKeys } from "./redact";
 import { toJsonSchema, validateWithSchema } from "./jsonSchema";
 import { InMemoryAgentMemory, trimMessages } from "./agentMemory";
 import { forMemory, resolveAttachments } from "./attachment";
-import { normalizeMessages, serializeToolResult } from "./messages";
+import { normalizeMessages, serializeToolResult, toolResultMedia } from "./messages";
 import { mcpToolsFor } from "./mcpClient";
+import { parsePartialJson } from "./partialJson";
 import { skillsBlock } from "./skills";
+import { trace } from "./tracing";
 import {
     EMPTY_USAGE,
     resolveProvider,
@@ -29,12 +32,15 @@ import type {
     AgentMessage,
     AgentOptions,
     AgentPendingApproval,
+    AgentRunState,
+    AgentRunStore,
     AgentSkill,
     AgentTextPart,
     AgentThinkingBlock,
     AgentThread,
     AgentProvider,
     AgentStreamChunk,
+    AgentStreamObjectResult,
     AgentStreamResult,
     AgentStep,
     AgentStructuredOutput,
@@ -43,6 +49,10 @@ import type {
     AgentToolContext,
     AgentToolMessage,
     AgentToolResultEntry,
+    AgentGuardrail,
+    AgentGuardrailContext,
+    AgentPricing,
+    AgentToolStats,
     AgentTools,
     AgentUsage,
     AgentWorkingMemory,
@@ -264,11 +274,13 @@ async function applyProcessors(
 function addUsage(base: AgentUsage, next: Partial<AgentUsage>): AgentUsage {
     const inputTokens = base.inputTokens + (next.inputTokens ?? 0);
     const outputTokens = base.outputTokens + (next.outputTokens ?? 0);
+    const cost = (base.cost ?? 0) + (next.cost ?? 0);
     return {
         inputTokens,
         outputTokens,
         totalTokens: inputTokens + outputTokens,
         requests: base.requests + (next.requests ?? 0),
+        ...(cost ? { cost } : {}),
     };
 }
 
@@ -328,6 +340,19 @@ function extractJson(text: string): { value?: any; error?: string } {
     }
 }
 
+/**
+ * The object of a running answer — `undefined` until the text parses, a partial
+ * parse after that. It **drains** the run's text: a caller picks
+ * `partialObjectStream` **or** `textStream`, never both.
+ */
+async function* partialObjects<T>(text: AsyncIterable<string>): AsyncIterable<T | undefined> {
+    let buffer = "";
+    for await (const token of text) {
+        buffer += token;
+        yield parsePartialJson(buffer) as T | undefined;
+    }
+}
+
 // ─── Default memory ──────────────────────────────────────────────────────
 // The stores live in `lib/agentMemory.ts` (`agents.memory.InMemory` / `.Mongo`).
 
@@ -359,6 +384,142 @@ async function withTimeout<T>(work: Promise<T>, ms: number, name: string): Promi
     }
 }
 
+/** Run `work` over `items` with at most `limit` in flight, preserving order. */
+async function mapLimited<T, R>(items: T[], limit: number, work: (item: T, index: number) => Promise<R>): Promise<R[]> {
+    const out = new Array<R>(items.length);
+    let next = 0;
+    const workers = Array.from({ length: Math.min(limit, items.length) }, async () => {
+        while (true) {
+            const index = next++;
+            if (index >= items.length) return;
+            out[index] = await work(items[index]!, index);
+        }
+    });
+    await Promise.all(workers);
+    return out;
+}
+
+/** A prompt budget is an order of magnitude: 4 chars ≈ 1 token, no tokenizer dependency. */
+function estimateTokens(message: AgentMessage): number {
+    const content = (message as any).content;
+    const text = typeof content === "string" ? content : JSON.stringify(content ?? "");
+    return Math.ceil((text?.length ?? 0) / 4) + 4;
+}
+
+/** Drop the oldest turns until the prompt fits `budget`; the last turn always stays. */
+function trimToTokens(messages: AgentMessage[], budget: number): AgentMessage[] {
+    let total = messages.reduce((sum, message) => sum + estimateTokens(message), 0);
+    let start = 0;
+    while (total > budget && start < messages.length - 1) {
+        total -= estimateTokens(messages[start]!);
+        start += 1;
+    }
+    return start ? messages.slice(start) : messages;
+}
+
+/** The text a guardrail reads — a message's text parts joined. */
+function messageText(message: AgentMessage | undefined): string {
+    if (!message) return "";
+    const content = (message as any).content;
+    if (typeof content === "string") return content;
+    return (content ?? [])
+        .filter((part: any) => part?.type === "text")
+        .map((part: any) => part.text)
+        .join("\n");
+}
+
+/**
+ * Run one side of the guardrails. A check that throws **blocks the run** with
+ * `AGENT_GUARDRAIL_BLOCKED`, naming the guardrail and carrying its own code.
+ */
+async function runGuardrails(
+    list: AgentGuardrail[] | undefined,
+    value: string,
+    ctx: AgentGuardrailContext,
+    kind: "prompt" | "answer",
+): Promise<void> {
+    for (const guardrail of list ?? []) {
+        try {
+            await guardrail.check(value, ctx);
+        } catch (err: any) {
+            throw new AppError(
+                `Guardrail '${guardrail.name}' blocked the ${kind}: ${err?.message ?? err}`,
+                { status: 400, code: "AGENT_GUARDRAIL_BLOCKED", reason: err?.code ? String(err.code) : undefined },
+            );
+        }
+    }
+}
+
+/**
+ * Check a **successful** tool result before the model reads it. A guardrail that
+ * throws blocks **that result only**: it becomes an error the model sees
+ * (`AGENT_GUARDRAIL_BLOCKED`), and the run goes on — a tool result is a turn of
+ * the conversation, not the answer. A failed result is not checked.
+ */
+async function guardToolResult(
+    entry: AgentToolResultEntry,
+    list: AgentGuardrail[],
+    ctx: AgentGuardrailContext,
+): Promise<void> {
+    if (entry.error || !list.length) return;
+
+    const value = serializeToolResult(entry);
+    const toolCtx: AgentGuardrailContext = { ...ctx, tool: entry.name, toolCallId: entry.id };
+
+    for (const guardrail of list) {
+        try {
+            await guardrail.check(value, toolCtx);
+        } catch (err: any) {
+            entry.error = `Guardrail '${guardrail.name}' blocked the tool result: ${err?.message ?? err}`;
+            entry.errorCode = "AGENT_GUARDRAIL_BLOCKED";
+            delete entry.result;
+            return;
+        }
+    }
+}
+
+/** What a model call cost, in USD, from its token usage. */
+function costOf(usage: AgentUsage, pricing: AgentPricing): number {
+    return (usage.inputTokens / 1_000_000) * pricing.input + (usage.outputTokens / 1_000_000) * pricing.output;
+}
+
+/** Whether another endpoint may answer — 429, 5xx, network, timeout; never a 4xx. */
+function worthFallback(err: any): boolean {
+    // The adapter reports the provider's own status under `reason.status`: a 400 from
+    // the upstream endpoint is the request's fault, and another model would fail too.
+    const providerStatus = Number(err?.reason?.status ?? 0);
+    if (providerStatus) return providerStatus === 429 || providerStatus >= 500;
+
+    const status = Number(err?.status ?? 0);
+    if (status === 429 || status === 504 || status >= 500) return true;
+    if (status) return false;
+
+    const text = String(err?.message ?? err);
+    return /network|fetch failed|timed out|timeout|ECONN|socket|ENOTFOUND|EAI_AGAIN/i.test(text);
+}
+
+/** Fold one tool result into the run's aggregate (`result.toolStats`). */
+function recordToolStats(stats: AgentToolStats, entry: AgentToolResultEntry): void {
+    stats.calls += 1;
+    stats.durationMs += entry.durationMs;
+    if (!entry.error) return;
+    stats.errors += 1;
+    const code = entry.errorCode ?? "AGENT_TOOL_FAILED";
+    stats.byCode[code] = (stats.byCode[code] ?? 0) + 1;
+    if (code === "AGENT_TOOL_TIMEOUT") stats.timeouts += 1;
+}
+
+/** Fold a delegated run's stats into its parent's — the calls are the caller's too. */
+function mergeToolStats(base: AgentToolStats, next: AgentToolStats): void {
+    base.calls += next.calls;
+    base.errors += next.errors;
+    base.timeouts += next.timeouts;
+    base.durationMs += next.durationMs;
+    for (const [code, count] of Object.entries(next.byCode)) {
+        base.byCode[code] = (base.byCode[code] ?? 0) + count;
+    }
+}
+
 /**
  * The tool calls the conversation has not answered yet — the ones of its last
  * assistant turn that carries calls, minus the `tool` turns that follow. That is
@@ -380,11 +541,26 @@ function unansweredToolCalls(messages: AgentMessage[]): AgentToolCall[] {
 
 /** The `tool` turn a result becomes in the conversation — one shape, one place. */
 function toolTurnFor(result: AgentToolResultEntry): AgentMessage {
+    const media = toolResultMedia(result);
+    if (!media.length) {
+        return {
+            role: "tool",
+            toolCallId: result.id,
+            name: result.name,
+            content: serializeToolResult(result),
+            isError: !!result.error,
+        };
+    }
+    // Media travels **beside** the text: Anthropic accepts image/document blocks in a
+    // `tool_result`; a text-only protocol gets the marker `serializeToolResult` built.
+    const text = serializeToolResult(result);
+    const content: any[] = text ? [{ type: "text", text }] : [];
+    content.push(...media);
     return {
         role: "tool",
         toolCallId: result.id,
         name: result.name,
-        content: serializeToolResult(result),
+        content,
         isError: !!result.error,
     };
 }
@@ -396,10 +572,24 @@ function toolTurnFor(result: AgentToolResultEntry): AgentMessage {
  * The sub-agent is built per call, bound to the **calling** `rest`: its own tools then
  * run with the right tenant, session and transaction.
  */
+/** What a delegated run must inherit from its parent. */
+type Delegation = {
+    signal?: AbortSignal;
+    maxSteps?: number;
+    /**
+     * The parent's `emit`, set when the parent **streams**: the child then streams
+     * too, and every one of its chunks is re-emitted wrapped as `subagent`.
+     */
+    emit?: (chunk: AgentStreamChunk) => void;
+    /** Folds the child's usage and tool stats into the parent's. */
+    onResult?: (result: { usage: AgentUsage; toolStats: AgentToolStats }) => void;
+};
+
 function subAgentTools(
     declared: AgentConfig | AgentConfig[] | undefined,
     rest: InstanceType<typeof useRest> | undefined,
     tenant: string | undefined,
+    delegation?: Delegation,
 ): Array<[string, AgentTool]> {
     const definitions = !declared ? [] : Array.isArray(declared) ? declared : [declared];
     const out: Array<[string, AgentTool]> = [];
@@ -421,9 +611,57 @@ function subAgentTools(
                 required: ["input"],
                 additionalProperties: false,
             },
-            execute: async ({ input, thread, resource }: any) => {
+            execute: async ({ input, thread, resource }: any, ctx) => {
+                if (!definition.description) {
+                    throw ctx.error(`Agent '${id}' has no description — it cannot be delegated to`, {
+                        status: 500, code: "AGENT_DELEGATION_INVALID",
+                    });
+                }
                 const sub = new Agent(definition, { rest, tenant });
-                const answer = await sub.generate(input, { thread, resource });
+                // The parent's lifetime is the child's: an aborted run must not
+                // leave a delegated model call running in the background
+                const inherited = {
+                    thread,
+                    resource,
+                    ...(delegation?.signal ? { signal: delegation.signal } : {}),
+                    ...(delegation?.maxSteps ? { maxSteps: delegation.maxSteps } : {}),
+                };
+
+                // A streaming parent gets a streaming child: the answer is still the
+                // tool result, it simply arrives as it is produced too
+                if (delegation?.emit) {
+                    const run = sub.stream(input, inherited);
+                    for await (const chunk of run.fullStream) {
+                        delegation.emit({ type: "subagent", name: id, chunk });
+                    }
+                    const [text, finishReason, usage, stats] = await Promise.all([
+                        run.text,
+                        run.finishReason,
+                        run.usage,
+                        run.toolStats,
+                    ]);
+                    delegation.onResult?.({ usage, toolStats: stats });
+                    // A step is all-or-nothing: a child that paused for a decision
+                    // cannot be resumed through a tool result
+                    if (finishReason === "approval_required") {
+                        throw ctx.error(
+                            `Sub-agent '${id}' paused for approval — delegation cannot carry a decision; call it directly`,
+                            { status: 409, code: "AGENT_APPROVAL_IN_DELEGATION" },
+                        );
+                    }
+                    return text;
+                }
+
+                const answer = await sub.generate(input, inherited);
+                delegation?.onResult?.({ usage: answer.usage, toolStats: answer.toolStats });
+                // A step is all-or-nothing: a child that paused for a decision cannot
+                // be resumed through a tool result — say so instead of losing the call
+                if (answer.pendingApprovals?.length) {
+                    throw ctx.error(
+                        `Sub-agent '${id}' paused for approval (${answer.pendingApprovals.map((approval) => approval.name).join(", ")}) — delegation cannot carry a decision; call it directly`,
+                        { status: 409, code: "AGENT_APPROVAL_IN_DELEGATION" },
+                    );
+                }
                 return answer.text;
             },
         }]);
@@ -438,6 +676,8 @@ type RunOptions = {
     streaming: boolean;
     controller: AbortController;
     emit?: (chunk: AgentStreamChunk) => void;
+    /** The stored state a `resumeRun()` continues — its conversation and counters. */
+    resumeState?: AgentRunState;
 };
 
 class Agent {
@@ -505,10 +745,18 @@ class Agent {
 
     // ── Provider ─────────────────────────────────────────────────────────
 
-    /** The provider as declared — the API key is **not** included. */
+    /** The provider as declared — the API key **and sensitive headers** are not included. */
     getProvider(): AgentProvider {
-        const { apiKey, ...rest } = this.#config.provider;
-        return { ...rest, options: { ...this.#config.provider.options } };
+        const { apiKey, headers, ...rest } = this.#config.provider;
+        const keys = redactKeys() ?? new Set(DEFAULT_REDACT_KEYS.map((key) => key.toLowerCase()));
+        const safeHeaders = headers
+            ? Object.fromEntries(Object.entries(headers).filter(([name]) => !isSensitiveKey(name, keys)))
+            : undefined;
+        return {
+            ...rest,
+            ...(safeHeaders && Object.keys(safeHeaders).length ? { headers: safeHeaders } : {}),
+            options: { ...this.#config.provider.options },
+        };
     }
 
     /** Sampling and transport options as declared (`provider.options`). */
@@ -916,6 +1164,54 @@ class Agent {
         return result.object;
     }
 
+    /**
+     * Continue a run that a crash, a restart or a provider failure interrupted —
+     * from its **last completed step**, without paying for the conversation again.
+     *
+     * ```ts
+     * const answer = await agent.resumeRun('run-42'); // 1 model call, not the whole loop
+     * ```
+     *
+     * The store is the call's `checkpoint`, then the agent's. The resumed run keeps
+     * the same `runId` and store, so its next steps are saved too and its success
+     * clears the state. Completed steps are never replayed through the model: a
+     * checkpoint sits on a step boundary, and the tool results of the last completed
+     * step are already part of the stored conversation — the model is asked for the
+     * **next** step only.
+     */
+    async resumeRun<T = any>(runId: string, options: AgentCallOptions = {}): Promise<AgentGenerateResult<T>> {
+        const store = options.checkpoint ?? this.#config.checkpoint;
+        if (!store) {
+            throw new AppError(
+                `Agent '${this.getId() ?? "?"}' cannot resume run '${runId}': no \`checkpoint\` store`,
+                { status: 400, code: "AGENT_CHECKPOINT_REQUIRED" },
+            );
+        }
+
+        const state = await store.load(runId);
+        if (!state) {
+            throw new AppError(`No checkpoint is stored for run '${runId}'`, {
+                status: 404,
+                code: "AGENT_CHECKPOINT_NOT_FOUND",
+            });
+        }
+
+        // The conversation comes from the state — the input is inert, and so are the
+        // injected turns, the attachments and a paused-run resume of the call
+        const call: AgentCallOptions = { ...options, runId, checkpoint: store };
+        delete call.resume;
+        delete call.messages;
+        delete call.files;
+
+        return this.#run<T>({
+            input: "",
+            call,
+            streaming: false,
+            controller: new AbortController(),
+            resumeState: state,
+        });
+    }
+
     /**
      * Run the agent and stream the answer token by token.
      * The promises of the returned object resolve when the **run** is over
@@ -936,12 +1232,15 @@ class Agent {
         const object = deferred<T | undefined>();
         const toolCalls = deferred<AgentToolCall[]>();
         const toolResults = deferred<AgentToolResultEntry[]>();
+        const toolStats = deferred<AgentToolStats>();
+        const modelUsed = deferred<string | undefined>();
+        const budgetHit = deferred<{ limit: "tokens" | "cost" | "duration"; value: number } | undefined>();
         const steps = deferred<AgentStep[]>();
         const usage = deferred<AgentUsage>();
         const finishReason = deferred<string>();
         const messages = deferred<AgentMessage[]>();
 
-        const deferreds = [text, reasoning, object, toolCalls, toolResults, steps, usage, finishReason, messages];
+        const deferreds = [text, reasoning, object, toolCalls, toolResults, toolStats, modelUsed, budgetHit, steps, usage, finishReason, messages];
 
         // The caller may only consume `textStream` — a rejected promise nobody
         // awaits must not become an unhandled rejection.
@@ -957,6 +1256,9 @@ class Agent {
                 object.resolve(result.object);
                 toolCalls.resolve(result.toolCalls);
                 toolResults.resolve(result.toolResults);
+                toolStats.resolve(result.toolStats);
+                modelUsed.resolve(result.model);
+                budgetHit.resolve(result.budget);
                 steps.resolve(result.steps);
                 usage.resolve(result.usage);
                 finishReason.resolve(result.finishReason);
@@ -980,6 +1282,9 @@ class Agent {
             object: object.promise,
             toolCalls: toolCalls.promise,
             toolResults: toolResults.promise,
+            toolStats: toolStats.promise,
+            model: modelUsed.promise,
+            budget: budgetHit.promise,
             steps: steps.promise,
             usage: usage.promise,
             finishReason: finishReason.promise,
@@ -988,6 +1293,36 @@ class Agent {
         };
     }
 
+    /**
+     * Stream the answer into a typed object — `stream()` bounded to a schema.
+     *
+     * Every member of `stream()` is there (`text`, `object`, `usage`,
+     * `fullStream`, `abort`…), plus `partialObjectStream`, which yields a
+     * **best-effort** parse of the answer so far (`undefined` until the partial
+     * JSON parses). The final object is still parsed and validated at the end:
+     * `await result.object` is the one to trust.
+     *
+     * ```ts
+     * const run = agent.streamObject('List 3 cities', { schema });
+     * for await (const partial of run.partialObjectStream) {
+     *   if (partial) render(partial);
+     * }
+     * ```
+     *
+     * `partialObjectStream` consumes the run's text: choose it **or**
+     * `textStream`, not both.
+     */
+    streamObject<T = any>(input: AgentInput, options: AgentCallOptions = {}): AgentStreamObjectResult<T> {
+        if (!options.schema && !options.structuredOutput) {
+            throw new AppError("`streamObject` requires a `schema` (or `structuredOutput`)", {
+                status: 500,
+                code: "AGENT_SCHEMA_REQUIRED",
+            });
+        }
+        const run = this.stream<T>(input, options);
+        return { ...run, partialObjectStream: partialObjects<T>(run.textStream) };
+    }
+
     // ── Internals ────────────────────────────────────────────────────────
 
     /** Provider merged with the call-level override — `options` merged too, not replaced. */
@@ -1001,6 +1336,22 @@ class Agent {
         };
     }
 
+    /** The primary provider, then its fallbacks, each with its options merged in. */
+    #providerChain(override?: Partial<AgentProvider>): AgentProvider[] {
+        const primary = this.#provider(override);
+        const fallbacks = primary.fallbacks ?? [];
+        return [
+            primary,
+            ...fallbacks.map((fallback) => ({
+                ...primary,
+                ...fallback,
+                options: { ...primary.options, ...fallback.options },
+                headers: { ...primary.headers, ...fallback.headers },
+                fallbacks: undefined,
+            })),
+        ];
+    }
+
     /**
      * The tools of one run — `true` (all), `false` (none), an array of **names**
      * (an allow-list) or extra tools to merge in.
@@ -1008,14 +1359,14 @@ class Agent {
      * The declared agents and the **remote MCP servers** are resolved here, before the
      * allow-list is applied, so every rule below treats them like any other tool.
      */
-    async #collectTools(requested?: boolean | string[] | AgentTools): Promise<Map<string, AgentTool>> {
+    async #collectTools(requested?: boolean | string[] | AgentTools, delegation?: Delegation): Promise<Map<string, AgentTool>> {
         // `tools: false` — the run answers directly (and no remote is even contacted)
         if (requested === false) return new Map();
 
         const tools = new Map(this.#tools);
 
         // The other agents this one may delegate to
-        for (const [name, tool] of subAgentTools(this.#config.agents, this.#rest, this._tenant_)) {
+        for (const [name, tool] of subAgentTools(this.#config.agents, this.#rest, this._tenant_, delegation)) {
             if (!tools.has(name)) tools.set(name, tool);
         }
 
@@ -1169,12 +1520,14 @@ class Agent {
         const tool = tools.get(call.name);
         if (!tool) {
             entry.error = `Unknown tool '${call.name}'`;
+            entry.errorCode = "AGENT_TOOL_UNKNOWN";
             return done();
         }
 
         const { value, error } = await validateWithSchema(tool.inputSchema, call.args);
         if (error) {
             entry.error = `Invalid arguments for tool '${call.name}': ${error}`;
+            entry.errorCode = "AGENT_TOOL_ARGS_INVALID";
             return done();
         }
 
@@ -1207,6 +1560,7 @@ class Agent {
             return done();
         } catch (err: any) {
             entry.error = err?.message ?? String(err);
+            entry.errorCode = err?.code ? String(err.code) : "AGENT_TOOL_FAILED";
             return done();
         }
     }
@@ -1214,10 +1568,31 @@ class Agent {
     async #run<T>(run: RunOptions): Promise<AgentGenerateResult<T>> {
         const { input, call, streaming, controller } = run;
         const emit = run.emit;
+        // A `resumeRun()` continues a stored state: its conversation and counters are
+        // the starting point, not the caller's input
+        const resumeState = run.resumeState;
+
+        // The run's signal comes first: the delegated agents inherit it, and the
+        // tools are collected before the first model call
+        const signal = call.signal ? AbortSignal.any([call.signal, controller.signal]) : controller.signal;
 
-        const tools = await this.#collectTools(call.tools);
-        const provider = this.#provider(call.provider);
-        const adapter = resolveProvider(provider);
+        // What delegated runs consume — folded into the parent's usage and stats at
+        // the end. A streaming parent hands its `emit` down, so the child streams too.
+        let delegatedUsage: AgentUsage = { ...EMPTY_USAGE };
+        const delegatedToolStats: AgentToolStats = { calls: 0, errors: 0, byCode: {}, timeouts: 0, durationMs: 0 };
+        const tools = await this.#collectTools(call.tools, {
+            signal,
+            maxSteps: call.maxSteps ?? this.#config.maxSteps,
+            ...(emit ? { emit } : {}),
+            onResult: ({ usage, toolStats: stats }) => {
+                delegatedUsage = addUsage(delegatedUsage, usage);
+                mergeToolStats(delegatedToolStats, stats);
+            },
+        });
+        const chain = this.#providerChain(call.provider);
+        const provider = chain[0]!;
+        // A resumed segment starts from the model that answered before the restart
+        let model = resumeState?.model ?? provider.model;
 
         const structured: AgentStructuredOutput | undefined = call.structuredOutput ?? (call.schema ? { schema: call.schema } : undefined);
         const schema = structured?.schema;
@@ -1233,18 +1608,44 @@ class Agent {
         // `…Id` spellings are still read as aliases.
         const thread = call.memory?.thread ?? call.memory?.threadId ?? call.thread ?? call.threadId;
         const resource = call.memory?.resource ?? call.memory?.resourceId ?? call.resource ?? call.resourceId;
-        const signal = call.signal ? AbortSignal.any([call.signal, controller.signal]) : controller.signal;
 
         // Per-tool deadline and tool parallelism: the call wins over the declaration
         const toolTimeout = call.toolTimeoutMs ?? this.#config.toolTimeoutMs;
         const parallel = call.parallelTools ?? this.#config.parallelTools ?? false;
+        const maxParallelToolCalls = Math.max(
+            1,
+            Math.trunc(call.maxParallelToolCalls ?? this.#config.maxParallelToolCalls ?? 8),
+        );
         const savePartial = call.savePartial ?? this.#config.savePartial ?? false;
 
+        // Durable checkpoint: the call's store wins over the declaration's, and both
+        // the store and a run id are required — a run without identity cannot be
+        // resumed, so it is not saved either
+        const checkpointStore = call.checkpoint ?? this.#config.checkpoint;
+        const runId = call.runId;
+        const checkpointing = !!checkpointStore && !!runId;
+
+        // Run budget — a run that reaches it stops **cleanly**, it does not throw
+        const maxTokens = call.maxTokens ?? this.#config.maxTokens;
+        const maxDurationMs = call.maxDurationMs ?? this.#config.maxDurationMs;
+        const maxCost = call.maxCost ?? this.#config.maxCost;
+        const contextTokens = call.contextTokens ?? this.#config.contextTokens;
+        const startedAt = Date.now();
+        let budget: { limit: "tokens" | "cost" | "duration"; value: number } | undefined;
+
+        // Guardrails of this run: the declaration's, then the call's
+        const inputGuardrails = [...(this.#config.guardrails?.input ?? []), ...(call.guardrails?.input ?? [])];
+        const outputGuardrails = [...(this.#config.guardrails?.output ?? []), ...(call.guardrails?.output ?? [])];
+        const toolGuardrails = [...(this.#config.guardrails?.tool ?? []), ...(call.guardrails?.tool ?? [])];
+
         // Resuming a paused run: the conversation is the caller's — the paused step is
-        // resolved from it instead of being asked of the model again
-        const resuming = call.resume ? await normalizeMessages(call.resume.messages) : undefined;
+        // resolved from it instead of being asked of the model again. A checkpoint
+        // resume has the same shape, with the stored conversation as the whole
+        // history (already shaped by the processors of the interrupted run).
+        const carried = resumeState?.messages;
+        const resuming = !carried && call.resume ? await normalizeMessages(call.resume.messages) : undefined;
 
-        const history = resuming ?? (thread && this.#config.memory
+        const history = carried ?? resuming ?? (thread && this.#config.memory
             ? [...(await this.#config.memory.get(thread, this.#memoryCtx(resource)))]
             : []);
 
@@ -1252,31 +1653,56 @@ class Agent {
         // history and before the question, and saved with the rest of the
         // conversation. A trailing assistant turn is legitimate ("continue from
         // here"), so the block is not forced to end on a user turn.
-        const injected = !resuming && call.messages ? await normalizeMessages(call.messages) : [];
+        const injected = !carried && !resuming && call.messages ? await normalizeMessages(call.messages) : [];
 
         // What this run **adds to the thread** — the injected turns and the input,
         // with the attachments on the turn being asked. The history window and the
         // `load` processors only shape the prompt: neither reaches the store.
         // A resumed run adds nothing: its input **is** the conversation it was given.
-        const additions: AgentMessage[] = resuming ? [] : [...injected, ...(await normalizeMessages(input))];
+        const additions: AgentMessage[] = carried || resuming ? [] : [...injected, ...(await normalizeMessages(input))];
 
         // Attachments of this run — read now, attached to the turn being asked
-        if (!resuming && call.files) {
+        if (!carried && !resuming && call.files) {
             const parts = await resolveAttachments(call.files, { baseDir: call.filesBaseDir });
             if (parts.length) attachToTurn(additions, parts);
         }
 
         // The history window: the thread is what the store keeps, the window is what
-        // the prompt replays (a cheap bound on the prompt)
+        // the prompt replays (a cheap bound on the prompt). A carried conversation is
+        // continued as-is — the interrupted run already applied both bounds to it.
         const window = call.lastMessages ?? this.#config.lastMessages;
-        const replay = window && window > 0 ? trimMessages(history, window) : history;
+        const replay = carried ?? (window && window > 0 ? trimMessages(history, window) : history);
 
         const memoryCtx = this.#memoryCtx(resource, thread);
         let messages: AgentMessage[] = [...replay, ...additions];
 
-        // Memory processors, `load` side: they see exactly what the model will see
-        // (history, injected turns, input and attachments included)
-        messages = await applyProcessors(this.#config.processors, "load", messages, memoryCtx);
+        if (!carried) {
+            // Memory processors, `load` side: they see exactly what the model will
+            // see (history, injected turns, input and attachments included)
+            messages = await applyProcessors(this.#config.processors, "load", messages, memoryCtx);
+
+            // The prompt budget, in tokens: the oldest turns are dropped first. It runs
+            // after the processors (they may have shortened the thread) and before the
+            // guardrails (which read the caller's prompt, not the window).
+            if (contextTokens && contextTokens > 0) messages = trimToTokens(messages, contextTokens);
+        }
+
+        // Input guardrails — the caller's prompt, before any model call
+        const guardrailCtx: AgentGuardrailContext = { agent: this, tenant: this._tenant_, thread, resource, signal };
+        const lastUser = [...messages].reverse().find((message) => message.role === "user");
+        await runGuardrails(inputGuardrails, messageText(lastUser), guardrailCtx, "prompt");
+
+        // Tracing — the caller's hook (the call's wins over the declaration's). It
+        // sees plain events; turning them into spans is its job, not the run's.
+        const traceHook = call.onTrace ?? this.#config.onTrace;
+        trace(traceHook, {
+            type: "run.start",
+            at: Date.now(),
+            ...(this.#config.id ? { agentId: this.#config.id } : {}),
+            ...(call.runId ? { runId: call.runId } : {}),
+            ...(thread ? { thread } : {}),
+            ...(resource ? { resource } : {}),
+        });
 
         let system = call.instructions ?? await this.resolveInstructions({ thread, resource });
 
@@ -1303,11 +1729,87 @@ class Agent {
         const steps: AgentStep[] = [];
         const allToolCalls: AgentToolCall[] = [];
         const allToolResults: AgentToolResultEntry[] = [];
-        let usage: AgentUsage = { ...EMPTY_USAGE };
+        // A resumed segment carries what the run already spent — a budget sees the
+        // whole run, not just the part after the restart
+        const toolStats: AgentToolStats = resumeState
+            ? { ...resumeState.toolStats, byCode: { ...resumeState.toolStats.byCode } }
+            : { calls: 0, errors: 0, byCode: {}, timeouts: 0, durationMs: 0 };
+        let usage: AgentUsage = resumeState ? { ...resumeState.usage } : { ...EMPTY_USAGE };
         let text = "";
         let reasoning = "";
         let finishReason = "stop";
 
+        // The durable checkpoint — written at a **step boundary** only: the assistant
+        // turn and every tool turn it asked for are in, so the stored conversation is
+        // always valid to replay. Best-effort: a store that fails is logged, never
+        // fatal — the run is worth more than its checkpoint.
+        const saveCheckpoint = async (completed: number): Promise<void> => {
+            if (!checkpointing) return;
+            const state: AgentRunState = {
+                runId: runId!,
+                ...(this.#config.id ? { agentId: this.#config.id } : {}),
+                step: completed,
+                messages: messages.map((message) => ({ ...message })),
+                usage: { ...usage },
+                toolStats: { ...toolStats, byCode: { ...toolStats.byCode } },
+                ...(model ? { model } : {}),
+                finishReason,
+                at: new Date().toISOString(),
+            };
+            try {
+                await checkpointStore!.save(runId!, state);
+            } catch (err: any) {
+                console.error(`The checkpoint of run '${runId}' was not saved: ${err?.message ?? err}`);
+            }
+        };
+
+        // The end of the run — traced once, on the answer or on the way out. The
+        // delegated work is folded in first: it is part of what the caller pays for.
+        const endRun = (reason: string, error?: unknown): void => {
+            usage = addUsage(usage, delegatedUsage);
+            mergeToolStats(toolStats, delegatedToolStats);
+            trace(traceHook, {
+                type: "run.end",
+                at: Date.now(),
+                finishReason: reason,
+                durationMs: Date.now() - startedAt,
+                usage,
+                toolStats,
+                ...(error !== undefined
+                    ? { error: String((error as any)?.message ?? error).split("\n")[0] }
+                    : {}),
+            });
+        };
+
+        // One tool call, traced — the start before it runs, the end after, whichever
+        // path executes it (sequential, parallel, or a resumed step)
+        const runTool = async (toolCall: AgentToolCall, step: number): Promise<AgentToolResultEntry> => {
+            const toolStartedAt = Date.now();
+            trace(traceHook, {
+                type: "tool.start",
+                at: toolStartedAt,
+                step,
+                tool: toolCall.name,
+                toolCallId: toolCall.id,
+            });
+            let result: AgentToolResultEntry | undefined;
+            try {
+                result = await this.#executeTool(toolCall, tools, step, signal, toolTimeout);
+                return result;
+            } finally {
+                const endedAt = Date.now();
+                trace(traceHook, {
+                    type: "tool.end",
+                    at: endedAt,
+                    step,
+                    tool: toolCall.name,
+                    toolCallId: toolCall.id,
+                    durationMs: endedAt - toolStartedAt,
+                    ...(result?.errorCode ? { errorCode: result.errorCode } : {}),
+                });
+            }
+        };
+
         // What the model may use, and how — the framework picks no tool itself
         const toolChoice = call.toolChoice ?? this.#config.toolChoice ?? "auto";
 
@@ -1337,7 +1839,10 @@ class Agent {
                 }
                 const result: AgentToolResultEntry = decision === false
                     ? { id: toolCall.id, name: toolCall.name, args: toolCall.args, durationMs: 0, error: "Refused by the caller" }
-                    : await this.#executeTool(toolCall, tools, 0, signal, toolTimeout);
+                    : await runTool(toolCall, 0);
+                // Checked before it enters the conversation — a blocked result is an
+                // error the model reads, and the run goes on
+                if (decision !== false) await guardToolResult(result, toolGuardrails, guardrailCtx);
                 const turn = toolTurnFor(result);
                 messages.push(turn);
                 additions.push({ ...turn });
@@ -1347,13 +1852,63 @@ class Agent {
             }
         }
 
+        // One model call, with the fallbacks: an endpoint that is rate-limited, down or
+        // unreachable hands over to the next one — as long as nothing was streamed yet
+        let streamed = false;
+        const guardedEmit = emit
+            ? (chunk: AgentStreamChunk) => { streamed = true; emit(chunk); }
+            : undefined;
+        const chat = async (request: ModelChatRequest, step: number) => {
+            const callStartedAt = Date.now();
+            let lastError: any;
+            for (const candidate of chain) {
+                const adapter = resolveProvider(candidate);
+                try {
+                    const turn = streaming && guardedEmit
+                        ? await this.#chatStream(adapter, request, guardedEmit)
+                        : await this.#chat(adapter, request);
+                    model = candidate.model;
+                    if (candidate.pricing) {
+                        turn.usage = { ...turn.usage, cost: costOf(turn.usage, candidate.pricing) };
+                    }
+                    const endedAt = Date.now();
+                    trace(traceHook, {
+                        type: "model.end",
+                        at: endedAt,
+                        step,
+                        durationMs: endedAt - callStartedAt,
+                        // What answered, not what was tried: a fallback is the model
+                        model: candidate.model,
+                        usage: turn.usage,
+                        finishReason: turn.finishReason,
+                    });
+                    return turn;
+                } catch (err: any) {
+                    lastError = err;
+                    // A stream that already reached the caller cannot be restarted
+                    if (signal.aborted || streamed || !worthFallback(err)) throw err;
+                }
+            }
+            throw lastError;
+        };
+
         try {
-            // Nothing to ask the model while a decision is pending
-            for (let step = 0; !pending.length && step < maxSteps; step++) {
+            // Nothing to ask the model while a decision is pending. A resumed run
+            // starts at the **next** step: the completed ones are never replayed through
+            // the model, they are simply already in the conversation.
+            for (let step = resumeState?.step ?? 0; !pending.length && step < maxSteps; step++) {
+                trace(traceHook, { type: "step.start", at: Date.now(), step });
                 if (signal.aborted) {
                     finishReason = "aborted";
                     break;
                 }
+                // The wall-clock budget is checked between steps: a model call already
+                // started is bounded by its own `provider.options.timeout`
+                if (maxDurationMs !== undefined && Date.now() - startedAt >= maxDurationMs) {
+                    finishReason = "budget";
+                    budget = { limit: "duration", value: maxDurationMs };
+                    break;
+                }
 
                 const request: ModelChatRequest = {
                     system,
@@ -1370,9 +1925,7 @@ class Agent {
                         : {}),
                 };
 
-                const turn = streaming && emit
-                    ? await this.#chatStream(adapter, request, emit)
-                    : await this.#chat(adapter, request);
+                const turn = await chat(request, step);
 
                 usage = addUsage(usage, turn.usage);
                 text = turn.text;
@@ -1401,10 +1954,45 @@ class Agent {
                 additions.push({ ...assistantTurn });
                 allToolCalls.push(...turn.toolCalls);
 
+                // The token/cost budget is checked **after** the assistant turn is in
+                // the conversation: the paid answer is kept, and the calls it asked for
+                // are answered with a refusal so the thread stays valid for a replay
+                const overTokens = maxTokens !== undefined && usage.totalTokens > maxTokens;
+                const overCost = maxCost !== undefined && (usage.cost ?? 0) > maxCost;
+                if (overTokens || overCost) {
+                    for (const toolCall of turn.toolCalls) {
+                        const skipped: AgentToolResultEntry = {
+                            id: toolCall.id,
+                            name: toolCall.name,
+                            args: toolCall.args,
+                            durationMs: 0,
+                            error: "Skipped: the run budget was exhausted",
+                            errorCode: "AGENT_BUDGET_EXHAUSTED",
+                        };
+                        current.toolResults.push(skipped);
+                        allToolResults.push(skipped);
+                        recordToolStats(toolStats, skipped);
+                        const toolTurn = toolTurnFor(skipped);
+                        messages.push(toolTurn);
+                        additions.push({ ...toolTurn });
+                        emit?.({ type: "tool_result", toolResult: skipped });
+                    }
+                    budget = overTokens
+                        ? { limit: "tokens", value: maxTokens! }
+                        : { limit: "cost", value: maxCost! };
+                    finishReason = "budget";
+                    steps.push(current);
+                    await call.onStepFinish?.(current);
+                    emit?.({ type: "step", step: current });
+                    await saveCheckpoint(step + 1);
+                    break;
+                }
+
                 if (!turn.toolCalls.length) {
                     finishReason = turn.finishReason;
                     steps.push(current);
                     await call.onStepFinish?.(current);
+                    await saveCheckpoint(step + 1);
                     break;
                 }
 
@@ -1424,23 +2012,33 @@ class Agent {
                 }
 
                 // The tools of one step: sequential by default (two writes must not
-                // interleave), together when the caller asks for it. The results are
-                // attached in the model's order either way.
+                // interleave), together when the caller asks for it **or** when every
+                // call is read-only. The results are attached in the model's order
+                // either way, and at most `maxParallelToolCalls` run at once.
                 const entries: AgentToolResultEntry[] = [];
-                if (parallel) {
-                    entries.push(...await Promise.all(
-                        turn.toolCalls.map((toolCall) => this.#executeTool(toolCall, tools, step, signal, toolTimeout)),
+                const readOnlyStep = turn.toolCalls.length > 1
+                    && turn.toolCalls.every((toolCall) => tools.get(toolCall.name)?.readOnly === true);
+                if ((parallel || readOnlyStep) && turn.toolCalls.length > 1) {
+                    entries.push(...await mapLimited(
+                        turn.toolCalls,
+                        maxParallelToolCalls,
+                        (toolCall) => runTool(toolCall, step),
                     ));
                 } else {
                     for (const toolCall of turn.toolCalls) {
-                        entries.push(await this.#executeTool(toolCall, tools, step, signal, toolTimeout));
+                        entries.push(await runTool(toolCall, step));
                     }
                 }
 
+                // Tool guardrails: a **successful** result is checked before the model
+                // reads it — a blocked one becomes an error and the run goes on
+                for (const result of entries) await guardToolResult(result, toolGuardrails, guardrailCtx);
+
                 turn.toolCalls.forEach((toolCall, index) => {
                     const result = entries[index]!;
                     current.toolResults.push(result);
                     allToolResults.push(result);
+                    recordToolStats(toolStats, result);
                     const toolTurn = toolTurnFor(result);
                     messages.push(toolTurn);
                     additions.push({ ...toolTurn });
@@ -1452,6 +2050,7 @@ class Agent {
                 emit?.({ type: "step", step: current });
                 finishReason = turn.finishReason;
                 if (step === maxSteps - 1) finishReason = "max_steps";
+                await saveCheckpoint(step + 1);
             }
         } catch (err: any) {
             if (signal.aborted) {
@@ -1462,6 +2061,7 @@ class Agent {
                 if (savePartial && thread && this.#config.memory && !call.memory?.readOnly) {
                     await this.#persist(thread, [...history, ...additions], memoryCtx).catch(() => {});
                 }
+                endRun("error", err);
                 throw err;
             }
         }
@@ -1471,53 +2071,62 @@ class Agent {
 
         // Structured output — one repair turn, then give up with a stable code.
         let object: T | undefined;
-        if (schema && !pending.length) {
-            const attempt = extractJson(text);
-            let candidate: any = attempt.value;
-            let failure = attempt.error;
-            if (!failure) {
-                const validated = await validateWithSchema(schema, candidate);
-                if (validated.error) failure = validated.error;
-                else candidate = validated.value;
-            }
-
-            if (failure && text.trim()) {
-                // One repair turn — `messages` already ends with the rejected answer.
-                const repair = [...messages, {
-                    role: "user" as const,
-                    content: `That answer was rejected: ${failure}. Reply with a single valid JSON object matching the schema — no markdown, no commentary.`,
-                }];
-                const turn = await this.#chat(adapter, {
-                    system,
-                    messages: repair,
-                    temperature: 0,
-                    signal,
-                    json: true,
-                    ...(jsonSchema ? { jsonSchema: { name: schemaName, schema: jsonSchema } } : {}),
-                });
-
-                usage = addUsage(usage, turn.usage);
-                text = turn.text;
-                messages.push({ role: "assistant", content: text });
-                additions.push({ role: "assistant", content: text });
-
-                const retry = extractJson(text);
-                failure = retry.error;
+        try {
+            if (schema && !pending.length) {
+                const attempt = extractJson(text);
+                let candidate: any = attempt.value;
+                let failure = attempt.error;
                 if (!failure) {
-                    const validated = await validateWithSchema(schema, retry.value);
+                    const validated = await validateWithSchema(schema, candidate);
                     if (validated.error) failure = validated.error;
                     else candidate = validated.value;
                 }
-            }
 
-            if (failure) {
-                throw new AppError(`Agent '${this.#config.id ?? "?"}' did not return valid structured output: ${failure}`, {
-                    status: 502,
-                    code: "AGENT_OUTPUT_INVALID",
-                    reason: text,
-                });
+                if (failure && text.trim()) {
+                    // One repair turn — `messages` already ends with the rejected answer.
+                    const repair = [...messages, {
+                        role: "user" as const,
+                        content: `That answer was rejected: ${failure}. Reply with a single valid JSON object matching the schema — no markdown, no commentary.`,
+                    }];
+                    const turn = await chat({
+                        system,
+                        messages: repair,
+                        temperature: 0,
+                        signal,
+                        json: true,
+                        ...(jsonSchema ? { jsonSchema: { name: schemaName, schema: jsonSchema } } : {}),
+                    }, Math.max(0, steps.length - 1));
+
+                    usage = addUsage(usage, turn.usage);
+                    text = turn.text;
+                    messages.push({ role: "assistant", content: text });
+                    additions.push({ role: "assistant", content: text });
+
+                    const retry = extractJson(text);
+                    failure = retry.error;
+                    if (!failure) {
+                        const validated = await validateWithSchema(schema, retry.value);
+                        if (validated.error) failure = validated.error;
+                        else candidate = validated.value;
+                    }
+                }
+
+                if (failure) {
+                    throw new AppError(`Agent '${this.#config.id ?? "?"}' did not return valid structured output: ${failure}`, {
+                        status: 502,
+                        code: "AGENT_OUTPUT_INVALID",
+                        reason: text,
+                    });
+                }
+                object = candidate as T;
             }
-            object = candidate as T;
+
+            // Output guardrails — the answer is checked **before** it is stored or
+            // returned: a blocked answer is not persisted, and the caller gets the reason
+            await runGuardrails(outputGuardrails, text, guardrailCtx, "answer");
+        } catch (err) {
+            endRun("error", err);
+            throw err;
         }
 
         // A paused run stores **nothing**: the caller resumes with the conversation it
@@ -1531,12 +2140,31 @@ class Agent {
             }
         }
 
+        // A run that came back with an answer has nothing to resume: its state is
+        // cleared. A paused or aborted run keeps it — that is what the next call
+        // continues from. Best-effort: a store that fails is logged, never fatal.
+        if (checkpointing && !pending.length && finishReason !== "aborted" && finishReason !== "error") {
+            try {
+                await checkpointStore!.clear(runId!);
+            } catch (err: any) {
+                console.error(`The checkpoint of run '${runId}' was not cleared: ${err?.message ?? err}`);
+            }
+        }
+
+        // A delegated run's tokens and cost are the parent's: they are what the caller
+        // pays for, and what a budget must see — `endRun` folds them in before
+        // emitting the final event.
+        endRun(finishReason);
+
         return {
             text,
+            model,
+            ...(budget ? { budget } : {}),
             ...(object !== undefined ? { object } : {}),
             ...(reasoning ? { reasoning } : {}),
             toolCalls: allToolCalls,
             toolResults: allToolResults,
+            toolStats,
             steps,
             usage,
             finishReason,