@nodetool-ai/runtime 0.7.0-rc.23 → 0.8.2

package/dist/providers/base-provider.js

@@ -1,56 +1,197 @@
+import { stringifyTraceContent } from "../run-trace-serialization.js";
+import { tracedProviderLoop, tracedToolCall, beginProviderRound, providerRoundContext, traceContentAllowed, messageTraceContent, traceTextContent } from "./loop-tracing.js";
+import { isProviderSessionUpdate, isProviderMessageEvent, isProviderStop, isProviderToolErrorResult, PROVIDER_STOP_ABORTED } from "./types.js";
+import { ProviderGenerationsUnsupportedError } from "./provider-generations.js";
 import { CostCalculator } from "./cost-calculator.js";
 import { getTracer } from "../telemetry.js";
-import { withUsageCapture, setLastUsage, consumeLastUsage, peekLastUsage, createUsageSlot } from "../tracing-helpers.js";
-import { SpanStatusCode } from "@opentelemetry/api";
-import { createLogger, getAssetFilePath } from "@nodetool-ai/config";
+import { withUsageCapture, withModalityCapture, setLastUsage, setLastRequest, consumeLastUsage, peekLastUsage, peekLastRequest, createUsageSlot } from "../tracing-helpers.js";
+import { isRunBudget } from "../turn-budget.js";
+import { countTokens } from "../token-counter.js";
+import { isCallable, isObjectLike, isRecord, isString } from "@nodetool-ai/protocol";
+import { logProviderRequestFailure } from "./provider-request-log.js";
+import { annotateProviderError, markContextExceeded } from "./provider-error.js";
+import { buildProviderCallFailure } from "./provider-call-failure.js";
+import { applyEntityReferences } from "./entity-references.js";
+import { expandVideoContentAsFrames, videoFrameFallbackEnabled } from "./video-frame-fallback.js";
+import { SpanStatusCode, context, trace } from "@opentelemetry/api";
+import { createLogger, getDefaultAssetsPath } from "@nodetool-ai/config";
 const log = createLogger("nodetool.runtime.provider");
 /**
- * Derive a provider's capability set by checking which optional `getAvailable*`
- * methods it overrides on its prototype. Every provider can always generate
- * messages; image/video/TTS/ASR/embedding are advertised only when the concrete
- * class overrides the matching base method.
+ * Split the `turnBudget` argument into the run-level bounds (when the caller
+ * passed a {@link RunBudget}) and the per-turn admission object every loop
+ * reserves against. One argument serves both, so callers holding only a bare
+ * {@link TurnBudget} — the supervisor — keep working unchanged.
  *
- * Shared by `packages/websocket/src/models-api.ts` (REST leftover) and the
- * `models` tRPC router so both report identical capabilities for the same
- * provider instance.
+ * A free function rather than a method: `generateLoop` is called unbound on
+ * duck-typed test providers (`BaseProvider.prototype.generateLoop.call(mock)`),
+ * where `this` has no class methods.
  */
-export function providerCapabilities(instance) {
-    const capabilities = [
-        "generate_message",
-        "generate_messages"
-    ];
-    if (instance.getAvailableImageModels !==
-        BaseProvider.prototype.getAvailableImageModels) {
-        capabilities.push("text_to_image", "image_to_image");
-    }
-    if (instance.getAvailableVideoModels !==
-        BaseProvider.prototype.getAvailableVideoModels) {
-        capabilities.push("text_to_video", "image_to_video");
-    }
-    if (instance.getAvailableTTSModels !==
-        BaseProvider.prototype.getAvailableTTSModels) {
-        capabilities.push("text_to_speech");
-    }
-    if (instance.getAvailableASRModels !==
-        BaseProvider.prototype.getAvailableASRModels) {
-        capabilities.push("automatic_speech_recognition");
-    }
-    if (instance.getAvailableEmbeddingModels !==
-        BaseProvider.prototype.getAvailableEmbeddingModels) {
-        capabilities.push("generate_embedding");
-    }
-    if (instance.getAvailable3DModels !==
-        BaseProvider.prototype.getAvailable3DModels) {
-        // Providers that expose 3D models are assumed to support both task types;
-        // individual `textTo3D` / `imageTo3D` calls will throw if not implemented.
-        capabilities.push("text_to_3d", "image_to_3d");
-    }
-    return capabilities;
+export function resolveTurnBudget(budget) {
+    if (isRunBudget(budget)) {
+        return { runBudget: budget, turnBudget: budget.turns };
+    }
+    return { runBudget: null, turnBudget: budget };
+}
+/**
+ * The stop item for a refused turn. A {@link RunBudget} says which ceiling
+ * refused it; a bare {@link TurnBudget} only knows that one did.
+ */
+export function budgetStopItem(runBudget) {
+    const exhausted = runBudget?.exhausted ?? null;
+    return {
+        type: "stop",
+        reason: exhausted?.kind === "deadline" ? "deadline" : "budget",
+        detail: exhausted?.detail ?? "the turn budget refused the next model turn"
+    };
+}
+/**
+ * Collapse a tool result to plain text. Used by string-only tool channels (the
+ * legacy inline callback, the Claude Agent SDK's MCP tools) when a tool returns
+ * {@link MessageContent} blocks they can't render — keep the text, drop images.
+ */
+export function toolResultToText(result) {
+    if (isString(result))
+        return result;
+    const text = result
+        .filter((c) => c.type === "text")
+        .map((c) => c.text)
+        .join("\n");
+    return text || "[image result]";
+}
+export function splitToolResultImages(result) {
+    if (isString(result)) {
+        return { toolContent: result, imageMessage: null };
+    }
+    const images = result.filter((c) => c.type === "image_url");
+    if (images.length === 0) {
+        return { toolContent: result, imageMessage: null };
+    }
+    const texts = result.filter((c) => c.type === "text");
+    const note = texts.map((c) => c.text).join("\n") ||
+        "Returned an image, shown in the next message.";
+    return {
+        toolContent: note,
+        imageMessage: { role: "user", content: [...texts, ...images] }
+    };
+}
+/**
+ * What one attached medium costs a vision/audio model, per block. These are
+ * modality constants, not measurements of the bytes: by the time a turn is
+ * estimated, `resolveMessageMediaUris` has inlined every `asset://` as a
+ * `data:` uri, and BPE-tokenizing that base64 scores a 50 KB image at ~49,000
+ * tokens against a real bill of ~1,600. A fixed figure per block is the right
+ * order of magnitude; the base64 length is off by two of them.
+ *
+ * - Image: 1,600 — the common vision-model charge for a full-size image
+ *   (Gemini bills a large image in the 1,000-1,600 band, GPT-4o high-detail
+ *   lands near 1,100). Take the top of the band, since a reservation is a
+ *   worst case.
+ * - Audio: 2,000 — roughly a minute at the ~32 tokens/second Gemini charges
+ *   for audio. Longer clips under-count, but audio in a chat turn is short.
+ * - Video: 4,000 — video is billed per sampled frame (~250-300 tokens/second
+ *   at 1 fps), so this covers a short clip of ~15 seconds.
+ * - Document: 3,000 — a few pages of extracted PDF text.
+ */
+const IMAGE_CONTENT_TOKENS = 1_600;
+const AUDIO_CONTENT_TOKENS = 2_000;
+const VIDEO_CONTENT_TOKENS = 4_000;
+const DOCUMENT_CONTENT_TOKENS = 3_000;
+/** Any content block that is neither text nor a modality named above. */
+const UNKNOWN_CONTENT_TOKENS = 1_600;
+function estimateContentPartTokens(part) {
+    switch (part.type) {
+        case "text":
+            return countTokens(part.text);
+        case "image_url":
+            return IMAGE_CONTENT_TOKENS;
+        case "audio":
+            return AUDIO_CONTENT_TOKENS;
+        case "video":
+            return VIDEO_CONTENT_TOKENS;
+        case "document":
+            return DOCUMENT_CONTENT_TOKENS;
+        default: {
+            // A block of some shape this build does not narrow — a `thought` block
+            // from the protocol union, or a newer type. Count it when it carries
+            // text; otherwise charge the unknown-block figure rather than tokenize
+            // it, since it may hold a `data:` uri.
+            const text = part.text;
+            return isString(text) ? countTokens(text) : UNKNOWN_CONTENT_TOKENS;
+        }
+    }
+}
+/**
+ * Prompt tokens a turn will send, for reservation purposes. Deliberately a
+ * rough count: a reservation is a worst case, and a per-provider exact count
+ * would cost a tokenizer round-trip per turn to refine a number that only has
+ * to be an upper-ish bound.
+ *
+ * Text is tokenized. Media blocks get a fixed per-modality figure, so an
+ * inlined `data:` uri is never fed to the tokenizer.
+ */
+export function estimatePromptTokens(messages) {
+    let total = 0;
+    for (const message of messages) {
+        const content = message.content;
+        if (isString(content)) {
+            total += countTokens(content);
+        }
+        else if (Array.isArray(content)) {
+            for (const part of content) {
+                total += estimateContentPartTokens(part);
+            }
+        }
+        else if (content != null) {
+            total += countTokens(JSON.stringify(content));
+        }
+        if (message.toolCalls?.length) {
+            total += countTokens(JSON.stringify(message.toolCalls));
+        }
+    }
+    return total;
 }
 export class BaseProvider {
     provider;
     _cost = 0;
+    _usageTotals = {
+        inputTokens: 0,
+        outputTokens: 0,
+        cachedTokens: 0
+    };
+    _unpricedReason = null;
+    _lastCallTokens = null;
     _emitMessage = null;
+    /**
+     * Whether this provider has a built-in, server-side web search it runs
+     * itself (e.g. Anthropic's `web_search_20250305` server tool, the Claude
+     * Agent SDK's `WebSearch`). When `true`, a tool named
+     * {@link WEB_SEARCH_TOOL_NAME} is fulfilled by the provider natively rather
+     * than the SerpAPI-backed `WebSearchTool`. Default `false` (use the fallback).
+     */
+    get supportsNativeWebSearch() {
+        return false;
+    }
+    /**
+     * Whether this provider has a built-in, server-side image generator (e.g.
+     * the OpenAI Responses `image_generation` tool). When `true`, a tool named
+     * {@link IMAGE_GENERATION_TOOL_NAME} is rendered as the provider's native
+     * tool — its result arrives as image content on the assistant message, not
+     * as a function-call round-trip. Default `false` (use the fallback function
+     * tool).
+     */
+    get supportsNativeImageGeneration() {
+        return false;
+    }
+    /**
+     * Whether this provider's chat API accepts a `video` content part. Only
+     * providers that read a whole clip natively answer `true` — an
+     * OpenAI-compatible endpoint re-hosting a video-capable model (AtlasCloud
+     * serving `google/gemini-2.5-flash`, say) does not, because the wire format
+     * has nowhere to put the bytes. Default `false`.
+     */
+    get supportsVideoInput() {
+        return false;
+    }
     setMessageEmitter(fn) {
         this._emitMessage = fn;
     }
@@ -58,29 +199,332 @@ export class BaseProvider {
         if (this._emitMessage)
             this._emitMessage(msg);
     }
+    /**
+     * Report one non-chat provider call (image, video, speech, ASR, embedding…)
+     * as an `llm_call` so the trace shows every provider call, not only chat.
+     * Carries no token counts: those modalities bill per unit, not per token.
+     * @internal
+     */
+    recordModalityCall(input) {
+        if (!this._emitMessage)
+            return;
+        const prompt = input.args
+            .map((arg) => isObjectLike(arg) ? arg.prompt : undefined)
+            .find(isString);
+        this.emitMessage({
+            type: "llm_call",
+            node_id: "",
+            provider: this.provider,
+            operation: input.operation,
+            model: extractModelId(input.args),
+            messages: prompt ? [{ role: "user", content: prompt }] : [],
+            response: null,
+            tool_calls: null,
+            tokens_input: null,
+            tokens_output: null,
+            cost: null,
+            duration_ms: Date.now() - input.startedAt,
+            error: input.error === undefined ? null : String(input.error),
+            timestamp: new Date(input.startedAt).toISOString()
+        });
+    }
+    /**
+     * Record a failed provider call: a server-side log with the exact request,
+     * and a `provider_call_failed` message so the surface showing the failure
+     * can report it without the user copying a status code out of a toast.
+     *
+     * One call, so the log and the report can never disagree about what failed.
+     * Public because the modality generator wrapper below is a free function.
+     * @internal
+     */
+    recordCallFailure(input) {
+        logProviderRequestFailure({
+            provider: this.provider,
+            model: input.model ?? "unknown",
+            request: input.request,
+            nodetoolArgs: input.nodetoolArgs,
+            error: input.error
+        });
+        if (!this._emitMessage)
+            return;
+        const failure = buildProviderCallFailure({
+            ...input,
+            provider: this.provider
+        });
+        if (failure)
+            this.emitMessage(failure);
+    }
+    /**
+     * Record the exact payload this provider is about to send to the upstream
+     * API. Call this right before the network request in `generateMessage` /
+     * `generateMessages`. On failure, the traced wrappers log precisely what was
+     * sent (secrets redacted, large fields truncated). Safe to call even outside
+     * a traced call — it is a no-op when no capture slot is active.
+     */
+    // HOLDOUT (anti-slop/no-unknown-parameters): see `setLastRequest` — the
+    // honest type is `object | null`, which `no-object-parameters` forbids.
+    recordRequestPayload(payload) {
+        setLastRequest(payload);
+    }
     constructor(provider) {
         this.provider = provider;
+        this.installModalityFailureLogging();
+        this.installVideoFrameFallback();
+    }
+    /**
+     * Chat already records what it sent and logs it on failure through
+     * {@link generateMessageTraced} / {@link generateMessagesTraced}. The
+     * non-chat modalities (image/video/audio/3D/embedding) are called directly,
+     * so wrap every one a concrete provider actually overrides with the same
+     * central failure logging.
+     *
+     * We replace methods on the instance (not the prototype), and only the ones
+     * that differ from the base — so an unsupported modality keeps its base
+     * "does not support" throw and never logs a spurious failure. Providers may
+     * additionally call {@link recordRequestPayload} to log the exact wire body;
+     * otherwise the call arguments are logged (binary buffers reduced to size
+     * markers, secrets redacted).
+     */
+    installModalityFailureLogging() {
+        for (const name of MODALITY_PROMISE_METHODS) {
+            const fn = this[name];
+            if (!isCallable(fn) || fn === BaseProvider.prototype[name]) {
+                continue;
+            }
+            // SAFETY: `name` comes from MODALITY_PROMISE_METHODS and the check above
+            // proved this instance overrides it, so `fn` is that modality method.
+            const original = fn;
+            Reflect.set(this, name, (...rawArgs) => withModalityCapture(async (alreadyActive) => {
+                // Central entity expansion: descriptors into the prompt, reference
+                // images onto the source list, before the concrete provider runs.
+                const args = applyEntityReferences(name, rawArgs);
+                const startedAt = Date.now();
+                try {
+                    const result = await original.apply(this, args);
+                    if (!alreadyActive) {
+                        this.recordModalityCall({ operation: name, args, startedAt });
+                    }
+                    return result;
+                }
+                catch (err) {
+                    annotateProviderError(err, {
+                        provider: this.provider,
+                        model: extractModelId(args)
+                    });
+                    if (!alreadyActive) {
+                        this.recordModalityCall({
+                            operation: name,
+                            args,
+                            startedAt,
+                            error: err
+                        });
+                        const failureArgs = name === "referenceToVideo"
+                            ? [referenceVideoDiagnostics(args)]
+                            : args;
+                        this.recordCallFailure({
+                            model: extractModelId(args),
+                            operation: name,
+                            request: name === "referenceToVideo" ? undefined : peekLastRequest(),
+                            nodetoolArgs: failureArgs,
+                            error: err,
+                            startedAt
+                        });
+                    }
+                    throw err;
+                }
+            }));
+        }
+        for (const name of MODALITY_GENERATOR_METHODS) {
+            const fn = this[name];
+            if (!isCallable(fn) || fn === BaseProvider.prototype[name]) {
+                continue;
+            }
+            const original = fn;
+            Reflect.set(this, name, (...args) => wrapModalityGenerator(this, original, args, name));
+        }
+    }
+    /**
+     * Make a `video` content part readable by a provider whose chat API has no
+     * place to put one: sample the clip into stills and send those instead.
+     *
+     * Wrapping the instance's own methods (the pattern
+     * {@link installModalityFailureLogging} uses) is what makes it transparent.
+     * Not every caller goes through the traced wrappers — `packages/llm-nodes`
+     * and the jtbd optimizer call `generateMessages` straight — and a seam only
+     * some callers pass through is not a fallback, it is a coin flip.
+     *
+     * A provider that reads video natively is left alone, and so is one whose
+     * `generateMessage` delegates to its own `generateMessages`: by then the
+     * video parts are already frames, and the second pass is a no-op.
+     */
+    installVideoFrameFallback() {
+        // An arrow, so the `async function*` below (which binds its own `this`)
+        // still reaches this instance.
+        const sample = (args) => this.sampleVideoAsFrames(args);
+        // `generateMessage` and `generateMessages` are abstract, so a subclass that
+        // implements only one leaves the other undefined at construction — wrap
+        // what is there, the way installModalityFailureLogging does.
+        const message = this.generateMessage;
+        if (isCallable(message)) {
+            const original = message.bind(this);
+            Reflect.set(this, "generateMessage", async (args) => original({ ...args, messages: await sample(args) }));
+        }
+        const messages = this.generateMessages;
+        if (isCallable(messages)) {
+            const original = messages.bind(this);
+            Reflect.set(this, "generateMessages", async function* (args) {
+                yield* original({
+                    ...args,
+                    messages: await sample(args)
+                });
+            });
+        }
+    }
+    /**
+     * The messages to actually send: unchanged unless this provider cannot take
+     * video and the conversation carries some.
+     */
+    async sampleVideoAsFrames(args) {
+        if (this.supportsVideoInput || !videoFrameFallbackEnabled()) {
+            return args.messages;
+        }
+        // Returns the same array when nothing carries video, which is every call
+        // but the rare one.
+        return await expandVideoContentAsFrames(args.messages, {
+            resolveUri: (uri) => this.resolveUri(uri),
+            signal: args.signal,
+            provider: this.provider
+        });
     }
     static requiredSecrets() {
         return [];
     }
+    /**
+     * Why this provider cannot serve a call right now, or `null` when it can.
+     *
+     * The credential check (`isProviderConfigured`) runs before a provider is
+     * ever constructed, so a provider that reaches this point has its keys. This
+     * answers what a key cannot: an optional SDK nobody installed, a local
+     * runtime the host lacks. `find_model` and `list_models` read it, so a model
+     * nothing here can run is not offered as the best match — the failure lands
+     * on the discovery call instead of on the paid generation call.
+     */
+    async unavailableReason() {
+        return null;
+    }
+    /**
+     * The operations beyond chat that this provider implements. Each concrete
+     * provider lists its own; the base implements none. A capability belongs
+     * here only when the matching method has a working implementation.
+     */
+    declaredCapabilities() {
+        return [];
+    }
+    /**
+     * The capabilities this provider exposes: chat generation, which every
+     * provider has, plus its {@link declaredCapabilities}.
+     */
+    getCapabilities() {
+        return [
+            ...new Set([
+                "generate_message",
+                "generate_messages",
+                ...this.declaredCapabilities()
+            ])
+        ];
+    }
+    /**
+     * The provider's own record of the generations this account ran, newest
+     * first. Only providers with a request-history API override this; the base
+     * throws {@link ProviderGenerationsUnsupportedError} so a caller can tell
+     * "this provider cannot answer" from "this provider ran nothing".
+     *
+     * This is the provider's truth, not NodeTool's: it covers generations made
+     * outside this installation and carries the billed cost rather than the
+     * estimate the local row holds until reconciliation.
+     */
+    async listGenerations(_query = {}) {
+        throw new ProviderGenerationsUnsupportedError(this.provider, "list");
+    }
+    /**
+     * One generation at the provider, by the provider's own request id — the id
+     * the local row keeps as `provider_request_id`. Returns null when the
+     * provider answers that it has no such request. Throws
+     * {@link ProviderGenerationsUnsupportedError} when the provider has no
+     * lookup API at all.
+     */
+    async getGeneration(_requestId, _options = {}) {
+        throw new ProviderGenerationsUnsupportedError(this.provider, "get");
+    }
     getContainerEnv() {
         return {};
     }
+    /**
+     * Release any resources held by this provider (HTTP connection pools, cached
+     * model metadata, child processes, etc.). Default is a no-op; providers that
+     * hold long-lived resources should override it. Safe to call more than once.
+     * Callers that own a provider instance should invoke this on shutdown.
+     */
+    async close() { }
     async hasToolSupport(_model) {
         return true;
     }
+    /**
+     * The input window, in tokens, this provider's own API reports for `model`,
+     * or null when it has no such endpoint. Callers go through
+     * `resolveContextWindow`, which falls back to models.dev.
+     */
+    async getContextWindow(_model) {
+        return null;
+    }
     trackUsage(model, usage) {
-        const cost = CostCalculator.calculate(model, usage, this.provider);
-        this._cost += cost;
-        setLastUsage({
+        // Accounting must never break a generation that already ran (and was
+        // already billed by the provider): keep the token counts and record the
+        // call as *unpriced*. A zero here would be written to the ledger as a real
+        // zero, which reads as "this call was free" — the one thing it is not.
+        let cost;
+        try {
+            cost = CostCalculator.calculate(model, usage, this.provider);
+        }
+        catch (error) {
+            const reason = error instanceof Error ? error.message : String(error);
+            this._unpricedReason = `No unit price for ${this.provider}/${model}: ${reason}`;
+            log.warn("Cost calculation failed; recording usage as unpriced", {
+                model,
+                provider: this.provider,
+                error: reason
+            });
+        }
+        this._cost += cost ?? 0;
+        this._usageTotals.inputTokens += usage.inputTokens ?? 0;
+        this._usageTotals.outputTokens += usage.outputTokens ?? 0;
+        this._usageTotals.cachedTokens += usage.cachedTokens ?? 0;
+        if (usage.inputTokens !== undefined) {
+            this._lastCallTokens = usage.inputTokens + (usage.outputTokens ?? 0);
+        }
+        const tracked = {
             inputTokens: usage.inputTokens ?? 0,
             outputTokens: usage.outputTokens ?? 0,
             cachedInputTokens: usage.cachedTokens,
-            cost
-        });
+            cacheWriteTokens: usage.cacheWriteTokens
+        };
+        // Left unset rather than zero: `setLastUsage` charges the enclosing node
+        // invocation with it, and an invented zero is spend nobody can audit.
+        if (cost !== undefined) {
+            tracked.cost = cost;
+        }
+        setLastUsage(tracked);
         log.debug("Cost tracked", { model, cost, total: this._cost });
-        return cost;
+        return cost ?? 0;
+    }
+    /**
+     * Why this provider's cost total is missing spend, when a price lookup
+     * failed. Null when every tracked call was priced. A host writing a ledger
+     * row reads it to record the row as unpriced (`cost: null`) instead of
+     * booking an unpriced call as free.
+     */
+    get unpricedReason() {
+        return this._unpricedReason;
     }
     /** Alias for getTotalCost() — mirrors Python's BaseProvider.cost property. */
     get cost() {
@@ -89,6 +533,20 @@ export class BaseProvider {
     getTotalCost() {
         return this._cost;
     }
+    /**
+     * Input (cache reads and writes included) plus output tokens of the most
+     * recent tracked call, as the provider reported them. The prompt of the next
+     * call on the same transcript is at least this large, and unlike a local
+     * estimate it counts the tool definitions and system prompt too. Null until
+     * a call reports usage.
+     */
+    get lastCallTokens() {
+        return this._lastCallTokens;
+    }
+    /** Running token counts across every call tracked on this instance. */
+    get usageTotals() {
+        return { ...this._usageTotals };
+    }
     /**
      * Log a provider call for cost tracking.
      * Base implementation logs to debug; subclasses or callers may persist to a Prediction model.
@@ -104,6 +562,9 @@ export class BaseProvider {
     }
     resetCost() {
         this._cost = 0;
+        this._unpricedReason = null;
+        this._usageTotals = { inputTokens: 0, outputTokens: 0, cachedTokens: 0 };
+        this._lastCallTokens = null;
     }
     async getAvailableLanguageModels() {
         return [];
@@ -120,9 +581,31 @@ export class BaseProvider {
     async getAvailableASRModels() {
         return [];
     }
+    /**
+     * Music **generation** models exposed by this provider (e.g. MusicGen,
+     * Stable Audio, Suno, MiniMax Music). Override on providers that can
+     * synthesize music from a text prompt; the base returns none so the
+     * `text_to_music` capability is advertised only when overridden.
+     */
+    async getAvailableMusicModels() {
+        return [];
+    }
+    /**
+     * Audio **transform** models exposed by this provider (voice changers, stem
+     * separators, denoisers, audio super-resolution). Override on providers that
+     * can rewrite a recording; the base returns none so the `audio_to_audio`
+     * capability is advertised only when overridden.
+     */
+    async getAvailableAudioToAudioModels() {
+        return [];
+    }
     async getAvailableEmbeddingModels() {
         return [];
     }
+    /** Reranking models exposed by this provider. */
+    async getAvailableRerankModels() {
+        return [];
+    }
     /**
      * 3D **generation** models exposed by this provider (e.g. Meshy's `meshy-4`,
      * Rodin's `rodin-regular`). Override on providers that can synthesize 3D
@@ -142,14 +625,16 @@ export class BaseProvider {
                 this.applyLlmRequestAttributes(span, args, false);
                 try {
                     const result = await this.generateMessage(args);
-                    const content = typeof result.content === "string"
+                    const content = isString(result.content)
                         ? result.content
-                        : JSON.stringify(result.content);
+                        : stringifyTraceContent(result.content);
                     span.setAttributes({
                         "llm.response.role": result.role,
-                        "llm.response.content": content.slice(0, 2000),
                         "llm.response.tool_calls_count": result.toolCalls?.length ?? 0
                     });
+                    if (traceContentAllowed()) {
+                        span.setAttribute("llm.response.content", traceTextContent(content));
+                    }
                     span.setStatus({ code: SpanStatusCode.OK });
                     return result;
                 }
@@ -170,18 +655,45 @@ export class BaseProvider {
         let result;
         let error;
         let usage = null;
+        let requestPayload = null;
+        const startedAt = Date.now();
         try {
             // withUsageCapture creates a per-call AsyncLocalStorage slot so
             // trackUsage() in the provider hands its numbers back to us.
             result = await withUsageCapture(async () => {
-                const r = await doCall();
-                usage = consumeLastUsage();
-                return r;
+                try {
+                    const r = await doCall();
+                    usage = consumeLastUsage();
+                    return r;
+                }
+                finally {
+                    // Read the recorded wire payload while still inside the capture slot.
+                    requestPayload = peekLastRequest();
+                }
             });
             return result;
         }
         catch (err) {
+            annotateProviderError(err, {
+                provider: this.provider,
+                model: args.model
+            });
+            if (this.isContextExceededError(err)) {
+                markContextExceeded(err, this.provider);
+            }
             error = String(err);
+            this.recordCallFailure({
+                model: args.model,
+                operation: "generateMessage",
+                request: requestPayload,
+                nodetoolArgs: {
+                    model: args.model,
+                    messages: args.messages,
+                    tools: args.tools
+                },
+                error: err,
+                startedAt
+            });
             throw err;
         }
         finally {
@@ -210,6 +722,10 @@ export class BaseProvider {
         }
     }
     applyLlmRequestAttributes(span, args, streaming) {
+        if (traceContentAllowed()) {
+            span.setAttribute("llm.request.messages", messageTraceContent(args.messages.filter((message) => typeof message === "object" && message !== null && "role" in message)));
+            span.setAttribute("llm.tool.names", stringifyTraceContent((args.tools ?? []).map((tool) => typeof tool === "object" && tool !== null && "name" in tool ? tool.name : null)));
+        }
         span.setAttributes({
             "llm.provider": this.provider,
             "llm.model": args.model,
@@ -245,12 +761,13 @@ export class BaseProvider {
         let error;
         // Per-call usage slot survives across the generator's yields by wrapping
         // each `next()` call in `runInSlot`.
-        const { runInSlot, getUsage } = createUsageSlot();
+        const { runInSlot, getUsage, getRequest } = createUsageSlot();
         const tracer = getTracer();
         const source = tracer
             ? this._tracedStream(args, tracer)
             : this.generateMessages(args);
         let exhausted = false;
+        const startedAt = Date.now();
         try {
             while (true) {
                 const result = await runInSlot(() => source.next());
@@ -280,7 +797,26 @@ export class BaseProvider {
             });
         }
         catch (err) {
+            annotateProviderError(err, {
+                provider: this.provider,
+                model: args.model
+            });
+            if (this.isContextExceededError(err)) {
+                markContextExceeded(err, this.provider);
+            }
             error = String(err);
+            this.recordCallFailure({
+                model: args.model,
+                operation: "generateMessages",
+                request: getRequest(),
+                nodetoolArgs: {
+                    model: args.model,
+                    messages: args.messages,
+                    tools: args.tools
+                },
+                error: err,
+                startedAt
+            });
             throw err;
         }
         finally {
@@ -288,7 +824,11 @@ export class BaseProvider {
             // called return()), give the underlying generator a chance to close
             // so _tracedStream's finally runs and the LLM stream span ends.
             if (!exhausted) {
-                await runInSlot(() => source.return?.(undefined) ?? Promise.resolve({ done: true, value: undefined })).catch(() => { });
+                await runInSlot(() => source.return?.(undefined) ??
+                    Promise.resolve({
+                        done: true,
+                        value: undefined
+                    })).catch(() => { });
             }
             const usage = getUsage();
             this.emitMessage({
@@ -311,17 +851,339 @@ export class BaseProvider {
             });
         }
     }
+    /**
+     * Run the agentic tool-calling loop and stream its events. This is the entry
+     * point an agent harness calls; the harness no longer implements the loop
+     * itself. The default implementation drives the standard external loop:
+     * stream one {@link generateMessages} turn, collect any tool calls, run them
+     * via the harness-supplied {@link executeTool}, append the results, and repeat
+     * until the model stops calling tools (or `maxIterations` is hit).
+     *
+     * It yields the same {@link Chunk}/{@link ToolCall}/{@link ProviderSessionUpdate}
+     * items a single turn does — for live display — plus a
+     * {@link ProviderMessageEvent} for every finalized message (assistant turns
+     * and tool results) so the harness can persist/collect them without
+     * reassembling them from chunks.
+     *
+     * Providers whose backend runs its own agent loop (e.g. the Claude Agent SDK)
+     * override this to drive that loop instead, while still emitting the same
+     * stream items. `generateMessages` remains the single-turn primitive.
+     */
+    /**
+     * Ask a {@link TurnBudget} to admit the turn that is about to be made.
+     * Exposed to subclasses because a provider driving a backend's own agent
+     * loop has to make the same call at the same point.
+     *
+     * Answers the reservation to hand back to `commit` after the turn, or `null`
+     * when the budget refused. The handle belongs to this turn alone: another
+     * loop sharing the budget settles its own.
+     */
+    _admitTurn(budget, model, messages) {
+        return budget.reserve({
+            model,
+            provider: this.provider,
+            inputTokens: estimatePromptTokens(messages)
+        });
+    }
+    async *generateLoop(args) {
+        return yield* tracedProviderLoop(this.provider, args.model, () => BaseProvider.prototype.generateBaseLoop.call(this, args));
+    }
+    async *generateBaseLoop(args) {
+        const maxIterations = args.maxIterations ?? 25;
+        const { executeTool, maxIterations: _omitMax, sequentialTools, providedToolsOnly: _omitProvidedToolsOnly, turnBudget: budgetArg, workspaceDir: _omitWorkspaceDir, skills: _omitSkills, resolveMedia, ...turnArgs } = args;
+        const { runBudget, turnBudget } = resolveTurnBudget(budgetArg);
+        const messages = [...args.messages];
+        // What the provider is actually sent: `messages` with every media
+        // reference dereferenced. Kept beside `messages` rather than replacing it,
+        // so the budget still reserves against the references (what the loop
+        // carries) and the tail is resolved once instead of once per round.
+        const sent = [];
+        let resolvedUpTo = 0;
+        const resolvedMessages = async () => {
+            if (!resolveMedia)
+                return messages;
+            if (resolvedUpTo < messages.length) {
+                sent.push(...(await resolveMedia(messages.slice(resolvedUpTo))));
+                resolvedUpTo = messages.length;
+            }
+            return sent;
+        };
+        // Tools may carry their own `execute` (provider-dispatched) and/or a
+        // `terminal` flag (end the loop after they run). Index by name so the
+        // tool-execution section can look up the matching ProviderTool per call.
+        const toolMap = new Map((args.tools ?? []).map((t) => [t.name, t]));
+        // Bridge to the legacy inline tool callback for any provider that executes
+        // tools mid-stream instead of yielding ToolCall items. Real providers yield
+        // ToolCall (handled below); this keeps the inline path working too. That
+        // channel is string-only, so image results collapse to their text.
+        let toolCallSeq = 0;
+        const onToolCall = executeTool
+            ? async (name, toolArgs) => {
+                const call = { id: `call_${++toolCallSeq}`, name, args: toolArgs };
+                const result = await tracedToolCall(call, () => executeTool(call));
+                if (isProviderToolErrorResult(result)) {
+                    return toolResultToText(result.content);
+                }
+                return isString(result) ? result : toolResultToText(result);
+            }
+            : turnArgs.onToolCall;
+        for (let iteration = 0; iteration < maxIterations; iteration++) {
+            if (turnArgs.signal?.aborted) {
+                yield PROVIDER_STOP_ABORTED;
+                return;
+            }
+            if (runBudget?.deadline.expired()) {
+                yield budgetStopItem(runBudget);
+                return;
+            }
+            let reservation = null;
+            if (turnBudget) {
+                reservation = this._admitTurn(turnBudget, args.model, messages);
+                if (reservation === null) {
+                    yield budgetStopItem(runBudget);
+                    return;
+                }
+            }
+            beginProviderRound(this.provider, args.model);
+            const costBeforeTurn = turnBudget ? this.getTotalCost() : 0;
+            let assistantText = "";
+            let finalizedAssistant;
+            const pending = [];
+            // The round's `done: true` chunk, withheld until we know if it ends the
+            // turn. Discarded when tools are pending — the next round supplies its own.
+            let terminalChunk;
+            // The reservation covers exactly one turn, so it is reconciled in a
+            // `finally`: a turn that throws must release it. Left outstanding, a
+            // single provider error would shrink the budget's headroom for good —
+            // it is run-level and outlives the decision that hit the error.
+            try {
+                for await (const item of this.generateMessagesTraced({
+                    ...turnArgs,
+                    messages: await resolvedMessages(),
+                    onToolCall
+                })) {
+                    if (turnArgs.signal?.aborted)
+                        break;
+                    if (isProviderMessageEvent(item)) {
+                        if (item.message.role === "assistant") {
+                            finalizedAssistant = item.message;
+                        }
+                        yield item;
+                        continue;
+                    }
+                    if (isProviderSessionUpdate(item)) {
+                        yield item;
+                        continue;
+                    }
+                    if ("id" in item && "name" in item && "args" in item) {
+                        pending.push(item);
+                        yield item;
+                        continue;
+                    }
+                    if (isProviderStop(item)) {
+                        // Only this loop ends a loop. A single turn never yields a stop,
+                        // so one arriving from `generateMessages` is not the loop's stop
+                        // and must not be forwarded as if it were.
+                        continue;
+                    }
+                    const chunk = item;
+                    // Only text belongs in the assistant message. Audio/image chunks carry
+                    // base64 in the same `content` field; concatenating those produced
+                    // assistant messages full of binary garbage.
+                    const isTextChunk = chunk.content_type === undefined || chunk.content_type === "text";
+                    if (isString(chunk.content) &&
+                        !chunk.thinking &&
+                        isTextChunk) {
+                        assistantText += chunk.content;
+                    }
+                    // `done: true` closes the provider's *completion*, which is not the same
+                    // as the end of the turn — every provider emits one per tool-calling
+                    // round. Forwarding them made consumers (the chat composer's Stop
+                    // button) treat each round as the end of the turn. Hold the terminal
+                    // chunk until we know whether tools are pending; release it below only
+                    // when this round really is the last one.
+                    if (chunk.done === true) {
+                        terminalChunk = item;
+                        continue;
+                    }
+                    yield item;
+                }
+            }
+            finally {
+                // Replace the reservation with what the turn really cost. A zero delta
+                // here is a real zero, not an unknown: `trackUsage` fires on every
+                // completed call, so nothing recorded means nothing was billed. (The
+                // Claude Agent SDK path cannot say that, which is why it commits
+                // `null` instead — see its override.)
+                if (turnBudget && reservation) {
+                    turnBudget.commit(reservation, this.getTotalCost() - costBeforeTurn);
+                }
+            }
+            if (pending.length === 0) {
+                if (terminalChunk !== undefined) {
+                    yield terminalChunk;
+                }
+                if (finalizedAssistant) {
+                    messages.push(finalizedAssistant);
+                }
+                else {
+                    const assistantMsg = {
+                        role: "assistant",
+                        content: assistantText || null
+                    };
+                    messages.push(assistantMsg);
+                    yield { type: "message", message: assistantMsg };
+                }
+                // An abort mid-stream cut the turn short; the model did not end it.
+                if (turnArgs.signal?.aborted) {
+                    yield PROVIDER_STOP_ABORTED;
+                }
+                return;
+            }
+            // The assistant turn requested tools — emit it (with the calls), run them,
+            // then loop with the results appended.
+            const assistantMsg = {
+                role: "assistant",
+                content: assistantText || null,
+                toolCalls: pending
+            };
+            // Carry Gemini thought-signature parts forward so multi-turn function
+            // calling keeps working — the loop owns the message now, so a tool call
+            // that arrived with raw parts can't stash them on the message itself.
+            const rawParts = pending.find((tc) => tc._rawGeminiParts)?._rawGeminiParts;
+            if (rawParts) {
+                assistantMsg._rawGeminiParts = rawParts;
+            }
+            const thinkingBlocks = pending.find((tc) => tc._anthropicThinkingBlocks)?._anthropicThinkingBlocks;
+            if (thinkingBlocks) {
+                assistantMsg._anthropicThinkingBlocks = thinkingBlocks;
+            }
+            messages.push(assistantMsg);
+            yield { type: "message", message: assistantMsg };
+            // Dispatch order: a tool's own `execute` (provider-driven) wins; else the
+            // harness-supplied `executeTool` callback; else the tool is unavailable.
+            const runTool = async (tc) => {
+                const tool = toolMap.get(tc.name);
+                try {
+                    if (tool?.execute) {
+                        const execute = tool.execute;
+                        return await tracedToolCall(tc, () => execute(tc.args ?? {}, tc.id));
+                    }
+                    if (executeTool)
+                        return await tracedToolCall(tc, () => executeTool(tc));
+                    return `Tool "${tc.name}" is not available`;
+                }
+                catch (err) {
+                    // A throwing tool must not destroy the whole assistant turn: turn the
+                    // failure into an error result so the model still receives output for
+                    // every tool call (and sibling results are preserved). Without this,
+                    // Promise.all in the parallel path would reject before any result is
+                    // emitted, discarding the turn.
+                    const message = err instanceof Error ? err.message : String(err);
+                    return {
+                        content: `Error executing tool "${tc.name}": ${message}`,
+                        isError: true
+                    };
+                }
+            };
+            // A tool flagged `terminal` ends the loop once its turn's results are
+            // emitted (e.g. a finish/submit tool).
+            let terminated = false;
+            const imageMessages = [];
+            const emitToolResult = function* (tc, result) {
+                let content;
+                let isError = false;
+                if (isProviderToolErrorResult(result)) {
+                    content = result.content;
+                    isError = true;
+                }
+                else {
+                    content = result;
+                }
+                const { toolContent, imageMessage } = splitToolResultImages(content);
+                const toolMsg = {
+                    role: "tool",
+                    toolCallId: tc.id,
+                    content: toolContent
+                };
+                if (isError) {
+                    toolMsg.isError = true;
+                }
+                messages.push(toolMsg);
+                yield { type: "message", message: toolMsg };
+                if (imageMessage) {
+                    // The pixels ride a user message for the next model turn (most
+                    // providers can't render an image in a tool-result message). It is
+                    // in-flight only — pushed to the loop's messages but never yielded,
+                    // so it is never persisted or echoed, keeping saved history cheap.
+                    imageMessages.push(imageMessage);
+                }
+            };
+            if (sequentialTools) {
+                // One at a time so a tool can read the state a prior tool wrote, and so
+                // a terminal tool can abort the loop mid-turn (skipping the rest).
+                for (const tc of pending) {
+                    if (turnArgs.signal?.aborted)
+                        break;
+                    yield* emitToolResult(tc, await runTool(tc));
+                    if (toolMap.get(tc.name)?.terminal) {
+                        terminated = true;
+                        break;
+                    }
+                }
+            }
+            else {
+                if (turnArgs.signal?.aborted) {
+                    yield PROVIDER_STOP_ABORTED;
+                    return;
+                }
+                const results = await Promise.all(pending.map(async (tc) => ({ tc, content: await runTool(tc) })));
+                for (const { tc, content } of results) {
+                    yield* emitToolResult(tc, content);
+                    if (toolMap.get(tc.name)?.terminal)
+                        terminated = true;
+                }
+            }
+            messages.push(...imageMessages);
+            // A terminal tool ran this turn — stop after emitting its results. That
+            // is the loop doing what it was asked, not a limit being hit, so it
+            // carries no stop item.
+            if (terminated)
+                return;
+        }
+        yield {
+            type: "stop",
+            reason: "iterations",
+            detail: `stopped after ${maxIterations} tool-calling round(s)`
+        };
+    }
     /** Internal: wrap generateMessages with OTel span */
     async *_tracedStream(args, tracer) {
-        const span = tracer.startSpan(`llm.stream ${this.provider}/${args.model}`);
+        const span = tracer.startSpan(`llm.stream ${this.provider}/${args.model}`, {}, providerRoundContext());
         this.applyLlmRequestAttributes(span, args, true);
         let chunkCount = 0;
+        let response = "";
+        const stream = this.generateMessages(args);
+        const active = trace.setSpan(providerRoundContext(), span);
+        let exhausted = false;
         try {
-            for await (const item of this.generateMessages(args)) {
+            while (true) {
+                const next = await context.with(active, () => stream.next());
+                if (next.done) {
+                    exhausted = true;
+                    break;
+                }
+                const item = next.value;
+                if (traceContentAllowed() && "content" in item && typeof item.content === "string" && !("thinking" in item && item.thinking) && (!("content_type" in item) || item.content_type === "text")) {
+                    response = traceTextContent(response + item.content);
+                }
                 chunkCount++;
                 yield item;
             }
-            span.setAttributes({ "llm.response.chunk_count": chunkCount });
+            span.setAttribute("llm.response.chunk_count", chunkCount);
+            if (traceContentAllowed()) {
+                span.setAttribute("llm.response.content", response);
+            }
             span.setStatus({ code: SpanStatusCode.OK });
         }
         catch (err) {
@@ -330,6 +1192,9 @@ export class BaseProvider {
             throw err;
         }
         finally {
+            if (!exhausted) {
+                await context.with(active, () => stream.return(undefined));
+            }
             applyUsageAttributes(span, peekLastUsage());
             span.end();
         }
@@ -351,7 +1216,14 @@ export class BaseProvider {
         }
         return results;
     }
-    async imageToImage(_image, _params) {
+    /**
+     * Transform one or more source images into a single output image.
+     *
+     * The first image is the primary subject; providers that accept multiple
+     * reference images (e.g. multi-image editing / composition) use the rest,
+     * while single-image providers operate on `images[0]`.
+     */
+    async imageToImage(_images, _params) {
         throw new Error(`${this.provider} does not support imageToImage`);
     }
     /**
@@ -360,13 +1232,76 @@ export class BaseProvider {
      * Default implementation calls imageToImage() sequentially as a fallback.
      * Providers that support native batch generation should override this.
      */
-    async imageToImages(image, params, numImages) {
+    async imageToImages(images, params, numImages) {
         const results = [];
         for (let i = 0; i < numImages; i++) {
-            results.push(await this.imageToImage(image, params));
+            results.push(await this.imageToImage(images, params));
         }
         return results;
     }
+    /**
+     * Inpaint: regenerate the masked region of one or more source images
+     * according to a prompt. Like {@link imageToImage} but mask-guided — the mask
+     * is routed to whatever mask input the target endpoint declares.
+     *
+     * The first image is the primary subject; the mask in `params.mask` marks the
+     * region to regenerate (white = regenerate).
+     */
+    async inpaint(_images, _params) {
+        throw new Error(`${this.provider} does not support inpaint`);
+    }
+    /**
+     * Generate multiple inpaint results in one logical call.
+     *
+     * Default implementation calls inpaint() sequentially as a fallback.
+     * Providers that support native batch generation should override this.
+     */
+    async inpaintImages(images, params, numImages) {
+        const results = [];
+        for (let i = 0; i < numImages; i++) {
+            results.push(await this.inpaint(images, params));
+        }
+        return results;
+    }
+    /**
+     * Outpaint (uncrop): extend an image past its frame, filling the added area
+     * with content that continues the scene. The first image is the subject;
+     * providers that take reference images use the rest.
+     */
+    async outpaintImage(_images, _params) {
+        throw new Error(`${this.provider} does not support outpaintImage`);
+    }
+    /** Increase the resolution / detail of an image. */
+    async upscaleImage(_image, _params) {
+        throw new Error(`${this.provider} does not support upscaleImage`);
+    }
+    /** Remove the background from an image, returning an image with alpha. */
+    async removeBackground(_image, _params) {
+        throw new Error(`${this.provider} does not support removeBackground`);
+    }
+    /**
+     * Estimate per-pixel depth. Returns a grayscale PNG at the input's size,
+     * where brighter pixels are nearer to the camera.
+     */
+    async estimateDepth(_image, _params) {
+        throw new Error(`${this.provider} does not support estimateDepth`);
+    }
+    /**
+     * Segment an image into object masks. Returns one entry per object the model
+     * found, in the model's own order; an empty array means it found nothing,
+     * which is an answer rather than a failure.
+     */
+    async segmentImage(_image, _params) {
+        throw new Error(`${this.provider} does not support segmentImage`);
+    }
+    /** Re-light a subject according to a prompt / background reference. */
+    async relightImage(_image, _params) {
+        throw new Error(`${this.provider} does not support relightImage`);
+    }
+    /** Convert a raster image into a vector (SVG) representation. */
+    async vectorizeImage(_image, _params) {
+        throw new Error(`${this.provider} does not support vectorizeImage`);
+    }
     async *textToSpeech(_args) {
         yield* [];
         throw new Error(`${this.provider} does not support textToSpeech`);
@@ -383,15 +1318,87 @@ export class BaseProvider {
     async textToSpeechEncoded(_args) {
         return null;
     }
+    /**
+     * True when this provider streams raw PCM via {@link textToSpeech}. Providers
+     * that only return encoded audio files (e.g. FAL/KIE) leave the base method in
+     * place and are consumed via {@link textToSpeechEncoded} instead.
+     */
+    supportsStreamingTextToSpeech() {
+        return this.textToSpeech !== BaseProvider.prototype.textToSpeech;
+    }
+    /**
+     * Generate music / instrumental audio from a text prompt. Music providers
+     * return a fully-encoded audio file (mp3/wav/flac), so this resolves to an
+     * {@link EncodedAudioResult} rather than streaming raw PCM. Providers that
+     * expose music models via {@link getAvailableMusicModels} must override this.
+     */
+    async textToMusic(_params) {
+        throw new Error(`${this.provider} does not support textToMusic`);
+    }
+    /**
+     * Optional video-conditioned sound task. Call through requestVideoToAudio
+     * for input and capability validation before dispatch.
+     */
+    async videoToAudio(_video, _params) {
+        throw new Error(`${this.provider} does not support videoToAudio`);
+    }
+    /**
+     * Rewrite a recording: convert the voice, isolate or denoise it, separate a
+     * stem, raise its sample rate. The source bytes are passed positionally, the
+     * direction rides in the params. Like {@link textToMusic} this resolves to an
+     * {@link EncodedAudioResult} — every provider that offers these endpoints
+     * hands back a file, not PCM.
+     */
+    async audioToAudio(_audio, _params) {
+        throw new Error(`${this.provider} does not support audioToAudio`);
+    }
     async automaticSpeechRecognition(_args) {
         throw new Error(`${this.provider} does not support automaticSpeechRecognition`);
     }
     async textToVideo(_params) {
         throw new Error(`${this.provider} does not support textToVideo`);
     }
+    /**
+     * Animate a single source image into a video. Reference media uses
+     * {@link referenceToVideo} so the input role remains explicit.
+     */
     async imageToVideo(_image, _params) {
         throw new Error(`${this.provider} does not support imageToVideo`);
     }
+    async referenceToVideo(_inputs, _params) {
+        throw new Error(`${this.provider} does not support referenceToVideo`);
+    }
+    /** Track a source-frame box and return samples in absolute source milliseconds. */
+    async trackObject(_video, _params) {
+        throw new Error(`${this.provider} does not support object tracking`);
+    }
+    /** Restyle / edit an existing video, guided by a prompt. */
+    async videoToVideo(_video, _params) {
+        throw new Error(`${this.provider} does not support videoToVideo`);
+    }
+    /** Add source time before or after an existing video window. */
+    async extendVideo(_video, _params) {
+        throw new Error(`${this.provider} does not support extendVideo`);
+    }
+    /** Increase the resolution / detail of a video. */
+    async upscaleVideo(_video, _params) {
+        throw new Error(`${this.provider} does not support upscaleVideo`);
+    }
+    /**
+     * Synthesize intermediate frames so a clip plays at a higher frame rate (or
+     * in slow motion at the same one).
+     */
+    async interpolateVideo(_video, _params) {
+        throw new Error(`${this.provider} does not support interpolateVideo`);
+    }
+    /** Extend a video past its frame, consistently across its frames. */
+    async outpaintVideo(_video, _params) {
+        throw new Error(`${this.provider} does not support outpaintVideo`);
+    }
+    /** Drive a face in a video/image to match the speech in an audio track. */
+    async lipSync(_video, _params) {
+        throw new Error(`${this.provider} does not support lipSync`);
+    }
     /**
      * Generate a 3D asset from a text prompt. Returns the encoded asset bytes
      * (typically GLB). Providers that support 3D generation should override.
@@ -410,6 +1417,20 @@ export class BaseProvider {
     async generateEmbedding(_args) {
         throw new Error(`${this.provider} does not support generateEmbedding`);
     }
+    /** Score each document against the query, most relevant first. */
+    async rerank(_params) {
+        throw new Error(`${this.provider} does not support rerank`);
+    }
+    /**
+     * Whether `error` is this provider refusing the request because it exceeds
+     * the model's context window. Default `false`: a provider whose signal has
+     * not been established must not have one guessed for it, because a false
+     * positive rewrites a transcript over an unrelated failure. Overridden by the
+     * providers in `context-exceeded.ts`.
+     */
+    isContextExceededError(_error) {
+        return false;
+    }
     isContextLengthError(error) {
         const msg = String(error).toLowerCase();
         return msg.includes("context length") || msg.includes("maximum context");
@@ -420,7 +1441,9 @@ export class BaseProvider {
      */
     isRateLimitError(error) {
         const msg = error instanceof Error ? error.message : String(error);
-        return /429|rate.?limit|too many requests/i.test(msg);
+        // Match 429 as a standalone token, not as digits inside a larger number
+        // (e.g. "request 4290", a byte count, or a model id like "gpt-4-0429").
+        return /\b429\b|rate.?limit|too many requests/i.test(msg);
     }
     /**
      * Check if an error is an authentication error (HTTP 401/403).
@@ -428,18 +1451,21 @@ export class BaseProvider {
      */
     isAuthError(error) {
         const msg = error instanceof Error ? error.message : String(error);
-        return /401|403|unauthorized|forbidden|invalid.*api.*key|authentication/i.test(msg);
+        // Match 401/403 as standalone tokens, not as digits inside a larger number
+        // (e.g. "model gpt-4-0403 not found") which would misclassify unrelated
+        // errors as auth failures and skip a retry that should have happened.
+        return /\b401\b|\b403\b|unauthorized|forbidden|invalid.*api.*key|authentication/i.test(msg);
     }
     parseToolCallArgs(raw) {
-        if (raw && typeof raw === "object" && !Array.isArray(raw)) {
+        if (isRecord(raw)) {
             return raw;
         }
-        if (typeof raw !== "string") {
+        if (!isString(raw)) {
             return {};
         }
         try {
             const parsed = JSON.parse(raw);
-            if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
+            if (isRecord(parsed)) {
                 return parsed;
             }
             return {};
@@ -458,42 +1484,202 @@ export class BaseProvider {
     /**
      * Resolve a URI to a `data:` URI that providers can consume directly.
      *
-     * - `file://...`        → read from disk, return `data:<mime>;base64,…`
+     * - `file://...`        → read an asset file, return `data:<mime>;base64,…`
      * - `/api/storage/<k>`  → read from getDefaultAssetsPath()/<k> (legacy fallback)
      * - Everything else     → returned unchanged (http/https/data: pass through)
      */
     async resolveUri(uri) {
-        const { readFile } = await import("node:fs/promises");
+        const { importNodeBuiltin } = await import("@nodetool-ai/config");
+        const fsP = await importNodeBuiltin("node:fs/promises");
+        if (!fsP) {
+            throw new Error("resolveUri requires node:fs/promises (Node-only feature)");
+        }
+        const pathMod = await importNodeBuiltin("node:path");
+        if (!pathMod) {
+            throw new Error("resolveUri requires node:path (Node-only feature)");
+        }
+        const getAssetsRoot = () => fsP.realpath(getDefaultAssetsPath());
+        const readAssetFile = async (filePath) => {
+            const assetsRoot = await getAssetsRoot();
+            const assertContained = (candidate) => {
+                const relative = pathMod.relative(assetsRoot, candidate);
+                if (relative.startsWith("..") || pathMod.isAbsolute(relative)) {
+                    throw new Error("Asset URI resolves outside the asset directory");
+                }
+            };
+            const requestedPath = pathMod.resolve(filePath);
+            assertContained(requestedPath);
+            const realPath = await fsP.realpath(requestedPath);
+            assertContained(realPath);
+            return await fsP.readFile(realPath);
+        };
         if (uri.startsWith("file://")) {
-            try {
-                const { fileURLToPath } = await import("node:url");
-                const filePath = fileURLToPath(uri);
-                const bytes = await readFile(filePath);
-                const ext = filePath.split(".").pop()?.toLowerCase() ?? "";
-                const mime = EXT_TO_MIME[ext] ?? "application/octet-stream";
-                return `data:${mime};base64,${bytes.toString("base64")}`;
-            }
-            catch {
-                return uri;
+            const urlMod = await importNodeBuiltin("node:url");
+            if (!urlMod) {
+                throw new Error("resolveUri file:// requires node:url");
             }
+            const filePath = urlMod.fileURLToPath(uri);
+            const bytes = await readAssetFile(filePath);
+            const ext = filePath.split(".").pop()?.toLowerCase() ?? "";
+            const mime = EXT_TO_MIME[ext] ?? "application/octet-stream";
+            return `data:${mime};base64,${Buffer.from(bytes).toString("base64")}`;
         }
         // Legacy: old messages stored with browser-facing /api/storage/ path
         if (uri.startsWith("/api/storage/")) {
             const key = uri.slice("/api/storage/".length);
             try {
-                const bytes = await readFile(getAssetFilePath(key));
+                const assetsRoot = await getAssetsRoot();
+                const decodedKey = decodeURIComponent(key);
+                const bytes = await readAssetFile(pathMod.resolve(assetsRoot, decodedKey.replace(/^\/+/, "")));
                 const ext = key.split(".").pop()?.toLowerCase() ?? "";
                 const mime = EXT_TO_MIME[ext] ?? "application/octet-stream";
-                return `data:${mime};base64,${bytes.toString("base64")}`;
+                return `data:${mime};base64,${Buffer.from(bytes).toString("base64")}`;
             }
-            catch {
+            catch (error) {
+                if (error instanceof Error &&
+                    error.message === "Asset URI resolves outside the asset directory") {
+                    throw error;
+                }
                 // Remote deployment: file not local, fall back to absolute HTTP
                 return `http://127.0.0.1:${process.env.PORT ?? 7777}${uri}`;
             }
         }
+        // A reference scheme reaching a provider means the caller never
+        // dereferenced it (see `ProcessingContext.resolveMessageMediaUris`). Say
+        // that, instead of handing `asset://…` to `fetch` and reporting the SSRF
+        // guard's "Refusing to fetch unsafe URL" — which names neither the asset
+        // nor the real cause.
+        if (uri.startsWith("asset://") || uri.startsWith("package://")) {
+            throw new Error(`Unresolved asset reference reached the model provider: ${uri}. ` +
+                `The asset bytes could not be loaded from storage.`);
+        }
         return uri;
     }
 }
+function referenceVideoDiagnostics(args) {
+    const inputs = args[0];
+    const params = args[1];
+    const media = isRecord(inputs) ? inputs : {};
+    const listBytes = (value) => Array.isArray(value)
+        ? value.reduce((total, item) => total + (item instanceof Uint8Array ? item.byteLength : 0), 0)
+        : 0;
+    return {
+        image_count: Array.isArray(media.images) ? media.images.length : 0,
+        video_count: Array.isArray(media.videos) ? media.videos.length : 0,
+        image_bytes: listBytes(media.images),
+        video_bytes: listBytes(media.videos),
+        model: extractModelId([params])
+    };
+}
+const MODALITY_PROMISE_METHODS = [
+    "textToImage",
+    "textToImages",
+    "imageToImage",
+    "imageToImages",
+    "inpaint",
+    "inpaintImages",
+    "outpaintImage",
+    "upscaleImage",
+    "removeBackground",
+    "estimateDepth",
+    "relightImage",
+    "vectorizeImage",
+    "segmentImage",
+    "textToSpeechEncoded",
+    "textToMusic",
+    "videoToAudio",
+    "automaticSpeechRecognition",
+    "textToVideo",
+    "imageToVideo",
+    "referenceToVideo",
+    "videoToVideo",
+    "trackObject",
+    "extendVideo",
+    "upscaleVideo",
+    "interpolateVideo",
+    "outpaintVideo",
+    "lipSync",
+    "textTo3D",
+    "imageTo3D",
+    "generateEmbedding",
+    "rerank"
+];
+/** Async-generator non-chat modality methods wrapped with failure logging. */
+const MODALITY_GENERATOR_METHODS = ["textToSpeech"];
+/**
+ * Best-effort model id for failure logging. Modality params carry the model
+ * either as a plain string (ASR, embeddings) or as a model object with an `id`
+ * (image/video tasks). Returns "unknown" when neither is present.
+ */
+function extractModelId(args) {
+    for (const arg of args) {
+        if (!isObjectLike(arg))
+            continue;
+        const model = arg.model;
+        if (isString(model))
+            return model;
+        if (isObjectLike(model)) {
+            const id = model.id;
+            if (isString(id))
+                return id;
+        }
+    }
+    return "unknown";
+}
+/**
+ * Drive a provider's async-generator modality method (e.g. textToSpeech)
+ * through a per-call slot — so `recordRequestPayload` is visible across yields —
+ * and log centrally if it throws. The slot is wrapped around each `next()`
+ * because AsyncLocalStorage does not persist across generator suspension.
+ */
+async function* wrapModalityGenerator(provider, original, args, operationName) {
+    const { runInSlot, getRequest } = createUsageSlot();
+    const startedAt = Date.now();
+    const source = original.apply(provider, args);
+    let exhausted = false;
+    let failure;
+    try {
+        while (true) {
+            const result = await runInSlot(() => source.next());
+            if (result.done) {
+                exhausted = true;
+                return result.value;
+            }
+            yield result.value;
+        }
+    }
+    catch (err) {
+        failure = err;
+        annotateProviderError(err, {
+            provider: provider.provider,
+            model: extractModelId(args)
+        });
+        provider.recordCallFailure({
+            model: extractModelId(args),
+            operation: operationName,
+            request: getRequest(),
+            nodetoolArgs: args,
+            error: err,
+            startedAt
+        });
+        throw err;
+    }
+    finally {
+        if (!exhausted) {
+            await runInSlot(() => source.return?.(undefined) ??
+                Promise.resolve({
+                    done: true,
+                    value: undefined
+                })).catch(() => { });
+        }
+        provider.recordModalityCall({
+            operation: operationName,
+            args,
+            startedAt,
+            error: failure
+        });
+    }
+}
 /** Attach gen_ai usage attributes to a span (no-op if usage is null). */
 function applyUsageAttributes(span, usage) {
     if (!usage)
@@ -505,14 +1691,21 @@ function applyUsageAttributes(span, usage) {
         ...(usage.cachedInputTokens !== undefined && {
             "gen_ai.usage.cached_input_tokens": usage.cachedInputTokens
         }),
-        ...(usage.cost !== undefined && { "gen_ai.usage.cost_credits": usage.cost })
+        ...(usage.cost !== undefined && { "gen_ai.usage.cost_usd": usage.cost })
     });
 }
 const EXT_TO_MIME = {
-    jpg: "image/jpeg", jpeg: "image/jpeg", png: "image/png",
-    gif: "image/gif", webp: "image/webp", svg: "image/svg+xml",
-    mp4: "video/mp4", webm: "video/webm",
-    mp3: "audio/mpeg", wav: "audio/wav", ogg: "audio/ogg",
+    jpg: "image/jpeg",
+    jpeg: "image/jpeg",
+    png: "image/png",
+    gif: "image/gif",
+    webp: "image/webp",
+    svg: "image/svg+xml",
+    mp4: "video/mp4",
+    webm: "video/webm",
+    mp3: "audio/mpeg",
+    wav: "audio/wav",
+    ogg: "audio/ogg",
     pdf: "application/pdf"
 };
 //# sourceMappingURL=base-provider.js.map