@nodetool-ai/runtime 0.7.0-rc.23 → 0.8.2

package/dist/providers/base-provider.d.ts

@@ -1,36 +1,256 @@
-import type { ASRModel, EmbeddingModel, ImageModel, ImageTo3DParams, ImageToImageParams, ImageToVideoParams, LanguageModel, Message, Model3D, ProviderId, ProviderStreamItem, ProviderTool, EncodedAudioResult, StreamingAudioChunk, TextTo3DParams, TextToImageParams, TextToVideoParams, ToolCall, TTSModel, VideoModel } from "./types.js";
+import type { ASRModel, EmbeddingModel, ImageModel, ImageTo3DParams, ImageToImageParams, InpaintingParams, ImageToVideoParams, ReferenceToVideoInputs, ReferenceToVideoParams, LanguageModel, LipSyncParams, InterpolateVideoParams, Message, MessageContent, Model3D, MusicModel, AudioToAudioModel, AudioToAudioParams, ProviderId, ProviderSession, ProviderStop, ProviderStreamItem, ProviderThinkingConfig, ProviderEffort, ProviderTool, ProviderSkill, ProviderToolResult, EncodedAudioResult, TextToMusicParams, TextToSpeechParams, RelightImageParams, RemoveBackgroundParams, EstimateDepthParams, RerankModel, RerankParams, RerankResult, SegmentImageParams, ImageSegmentationMask, StreamingAudioChunk, TextTo3DParams, TextToImageParams, TextToVideoParams, ToolCall, TTSModel, OutpaintImageParams, OutpaintVideoParams, UpscaleImageParams, UpscaleVideoParams, VectorizeImageParams, VideoModel, VideoToVideoParams, ExtendVideoParams } from "./types.js";
+import type { ProviderGeneration, ProviderGenerationLookup, ProviderGenerationPage, ProviderGenerationQuery } from "./provider-generations.js";
+import type { VideoToAudioParams } from "./video-to-audio.js";
+import type { ObjectTrackingParams, ObjectTrackingResult } from "./object-tracking.js";
 import type { UsageInfo } from "./cost-calculator.js";
+import type { RunBudget, TurnBudget, TurnReservationHandle } from "../turn-budget.js";
+import { type ProviderCallFailureInput } from "./provider-call-failure.js";
+/**
+ * Split the `turnBudget` argument into the run-level bounds (when the caller
+ * passed a {@link RunBudget}) and the per-turn admission object every loop
+ * reserves against. One argument serves both, so callers holding only a bare
+ * {@link TurnBudget} — the supervisor — keep working unchanged.
+ *
+ * A free function rather than a method: `generateLoop` is called unbound on
+ * duck-typed test providers (`BaseProvider.prototype.generateLoop.call(mock)`),
+ * where `this` has no class methods.
+ */
+export declare function resolveTurnBudget(budget: TurnBudget | RunBudget | undefined): {
+    runBudget: RunBudget | null;
+    turnBudget: TurnBudget | undefined;
+};
+/**
+ * The stop item for a refused turn. A {@link RunBudget} says which ceiling
+ * refused it; a bare {@link TurnBudget} only knows that one did.
+ */
+export declare function budgetStopItem(runBudget: RunBudget | null): ProviderStop;
+/**
+ * Collapse a tool result to plain text. Used by string-only tool channels (the
+ * legacy inline callback, the Claude Agent SDK's MCP tools) when a tool returns
+ * {@link MessageContent} blocks they can't render — keep the text, drop images.
+ */
+export declare function toolResultToText(result: string | MessageContent[]): string;
+/**
+ * Split a tool result into the text that belongs in the `role:"tool"` message
+ * and, when the result carries images, a follow-up `role:"user"` message that
+ * actually carries the pixels.
+ *
+ * Most providers cannot render an image inside a tool-result message — OpenAI
+ * stringifies it to a base64 *text* blob (massive token blowup, and the model
+ * never sees an image). An image content block is only honored on a user (or
+ * assistant) message, so image-bearing tool results must ride a user message.
+ * Returns `imageMessage: null` for plain (text/no-image) results.
+ */
+export type ToolResultSplit = {
+    toolContent: string | MessageContent[];
+    imageMessage: Message | null;
+};
+export declare function splitToolResultImages(result: string | MessageContent[]): ToolResultSplit;
 /**
  * Capability names a provider can expose. These correspond to the `capability`
  * strings consumed by client-side filters like `useImageModelProviders`,
  * `useTTSProviders`, etc.
  */
-export type ProviderCapability = "generate_message" | "generate_messages" | "text_to_image" | "image_to_image" | "text_to_video" | "image_to_video" | "text_to_speech" | "automatic_speech_recognition" | "generate_embedding" | "text_to_3d" | "image_to_3d";
+export type ProviderCapability = "generate_message" | "generate_messages" | "text_to_image" | "image_to_image" | "inpainting" | "outpaint_image" | "upscale_image" | "remove_background" | "relight_image" | "segment_image" | "vectorize_image" | "estimate_depth" | "text_to_video" | "image_to_video" | "reference_to_video" | "video_to_video" | "extend_video" | "upscale_video" | "interpolate_video" | "outpaint_video" | "lip_sync" | "text_to_speech" | "text_to_music" | "audio_to_audio" | "video_to_audio" | "automatic_speech_recognition" | "generate_embedding" | "rerank" | "text_to_3d" | "image_to_3d" | "list_generations" | "get_generation" | "track_object";
 /**
- * Derive a provider's capability set by checking which optional `getAvailable*`
- * methods it overrides on its prototype. Every provider can always generate
- * messages; image/video/TTS/ASR/embedding are advertised only when the concrete
- * class overrides the matching base method.
+ * Prompt tokens a turn will send, for reservation purposes. Deliberately a
+ * rough count: a reservation is a worst case, and a per-provider exact count
+ * would cost a tokenizer round-trip per turn to refine a number that only has
+ * to be an upper-ish bound.
  *
- * Shared by `packages/websocket/src/models-api.ts` (REST leftover) and the
- * `models` tRPC router so both report identical capabilities for the same
- * provider instance.
+ * Text is tokenized. Media blocks get a fixed per-modality figure, so an
+ * inlined `data:` uri is never fed to the tokenizer.
+ */
+export declare function estimatePromptTokens(messages: readonly Message[]): number;
+/**
+ * Running token counts for one provider instance, alongside its running
+ * `cost`. A caller that owns a stretch of work — a workflow node driving
+ * `generateLoop` — takes a before/after snapshot and reports the difference as
+ * that node's spend. Monotonic until {@link BaseProvider.resetCost}.
  */
-export declare function providerCapabilities(instance: BaseProvider): ProviderCapability[];
+export interface ProviderUsageTotals {
+    inputTokens: number;
+    outputTokens: number;
+    cachedTokens: number;
+}
 export declare abstract class BaseProvider {
     readonly provider: ProviderId;
     private _cost;
+    private _usageTotals;
+    private _unpricedReason;
+    private _lastCallTokens;
     private _emitMessage;
+    /**
+     * Whether this provider has a built-in, server-side web search it runs
+     * itself (e.g. Anthropic's `web_search_20250305` server tool, the Claude
+     * Agent SDK's `WebSearch`). When `true`, a tool named
+     * {@link WEB_SEARCH_TOOL_NAME} is fulfilled by the provider natively rather
+     * than the SerpAPI-backed `WebSearchTool`. Default `false` (use the fallback).
+     */
+    get supportsNativeWebSearch(): boolean;
+    /**
+     * Whether this provider has a built-in, server-side image generator (e.g.
+     * the OpenAI Responses `image_generation` tool). When `true`, a tool named
+     * {@link IMAGE_GENERATION_TOOL_NAME} is rendered as the provider's native
+     * tool — its result arrives as image content on the assistant message, not
+     * as a function-call round-trip. Default `false` (use the fallback function
+     * tool).
+     */
+    get supportsNativeImageGeneration(): boolean;
+    /**
+     * Whether this provider's chat API accepts a `video` content part. Only
+     * providers that read a whole clip natively answer `true` — an
+     * OpenAI-compatible endpoint re-hosting a video-capable model (AtlasCloud
+     * serving `google/gemini-2.5-flash`, say) does not, because the wire format
+     * has nowhere to put the bytes. Default `false`.
+     */
+    get supportsVideoInput(): boolean;
     setMessageEmitter(fn: (msg: unknown) => void): void;
     protected emitMessage(msg: unknown): void;
+    /**
+     * Report one non-chat provider call (image, video, speech, ASR, embedding…)
+     * as an `llm_call` so the trace shows every provider call, not only chat.
+     * Carries no token counts: those modalities bill per unit, not per token.
+     * @internal
+     */
+    recordModalityCall(input: {
+        operation: string;
+        args: unknown[];
+        startedAt: number;
+        error?: unknown;
+    }): void;
+    /**
+     * Record a failed provider call: a server-side log with the exact request,
+     * and a `provider_call_failed` message so the surface showing the failure
+     * can report it without the user copying a status code out of a toast.
+     *
+     * One call, so the log and the report can never disagree about what failed.
+     * Public because the modality generator wrapper below is a free function.
+     * @internal
+     */
+    recordCallFailure(input: Omit<ProviderCallFailureInput, "provider">): void;
+    /**
+     * Record the exact payload this provider is about to send to the upstream
+     * API. Call this right before the network request in `generateMessage` /
+     * `generateMessages`. On failure, the traced wrappers log precisely what was
+     * sent (secrets redacted, large fields truncated). Safe to call even outside
+     * a traced call — it is a no-op when no capture slot is active.
+     */
+    protected recordRequestPayload(payload: unknown): void;
     protected constructor(provider: ProviderId);
+    /**
+     * Chat already records what it sent and logs it on failure through
+     * {@link generateMessageTraced} / {@link generateMessagesTraced}. The
+     * non-chat modalities (image/video/audio/3D/embedding) are called directly,
+     * so wrap every one a concrete provider actually overrides with the same
+     * central failure logging.
+     *
+     * We replace methods on the instance (not the prototype), and only the ones
+     * that differ from the base — so an unsupported modality keeps its base
+     * "does not support" throw and never logs a spurious failure. Providers may
+     * additionally call {@link recordRequestPayload} to log the exact wire body;
+     * otherwise the call arguments are logged (binary buffers reduced to size
+     * markers, secrets redacted).
+     */
+    private installModalityFailureLogging;
+    /**
+     * Make a `video` content part readable by a provider whose chat API has no
+     * place to put one: sample the clip into stills and send those instead.
+     *
+     * Wrapping the instance's own methods (the pattern
+     * {@link installModalityFailureLogging} uses) is what makes it transparent.
+     * Not every caller goes through the traced wrappers — `packages/llm-nodes`
+     * and the jtbd optimizer call `generateMessages` straight — and a seam only
+     * some callers pass through is not a fallback, it is a coin flip.
+     *
+     * A provider that reads video natively is left alone, and so is one whose
+     * `generateMessage` delegates to its own `generateMessages`: by then the
+     * video parts are already frames, and the second pass is a no-op.
+     */
+    private installVideoFrameFallback;
+    /**
+     * The messages to actually send: unchanged unless this provider cannot take
+     * video and the conversation carries some.
+     */
+    private sampleVideoAsFrames;
     static requiredSecrets(): string[];
+    /**
+     * Why this provider cannot serve a call right now, or `null` when it can.
+     *
+     * The credential check (`isProviderConfigured`) runs before a provider is
+     * ever constructed, so a provider that reaches this point has its keys. This
+     * answers what a key cannot: an optional SDK nobody installed, a local
+     * runtime the host lacks. `find_model` and `list_models` read it, so a model
+     * nothing here can run is not offered as the best match — the failure lands
+     * on the discovery call instead of on the paid generation call.
+     */
+    unavailableReason(): Promise<string | null>;
+    /**
+     * The operations beyond chat that this provider implements. Each concrete
+     * provider lists its own; the base implements none. A capability belongs
+     * here only when the matching method has a working implementation.
+     */
+    protected declaredCapabilities(): readonly ProviderCapability[];
+    /**
+     * The capabilities this provider exposes: chat generation, which every
+     * provider has, plus its {@link declaredCapabilities}.
+     */
+    getCapabilities(): ProviderCapability[];
+    /**
+     * The provider's own record of the generations this account ran, newest
+     * first. Only providers with a request-history API override this; the base
+     * throws {@link ProviderGenerationsUnsupportedError} so a caller can tell
+     * "this provider cannot answer" from "this provider ran nothing".
+     *
+     * This is the provider's truth, not NodeTool's: it covers generations made
+     * outside this installation and carries the billed cost rather than the
+     * estimate the local row holds until reconciliation.
+     */
+    listGenerations(_query?: ProviderGenerationQuery): Promise<ProviderGenerationPage>;
+    /**
+     * One generation at the provider, by the provider's own request id — the id
+     * the local row keeps as `provider_request_id`. Returns null when the
+     * provider answers that it has no such request. Throws
+     * {@link ProviderGenerationsUnsupportedError} when the provider has no
+     * lookup API at all.
+     */
+    getGeneration(_requestId: string, _options?: ProviderGenerationLookup): Promise<ProviderGeneration | null>;
     getContainerEnv(): Record<string, string>;
+    /**
+     * Release any resources held by this provider (HTTP connection pools, cached
+     * model metadata, child processes, etc.). Default is a no-op; providers that
+     * hold long-lived resources should override it. Safe to call more than once.
+     * Callers that own a provider instance should invoke this on shutdown.
+     */
+    close(): Promise<void>;
     hasToolSupport(_model: string): Promise<boolean>;
+    /**
+     * The input window, in tokens, this provider's own API reports for `model`,
+     * or null when it has no such endpoint. Callers go through
+     * `resolveContextWindow`, which falls back to models.dev.
+     */
+    getContextWindow(_model: string): Promise<number | null>;
     trackUsage(model: string, usage: UsageInfo): number;
+    /**
+     * Why this provider's cost total is missing spend, when a price lookup
+     * failed. Null when every tracked call was priced. A host writing a ledger
+     * row reads it to record the row as unpriced (`cost: null`) instead of
+     * booking an unpriced call as free.
+     */
+    get unpricedReason(): string | null;
     /** Alias for getTotalCost() — mirrors Python's BaseProvider.cost property. */
     get cost(): number;
     getTotalCost(): number;
+    /**
+     * Input (cache reads and writes included) plus output tokens of the most
+     * recent tracked call, as the provider reported them. The prompt of the next
+     * call on the same transcript is at least this large, and unlike a local
+     * estimate it counts the tool definitions and system prompt too. Null until
+     * a call reports usage.
+     */
+    get lastCallTokens(): number | null;
+    /** Running token counts across every call tracked on this instance. */
+    get usageTotals(): ProviderUsageTotals;
     /**
      * Log a provider call for cost tracking.
      * Base implementation logs to debug; subclasses or callers may persist to a Prediction model.
@@ -48,7 +268,23 @@ export declare abstract class BaseProvider {
     getAvailableVideoModels(): Promise<VideoModel[]>;
     getAvailableTTSModels(): Promise<TTSModel[]>;
     getAvailableASRModels(): Promise<ASRModel[]>;
+    /**
+     * Music **generation** models exposed by this provider (e.g. MusicGen,
+     * Stable Audio, Suno, MiniMax Music). Override on providers that can
+     * synthesize music from a text prompt; the base returns none so the
+     * `text_to_music` capability is advertised only when overridden.
+     */
+    getAvailableMusicModels(): Promise<MusicModel[]>;
+    /**
+     * Audio **transform** models exposed by this provider (voice changers, stem
+     * separators, denoisers, audio super-resolution). Override on providers that
+     * can rewrite a recording; the base returns none so the `audio_to_audio`
+     * capability is advertised only when overridden.
+     */
+    getAvailableAudioToAudioModels(): Promise<AudioToAudioModel[]>;
     getAvailableEmbeddingModels(): Promise<EmbeddingModel[]>;
+    /** Reranking models exposed by this provider. */
+    getAvailableRerankModels(): Promise<RerankModel[]>;
     /**
      * 3D **generation** models exposed by this provider (e.g. Meshy's `meshy-4`,
      * Rodin's `rodin-regular`). Override on providers that can synthesize 3D
@@ -66,8 +302,16 @@ export declare abstract class BaseProvider {
         topP?: number;
         presencePenalty?: number;
         frequencyPenalty?: number;
+        thinking?: ProviderThinkingConfig;
+        effort?: ProviderEffort;
         /** Optional thread/conversation identifier for session-based providers. */
         threadId?: string | null;
+        /**
+         * Opaque continuation token from a prior turn (read off the last assistant
+         * message). Providers that support server/SDK-side sessions resume from it
+         * and send only the delta; stateless providers ignore it.
+         */
+        providerSession?: ProviderSession | null;
         /** Optional callback for native tool execution (used by providers with in-process MCP). */
         onToolCall?: (name: string, args: Record<string, unknown>) => Promise<string>;
         /** Optional signal to abort the request. */
@@ -82,7 +326,7 @@ export declare abstract class BaseProvider {
         maxTokens?: number;
         /**
          * Upper bound on internal agentic turns for providers that handle
-         * multi-turn tool execution within a single call (e.g. claude_agent).
+         * multi-turn tool execution within a single call.
          * Ignored by providers that delegate the loop to the caller.
          */
         maxTurns?: number;
@@ -90,9 +334,25 @@ export declare abstract class BaseProvider {
         topP?: number;
         presencePenalty?: number;
         frequencyPenalty?: number;
+        thinking?: ProviderThinkingConfig;
+        effort?: ProviderEffort;
         audio?: Record<string, unknown>;
         /** Optional thread/conversation identifier for session-based providers. */
         threadId?: string | null;
+        /**
+         * Opaque continuation token from a prior turn (read off the last assistant
+         * message). Providers that support server/SDK-side sessions resume from it
+         * and send only the delta; stateless providers ignore it.
+         */
+        providerSession?: ProviderSession | null;
+        /**
+         * Lazily fetch the full conversation. When a session-based provider is
+         * handed only the new delta in `messages` (to avoid reloading the whole
+         * thread), it calls this to recover the complete history if it must prime
+         * context — i.e. when a resume fails or the system prompt changed. Absent
+         * when `messages` already holds the full history.
+         */
+        loadFullHistory?: () => Promise<Message[]>;
         /** Optional callback for native tool execution (used by providers with in-process MCP). */
         onToolCall?: (name: string, args: Record<string, unknown>) => Promise<string>;
         /** Optional signal to abort the request. */
@@ -103,6 +363,125 @@ export declare abstract class BaseProvider {
     private applyLlmRequestAttributes;
     /** Traced wrapper around generateMessages. Use this instead of calling generateMessages directly. */
     generateMessagesTraced(args: Parameters<this["generateMessages"]>[0]): AsyncGenerator<ProviderStreamItem>;
+    /**
+     * Run the agentic tool-calling loop and stream its events. This is the entry
+     * point an agent harness calls; the harness no longer implements the loop
+     * itself. The default implementation drives the standard external loop:
+     * stream one {@link generateMessages} turn, collect any tool calls, run them
+     * via the harness-supplied {@link executeTool}, append the results, and repeat
+     * until the model stops calling tools (or `maxIterations` is hit).
+     *
+     * It yields the same {@link Chunk}/{@link ToolCall}/{@link ProviderSessionUpdate}
+     * items a single turn does — for live display — plus a
+     * {@link ProviderMessageEvent} for every finalized message (assistant turns
+     * and tool results) so the harness can persist/collect them without
+     * reassembling them from chunks.
+     *
+     * Providers whose backend runs its own agent loop (e.g. the Claude Agent SDK)
+     * override this to drive that loop instead, while still emitting the same
+     * stream items. `generateMessages` remains the single-turn primitive.
+     */
+    /**
+     * Ask a {@link TurnBudget} to admit the turn that is about to be made.
+     * Exposed to subclasses because a provider driving a backend's own agent
+     * loop has to make the same call at the same point.
+     *
+     * Answers the reservation to hand back to `commit` after the turn, or `null`
+     * when the budget refused. The handle belongs to this turn alone: another
+     * loop sharing the budget settles its own.
+     */
+    protected _admitTurn(budget: TurnBudget, model: string, messages: readonly Message[]): TurnReservationHandle | null;
+    generateLoop(args: Parameters<this["generateMessages"]>[0] & {
+        /**
+         * Execute one tool call and return the result to feed back to the model
+         * (and to store in the tool message). Usually result text; a tool may
+         * instead return {@link MessageContent} blocks (e.g. an image) for
+         * vision-capable providers. The harness owns tool resolution, gating, and
+         * side effects; the provider only orchestrates.
+         */
+        executeTool?: (toolCall: ToolCall) => Promise<ProviderToolResult>;
+        /** Cap on tool-calling rounds before stopping. Defaults to 25. */
+        maxIterations?: number;
+        /**
+         * Run a turn's tool calls one at a time (in array order) instead of in
+         * parallel. Required by consumers whose tools mutate shared state per call
+         * and read it back before the next (e.g. the planner's plan builder), and
+         * by consumers that abort the loop from inside a terminal tool. Defaults to
+         * false (parallel) to keep chat's concurrent tool execution fast.
+         */
+        sequentialTools?: boolean;
+        /**
+         * Restrict this loop to the supplied tools. Defaults to false, retaining
+         * provider-specific tool defaults. Overrides must not add implicit native
+         * tools or skill tools outside the supplied set when true.
+         */
+        providedToolsOnly?: boolean;
+        /**
+         * Spend admission, consulted before every model turn. A refusal ends the
+         * loop without making the call.
+         *
+         * **Contract for overrides.** A provider that overrides `generateLoop`
+         * to drive a backend's own agent loop must call {@link _admitTurn}
+         * before each model turn and `commit` after it — a ceiling only holds
+         * where the turns are, and an override that ignores this makes the cap
+         * advisory for every model it serves. Both current overrides honor it
+         * (the Claude Agent SDK loop and OpenAI's Responses loop). TypeScript
+         * cannot enforce this, so a new override that owns its turns owns this
+         * too — including yielding one {@link ProviderStop} as its final item
+         * when a refusal, rather than the model, ended the loop. Returning
+         * silently reads to every consumer as a finished answer.
+         *
+         * Accepts a {@link RunBudget} as well, and then also honors that run's
+         * deadline and cumulative turn count; the stop item names which ceiling
+         * refused the turn.
+         *
+         * Reserve against the *evolving* transcript, not the opening prompt: a
+         * backend that keeps the conversation server-side still bills for it,
+         * and reserving against the delta prices a turn as if the conversation
+         * had restarted.
+         *
+         * Reconciliation reads this provider instance's running cost, so a
+         * budget must not be threaded through an instance two callers are
+         * driving concurrently; the supervisor owns its provider for exactly
+         * this reason.
+         */
+        turnBudget?: TurnBudget | RunBudget;
+        /**
+         * The run's workspace directory. Only providers that drive their own
+         * agent loop over host tools read it — the Claude Agent SDK uses it as
+         * the session `cwd` so its path-scoped built-ins resolve where the
+         * NodeTool tools they replace would have. Ignored here.
+         */
+        workspaceDir?: string;
+        /**
+         * User-defined skills to expose to a provider that drives its own agent
+         * loop with a native skill mechanism (the Claude Agent SDK). That
+         * override materializes them as an isolated local plugin so its `Skill`
+         * tool can load them on demand. Ignored by this base loop and every
+         * provider that doesn't own its loop — those receive skills baked into
+         * the system prompt by the caller instead.
+         */
+        skills?: ProviderSkill[];
+        /**
+         * Dereference the media references a message carries, just before the
+         * request that needs the bytes. Usually
+         * `ProcessingContext.resolveMessageMediaUris`.
+         *
+         * Tool results are appended inside this loop, so a host cannot resolve
+         * them the way it resolves the opening history. Doing it here rather than
+         * at the tool keeps pixels out of the conversation entirely: `view_image`
+         * answers with an `asset://` reference, the transcript and the budget
+         * estimate carry that reference, and the base64 exists only in the one
+         * request body it is needed for. A ~200KB screenshot inlined into the
+         * transcript instead estimates at ~190k tokens and is re-sent every
+         * round.
+         *
+         * Only messages appended since the last turn are resolved, so a loop that
+         * looks at three images reads three assets, not three per round.
+         */
+        resolveMedia?: (messages: Message[]) => Promise<Message[]>;
+    }): AsyncGenerator<ProviderStreamItem>;
+    private generateBaseLoop;
     /** Internal: wrap generateMessages with OTel span */
     private _tracedStream;
     textToImage(_params: TextToImageParams): Promise<Uint8Array>;
@@ -114,25 +493,63 @@ export declare abstract class BaseProvider {
      * should override this for efficiency.
      */
     textToImages(params: TextToImageParams, numImages: number): Promise<Uint8Array[]>;
-    imageToImage(_image: Uint8Array, _params: ImageToImageParams): Promise<Uint8Array>;
+    /**
+     * Transform one or more source images into a single output image.
+     *
+     * The first image is the primary subject; providers that accept multiple
+     * reference images (e.g. multi-image editing / composition) use the rest,
+     * while single-image providers operate on `images[0]`.
+     */
+    imageToImage(_images: Uint8Array[], _params: ImageToImageParams): Promise<Uint8Array>;
     /**
      * Generate multiple image-to-image results in one logical call.
      *
      * Default implementation calls imageToImage() sequentially as a fallback.
      * Providers that support native batch generation should override this.
      */
-    imageToImages(image: Uint8Array, params: ImageToImageParams, numImages: number): Promise<Uint8Array[]>;
-    textToSpeech(_args: {
-        text: string;
-        model: string;
-        voice?: string;
-        speed?: number;
-        /**
-         * Requested output container. Providers that stream raw PCM may ignore
-         * this hint; the caller is responsible for wrapping/encoding the result.
-         */
-        audioFormat?: string;
-    }): AsyncGenerator<StreamingAudioChunk>;
+    imageToImages(images: Uint8Array[], params: ImageToImageParams, numImages: number): Promise<Uint8Array[]>;
+    /**
+     * Inpaint: regenerate the masked region of one or more source images
+     * according to a prompt. Like {@link imageToImage} but mask-guided — the mask
+     * is routed to whatever mask input the target endpoint declares.
+     *
+     * The first image is the primary subject; the mask in `params.mask` marks the
+     * region to regenerate (white = regenerate).
+     */
+    inpaint(_images: Uint8Array[], _params: InpaintingParams): Promise<Uint8Array>;
+    /**
+     * Generate multiple inpaint results in one logical call.
+     *
+     * Default implementation calls inpaint() sequentially as a fallback.
+     * Providers that support native batch generation should override this.
+     */
+    inpaintImages(images: Uint8Array[], params: InpaintingParams, numImages: number): Promise<Uint8Array[]>;
+    /**
+     * Outpaint (uncrop): extend an image past its frame, filling the added area
+     * with content that continues the scene. The first image is the subject;
+     * providers that take reference images use the rest.
+     */
+    outpaintImage(_images: Uint8Array[], _params: OutpaintImageParams): Promise<Uint8Array>;
+    /** Increase the resolution / detail of an image. */
+    upscaleImage(_image: Uint8Array, _params: UpscaleImageParams): Promise<Uint8Array>;
+    /** Remove the background from an image, returning an image with alpha. */
+    removeBackground(_image: Uint8Array, _params: RemoveBackgroundParams): Promise<Uint8Array>;
+    /**
+     * Estimate per-pixel depth. Returns a grayscale PNG at the input's size,
+     * where brighter pixels are nearer to the camera.
+     */
+    estimateDepth(_image: Uint8Array, _params: EstimateDepthParams): Promise<Uint8Array>;
+    /**
+     * Segment an image into object masks. Returns one entry per object the model
+     * found, in the model's own order; an empty array means it found nothing,
+     * which is an answer rather than a failure.
+     */
+    segmentImage(_image: Uint8Array, _params: SegmentImageParams): Promise<ImageSegmentationMask[]>;
+    /** Re-light a subject according to a prompt / background reference. */
+    relightImage(_image: Uint8Array, _params: RelightImageParams): Promise<Uint8Array>;
+    /** Convert a raster image into a vector (SVG) representation. */
+    vectorizeImage(_image: Uint8Array, _params: VectorizeImageParams): Promise<Uint8Array>;
+    textToSpeech(_args: TextToSpeechParams): AsyncGenerator<StreamingAudioChunk>;
     /**
      * Override this for providers that return fully-encoded audio (FLAC, WAV,
      * MP3) instead of raw PCM samples. Returns null by default, meaning the
@@ -142,13 +559,33 @@ export declare abstract class BaseProvider {
      * when possible and fall back to their default when the format is not
      * supported. The returned `mimeType` reflects the actual bytes produced.
      */
-    textToSpeechEncoded(_args: {
-        text: string;
-        model: string;
-        voice?: string;
-        speed?: number;
-        audioFormat?: string;
-    }): Promise<EncodedAudioResult | null>;
+    textToSpeechEncoded(_args: TextToSpeechParams): Promise<EncodedAudioResult | null>;
+    /**
+     * True when this provider streams raw PCM via {@link textToSpeech}. Providers
+     * that only return encoded audio files (e.g. FAL/KIE) leave the base method in
+     * place and are consumed via {@link textToSpeechEncoded} instead.
+     */
+    supportsStreamingTextToSpeech(): boolean;
+    /**
+     * Generate music / instrumental audio from a text prompt. Music providers
+     * return a fully-encoded audio file (mp3/wav/flac), so this resolves to an
+     * {@link EncodedAudioResult} rather than streaming raw PCM. Providers that
+     * expose music models via {@link getAvailableMusicModels} must override this.
+     */
+    textToMusic(_params: TextToMusicParams): Promise<EncodedAudioResult>;
+    /**
+     * Optional video-conditioned sound task. Call through requestVideoToAudio
+     * for input and capability validation before dispatch.
+     */
+    videoToAudio(_video: Uint8Array, _params: VideoToAudioParams): Promise<EncodedAudioResult>;
+    /**
+     * Rewrite a recording: convert the voice, isolate or denoise it, separate a
+     * stem, raise its sample rate. The source bytes are passed positionally, the
+     * direction rides in the params. Like {@link textToMusic} this resolves to an
+     * {@link EncodedAudioResult} — every provider that offers these endpoints
+     * hands back a file, not PCM.
+     */
+    audioToAudio(_audio: Uint8Array, _params: AudioToAudioParams): Promise<EncodedAudioResult>;
     automaticSpeechRecognition(_args: {
         audio: Uint8Array;
         model: string;
@@ -158,7 +595,29 @@ export declare abstract class BaseProvider {
         word_timestamps?: boolean;
     }): Promise<import("./types.js").ASRResult>;
     textToVideo(_params: TextToVideoParams): Promise<Uint8Array>;
+    /**
+     * Animate a single source image into a video. Reference media uses
+     * {@link referenceToVideo} so the input role remains explicit.
+     */
     imageToVideo(_image: Uint8Array, _params: ImageToVideoParams): Promise<Uint8Array>;
+    referenceToVideo(_inputs: ReferenceToVideoInputs, _params: ReferenceToVideoParams): Promise<Uint8Array>;
+    /** Track a source-frame box and return samples in absolute source milliseconds. */
+    trackObject(_video: Uint8Array, _params: ObjectTrackingParams): Promise<ObjectTrackingResult>;
+    /** Restyle / edit an existing video, guided by a prompt. */
+    videoToVideo(_video: Uint8Array, _params: VideoToVideoParams): Promise<Uint8Array>;
+    /** Add source time before or after an existing video window. */
+    extendVideo(_video: Uint8Array, _params: ExtendVideoParams): Promise<Uint8Array>;
+    /** Increase the resolution / detail of a video. */
+    upscaleVideo(_video: Uint8Array, _params: UpscaleVideoParams): Promise<Uint8Array>;
+    /**
+     * Synthesize intermediate frames so a clip plays at a higher frame rate (or
+     * in slow motion at the same one).
+     */
+    interpolateVideo(_video: Uint8Array, _params: InterpolateVideoParams): Promise<Uint8Array>;
+    /** Extend a video past its frame, consistently across its frames. */
+    outpaintVideo(_video: Uint8Array, _params: OutpaintVideoParams): Promise<Uint8Array>;
+    /** Drive a face in a video/image to match the speech in an audio track. */
+    lipSync(_video: Uint8Array, _params: LipSyncParams): Promise<Uint8Array>;
     /**
      * Generate a 3D asset from a text prompt. Returns the encoded asset bytes
      * (typically GLB). Providers that support 3D generation should override.
@@ -175,6 +634,16 @@ export declare abstract class BaseProvider {
         model: string;
         dimensions?: number;
     }): Promise<number[][]>;
+    /** Score each document against the query, most relevant first. */
+    rerank(_params: RerankParams): Promise<RerankResult[]>;
+    /**
+     * Whether `error` is this provider refusing the request because it exceeds
+     * the model's context window. Default `false`: a provider whose signal has
+     * not been established must not have one guessed for it, because a false
+     * positive rewrites a transcript over an unrelated failure. Overridden by the
+     * providers in `context-exceeded.ts`.
+     */
+    isContextExceededError(_error: unknown): boolean;
     isContextLengthError(error: unknown): boolean;
     /**
      * Check if an error is a rate limit error (HTTP 429).
@@ -191,7 +660,7 @@ export declare abstract class BaseProvider {
     /**
      * Resolve a URI to a `data:` URI that providers can consume directly.
      *
-     * - `file://...`        → read from disk, return `data:<mime>;base64,…`
+     * - `file://...`        → read an asset file, return `data:<mime>;base64,…`
      * - `/api/storage/<k>`  → read from getDefaultAssetsPath()/<k> (legacy fallback)
      * - Everything else     → returned unchanged (http/https/data: pass through)
      */