@shop-os/modelkit 0.4.7 → 0.5.0

package/dist/index.d.ts

@@ -34,6 +34,19 @@ type MediaRef = {
     /** Measured length of a video/audio; the kit never measures it. */
     durationSeconds?: number;
 };
+/** A click on the image for `mask` rows, in source pixels; `background` excludes the spot. */
+interface MaskPoint {
+    x: number;
+    y: number;
+    label?: "foreground" | "background";
+}
+/** A box around the object to mask, in source pixels. */
+interface MaskBox {
+    xMin: number;
+    yMin: number;
+    xMax: number;
+    yMax: number;
+}
 /** Caller-driven Topaz upscale controls; see the topaz row and TopazImageInput. */
 interface UpscaleOptions {
     enhanceModel?: string;
@@ -42,13 +55,28 @@ interface UpscaleOptions {
     faceEnhancementCreativity?: number;
     faceEnhancementStrength?: number;
 }
-type ModelType = "text" | "image" | "video" | "audio" | "upscale" | "bg-remove"
-/** Video in, video out, no prompt: upscale, frame interpolation, SDR to HDR. */
- | "video-tool";
+type ModelType = "text" | "image" | "video" | "audio"
+/** Sound effects (`generateAudio`): a text description, or a `sourceVideo` to score when the row requires one. */
+ | "sound-effect" | "upscale" | "bg-remove"
+/** Image segmentation (`generateImage`): one image and a prompt in, one mask per match out. */
+ | "mask"
+/** Video in, video out, no prompt: upscale, frame interpolation, SDR to HDR, background removal, subtitles. */
+ | "video-tool"
+/** Text to speech (`generateSpeech`). */
+ | "speech"
+/** Speech to speech in another voice (`changeVoice`). */
+ | "voice-change"
+/** Speech translated into `targetLanguage` (`dub`), from a video or an audio file. */
+ | "dubbing"
+/** A face driven by speech (`lipsync`): a `sourceVideo` re-timed or a still `image` animated. */
+ | "lipsync"
+/** Speech to text (`transcribe`). */
+ | "transcription";
 /** What a `video-tool` row does to its source clip. */
-type VideoToolKind = "upscale" | "interpolate" | "hdr";
+type VideoToolKind = "upscale" | "interpolate" | "hdr" | "bg-remove" | "subtitles";
 /** Container/codec a video-tool row can write. */
-type VideoToolOutputFormat = "h265" | "prores";
+/** `exr`: a ZIP of 16-bit EXR frames in `VideoToolResult.frames`, with an MP4 preview as `video`. */
+type VideoToolOutputFormat = "h265" | "prores" | "exr";
 type Lifecycle = "active" | "deprecated" | "retired";
 type Surface = "cowork" | "space_builder";
 /** @deprecated use Surface[] */
@@ -78,6 +106,10 @@ interface ImageRequest {
     temperature?: number;
     /** Upscaler controls beyond `scaleFactor`/`outputFormat` (Topaz). */
     upscale?: UpscaleOptions;
+    /** Mask rows with `supports: points`: spots that refine what the prompt selects. */
+    points?: MaskPoint[];
+    /** Mask rows with `supports: boxes`: boxes around the objects to mask. */
+    boxes?: MaskBox[];
 }
 /**
  * What a caller passes to generateImage. Same field names as the AI SDK's
@@ -125,6 +157,12 @@ interface VideoRequest {
      * adapter sends the prompt alone; `taskType: "extend"` still selects the extend path. */
     previousInteractionId?: string;
     mode?: "standard" | "pro";
+    /** Rows with `supports: ["insertWindow"]`: where the generated scene starts in the
+     * source clip and where the source footage resumes, in seconds. */
+    insertWindow?: {
+        startSeconds: number;
+        resumeSeconds: number;
+    };
 }
 /**
  * What a caller passes to generateVideo. `duration` is the AI SDK spelling of
@@ -152,12 +190,38 @@ interface VideoToolRequest {
     source: VideoToolSource;
     /** Upscale rows. */
     upscaleFactor?: number;
-    /** Upscale rows whose `constraints.tool.prompt` is set; ignored by every other row. */
+    /** Rows whose `constraints.tool.prompt` is set; ignored by every other row. */
     prompt?: string;
     /** Interpolate rows. */
     targetFps?: number;
     /** HDR rows; the row's first `outputFormats` entry when unset. */
     outputFormat?: VideoToolOutputFormat;
+    /** Subtitles rows: caption style and language; the row's first preset when unset. */
+    subtitles?: SubtitleOptions;
+}
+/** Per-tier caption text overrides; an unset field keeps the preset's own. */
+interface SubtitleTextStyle {
+    /** A Google Font name the provider supports. */
+    font?: string;
+    /** 100 to 900. */
+    weight?: number;
+    /** `#rgb` or `#rrggbb`. */
+    color?: string;
+}
+/** Burned-in caption options; every unset field keeps the preset's own. */
+interface SubtitleOptions {
+    /** One of `constraints.tool.presets`. */
+    preset?: string;
+    /** The spoken language (one of `tool.languages`); the provider detects it when unset. */
+    language?: string;
+    /** Caption in this language instead (one of `tool.translationLanguages`). */
+    translationLanguage?: string;
+    position?: "top" | "center" | "bottom";
+    shadow?: "none" | "min" | "mid" | "max";
+    /** Every word. */
+    baseline?: SubtitleTextStyle;
+    /** The words the provider marks as key. */
+    highlighted?: SubtitleTextStyle;
 }
 /** The output a video-tool request will produce, from the request alone. */
 interface VideoToolOutputSpec {
@@ -165,8 +229,11 @@ interface VideoToolOutputSpec {
     height: number;
     durationSeconds: number;
     fps?: number;
+    /** HDR: the frames the row renders at `fps`, which the request sends. */
+    frames?: number;
     resolution: VideoResolution;
-    format: "h264" | VideoToolOutputFormat;
+    /** `vp9` (WebM with alpha) is what a `bg-remove` row writes. */
+    format: "h264" | "vp9" | VideoToolOutputFormat;
 }
 interface VideoToolCallContext extends CallContext {
     /** Where the provider posts the result when the job ends. */
@@ -194,6 +261,12 @@ interface VideoToolResult {
         mime?: string;
         bytes?: number;
     };
+    /** The frame archive of an `exr` row; `video` is then its preview. */
+    frames?: {
+        url: string;
+        mime?: string;
+        bytes?: number;
+    };
     output: VideoToolOutputSpec;
     usage: UsageEvent;
     raw?: unknown;
@@ -209,22 +282,118 @@ interface AudioRequest {
     seed?: number;
     /** Music models: no vocals. */
     instrumental?: boolean;
+    /** Music rows with `supports: lyrics`: the words to sing, with [verse]/[chorus] tags. */
+    lyrics?: string;
+    /** One of the row's `constraints.outputFormats`; the row default when unset. */
+    outputFormat?: string;
     /** Cancellation; combined with CallContext.signal before dispatch. */
     abortSignal?: AbortSignal;
 }
+/** A provider file: a URL it hosts, or the bytes when it returns them inline. */
+type MediaOutput = {
+    bytes: Uint8Array;
+    mime: string;
+} | {
+    url: string;
+    mime?: string;
+    expiresAt?: Date;
+};
 interface AudioResult {
-    audio: {
-        bytes: Uint8Array;
-        mime: string;
-    } | {
-        url: string;
-        expiresAt?: Date;
-    };
+    audio: MediaOutput;
     /** Lyria returns lyrics as text parts next to the audio. */
     text?: string;
     usage: UsageEvent;
     raw?: JsonValue;
 }
+/** `generateSpeech` on a `speech` row. Voice, language and format are ids the row lists. */
+interface SpeechRequest {
+    model: string;
+    /** The script, spoken verbatim. */
+    text: string;
+    /** One of `constraints.voices[].id`; the row default when unset. */
+    voice?: string;
+    /** One of `constraints.languages[].code`; the provider detects it when unset. */
+    language?: string;
+    /** Rows with `supports: style`: how to speak it ("warm and slow"), kept out of the script. */
+    style?: string;
+    /** One of `constraints.outputFormats`; the row default when unset. */
+    outputFormat?: string;
+    abortSignal?: AbortSignal;
+}
+/** `changeVoice` on a `voice-change` row: the recording re-spoken in another voice. */
+interface VoiceChangeRequest {
+    model: string;
+    /** Set `durationSeconds` so `constraints.references.audioSeconds` can be checked. */
+    sourceAudio: MediaRef;
+    voice?: string;
+    outputFormat?: string;
+    abortSignal?: AbortSignal;
+}
+/**
+ * `lipsync` on a `lipsync` row. The row's `requires` names the face input: `sourceVideo` (a
+ * video re-timed to the speech) or `firstFrame` (a still `image` animated).
+ */
+interface LipsyncRequest {
+    model: string;
+    sourceVideo?: MediaRef;
+    image?: MediaRef;
+    /** The speech. Set `durationSeconds` so `constraints.references.audioSeconds` can be checked. */
+    audio: MediaRef;
+    /** Rows that list `constraints.resolutions`; the row default when unset. */
+    resolution?: VideoResolution;
+    abortSignal?: AbortSignal;
+}
+interface LipsyncResult {
+    video: MediaOutput;
+    usage: UsageEvent;
+    raw?: unknown;
+}
+/** `dub` on a `dubbing` row: exactly one of `sourceVideo` / `sourceAudio`. */
+interface DubbingRequest {
+    model: string;
+    sourceVideo?: MediaRef;
+    sourceAudio?: MediaRef;
+    /** One of `constraints.languages[].code`. */
+    targetLanguage: string;
+    /** One of `constraints.outputFormats` on rows that list them. */
+    outputFormat?: string;
+    abortSignal?: AbortSignal;
+}
+/** The dubbed media, of the same kind as the source. */
+interface DubbingResult {
+    video?: MediaOutput;
+    audio?: MediaOutput;
+    usage: UsageEvent;
+    raw?: unknown;
+}
+interface TranscriptionRequest {
+    model: string;
+    /** The speech, audio or video. Set `durationSeconds` so limits and usage can use it. */
+    source: MediaRef;
+    /** One of `constraints.languages[].code`; the provider detects the language when unset. */
+    language?: string;
+    /** Also return `subtitles` (SRT) built from the word timings. */
+    timestamps?: boolean;
+    abortSignal?: AbortSignal;
+}
+interface TranscriptWord {
+    text: string;
+    /** `word`, `spacing` or `audio_event` (ElevenLabs Scribe). */
+    type: string;
+    start?: number;
+    end?: number;
+    speakerId?: string;
+}
+interface TranscriptionResult {
+    text: string;
+    /** SubRip (SRT) cues, when the request asked for `timestamps`. */
+    subtitles?: string;
+    /** The language the provider detected or was told. */
+    language?: string;
+    words: TranscriptWord[];
+    usage: UsageEvent;
+    raw?: JsonValue;
+}
 /** An AI SDK file part ready for generateText/generateObject, e.g. a video for understanding. */
 interface MediaPart {
     type: "file";
@@ -233,10 +402,18 @@ interface MediaPart {
     /** Set when the bytes were uploaded to a provider file store (Gemini Files API). Call `cleanup` after the request. */
     cleanup?: () => Promise<void>;
 }
-type RequiredInput = "firstFrame" | "sourceVideo" | "references";
+type RequiredInput = "firstFrame" | "sourceVideo" | "references" | "sourceAudio";
 type SupportedOption = "seed" | "quality" | "negativePrompt" | "cfgScale" | "generateAudio" | "promptExpansion" | "lastFrame" | "count" | "outputFormat" | "scaleFactor" | "safetySettings" | "temperature" | "upscale" | "watermark" | "safetyChecker" | "taskType" | "instrumental"
+/** Music rows that sing `AudioRequest.lyrics`. */
+ | "lyrics"
+/** Speech rows that take `SpeechRequest.style`, a delivery instruction. */
+ | "style"
 /** Video rows that extend a clip in fixed segments up to a cumulative cap (see `ModelConstraints.extension`). */
- | "videoExtension";
+ | "videoExtension"
+/** Video rows that generate a scene inside a source clip (`VideoRequest.insertWindow`). */
+ | "insertWindow"
+/** Mask rows: `ImageRequest.points` and `ImageRequest.boxes`. */
+ | "points" | "boxes";
 interface VideoSecondsLimits {
     /** Minimum length of each reference video, in seconds. */
     min?: number;
@@ -268,6 +445,16 @@ interface ReferenceLimits {
     audioFormats?: string[];
     /** Largest reference audio file the provider takes, in bytes. */
     audioMaxBytes?: number;
+    /**
+     * Video rows: the fewest references of each kind a request needs (Kling 3.0 Motion Control:
+     * one image and one video). `requires: ["references"]` needs one of any kind; this names
+     * the kinds. Kits before 0.5.0 ignore it.
+     */
+    min?: {
+        images?: number;
+        videos?: number;
+        audios?: number;
+    };
 }
 interface VideoToolConstraints {
     kind: VideoToolKind;
@@ -278,7 +465,7 @@ interface VideoToolConstraints {
     };
     /** Upscale: the output's long edge may not pass this, in pixels. */
     maxOutputLongEdge?: number;
-    /** Upscale: the row takes an optional prompt that guides added detail. */
+    /** Upscale and HDR: the row takes an optional prompt that guides the result. */
     prompt?: boolean;
     /** Interpolate: allowed target frame rate range. */
     fps?: {
@@ -287,11 +474,45 @@ interface VideoToolConstraints {
     };
     /** HDR: the formats the row writes, first is the default. */
     outputFormats?: VideoToolOutputFormat[];
+    /** The output's short edge is scaled down to this many pixels, keeping the aspect ratio. */
+    maxOutputShortEdge?: number;
+    /** The most frames the row renders (duration times frame rate). */
+    maxOutputFrames?: number;
+    /** HDR: the fewest frames the row renders; a shorter source is padded to it. */
+    minOutputFrames?: number;
+    /** HDR: the fastest output rate; a faster source renders at this rate. */
+    maxOutputFps?: number;
+    /** Subtitles: the caption styles, first is the default. */
+    presets?: string[];
+    /** Subtitles: spoken-language codes the provider transcribes. */
+    languages?: string[];
+    /** Subtitles: codes the provider can translate the captions into. */
+    translationLanguages?: string[];
+    /** Subtitles: the longest source above 1080p, in seconds (`sourceVideoSeconds.max` covers the rest). */
+    maxSecondsAbove1080p?: number;
 }
 /** Whether a video row's output has sound: always, switchable (`supports: generateAudio`), or silent. */
 type AudioOutput = "always" | "optional" | "never";
 /** How a prompt points at reference N: "image N" numbering, a caption beside each image, or prose. */
 type ReferenceSyntax = "numbered" | "caption" | "descriptive";
+type VoiceGender = "male" | "female" | "neutral";
+/** A voice a row offers. `id` is the value sent to the provider. */
+interface VoiceOption {
+    id: string;
+    name: string;
+    gender?: VoiceGender;
+    accent?: string;
+    /** Language codes the provider lists for the voice. */
+    languages?: string[];
+    description?: string;
+    /** An https sample of the voice. */
+    previewUrl?: string;
+}
+/** A language a row takes. `code` is the value sent to the provider. */
+interface LanguageOption {
+    code: string;
+    name: string;
+}
 interface PromptConventions {
     referenceSyntax: ReferenceSyntax;
     /** Provider-documented prompt length limit, in characters. */
@@ -309,6 +530,8 @@ interface ModelConstraints {
     durationPresets?: number[];
     /** Curated quality choices for rows whose `supports` includes "quality". */
     qualityOptions?: string[];
+    /** Upscale rows: the `scaleFactor` values the row offers; absent means any. */
+    scaleFactors?: number[];
     /**
      * Exact pixel size per aspect ratio for rows an app must size verbatim
      * (gpt-image-2 keeps its historic WxH contract); callers pass it as `exactSize`.
@@ -316,9 +539,26 @@ interface ModelConstraints {
     exactSizes?: Record<string, string>;
     references?: ReferenceLimits;
     requires?: RequiredInput[];
+    /**
+     * Video rows: at least one of these inputs must be present (`requires` needs all of its
+     * own). Kits before 0.5.0 ignore it.
+     */
+    requiresOneOf?: RequiredInput[];
+    /**
+     * Video rows whose endpoint has no prompt field (MiniMax H3 Max Relight): the request needs
+     * no prompt, and a non-empty one is dropped with an `option_ignored` warning. Absent means
+     * the row takes a prompt. Kits before 0.5.0 ignore it.
+     */
+    noPrompt?: boolean;
     supports?: SupportedOption[];
     /** Ratio is dropped, with a warning, when one of these inputs is present. */
     ratioIgnoredWith?: ("firstFrame" | "sourceVideo")[];
+    /** Speech and voice-change rows: the voices offered; `defaults.voice` is one of their ids. */
+    voices?: VoiceOption[];
+    /** Speech, transcription and dubbing (target) rows: the languages taken. */
+    languages?: LanguageOption[];
+    /** Audio-output rows whose provider lets the caller pick a file format; `defaults.outputFormat` is one. */
+    outputFormats?: string[];
     /** Length limits for the source video the row edits or extends. */
     sourceVideoSeconds?: Omit<VideoSecondsLimits, "total"> & {
         /** The provider sends the source as a reference video, so it counts toward `references.videoSeconds.total` (Seedance). */
@@ -411,6 +651,15 @@ interface CatalogCapabilities {
     vision?: boolean;
     contextWindow?: number;
 }
+/** Row defaults the request types do not share. */
+interface MediaNodeDefaults {
+    /** Image rows: png, jpeg or webp. Audio-output rows: one of `constraints.outputFormats`. */
+    outputFormat?: string;
+    /** One of `constraints.voices[].id`. */
+    voice?: string;
+    /** One of `constraints.languages[].code`. */
+    language?: string;
+}
 interface CatalogModel {
     /** Canonical id. Equals the pricing key in pro-backend credit_pricing_* tables. */
     id: string;
@@ -430,7 +679,7 @@ interface CatalogModel {
      */
     exposure: Surface[];
     constraints: ModelConstraints;
-    defaults: Partial<Omit<ImageRequest, "model" | "prompt"> & Omit<VideoRequest, "model" | "prompt"> & Omit<AudioRequest, "model" | "prompt">>;
+    defaults: Partial<Omit<ImageRequest, "model" | "prompt" | "outputFormat"> & Omit<VideoRequest, "model" | "prompt"> & Omit<AudioRequest, "model" | "prompt" | "outputFormat">> & MediaNodeDefaults;
     /** Provider options merged under the caller's on every call. Chat-only tuning belongs in display.chat. */
     providerOptions?: ProviderOptions;
     /**
@@ -676,7 +925,11 @@ declare function isListedOn(row: CatalogModel, surface: Surface): boolean;
 /** True when an active row is listed on at least one surface. */
 declare function isListed(row: CatalogModel): boolean;
 
-/** Seeded from Cowork, Space Builder and trigger-workflows. See catalog/*.ts. */
+/**
+ * Every row, aggregated from the exported per-type collections only, so a row added anywhere
+ * else never reaches MODELS without its collection. Seeded from Cowork, Space Builder and
+ * trigger-workflows. See catalog/*.ts.
+ */
 declare const MODELS: CatalogModel[];
 
 declare const ADAPTER_IDS: readonly ["ai-sdk-text", "google-image", "openai-image", "google-genai", "ark", "fal", "replicate", "prodia", "higgsfield", "xai-video"];
@@ -704,9 +957,16 @@ declare const CatalogModelSchemaBase: z.ZodObject<{
         image: "image";
         video: "video";
         audio: "audio";
+        "sound-effect": "sound-effect";
         upscale: "upscale";
         "bg-remove": "bg-remove";
+        mask: "mask";
         "video-tool": "video-tool";
+        speech: "speech";
+        "voice-change": "voice-change";
+        dubbing: "dubbing";
+        lipsync: "lipsync";
+        transcription: "transcription";
     }>;
     adapter: z.ZodEnum<{
         "ai-sdk-text": "ai-sdk-text";
@@ -769,6 +1029,7 @@ declare const CatalogModelSchemaBase: z.ZodObject<{
         }, z.core.$loose>, z.ZodArray<z.ZodNumber>]>>;
         durationPresets: z.ZodOptional<z.ZodArray<z.ZodNumber>>;
         qualityOptions: z.ZodOptional<z.ZodArray<z.ZodString>>;
+        scaleFactors: z.ZodOptional<z.ZodArray<z.ZodNumber>>;
         exactSizes: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodString>>;
         references: z.ZodOptional<z.ZodObject<{
             images: z.ZodOptional<z.ZodNumber>;
@@ -790,12 +1051,25 @@ declare const CatalogModelSchemaBase: z.ZodObject<{
             }, z.core.$loose>>;
             audioFormats: z.ZodOptional<z.ZodArray<z.ZodString>>;
             audioMaxBytes: z.ZodOptional<z.ZodNumber>;
+            min: z.ZodOptional<z.ZodObject<{
+                images: z.ZodOptional<z.ZodNumber>;
+                videos: z.ZodOptional<z.ZodNumber>;
+                audios: z.ZodOptional<z.ZodNumber>;
+            }, z.core.$loose>>;
         }, z.core.$loose>>;
         requires: z.ZodOptional<z.ZodArray<z.ZodEnum<{
             references: "references";
             firstFrame: "firstFrame";
             sourceVideo: "sourceVideo";
+            sourceAudio: "sourceAudio";
+        }>>>;
+        requiresOneOf: z.ZodOptional<z.ZodArray<z.ZodEnum<{
+            references: "references";
+            firstFrame: "firstFrame";
+            sourceVideo: "sourceVideo";
+            sourceAudio: "sourceAudio";
         }>>>;
+        noPrompt: z.ZodOptional<z.ZodBoolean>;
         supports: z.ZodOptional<z.ZodArray<z.ZodEnum<{
             upscale: "upscale";
             quality: "quality";
@@ -805,6 +1079,8 @@ declare const CatalogModelSchemaBase: z.ZodObject<{
             scaleFactor: "scaleFactor";
             safetySettings: "safetySettings";
             temperature: "temperature";
+            points: "points";
+            boxes: "boxes";
             lastFrame: "lastFrame";
             generateAudio: "generateAudio";
             negativePrompt: "negativePrompt";
@@ -813,13 +1089,34 @@ declare const CatalogModelSchemaBase: z.ZodObject<{
             watermark: "watermark";
             safetyChecker: "safetyChecker";
             taskType: "taskType";
+            insertWindow: "insertWindow";
             instrumental: "instrumental";
+            lyrics: "lyrics";
+            style: "style";
             videoExtension: "videoExtension";
         }>>>;
         ratioIgnoredWith: z.ZodOptional<z.ZodArray<z.ZodEnum<{
             firstFrame: "firstFrame";
             sourceVideo: "sourceVideo";
         }>>>;
+        voices: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            id: z.ZodString;
+            name: z.ZodString;
+            gender: z.ZodOptional<z.ZodEnum<{
+                male: "male";
+                female: "female";
+                neutral: "neutral";
+            }>>;
+            accent: z.ZodOptional<z.ZodString>;
+            languages: z.ZodOptional<z.ZodArray<z.ZodString>>;
+            description: z.ZodOptional<z.ZodString>;
+            previewUrl: z.ZodOptional<z.ZodURL>;
+        }, z.core.$loose>>>;
+        languages: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            code: z.ZodString;
+            name: z.ZodString;
+        }, z.core.$loose>>>;
+        outputFormats: z.ZodOptional<z.ZodArray<z.ZodString>>;
         sourceVideoSeconds: z.ZodOptional<z.ZodObject<{
             max: z.ZodOptional<z.ZodNumber>;
             min: z.ZodOptional<z.ZodNumber>;
@@ -837,8 +1134,10 @@ declare const CatalogModelSchemaBase: z.ZodObject<{
         tool: z.ZodOptional<z.ZodObject<{
             kind: z.ZodEnum<{
                 upscale: "upscale";
+                "bg-remove": "bg-remove";
                 interpolate: "interpolate";
                 hdr: "hdr";
+                subtitles: "subtitles";
             }>;
             upscaleFactor: z.ZodOptional<z.ZodObject<{
                 min: z.ZodNumber;
@@ -853,7 +1152,16 @@ declare const CatalogModelSchemaBase: z.ZodObject<{
             outputFormats: z.ZodOptional<z.ZodArray<z.ZodEnum<{
                 h265: "h265";
                 prores: "prores";
+                exr: "exr";
             }>>>;
+            maxOutputShortEdge: z.ZodOptional<z.ZodNumber>;
+            maxOutputFrames: z.ZodOptional<z.ZodNumber>;
+            minOutputFrames: z.ZodOptional<z.ZodNumber>;
+            maxOutputFps: z.ZodOptional<z.ZodNumber>;
+            presets: z.ZodOptional<z.ZodArray<z.ZodString>>;
+            languages: z.ZodOptional<z.ZodArray<z.ZodString>>;
+            translationLanguages: z.ZodOptional<z.ZodArray<z.ZodString>>;
+            maxSecondsAbove1080p: z.ZodOptional<z.ZodNumber>;
         }, z.core.$loose>>;
         audio: z.ZodOptional<z.ZodUnknown>;
         prompt: z.ZodOptional<z.ZodUnknown>;
@@ -905,11 +1213,11 @@ declare const CatalogModelSchemaBase: z.ZodObject<{
         }>>;
         seed: z.ZodOptional<z.ZodNumber>;
         count: z.ZodOptional<z.ZodNumber>;
-        outputFormat: z.ZodOptional<z.ZodEnum<{
+        outputFormat: z.ZodOptional<z.ZodUnion<readonly [z.ZodEnum<{
             png: "png";
             jpeg: "jpeg";
             webp: "webp";
-        }>>;
+        }>, z.ZodString]>>;
         scaleFactor: z.ZodOptional<z.ZodNumber>;
         safetySettings: z.ZodOptional<z.ZodArray<z.ZodObject<{
             category: z.ZodString;
@@ -923,6 +1231,20 @@ declare const CatalogModelSchemaBase: z.ZodObject<{
             faceEnhancementCreativity: z.ZodOptional<z.ZodNumber>;
             faceEnhancementStrength: z.ZodOptional<z.ZodNumber>;
         }, z.core.$loose>>;
+        points: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            x: z.ZodNumber;
+            y: z.ZodNumber;
+            label: z.ZodOptional<z.ZodEnum<{
+                foreground: "foreground";
+                background: "background";
+            }>>;
+        }, z.core.$loose>>>;
+        boxes: z.ZodOptional<z.ZodArray<z.ZodObject<{
+            xMin: z.ZodNumber;
+            yMin: z.ZodNumber;
+            xMax: z.ZodNumber;
+            yMax: z.ZodNumber;
+        }, z.core.$loose>>>;
         firstFrame: z.ZodOptional<z.ZodUnion<readonly [z.ZodObject<{
             url: z.ZodString;
             label: z.ZodOptional<z.ZodString>;
@@ -1003,6 +1325,10 @@ declare const CatalogModelSchemaBase: z.ZodObject<{
         }>>;
         watermark: z.ZodOptional<z.ZodBoolean>;
         safetyChecker: z.ZodOptional<z.ZodBoolean>;
+        insertWindow: z.ZodOptional<z.ZodObject<{
+            startSeconds: z.ZodNumber;
+            resumeSeconds: z.ZodNumber;
+        }, z.core.$loose>>;
         taskType: z.ZodOptional<z.ZodEnum<{
             reference: "reference";
             edit: "edit";
@@ -1013,6 +1339,9 @@ declare const CatalogModelSchemaBase: z.ZodObject<{
             pro: "pro";
         }>>;
         instrumental: z.ZodOptional<z.ZodBoolean>;
+        lyrics: z.ZodOptional<z.ZodString>;
+        voice: z.ZodOptional<z.ZodString>;
+        language: z.ZodOptional<z.ZodString>;
     }, z.core.$loose>;
     providerOptions: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodRecord<z.ZodString, z.ZodType<ProviderOptionValue, unknown, z.core.$ZodTypeInternals<ProviderOptionValue, unknown>>>>>;
     usage: z.ZodObject<{
@@ -1144,6 +1473,14 @@ interface AdapterContext {
     call: CallContext;
     warnings: Warning[];
 }
+/** A dubbed video is a provider job the client waits on; dubbed audio arrives in one call. */
+type DubbingOutcome = {
+    kind: "video";
+    job: VideoJob;
+} | {
+    kind: "audio";
+    result: AudioResult;
+};
 /**
  * One adapter per provider API. The request the adapter receives has already
  * passed resolveOptions: every option is inside the catalog constraints and the
@@ -1156,6 +1493,11 @@ interface Adapter {
     /** Reconnect to a job `generateVideo` submitted earlier; never submits. */
     resumeVideo?(req: VideoRequest, jobId: string, ctx: AdapterContext): VideoJob;
     generateAudio?(req: AudioRequest, ctx: AdapterContext): Promise<AudioResult>;
+    generateSpeech?(req: SpeechRequest, ctx: AdapterContext): Promise<AudioResult>;
+    changeVoice?(req: VoiceChangeRequest, ctx: AdapterContext): Promise<AudioResult>;
+    lipsync?(req: LipsyncRequest, ctx: AdapterContext): Promise<VideoJob>;
+    dub?(req: DubbingRequest, ctx: AdapterContext): Promise<DubbingOutcome>;
+    transcribe?(req: TranscriptionRequest, ctx: AdapterContext): Promise<TranscriptionResult>;
     /** Video tools run as provider jobs the caller tracks; see ModelKit.submitVideoTool. */
     submitVideoTool?(req: VideoToolRequest, ctx: AdapterContext, opts: {
         webhookUrl?: string;
@@ -1254,6 +1596,17 @@ declare function resolveImageOptions(model: CatalogModel, request: ImageRequest,
 declare function resolveVideoOptions(model: CatalogModel, request: VideoRequest, onUnsupported?: OnUnsupported): ResolveResult<VideoRequest>;
 declare function resolveAudioOptions(model: CatalogModel, request: AudioRequest, onUnsupported?: OnUnsupported): ResolveResult<AudioRequest>;
 
+/** `generateSpeech` on a `speech` row. */
+declare function resolveSpeechOptions(model: CatalogModel, request: SpeechRequest, onUnsupported?: OnUnsupported): ResolveResult<SpeechRequest>;
+/** `changeVoice` on a `voice-change` row. */
+declare function resolveVoiceChangeOptions(model: CatalogModel, request: VoiceChangeRequest, onUnsupported?: OnUnsupported): ResolveResult<VoiceChangeRequest>;
+/** `lipsync` on a `lipsync` row: the face input is the one its `requires` names. */
+declare function resolveLipsyncOptions(model: CatalogModel, request: LipsyncRequest, onUnsupported?: OnUnsupported): ResolveResult<LipsyncRequest>;
+/** `dub` on a `dubbing` row: one source, a target language the row lists. */
+declare function resolveDubbingOptions(model: CatalogModel, request: DubbingRequest, onUnsupported?: OnUnsupported): ResolveResult<DubbingRequest>;
+/** `transcribe` on a `transcription` row. */
+declare function resolveTranscriptionOptions(model: CatalogModel, request: TranscriptionRequest, onUnsupported?: OnUnsupported): ResolveResult<TranscriptionRequest>;
+
 /** Which key names an adapter needs. All of them must be set on a tier for that tier to win. */
 declare const ADAPTER_KEYS: Record<AdapterId, ProviderKeyName[]>;
 /** The required names the given non-pro tier is missing. */
@@ -1293,6 +1646,27 @@ declare function pollUntil<T>(check: () => Promise<T | undefined>, opts: {
     maxIntervalMs?: number;
 }): Promise<T>;
 
+/** `lipsync` and a video `dub` wait on a provider job; `wait` tunes that wait. */
+interface MediaJobCallContext extends CallContext {
+    wait?: {
+        pollIntervalMs?: number;
+        timeoutMs?: number;
+    };
+}
+/** The single-purpose audio and lipsync node methods of the kit. */
+interface AudioNodeMethods {
+    /** Text to speech on a `speech` row. Blocks until the provider returns the audio. */
+    generateSpeech(req: SpeechRequest, call?: CallContext): Promise<AudioResult>;
+    /** Re-speaks a recording in another voice on a `voice-change` row. */
+    changeVoice(req: VoiceChangeRequest, call?: CallContext): Promise<AudioResult>;
+    /** Drives a face (`sourceVideo` or `image`) with speech on a `lipsync` row. */
+    lipsync(req: LipsyncRequest, call?: MediaJobCallContext): Promise<LipsyncResult>;
+    /** Speech to text on a `transcription` row; `timestamps` adds SRT `subtitles`. */
+    transcribe(req: TranscriptionRequest, call?: CallContext): Promise<TranscriptionResult>;
+    /** Dubs a video or an audio file into `targetLanguage`; the result is the same kind. */
+    dub(req: DubbingRequest, call?: MediaJobCallContext): Promise<DubbingResult>;
+}
+
 /**
  * The AI SDK `LanguageModel` type. A string is a catalog id or alias; a provider
  * instance (`openai("gpt-4o")`) runs through the SDK unchanged, with usage
@@ -1349,7 +1723,7 @@ type StreamTextCallOptions<TOOLS extends ToolSet, OUTPUT extends Output.Output =
     /** @deprecated Use `prepareStep`; accepted for parity with `generateText`. */
     experimental_prepareStep?: Parameters<typeof streamText$1<TOOLS, OUTPUT>>[0]["prepareStep"];
 };
-interface ModelKit extends TextMethods {
+interface ModelKit extends TextMethods, AudioNodeMethods {
     catalog: Catalog;
     /**
      * Generate one or more images. Blocks until the provider returns bytes.
@@ -1371,7 +1745,11 @@ interface ModelKit extends TextMethods {
     languageModel(modelId: string, call?: CallContext): LanguageModelHandle;
     /** Usage-event `provider` for a catalog model, without building it: "bedrock" on Bedrock, else the vendor. */
     usageProvider(modelId: string, call?: CallContext): string;
-    /** Generate music or a soundtrack. Blocks until the provider returns audio. */
+    /**
+     * Generate music (`audio`) or a sound effect (`sound-effect`). Blocks until the provider
+     * returns audio. Speech, voice change, lipsync, dubbing and transcription have their own
+     * methods (AudioNodeMethods).
+     */
     generateAudio(req: AudioRequest, call?: CallContext): Promise<AudioResult>;
     /**
      * Stream text. Mirrors `TextMethods.streamText` and also accepts the
@@ -1508,4 +1886,4 @@ declare function describeThrown(cause: unknown): ThrownErrorInfo;
 /** Add the AI SDK `GeneratedFile` accessors to adapter image bytes. base64 is computed on first read. */
 declare function toGeneratedImage(image: ImageBytes): GeneratedImage;
 
-export { ADAPTER_IDS, ADAPTER_KEYS, ARRAY_INPUT_SPEC_KEYS, AUDIO_INPUT_SPEC_KEYS, type Adapter, type AdapterContext, type AdapterEndpoints, type AdapterId, type AdapterRegistry, type AiTextProviderOptions, type ArrayInputSpecKey, type AspectRatio, type AudioOutput, type AudioRequest, type AudioResult, type CallContext, Catalog, type CatalogAuth, type CatalogCache, type CatalogCapabilities, type CatalogDisplay, type CatalogEnvelope, type CatalogModel, CatalogModelSchema, CatalogModelSchemaBase, type CatalogOrigin, CatalogSchema, type CatalogSource, type CatalogTextModel, DEFAULT_ADAPTERS, type DataOnlyRow, type DataOnlyVerdict, type DefaultObjectOutput, type Exposure, ExposureSchema, type FalWebhookBody, type GeneratedImage, IMAGE_INPUT_SPEC_KEYS, INPUT_SPEC_SUPPORTS_KEYS, type ImageBytes, type ImageCallOptions, type ImageCallResult, type ImageQuality, type ImageRequest, type ImageResult, type ImageSafetySetting, type ImageSize, type InputSpec, type InputSpecDimensions, type InputSpecIssue, type InputSpecKey, InputSpecSchema, type JsonObject, JsonObjectSchema, type JsonPrimitive, type JsonValue, JsonValueSchema, type KeyTier, type LanguageModelHandle, type Lifecycle, type LiveCatalogState, MODELS, type MediaPart, type MediaRef, type ModelConstraints, type ModelKit, type ModelKitConfig, ModelKitError, type ModelKitErrorCode, type ModelKitErrorInit, type ModelType, type ModelkitGenerateObjectOptions, type ModelkitGenerateTextOptions, type ModelkitStreamObjectOptions, type ModelkitStreamTextOptions, type ObjectOutput, type ObjectResult, type OnUnsupported, type OutputFormat, type PricingKind, type PrimeCatalogResult, type PromptConventions, type ProviderErrorBody, type ProviderKeyName, type ProviderKeys, type ProviderOptionValue, type ProviderOptions, type QuarantineResult, type ReferenceLimits, type ReferenceSyntax, type RequiredInput, type RequiredKeyName, type ResolveResult, type Routing, SURFACES, type SupportedOption, type Surface, SurfaceSchema, type TextMethods, type TextModelInstance, type UpscaleOptions, type UsageEvent, type UsageModality, type UsageUnit, VIDEO_INPUT_SPEC_KEYS, VIDEO_TOOL_MODELS, type VideoCallJob, type VideoCallOptions, type VideoJob, type VideoModel, type VideoRequest, type VideoResolution, VideoResolutionSchema, type VideoResult, type VideoSecondsLimits, type VideoToolCallContext, type VideoToolConstraints, type VideoToolKind, type VideoToolOutputFormat, type VideoToolOutputSpec, type VideoToolPlan, type VideoToolRequest, type VideoToolResult, type VideoToolSource, type VideoToolStatus, type VideoToolSubmission, type Warning, adapterServesType, aiSdkTextAdapter, arkAdapter, aspectRatioToSize, classifyHttpError, compareCatalogOrder, createDefaultModelKit, createModelKit, dataOnlyCapable, describe, describeThrown, generateImage as experimental_generateImage, falAdapter, falWebhookBodySchema, fileCatalogCache, generateAudio, generateImage, generateObject, generateText, generateVideo, googleGenaiAdapter, googleImageAdapter, hasAwsCredentials, higgsfieldAdapter, isArrayInputSpecKey, isInputSpecKey, isListed, isListedOn, isModelKitError, keysFromEnv, languageModel, liveCatalog, liveState, mergeTextProviderOptions, missingTierKeys, nearestAspectRatio, normalizeExposure, openaiImageAdapter, passthroughTextModel, planVideoTool, pollUntil, prepareMediaPart, primeCatalog, prodiaAdapter, readErrorBody, replicateAdapter, resetLiveCatalog, resolutionTier, resolveAudioOptions, resolveImageOptions, resolveKeys, resolveVideoOptions, shouldRetry, sleep, sortCatalogRows, streamObject, streamText, textUsageEvent, toGeneratedImage, validateInputSpec, videoLengthViolation, videoToolConstraints, withRetry, xaiVideoAdapter };
+export { ADAPTER_IDS, ADAPTER_KEYS, ARRAY_INPUT_SPEC_KEYS, AUDIO_INPUT_SPEC_KEYS, type Adapter, type AdapterContext, type AdapterEndpoints, type AdapterId, type AdapterRegistry, type AiTextProviderOptions, type ArrayInputSpecKey, type AspectRatio, type AudioNodeMethods, type AudioOutput, type AudioRequest, type AudioResult, type CallContext, Catalog, type CatalogAuth, type CatalogCache, type CatalogCapabilities, type CatalogDisplay, type CatalogEnvelope, type CatalogModel, CatalogModelSchema, CatalogModelSchemaBase, type CatalogOrigin, CatalogSchema, type CatalogSource, type CatalogTextModel, DEFAULT_ADAPTERS, type DataOnlyRow, type DataOnlyVerdict, type DefaultObjectOutput, type DubbingOutcome, type DubbingRequest, type DubbingResult, type Exposure, ExposureSchema, type FalWebhookBody, type GeneratedImage, IMAGE_INPUT_SPEC_KEYS, INPUT_SPEC_SUPPORTS_KEYS, type ImageBytes, type ImageCallOptions, type ImageCallResult, type ImageQuality, type ImageRequest, type ImageResult, type ImageSafetySetting, type ImageSize, type InputSpec, type InputSpecDimensions, type InputSpecIssue, type InputSpecKey, InputSpecSchema, type JsonObject, JsonObjectSchema, type JsonPrimitive, type JsonValue, JsonValueSchema, type KeyTier, type LanguageModelHandle, type LanguageOption, type Lifecycle, type LipsyncRequest, type LipsyncResult, type LiveCatalogState, MODELS, type MaskBox, type MaskPoint, type MediaJobCallContext, type MediaNodeDefaults, type MediaOutput, type MediaPart, type MediaRef, type ModelConstraints, type ModelKit, type ModelKitConfig, ModelKitError, type ModelKitErrorCode, type ModelKitErrorInit, type ModelType, type ModelkitGenerateObjectOptions, type ModelkitGenerateTextOptions, type ModelkitStreamObjectOptions, type ModelkitStreamTextOptions, type ObjectOutput, type ObjectResult, type OnUnsupported, type OutputFormat, type PricingKind, type PrimeCatalogResult, type PromptConventions, type ProviderErrorBody, type ProviderKeyName, type ProviderKeys, type ProviderOptionValue, type ProviderOptions, type QuarantineResult, type ReferenceLimits, type ReferenceSyntax, type RequiredInput, type RequiredKeyName, type ResolveResult, type Routing, SURFACES, type SpeechRequest, type SubtitleOptions, type SubtitleTextStyle, type SupportedOption, type Surface, SurfaceSchema, type TextMethods, type TextModelInstance, type TranscriptWord, type TranscriptionRequest, type TranscriptionResult, type UpscaleOptions, type UsageEvent, type UsageModality, type UsageUnit, VIDEO_INPUT_SPEC_KEYS, VIDEO_TOOL_MODELS, type VideoCallJob, type VideoCallOptions, type VideoJob, type VideoModel, type VideoRequest, type VideoResolution, VideoResolutionSchema, type VideoResult, type VideoSecondsLimits, type VideoToolCallContext, type VideoToolConstraints, type VideoToolKind, type VideoToolOutputFormat, type VideoToolOutputSpec, type VideoToolPlan, type VideoToolRequest, type VideoToolResult, type VideoToolSource, type VideoToolStatus, type VideoToolSubmission, type VoiceChangeRequest, type VoiceGender, type VoiceOption, type Warning, adapterServesType, aiSdkTextAdapter, arkAdapter, aspectRatioToSize, classifyHttpError, compareCatalogOrder, createDefaultModelKit, createModelKit, dataOnlyCapable, describe, describeThrown, generateImage as experimental_generateImage, falAdapter, falWebhookBodySchema, fileCatalogCache, generateAudio, generateImage, generateObject, generateText, generateVideo, googleGenaiAdapter, googleImageAdapter, hasAwsCredentials, higgsfieldAdapter, isArrayInputSpecKey, isInputSpecKey, isListed, isListedOn, isModelKitError, keysFromEnv, languageModel, liveCatalog, liveState, mergeTextProviderOptions, missingTierKeys, nearestAspectRatio, normalizeExposure, openaiImageAdapter, passthroughTextModel, planVideoTool, pollUntil, prepareMediaPart, primeCatalog, prodiaAdapter, readErrorBody, replicateAdapter, resetLiveCatalog, resolutionTier, resolveAudioOptions, resolveDubbingOptions, resolveImageOptions, resolveKeys, resolveLipsyncOptions, resolveSpeechOptions, resolveTranscriptionOptions, resolveVideoOptions, resolveVoiceChangeOptions, shouldRetry, sleep, sortCatalogRows, streamObject, streamText, textUsageEvent, toGeneratedImage, validateInputSpec, videoLengthViolation, videoToolConstraints, withRetry, xaiVideoAdapter };