@nodetool-ai/runtime 0.7.0-rc.23 → 0.8.2

package/dist/providers/replicate-provider.js

@@ -1,8 +1,109 @@
 import Replicate from "replicate";
 import { createLogger } from "@nodetool-ai/config";
+import { isCallable, isObjectLike, isString } from "@nodetool-ai/protocol";
 import { BaseProvider } from "./base-provider.js";
-import { loadImageModels, loadVideoModels } from "./manifest-models.js";
+import { safeFetch } from "./safe-url.js";
+import { withReplicateRetry } from "./replicate-retry.js";
+import { sniffAudioMime } from "./audio-mime.js";
+import { sniffVideoMime } from "./video-mime.js";
+import { detectImageMime } from "./image-mime.js";
+import { getModelImageInputs, getModelReferenceInputs, getModelInputFields, validateReferenceInputs, getModelInputNames, loadImageModels, loadMusicModels, loadAudioToAudioModels, loadVideoModels, selectMaskImageInput, selectPrimaryImageInput } from "./manifest-models.js";
 const log = createLogger("nodetool.runtime.providers.replicate");
+/** A nested string counts as a locator only when it looks like one. */
+const URL_LIKE = /^(https?:|data:)/;
+/** The walk is over a provider's JSON, so bound how deep it goes. */
+const MAX_OUTPUT_DEPTH = 6;
+/**
+ * Resolve a `replicate.run()` result to the one file it carries.
+ *
+ * The SDK walks the prediction's output and swaps every `https:`/`data:`
+ * string for a `FileOutput`, objects and nested arrays included, so a model
+ * answering `{"image": <FileOutput>}` is as ordinary as one answering
+ * `["https://…"]`. Reading only `output[0]` drops the first and reports "no
+ * usable output" for a prediction that succeeded.
+ */
+export function decodeReplicateOutput(output) {
+    // A bare string output is itself the locator (the model's contract),
+    // whether or not it parses as a URL.
+    if (isString(output)) {
+        return output ? { kind: "url", url: output } : null;
+    }
+    return decodeOutputValue(output, 0);
+}
+function decodeOutputValue(value, depth) {
+    if (depth > MAX_OUTPUT_DEPTH || !isObjectLike(value))
+        return null;
+    // FileOutput extends ReadableStream, so `getReader` is on its prototype.
+    // Prefer it over `url()`: those bytes are already in flight.
+    if ("getReader" in value) {
+        return {
+            kind: "stream",
+            stream: value
+        };
+    }
+    // FileOutput also exposes `url()`, and some models answer a plain `{url}`.
+    if ("url" in value) {
+        const raw = value.url;
+        const url = isCallable(raw) ? raw.call(value) : raw;
+        if (url)
+            return { kind: "url", url: String(url) };
+    }
+    const items = Array.isArray(value)
+        ? value
+        : Object.values(value);
+    for (const item of items) {
+        const found = isString(item)
+            ? decodeOutputLeaf(item)
+            : decodeOutputValue(item, depth + 1);
+        if (found)
+            return found;
+    }
+    return null;
+}
+function decodeOutputLeaf(item) {
+    return URL_LIKE.test(item) ? { kind: "url", url: item } : null;
+}
+async function drainStream(stream) {
+    const reader = stream.getReader();
+    const chunks = [];
+    let total = 0;
+    while (true) {
+        const { done, value } = await reader.read();
+        if (done)
+            break;
+        chunks.push(value);
+        total += value.length;
+    }
+    const result = new Uint8Array(total);
+    let offset = 0;
+    for (const chunk of chunks) {
+        result.set(chunk, offset);
+        offset += chunk.length;
+    }
+    return result;
+}
+/** `safeFetch` speaks https only, so an inline output is decoded here. */
+function decodeDataUri(uri) {
+    const comma = uri.indexOf(",");
+    if (comma < 0)
+        throw new Error("Malformed data URI in Replicate output");
+    const body = uri.slice(comma + 1);
+    if (uri.slice(0, comma).endsWith(";base64")) {
+        return new Uint8Array(Buffer.from(body, "base64"));
+    }
+    return new TextEncoder().encode(decodeURIComponent(body));
+}
+/** A short, credential-free description of an output the decoder rejected. */
+function describeOutputShape(output) {
+    if (output == null)
+        return String(output);
+    if (Array.isArray(output))
+        return `an array of ${output.length}`;
+    if (isObjectLike(output)) {
+        return `an object with keys [${Object.keys(output).slice(0, 8).join(", ")}]`;
+    }
+    return `a ${typeof output}`;
+}
 /**
  * Provider for Replicate's LLM, image, video, and audio models.
  *
@@ -11,6 +112,28 @@ const log = createLogger("nodetool.runtime.providers.replicate");
  * - Image / Video / TTS:  `replicate.run()` with file output
  */
 export class ReplicateProvider extends BaseProvider {
+    declaredCapabilities() {
+        return [
+            "text_to_image",
+            "image_to_image",
+            "text_to_video",
+            "image_to_video",
+            "reference_to_video",
+            "inpainting",
+            "outpaint_image",
+            "upscale_image",
+            "remove_background",
+            "relight_image",
+            "vectorize_image",
+            "video_to_video",
+            "extend_video",
+            "upscale_video",
+            "lip_sync",
+            "text_to_speech",
+            "text_to_music",
+            "audio_to_audio"
+        ];
+    }
     static requiredSecrets() {
         return ["REPLICATE_API_TOKEN"];
     }
@@ -23,12 +146,13 @@ export class ReplicateProvider extends BaseProvider {
             throw new Error("REPLICATE_API_TOKEN is required");
         }
         this.apiKey = apiKey;
-        this._client =
-            options.client ??
-                new Replicate({
-                    auth: apiKey,
-                    ...(options.fetchFn ? { fetch: options.fetchFn } : {})
-                });
+        const replicateOptions = {
+            auth: apiKey
+        };
+        if (options.fetchFn) {
+            replicateOptions.fetch = options.fetchFn;
+        }
+        this._client = options.client ?? new Replicate(replicateOptions);
     }
     getContainerEnv() {
         return { REPLICATE_API_TOKEN: this.apiKey };
@@ -37,6 +161,14 @@ export class ReplicateProvider extends BaseProvider {
         // Replicate predictions API does not support OpenAI-style tool calling
         return false;
     }
+    /** Create a prediction, waiting out the low-credit throttle. */
+    runModel(modelId, input, signal) {
+        signal?.throwIfAborted();
+        const options = { input };
+        if (signal)
+            options.signal = signal;
+        return withReplicateRetry(modelId, () => this._client.run(modelId, options), signal);
+    }
     // ---------------------------------------------------------------------------
     // Chat completions via Replicate SDK
     // ---------------------------------------------------------------------------
@@ -48,7 +180,7 @@ export class ReplicateProvider extends BaseProvider {
         let systemPrompt = "";
         const parts = [];
         for (const msg of messages) {
-            const text = typeof msg.content === "string"
+            const text = isString(msg.content)
                 ? msg.content
                 : Array.isArray(msg.content)
                     ? msg.content
@@ -79,13 +211,11 @@ export class ReplicateProvider extends BaseProvider {
         if (args.topP != null)
             input.top_p = args.topP;
         log.debug("generateMessage", { model: args.model });
-        const output = await this._client.run(args.model, {
-            input
-        });
+        const output = await this.runModel(args.model, input);
         // Replicate LLMs return output as a string, array of token strings,
         // or a ReadableStream/FileOutput.
         let text;
-        if (typeof output === "string") {
+        if (isString(output)) {
             text = output;
         }
         else if (Array.isArray(output)) {
@@ -211,6 +341,11 @@ export class ReplicateProvider extends BaseProvider {
                 name: "Claude Opus 4.6",
                 provider: "replicate"
             },
+            {
+                id: "anthropic/claude-opus-4.7",
+                name: "Claude Opus 4.7",
+                provider: "replicate"
+            },
             { id: "openai/gpt-4o", name: "GPT-4o", provider: "replicate" },
             { id: "openai/gpt-4o-mini", name: "GPT-4o Mini", provider: "replicate" },
             { id: "openai/gpt-4.1", name: "GPT-4.1", provider: "replicate" },
@@ -219,8 +354,14 @@ export class ReplicateProvider extends BaseProvider {
                 name: "GPT-4.1 Mini",
                 provider: "replicate"
             },
+            {
+                id: "openai/gpt-4.1-nano",
+                name: "GPT-4.1 Nano",
+                provider: "replicate"
+            },
             { id: "openai/gpt-5", name: "GPT-5", provider: "replicate" },
             { id: "openai/gpt-5-mini", name: "GPT-5 Mini", provider: "replicate" },
+            { id: "openai/gpt-5-nano", name: "GPT-5 Nano", provider: "replicate" },
             { id: "openai/gpt-5.2", name: "GPT-5.2", provider: "replicate" },
             { id: "openai/o1", name: "O1", provider: "replicate" },
             { id: "openai/o4-mini", name: "O4 Mini", provider: "replicate" },
@@ -231,10 +372,136 @@ export class ReplicateProvider extends BaseProvider {
                 provider: "replicate"
             },
             { id: "moonshotai/kimi-k2.5", name: "Kimi K2.5", provider: "replicate" },
+            { id: "moonshotai/kimi-k2.6", name: "Kimi K2.6", provider: "replicate" },
+            {
+                id: "ibm-granite/granite-4.1-8b",
+                name: "Granite 4.1 8B",
+                provider: "replicate"
+            },
             {
                 id: "snowflake/snowflake-arctic-instruct",
                 name: "Snowflake Arctic Instruct",
                 provider: "replicate"
+            },
+            {
+                id: "anthropic/claude-sonnet-5",
+                name: "Claude Sonnet 5",
+                provider: "replicate"
+            },
+            {
+                id: "anthropic/claude-sonnet-4.6",
+                name: "Claude Sonnet 4.6",
+                provider: "replicate"
+            },
+            {
+                id: "anthropic/claude-fable-5",
+                name: "Claude Fable 5",
+                provider: "replicate"
+            },
+            { id: "openai/gpt-5.4", name: "GPT-5.4", provider: "replicate" },
+            {
+                id: "openai/gpt-5.6-sol",
+                name: "GPT-5.6 Sol",
+                provider: "replicate"
+            },
+            {
+                id: "openai/gpt-5.6-terra",
+                name: "GPT-5.6 Terra",
+                provider: "replicate"
+            },
+            {
+                id: "openai/gpt-5.6-luna",
+                name: "GPT-5.6 Luna",
+                provider: "replicate"
+            },
+            {
+                id: "google/gemini-3.5-flash",
+                name: "Gemini 3.5 Flash",
+                provider: "replicate"
+            },
+            {
+                id: "qwen/qwen3-7-plus",
+                name: "Qwen3.7 Plus",
+                provider: "replicate"
+            },
+            {
+                id: "ibm-granite/granite-vision-4.1-4b",
+                name: "Granite Vision 4.1 4B",
+                provider: "replicate"
+            },
+            {
+                id: "anthropic/claude-3.5-haiku",
+                name: "Claude 3.5 Haiku",
+                provider: "replicate"
+            },
+            { id: "openai/gpt-5.1", name: "GPT-5.1", provider: "replicate" },
+            { id: "openai/gpt-5-pro", name: "GPT-5 Pro", provider: "replicate" },
+            { id: "openai/o1-mini", name: "O1 Mini", provider: "replicate" },
+            {
+                id: "openai/gpt-oss-120b",
+                name: "GPT OSS 120B",
+                provider: "replicate"
+            },
+            { id: "openai/gpt-oss-20b", name: "GPT OSS 20B", provider: "replicate" },
+            {
+                id: "meta/llama-4-maverick-instruct",
+                name: "Llama 4 Maverick Instruct",
+                provider: "replicate"
+            },
+            {
+                id: "meta/llama-4-scout-instruct",
+                name: "Llama 4 Scout Instruct",
+                provider: "replicate"
+            },
+            {
+                id: "ibm-granite/granite-3.2-8b-instruct",
+                name: "Granite 3.2 8B Instruct",
+                provider: "replicate"
+            },
+            {
+                id: "ibm-granite/granite-3.3-8b-instruct",
+                name: "Granite 3.3 8B Instruct",
+                provider: "replicate"
+            },
+            {
+                id: "ibm-granite/granite-vision-3.3-2b",
+                name: "Granite Vision 3.3 2B",
+                provider: "replicate"
+            },
+            {
+                id: "lucataco/qwen2.5-omni-7b",
+                name: "Qwen2.5 Omni 7B",
+                provider: "replicate"
+            },
+            {
+                id: "google/gemini-3-flash",
+                name: "Gemini 3 Flash",
+                provider: "replicate"
+            },
+            {
+                id: "openai/gpt-5-structured",
+                name: "GPT-5 Structured",
+                provider: "replicate"
+            },
+            {
+                id: "moonshotai/kimi-k2-thinking",
+                name: "Kimi K2 Thinking",
+                provider: "replicate"
+            },
+            {
+                id: "ibm-granite/granite-4.0-h-small",
+                name: "Granite 4.0 H Small",
+                provider: "replicate"
+            },
+            {
+                id: "nvidia/nemotron-3-nano-30b-a3b",
+                name: "Nemotron 3 Nano 30B A3B",
+                provider: "replicate"
+            },
+            {
+                id: "nvidia/nemotron-nano-v2-12b-vl",
+                name: "Nemotron Nano V2 12B VL",
+                provider: "replicate"
             }
         ];
     }
@@ -252,22 +519,129 @@ export class ReplicateProvider extends BaseProvider {
                 name: "MiniMax Speech 2.8 HD",
                 provider: "replicate"
             },
+            {
+                id: "minimax/speech-2.8-turbo",
+                name: "MiniMax Speech 2.8 Turbo",
+                provider: "replicate"
+            },
+            {
+                id: "google/gemini-3.1-flash-tts",
+                name: "Gemini 3.1 Flash TTS",
+                provider: "replicate"
+            },
+            { id: "qwen/qwen3-tts", name: "Qwen3 TTS", provider: "replicate" },
+            {
+                id: "inworld/tts-1.5-max",
+                name: "Inworld TTS 1.5 Max",
+                provider: "replicate"
+            },
+            {
+                id: "inworld/tts-1.5-mini",
+                name: "Inworld TTS 1.5 Mini",
+                provider: "replicate"
+            },
+            {
+                id: "inworld/realtime-tts-2",
+                name: "Inworld Realtime TTS 2",
+                provider: "replicate"
+            },
+            {
+                id: "xai/grok-text-to-speech",
+                name: "Grok Text To Speech",
+                provider: "replicate"
+            },
             { id: "jaaari/kokoro-82m", name: "Kokoro 82M", provider: "replicate" },
             {
                 id: "resemble-ai/chatterbox-pro",
                 name: "Chatterbox Pro",
                 provider: "replicate"
             },
-            { id: "x-lance/f5-tts", name: "F5 TTS", provider: "replicate" }
+            {
+                id: "resemble-ai/chatterbox-turbo",
+                name: "Chatterbox Turbo",
+                provider: "replicate"
+            },
+            {
+                id: "resemble-ai/chatterbox-multilingual",
+                name: "Chatterbox Multilingual",
+                provider: "replicate"
+            },
+            { id: "x-lance/f5-tts", name: "F5 TTS", provider: "replicate" },
+            {
+                id: "inworld/realtime-tts-1.5-max",
+                name: "Inworld Realtime TTS 1.5 Max",
+                provider: "replicate"
+            },
+            {
+                id: "inworld/realtime-tts-1.5-mini",
+                name: "Inworld Realtime TTS 1.5 Mini",
+                provider: "replicate"
+            },
+            {
+                id: "elevenlabs/v2-multilingual",
+                name: "ElevenLabs V2 Multilingual",
+                provider: "replicate"
+            },
+            {
+                id: "elevenlabs/turbo-v2.5",
+                name: "ElevenLabs Turbo V2.5",
+                provider: "replicate"
+            },
+            {
+                id: "elevenlabs/flash-v2.5",
+                name: "ElevenLabs Flash V2.5",
+                provider: "replicate"
+            },
+            {
+                id: "minimax/speech-2.6-hd",
+                name: "MiniMax Speech 2.6 HD",
+                provider: "replicate"
+            },
+            {
+                id: "minimax/speech-2.6-turbo",
+                name: "MiniMax Speech 2.6 Turbo",
+                provider: "replicate"
+            },
+            {
+                id: "resemble-ai/chatterbox",
+                name: "Chatterbox",
+                provider: "replicate"
+            }
         ];
     }
+    async getAvailableMusicModels() {
+        return loadMusicModels("@nodetool-ai/replicate-nodes", "replicate-manifest.json", "replicate");
+    }
+    async getAvailableAudioToAudioModels() {
+        return loadAudioToAudioModels("@nodetool-ai/replicate-nodes", "replicate-manifest.json", "replicate");
+    }
     async getAvailableASRModels() {
         return [
+            {
+                id: "elevenlabs/scribe-v2",
+                name: "ElevenLabs Scribe V2",
+                provider: "replicate"
+            },
             {
                 id: "openai/gpt-4o-transcribe",
                 name: "GPT-4o Transcribe",
                 provider: "replicate"
             },
+            {
+                id: "openai/gpt-4o-mini-transcribe",
+                name: "GPT-4o Mini Transcribe",
+                provider: "replicate"
+            },
+            {
+                id: "xai/grok-speech-to-text",
+                name: "Grok Speech To Text",
+                provider: "replicate"
+            },
+            {
+                id: "nvidia/parakeet-rnnt-1.1b",
+                name: "Parakeet RNNT 1.1B",
+                provider: "replicate"
+            },
             {
                 id: "vaibhavs10/incredibly-fast-whisper",
                 name: "Incredibly Fast Whisper",
@@ -279,6 +653,31 @@ export class ReplicateProvider extends BaseProvider {
                 id: "thomasmol/whisper-diarization",
                 name: "Whisper Diarization",
                 provider: "replicate"
+            },
+            {
+                id: "rafaelgalle/whisper-diarization-advanced",
+                name: "Whisper Diarization Advanced",
+                provider: "replicate"
+            },
+            {
+                id: "ibm-granite/granite-speech-3.3-8b",
+                name: "Granite Speech 3.3 8B",
+                provider: "replicate"
+            },
+            {
+                id: "ibm-granite/granite-speech-4.1-2b",
+                name: "Granite Speech 4.1 2B",
+                provider: "replicate"
+            },
+            {
+                id: "nvidia/canary-qwen-2.5b",
+                name: "Canary Qwen 2.5B",
+                provider: "replicate"
+            },
+            {
+                id: "victor-upmeet/whisperx",
+                name: "WhisperX Large V3",
+                provider: "replicate"
             }
         ];
     }
@@ -314,6 +713,30 @@ export class ReplicateProvider extends BaseProvider {
                 provider: "replicate",
                 dimensions: 768
             },
+            {
+                id: "ibm-granite/granite-embedding-small-english-r2",
+                name: "Granite Embedding Small English R2",
+                provider: "replicate",
+                dimensions: 384
+            },
+            {
+                id: "adirik/e5-mistral-7b-instruct",
+                name: "E5 Mistral 7B Instruct",
+                provider: "replicate",
+                dimensions: 4096
+            },
+            {
+                id: "mark3labs/embeddings-gte-base",
+                name: "GTE Base",
+                provider: "replicate",
+                dimensions: 768
+            },
+            {
+                id: "center-for-curriculum-redesign/bge_1-5_query_embeddings",
+                name: "BGE 1.5 Query Embeddings",
+                provider: "replicate",
+                dimensions: 1024
+            },
             {
                 id: "beautyyuyanli/multilingual-e5-large",
                 name: "Multilingual E5 Large",
@@ -343,46 +766,42 @@ export class ReplicateProvider extends BaseProvider {
                 name: "ImageBind",
                 provider: "replicate",
                 dimensions: 1024
+            },
+            {
+                id: "openai/clip",
+                name: "CLIP ViT-L/14",
+                provider: "replicate",
+                dimensions: 768
+            },
+            {
+                id: "krthr/clip-embeddings",
+                name: "CLIP Embeddings",
+                provider: "replicate",
+                dimensions: 768
             }
         ];
     }
     // ---------------------------------------------------------------------------
     // Image / Video / Audio capabilities
     // ---------------------------------------------------------------------------
-    async _fetchOutputBytes(output) {
-        // replicate.run() returns FileOutput (ReadableStream) for file models,
-        // or a string URL, or an array of URLs/FileOutputs.
-        let target = output;
-        if (Array.isArray(output)) {
-            target = output[0];
-        }
-        // FileOutput is a ReadableStream — read it to bytes
-        if (target && typeof target === "object" && "getReader" in target) {
-            const reader = target.getReader();
-            const chunks = [];
-            while (true) {
-                const { done, value } = await reader.read();
-                if (done)
-                    break;
-                chunks.push(value);
-            }
-            const totalLength = chunks.reduce((sum, c) => sum + c.length, 0);
-            const result = new Uint8Array(totalLength);
-            let offset = 0;
-            for (const chunk of chunks) {
-                result.set(chunk, offset);
-                offset += chunk.length;
-            }
-            return result;
-        }
-        // String URL — fetch the bytes
-        if (typeof target === "string") {
-            const res = await fetch(target);
-            if (!res.ok)
-                throw new Error(`Failed to fetch output: ${res.status}`);
-            return new Uint8Array(await res.arrayBuffer());
+    async _fetchOutputBytes(output, signal) {
+        signal?.throwIfAborted();
+        const target = decodeReplicateOutput(output);
+        if (!target) {
+            throw new Error("Replicate prediction returned no usable output " +
+                `(output was ${describeOutputShape(output)})`);
         }
-        throw new Error("Replicate prediction returned no usable output");
+        return target.kind === "stream"
+            ? drainStream(target.stream)
+            : this._fetchUrlBytes(target.url, signal);
+    }
+    async _fetchUrlBytes(url, signal) {
+        if (url.startsWith("data:"))
+            return decodeDataUri(url);
+        const res = await safeFetch(url, signal ? { signal } : undefined);
+        if (!res.ok)
+            throw new Error(`Failed to fetch output: ${res.status}`);
+        return new Uint8Array(await res.arrayBuffer());
     }
     async textToImage(params) {
         if (!params.prompt)
@@ -403,13 +822,11 @@ export class ReplicateProvider extends BaseProvider {
         if (params.scheduler)
             input.scheduler = params.scheduler;
         log.debug("textToImage", { model: params.model.id });
-        const output = await this._client.run(params.model.id, { input });
+        const output = await this.runModel(params.model.id, input);
         return this._fetchOutputBytes(output);
     }
-    async imageToImage(image, params) {
-        const base64 = Buffer.from(image).toString("base64");
-        const dataUri = `data:image/png;base64,${base64}`;
-        const input = { image: dataUri };
+    async imageToImage(images, params) {
+        const input = this.imageInput(params.model.id, images);
         if (params.prompt)
             input.prompt = params.prompt;
         if (params.negativePrompt)
@@ -423,7 +840,31 @@ export class ReplicateProvider extends BaseProvider {
         if (params.seed != null)
             input.seed = params.seed;
         log.debug("imageToImage", { model: params.model.id });
-        const output = await this._client.run(params.model.id, { input });
+        const output = await this.runModel(params.model.id, input);
+        return this._fetchOutputBytes(output);
+    }
+    async inpaint(images, params) {
+        const input = this.imageInput(params.model.id, images);
+        if (params.prompt)
+            input.prompt = params.prompt;
+        if (params.negativePrompt)
+            input.negative_prompt = params.negativePrompt;
+        if (params.guidanceScale != null)
+            input.guidance_scale = params.guidanceScale;
+        if (params.numInferenceSteps != null)
+            input.num_inference_steps = params.numInferenceSteps;
+        if (params.strength != null)
+            input.strength = params.strength;
+        if (params.seed != null)
+            input.seed = params.seed;
+        if (params.mask && params.mask.length > 0) {
+            const maskUri = this.imageDataUri(params.mask);
+            const maskField = selectMaskImageInput(getModelImageInputs("@nodetool-ai/replicate-nodes", "replicate-manifest.json", params.model.id));
+            const fieldName = maskField?.apiName ?? "mask_url";
+            input[fieldName] = maskField?.isList ? [maskUri] : maskUri;
+        }
+        log.debug("inpaint", { model: params.model.id });
+        const output = await this.runModel(params.model.id, input);
         return this._fetchOutputBytes(output);
     }
     async textToVideo(params) {
@@ -441,13 +882,17 @@ export class ReplicateProvider extends BaseProvider {
         if (params.seed != null)
             input.seed = params.seed;
         log.debug("textToVideo", { model: params.model.id });
-        const output = await this._client.run(params.model.id, { input });
+        const output = await this.runModel(params.model.id, input);
         return this._fetchOutputBytes(output);
     }
     async imageToVideo(image, params) {
-        const base64 = Buffer.from(image).toString("base64");
-        const dataUri = `data:image/png;base64,${base64}`;
-        const input = { image: dataUri };
+        const model = (await this.getAvailableVideoModels()).find((item) => item.id === params.model.id);
+        if (model && !model.supportedTasks?.includes("image_to_video")) {
+            throw new Error(`Replicate model ${params.model.id} does not support image_to_video`);
+        }
+        if (image.length === 0)
+            throw new Error("Replicate image_to_video requires a start image");
+        const input = this.imageInput(params.model.id, [image]);
         if (params.prompt)
             input.prompt = params.prompt;
         if (params.negativePrompt)
@@ -461,24 +906,337 @@ export class ReplicateProvider extends BaseProvider {
         if (params.seed != null)
             input.seed = params.seed;
         log.debug("imageToVideo", { model: params.model.id });
-        const output = await this._client.run(params.model.id, { input });
+        const output = await this.runModel(params.model.id, input);
         return this._fetchOutputBytes(output);
     }
-    async *textToSpeech(args) {
+    async referenceToVideo(inputs, params) {
+        const modelId = params.model.id;
+        const model = (await this.getAvailableVideoModels()).find((item) => item.id === modelId);
+        if (!model?.supportedTasks?.includes("reference_to_video")) {
+            throw new Error(`Replicate model ${modelId} does not support reference_to_video`);
+        }
+        const fields = getModelReferenceInputs("@nodetool-ai/replicate-nodes", "replicate-manifest.json", modelId);
+        const references = {
+            images: inputs.images.filter((bytes) => bytes.length > 0),
+            videos: inputs.videos.filter((bytes) => bytes.length > 0)
+        };
+        validateReferenceInputs("Replicate", modelId, references, fields);
+        const declared = getModelInputFields("@nodetool-ai/replicate-nodes", "replicate-manifest.json", modelId);
+        const hasField = (name) => declared.some((field) => field.name === name);
+        if (params.useReferenceVideoAudio === true && (!hasField("use_reference_video_audio") || references.videos.length === 0)) {
+            throw new Error(`Replicate model ${modelId} does not support reference video audio for these inputs`);
+        }
+        const input = {};
+        for (const field of fields) {
+            const buffers = field.kind === "image" ? references.images : references.videos;
+            if (buffers.length === 0)
+                continue;
+            const urls = buffers.map((bytes) => field.kind === "image"
+                ? this.imageDataUri(bytes)
+                : this.dataUri(bytes, sniffVideoMime(bytes)));
+            input[field.apiName] = field.isList ? urls : urls[0];
+        }
+        const options = {
+            prompt: params.prompt,
+            negative_prompt: params.negativePrompt,
+            duration: params.durationSeconds,
+            aspect_ratio: params.aspectRatio,
+            resolution: params.resolution,
+            num_frames: params.numFrames,
+            guidance_scale: params.guidanceScale,
+            seed: params.seed,
+            use_reference_video_audio: params.useReferenceVideoAudio
+        };
+        for (const [name, value] of Object.entries(options)) {
+            if (value != null && hasField(name))
+                input[name] = value;
+        }
+        const timeout = params.timeoutSeconds && params.timeoutSeconds > 0
+            ? AbortSignal.timeout(params.timeoutSeconds * 1000)
+            : undefined;
+        const signal = params.signal && timeout
+            ? AbortSignal.any([params.signal, timeout])
+            : params.signal ?? timeout;
+        const output = await this.runModel(modelId, input, signal);
+        return this._fetchOutputBytes(output, signal);
+    }
+    dataUri(bytes, mimeType) {
+        return `data:${mimeType};base64,${Buffer.from(bytes).toString("base64")}`;
+    }
+    // Replicate models decode by the declared type, so feeding back a WebP this
+    // provider itself produced as `image/png` fails inside the model.
+    imageDataUri(bytes) {
+        return this.dataUri(bytes, detectImageMime(bytes));
+    }
+    /**
+     * Build the image portion of a Replicate prediction input from one or more
+     * source images, using the model's declared schema (replicate-manifest) to
+     * pick the correct input field — single (`image`, `input_image`, …) vs list
+     * (`image_input`, `images`, …). Falls back to the common conventions when the
+     * model isn't in the manifest. Empty buffers are dropped.
+     */
+    imageInput(modelId, images) {
+        const dataUris = images
+            .filter((b) => b && b.length > 0)
+            .map((b) => this.imageDataUri(b));
+        if (dataUris.length === 0)
+            return {};
+        const field = selectPrimaryImageInput(getModelImageInputs("@nodetool-ai/replicate-nodes", "replicate-manifest.json", modelId), dataUris.length);
+        if (field) {
+            return { [field.apiName]: field.isList ? dataUris : dataUris[0] };
+        }
+        // Unknown model: fall back to the common conventions.
+        return dataUris.length === 1
+            ? { image: dataUris[0] }
+            : { image_input: dataUris };
+    }
+    async runWithInput(modelId, input) {
+        return this._fetchOutputBytes(await this.runModel(modelId, input));
+    }
+    async upscaleImage(image, params) {
+        const input = {
+            image: this.imageDataUri(image)
+        };
+        if (params.scale != null)
+            input.scale = params.scale;
+        if (params.prompt)
+            input.prompt = params.prompt;
+        if (params.creativity != null)
+            input.creativity = params.creativity;
+        if (params.seed != null)
+            input.seed = params.seed;
+        log.debug("upscaleImage", { model: params.model.id });
+        return this.runWithInput(params.model.id, input);
+    }
+    /**
+     * Outpaint an image. Replicate's expand models take the target canvas rather
+     * than per-side padding (`bria/expand-image`: `canvas_size` plus where the
+     * original sits inside it), so the padding is turned into that geometry and
+     * pruned to whatever the model declares.
+     */
+    async outpaintImage(images, params) {
+        const input = this.imageInput(params.model.id, images);
+        if (params.prompt)
+            input.prompt = params.prompt;
+        if (params.negativePrompt)
+            input.negative_prompt = params.negativePrompt;
+        if (params.aspectRatio)
+            input.aspect_ratio = params.aspectRatio;
+        const padding = params.padding;
+        if (padding) {
+            const left = padding.left ?? 0;
+            const top = padding.top ?? 0;
+            input.expand_left = left;
+            input.expand_right = padding.right ?? 0;
+            input.expand_top = top;
+            input.expand_bottom = padding.bottom ?? 0;
+            // Where the source image lands on the expanded canvas, for the models
+            // that place it explicitly instead of taking per-side amounts.
+            input.original_image_location = [left, top];
+        }
+        if (params.seed != null)
+            input.seed = params.seed;
+        log.debug("outpaintImage", { model: params.model.id });
+        return this.runWithInput(params.model.id, this.pruneToDeclaredInputs(params.model.id, input));
+    }
+    async removeBackground(image, params) {
+        log.debug("removeBackground", { model: params.model.id });
+        return this.runWithInput(params.model.id, {
+            image: this.imageDataUri(image)
+        });
+    }
+    async relightImage(image, params) {
+        const input = {
+            image: this.imageDataUri(image)
+        };
+        if (params.prompt)
+            input.prompt = params.prompt;
+        if (params.negativePrompt)
+            input.negative_prompt = params.negativePrompt;
+        if (params.seed != null)
+            input.seed = params.seed;
+        log.debug("relightImage", { model: params.model.id });
+        return this.runWithInput(params.model.id, input);
+    }
+    async vectorizeImage(image, params) {
+        log.debug("vectorizeImage", { model: params.model.id });
+        return this.runWithInput(params.model.id, {
+            image: this.imageDataUri(image)
+        });
+    }
+    /**
+     * Drop input fields the model does not declare in the manifest.
+     *
+     * Replicate fails the whole prediction on an undeclared field instead of
+     * ignoring it, so a generic param bag breaks any endpoint with a narrower
+     * schema. An empty declaration set means the model is not in the manifest —
+     * send everything rather than silently stripping the request bare.
+     */
+    pruneToDeclaredInputs(modelId, input) {
+        const declared = getModelInputNames("@nodetool-ai/replicate-nodes", "replicate-manifest.json", modelId);
+        if (declared.size === 0)
+            return input;
+        const pruned = {};
+        const dropped = [];
+        for (const [key, value] of Object.entries(input)) {
+            if (declared.has(key))
+                pruned[key] = value;
+            else
+                dropped.push(key);
+        }
+        if (dropped.length > 0) {
+            log.debug("pruned undeclared inputs", { model: modelId, dropped });
+        }
+        return pruned;
+    }
+    async videoToVideo(video, params) {
+        const images = params.referenceImages ?? [];
+        const fields = getModelReferenceInputs("@nodetool-ai/replicate-nodes", "replicate-manifest.json", params.model.id);
+        if (images.length) {
+            validateReferenceInputs("Replicate", params.model.id, { images, videos: [] }, fields);
+        }
+        params.signal?.throwIfAborted();
+        const input = {
+            video: this.dataUri(video, "video/mp4")
+        };
+        if (images.length) {
+            const urls = images.map((image) => this.imageDataUri(image));
+            for (const field of fields.filter((candidate) => candidate.kind === "image")) {
+                input[field.apiName] = field.isList ? urls : urls[0];
+            }
+        }
+        if (params.prompt)
+            input.prompt = params.prompt;
+        if (params.negativePrompt)
+            input.negative_prompt = params.negativePrompt;
+        if (params.strength != null)
+            input.strength = params.strength;
+        if (params.seed != null)
+            input.seed = params.seed;
+        log.debug("videoToVideo", { model: params.model.id });
+        return this._fetchOutputBytes(await this.runModel(params.model.id, this.pruneToDeclaredInputs(params.model.id, input), params.signal), params.signal);
+    }
+    async extendVideo(video, params) {
+        if (video.length === 0) {
+            throw new Error("video must not be empty");
+        }
+        const model = (await this.getAvailableVideoModels()).find((candidate) => candidate.id === params.model.id);
+        if (!model?.supportedTasks?.includes("extend_video")) {
+            throw new Error(`Model ${params.model.id} does not support extend_video`);
+        }
+        const directionTasks = model.supportedTasks.filter((task) => task.startsWith("extend_video_"));
+        if (directionTasks.length > 0 &&
+            !directionTasks.includes(`extend_video_${params.mode}`)) {
+            throw new Error(`Model ${params.model.id} does not support ${params.mode} extension`);
+        }
+        if (!Number.isFinite(params.durationSeconds) ||
+            params.durationSeconds <= 0) {
+            throw new Error("Extension duration must be positive");
+        }
+        const input = this.pruneToDeclaredInputs(params.model.id, {
+            video: this.dataUri(video, "video/mp4"),
+            prompt: params.prompt,
+            mode: params.mode,
+            duration: params.durationSeconds
+        });
+        log.debug("extendVideo", { model: params.model.id, mode: params.mode });
+        return this._fetchOutputBytes(await this.runModel(params.model.id, input));
+    }
+    async upscaleVideo(video, params) {
+        const input = {
+            video: this.dataUri(video, "video/mp4")
+        };
+        // One concept, four spellings across the catalog's upscalers; the prune
+        // below drops whichever names this model does not declare.
+        if (params.scale != null) {
+            input.scale = params.scale;
+            input.upscale_factor = params.scale;
+            input.resolution = params.scale;
+        }
+        if (params.targetResolution) {
+            input.target_resolution = params.targetResolution;
+        }
+        if (params.prompt)
+            input.prompt = params.prompt;
+        if (params.creativity != null)
+            input.creativity = params.creativity;
+        if (params.seed != null)
+            input.seed = params.seed;
+        log.debug("upscaleVideo", { model: params.model.id });
+        return this.runWithInput(params.model.id, this.pruneToDeclaredInputs(params.model.id, input));
+    }
+    async lipSync(video, params) {
+        const input = {
+            video: this.dataUri(video, "video/mp4"),
+            audio: this.dataUri(params.audio, "audio/mpeg")
+        };
+        if (params.seed != null)
+            input.seed = params.seed;
+        log.debug("lipSync", { model: params.model.id });
+        return this.runWithInput(params.model.id, this.pruneToDeclaredInputs(params.model.id, input));
+    }
+    /**
+     * Replicate TTS models (elevenlabs, minimax, kokoro, chatterbox, f5-tts)
+     * return an ENCODED audio file (MP3/WAV), not raw little-endian PCM. Expose
+     * it through the encoded path (like textToMusic / FAL) rather than overriding
+     * the streaming-PCM textToSpeech — the previous override blindly cast the
+     * encoded bytes to Int16Array, producing noise (MP3) or a header click +
+     * misaligned waveform (WAV).
+     */
+    async textToSpeechEncoded(args) {
         const input = { text: args.text };
         if (args.voice)
             input.voice = args.voice;
         if (args.speed != null)
             input.speed = args.speed;
-        log.debug("textToSpeech", { model: args.model });
-        const output = await this._client.run(args.model, {
-            input
-        });
+        log.debug("textToSpeechEncoded", { model: args.model });
+        const output = await this.runModel(args.model, input);
         const bytes = await this._fetchOutputBytes(output);
-        // Convert to Int16Array (assume raw PCM or WAV)
-        yield {
-            samples: new Int16Array(bytes.buffer, bytes.byteOffset, bytes.byteLength / 2)
+        return { data: bytes, mimeType: sniffAudioMime(bytes) };
+    }
+    /**
+     * Generate music as an encoded audio file. Replicate music models (MusicGen,
+     * MiniMax Music, ElevenLabs Music, …) return a file URL / FileOutput, so this
+     * runs the model and downloads the encoded audio bytes.
+     */
+    async textToMusic(params) {
+        if (!params.prompt)
+            throw new Error("Prompt is required");
+        const input = { prompt: params.prompt };
+        if (params.lyrics)
+            input.lyrics = params.lyrics;
+        if (params.durationSeconds != null) {
+            input.duration = Math.round(params.durationSeconds);
+        }
+        if (params.seed != null)
+            input.seed = params.seed;
+        log.debug("textToMusic", { model: params.model.id });
+        const output = await this.runModel(params.model.id, input);
+        const bytes = await this._fetchOutputBytes(output);
+        return { data: bytes, mimeType: sniffAudioMime(bytes) };
+    }
+    /**
+     * Rewrite a recording. Replicate's audio transforms (super-resolution, stem
+     * separation, enhancers) take the source under `audio`; the direction fields
+     * go through {@link pruneToDeclaredInputs} like {@link lipSync} does, so a
+     * model that declares none of them does not get sent one.
+     */
+    async audioToAudio(audio, params) {
+        const input = {
+            audio: this.dataUri(audio, "audio/mpeg")
         };
+        if (params.prompt)
+            input.prompt = params.prompt;
+        if (params.voice)
+            input.voice = params.voice;
+        if (params.strength != null)
+            input.strength = params.strength;
+        if (params.seed != null)
+            input.seed = params.seed;
+        log.debug("audioToAudio", { model: params.model.id });
+        const output = await this.runModel(params.model.id, this.pruneToDeclaredInputs(params.model.id, input));
+        const bytes = await this._fetchOutputBytes(output);
+        return { data: bytes, mimeType: sniffAudioMime(bytes) };
     }
 }
 //# sourceMappingURL=replicate-provider.js.map