@nodetool-ai/agents 0.7.0-rc.11

package/dist/tools/media-tools.js

@@ -0,0 +1,682 @@
+/**
+ * Generic provider-backed media generation tools for any agent loop.
+ *
+ * Each tool is a thin wrapper around `ProcessingContext.runProviderPrediction`
+ * (or `streamProviderPrediction` for capabilities that require it). They take
+ * `provider` + `model_id` + capability-specific params, dispatch to whichever
+ * `BaseProvider` is registered for that id, and return a JSON-friendly result.
+ *
+ * Persistence: outputs are saved as proper **assets** when the context
+ * exposes a `createAsset` model interface — the tool returns an
+ * `asset_id` + `asset_uri` ("asset://<id>.<ext>") that subsequent steps and
+ * the chat UI can reference. When no asset interface is wired (CLI without
+ * a model layer, tests), the tool falls back to writing the bytes to the
+ * caller-supplied `output_file` under the workspace.
+ *
+ * Pair with `find_model` so the agent first picks a `{provider, model_id}`
+ * and then calls the right generation tool.
+ */
+import { Buffer } from "node:buffer";
+import { promises as fs } from "node:fs";
+import path from "node:path";
+import { Tool } from "./base-tool.js";
+const MAX_INLINE_TEXT_PREVIEW = 500;
+const MIME_TO_EXT = {
+    "image/png": "png",
+    "image/jpeg": "jpg",
+    "image/jpg": "jpg",
+    "image/webp": "webp",
+    "image/gif": "gif",
+    "video/mp4": "mp4",
+    "video/webm": "webm",
+    "audio/mpeg": "mp3",
+    "audio/mp3": "mp3",
+    "audio/wav": "wav",
+    "audio/x-wav": "wav",
+    "audio/flac": "flac",
+    "audio/ogg": "ogg",
+    "application/octet-stream": "bin"
+};
+function workspaceDir(context) {
+    const ws = context["workspaceDir"];
+    return typeof ws === "string" && ws ? ws : null;
+}
+function timestampedName(prefix, ext) {
+    const ts = new Date().toISOString().replace(/[:.]/g, "-");
+    return `${prefix}-${ts}.${ext}`;
+}
+function inferImageMime(bytes) {
+    if (bytes.length >= 8 &&
+        bytes[0] === 0x89 && bytes[1] === 0x50 &&
+        bytes[2] === 0x4e && bytes[3] === 0x47)
+        return "image/png";
+    if (bytes.length >= 3 &&
+        bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff)
+        return "image/jpeg";
+    if (bytes.length >= 6 &&
+        bytes[0] === 0x47 && bytes[1] === 0x49 && bytes[2] === 0x46)
+        return "image/gif";
+    if (bytes.length >= 12 &&
+        bytes[8] === 0x57 && bytes[9] === 0x45 &&
+        bytes[10] === 0x42 && bytes[11] === 0x50)
+        return "image/webp";
+    return "image/png";
+}
+/**
+ * Persist generated bytes. Prefers `context.createAsset` (writes to storage
+ * + DB), falling back to the workspace filesystem when the model interface
+ * isn't wired. Returns whichever location is available so the agent can
+ * reference the output in subsequent steps.
+ */
+async function persistOutput(context, bytes, opts) {
+    const ext = MIME_TO_EXT[opts.mime] ?? "bin";
+    const result = { bytes: bytes.length, mime_type: opts.mime };
+    // Asset path: persisted in storage + DB, returns asset:// URI.
+    if (typeof context.createAsset === "function") {
+        try {
+            const name = opts.outputFile ?? timestampedName(opts.namePrefix, ext);
+            const asset = (await context.createAsset({
+                name,
+                contentType: opts.mime,
+                content: bytes
+            }));
+            if (asset && typeof asset.id === "string") {
+                result.asset_id = asset.id;
+                result.asset_uri = `asset://${asset.id}.${ext}`;
+            }
+        }
+        catch {
+            // Fall through to filesystem fallback.
+        }
+    }
+    // Workspace fallback. Always write when the caller supplied output_file
+    // OR when asset creation didn't happen, so the agent has a concrete
+    // location to reference.
+    if (opts.outputFile || !result.asset_id) {
+        const fileName = opts.outputFile ?? timestampedName(opts.namePrefix, ext);
+        const ws = workspaceDir(context);
+        const filePath = ws ? path.join(ws, fileName) : fileName;
+        await fs.mkdir(path.dirname(filePath), { recursive: true });
+        await fs.writeFile(filePath, Buffer.from(bytes));
+        result.path = filePath;
+    }
+    return result;
+}
+async function readWorkspaceOrAssetFile(context, inputFile) {
+    const ws = workspaceDir(context);
+    const filePath = ws ? path.join(ws, inputFile) : inputFile;
+    const buf = await fs.readFile(filePath);
+    return new Uint8Array(buf);
+}
+function parseModelArgs(params) {
+    const provider = params["provider"];
+    const model = params["model"];
+    if (typeof provider !== "string" || !provider) {
+        return { error: "provider must be a non-empty string (use find_model)" };
+    }
+    if (typeof model !== "string" || !model) {
+        return { error: "model must be a non-empty string (use find_model)" };
+    }
+    return { provider, model };
+}
+/* ------------------------------------------------------------------ */
+/*  Image generation                                                  */
+/* ------------------------------------------------------------------ */
+export class GenerateImageTool extends Tool {
+    name = "generate_image";
+    description = "Generate an image from a text prompt using a provider+model selected via find_model (capability=text_to_image). The result is saved as an asset (asset:// URI returned); pass `output_file` to also write a workspace copy.";
+    inputSchema = {
+        type: "object",
+        properties: {
+            provider: { type: "string", description: "Provider id from find_model." },
+            model: { type: "string", description: "Model id from find_model." },
+            prompt: { type: "string", description: "Text prompt." },
+            output_file: {
+                type: "string",
+                description: "Optional workspace-relative path to also write the result. Omit to rely on the asset URI."
+            },
+            negative_prompt: { type: "string" },
+            width: { type: "number" },
+            height: { type: "number" },
+            quality: { type: "string" }
+        },
+        required: ["provider", "model", "prompt"]
+    };
+    async process(context, params) {
+        const m = parseModelArgs(params);
+        if ("error" in m)
+            return m;
+        const prompt = params["prompt"];
+        if (typeof prompt !== "string" || !prompt)
+            return { error: "prompt is required" };
+        try {
+            const result = (await context.runProviderPrediction({
+                provider: m.provider,
+                capability: "text_to_image",
+                model: m.model,
+                params: {
+                    prompt,
+                    negative_prompt: params["negative_prompt"],
+                    width: params["width"],
+                    height: params["height"],
+                    quality: params["quality"]
+                }
+            }));
+            const persisted = await persistOutput(context, result, {
+                namePrefix: "generated-image",
+                mime: inferImageMime(result),
+                outputFile: typeof params["output_file"] === "string"
+                    ? params["output_file"]
+                    : undefined
+            });
+            return { type: "image", provider: m.provider, model: m.model, ...persisted };
+        }
+        catch (e) {
+            return { error: `text_to_image failed: ${e instanceof Error ? e.message : String(e)}` };
+        }
+    }
+    userMessage(params) {
+        return `Generating image with ${String(params["provider"])}:${String(params["model"])}`;
+    }
+}
+export class EditImageTool extends Tool {
+    name = "edit_image";
+    description = "Transform a source image with a text prompt using a provider+model selected via find_model (capability=image_to_image). Source can be an asset URI (asset://...) or a workspace path. Result is saved as an asset.";
+    inputSchema = {
+        type: "object",
+        properties: {
+            provider: { type: "string", description: "Provider id from find_model." },
+            model: { type: "string", description: "Model id from find_model." },
+            input_file: {
+                type: "string",
+                description: "Workspace-relative path of the source image (or asset:// URI)."
+            },
+            prompt: { type: "string", description: "Text prompt describing the desired transformation." },
+            output_file: {
+                type: "string",
+                description: "Optional workspace-relative path to also write the result."
+            },
+            negative_prompt: { type: "string" },
+            target_width: { type: "number" },
+            target_height: { type: "number" },
+            strength: { type: "number" }
+        },
+        required: ["provider", "model", "input_file", "prompt"]
+    };
+    async process(context, params) {
+        const m = parseModelArgs(params);
+        if ("error" in m)
+            return m;
+        const inputFile = params["input_file"];
+        const prompt = params["prompt"];
+        if (typeof inputFile !== "string" || !inputFile)
+            return { error: "input_file is required" };
+        if (typeof prompt !== "string" || !prompt)
+            return { error: "prompt is required" };
+        try {
+            const image = await readWorkspaceOrAssetFile(context, inputFile);
+            const result = (await context.runProviderPrediction({
+                provider: m.provider,
+                capability: "image_to_image",
+                model: m.model,
+                params: {
+                    image,
+                    prompt,
+                    negative_prompt: params["negative_prompt"],
+                    target_width: params["target_width"],
+                    target_height: params["target_height"],
+                    strength: params["strength"]
+                }
+            }));
+            const persisted = await persistOutput(context, result, {
+                namePrefix: "edited-image",
+                mime: inferImageMime(result),
+                outputFile: typeof params["output_file"] === "string"
+                    ? params["output_file"]
+                    : undefined
+            });
+            return { type: "image", provider: m.provider, model: m.model, ...persisted };
+        }
+        catch (e) {
+            return { error: `image_to_image failed: ${e instanceof Error ? e.message : String(e)}` };
+        }
+    }
+    userMessage(params) {
+        return `Editing image with ${String(params["provider"])}:${String(params["model"])}`;
+    }
+}
+/* ------------------------------------------------------------------ */
+/*  Video generation                                                  */
+/* ------------------------------------------------------------------ */
+export class GenerateVideoTool extends Tool {
+    name = "generate_video";
+    description = "Generate a video from a text prompt using a provider+model selected via find_model (capability=text_to_video). Result is saved as an asset (asset:// URI returned).";
+    inputSchema = {
+        type: "object",
+        properties: {
+            provider: { type: "string" },
+            model: { type: "string" },
+            prompt: { type: "string" },
+            output_file: {
+                type: "string",
+                description: "Optional workspace-relative path to also write the result."
+            },
+            negative_prompt: { type: "string" },
+            num_frames: { type: "number" },
+            aspect_ratio: { type: "string" },
+            resolution: { type: "string" }
+        },
+        required: ["provider", "model", "prompt"]
+    };
+    async process(context, params) {
+        const m = parseModelArgs(params);
+        if ("error" in m)
+            return m;
+        const prompt = params["prompt"];
+        if (typeof prompt !== "string" || !prompt)
+            return { error: "prompt is required" };
+        try {
+            const result = (await context.runProviderPrediction({
+                provider: m.provider,
+                capability: "text_to_video",
+                model: m.model,
+                params: {
+                    prompt,
+                    negative_prompt: params["negative_prompt"],
+                    num_frames: params["num_frames"],
+                    aspect_ratio: params["aspect_ratio"],
+                    resolution: params["resolution"]
+                }
+            }));
+            const persisted = await persistOutput(context, result, {
+                namePrefix: "generated-video",
+                mime: "video/mp4",
+                outputFile: typeof params["output_file"] === "string"
+                    ? params["output_file"]
+                    : undefined
+            });
+            return { type: "video", provider: m.provider, model: m.model, ...persisted };
+        }
+        catch (e) {
+            return { error: `text_to_video failed: ${e instanceof Error ? e.message : String(e)}` };
+        }
+    }
+    userMessage(params) {
+        return `Generating video with ${String(params["provider"])}:${String(params["model"])}`;
+    }
+}
+export class AnimateImageTool extends Tool {
+    name = "animate_image";
+    description = "Animate a source image into a video using a provider+model selected via find_model (capability=image_to_video). Source can be a workspace path or asset URI; result is saved as an asset.";
+    inputSchema = {
+        type: "object",
+        properties: {
+            provider: { type: "string" },
+            model: { type: "string" },
+            input_file: {
+                type: "string",
+                description: "Workspace-relative path of the source image (or asset:// URI)."
+            },
+            output_file: {
+                type: "string",
+                description: "Optional workspace-relative path to also write the result."
+            },
+            prompt: { type: "string" },
+            num_frames: { type: "number" },
+            aspect_ratio: { type: "string" },
+            resolution: { type: "string" }
+        },
+        required: ["provider", "model", "input_file"]
+    };
+    async process(context, params) {
+        const m = parseModelArgs(params);
+        if ("error" in m)
+            return m;
+        const inputFile = params["input_file"];
+        if (typeof inputFile !== "string" || !inputFile)
+            return { error: "input_file is required" };
+        try {
+            const image = await readWorkspaceOrAssetFile(context, inputFile);
+            const result = (await context.runProviderPrediction({
+                provider: m.provider,
+                capability: "image_to_video",
+                model: m.model,
+                params: {
+                    image,
+                    prompt: params["prompt"],
+                    num_frames: params["num_frames"],
+                    aspect_ratio: params["aspect_ratio"],
+                    resolution: params["resolution"]
+                }
+            }));
+            const persisted = await persistOutput(context, result, {
+                namePrefix: "animated-video",
+                mime: "video/mp4",
+                outputFile: typeof params["output_file"] === "string"
+                    ? params["output_file"]
+                    : undefined
+            });
+            return { type: "video", provider: m.provider, model: m.model, ...persisted };
+        }
+        catch (e) {
+            return { error: `image_to_video failed: ${e instanceof Error ? e.message : String(e)}` };
+        }
+    }
+    userMessage(params) {
+        return `Animating image with ${String(params["provider"])}:${String(params["model"])}`;
+    }
+}
+function int16ToUint8(samples) {
+    const bytes = new Uint8Array(samples.byteLength);
+    const view = new DataView(bytes.buffer);
+    for (let i = 0; i < samples.length; i++) {
+        view.setInt16(i * 2, samples[i], true);
+    }
+    return bytes;
+}
+/**
+ * Wrap raw little-endian int16 PCM bytes in a minimal RIFF/WAVE container so
+ * the file is playable. Defaults match OpenAI's TTS PCM stream (24 kHz mono).
+ */
+function wrapPcmAsWav(pcm, sampleRate = 24000, channels = 1) {
+    const bitsPerSample = 16;
+    const byteRate = (sampleRate * channels * bitsPerSample) / 8;
+    const blockAlign = (channels * bitsPerSample) / 8;
+    const header = new Uint8Array(44);
+    const v = new DataView(header.buffer);
+    const w = (s, off) => {
+        for (let i = 0; i < s.length; i++)
+            v.setUint8(off + i, s.charCodeAt(i));
+    };
+    w("RIFF", 0);
+    v.setUint32(4, 36 + pcm.length, true);
+    w("WAVE", 8);
+    w("fmt ", 12);
+    v.setUint32(16, 16, true); // PCM chunk size
+    v.setUint16(20, 1, true); // PCM format
+    v.setUint16(22, channels, true);
+    v.setUint32(24, sampleRate, true);
+    v.setUint32(28, byteRate, true);
+    v.setUint16(32, blockAlign, true);
+    v.setUint16(34, bitsPerSample, true);
+    w("data", 36);
+    v.setUint32(40, pcm.length, true);
+    const out = new Uint8Array(header.length + pcm.length);
+    out.set(header, 0);
+    out.set(pcm, header.length);
+    return out;
+}
+const AUDIO_EXT_TO_FORMAT = {
+    mp3: "mp3",
+    wav: "wav",
+    flac: "flac",
+    opus: "opus",
+    ogg: "opus",
+    aac: "aac",
+    m4a: "aac"
+};
+function audioFormatFromOutputFile(outputFile) {
+    if (!outputFile)
+        return null;
+    const ext = path.extname(outputFile).slice(1).toLowerCase();
+    return AUDIO_EXT_TO_FORMAT[ext] ?? null;
+}
+function concatUint8(parts) {
+    let total = 0;
+    for (const p of parts)
+        total += p.length;
+    const out = new Uint8Array(total);
+    let offset = 0;
+    for (const p of parts) {
+        out.set(p, offset);
+        offset += p.length;
+    }
+    return out;
+}
+export class GenerateSpeechTool extends Tool {
+    name = "generate_speech";
+    description = "Synthesize speech audio from text using a provider+model selected via find_model (capability=text_to_speech). Result is saved as an asset (asset:// URI returned).";
+    inputSchema = {
+        type: "object",
+        properties: {
+            provider: { type: "string" },
+            model: { type: "string" },
+            text: { type: "string", description: "Text to speak." },
+            output_file: {
+                type: "string",
+                description: "Optional workspace-relative path to also write the audio file (mp3/wav/pcm depending on provider)."
+            },
+            voice: { type: "string" },
+            speed: { type: "number", description: "Speech speed (e.g. 0.25–4.0)." }
+        },
+        required: ["provider", "model", "text"]
+    };
+    async process(context, params) {
+        const m = parseModelArgs(params);
+        if ("error" in m)
+            return m;
+        const text = params["text"];
+        if (typeof text !== "string" || !text)
+            return { error: "text is required" };
+        const outputFile = typeof params["output_file"] === "string"
+            ? params["output_file"]
+            : undefined;
+        const desiredFormat = audioFormatFromOutputFile(outputFile) ?? "mp3";
+        try {
+            // Preferred path: ask the provider for fully-encoded audio in the
+            // desired container (mp3/wav/flac/...). Returns null when the provider
+            // doesn't support encoded TTS — we then fall through to streaming PCM.
+            let audio = null;
+            let mimeType;
+            let outputFileFinal = outputFile;
+            try {
+                const provider = await context.getProvider(m.provider);
+                const encoded = await provider.textToSpeechEncoded({
+                    text,
+                    model: m.model,
+                    voice: params["voice"],
+                    speed: params["speed"],
+                    audioFormat: desiredFormat
+                });
+                if (encoded && encoded.data) {
+                    audio = encoded.data;
+                    mimeType = encoded.mimeType;
+                }
+            }
+            catch {
+                // Fall through to streaming path.
+            }
+            if (!audio) {
+                // Streaming path — provider returns either pre-encoded chunks
+                // (carrying mimeType) or raw int16 PCM samples that we must wrap in
+                // a WAV container before writing to disk so the file is playable.
+                const parts = [];
+                let pcmOnly = true;
+                for await (const item of context.streamProviderPrediction({
+                    provider: m.provider,
+                    capability: "text_to_speech",
+                    model: m.model,
+                    params: {
+                        text,
+                        voice: params["voice"],
+                        speed: params["speed"]
+                    }
+                })) {
+                    const chunk = item;
+                    if (chunk.data instanceof Uint8Array) {
+                        parts.push(chunk.data);
+                        if (chunk.mimeType)
+                            mimeType = chunk.mimeType;
+                        pcmOnly = false;
+                    }
+                    else if (typeof chunk.data === "string") {
+                        parts.push(Buffer.from(chunk.data, "base64"));
+                        if (chunk.mimeType)
+                            mimeType = chunk.mimeType;
+                        pcmOnly = false;
+                    }
+                    else if (chunk.samples) {
+                        parts.push(int16ToUint8(chunk.samples));
+                    }
+                }
+                if (parts.length === 0)
+                    return { error: "Provider returned no audio data" };
+                const merged = concatUint8(parts);
+                if (pcmOnly && !mimeType) {
+                    // Wrap raw PCM in WAV so the bytes are playable. Rename .mp3 →
+                    // .wav since the actual data is now WAV, not MP3.
+                    audio = wrapPcmAsWav(merged);
+                    mimeType = "audio/wav";
+                    if (outputFileFinal) {
+                        const dir = path.dirname(outputFileFinal);
+                        const base = path.basename(outputFileFinal, path.extname(outputFileFinal));
+                        outputFileFinal = path.join(dir === "." ? "" : dir, `${base}.wav`);
+                    }
+                }
+                else {
+                    audio = merged;
+                }
+            }
+            const persisted = await persistOutput(context, audio, {
+                namePrefix: "generated-speech",
+                mime: mimeType ?? "audio/mpeg",
+                outputFile: outputFileFinal
+            });
+            return {
+                type: "audio",
+                provider: m.provider,
+                model: m.model,
+                ...persisted
+            };
+        }
+        catch (e) {
+            return { error: `text_to_speech failed: ${e instanceof Error ? e.message : String(e)}` };
+        }
+    }
+    userMessage(params) {
+        return `Synthesizing speech with ${String(params["provider"])}:${String(params["model"])}`;
+    }
+}
+export class TranscribeAudioTool extends Tool {
+    name = "transcribe_audio";
+    description = "Transcribe an audio file to text using a provider+model selected via find_model (capability=automatic_speech_recognition). Source can be a workspace path or asset:// URI.";
+    inputSchema = {
+        type: "object",
+        properties: {
+            provider: { type: "string" },
+            model: { type: "string" },
+            input_file: {
+                type: "string",
+                description: "Workspace-relative path of the audio file to transcribe (or asset:// URI)."
+            },
+            language: {
+                type: "string",
+                description: "Optional ISO 639-1 language hint (e.g. 'en')."
+            },
+            prompt: {
+                type: "string",
+                description: "Optional context to bias the model."
+            }
+        },
+        required: ["provider", "model", "input_file"]
+    };
+    async process(context, params) {
+        const m = parseModelArgs(params);
+        if ("error" in m)
+            return m;
+        const inputFile = params["input_file"];
+        if (typeof inputFile !== "string" || !inputFile)
+            return { error: "input_file is required" };
+        try {
+            const audio = await readWorkspaceOrAssetFile(context, inputFile);
+            const result = (await context.runProviderPrediction({
+                provider: m.provider,
+                capability: "automatic_speech_recognition",
+                model: m.model,
+                params: {
+                    audio,
+                    language: params["language"],
+                    prompt: params["prompt"]
+                }
+            }));
+            const text = String(result.text ?? "");
+            return {
+                type: "transcription",
+                provider: m.provider,
+                model: m.model,
+                text: text.length > MAX_INLINE_TEXT_PREVIEW
+                    ? text.slice(0, MAX_INLINE_TEXT_PREVIEW) +
+                        `… [${text.length - MAX_INLINE_TEXT_PREVIEW} chars truncated]`
+                    : text,
+                full_length: text.length,
+                chunks: Array.isArray(result.chunks) ? result.chunks.length : 0
+            };
+        }
+        catch (e) {
+            return { error: `transcribe failed: ${e instanceof Error ? e.message : String(e)}` };
+        }
+    }
+    userMessage(params) {
+        return `Transcribing audio with ${String(params["provider"])}:${String(params["model"])}`;
+    }
+}
+/* ------------------------------------------------------------------ */
+/*  Embeddings                                                        */
+/* ------------------------------------------------------------------ */
+export class EmbedTextTool extends Tool {
+    name = "embed_text";
+    description = "Compute embedding vector(s) for a text or list of texts using a provider+model selected via find_model (capability=generate_embedding).";
+    inputSchema = {
+        type: "object",
+        properties: {
+            provider: { type: "string" },
+            model: { type: "string" },
+            text: {
+                oneOf: [
+                    { type: "string" },
+                    { type: "array", items: { type: "string" } }
+                ],
+                description: "A single string or an array of strings to embed."
+            },
+            dimensions: {
+                type: "number",
+                description: "Optional target dimensions if the model supports truncation."
+            }
+        },
+        required: ["provider", "model", "text"]
+    };
+    async process(context, params) {
+        const m = parseModelArgs(params);
+        if ("error" in m)
+            return m;
+        const text = params["text"];
+        if (typeof text !== "string" && !Array.isArray(text))
+            return { error: "text must be a string or array of strings" };
+        try {
+            const result = (await context.runProviderPrediction({
+                provider: m.provider,
+                capability: "generate_embedding",
+                model: m.model,
+                params: {
+                    text,
+                    dimensions: params["dimensions"]
+                }
+            }));
+            return {
+                type: "embedding",
+                provider: m.provider,
+                model: m.model,
+                count: result.length,
+                dimensions: result[0]?.length ?? 0,
+                embeddings: result
+            };
+        }
+        catch (e) {
+            return { error: `generate_embedding failed: ${e instanceof Error ? e.message : String(e)}` };
+        }
+    }
+    userMessage(params) {
+        return `Embedding text with ${String(params["provider"])}:${String(params["model"])}`;
+    }
+}
+//# sourceMappingURL=media-tools.js.map