@stabgan/openrouter-mcp-multimodal 2.0.0 → 3.0.0

package/dist/tool-handlers/generate-image.js

@@ -1,43 +1,95 @@
 import { promises as fs } from 'fs';
-import { dirname } from 'path';
+import { resolveSafeOutputPath, UnsafeOutputPathError } from './path-safety.js';
+import { ErrorCode, toolError, toolErrorFrom } from '../errors.js';
+import { classifyUpstreamError } from './openrouter-errors.js';
 const DEFAULT_MODEL = 'google/gemini-2.5-flash-image';
 export async function handleGenerateImage(request, openai) {
-    const { prompt, model, save_path } = request.params.arguments;
+    const { prompt, model, save_path } = request.params.arguments ?? { prompt: '' };
     if (!prompt?.trim()) {
-        return { content: [{ type: 'text', text: 'Prompt is required.' }], isError: true };
+        return toolError(ErrorCode.INVALID_INPUT, 'prompt is required.');
     }
+    // Fail-fast on unsafe paths BEFORE spending tokens.
+    let safePathResolved = null;
+    if (save_path) {
+        try {
+            safePathResolved = await resolveSafeOutputPath(save_path);
+        }
+        catch (err) {
+            if (err instanceof UnsafeOutputPathError) {
+                return toolErrorFrom(ErrorCode.UNSAFE_PATH, err);
+            }
+            return toolErrorFrom(ErrorCode.INTERNAL, err);
+        }
+    }
+    let completion;
     try {
-        const completion = await openai.chat.completions.create({
+        completion = await openai.chat.completions.create({
             model: model || DEFAULT_MODEL,
             messages: [{ role: 'user', content: `Generate an image: ${prompt}` }],
         });
-        const message = completion.choices[0]?.message;
-        if (!message) {
-            return { content: [{ type: 'text', text: 'No response from model.' }], isError: true };
-        }
-        const base64 = extractBase64(message);
-        if (base64) {
-            if (save_path) {
-                const dir = dirname(save_path);
-                await fs.mkdir(dir, { recursive: true });
-                await fs.writeFile(save_path, Buffer.from(base64.data, 'base64'));
-                return {
-                    content: [
-                        { type: 'text', text: `Image saved to: ${save_path}` },
-                        { type: 'image', mimeType: base64.mime, data: base64.data },
-                    ],
-                };
-            }
-            return { content: [{ type: 'image', mimeType: base64.mime, data: base64.data }] };
-        }
+    }
+    catch (err) {
+        return classifyUpstreamError(err, 'generate_image');
+    }
+    const message = completion.choices[0]?.message;
+    if (!message) {
+        return toolError(ErrorCode.INTERNAL, 'No response from model.');
+    }
+    const base64 = extractBase64(message);
+    if (!base64) {
+        // Model talked but did not emit an image. Surface this as a distinct
+        // condition so callers don't treat chatter as a successful image.
         const content = message.content;
         const text = typeof content === 'string' ? content : JSON.stringify(content);
-        return { content: [{ type: 'text', text }] };
+        return toolError(ErrorCode.UPSTREAM_REFUSED, `Model returned no image. Text response: ${text.slice(0, 300)}`, {
+            reason: 'no_image_in_response',
+            finish_reason: completion.choices[0]?.finish_reason,
+        });
     }
-    catch (error) {
-        const msg = error instanceof Error ? error.message : String(error);
-        return { content: [{ type: 'text', text: `Error: ${msg}` }], isError: true };
+    if (safePathResolved) {
+        try {
+            await fs.writeFile(safePathResolved, Buffer.from(base64.data, 'base64'));
+        }
+        catch (err) {
+            return toolErrorFrom(ErrorCode.INTERNAL, err, 'Write');
+        }
+        const usage = completion.usage;
+        return {
+            content: [
+                { type: 'text', text: `Image saved to: ${safePathResolved}` },
+                { type: 'image', mimeType: base64.mime, data: base64.data },
+            ],
+            _meta: {
+                save_path: safePathResolved,
+                mime: base64.mime,
+                ...(usage
+                    ? {
+                        usage: {
+                            prompt_tokens: usage.prompt_tokens,
+                            completion_tokens: usage.completion_tokens,
+                            total_tokens: usage.total_tokens,
+                        },
+                    }
+                    : {}),
+            },
+        };
     }
+    const usage = completion.usage;
+    return {
+        content: [{ type: 'image', mimeType: base64.mime, data: base64.data }],
+        _meta: {
+            mime: base64.mime,
+            ...(usage
+                ? {
+                    usage: {
+                        prompt_tokens: usage.prompt_tokens,
+                        completion_tokens: usage.completion_tokens,
+                        total_tokens: usage.total_tokens,
+                    },
+                }
+                : {}),
+        },
+    };
 }
 function extractBase64(message) {
     const images = message.images;

package/dist/tool-handlers/generate-video.d.ts

@@ -0,0 +1,78 @@
+import type { OpenRouterAPIClient, VideoJobStatus } from '../openrouter-api.js';
+export interface GenerateVideoToolRequest {
+    prompt: string;
+    model?: string;
+    resolution?: string;
+    aspect_ratio?: string;
+    duration?: number;
+    seed?: number;
+    first_frame_image?: string;
+    last_frame_image?: string;
+    reference_images?: string[];
+    provider?: Record<string, unknown>;
+    save_path?: string;
+    max_wait_ms?: number;
+    poll_interval_ms?: number;
+}
+export interface GetVideoStatusToolRequest {
+    video_id: string;
+    save_path?: string;
+    polling_url?: string;
+}
+type ProgressHook = (update: {
+    status: string;
+    progress?: number;
+    attempt: number;
+    video_id: string;
+}) => void | Promise<void>;
+declare function buildRequestBody(args: GenerateVideoToolRequest, model: string): Record<string, unknown>;
+declare function extractJobError(status: VideoJobStatus): string;
+declare function stripAndReplaceExt(p: string, newExt: string): string;
+export declare function handleGenerateVideo(request: {
+    params: {
+        arguments: GenerateVideoToolRequest;
+    };
+}, apiClient: OpenRouterAPIClient, progress?: ProgressHook): Promise<import("../errors.js").ToolErrorResult | {
+    content: {
+        type: "text";
+        text: string;
+    }[];
+    isError: false;
+    _meta: {
+        code: "JOB_STILL_RUNNING";
+        video_id: string;
+        polling_url: string;
+        last_status: string | undefined;
+    };
+} | {
+    content: Record<string, unknown>[];
+    _meta: Record<string, unknown>;
+    isError?: undefined;
+}>;
+export declare function handleGetVideoStatus(request: {
+    params: {
+        arguments: GetVideoStatusToolRequest;
+    };
+}, apiClient: OpenRouterAPIClient): Promise<import("../errors.js").ToolErrorResult | {
+    content: Record<string, unknown>[];
+    _meta: Record<string, unknown>;
+    isError?: undefined;
+} | {
+    content: {
+        type: "text";
+        text: string;
+    }[];
+    isError: false;
+    _meta: {
+        code: "JOB_STILL_RUNNING";
+        video_id: string;
+        last_status: string;
+        progress: number | undefined;
+    };
+}>;
+export declare const _internals: {
+    buildRequestBody: typeof buildRequestBody;
+    stripAndReplaceExt: typeof stripAndReplaceExt;
+    extractJobError: typeof extractJobError;
+};
+export {};

package/dist/tool-handlers/generate-video.js

@@ -0,0 +1,353 @@
+import { promises as fs } from 'node:fs';
+import { extname } from 'node:path';
+import { ErrorCode, toolError, toolErrorFrom } from '../errors.js';
+import { logger } from '../logger.js';
+import { resolveSafeOutputPath, UnsafeOutputPathError, } from './path-safety.js';
+import { readEnvInt } from './fetch-utils.js';
+import { classifyUpstreamError } from './openrouter-errors.js';
+const FALLBACK_MODEL = 'google/veo-3.1';
+const DEFAULT_POLL_INTERVAL_MS = 15_000;
+const DEFAULT_MAX_WAIT_MS = 10 * 60_000;
+const MIN_POLL_INTERVAL_MS = 50; // just to avoid a 0ms busy-loop if a caller omits
+const INLINE_RETURN_CEILING_BYTES = 10 * 1024 * 1024;
+function getMaxInlineBytes() {
+    return readEnvInt('OPENROUTER_VIDEO_INLINE_MAX_BYTES', INLINE_RETURN_CEILING_BYTES, 4096);
+}
+function getDefaultPollInterval() {
+    return readEnvInt('OPENROUTER_VIDEO_POLL_INTERVAL_MS', DEFAULT_POLL_INTERVAL_MS, MIN_POLL_INTERVAL_MS);
+}
+function getDefaultMaxWait() {
+    return readEnvInt('OPENROUTER_VIDEO_MAX_WAIT_MS', DEFAULT_MAX_WAIT_MS, 10_000);
+}
+function getMaxDownloadBytes() {
+    // Generation output can be bigger than the input cap since it's our own
+    // content. Default 256 MB, override via env.
+    return readEnvInt('OPENROUTER_VIDEO_GEN_MAX_BYTES', 256 * 1024 * 1024, 1024 * 1024);
+}
+/**
+ * Fold a caller-supplied image source (local path, http URL, or data URL)
+ * into the `{ url: "data:video|image/...base64,..." }` shape OpenRouter
+ * expects inside `frame_images[].image` / `input_references[]`.
+ *
+ * We reuse `prepareVideoData` for videos but images live in `image-utils`.
+ * Since generate_video's references are images, not videos, we do a small
+ * image-specific fetch here (data URL pass-through, HTTP via fetch-utils,
+ * local via fs). We deliberately do NOT run them through sharp — the model
+ * wants the pristine frame.
+ */
+async function prepareImageInput(source) {
+    if (!source)
+        return null;
+    if (source.startsWith('data:')) {
+        const match = source.match(/^data:([^;,]+)(?:;[^,]*)*;base64,(.+)$/);
+        if (!match)
+            throw new Error(`Invalid image data URL: ${source.slice(0, 40)}…`);
+        return { mime: match[1], data: match[2] };
+    }
+    if (source.startsWith('http://') || source.startsWith('https://')) {
+        const { fetchHttpResource } = await import('./fetch-utils.js');
+        const { buffer, contentType } = await fetchHttpResource(source, {
+            timeoutMs: 30_000,
+            maxBytes: 25 * 1024 * 1024,
+            maxRedirects: 8,
+        });
+        const mime = (contentType?.split(';')[0]?.trim() || 'image/jpeg').toLowerCase();
+        return { mime, data: buffer.toString('base64') };
+    }
+    const buf = await fs.readFile(source);
+    const ext = extname(source).toLowerCase();
+    const mime = ext === '.png'
+        ? 'image/png'
+        : ext === '.webp'
+            ? 'image/webp'
+            : ext === '.gif'
+                ? 'image/gif'
+                : 'image/jpeg';
+    return { mime, data: buf.toString('base64') };
+}
+function buildRequestBody(args, model) {
+    const body = { model, prompt: args.prompt };
+    if (args.resolution)
+        body.resolution = args.resolution;
+    if (args.aspect_ratio)
+        body.aspect_ratio = args.aspect_ratio;
+    if (typeof args.duration === 'number')
+        body.duration = args.duration;
+    if (typeof args.seed === 'number')
+        body.seed = args.seed;
+    if (args.provider && typeof args.provider === 'object')
+        body.provider = args.provider;
+    return body;
+}
+async function attachFrameImages(args, body) {
+    const frameImages = [];
+    if (args.first_frame_image) {
+        const img = await prepareImageInput(args.first_frame_image);
+        if (img) {
+            frameImages.push({
+                frame_type: 'first_frame',
+                image: { url: `data:${img.mime};base64,${img.data}` },
+            });
+        }
+    }
+    if (args.last_frame_image) {
+        const img = await prepareImageInput(args.last_frame_image);
+        if (img) {
+            frameImages.push({
+                frame_type: 'last_frame',
+                image: { url: `data:${img.mime};base64,${img.data}` },
+            });
+        }
+    }
+    if (frameImages.length)
+        body.frame_images = frameImages;
+    if (args.reference_images?.length) {
+        const refs = [];
+        for (const src of args.reference_images) {
+            const img = await prepareImageInput(src);
+            if (img)
+                refs.push({ image: { url: `data:${img.mime};base64,${img.data}` } });
+        }
+        if (refs.length)
+            body.input_references = refs;
+    }
+}
+async function pollUntilTerminal(apiClient, envelope, opts) {
+    let attempt = 0;
+    let last = null;
+    const initialStatus = (envelope.status ?? 'pending');
+    await opts.onProgress?.({ status: initialStatus, attempt: 0, video_id: envelope.id });
+    while (Date.now() < opts.deadlineAt) {
+        attempt += 1;
+        await sleep(Math.min(opts.pollIntervalMs, Math.max(0, opts.deadlineAt - Date.now())));
+        try {
+            last = await apiClient.pollVideoJob(envelope.id);
+        }
+        catch (err) {
+            logger.warn('generate_video.poll_error', {
+                id: envelope.id,
+                err: err instanceof Error ? err.message : String(err),
+            });
+            continue; // transient; try again until deadline
+        }
+        await opts.onProgress?.({
+            status: last.status,
+            progress: typeof last.progress === 'number' ? last.progress : undefined,
+            attempt,
+            video_id: envelope.id,
+        });
+        if (last.status === 'completed')
+            return { kind: 'completed', status: last };
+        if (last.status === 'failed')
+            return { kind: 'failed', status: last };
+    }
+    return { kind: 'timeout', last };
+}
+function sleep(ms) {
+    if (ms <= 0)
+        return Promise.resolve();
+    return new Promise((r) => setTimeout(r, ms));
+}
+function extractJobError(status) {
+    if (!status.error)
+        return 'Upstream marked the job failed.';
+    if (typeof status.error === 'string')
+        return status.error;
+    return status.error.message ?? 'Upstream marked the job failed.';
+}
+async function finalizeCompletedJob(apiClient, status, savePath) {
+    const url = status.unsigned_urls?.[0];
+    if (!url) {
+        throw new Error('Completed job returned no content URLs.');
+    }
+    const { buffer, contentType } = await apiClient.downloadVideoContent(status.id, 0, getMaxDownloadBytes());
+    const mime = (contentType?.split(';')[0]?.trim() || 'video/mp4').toLowerCase();
+    const ext = mime.includes('webm')
+        ? '.webm'
+        : mime.includes('mov')
+            ? '.mov'
+            : mime.includes('mpeg')
+                ? '.mpeg'
+                : '.mp4';
+    const baseMeta = {
+        video_id: status.id,
+        mime,
+        size_bytes: buffer.length,
+    };
+    if (status.usage)
+        baseMeta.usage = status.usage;
+    if (status.unsigned_urls)
+        baseMeta.unsigned_urls = status.unsigned_urls;
+    if (savePath) {
+        const finalPath = extname(savePath) === ext ? savePath : stripAndReplaceExt(savePath, ext);
+        await fs.writeFile(finalPath, buffer);
+        baseMeta.save_path = finalPath;
+        const summaryNote = finalPath !== savePath ? ` (detected ${mime}, saved as ${finalPath})` : '';
+        const content = [
+            { type: 'text', text: `Video saved to: ${finalPath}${summaryNote}` },
+        ];
+        if (buffer.length <= getMaxInlineBytes()) {
+            content.push({
+                type: 'video',
+                mimeType: mime,
+                data: buffer.toString('base64'),
+            });
+        }
+        return { content, _meta: baseMeta };
+    }
+    // No save_path — return inline if small enough, otherwise just the URL.
+    if (buffer.length <= getMaxInlineBytes()) {
+        return {
+            content: [
+                { type: 'text', text: `Video generated (${buffer.length} bytes, ${mime}).` },
+                { type: 'video', mimeType: mime, data: buffer.toString('base64') },
+            ],
+            _meta: baseMeta,
+        };
+    }
+    return {
+        content: [
+            {
+                type: 'text',
+                text: `Video generated (${buffer.length} bytes, ${mime}). Too large to inline; pass save_path to persist. URL: ${url}`,
+            },
+        ],
+        _meta: baseMeta,
+    };
+}
+function stripAndReplaceExt(p, newExt) {
+    const cur = extname(p);
+    const base = cur ? p.slice(0, -cur.length) : p;
+    return base + newExt;
+}
+export async function handleGenerateVideo(request, apiClient, progress) {
+    const args = request.params.arguments ?? {};
+    if (!args.prompt || !args.prompt.trim()) {
+        return toolError(ErrorCode.INVALID_INPUT, 'prompt is required.');
+    }
+    // Fail-fast on unsafe save_path BEFORE spending credits on the job.
+    let safeSavePath = null;
+    if (args.save_path) {
+        try {
+            safeSavePath = await resolveSafeOutputPath(args.save_path);
+        }
+        catch (err) {
+            if (err instanceof UnsafeOutputPathError)
+                return toolErrorFrom(ErrorCode.UNSAFE_PATH, err);
+            return toolErrorFrom(ErrorCode.INTERNAL, err);
+        }
+    }
+    const model = args.model ||
+        process.env.OPENROUTER_DEFAULT_VIDEO_GEN_MODEL ||
+        FALLBACK_MODEL;
+    const body = buildRequestBody(args, model);
+    try {
+        await attachFrameImages(args, body);
+    }
+    catch (err) {
+        return toolErrorFrom(ErrorCode.UNSUPPORTED_FORMAT, err, 'Reference/frame image');
+    }
+    let envelope;
+    try {
+        logger.info('generate_video.submit', { model, keys: Object.keys(body) });
+        envelope = await apiClient.submitVideoJob(body);
+    }
+    catch (err) {
+        return classifyUpstreamError(err, 'generate_video.submit');
+    }
+    const pollIntervalMs = Math.max(MIN_POLL_INTERVAL_MS, args.poll_interval_ms ?? getDefaultPollInterval());
+    const maxWaitMs = Math.max(100, args.max_wait_ms ?? getDefaultMaxWait());
+    const deadlineAt = Date.now() + maxWaitMs;
+    const outcome = await pollUntilTerminal(apiClient, envelope, {
+        pollIntervalMs,
+        deadlineAt,
+        onProgress: progress,
+    });
+    if (outcome.kind === 'failed') {
+        return toolError(ErrorCode.JOB_FAILED, extractJobError(outcome.status), {
+            video_id: outcome.status.id,
+        });
+    }
+    if (outcome.kind === 'timeout') {
+        return {
+            content: [
+                {
+                    type: 'text',
+                    text: `Video still generating after ${maxWaitMs}ms. Use get_video_status with video_id=${envelope.id} to resume.`,
+                },
+            ],
+            isError: false,
+            _meta: {
+                code: ErrorCode.JOB_STILL_RUNNING,
+                video_id: envelope.id,
+                polling_url: envelope.polling_url ?? `https://openrouter.ai/api/v1/videos/${envelope.id}`,
+                last_status: outcome.last?.status,
+            },
+        };
+    }
+    try {
+        const { content, _meta } = await finalizeCompletedJob(apiClient, outcome.status, safeSavePath);
+        return { content, _meta };
+    }
+    catch (err) {
+        if (err instanceof UnsafeOutputPathError) {
+            return toolErrorFrom(ErrorCode.UNSAFE_PATH, err);
+        }
+        return toolErrorFrom(ErrorCode.UPSTREAM_HTTP, err, 'Download');
+    }
+}
+export async function handleGetVideoStatus(request, apiClient) {
+    const args = request.params.arguments ?? {};
+    const id = args.video_id?.trim();
+    if (!id)
+        return toolError(ErrorCode.INVALID_INPUT, 'video_id is required.');
+    // Pre-resolve save_path so the poll surfaces a fast error before hitting OpenRouter.
+    let safeSavePath = null;
+    if (args.save_path) {
+        try {
+            safeSavePath = await resolveSafeOutputPath(args.save_path);
+        }
+        catch (err) {
+            if (err instanceof UnsafeOutputPathError)
+                return toolErrorFrom(ErrorCode.UNSAFE_PATH, err);
+            return toolErrorFrom(ErrorCode.INTERNAL, err);
+        }
+    }
+    let status;
+    try {
+        status = await apiClient.pollVideoJob(id);
+    }
+    catch (err) {
+        return classifyUpstreamError(err, 'get_video_status.poll');
+    }
+    if (status.status === 'failed') {
+        return toolError(ErrorCode.JOB_FAILED, extractJobError(status), { video_id: id });
+    }
+    if (status.status === 'completed') {
+        try {
+            const { content, _meta } = await finalizeCompletedJob(apiClient, status, safeSavePath);
+            return { content, _meta };
+        }
+        catch (err) {
+            if (err instanceof UnsafeOutputPathError)
+                return toolErrorFrom(ErrorCode.UNSAFE_PATH, err);
+            return toolErrorFrom(ErrorCode.UPSTREAM_HTTP, err, 'Download');
+        }
+    }
+    return {
+        content: [
+            {
+                type: 'text',
+                text: `Video ${id} status: ${status.status}${typeof status.progress === 'number' ? ` (progress=${status.progress})` : ''}`,
+            },
+        ],
+        isError: false,
+        _meta: {
+            code: ErrorCode.JOB_STILL_RUNNING,
+            video_id: id,
+            last_status: status.status,
+            progress: status.progress,
+        },
+    };
+}
+export const _internals = { buildRequestBody, stripAndReplaceExt, extractJobError };

package/dist/tool-handlers/get-model-info.js

@@ -1,7 +1,13 @@
 export async function handleGetModelInfo(request, modelCache, apiClient) {
     const { model } = request.params.arguments;
-    if (!modelCache.isValid() && apiClient) {
-        modelCache.setModels(await apiClient.getModels());
+    if (apiClient) {
+        try {
+            await modelCache.ensureFresh(() => apiClient.getModels());
+        }
+        catch (error) {
+            const msg = error instanceof Error ? error.message : String(error);
+            return { content: [{ type: 'text', text: `Error: ${msg}` }], isError: true };
+        }
     }
     if (!modelCache.isValid()) {
         return { content: [{ type: 'text', text: 'No model data available.' }], isError: true };

package/dist/tool-handlers/image-utils.d.ts

@@ -6,5 +6,21 @@ export declare function getImageJpegQuality(): number;
 export declare function getMimeType(filePath: string): string;
 export declare function fetchHttpImage(urlString: string): Promise<Buffer>;
 export declare function fetchImage(source: string): Promise<Buffer>;
-export declare function optimizeImage(buffer: Buffer): Promise<string>;
+/**
+ * Sniff image MIME type from magic bytes. Used to label the output of a
+ * failed `sharp` optimization (where we return original bytes but don't
+ * know the source MIME yet) and HTTP image responses whose Content-Type
+ * header is missing or wrong.
+ */
+export declare function sniffImageMime(buffer: Buffer): string | null;
+/**
+ * Optimize an image buffer and return both the base64 payload AND the MIME
+ * type that matches that payload. Callers should NOT assume JPEG — the
+ * pipeline falls back to the original bytes (with its detected MIME) when
+ * sharp is unavailable or fails. This closes BUG-012.
+ */
+export declare function optimizeImage(buffer: Buffer): Promise<{
+    base64: string;
+    mime: string;
+}>;
 export declare function prepareImageUrl(source: string): Promise<string>;