@elizaos/agent 2.0.0-beta.2 → 2.0.11-beta.6

package/providers/media-provider.js

@@ -0,0 +1,1511 @@
+/**
+ * Media Provider Abstraction Layer
+ *
+ * Provides a single interface for media generation across multiple providers:
+ * - Image Generation: FAL.ai, OpenAI (DALL-E), Google (Imagen), xAI, Eliza Cloud
+ * - Video Generation: FAL.ai, OpenAI (Sora), Google (Veo), Eliza Cloud
+ * - Audio Generation: Suno, ElevenLabs (SFX), Eliza Cloud
+ * - Vision (Analysis): OpenAI, Google, Anthropic, xAI, Eliza Cloud
+ *
+ * Follows the same pattern as TTS provider selection:
+ * - "cloud" mode uses Eliza Cloud (no API key needed)
+ * - "own-key" mode uses the user's own API keys
+ */
+import { logger } from "@elizaos/core";
+// ============================================================================
+// Fetch Utilities
+// ============================================================================
+/** Fetch with an AbortController-based timeout (default 30s). */
+export function fetchWithTimeout(url, init, timeoutMs = 30_000) {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    return fetch(url, { ...init, signal: controller.signal }).finally(() => clearTimeout(timer));
+}
+async function withProviderErrorBoundary(providerName, run) {
+    try {
+        return await run();
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        return {
+            success: false,
+            error: `[${providerName}] Network error: ${message}`,
+        };
+    }
+}
+const DEFAULT_ELIZA_CLOUD_BASE_URL = "https://elizacloud.ai/api/v1";
+const DEFAULT_AUDIO_TIMEOUT_MS = 120_000;
+const VEO_OPERATION_POLL_INTERVAL_MS = 10_000;
+const VEO_OPERATION_TIMEOUT_MS = 300_000;
+function delay(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+function extractVeoVideoUri(operation) {
+    return (operation.response?.predictions?.[0]?.videoUri ??
+        operation.response?.generateVideoResponse?.generatedSamples?.[0]?.video?.uri);
+}
+function normalizeAudioKind(value) {
+    switch (value) {
+        case "music":
+            return "music";
+        case "sfx":
+        case "sound-effect":
+        case "sound_effect":
+            return "sfx";
+        case "tts":
+        case "speech":
+        case "text-to-speech":
+        case "text_to_speech":
+            return "tts";
+        default:
+            return undefined;
+    }
+}
+function getAudioKind(options, config) {
+    return (normalizeAudioKind(options.audioKind) ??
+        normalizeAudioKind(options.kind) ??
+        normalizeAudioKind(config?.audioKind) ??
+        normalizeAudioKind(config?.kind) ??
+        normalizeAudioKind(config?.defaultKind) ??
+        "music");
+}
+function createCloudAudioProvider(options) {
+    return new ElizaCloudAudioProvider(options.elizaCloudBaseUrl ?? DEFAULT_ELIZA_CLOUD_BASE_URL, options.elizaCloudApiKey);
+}
+async function responseToAudioResult(response, title, duration) {
+    const contentType = response.headers.get("content-type")?.split(";")[0] || "audio/mpeg";
+    const arrayBuffer = await response.arrayBuffer();
+    const audioBase64 = Buffer.from(arrayBuffer).toString("base64");
+    return {
+        audioUrl: `data:${contentType};base64,${audioBase64}`,
+        audioBase64,
+        mimeType: contentType,
+        id: response.headers.get("song-id") ?? undefined,
+        title,
+        duration,
+    };
+}
+function buildOutputFormatQuery(outputFormat) {
+    if (!outputFormat)
+        return "";
+    return `?output_format=${encodeURIComponent(outputFormat)}`;
+}
+// ============================================================================
+// Eliza Cloud Provider Implementations
+// ============================================================================
+class ElizaCloudImageProvider {
+    name = "eliza-cloud";
+    baseUrl;
+    apiKey;
+    constructor(baseUrl, apiKey) {
+        this.baseUrl = baseUrl;
+        this.apiKey = apiKey;
+    }
+    async generate(options) {
+        const response = await fetchWithTimeout(`${this.baseUrl}/media/image/generate`, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                ...(this.apiKey ? { Authorization: `Bearer ${this.apiKey}` } : {}),
+            },
+            body: JSON.stringify({
+                prompt: options.prompt,
+                size: options.size,
+                quality: options.quality,
+                style: options.style,
+            }),
+        });
+        if (!response.ok) {
+            const text = await response.text();
+            return { success: false, error: `Eliza Cloud error: ${text}` };
+        }
+        const data = (await response.json());
+        return {
+            success: true,
+            data: {
+                imageUrl: data.imageUrl,
+                imageBase64: data.imageBase64,
+                revisedPrompt: data.revisedPrompt,
+            },
+        };
+    }
+}
+class ElizaCloudVideoProvider {
+    name = "eliza-cloud";
+    baseUrl;
+    apiKey;
+    constructor(baseUrl, apiKey) {
+        this.baseUrl = baseUrl;
+        this.apiKey = apiKey;
+    }
+    async generate(options) {
+        const response = await fetchWithTimeout(`${this.baseUrl}/media/video/generate`, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                ...(this.apiKey ? { Authorization: `Bearer ${this.apiKey}` } : {}),
+            },
+            body: JSON.stringify({
+                prompt: options.prompt,
+                duration: options.duration,
+                aspectRatio: options.aspectRatio,
+                imageUrl: options.imageUrl,
+            }),
+        });
+        if (!response.ok) {
+            const text = await response.text();
+            return { success: false, error: `Eliza Cloud error: ${text}` };
+        }
+        const data = (await response.json());
+        return {
+            success: true,
+            data: {
+                videoUrl: data.videoUrl,
+                thumbnailUrl: data.thumbnailUrl,
+                duration: data.duration,
+            },
+        };
+    }
+}
+class ElizaCloudAudioProvider {
+    name = "eliza-cloud";
+    baseUrl;
+    apiKey;
+    constructor(baseUrl, apiKey) {
+        this.baseUrl = baseUrl;
+        this.apiKey = apiKey;
+    }
+    async generate(options) {
+        const response = await fetchWithTimeout(`${this.baseUrl}/media/audio/generate`, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                ...(this.apiKey ? { Authorization: `Bearer ${this.apiKey}` } : {}),
+            },
+            body: JSON.stringify({
+                prompt: options.prompt,
+                kind: options.kind,
+                audioKind: options.audioKind,
+                duration: options.duration,
+                instrumental: options.instrumental,
+                genre: options.genre,
+                voiceId: options.voiceId,
+                modelId: options.modelId,
+                outputFormat: options.outputFormat,
+            }),
+        });
+        if (!response.ok) {
+            const text = await response.text();
+            return { success: false, error: `Eliza Cloud error: ${text}` };
+        }
+        const data = (await response.json());
+        return {
+            success: true,
+            data: {
+                audioUrl: data.audioUrl,
+                title: data.title,
+                duration: data.duration,
+            },
+        };
+    }
+}
+class ElizaCloudVisionProvider {
+    name = "eliza-cloud";
+    baseUrl;
+    apiKey;
+    constructor(baseUrl, apiKey) {
+        this.baseUrl = baseUrl;
+        this.apiKey = apiKey;
+    }
+    async analyze(options) {
+        const response = await fetchWithTimeout(`${this.baseUrl}/media/vision/analyze`, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                ...(this.apiKey ? { Authorization: `Bearer ${this.apiKey}` } : {}),
+            },
+            body: JSON.stringify({
+                imageUrl: options.imageUrl,
+                imageBase64: options.imageBase64,
+                prompt: options.prompt,
+                maxTokens: options.maxTokens,
+            }),
+        });
+        if (!response.ok) {
+            const text = await response.text();
+            return { success: false, error: `Eliza Cloud error: ${text}` };
+        }
+        const data = (await response.json());
+        return {
+            success: true,
+            data: {
+                description: data.description,
+                labels: data.labels,
+                confidence: data.confidence,
+            },
+        };
+    }
+}
+// ============================================================================
+// FAL.ai Provider Implementations
+// ============================================================================
+export class FalImageProvider {
+    name = "fal";
+    apiKey;
+    model;
+    baseUrl;
+    constructor(config) {
+        if (!config.apiKey) {
+            throw new Error(`${this.name} API key is required`);
+        }
+        this.apiKey = config.apiKey;
+        this.model = config.model ?? "fal-ai/flux-pro";
+        this.baseUrl = config.baseUrl ?? "https://fal.run";
+    }
+    async generate(options) {
+        return withProviderErrorBoundary(this.name, async () => {
+            const response = await fetchWithTimeout(`${this.baseUrl}/${this.model}`, {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    Authorization: `Key ${this.apiKey}`,
+                },
+                body: JSON.stringify({
+                    prompt: options.prompt,
+                    image_size: options.size ?? "landscape_4_3",
+                    num_images: 1,
+                    ...(options.negativePrompt
+                        ? { negative_prompt: options.negativePrompt }
+                        : {}),
+                    ...(options.seed ? { seed: options.seed } : {}),
+                }),
+            });
+            if (!response.ok) {
+                const text = await response.text();
+                return { success: false, error: `FAL error: ${text}` };
+            }
+            const data = (await response.json());
+            const imageUrl = data.images?.[0]?.url;
+            if (!imageUrl) {
+                return { success: false, error: "No image returned from FAL" };
+            }
+            return {
+                success: true,
+                data: { imageUrl },
+            };
+        });
+    }
+}
+export class FalVideoProvider {
+    name = "fal";
+    apiKey;
+    model;
+    baseUrl;
+    constructor(config) {
+        if (!config.apiKey) {
+            throw new Error(`${this.name} API key is required`);
+        }
+        this.apiKey = config.apiKey;
+        this.model = config.model ?? "fal-ai/minimax-video";
+        this.baseUrl = config.baseUrl ?? "https://fal.run";
+    }
+    async generate(options) {
+        return withProviderErrorBoundary(this.name, async () => {
+            const response = await fetchWithTimeout(`${this.baseUrl}/${this.model}`, {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    Authorization: `Key ${this.apiKey}`,
+                },
+                body: JSON.stringify({
+                    prompt: options.prompt,
+                    ...(options.duration ? { duration: options.duration } : {}),
+                    ...(options.aspectRatio ? { aspect_ratio: options.aspectRatio } : {}),
+                    ...(options.imageUrl ? { image_url: options.imageUrl } : {}),
+                }),
+            });
+            if (!response.ok) {
+                const text = await response.text();
+                return { success: false, error: `FAL error: ${text}` };
+            }
+            const data = (await response.json());
+            const videoUrl = data.video?.url;
+            if (!videoUrl) {
+                return { success: false, error: "No video returned from FAL" };
+            }
+            return {
+                success: true,
+                data: {
+                    videoUrl,
+                    thumbnailUrl: data.thumbnail?.url,
+                    duration: data.duration,
+                },
+            };
+        });
+    }
+}
+// ============================================================================
+// OpenAI Provider Implementations
+// ============================================================================
+export class OpenAIImageProvider {
+    name = "openai";
+    apiKey;
+    model;
+    quality;
+    style;
+    constructor(config) {
+        if (!config.apiKey) {
+            throw new Error(`${this.name} API key is required`);
+        }
+        this.apiKey = config.apiKey;
+        this.model = config.model ?? "dall-e-3";
+        this.quality = config.quality ?? "standard";
+        this.style = config.style ?? "vivid";
+    }
+    async generate(options) {
+        return withProviderErrorBoundary(this.name, async () => {
+            const response = await fetchWithTimeout("https://api.openai.com/v1/images/generations", {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    Authorization: `Bearer ${this.apiKey}`,
+                },
+                body: JSON.stringify({
+                    model: this.model,
+                    prompt: options.prompt,
+                    n: 1,
+                    size: options.size ?? "1024x1024",
+                    quality: options.quality ?? this.quality,
+                    style: options.style ?? this.style,
+                }),
+            });
+            if (!response.ok) {
+                const text = await response.text();
+                return { success: false, error: `OpenAI error: ${text}` };
+            }
+            const data = (await response.json());
+            const image = data.data?.[0];
+            if (!image?.url) {
+                return { success: false, error: "No image returned from OpenAI" };
+            }
+            return {
+                success: true,
+                data: {
+                    imageUrl: image.url,
+                    revisedPrompt: image.revised_prompt,
+                },
+            };
+        });
+    }
+}
+export class OpenAIVideoProvider {
+    name = "openai";
+    apiKey;
+    model;
+    constructor(config) {
+        if (!config.apiKey) {
+            throw new Error(`${this.name} API key is required`);
+        }
+        this.apiKey = config.apiKey;
+        this.model = config.model ?? "sora-1.0-turbo";
+    }
+    async generate(options) {
+        return withProviderErrorBoundary(this.name, async () => {
+            // OpenAI Sora API (video generation)
+            const response = await fetchWithTimeout("https://api.openai.com/v1/videos/generations", {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    Authorization: `Bearer ${this.apiKey}`,
+                },
+                body: JSON.stringify({
+                    model: this.model,
+                    prompt: options.prompt,
+                    n: 1,
+                    duration: options.duration ?? 5,
+                    aspect_ratio: options.aspectRatio ?? "16:9",
+                    ...(options.imageUrl ? { image: options.imageUrl } : {}),
+                }),
+            });
+            if (!response.ok) {
+                const text = await response.text();
+                return { success: false, error: `OpenAI Sora error: ${text}` };
+            }
+            const data = (await response.json());
+            const video = data.data?.[0];
+            if (!video?.url) {
+                return { success: false, error: "No video returned from OpenAI Sora" };
+            }
+            return {
+                success: true,
+                data: {
+                    videoUrl: video.url,
+                    duration: video.duration,
+                },
+            };
+        });
+    }
+}
+export class OpenAIVisionProvider {
+    name = "openai";
+    apiKey;
+    model;
+    maxTokens;
+    constructor(config) {
+        if (!config.apiKey) {
+            throw new Error(`${this.name} API key is required`);
+        }
+        this.apiKey = config.apiKey;
+        this.model = config.model ?? "gpt-4o";
+        this.maxTokens = config.maxTokens ?? 1024;
+    }
+    async analyze(options) {
+        const imageContent = options.imageBase64
+            ? {
+                type: "image_url",
+                image_url: { url: `data:image/jpeg;base64,${options.imageBase64}` },
+            }
+            : {
+                type: "image_url",
+                image_url: { url: options.imageUrl ?? "" },
+            };
+        return withProviderErrorBoundary(this.name, async () => {
+            const response = await fetchWithTimeout("https://api.openai.com/v1/chat/completions", {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    Authorization: `Bearer ${this.apiKey}`,
+                },
+                body: JSON.stringify({
+                    model: this.model,
+                    max_tokens: options.maxTokens ?? this.maxTokens,
+                    messages: [
+                        {
+                            role: "user",
+                            content: [
+                                {
+                                    type: "text",
+                                    text: options.prompt ?? "Describe this image in detail.",
+                                },
+                                imageContent,
+                            ],
+                        },
+                    ],
+                }),
+            });
+            if (!response.ok) {
+                const text = await response.text();
+                return { success: false, error: `OpenAI error: ${text}` };
+            }
+            const data = (await response.json());
+            const description = data.choices?.[0]?.message?.content;
+            if (!description) {
+                return {
+                    success: false,
+                    error: "No description returned from OpenAI",
+                };
+            }
+            return {
+                success: true,
+                data: { description },
+            };
+        });
+    }
+}
+// ============================================================================
+// Google Provider Implementations
+// ============================================================================
+export class GoogleImageProvider {
+    name = "google";
+    apiKey;
+    model;
+    aspectRatio;
+    constructor(config) {
+        if (!config.apiKey) {
+            throw new Error(`${this.name} API key is required`);
+        }
+        this.apiKey = config.apiKey;
+        this.model = config.model ?? "imagen-3.0-generate-002";
+        this.aspectRatio = config.aspectRatio ?? "1:1";
+    }
+    async generate(options) {
+        return withProviderErrorBoundary(this.name, async () => {
+            const response = await fetchWithTimeout(`https://generativelanguage.googleapis.com/v1beta/models/${this.model}:predict`, {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    "x-goog-api-key": this.apiKey,
+                },
+                body: JSON.stringify({
+                    instances: [{ prompt: options.prompt }],
+                    parameters: {
+                        sampleCount: 1,
+                        aspectRatio: options.size ?? this.aspectRatio,
+                        personGeneration: "allow_adult",
+                        safetyFilterLevel: "block_few",
+                    },
+                }),
+            });
+            if (!response.ok) {
+                const text = await response.text();
+                return { success: false, error: `Google Imagen error: ${text}` };
+            }
+            const data = (await response.json());
+            const imageData = data.predictions?.[0]?.bytesBase64Encoded;
+            if (!imageData) {
+                return {
+                    success: false,
+                    error: "No image returned from Google Imagen",
+                };
+            }
+            return {
+                success: true,
+                data: {
+                    imageBase64: imageData,
+                },
+            };
+        });
+    }
+}
+export class GoogleVideoProvider {
+    name = "google";
+    apiKey;
+    model;
+    constructor(config) {
+        if (!config.apiKey) {
+            throw new Error(`${this.name} API key is required`);
+        }
+        this.apiKey = config.apiKey;
+        this.model = config.model ?? "veo-2.0-generate-001";
+    }
+    async generate(options) {
+        return withProviderErrorBoundary(this.name, async () => {
+            // Google Veo uses a different endpoint structure
+            const response = await fetchWithTimeout(`https://generativelanguage.googleapis.com/v1beta/models/${this.model}:predictLongRunning`, {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    "x-goog-api-key": this.apiKey,
+                },
+                body: JSON.stringify({
+                    instances: [
+                        {
+                            prompt: options.prompt,
+                            ...(options.imageUrl
+                                ? { image: { gcsUri: options.imageUrl } }
+                                : {}),
+                        },
+                    ],
+                    parameters: {
+                        aspectRatio: options.aspectRatio ?? "16:9",
+                        durationSeconds: options.duration ?? 5,
+                        personGeneration: "allow_adult",
+                    },
+                }),
+            });
+            if (!response.ok) {
+                const text = await response.text();
+                return { success: false, error: `Google Veo error: ${text}` };
+            }
+            // Veo returns a long-running operation; poll it to completion.
+            const started = (await response.json());
+            if (!started.name) {
+                return {
+                    success: false,
+                    error: "Google Veo error: operation response missing name",
+                };
+            }
+            const completed = await this.pollOperation(started);
+            const videoUri = extractVeoVideoUri(completed);
+            if (!videoUri) {
+                return {
+                    success: false,
+                    error: "Google Veo error: operation completed without a video URI in the response",
+                };
+            }
+            return { success: true, data: { videoUrl: videoUri } };
+        });
+    }
+    async pollOperation(started) {
+        if (started.done)
+            return started;
+        const deadline = Date.now() + VEO_OPERATION_TIMEOUT_MS;
+        let operation = started;
+        while (!operation.done) {
+            if (Date.now() >= deadline) {
+                throw new Error(`Google Veo error: video generation did not complete within ${Math.round(VEO_OPERATION_TIMEOUT_MS / 1000)}s`);
+            }
+            await delay(VEO_OPERATION_POLL_INTERVAL_MS);
+            const pollResponse = await fetchWithTimeout(`https://generativelanguage.googleapis.com/v1beta/${operation.name}`, {
+                method: "GET",
+                headers: { "x-goog-api-key": this.apiKey },
+            });
+            if (!pollResponse.ok) {
+                const text = await pollResponse.text();
+                throw new Error(`Google Veo error: ${text}`);
+            }
+            operation = (await pollResponse.json());
+            if (operation.error) {
+                throw new Error(`Google Veo error: ${operation.error.message ?? "operation failed"}`);
+            }
+        }
+        return operation;
+    }
+}
+export class GoogleVisionProvider {
+    name = "google";
+    apiKey;
+    model;
+    constructor(config) {
+        if (!config.apiKey) {
+            throw new Error(`${this.name} API key is required`);
+        }
+        this.apiKey = config.apiKey;
+        this.model = config.model ?? "gemini-2.0-flash";
+    }
+    async analyze(options) {
+        const imagePart = options.imageBase64
+            ? { inline_data: { mime_type: "image/jpeg", data: options.imageBase64 } }
+            : { file_data: { file_uri: options.imageUrl, mime_type: "image/jpeg" } };
+        return withProviderErrorBoundary(this.name, async () => {
+            const response = await fetchWithTimeout(`https://generativelanguage.googleapis.com/v1beta/models/${this.model}:generateContent`, {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    "x-goog-api-key": this.apiKey,
+                },
+                body: JSON.stringify({
+                    contents: [
+                        {
+                            parts: [
+                                { text: options.prompt ?? "Describe this image in detail." },
+                                imagePart,
+                            ],
+                        },
+                    ],
+                }),
+            });
+            if (!response.ok) {
+                const text = await response.text();
+                return { success: false, error: `Google error: ${text}` };
+            }
+            const data = (await response.json());
+            const description = data.candidates?.[0]?.content?.parts?.[0]?.text;
+            if (!description) {
+                return {
+                    success: false,
+                    error: "No description returned from Google",
+                };
+            }
+            return {
+                success: true,
+                data: { description },
+            };
+        });
+    }
+}
+// ============================================================================
+// xAI Provider Implementations
+// ============================================================================
+export class XAIImageProvider {
+    name = "xai";
+    apiKey;
+    model;
+    constructor(config) {
+        if (!config.apiKey) {
+            throw new Error(`${this.name} API key is required`);
+        }
+        this.apiKey = config.apiKey;
+        this.model = config.model ?? "grok-2-image";
+    }
+    async generate(options) {
+        return withProviderErrorBoundary(this.name, async () => {
+            // xAI uses OpenAI-compatible API format for image generation
+            const response = await fetchWithTimeout("https://api.x.ai/v1/images/generations", {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    Authorization: `Bearer ${this.apiKey}`,
+                },
+                body: JSON.stringify({
+                    model: this.model,
+                    prompt: options.prompt,
+                    n: 1,
+                    size: options.size ?? "1024x1024",
+                    response_format: "url",
+                }),
+            });
+            if (!response.ok) {
+                const text = await response.text();
+                return { success: false, error: `xAI error: ${text}` };
+            }
+            const data = (await response.json());
+            const image = data.data?.[0];
+            if (!image?.url) {
+                return { success: false, error: "No image returned from xAI" };
+            }
+            return {
+                success: true,
+                data: {
+                    imageUrl: image.url,
+                    revisedPrompt: image.revised_prompt,
+                },
+            };
+        });
+    }
+}
+export class XAIVisionProvider {
+    name = "xai";
+    apiKey;
+    model;
+    constructor(config) {
+        if (!config.apiKey) {
+            throw new Error(`${this.name} API key is required`);
+        }
+        this.apiKey = config.apiKey;
+        this.model = config.model ?? "grok-2-vision-1212";
+    }
+    async analyze(options) {
+        // xAI uses OpenAI-compatible API format
+        const imageContent = options.imageBase64
+            ? {
+                type: "image_url",
+                image_url: { url: `data:image/jpeg;base64,${options.imageBase64}` },
+            }
+            : {
+                type: "image_url",
+                image_url: { url: options.imageUrl ?? "" },
+            };
+        return withProviderErrorBoundary(this.name, async () => {
+            const response = await fetchWithTimeout("https://api.x.ai/v1/chat/completions", {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    Authorization: `Bearer ${this.apiKey}`,
+                },
+                body: JSON.stringify({
+                    model: this.model,
+                    max_tokens: options.maxTokens ?? 1024,
+                    messages: [
+                        {
+                            role: "user",
+                            content: [
+                                {
+                                    type: "text",
+                                    text: options.prompt ?? "Describe this image in detail.",
+                                },
+                                imageContent,
+                            ],
+                        },
+                    ],
+                }),
+            });
+            if (!response.ok) {
+                const text = await response.text();
+                return { success: false, error: `xAI error: ${text}` };
+            }
+            const data = (await response.json());
+            const description = data.choices?.[0]?.message?.content;
+            if (!description) {
+                return { success: false, error: "No description returned from xAI" };
+            }
+            return {
+                success: true,
+                data: { description },
+            };
+        });
+    }
+}
+// ============================================================================
+// Anthropic Provider Implementation
+// ============================================================================
+// ============================================================================
+// Ollama Provider Implementation (Local Vision)
+// ============================================================================
+class OllamaVisionProvider {
+    name = "ollama";
+    baseUrl;
+    model;
+    maxTokens;
+    autoDownload;
+    modelChecked = false;
+    constructor(config) {
+        this.baseUrl = config.baseUrl ?? "http://localhost:11434";
+        this.model = config.model ?? "llava";
+        this.maxTokens = config.maxTokens ?? 1024;
+        this.autoDownload = config.autoDownload ?? true;
+    }
+    async ensureModelAvailable() {
+        if (this.modelChecked)
+            return;
+        try {
+            // Check if model exists
+            const response = await fetchWithTimeout(`${this.baseUrl}/api/tags`, {}, 120_000);
+            if (!response.ok) {
+                throw new Error(`Ollama server not reachable: ${response.statusText}`);
+            }
+            const data = (await response.json());
+            const models = data.models ?? [];
+            const hasModel = models.some((m) => m.name === this.model || m.name.startsWith(`${this.model}:`));
+            if (!hasModel && this.autoDownload) {
+                logger.info(`[ollama-vision] Model ${this.model} not found, downloading...`);
+                await this.downloadModel();
+            }
+            else if (!hasModel) {
+                throw new Error(`Ollama model ${this.model} not found. Run 'ollama pull ${this.model}' or enable autoDownload.`);
+            }
+            this.modelChecked = true;
+        }
+        catch (err) {
+            if (err instanceof Error &&
+                err.message.includes("Ollama server not reachable")) {
+                throw err;
+            }
+            throw new Error(`Failed to check Ollama models: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    async downloadModel() {
+        const response = await fetchWithTimeout(`${this.baseUrl}/api/pull`, {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({ name: this.model, stream: false }),
+        }, 300_000);
+        if (!response.ok) {
+            const text = await response.text();
+            throw new Error(`Failed to download model ${this.model}: ${text}`);
+        }
+        // Wait for download to complete (non-streaming mode)
+        await response.json();
+        logger.info(`[ollama-vision] Model ${this.model} downloaded successfully`);
+    }
+    async analyze(options) {
+        try {
+            await this.ensureModelAvailable();
+        }
+        catch (err) {
+            return {
+                success: false,
+                error: `Ollama setup failed: ${err instanceof Error ? err.message : String(err)}`,
+            };
+        }
+        // Ollama uses a different format for vision - images must be base64
+        let imageData = options.imageBase64;
+        if (!imageData && options.imageUrl) {
+            // Fetch the image and convert to base64
+            try {
+                const imageResponse = await fetchWithTimeout(options.imageUrl, {}, 120_000);
+                if (!imageResponse.ok) {
+                    return {
+                        success: false,
+                        error: `Failed to fetch image: ${imageResponse.statusText}`,
+                    };
+                }
+                const buffer = await imageResponse.arrayBuffer();
+                imageData = Buffer.from(buffer).toString("base64");
+            }
+            catch (err) {
+                return {
+                    success: false,
+                    error: `Failed to fetch image: ${err instanceof Error ? err.message : String(err)}`,
+                };
+            }
+        }
+        if (!imageData) {
+            return {
+                success: false,
+                error: "No image provided (imageUrl or imageBase64 required)",
+            };
+        }
+        return withProviderErrorBoundary(this.name, async () => {
+            const response = await fetchWithTimeout(`${this.baseUrl}/api/chat`, {
+                method: "POST",
+                headers: { "Content-Type": "application/json" },
+                body: JSON.stringify({
+                    model: this.model,
+                    messages: [
+                        {
+                            role: "user",
+                            content: options.prompt ?? "Describe this image in detail.",
+                            images: [imageData],
+                        },
+                    ],
+                    stream: false,
+                    options: {
+                        num_predict: this.maxTokens,
+                    },
+                }),
+            }, 120_000);
+            if (!response.ok) {
+                const text = await response.text();
+                return { success: false, error: `Ollama error: ${text}` };
+            }
+            const data = (await response.json());
+            const description = data.message?.content;
+            if (!description) {
+                return { success: false, error: "No description returned from Ollama" };
+            }
+            return {
+                success: true,
+                data: { description },
+            };
+        });
+    }
+}
+// ============================================================================
+// Anthropic Provider Implementation
+// ============================================================================
+export class AnthropicVisionProvider {
+    name = "anthropic";
+    apiKey;
+    model;
+    constructor(config) {
+        if (!config.apiKey) {
+            throw new Error(`${this.name} API key is required`);
+        }
+        this.apiKey = config.apiKey;
+        this.model = config.model ?? "claude-opus-4-7";
+    }
+    async analyze(options) {
+        const imageSource = options.imageBase64
+            ? {
+                type: "base64",
+                media_type: "image/jpeg",
+                data: options.imageBase64,
+            }
+            : { type: "url", url: options.imageUrl ?? "" };
+        return withProviderErrorBoundary(this.name, async () => {
+            const response = await fetchWithTimeout("https://api.anthropic.com/v1/messages", {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    "x-api-key": this.apiKey,
+                    "anthropic-version": "2023-06-01",
+                },
+                body: JSON.stringify({
+                    model: this.model,
+                    max_tokens: options.maxTokens ?? 1024,
+                    messages: [
+                        {
+                            role: "user",
+                            content: [
+                                { type: "image", source: imageSource },
+                                {
+                                    type: "text",
+                                    text: options.prompt ?? "Describe this image in detail.",
+                                },
+                            ],
+                        },
+                    ],
+                }),
+            });
+            if (!response.ok) {
+                const text = await response.text();
+                return { success: false, error: `Anthropic error: ${text}` };
+            }
+            const data = (await response.json());
+            const textBlock = data.content?.find((c) => c.type === "text");
+            if (!textBlock?.text) {
+                return {
+                    success: false,
+                    error: "No description returned from Anthropic",
+                };
+            }
+            return {
+                success: true,
+                data: { description: textBlock.text },
+            };
+        });
+    }
+}
+// ============================================================================
+// FAL Audio Provider Implementation
+// ============================================================================
+export class FalAudioProvider {
+    name = "fal";
+    apiKey;
+    model;
+    baseUrl;
+    secondsStart;
+    secondsTotal;
+    steps;
+    timeoutMs;
+    extraInput;
+    constructor(config) {
+        if (!config.apiKey) {
+            throw new Error(`${this.name} API key is required`);
+        }
+        this.apiKey = config.apiKey;
+        this.model = config.model ?? "fal-ai/stable-audio";
+        this.baseUrl = config.baseUrl ?? "https://fal.run";
+        this.secondsStart = config.secondsStart;
+        this.secondsTotal = config.secondsTotal;
+        this.steps = config.steps;
+        this.timeoutMs = config.timeoutMs ?? DEFAULT_AUDIO_TIMEOUT_MS;
+        this.extraInput = config.extraInput;
+    }
+    async generate(options) {
+        return withProviderErrorBoundary(this.name, async () => {
+            const response = await fetchWithTimeout(`${this.baseUrl}/${this.model}`, {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    Authorization: `Key ${this.apiKey}`,
+                },
+                body: JSON.stringify({
+                    ...(this.extraInput ?? {}),
+                    prompt: options.prompt,
+                    ...(this.secondsStart !== undefined
+                        ? { seconds_start: this.secondsStart }
+                        : {}),
+                    seconds_total: options.duration ??
+                        this.secondsTotal ??
+                        this.extraInput?.duration,
+                    ...(this.steps !== undefined ? { steps: this.steps } : {}),
+                }),
+            }, this.timeoutMs);
+            if (!response.ok) {
+                const text = await response.text();
+                return { success: false, error: `FAL audio error: ${text}` };
+            }
+            const data = (await response.json());
+            const file = data.audio_file ?? data.audio ?? data.file;
+            const audioUrl = file?.url ?? data.audio_url ?? data.url;
+            if (!audioUrl) {
+                return { success: false, error: "No audio returned from FAL" };
+            }
+            return {
+                success: true,
+                data: {
+                    audioUrl,
+                    mimeType: file?.content_type,
+                    fileName: file?.file_name,
+                    title: "Generated Audio",
+                    duration: data.duration ?? options.duration,
+                },
+            };
+        });
+    }
+}
+// ============================================================================
+// ElevenLabs Audio Provider Implementation
+// ============================================================================
+export class ElevenLabsAudioProvider {
+    name = "elevenlabs";
+    apiKey;
+    baseUrl;
+    config;
+    timeoutMs;
+    constructor(config) {
+        if (!config.apiKey) {
+            throw new Error(`${this.name} API key is required`);
+        }
+        this.apiKey = config.apiKey;
+        this.baseUrl = config.baseUrl ?? "https://api.elevenlabs.io/v1";
+        this.config = config;
+        this.timeoutMs = config.timeoutMs ?? DEFAULT_AUDIO_TIMEOUT_MS;
+    }
+    async generate(options) {
+        const kind = normalizeAudioKind(options.audioKind ?? options.kind) ?? "sfx";
+        switch (kind) {
+            case "tts":
+                return this.generateSpeech(options);
+            case "music":
+                return this.generateMusic(options);
+            default:
+                return this.generateSoundEffect(options);
+        }
+    }
+    headers() {
+        return {
+            "Content-Type": "application/json",
+            "xi-api-key": this.apiKey,
+        };
+    }
+    async generateSoundEffect(options) {
+        return withProviderErrorBoundary(this.name, async () => {
+            const outputFormat = options.outputFormat ?? this.config.outputFormat;
+            const response = await fetchWithTimeout(`${this.baseUrl}/sound-generation${buildOutputFormatQuery(outputFormat)}`, {
+                method: "POST",
+                headers: this.headers(),
+                body: JSON.stringify({
+                    text: options.prompt,
+                    ...(options.duration !== undefined ||
+                        this.config.duration !== undefined
+                        ? { duration_seconds: options.duration ?? this.config.duration }
+                        : {}),
+                    ...(options.promptInfluence !== undefined ||
+                        this.config.promptInfluence !== undefined
+                        ? {
+                            prompt_influence: options.promptInfluence ?? this.config.promptInfluence,
+                        }
+                        : {}),
+                    ...(options.loop !== undefined || this.config.loop !== undefined
+                        ? { loop: options.loop ?? this.config.loop }
+                        : {}),
+                    ...(options.seed !== undefined || this.config.seed !== undefined
+                        ? { seed: options.seed ?? this.config.seed }
+                        : {}),
+                    model_id: options.modelId ??
+                        this.config.sfxModelId ??
+                        this.config.modelId ??
+                        "eleven_text_to_sound_v2",
+                }),
+            }, this.timeoutMs);
+            if (!response.ok) {
+                const text = await response.text();
+                return { success: false, error: `ElevenLabs SFX error: ${text}` };
+            }
+            return {
+                success: true,
+                data: await responseToAudioResult(response, "Generated Sound Effect", options.duration ?? this.config.duration),
+            };
+        });
+    }
+    async generateMusic(options) {
+        return withProviderErrorBoundary(this.name, async () => {
+            const outputFormat = options.outputFormat ?? this.config.outputFormat;
+            const prompt = options.genre
+                ? `${options.prompt}\nGenre: ${options.genre}`
+                : options.prompt;
+            const response = await fetchWithTimeout(`${this.baseUrl}/music${buildOutputFormatQuery(outputFormat)}`, {
+                method: "POST",
+                headers: this.headers(),
+                body: JSON.stringify({
+                    prompt,
+                    ...(options.duration
+                        ? { music_length_ms: Math.round(options.duration * 1000) }
+                        : {}),
+                    ...(options.instrumental !== undefined
+                        ? { force_instrumental: options.instrumental }
+                        : {}),
+                    ...(options.seed !== undefined || this.config.seed !== undefined
+                        ? { seed: options.seed ?? this.config.seed }
+                        : {}),
+                    ...(this.config.signWithC2pa !== undefined
+                        ? { sign_with_c2pa: this.config.signWithC2pa }
+                        : {}),
+                    model_id: options.modelId ??
+                        this.config.musicModelId ??
+                        this.config.modelId ??
+                        "music_v1",
+                }),
+            }, this.timeoutMs);
+            if (!response.ok) {
+                const text = await response.text();
+                return { success: false, error: `ElevenLabs music error: ${text}` };
+            }
+            return {
+                success: true,
+                data: await responseToAudioResult(response, "Generated Music", options.duration),
+            };
+        });
+    }
+    async generateSpeech(options) {
+        const voiceId = options.voiceId ?? this.config.voiceId;
+        if (!voiceId) {
+            return {
+                success: false,
+                error: "ElevenLabs TTS requires a voiceId in config or request options",
+            };
+        }
+        return withProviderErrorBoundary(this.name, async () => {
+            const outputFormat = options.outputFormat ?? this.config.outputFormat;
+            const response = await fetchWithTimeout(`${this.baseUrl}/text-to-speech/${encodeURIComponent(voiceId)}${buildOutputFormatQuery(outputFormat)}`, {
+                method: "POST",
+                headers: this.headers(),
+                body: JSON.stringify({
+                    text: options.text ?? options.prompt,
+                    model_id: options.modelId ??
+                        this.config.ttsModelId ??
+                        this.config.modelId ??
+                        "eleven_multilingual_v2",
+                    ...(options.languageCode !== undefined ||
+                        this.config.languageCode !== undefined
+                        ? {
+                            language_code: options.languageCode ?? this.config.languageCode,
+                        }
+                        : {}),
+                    ...(options.voiceSettings !== undefined ||
+                        this.config.voiceSettings !== undefined
+                        ? {
+                            voice_settings: options.voiceSettings ?? this.config.voiceSettings,
+                        }
+                        : {}),
+                    ...(options.seed !== undefined || this.config.seed !== undefined
+                        ? { seed: options.seed ?? this.config.seed }
+                        : {}),
+                    ...(this.config.applyTextNormalization
+                        ? {
+                            apply_text_normalization: this.config.applyTextNormalization,
+                        }
+                        : {}),
+                }),
+            }, this.timeoutMs);
+            if (!response.ok) {
+                const text = await response.text();
+                return { success: false, error: `ElevenLabs TTS error: ${text}` };
+            }
+            return {
+                success: true,
+                data: await responseToAudioResult(response, "Generated Speech"),
+            };
+        });
+    }
+}
+// ============================================================================
+// Suno Provider Implementation
+// ============================================================================
+export class SunoAudioProvider {
+    name = "suno";
+    apiKey;
+    model;
+    baseUrl;
+    constructor(config) {
+        if (!config.apiKey) {
+            throw new Error(`${this.name} API key is required`);
+        }
+        this.apiKey = config.apiKey;
+        this.model = config.model ?? "chirp-v3.5";
+        this.baseUrl = config.baseUrl ?? "https://api.suno.ai/v1";
+    }
+    async generate(options) {
+        return withProviderErrorBoundary(this.name, async () => {
+            const response = await fetchWithTimeout(`${this.baseUrl}/generate`, {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    Authorization: `Bearer ${this.apiKey}`,
+                },
+                body: JSON.stringify({
+                    prompt: options.prompt,
+                    model: this.model,
+                    duration: options.duration,
+                    instrumental: options.instrumental,
+                    genre: options.genre,
+                }),
+            });
+            if (!response.ok) {
+                const text = await response.text();
+                return { success: false, error: `Suno error: ${text}` };
+            }
+            const data = (await response.json());
+            return {
+                success: true,
+                data: {
+                    audioUrl: data.audio_url,
+                    title: data.title,
+                    duration: data.duration,
+                },
+            };
+        });
+    }
+}
+export function createImageProvider(config, options) {
+    const mode = config?.mode ?? (options.cloudMediaDisabled ? "local" : "cloud");
+    const provider = mode === "cloud" ? "cloud" : (config?.provider ?? "cloud");
+    switch (provider) {
+        case "fal":
+            if (config?.fal?.apiKey) {
+                return new FalImageProvider(config.fal);
+            }
+            break;
+        case "openai":
+            if (config?.openai?.apiKey) {
+                return new OpenAIImageProvider(config.openai);
+            }
+            break;
+        case "google":
+            if (config?.google?.apiKey) {
+                return new GoogleImageProvider(config.google);
+            }
+            break;
+        case "xai":
+            if (config?.xai?.apiKey) {
+                return new XAIImageProvider(config.xai);
+            }
+            break;
+    }
+    if (options.cloudMediaDisabled) {
+        throw new Error("No image provider configured and cloud media is disabled. " +
+            "Configure a direct provider (fal, openai, google, xai) or enable cloud media.");
+    }
+    return new ElizaCloudImageProvider(options.elizaCloudBaseUrl ?? "https://elizacloud.ai/api/v1", options.elizaCloudApiKey);
+}
+export function createVideoProvider(config, options) {
+    const mode = config?.mode ?? (options.cloudMediaDisabled ? "local" : "cloud");
+    const provider = mode === "cloud" ? "cloud" : (config?.provider ?? "cloud");
+    switch (provider) {
+        case "fal":
+            if (config?.fal?.apiKey) {
+                return new FalVideoProvider(config.fal);
+            }
+            break;
+        case "openai":
+            if (config?.openai?.apiKey) {
+                return new OpenAIVideoProvider(config.openai);
+            }
+            break;
+        case "google":
+            if (config?.google?.apiKey) {
+                return new GoogleVideoProvider(config.google);
+            }
+            break;
+    }
+    if (options.cloudMediaDisabled) {
+        throw new Error("No video provider configured and cloud media is disabled. " +
+            "Configure a direct provider (fal, openai, google) or enable cloud media.");
+    }
+    return new ElizaCloudVideoProvider(options.elizaCloudBaseUrl ?? "https://elizacloud.ai/api/v1", options.elizaCloudApiKey);
+}
+function hasUsableDirectAudioProvider(config) {
+    return !!(config?.suno?.apiKey ||
+        config?.elevenlabs?.apiKey ||
+        config?.fal?.apiKey);
+}
+function getConfiguredAudioProvider(config, kind) {
+    return (config?.providers?.[kind] ?? config?.providers?.default ?? config?.provider);
+}
+function getAutoAudioProvider(config, kind) {
+    if (kind === "music") {
+        if (config?.suno?.apiKey)
+            return "suno";
+        if (config?.elevenlabs?.apiKey)
+            return "elevenlabs";
+        if (config?.fal?.apiKey)
+            return "fal";
+        return undefined;
+    }
+    if (config?.elevenlabs?.apiKey)
+        return "elevenlabs";
+    if (config?.fal?.apiKey)
+        return "fal";
+    return undefined;
+}
+function missingAudioProviderError(kind) {
+    return (`No ${kind} audio provider configured and cloud media is disabled. ` +
+        "Configure a direct provider (suno, elevenlabs, fal) or enable cloud media.");
+}
+class RoutedAudioProvider {
+    name = "audio-router";
+    config;
+    options;
+    constructor(config, options) {
+        this.config = config;
+        this.options = options;
+    }
+    async generate(options) {
+        const kind = getAudioKind(options, this.config);
+        const providerName = getConfiguredAudioProvider(this.config, kind) ??
+            getAutoAudioProvider(this.config, kind) ??
+            "cloud";
+        if (providerName === "cloud") {
+            if (this.options.cloudMediaDisabled) {
+                return { success: false, error: missingAudioProviderError(kind) };
+            }
+            return createCloudAudioProvider(this.options).generate({
+                ...options,
+                kind,
+            });
+        }
+        try {
+            switch (providerName) {
+                case "suno":
+                    if (!this.config?.suno?.apiKey)
+                        break;
+                    if (kind !== "music") {
+                        return {
+                            success: false,
+                            error: "Suno audio generation only supports music requests",
+                        };
+                    }
+                    return new SunoAudioProvider(this.config.suno).generate({
+                        ...options,
+                        kind,
+                    });
+                case "elevenlabs":
+                    if (!this.config?.elevenlabs?.apiKey)
+                        break;
+                    return new ElevenLabsAudioProvider(this.config.elevenlabs).generate({
+                        ...options,
+                        kind,
+                    });
+                case "fal":
+                    if (!this.config?.fal?.apiKey)
+                        break;
+                    return new FalAudioProvider(this.config.fal).generate({
+                        ...options,
+                        kind,
+                    });
+            }
+        }
+        catch (err) {
+            return {
+                success: false,
+                error: err instanceof Error ? err.message : String(err),
+            };
+        }
+        if (this.options.cloudMediaDisabled) {
+            return { success: false, error: missingAudioProviderError(kind) };
+        }
+        return createCloudAudioProvider(this.options).generate({
+            ...options,
+            kind,
+        });
+    }
+}
+export function createAudioProvider(config, options) {
+    const mode = config?.mode ?? (options.cloudMediaDisabled ? "own-key" : "cloud");
+    if (mode === "cloud") {
+        if (options.cloudMediaDisabled) {
+            throw new Error("Audio media is configured for cloud mode but cloud media is disabled.");
+        }
+        return createCloudAudioProvider(options);
+    }
+    if (options.cloudMediaDisabled && !hasUsableDirectAudioProvider(config)) {
+        throw new Error("No audio provider configured and cloud media is disabled. " +
+            "Configure a direct provider (suno, elevenlabs, fal) or enable cloud media.");
+    }
+    return new RoutedAudioProvider(config, options);
+}
+export function createVisionProvider(config, options) {
+    const mode = config?.mode ?? (options.cloudMediaDisabled ? "local" : "cloud");
+    const provider = mode === "cloud" ? "cloud" : (config?.provider ?? "cloud");
+    switch (provider) {
+        case "openai":
+            if (config?.openai?.apiKey) {
+                return new OpenAIVisionProvider(config.openai);
+            }
+            break;
+        case "google":
+            if (config?.google?.apiKey) {
+                return new GoogleVisionProvider(config.google);
+            }
+            break;
+        case "anthropic":
+            if (config?.anthropic?.apiKey) {
+                return new AnthropicVisionProvider(config.anthropic);
+            }
+            break;
+        case "xai":
+            if (config?.xai?.apiKey) {
+                return new XAIVisionProvider(config.xai);
+            }
+            break;
+        case "ollama":
+            // Ollama doesn't require an API key, just a base URL
+            return new OllamaVisionProvider(config?.ollama ?? {});
+    }
+    if (options.cloudMediaDisabled) {
+        throw new Error("No vision provider configured and cloud media is disabled. " +
+            "Configure a direct provider (openai, google, anthropic, xai, ollama) or enable cloud media.");
+    }
+    return new ElizaCloudVisionProvider(options.elizaCloudBaseUrl ?? "https://elizacloud.ai/api/v1", options.elizaCloudApiKey);
+}
+export function createMediaProviders(config, options) {
+    return {
+        image: createImageProvider(config?.image, options),
+        video: createVideoProvider(config?.video, options),
+        audio: createAudioProvider(config?.audio, options),
+        vision: createVisionProvider(config?.vision, options),
+    };
+}