@elizaos/agent 2.0.0-alpha.144 → 2.0.0-alpha.151

package/packages/agent/src/providers/media-provider.js

@@ -0,0 +1,1088 @@
+/**
+ * Media Provider Abstraction Layer
+ *
+ * Provides a unified interface for media generation across multiple providers:
+ * - Image Generation: FAL.ai, OpenAI (DALL-E), Google (Imagen), xAI, Eliza Cloud
+ * - Video Generation: FAL.ai, OpenAI (Sora), Google (Veo), Eliza Cloud
+ * - Audio Generation: Suno, ElevenLabs (SFX), Eliza Cloud
+ * - Vision (Analysis): OpenAI, Google, Anthropic, xAI, Eliza Cloud
+ *
+ * Follows the same pattern as TTS provider selection:
+ * - "cloud" mode uses Eliza Cloud (no API key needed)
+ * - "own-key" mode uses the user's own API keys
+ */
+// ============================================================================
+// Fetch Utilities
+// ============================================================================
+/** Fetch with an AbortController-based timeout (default 30s). */
+export function fetchWithTimeout(url, init, timeoutMs = 30_000) {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    return fetch(url, { ...init, signal: controller.signal }).finally(() => clearTimeout(timer));
+}
+async function withProviderErrorBoundary(providerName, run) {
+    try {
+        return await run();
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        return {
+            success: false,
+            error: `[${providerName}] Network error: ${message}`,
+        };
+    }
+}
+// ============================================================================
+// Eliza Cloud Provider Implementations
+// ============================================================================
+class ElizaCloudImageProvider {
+    name = "eliza-cloud";
+    baseUrl;
+    apiKey;
+    constructor(baseUrl, apiKey) {
+        this.baseUrl = baseUrl;
+        this.apiKey = apiKey;
+    }
+    async generate(options) {
+        const response = await fetchWithTimeout(`${this.baseUrl}/media/image/generate`, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                ...(this.apiKey ? { Authorization: `Bearer ${this.apiKey}` } : {}),
+            },
+            body: JSON.stringify({
+                prompt: options.prompt,
+                size: options.size,
+                quality: options.quality,
+                style: options.style,
+            }),
+        });
+        if (!response.ok) {
+            const text = await response.text();
+            return { success: false, error: `Eliza Cloud error: ${text}` };
+        }
+        const data = (await response.json());
+        return {
+            success: true,
+            data: {
+                imageUrl: data.imageUrl,
+                imageBase64: data.imageBase64,
+                revisedPrompt: data.revisedPrompt,
+            },
+        };
+    }
+}
+class ElizaCloudVideoProvider {
+    name = "eliza-cloud";
+    baseUrl;
+    apiKey;
+    constructor(baseUrl, apiKey) {
+        this.baseUrl = baseUrl;
+        this.apiKey = apiKey;
+    }
+    async generate(options) {
+        const response = await fetchWithTimeout(`${this.baseUrl}/media/video/generate`, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                ...(this.apiKey ? { Authorization: `Bearer ${this.apiKey}` } : {}),
+            },
+            body: JSON.stringify({
+                prompt: options.prompt,
+                duration: options.duration,
+                aspectRatio: options.aspectRatio,
+                imageUrl: options.imageUrl,
+            }),
+        });
+        if (!response.ok) {
+            const text = await response.text();
+            return { success: false, error: `Eliza Cloud error: ${text}` };
+        }
+        const data = (await response.json());
+        return {
+            success: true,
+            data: {
+                videoUrl: data.videoUrl,
+                thumbnailUrl: data.thumbnailUrl,
+                duration: data.duration,
+            },
+        };
+    }
+}
+class ElizaCloudAudioProvider {
+    name = "eliza-cloud";
+    baseUrl;
+    apiKey;
+    constructor(baseUrl, apiKey) {
+        this.baseUrl = baseUrl;
+        this.apiKey = apiKey;
+    }
+    async generate(options) {
+        const response = await fetchWithTimeout(`${this.baseUrl}/media/audio/generate`, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                ...(this.apiKey ? { Authorization: `Bearer ${this.apiKey}` } : {}),
+            },
+            body: JSON.stringify({
+                prompt: options.prompt,
+                duration: options.duration,
+                instrumental: options.instrumental,
+                genre: options.genre,
+            }),
+        });
+        if (!response.ok) {
+            const text = await response.text();
+            return { success: false, error: `Eliza Cloud error: ${text}` };
+        }
+        const data = (await response.json());
+        return {
+            success: true,
+            data: {
+                audioUrl: data.audioUrl,
+                title: data.title,
+                duration: data.duration,
+            },
+        };
+    }
+}
+class ElizaCloudVisionProvider {
+    name = "eliza-cloud";
+    baseUrl;
+    apiKey;
+    constructor(baseUrl, apiKey) {
+        this.baseUrl = baseUrl;
+        this.apiKey = apiKey;
+    }
+    async analyze(options) {
+        const response = await fetchWithTimeout(`${this.baseUrl}/media/vision/analyze`, {
+            method: "POST",
+            headers: {
+                "Content-Type": "application/json",
+                ...(this.apiKey ? { Authorization: `Bearer ${this.apiKey}` } : {}),
+            },
+            body: JSON.stringify({
+                imageUrl: options.imageUrl,
+                imageBase64: options.imageBase64,
+                prompt: options.prompt,
+                maxTokens: options.maxTokens,
+            }),
+        });
+        if (!response.ok) {
+            const text = await response.text();
+            return { success: false, error: `Eliza Cloud error: ${text}` };
+        }
+        const data = (await response.json());
+        return {
+            success: true,
+            data: {
+                description: data.description,
+                labels: data.labels,
+                confidence: data.confidence,
+            },
+        };
+    }
+}
+// ============================================================================
+// FAL.ai Provider Implementations
+// ============================================================================
+export class FalImageProvider {
+    name = "fal";
+    apiKey;
+    model;
+    baseUrl;
+    constructor(config) {
+        if (!config.apiKey) {
+            throw new Error(`${this.name} API key is required`);
+        }
+        this.apiKey = config.apiKey;
+        this.model = config.model ?? "fal-ai/flux-pro";
+        this.baseUrl = config.baseUrl ?? "https://fal.run";
+    }
+    async generate(options) {
+        return withProviderErrorBoundary(this.name, async () => {
+            const response = await fetchWithTimeout(`${this.baseUrl}/${this.model}`, {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    Authorization: `Key ${this.apiKey}`,
+                },
+                body: JSON.stringify({
+                    prompt: options.prompt,
+                    image_size: options.size ?? "landscape_4_3",
+                    num_images: 1,
+                    ...(options.negativePrompt
+                        ? { negative_prompt: options.negativePrompt }
+                        : {}),
+                    ...(options.seed ? { seed: options.seed } : {}),
+                }),
+            });
+            if (!response.ok) {
+                const text = await response.text();
+                return { success: false, error: `FAL error: ${text}` };
+            }
+            const data = (await response.json());
+            const imageUrl = data.images?.[0]?.url;
+            if (!imageUrl) {
+                return { success: false, error: "No image returned from FAL" };
+            }
+            return {
+                success: true,
+                data: { imageUrl },
+            };
+        });
+    }
+}
+export class FalVideoProvider {
+    name = "fal";
+    apiKey;
+    model;
+    baseUrl;
+    constructor(config) {
+        if (!config.apiKey) {
+            throw new Error(`${this.name} API key is required`);
+        }
+        this.apiKey = config.apiKey;
+        this.model = config.model ?? "fal-ai/minimax-video";
+        this.baseUrl = config.baseUrl ?? "https://fal.run";
+    }
+    async generate(options) {
+        return withProviderErrorBoundary(this.name, async () => {
+            const response = await fetchWithTimeout(`${this.baseUrl}/${this.model}`, {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    Authorization: `Key ${this.apiKey}`,
+                },
+                body: JSON.stringify({
+                    prompt: options.prompt,
+                    ...(options.duration ? { duration: options.duration } : {}),
+                    ...(options.aspectRatio ? { aspect_ratio: options.aspectRatio } : {}),
+                    ...(options.imageUrl ? { image_url: options.imageUrl } : {}),
+                }),
+            });
+            if (!response.ok) {
+                const text = await response.text();
+                return { success: false, error: `FAL error: ${text}` };
+            }
+            const data = (await response.json());
+            return {
+                success: true,
+                data: {
+                    videoUrl: data.video?.url,
+                    thumbnailUrl: data.thumbnail?.url,
+                    duration: data.duration,
+                },
+            };
+        });
+    }
+}
+// ============================================================================
+// OpenAI Provider Implementations
+// ============================================================================
+export class OpenAIImageProvider {
+    name = "openai";
+    apiKey;
+    model;
+    quality;
+    style;
+    constructor(config) {
+        if (!config.apiKey) {
+            throw new Error(`${this.name} API key is required`);
+        }
+        this.apiKey = config.apiKey;
+        this.model = config.model ?? "dall-e-3";
+        this.quality = config.quality ?? "standard";
+        this.style = config.style ?? "vivid";
+    }
+    async generate(options) {
+        return withProviderErrorBoundary(this.name, async () => {
+            const response = await fetchWithTimeout("https://api.openai.com/v1/images/generations", {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    Authorization: `Bearer ${this.apiKey}`,
+                },
+                body: JSON.stringify({
+                    model: this.model,
+                    prompt: options.prompt,
+                    n: 1,
+                    size: options.size ?? "1024x1024",
+                    quality: options.quality ?? this.quality,
+                    style: options.style ?? this.style,
+                }),
+            });
+            if (!response.ok) {
+                const text = await response.text();
+                return { success: false, error: `OpenAI error: ${text}` };
+            }
+            const data = (await response.json());
+            const image = data.data?.[0];
+            if (!image?.url) {
+                return { success: false, error: "No image returned from OpenAI" };
+            }
+            return {
+                success: true,
+                data: {
+                    imageUrl: image.url,
+                    revisedPrompt: image.revised_prompt,
+                },
+            };
+        });
+    }
+}
+export class OpenAIVideoProvider {
+    name = "openai";
+    apiKey;
+    model;
+    constructor(config) {
+        if (!config.apiKey) {
+            throw new Error(`${this.name} API key is required`);
+        }
+        this.apiKey = config.apiKey;
+        this.model = config.model ?? "sora-1.0-turbo";
+    }
+    async generate(options) {
+        return withProviderErrorBoundary(this.name, async () => {
+            // OpenAI Sora API (video generation)
+            const response = await fetchWithTimeout("https://api.openai.com/v1/videos/generations", {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    Authorization: `Bearer ${this.apiKey}`,
+                },
+                body: JSON.stringify({
+                    model: this.model,
+                    prompt: options.prompt,
+                    n: 1,
+                    duration: options.duration ?? 5,
+                    aspect_ratio: options.aspectRatio ?? "16:9",
+                    ...(options.imageUrl ? { image: options.imageUrl } : {}),
+                }),
+            });
+            if (!response.ok) {
+                const text = await response.text();
+                return { success: false, error: `OpenAI Sora error: ${text}` };
+            }
+            const data = (await response.json());
+            const video = data.data?.[0];
+            if (!video?.url) {
+                return { success: false, error: "No video returned from OpenAI Sora" };
+            }
+            return {
+                success: true,
+                data: {
+                    videoUrl: video.url,
+                    duration: video.duration,
+                },
+            };
+        });
+    }
+}
+export class OpenAIVisionProvider {
+    name = "openai";
+    apiKey;
+    model;
+    maxTokens;
+    constructor(config) {
+        if (!config.apiKey) {
+            throw new Error(`${this.name} API key is required`);
+        }
+        this.apiKey = config.apiKey;
+        this.model = config.model ?? "gpt-4o";
+        this.maxTokens = config.maxTokens ?? 1024;
+    }
+    async analyze(options) {
+        const imageContent = options.imageBase64
+            ? {
+                type: "image_url",
+                image_url: { url: `data:image/jpeg;base64,${options.imageBase64}` },
+            }
+            : {
+                type: "image_url",
+                image_url: { url: options.imageUrl ?? "" },
+            };
+        return withProviderErrorBoundary(this.name, async () => {
+            const response = await fetchWithTimeout("https://api.openai.com/v1/chat/completions", {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    Authorization: `Bearer ${this.apiKey}`,
+                },
+                body: JSON.stringify({
+                    model: this.model,
+                    max_tokens: options.maxTokens ?? this.maxTokens,
+                    messages: [
+                        {
+                            role: "user",
+                            content: [
+                                {
+                                    type: "text",
+                                    text: options.prompt ?? "Describe this image in detail.",
+                                },
+                                imageContent,
+                            ],
+                        },
+                    ],
+                }),
+            });
+            if (!response.ok) {
+                const text = await response.text();
+                return { success: false, error: `OpenAI error: ${text}` };
+            }
+            const data = (await response.json());
+            const description = data.choices?.[0]?.message?.content;
+            if (!description) {
+                return {
+                    success: false,
+                    error: "No description returned from OpenAI",
+                };
+            }
+            return {
+                success: true,
+                data: { description },
+            };
+        });
+    }
+}
+// ============================================================================
+// Google Provider Implementations
+// ============================================================================
+export class GoogleImageProvider {
+    name = "google";
+    apiKey;
+    model;
+    aspectRatio;
+    constructor(config) {
+        if (!config.apiKey) {
+            throw new Error(`${this.name} API key is required`);
+        }
+        this.apiKey = config.apiKey;
+        this.model = config.model ?? "imagen-3.0-generate-002";
+        this.aspectRatio = config.aspectRatio ?? "1:1";
+    }
+    async generate(options) {
+        return withProviderErrorBoundary(this.name, async () => {
+            const response = await fetchWithTimeout(`https://generativelanguage.googleapis.com/v1beta/models/${this.model}:predict`, {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    "x-goog-api-key": this.apiKey,
+                },
+                body: JSON.stringify({
+                    instances: [{ prompt: options.prompt }],
+                    parameters: {
+                        sampleCount: 1,
+                        aspectRatio: options.size ?? this.aspectRatio,
+                        personGeneration: "allow_adult",
+                        safetyFilterLevel: "block_few",
+                    },
+                }),
+            });
+            if (!response.ok) {
+                const text = await response.text();
+                return { success: false, error: `Google Imagen error: ${text}` };
+            }
+            const data = (await response.json());
+            const imageData = data.predictions?.[0]?.bytesBase64Encoded;
+            if (!imageData) {
+                return {
+                    success: false,
+                    error: "No image returned from Google Imagen",
+                };
+            }
+            return {
+                success: true,
+                data: {
+                    imageBase64: imageData,
+                },
+            };
+        });
+    }
+}
+export class GoogleVideoProvider {
+    name = "google";
+    apiKey;
+    model;
+    constructor(config) {
+        if (!config.apiKey) {
+            throw new Error(`${this.name} API key is required`);
+        }
+        this.apiKey = config.apiKey;
+        this.model = config.model ?? "veo-2.0-generate-001";
+    }
+    async generate(options) {
+        return withProviderErrorBoundary(this.name, async () => {
+            // Google Veo uses a different endpoint structure
+            const response = await fetchWithTimeout(`https://generativelanguage.googleapis.com/v1beta/models/${this.model}:predictLongRunning`, {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    "x-goog-api-key": this.apiKey,
+                },
+                body: JSON.stringify({
+                    instances: [
+                        {
+                            prompt: options.prompt,
+                            ...(options.imageUrl
+                                ? { image: { gcsUri: options.imageUrl } }
+                                : {}),
+                        },
+                    ],
+                    parameters: {
+                        aspectRatio: options.aspectRatio ?? "16:9",
+                        durationSeconds: options.duration ?? 5,
+                        personGeneration: "allow_adult",
+                    },
+                }),
+            });
+            if (!response.ok) {
+                const text = await response.text();
+                return { success: false, error: `Google Veo error: ${text}` };
+            }
+            // Veo returns a long-running operation - for now we return the operation name
+            // In production, you'd poll the operation until it completes
+            const data = (await response.json());
+            if (data.done && data.response?.predictions?.[0]?.videoUri) {
+                return {
+                    success: true,
+                    data: {
+                        videoUrl: data.response.predictions[0].videoUri,
+                    },
+                };
+            }
+            // Operation started but not complete - return operation reference
+            // Client should poll the operation endpoint
+            return {
+                success: true,
+                data: {
+                    videoUrl: `pending:${data.name}`,
+                },
+            };
+        });
+    }
+}
+export class GoogleVisionProvider {
+    name = "google";
+    apiKey;
+    model;
+    constructor(config) {
+        if (!config.apiKey) {
+            throw new Error(`${this.name} API key is required`);
+        }
+        this.apiKey = config.apiKey;
+        this.model = config.model ?? "gemini-2.0-flash";
+    }
+    async analyze(options) {
+        const imagePart = options.imageBase64
+            ? { inline_data: { mime_type: "image/jpeg", data: options.imageBase64 } }
+            : { file_data: { file_uri: options.imageUrl, mime_type: "image/jpeg" } };
+        return withProviderErrorBoundary(this.name, async () => {
+            const response = await fetchWithTimeout(`https://generativelanguage.googleapis.com/v1beta/models/${this.model}:generateContent`, {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    "x-goog-api-key": this.apiKey,
+                },
+                body: JSON.stringify({
+                    contents: [
+                        {
+                            parts: [
+                                { text: options.prompt ?? "Describe this image in detail." },
+                                imagePart,
+                            ],
+                        },
+                    ],
+                }),
+            });
+            if (!response.ok) {
+                const text = await response.text();
+                return { success: false, error: `Google error: ${text}` };
+            }
+            const data = (await response.json());
+            const description = data.candidates?.[0]?.content?.parts?.[0]?.text;
+            if (!description) {
+                return {
+                    success: false,
+                    error: "No description returned from Google",
+                };
+            }
+            return {
+                success: true,
+                data: { description },
+            };
+        });
+    }
+}
+// ============================================================================
+// xAI Provider Implementations
+// ============================================================================
+export class XAIImageProvider {
+    name = "xai";
+    apiKey;
+    model;
+    constructor(config) {
+        if (!config.apiKey) {
+            throw new Error(`${this.name} API key is required`);
+        }
+        this.apiKey = config.apiKey;
+        this.model = config.model ?? "grok-2-image";
+    }
+    async generate(options) {
+        return withProviderErrorBoundary(this.name, async () => {
+            // xAI uses OpenAI-compatible API format for image generation
+            const response = await fetchWithTimeout("https://api.x.ai/v1/images/generations", {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    Authorization: `Bearer ${this.apiKey}`,
+                },
+                body: JSON.stringify({
+                    model: this.model,
+                    prompt: options.prompt,
+                    n: 1,
+                    size: options.size ?? "1024x1024",
+                    response_format: "url",
+                }),
+            });
+            if (!response.ok) {
+                const text = await response.text();
+                return { success: false, error: `xAI error: ${text}` };
+            }
+            const data = (await response.json());
+            const image = data.data?.[0];
+            if (!image?.url) {
+                return { success: false, error: "No image returned from xAI" };
+            }
+            return {
+                success: true,
+                data: {
+                    imageUrl: image.url,
+                    revisedPrompt: image.revised_prompt,
+                },
+            };
+        });
+    }
+}
+export class XAIVisionProvider {
+    name = "xai";
+    apiKey;
+    model;
+    constructor(config) {
+        if (!config.apiKey) {
+            throw new Error(`${this.name} API key is required`);
+        }
+        this.apiKey = config.apiKey;
+        this.model = config.model ?? "grok-2-vision-1212";
+    }
+    async analyze(options) {
+        // xAI uses OpenAI-compatible API format
+        const imageContent = options.imageBase64
+            ? {
+                type: "image_url",
+                image_url: { url: `data:image/jpeg;base64,${options.imageBase64}` },
+            }
+            : {
+                type: "image_url",
+                image_url: { url: options.imageUrl ?? "" },
+            };
+        return withProviderErrorBoundary(this.name, async () => {
+            const response = await fetchWithTimeout("https://api.x.ai/v1/chat/completions", {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    Authorization: `Bearer ${this.apiKey}`,
+                },
+                body: JSON.stringify({
+                    model: this.model,
+                    max_tokens: options.maxTokens ?? 1024,
+                    messages: [
+                        {
+                            role: "user",
+                            content: [
+                                {
+                                    type: "text",
+                                    text: options.prompt ?? "Describe this image in detail.",
+                                },
+                                imageContent,
+                            ],
+                        },
+                    ],
+                }),
+            });
+            if (!response.ok) {
+                const text = await response.text();
+                return { success: false, error: `xAI error: ${text}` };
+            }
+            const data = (await response.json());
+            const description = data.choices?.[0]?.message?.content;
+            if (!description) {
+                return { success: false, error: "No description returned from xAI" };
+            }
+            return {
+                success: true,
+                data: { description },
+            };
+        });
+    }
+}
+// ============================================================================
+// Anthropic Provider Implementation
+// ============================================================================
+// ============================================================================
+// Ollama Provider Implementation (Local Vision)
+// ============================================================================
+class OllamaVisionProvider {
+    name = "ollama";
+    baseUrl;
+    model;
+    maxTokens;
+    autoDownload;
+    modelChecked = false;
+    constructor(config) {
+        this.baseUrl = config.baseUrl ?? "http://localhost:11434";
+        this.model = config.model ?? "llava";
+        this.maxTokens = config.maxTokens ?? 1024;
+        this.autoDownload = config.autoDownload ?? true;
+    }
+    async ensureModelAvailable() {
+        if (this.modelChecked)
+            return;
+        try {
+            // Check if model exists
+            const response = await fetchWithTimeout(`${this.baseUrl}/api/tags`, {}, 120_000);
+            if (!response.ok) {
+                throw new Error(`Ollama server not reachable: ${response.statusText}`);
+            }
+            const data = (await response.json());
+            const models = data.models ?? [];
+            const hasModel = models.some((m) => m.name === this.model || m.name.startsWith(`${this.model}:`));
+            if (!hasModel && this.autoDownload) {
+                console.log(`[ollama-vision] Model ${this.model} not found, downloading...`);
+                await this.downloadModel();
+            }
+            else if (!hasModel) {
+                throw new Error(`Ollama model ${this.model} not found. Run 'ollama pull ${this.model}' or enable autoDownload.`);
+            }
+            this.modelChecked = true;
+        }
+        catch (err) {
+            if (err instanceof Error &&
+                err.message.includes("Ollama server not reachable")) {
+                throw err;
+            }
+            throw new Error(`Failed to check Ollama models: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    async downloadModel() {
+        const response = await fetchWithTimeout(`${this.baseUrl}/api/pull`, {
+            method: "POST",
+            headers: { "Content-Type": "application/json" },
+            body: JSON.stringify({ name: this.model, stream: false }),
+        }, 300_000);
+        if (!response.ok) {
+            const text = await response.text();
+            throw new Error(`Failed to download model ${this.model}: ${text}`);
+        }
+        // Wait for download to complete (non-streaming mode)
+        await response.json();
+        console.log(`[ollama-vision] Model ${this.model} downloaded successfully`);
+    }
+    async analyze(options) {
+        try {
+            await this.ensureModelAvailable();
+        }
+        catch (err) {
+            return {
+                success: false,
+                error: `Ollama setup failed: ${err instanceof Error ? err.message : String(err)}`,
+            };
+        }
+        // Ollama uses a different format for vision - images must be base64
+        let imageData = options.imageBase64;
+        if (!imageData && options.imageUrl) {
+            // Fetch the image and convert to base64
+            try {
+                const imageResponse = await fetchWithTimeout(options.imageUrl, {}, 120_000);
+                if (!imageResponse.ok) {
+                    return {
+                        success: false,
+                        error: `Failed to fetch image: ${imageResponse.statusText}`,
+                    };
+                }
+                const buffer = await imageResponse.arrayBuffer();
+                imageData = Buffer.from(buffer).toString("base64");
+            }
+            catch (err) {
+                return {
+                    success: false,
+                    error: `Failed to fetch image: ${err instanceof Error ? err.message : String(err)}`,
+                };
+            }
+        }
+        if (!imageData) {
+            return {
+                success: false,
+                error: "No image provided (imageUrl or imageBase64 required)",
+            };
+        }
+        return withProviderErrorBoundary(this.name, async () => {
+            const response = await fetchWithTimeout(`${this.baseUrl}/api/chat`, {
+                method: "POST",
+                headers: { "Content-Type": "application/json" },
+                body: JSON.stringify({
+                    model: this.model,
+                    messages: [
+                        {
+                            role: "user",
+                            content: options.prompt ?? "Describe this image in detail.",
+                            images: [imageData],
+                        },
+                    ],
+                    stream: false,
+                    options: {
+                        num_predict: this.maxTokens,
+                    },
+                }),
+            }, 120_000);
+            if (!response.ok) {
+                const text = await response.text();
+                return { success: false, error: `Ollama error: ${text}` };
+            }
+            const data = (await response.json());
+            const description = data.message?.content;
+            if (!description) {
+                return { success: false, error: "No description returned from Ollama" };
+            }
+            return {
+                success: true,
+                data: { description },
+            };
+        });
+    }
+}
+// ============================================================================
+// Anthropic Provider Implementation
+// ============================================================================
+export class AnthropicVisionProvider {
+    name = "anthropic";
+    apiKey;
+    model;
+    constructor(config) {
+        if (!config.apiKey) {
+            throw new Error(`${this.name} API key is required`);
+        }
+        this.apiKey = config.apiKey;
+        this.model = config.model ?? "claude-sonnet-4-20250514";
+    }
+    async analyze(options) {
+        const imageSource = options.imageBase64
+            ? {
+                type: "base64",
+                media_type: "image/jpeg",
+                data: options.imageBase64,
+            }
+            : { type: "url", url: options.imageUrl ?? "" };
+        return withProviderErrorBoundary(this.name, async () => {
+            const response = await fetchWithTimeout("https://api.anthropic.com/v1/messages", {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    "x-api-key": this.apiKey,
+                    "anthropic-version": "2023-06-01",
+                },
+                body: JSON.stringify({
+                    model: this.model,
+                    max_tokens: options.maxTokens ?? 1024,
+                    messages: [
+                        {
+                            role: "user",
+                            content: [
+                                { type: "image", source: imageSource },
+                                {
+                                    type: "text",
+                                    text: options.prompt ?? "Describe this image in detail.",
+                                },
+                            ],
+                        },
+                    ],
+                }),
+            });
+            if (!response.ok) {
+                const text = await response.text();
+                return { success: false, error: `Anthropic error: ${text}` };
+            }
+            const data = (await response.json());
+            const textBlock = data.content?.find((c) => c.type === "text");
+            if (!textBlock?.text) {
+                return {
+                    success: false,
+                    error: "No description returned from Anthropic",
+                };
+            }
+            return {
+                success: true,
+                data: { description: textBlock.text },
+            };
+        });
+    }
+}
+// ============================================================================
+// Suno Provider Implementation
+// ============================================================================
+export class SunoAudioProvider {
+    name = "suno";
+    apiKey;
+    model;
+    baseUrl;
+    constructor(config) {
+        if (!config.apiKey) {
+            throw new Error(`${this.name} API key is required`);
+        }
+        this.apiKey = config.apiKey;
+        this.model = config.model ?? "chirp-v3.5";
+        this.baseUrl = config.baseUrl ?? "https://api.suno.ai/v1";
+    }
+    async generate(options) {
+        return withProviderErrorBoundary(this.name, async () => {
+            const response = await fetchWithTimeout(`${this.baseUrl}/generate`, {
+                method: "POST",
+                headers: {
+                    "Content-Type": "application/json",
+                    Authorization: `Bearer ${this.apiKey}`,
+                },
+                body: JSON.stringify({
+                    prompt: options.prompt,
+                    model: this.model,
+                    duration: options.duration,
+                    instrumental: options.instrumental,
+                    genre: options.genre,
+                }),
+            });
+            if (!response.ok) {
+                const text = await response.text();
+                return { success: false, error: `Suno error: ${text}` };
+            }
+            const data = (await response.json());
+            return {
+                success: true,
+                data: {
+                    audioUrl: data.audio_url,
+                    title: data.title,
+                    duration: data.duration,
+                },
+            };
+        });
+    }
+}
+export function createImageProvider(config, options) {
+    const mode = config?.mode ?? (options.cloudMediaDisabled ? "local" : "cloud");
+    const provider = mode === "cloud" ? "cloud" : (config?.provider ?? "cloud");
+    switch (provider) {
+        case "fal":
+            if (config?.fal?.apiKey) {
+                return new FalImageProvider(config.fal);
+            }
+            break;
+        case "openai":
+            if (config?.openai?.apiKey) {
+                return new OpenAIImageProvider(config.openai);
+            }
+            break;
+        case "google":
+            if (config?.google?.apiKey) {
+                return new GoogleImageProvider(config.google);
+            }
+            break;
+        case "xai":
+            if (config?.xai?.apiKey) {
+                return new XAIImageProvider(config.xai);
+            }
+            break;
+    }
+    if (options.cloudMediaDisabled) {
+        throw new Error("No image provider configured and cloud media is disabled. " +
+            "Configure a direct provider (fal, openai, google, xai) or enable cloud media.");
+    }
+    return new ElizaCloudImageProvider(options.elizaCloudBaseUrl ?? "https://elizacloud.ai/api/v1", options.elizaCloudApiKey);
+}
+export function createVideoProvider(config, options) {
+    const mode = config?.mode ?? (options.cloudMediaDisabled ? "local" : "cloud");
+    const provider = mode === "cloud" ? "cloud" : (config?.provider ?? "cloud");
+    switch (provider) {
+        case "fal":
+            if (config?.fal?.apiKey) {
+                return new FalVideoProvider(config.fal);
+            }
+            break;
+        case "openai":
+            if (config?.openai?.apiKey) {
+                return new OpenAIVideoProvider(config.openai);
+            }
+            break;
+        case "google":
+            if (config?.google?.apiKey) {
+                return new GoogleVideoProvider(config.google);
+            }
+            break;
+    }
+    if (options.cloudMediaDisabled) {
+        throw new Error("No video provider configured and cloud media is disabled. " +
+            "Configure a direct provider (fal, openai, google) or enable cloud media.");
+    }
+    return new ElizaCloudVideoProvider(options.elizaCloudBaseUrl ?? "https://elizacloud.ai/api/v1", options.elizaCloudApiKey);
+}
+export function createAudioProvider(config, options) {
+    const mode = config?.mode ?? (options.cloudMediaDisabled ? "local" : "cloud");
+    const provider = mode === "cloud" ? "cloud" : (config?.provider ?? "cloud");
+    if (provider === "suno" && config?.suno?.apiKey) {
+        return new SunoAudioProvider(config.suno);
+    }
+    if (options.cloudMediaDisabled) {
+        throw new Error("No audio provider configured and cloud media is disabled. " +
+            "Configure a direct provider (suno) or enable cloud media.");
+    }
+    return new ElizaCloudAudioProvider(options.elizaCloudBaseUrl ?? "https://elizacloud.ai/api/v1", options.elizaCloudApiKey);
+}
+export function createVisionProvider(config, options) {
+    const mode = config?.mode ?? (options.cloudMediaDisabled ? "local" : "cloud");
+    const provider = mode === "cloud" ? "cloud" : (config?.provider ?? "cloud");
+    switch (provider) {
+        case "openai":
+            if (config?.openai?.apiKey) {
+                return new OpenAIVisionProvider(config.openai);
+            }
+            break;
+        case "google":
+            if (config?.google?.apiKey) {
+                return new GoogleVisionProvider(config.google);
+            }
+            break;
+        case "anthropic":
+            if (config?.anthropic?.apiKey) {
+                return new AnthropicVisionProvider(config.anthropic);
+            }
+            break;
+        case "xai":
+            if (config?.xai?.apiKey) {
+                return new XAIVisionProvider(config.xai);
+            }
+            break;
+        case "ollama":
+            // Ollama doesn't require an API key, just a base URL
+            return new OllamaVisionProvider(config?.ollama ?? {});
+    }
+    if (options.cloudMediaDisabled) {
+        throw new Error("No vision provider configured and cloud media is disabled. " +
+            "Configure a direct provider (openai, google, anthropic, xai, ollama) or enable cloud media.");
+    }
+    return new ElizaCloudVisionProvider(options.elizaCloudBaseUrl ?? "https://elizacloud.ai/api/v1", options.elizaCloudApiKey);
+}
+export function createMediaProviders(config, options) {
+    return {
+        image: createImageProvider(config?.image, options),
+        video: createVideoProvider(config?.video, options),
+        audio: createAudioProvider(config?.audio, options),
+        vision: createVisionProvider(config?.vision, options),
+    };
+}