noosphere 0.8.0 → 0.9.0

package/README.md

@@ -7,6 +7,8 @@ One import. Every model. Every modality.
 ## Features
 
 - **7 modalities** — LLM, image, video, TTS, STT, music, and embeddings
+- **OpenAI media** — GPT-Image-1/1.5, DALL-E 2/3, Sora 2/Pro (video), TTS-1/HD, Whisper — all auto-fetched from `OPENAI_API_KEY`
+- **Google media** — Imagen 4.0 (image), Veo 2/3/3.1 (video), Gemini TTS — all auto-fetched from `GEMINI_API_KEY`
 - **Always up-to-date models** — Dynamic auto-fetch from ALL provider APIs at runtime (OpenAI, Anthropic, Google, Groq, Mistral, xAI, Cerebras, OpenRouter)
 - **Dynamic descriptions** — Model descriptions fetched from source (Ollama library, HuggingFace READMEs, CivitAI API) — no hardcoded strings
 - **Modality-filtered sync** — `syncModels('llm')` only fetches LLM providers, avoiding unnecessary requests
@@ -38,13 +40,29 @@ const response = await ai.chat({
 });
 console.log(response.content);
 
-// Generate an image
+// Generate an image with GPT-Image-1 (OpenAI) — just needs OPENAI_API_KEY
 const image = await ai.image({
   prompt: 'A sunset over mountains',
+  provider: 'openai-media',
+});
+// image.buffer contains the PNG data
+
+// Generate an image with Imagen 4.0 (Google) — just needs GEMINI_API_KEY
+const googleImage = await ai.image({
+  prompt: 'A sunset over mountains',
+  provider: 'google-media',
+});
+// googleImage.buffer contains the PNG data
+
+// Generate an image with DALL-E 3
+const dalle = await ai.image({
+  prompt: 'A sunset over mountains',
+  provider: 'openai-media',
+  model: 'dall-e-3',
   width: 1024,
   height: 1024,
 });
-console.log(image.url);
+console.log(dalle.url);
 
 // Generate a video
 const video = await ai.video({
@@ -53,7 +71,7 @@ const video = await ai.video({
 });
 console.log(video.url);
 
-// Text-to-speech
+// Text-to-speech with OpenAI TTS — just needs OPENAI_API_KEY
 const audio = await ai.speak({
   text: 'Welcome to Noosphere',
   voice: 'alloy',
@@ -368,6 +386,8 @@ await ai.uninstallModel('deepseek-r1:14b');
 | Provider | Modality | Models | Source | Auto-Detect |
 |---|---|---|---|---|
 | **pi-ai** | LLM | 482 | OpenAI, Anthropic, Google, Groq, Mistral, xAI, OpenRouter, Cerebras | API keys |
+| **openai-media** | image, video, tts, stt | 12 | GPT-Image-1/1.5, DALL-E 2/3, Sora 2/Pro, TTS-1/HD, Whisper | `OPENAI_API_KEY` |
+| **google-media** | image, video, tts | 10 | Imagen 4.0, Veo 2/3/3.1, Gemini TTS (Flash/Pro) | `GEMINI_API_KEY` |
 | **ollama** | LLM, embedding | 70 | 38 installed + 32 from Ollama web catalog | `localhost:11434` |
 | **hf-local** | image, video, tts, stt, music | 220 | HuggingFace catalog (FLUX, SDXL, Wan2.2, Whisper, MusicGen) | Always (no API key) |
 | **huggingface** | LLM, image, tts | dynamic | HuggingFace Inference API | `HUGGINGFACE_TOKEN` |
@@ -400,11 +420,11 @@ await ai.syncModels();
 
 | Modality | Providers Synced |
 |---|---|
-| `llm` | pi-ai, ollama, openai-compat, huggingface (cloud, needs API key) |
-| `image` | hf-local, comfyui, fal, huggingface (cloud) |
-| `video` | hf-local, comfyui, fal |
-| `tts` | hf-local (speech models), fal, piper, kokoro, huggingface (cloud) |
-| `stt` | hf-local, whisper-local |
+| `llm` | pi-ai, ollama, openai-compat, huggingface (cloud) |
+| `image` | **openai-media** (GPT-Image-1, DALL-E), **google-media** (Imagen 4.0), hf-local, comfyui, fal, huggingface (cloud) |
+| `video` | **openai-media** (Sora 2/Pro), **google-media** (Veo 2/3/3.1), hf-local, comfyui, fal |
+| `tts` | **openai-media** (TTS-1, TTS-1-HD), **google-media** (Gemini TTS), hf-local, fal, piper, kokoro, huggingface (cloud) |
+| `stt` | **openai-media** (Whisper), hf-local, whisper-local |
 | `music` | hf-local (MusicGen, AudioLDM, etc.), audiocraft |
 | `embedding` | ollama |
 

package/dist/index.cjs

@@ -21,11 +21,13 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 var index_exports = {};
 __export(index_exports, {
   AudioCraftProvider: () => AudioCraftProvider,
+  GoogleMediaProvider: () => GoogleMediaProvider,
   HfLocalProvider: () => HfLocalProvider,
   Noosphere: () => Noosphere,
   NoosphereError: () => NoosphereError,
   OllamaProvider: () => OllamaProvider,
   OpenAICompatProvider: () => OpenAICompatProvider,
+  OpenAIMediaProvider: () => OpenAIMediaProvider,
   PROVIDER_IDS: () => PROVIDER_IDS,
   PROVIDER_LOGOS: () => PROVIDER_LOGOS,
   WhisperLocalProvider: () => WhisperLocalProvider,
@@ -2538,6 +2540,501 @@ async function detectOpenAICompatServers() {
   return providers;
 }
 
+// src/providers/openai-media.ts
+var OPENAI_API_BASE = "https://api.openai.com/v1";
+var FETCH_TIMEOUT_MS5 = 8e3;
+var MODEL_PREFIX_MAP = [
+  { prefix: "dall-e-", modality: "image" },
+  { prefix: "gpt-image-", modality: "image" },
+  { prefix: "sora-", modality: "video" },
+  { prefix: "tts-", modality: "tts" },
+  { prefix: "whisper-", modality: "stt" }
+];
+function classifyModel(id) {
+  for (const { prefix, modality } of MODEL_PREFIX_MAP) {
+    if (id.startsWith(prefix)) return modality;
+  }
+  return null;
+}
+var OpenAIMediaProvider = class {
+  constructor(apiKey) {
+    this.apiKey = apiKey;
+  }
+  id = "openai-media";
+  name = "OpenAI (Image, Video, TTS, STT)";
+  modalities = ["image", "video", "tts", "stt"];
+  isLocal = false;
+  modelsCache = null;
+  async ping() {
+    try {
+      const controller = new AbortController();
+      const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS5);
+      try {
+        const res = await fetch(`${OPENAI_API_BASE}/models`, {
+          headers: { Authorization: `Bearer ${this.apiKey}` },
+          signal: controller.signal
+        });
+        return res.ok;
+      } finally {
+        clearTimeout(timer);
+      }
+    } catch {
+      return false;
+    }
+  }
+  async listModels(modality) {
+    if (this.modelsCache) {
+      return modality ? this.modelsCache.filter((m) => m.modality === modality) : this.modelsCache;
+    }
+    try {
+      const controller = new AbortController();
+      const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS5);
+      let data;
+      try {
+        const res = await fetch(`${OPENAI_API_BASE}/models`, {
+          headers: { Authorization: `Bearer ${this.apiKey}` },
+          signal: controller.signal
+        });
+        if (!res.ok) return [];
+        data = await res.json();
+      } finally {
+        clearTimeout(timer);
+      }
+      const entries = data?.data ?? [];
+      const logo = getProviderLogo("openai");
+      const models = [];
+      for (const entry of entries) {
+        const mod = classifyModel(entry.id);
+        if (!mod) continue;
+        const info = {
+          id: entry.id,
+          provider: "openai-media",
+          name: entry.id,
+          modality: mod,
+          local: false,
+          cost: { price: 0, unit: "per_request" },
+          logo,
+          description: entry.description,
+          capabilities: this.getCapabilities(entry.id, mod)
+        };
+        models.push(info);
+      }
+      this.modelsCache = models;
+      return modality ? models.filter((m) => m.modality === modality) : models;
+    } catch {
+      return [];
+    }
+  }
+  async image(options) {
+    const model = options.model ?? "gpt-image-1";
+    const width = options.width ?? 1024;
+    const height = options.height ?? 1024;
+    const start = Date.now();
+    const isGptImage = model.startsWith("gpt-image-");
+    const body = {
+      model,
+      prompt: options.prompt,
+      n: 1,
+      size: `${width}x${height}`
+    };
+    if (!isGptImage) {
+      body.response_format = "url";
+    }
+    const res = await fetch(`${OPENAI_API_BASE}/images/generations`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${this.apiKey}`
+      },
+      body: JSON.stringify(body)
+    });
+    if (!res.ok) {
+      const errorBody = await res.text();
+      throw new Error(`OpenAI image generation failed (${res.status}): ${errorBody}`);
+    }
+    const data = await res.json();
+    const item = data?.data?.[0];
+    const result = {
+      provider: "openai-media",
+      model,
+      modality: "image",
+      latencyMs: Date.now() - start,
+      usage: {
+        cost: 0,
+        unit: "per_image"
+      },
+      media: {
+        width,
+        height,
+        format: "png"
+      }
+    };
+    if (item?.b64_json) {
+      result.buffer = Buffer.from(item.b64_json, "base64");
+    } else if (item?.url) {
+      result.url = item.url;
+    }
+    return result;
+  }
+  async speak(options) {
+    const model = options.model ?? "tts-1";
+    const voice = options.voice ?? "alloy";
+    const format = options.format ?? "mp3";
+    const speed = options.speed ?? 1;
+    const start = Date.now();
+    const body = {
+      model,
+      input: options.text,
+      voice,
+      response_format: format,
+      speed
+    };
+    const res = await fetch(`${OPENAI_API_BASE}/audio/speech`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${this.apiKey}`
+      },
+      body: JSON.stringify(body)
+    });
+    if (!res.ok) {
+      const errorBody = await res.text();
+      throw new Error(`OpenAI TTS failed (${res.status}): ${errorBody}`);
+    }
+    const arrayBuffer = await res.arrayBuffer();
+    const buffer = Buffer.from(arrayBuffer);
+    return {
+      buffer,
+      provider: "openai-media",
+      model,
+      modality: "tts",
+      latencyMs: Date.now() - start,
+      usage: {
+        cost: 0,
+        input: options.text.length,
+        unit: "per_1k_chars"
+      },
+      media: {
+        format
+      }
+    };
+  }
+  async video(options) {
+    const model = options.model ?? "sora-2";
+    const start = Date.now();
+    const body = {
+      model,
+      prompt: options.prompt,
+      n: 1
+    };
+    if (options.duration) body.duration = options.duration;
+    if (options.width && options.height) body.size = `${options.width}x${options.height}`;
+    const res = await fetch(`${OPENAI_API_BASE}/videos/generations`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${this.apiKey}`
+      },
+      body: JSON.stringify(body)
+    });
+    if (!res.ok) {
+      const errorBody = await res.text();
+      throw new Error(`OpenAI video generation failed (${res.status}): ${errorBody}`);
+    }
+    const data = await res.json();
+    const videoUrl = data?.data?.[0]?.url;
+    return {
+      url: videoUrl,
+      provider: "openai-media",
+      model,
+      modality: "video",
+      latencyMs: Date.now() - start,
+      usage: {
+        cost: 0,
+        unit: "per_video"
+      },
+      media: {
+        duration: options.duration,
+        width: options.width,
+        height: options.height
+      }
+    };
+  }
+  getCapabilities(id, modality) {
+    if (modality === "image") {
+      return {
+        maxWidth: id.startsWith("dall-e-3") ? 1792 : 1024,
+        maxHeight: id.startsWith("dall-e-3") ? 1792 : 1024
+      };
+    }
+    if (modality === "tts") {
+      return {
+        voices: ["alloy", "ash", "coral", "echo", "fable", "onyx", "nova", "sage", "shimmer"]
+      };
+    }
+    if (modality === "video") {
+      return {
+        maxDuration: id.includes("pro") ? 20 : 10,
+        supportsStreaming: false
+      };
+    }
+    if (modality === "stt") {
+      return {
+        languages: ["en", "zh", "de", "es", "ru", "ko", "fr", "ja", "pt", "tr", "pl", "ca", "nl", "ar", "sv", "it", "id", "hi", "fi", "vi", "he", "uk", "el", "ms", "cs", "ro", "da", "hu", "ta", "no", "th", "ur", "hr", "bg", "lt", "la", "mi", "ml", "cy", "sk", "te", "fa", "lv", "bn", "sr", "az", "sl", "kn", "et", "mk", "br", "eu", "is", "hy", "ne", "mn", "bs", "kk", "sq", "sw", "gl", "mr", "pa", "si", "km", "sn", "yo", "so", "af", "oc", "ka", "be", "tg", "sd", "gu", "am", "yi", "lo", "uz", "fo", "ht", "ps", "tk", "nn", "mt", "sa", "lb", "my", "bo", "tl", "mg", "as", "tt", "haw", "ln", "ha", "ba", "jw", "su"]
+      };
+    }
+    return void 0;
+  }
+};
+
+// src/providers/google-media.ts
+var GOOGLE_API_BASE = "https://generativelanguage.googleapis.com/v1beta";
+var FETCH_TIMEOUT_MS6 = 8e3;
+var GOOGLE_TTS_VOICES = [
+  "Aoede",
+  "Charon",
+  "Fenrir",
+  "Kore",
+  "Puck",
+  "Leda",
+  "Orus",
+  "Perseus",
+  "Zephyr",
+  "Callirrhoe"
+];
+function classifyGoogleModel(model) {
+  const name = (model.name ?? "").replace("models/", "");
+  const methods = model.supportedGenerationMethods ?? [];
+  if (name.startsWith("imagen") && methods.includes("predict")) return "image";
+  if (name.startsWith("veo") && methods.includes("predictLongRunning")) return "video";
+  if (name.includes("-tts") && methods.includes("generateContent")) return "tts";
+  return null;
+}
+var GoogleMediaProvider = class {
+  constructor(apiKey) {
+    this.apiKey = apiKey;
+  }
+  id = "google-media";
+  name = "Google (Image, Video, TTS)";
+  modalities = ["image", "video", "tts"];
+  isLocal = false;
+  modelsCache = null;
+  async ping() {
+    try {
+      const controller = new AbortController();
+      const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS6);
+      try {
+        const res = await fetch(`${GOOGLE_API_BASE}/models?key=${this.apiKey}`, {
+          signal: controller.signal
+        });
+        return res.ok;
+      } finally {
+        clearTimeout(timer);
+      }
+    } catch {
+      return false;
+    }
+  }
+  async listModels(modality) {
+    if (this.modelsCache) {
+      return modality ? this.modelsCache.filter((m) => m.modality === modality) : this.modelsCache;
+    }
+    try {
+      const controller = new AbortController();
+      const timer = setTimeout(() => controller.abort(), FETCH_TIMEOUT_MS6);
+      let data;
+      try {
+        const res = await fetch(`${GOOGLE_API_BASE}/models?key=${this.apiKey}`, {
+          signal: controller.signal
+        });
+        if (!res.ok) return [];
+        data = await res.json();
+      } finally {
+        clearTimeout(timer);
+      }
+      const entries = data?.models ?? [];
+      const logo = getProviderLogo("google");
+      const models = [];
+      for (const entry of entries) {
+        const modality2 = classifyGoogleModel(entry);
+        if (!modality2) continue;
+        const fullName = entry.name ?? "";
+        const modelId = fullName.startsWith("models/") ? fullName.slice("models/".length) : fullName;
+        const info = {
+          id: modelId,
+          provider: "google-media",
+          name: entry.displayName ?? modelId,
+          modality: modality2,
+          local: false,
+          cost: { price: 0, unit: modality2 === "video" ? "per_video" : "per_image" },
+          logo,
+          description: entry.description,
+          capabilities: modality2 === "video" ? { maxDuration: 8, supportsStreaming: false } : modality2 === "tts" ? { voices: GOOGLE_TTS_VOICES } : void 0
+        };
+        models.push(info);
+      }
+      this.modelsCache = models;
+      return modality ? models.filter((m) => m.modality === modality) : models;
+    } catch {
+      return [];
+    }
+  }
+  async image(options) {
+    const model = options.model ?? "imagen-4.0-generate-001";
+    const start = Date.now();
+    const body = {
+      instances: [{ prompt: options.prompt }],
+      parameters: {
+        sampleCount: 1
+      }
+    };
+    const res = await fetch(
+      `${GOOGLE_API_BASE}/models/${model}:predict?key=${this.apiKey}`,
+      {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify(body)
+      }
+    );
+    if (!res.ok) {
+      const errorBody = await res.text();
+      throw new Error(`Google image generation failed (${res.status}): ${errorBody}`);
+    }
+    const data = await res.json();
+    const base64 = data?.predictions?.[0]?.bytesBase64Encoded ?? data?.generatedImages?.[0]?.image?.imageBytes;
+    if (!base64) {
+      throw new Error("Google image generation returned no image data");
+    }
+    const buffer = Buffer.from(base64, "base64");
+    return {
+      buffer,
+      provider: "google-media",
+      model,
+      modality: "image",
+      latencyMs: Date.now() - start,
+      usage: {
+        cost: 0,
+        unit: "per_image"
+      },
+      media: {
+        format: "png"
+      }
+    };
+  }
+  async speak(options) {
+    const model = options.model ?? "gemini-2.5-flash-preview-tts";
+    const voice = options.voice ?? "Kore";
+    const start = Date.now();
+    const body = {
+      contents: [{ parts: [{ text: options.text }] }],
+      generationConfig: {
+        response_modalities: ["AUDIO"],
+        speech_config: {
+          voiceConfig: {
+            prebuiltVoiceConfig: { voiceName: voice }
+          }
+        }
+      }
+    };
+    const res = await fetch(
+      `${GOOGLE_API_BASE}/models/${model}:generateContent?key=${this.apiKey}`,
+      {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify(body)
+      }
+    );
+    if (!res.ok) {
+      const errorBody = await res.text();
+      throw new Error(`Google TTS failed (${res.status}): ${errorBody}`);
+    }
+    const data = await res.json();
+    const inlineData = data?.candidates?.[0]?.content?.parts?.[0]?.inlineData;
+    if (!inlineData?.data) {
+      throw new Error("Google TTS returned no audio data");
+    }
+    const buffer = Buffer.from(inlineData.data, "base64");
+    return {
+      buffer,
+      provider: "google-media",
+      model,
+      modality: "tts",
+      latencyMs: Date.now() - start,
+      usage: {
+        cost: 0,
+        input: options.text.length,
+        unit: "per_1k_chars"
+      },
+      media: {
+        format: "wav"
+        // Google returns PCM L16, essentially WAV
+      }
+    };
+  }
+  async video(options) {
+    const model = options.model ?? "veo-3.0-generate-001";
+    const start = Date.now();
+    const body = {
+      instances: [{ prompt: options.prompt }],
+      parameters: {
+        sampleCount: 1
+      }
+    };
+    if (options.duration) body.parameters.durationSeconds = options.duration;
+    const res = await fetch(
+      `${GOOGLE_API_BASE}/models/${model}:predictLongRunning?key=${this.apiKey}`,
+      {
+        method: "POST",
+        headers: { "Content-Type": "application/json" },
+        body: JSON.stringify(body)
+      }
+    );
+    if (!res.ok) {
+      const errorBody = await res.text();
+      throw new Error(`Google video generation failed (${res.status}): ${errorBody}`);
+    }
+    const operation = await res.json();
+    const operationName = operation?.name;
+    if (!operationName) {
+      throw new Error("Google video generation returned no operation name");
+    }
+    const deadline = Date.now() + 3e5;
+    while (Date.now() < deadline) {
+      await new Promise((r) => setTimeout(r, 5e3));
+      const pollRes = await fetch(
+        `${GOOGLE_API_BASE}/${operationName}?key=${this.apiKey}`
+      );
+      if (!pollRes.ok) continue;
+      const status = await pollRes.json();
+      if (status.done) {
+        const videoBase64 = status.response?.generatedSamples?.[0]?.video?.bytesBase64Encoded;
+        if (videoBase64) {
+          return {
+            buffer: Buffer.from(videoBase64, "base64"),
+            provider: "google-media",
+            model,
+            modality: "video",
+            latencyMs: Date.now() - start,
+            usage: { cost: 0, unit: "per_video" },
+            media: { format: "mp4", duration: options.duration }
+          };
+        }
+        const videoUrl = status.response?.generatedSamples?.[0]?.video?.uri;
+        return {
+          url: videoUrl,
+          provider: "google-media",
+          model,
+          modality: "video",
+          latencyMs: Date.now() - start,
+          usage: { cost: 0, unit: "per_video" },
+          media: { format: "mp4", duration: options.duration }
+        };
+      }
+    }
+    throw new Error(`Google video generation timed out after 5 minutes`);
+  }
+};
+
 // src/noosphere.ts
 var Noosphere = class {
   config;
@@ -2780,6 +3277,12 @@ var Noosphere = class {
     if (hasAnyLLMKey) {
       this.registry.addProvider(new PiAiProvider(llmKeys));
     }
+    if (keys.openai) {
+      this.registry.addProvider(new OpenAIMediaProvider(keys.openai));
+    }
+    if (keys.google) {
+      this.registry.addProvider(new GoogleMediaProvider(keys.google));
+    }
     if (keys.fal) {
       this.registry.addProvider(new FalProvider(keys.fal));
     }
@@ -2959,11 +3462,13 @@ var Noosphere = class {
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   AudioCraftProvider,
+  GoogleMediaProvider,
   HfLocalProvider,
   Noosphere,
   NoosphereError,
   OllamaProvider,
   OpenAICompatProvider,
+  OpenAIMediaProvider,
   PROVIDER_IDS,
   PROVIDER_LOGOS,
   WhisperLocalProvider,