noosphere 0.7.0 → 0.8.0

package/dist/index.d.cts

@@ -92,6 +92,7 @@ interface ModelInfo {
     id: string;
     provider: string;
     name: string;
+    description?: string;
     modality: Modality;
     local: boolean;
     cost: {
@@ -317,7 +318,7 @@ declare class Noosphere {
     getProviders(modality?: Modality): Promise<ProviderInfo[]>;
     getModels(modality?: Modality): Promise<ModelInfo[]>;
     getModel(provider: string, modelId: string): Promise<ModelInfo | null>;
-    syncModels(): Promise<SyncResult>;
+    syncModels(modality?: Modality): Promise<SyncResult>;
     getUsage(options?: UsageQueryOptions): UsageSummary;
     installModel(name: string): Promise<AsyncGenerator<OllamaPullProgress>>;
     uninstallModel(name: string): Promise<void>;

package/dist/index.d.ts

@@ -92,6 +92,7 @@ interface ModelInfo {
     id: string;
     provider: string;
     name: string;
+    description?: string;
     modality: Modality;
     local: boolean;
     cost: {
@@ -317,7 +318,7 @@ declare class Noosphere {
     getProviders(modality?: Modality): Promise<ProviderInfo[]>;
     getModels(modality?: Modality): Promise<ModelInfo[]>;
     getModel(provider: string, modelId: string): Promise<ModelInfo | null>;
-    syncModels(): Promise<SyncResult>;
+    syncModels(modality?: Modality): Promise<SyncResult>;
     getUsage(options?: UsageQueryOptions): UsageSummary;
     installModel(name: string): Promise<AsyncGenerator<OllamaPullProgress>>;
     uninstallModel(name: string): Promise<void>;

package/dist/index.js

@@ -312,20 +312,22 @@ var Registry = class {
     const cached = this.modelCache.get(provider);
     return cached?.models.find((m) => m.id === modelId) ?? null;
   }
-  async syncProvider(providerId) {
+  async syncProvider(providerId, modality) {
     const provider = this.providers.get(providerId);
     if (!provider) return 0;
-    const models = await provider.listModels();
+    if (modality && !provider.modalities.includes(modality)) return 0;
+    const models = await provider.listModels(modality);
     this.modelCache.set(providerId, { models, syncedAt: Date.now() });
     return models.length;
   }
-  async syncAll() {
+  async syncAll(modality) {
     const byProvider = {};
     const errors = [];
     let synced = 0;
     for (const provider of this.providers.values()) {
+      if (modality && !provider.modalities.includes(modality)) continue;
       try {
-        const count = await this.syncProvider(provider.id);
+        const count = await this.syncProvider(provider.id, modality);
         byProvider[provider.id] = count;
         synced += count;
       } catch (err) {
@@ -1110,6 +1112,7 @@ var ComfyUIProvider = class {
                 id: `civitai-${item.id}`,
                 provider: "comfyui",
                 name: item.name ?? `CivitAI Model ${item.id}`,
+                description: item.description ? item.description.replace(/<[^>]+>/g, "").trim().slice(0, 300) || void 0 : void 0,
                 modality: "image",
                 local: true,
                 cost: { price: 0, unit: "free" },
@@ -1548,6 +1551,28 @@ async function fetchJson(url, options) {
     clearTimeout(timer);
   }
 }
+async function fetchOllamaDescriptions() {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), 5e3);
+  try {
+    const res = await fetch("https://ollama.com/library", { signal: controller.signal });
+    if (!res.ok) throw new Error(`HTTP ${res.status}`);
+    const html = await res.text();
+    const descriptions = /* @__PURE__ */ new Map();
+    const cardRegex = /href="\/library\/([^"]+)"[\s\S]*?<p[^>]*>([\s\S]*?)<\/p>/g;
+    let match;
+    while ((match = cardRegex.exec(html)) !== null) {
+      const modelName = match[1].trim();
+      const desc = match[2].replace(/<[^>]*>/g, "").replace(/&amp;/g, "&").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&quot;/g, '"').replace(/&#39;/g, "'").replace(/\s+/g, " ").trim();
+      if (modelName && desc) {
+        descriptions.set(modelName, desc);
+      }
+    }
+    return descriptions;
+  } finally {
+    clearTimeout(timer);
+  }
+}
 var OllamaProvider = class {
   id = "ollama";
   name = "Ollama (Local)";
@@ -1571,10 +1596,11 @@ var OllamaProvider = class {
   }
   async listModels(_modality) {
     if (_modality && _modality !== "llm") return [];
-    const [localData, catalogData, runningData] = await Promise.all([
+    const [localData, catalogData, runningData, descriptions] = await Promise.all([
       fetchJson(`${this.baseUrl}/api/tags`, { timeoutMs: 5e3 }).catch(() => null),
       fetchJson("https://ollama.com/api/tags", { timeoutMs: 5e3 }).catch(() => null),
-      fetchJson(`${this.baseUrl}/api/ps`, { timeoutMs: 5e3 }).catch(() => null)
+      fetchJson(`${this.baseUrl}/api/ps`, { timeoutMs: 5e3 }).catch(() => null),
+      fetchOllamaDescriptions().catch(() => /* @__PURE__ */ new Map())
     ]);
     const runningNames = /* @__PURE__ */ new Set();
     if (runningData?.models) {
@@ -1587,27 +1613,30 @@ var OllamaProvider = class {
     if (localData?.models) {
       for (const m of localData.models) {
         const isRunning = runningNames.has(m.name) || runningNames.has(m.model);
-        models.set(m.name, this.toModelInfo(m, isRunning ? "running" : "installed", true));
+        models.set(m.name, this.toModelInfo(m, isRunning ? "running" : "installed", true, descriptions));
       }
     }
     if (catalogData?.models) {
       for (const m of catalogData.models) {
         const name = m.name;
         if (!models.has(name)) {
-          models.set(name, this.toModelInfo(m, "available", false));
+          models.set(name, this.toModelInfo(m, "available", false, descriptions));
         }
       }
     }
     return Array.from(models.values());
   }
-  toModelInfo(m, status, isLocal) {
+  toModelInfo(m, status, isLocal, descriptions) {
     const name = m.name ?? m.model ?? "unknown";
     const family = m.details?.family;
     const logoProvider = inferLogoProvider(name, family);
+    const baseName = name.split(":")[0];
+    const description = descriptions?.get(baseName);
     return {
       id: name,
       provider: "ollama",
       name,
+      ...description ? { description } : {},
       modality: "llm",
       local: true,
       cost: { price: 0, unit: "free" },
@@ -1816,16 +1845,79 @@ var OllamaProvider = class {
   }
 };
 
+// src/utils/parse-readme.ts
+async function fetchReadmeDescription(modelId, timeoutMs = 5e3) {
+  try {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeoutMs);
+    try {
+      const res = await fetch(
+        `https://huggingface.co/${modelId}/raw/main/README.md`,
+        { signal: controller.signal }
+      );
+      if (!res.ok) return void 0;
+      const text = await res.text();
+      return parseReadmeDescription(text);
+    } finally {
+      clearTimeout(timer);
+    }
+  } catch {
+    return void 0;
+  }
+}
+function parseReadmeDescription(readme) {
+  const withoutFrontmatter = readme.replace(/^---[\s\S]*?---\s*/, "");
+  const lines = withoutFrontmatter.split("\n");
+  let paragraph = "";
+  for (const line of lines) {
+    const trimmed = line.trim();
+    if (!trimmed) {
+      if (paragraph) break;
+      continue;
+    }
+    if (trimmed.startsWith("#")) {
+      if (paragraph) break;
+      continue;
+    }
+    if (/^\[?!\[/.test(trimmed) || /^</.test(trimmed)) continue;
+    if (/^\[.*\]\(.*\)$/.test(trimmed)) continue;
+    paragraph += (paragraph ? " " : "") + trimmed;
+  }
+  if (!paragraph) return void 0;
+  paragraph = paragraph.replace(/\[([^\]]+)\]\([^)]+\)/g, "$1").replace(/\*\*([^*]+)\*\*/g, "$1").replace(/`([^`]+)`/g, "$1").replace(/<[^>]+>/g, "").trim();
+  if (!paragraph) return void 0;
+  if (paragraph.length > 300) paragraph = paragraph.slice(0, 297) + "...";
+  return paragraph;
+}
+
 // src/providers/hf-local.ts
 import { readdir, readFile } from "fs/promises";
 import { join } from "path";
 import { homedir } from "os";
 var FETCH_TIMEOUT_MS3 = 5e3;
 var HF_HUB_API2 = "https://huggingface.co/api/models";
+var HF_ORG_TO_LOGO_PROVIDER = {
+  "meta-llama": "meta",
+  "facebook": "meta",
+  "google": "google",
+  "microsoft": "microsoft",
+  "nvidia": "nvidia",
+  "mistralai": "mistral",
+  "Qwen": "qwen",
+  "deepseek-ai": "deepseek",
+  "openai": "openai",
+  "CohereForAI": "cohere",
+  "rhasspy": "piper",
+  "stabilityai": "huggingface",
+  "black-forest-labs": "huggingface",
+  "tiiuae": "huggingface",
+  "allenai": "huggingface",
+  "Salesforce": "huggingface"
+};
 var PIPELINE_TAG_TO_MODALITY = {
   "text-to-image": "image",
   "text-to-video": "video",
-  "text-to-audio": "tts",
+  "text-to-audio": "music",
   "text-to-speech": "tts",
   "automatic-speech-recognition": "stt"
 };
@@ -1853,7 +1945,7 @@ async function fetchJsonTimeout(url, timeoutMs = FETCH_TIMEOUT_MS3) {
 var HfLocalProvider = class {
   id = "hf-local";
   name = "HuggingFace Local Models";
-  modalities = ["image", "video", "tts", "stt"];
+  modalities = ["image", "video", "tts", "stt", "music"];
   isLocal = true;
   cachedModels = null;
   async ping() {
@@ -1875,8 +1967,7 @@ var HfLocalProvider = class {
   }
   async fetchCatalog() {
     const seen = /* @__PURE__ */ new Set();
-    const models = [];
-    const logo = getProviderLogo("huggingface");
+    const entries = [];
     const results = await Promise.allSettled(
       CATALOG_QUERIES.map(async (q) => {
         const params = new URLSearchParams({
@@ -1894,32 +1985,56 @@ var HfLocalProvider = class {
         const id = entry.id ?? entry.modelId;
         if (!id || seen.has(id)) continue;
         seen.add(id);
-        const pipelineTag = entry.pipeline_tag ?? "";
-        const modality = PIPELINE_TAG_TO_MODALITY[pipelineTag] ?? "image";
-        models.push({
+        entries.push({
           id,
-          provider: "hf-local",
-          name: id.split("/").pop() ?? id,
-          modality,
-          local: true,
-          cost: { price: 0, unit: "free" },
-          logo,
-          status: "available",
-          localInfo: {
-            sizeBytes: 0,
-            runtime: "huggingface",
-            family: entry.library_name
-          },
-          capabilities: {}
+          pipelineTag: entry.pipeline_tag ?? "",
+          libraryName: entry.library_name
         });
       }
     }
+    const descriptionMap = /* @__PURE__ */ new Map();
+    for (let i = 0; i < entries.length; i += 10) {
+      const batch = entries.slice(i, i + 10);
+      const descs = await Promise.allSettled(
+        batch.map(async (e) => {
+          const desc = await fetchReadmeDescription(e.id);
+          return { id: e.id, desc };
+        })
+      );
+      for (const d of descs) {
+        if (d.status === "fulfilled" && d.value.desc) {
+          descriptionMap.set(d.value.id, d.value.desc);
+        }
+      }
+    }
+    const models = [];
+    for (const e of entries) {
+      const modality = PIPELINE_TAG_TO_MODALITY[e.pipelineTag] ?? "image";
+      const org = e.id.includes("/") ? e.id.split("/")[0] : void 0;
+      const logoProvider = org ? HF_ORG_TO_LOGO_PROVIDER[org] ?? "huggingface" : "huggingface";
+      models.push({
+        id: e.id,
+        provider: "hf-local",
+        name: e.id.split("/").pop() ?? e.id,
+        modality,
+        local: true,
+        cost: { price: 0, unit: "free" },
+        logo: getProviderLogo(logoProvider),
+        description: descriptionMap.get(e.id),
+        status: "available",
+        localInfo: {
+          sizeBytes: 0,
+          runtime: "huggingface",
+          family: e.libraryName
+        },
+        capabilities: {}
+      });
+    }
     return models;
   }
   async scanLocalCache() {
     const models = [];
     const cacheDir = join(homedir(), ".cache", "huggingface", "hub");
-    const logo = getProviderLogo("huggingface");
     try {
       const entries = await readdir(cacheDir, { withFileTypes: true });
       for (const entry of entries) {
@@ -1951,6 +2066,8 @@ var HfLocalProvider = class {
           }
         }
         const modality = PIPELINE_TAG_TO_MODALITY[pipelineTag] ?? "image";
+        const org = modelId.includes("/") ? modelId.split("/")[0] : void 0;
+        const logoProvider = org ? HF_ORG_TO_LOGO_PROVIDER[org] ?? "huggingface" : "huggingface";
         models.push({
           id: modelId,
           provider: "hf-local",
@@ -1958,7 +2075,7 @@ var HfLocalProvider = class {
           modality,
           local: true,
           cost: { price: 0, unit: "free" },
-          logo,
+          logo: getProviderLogo(logoProvider),
           status: "installed",
           localInfo: {
             sizeBytes: 0,
@@ -1979,6 +2096,25 @@ import { access } from "fs/promises";
 import { join as join2 } from "path";
 import { homedir as homedir2 } from "os";
 var WHISPER_MODELS = ["tiny", "base", "small", "medium", "large", "large-v2", "large-v3", "turbo"];
+var WHISPER_HF_REPOS = {
+  "tiny": "openai/whisper-tiny",
+  "base": "openai/whisper-base",
+  "small": "openai/whisper-small",
+  "medium": "openai/whisper-medium",
+  "large": "openai/whisper-large",
+  "large-v2": "openai/whisper-large-v2",
+  "large-v3": "openai/whisper-large-v3",
+  "turbo": "openai/whisper-large-v3-turbo"
+};
+async function fetchWhisperDescriptions() {
+  const descriptions = /* @__PURE__ */ new Map();
+  const fetches = Object.entries(WHISPER_HF_REPOS).map(async ([size, repo]) => {
+    const desc = await fetchReadmeDescription(repo, 8e3);
+    if (desc) descriptions.set(size, desc);
+  });
+  await Promise.allSettled(fetches);
+  return descriptions;
+}
 function runPython(code, timeoutMs = 5e3) {
   return new Promise((resolve, reject) => {
     const proc = execFile("python3", ["-c", code], { timeout: timeoutMs }, (err, stdout) => {
@@ -2024,7 +2160,8 @@ var WhisperLocalProvider = class {
     if (_modality && _modality !== "stt") return [];
     const runtime = await this.detectRuntime();
     if (!runtime) return [];
-    const logo = getProviderLogo("huggingface");
+    const descMap = await fetchWhisperDescriptions().catch(() => /* @__PURE__ */ new Map());
+    const logo = getProviderLogo("openai");
     const models = [];
     for (const name of WHISPER_MODELS) {
       const installed = await this.isModelCached(name, runtime);
@@ -2032,6 +2169,7 @@ var WhisperLocalProvider = class {
         id: `whisper-${name}`,
         provider: "whisper-local",
         name: `Whisper ${name}`,
+        description: descMap.get(name),
         modality: "stt",
         local: true,
         cost: { price: 0, unit: "free" },
@@ -2095,6 +2233,22 @@ var AUDIOCRAFT_MODELS = [
   { id: "musicgen-melody", name: "MusicGen Melody" },
   { id: "audiogen-medium", name: "AudioGen Medium" }
 ];
+var AUDIOCRAFT_HF_REPOS = {
+  "musicgen-small": "facebook/musicgen-small",
+  "musicgen-medium": "facebook/musicgen-medium",
+  "musicgen-large": "facebook/musicgen-large",
+  "musicgen-melody": "facebook/musicgen-melody",
+  "audiogen-medium": "facebook/audiogen-medium"
+};
+async function fetchAudioCraftDescriptions() {
+  const descriptions = /* @__PURE__ */ new Map();
+  const fetches = Object.entries(AUDIOCRAFT_HF_REPOS).map(async ([id, repo]) => {
+    const desc = await fetchReadmeDescription(repo, 8e3);
+    if (desc) descriptions.set(id, desc);
+  });
+  await Promise.allSettled(fetches);
+  return descriptions;
+}
 function runPython2(code, timeoutMs = 5e3) {
   return new Promise((resolve, reject) => {
     execFile2("python3", ["-c", code], { timeout: timeoutMs }, (err, stdout) => {
@@ -2130,6 +2284,7 @@ var AudioCraftProvider = class {
   async listModels(_modality) {
     if (_modality && _modality !== "music") return [];
     if (!await this.ping()) return [];
+    const descMap = await fetchAudioCraftDescriptions().catch(() => /* @__PURE__ */ new Map());
     const logo = getProviderLogo("meta");
     const models = [];
     for (const m of AUDIOCRAFT_MODELS) {
@@ -2139,6 +2294,7 @@ var AudioCraftProvider = class {
         id: m.id,
         provider: "audiocraft",
         name: m.name,
+        description: descMap.get(m.id),
         modality: "music",
         local: true,
         cost: { price: 0, unit: "free" },
@@ -2534,9 +2690,9 @@ var Noosphere = class {
     if (!this.initialized) await this.init();
     return this.registry.getModel(provider, modelId);
   }
-  async syncModels() {
+  async syncModels(modality) {
     if (!this.initialized) await this.init();
-    return this.registry.syncAll();
+    return this.registry.syncAll(modality);
   }
   // --- Tracking Methods ---
   getUsage(options) {