@webmcp-auto-ui/agent 2.5.26 → 2.5.27

package/src/types.ts

@@ -6,11 +6,22 @@ export type { Recipe, McpRecipe } from './recipes/types.js';
 // Short model IDs for remote (Anthropic-compatible) providers
 export type RemoteModelId = 'haiku' | 'sonnet' | 'opus' | string;
 
-// Model IDs for in-browser WASM providers
+// Model IDs for in-browser WASM providers (MediaPipe/LiteRT)
 export type WasmModelId = 'gemma-e2b' | 'gemma-e4b' | string;
 
+// Model IDs for in-browser transformers.js providers (ONNX + WebGPU)
+// Canonical list in ./providers/transformers-models.ts.
+export type TransformersModelId =
+  | 'transformers-gemma-4-e2b'
+  | 'transformers-gemma-4-e4b'
+  | 'transformers-qwen-3-4b'
+  | 'transformers-qwen-3.5-2b'
+  | 'transformers-qwen-3.5-4b'
+  | 'transformers-ministral-3-3b'
+  | string;
+
 // Union of all LLM IDs used by canvas.llm and LLMSelector
-export type LLMId = RemoteModelId | WasmModelId;
+export type LLMId = RemoteModelId | WasmModelId | TransformersModelId;
 
 // Backward compat alias
 export type ModelId = LLMId;
@@ -22,6 +33,7 @@ export interface ChatMessage {
 
 export type ContentBlock =
   | { type: 'text'; text: string }
+  | { type: 'image'; data: string; mediaType: string }
   | { type: 'tool_use'; id: string; name: string; input: Record<string, unknown> }
   | { type: 'tool_result'; tool_use_id: string; content: string };
 
@@ -50,9 +62,9 @@ export interface LLMProvider {
   readonly model: string;
   /** Hint for system prompt builders: which syntax this provider expects for tool
    *  references. `undefined` → treated as `'generic'`. Providers using a non-standard
-   *  native call syntax (e.g. Gemma) should set this so the agent loop can build
-   *  the prompt with the correct formatting. */
-  readonly promptKind?: 'generic' | 'gemma';
+   *  native call syntax (e.g. Gemma, Qwen ChatML, Mistral [INST]) should set this so
+   *  the agent loop can build the prompt with the correct formatting. */
+  readonly promptKind?: 'generic' | 'gemma' | 'qwen' | 'mistral';
   chat(
     messages: ChatMessage[],
     tools: ProviderTool[],
@@ -67,8 +79,6 @@ export interface ToolCall {
   result?: string;
   error?: string;
   elapsed?: number;
-  /** true if this call was preceded by a discovery tool (search_recipes, get_recipe, etc.) */
-  guided?: boolean;
 }
 
 export interface AgentMetrics {

package/src/util/opfs-cache.ts

@@ -0,0 +1,265 @@
+/**
+ * OPFS model cache — download & cache N files per model repo in the
+ * Origin Private File System, returning a streamable Uint8Array for each file.
+ *
+ * Designed for large-model scenarios (MediaPipe .task, transformers.js ONNX
+ * multi-file bundles, tokenizers, configs). Each file is streamed from network
+ * with `tee()` so the consumer and the OPFS writer share a single download.
+ *
+ * Cache validation strategy: for every file we write a sibling `<file>.complete`
+ * marker containing the total size in bytes. A cache is considered valid only
+ * when the file exists AND the marker exists AND their sizes match (and if the
+ * caller provided `expectedSize`, that too). This avoids serving half-written
+ * files when a tab is closed mid-download.
+ */
+
+export interface ModelFileSpec {
+  /** Path relative to the repo root (e.g. "onnx/model.onnx", "tokenizer.json") */
+  path: string;
+  /** Expected byte size if known — enables exact match cache validation */
+  expectedSize?: number;
+}
+
+export interface CacheProgress {
+  /** 0-1 progress of the file currently being processed */
+  fileProgress: number;
+  /** 0-1 aggregate progress across all files */
+  totalProgress: number;
+  status: 'cached' | 'downloading' | 'initializing' | 'error';
+  currentFile?: string;
+  /** Bytes loaded for current file */
+  loaded: number;
+  /** Total bytes expected for current file */
+  total: number;
+}
+
+/** Sanitize a repo id (e.g. "owner/name") into a valid OPFS folder name. */
+function sanitizeRepoKey(repo: string): string {
+  return repo.replace(/\//g, '__').replace(/[^a-zA-Z0-9_.-]/g, '_');
+}
+
+/**
+ * Walk nested directory segments, creating subdirectories as needed.
+ * Returns `{ parent, filename }` where `parent` is the deepest directory
+ * handle and `filename` is the leaf name.
+ */
+async function resolveFileParent(
+  repoDir: FileSystemDirectoryHandle,
+  relPath: string,
+): Promise<{ parent: FileSystemDirectoryHandle; filename: string }> {
+  const segments = relPath.split('/').filter((s) => s.length > 0);
+  if (segments.length === 0) throw new Error(`Invalid file path: ${relPath}`);
+  let parent = repoDir;
+  for (let i = 0; i < segments.length - 1; i++) {
+    parent = await parent.getDirectoryHandle(segments[i], { create: true });
+  }
+  return { parent, filename: segments[segments.length - 1] };
+}
+
+/**
+ * Removes legacy cache entries: pre-refactor the helper stored model files
+ * directly under `webmcp-models/<filename>`. The new layout nests them under
+ * `webmcp-models/<repo-key>/<filename>`, so the old top-level files are
+ * orphaned and can each weigh several GB.
+ * Runs once per process.
+ */
+let legacyCleanupDone = false;
+async function cleanupLegacyModelFiles(modelsDir: FileSystemDirectoryHandle): Promise<void> {
+  if (legacyCleanupDone) return;
+  legacyCleanupDone = true;
+  try {
+    const dir = modelsDir as unknown as {
+      entries: () => AsyncIterable<[string, FileSystemHandle]>;
+    };
+    for await (const [name, handle] of dir.entries()) {
+      if (handle.kind === 'file') {
+        try { await modelsDir.removeEntry(name); } catch { /* best-effort */ }
+      }
+    }
+  } catch { /* iteration unsupported or blocked — skip silently */ }
+}
+
+/**
+ * Load every requested file from the OPFS cache if valid, otherwise stream it
+ * from the HuggingFace repo and cache it in the background.
+ *
+ * The returned Map is keyed by the original `file.path` (including any
+ * subdirectory prefix) and contains a `ReadableStream<Uint8Array>` per file.
+ * Consumers are responsible for consuming the streams.
+ */
+export async function loadOrDownloadModel(
+  repo: string,
+  files: ModelFileSpec[],
+  onProgress?: (progress: CacheProgress) => void,
+): Promise<Map<string, ReadableStream<Uint8Array>>> {
+  const root = await navigator.storage.getDirectory();
+  const modelsDir = await root.getDirectoryHandle('webmcp-models', { create: true });
+  await cleanupLegacyModelFiles(modelsDir);
+  const repoKey = sanitizeRepoKey(repo);
+  const repoDir = await modelsDir.getDirectoryHandle(repoKey, { create: true });
+
+  const totalExpected = files.reduce((s, f) => s + (f.expectedSize ?? 0), 0);
+  let totalLoaded = 0;
+  const result = new Map<string, ReadableStream<Uint8Array>>();
+
+  for (const file of files) {
+    const { parent, filename } = await resolveFileParent(repoDir, file.path);
+    const markerName = `${filename}.complete`;
+
+    // Clean orphan .crswap files (Chrome WritableStream leftovers).
+    try { await parent.removeEntry(`${filename}.crswap`); } catch { /* no swap — OK */ }
+
+    // ── Cache hit attempt ───────────────────────────────────────────
+    let cacheHitSize: number | null = null;
+    try {
+      const fileHandle = await parent.getFileHandle(filename);
+      const fileObj = await fileHandle.getFile();
+      let expectedFromMarker: number | null = null;
+      try {
+        const markerHandle = await parent.getFileHandle(markerName);
+        const markerText = await (await markerHandle.getFile()).text();
+        expectedFromMarker = Number(markerText.trim());
+      } catch {
+        // Marker missing — try backfill via HEAD request
+        try {
+          const head = await fetch(`https://huggingface.co/${repo}/resolve/main/${file.path}`, { method: 'HEAD' });
+          if (head.ok) {
+            const headerSize = Number(head.headers.get('content-length'));
+            if (Number.isFinite(headerSize) && headerSize > 0 && fileObj.size === headerSize) {
+              // Backfill marker
+              const markerHandle = await parent.getFileHandle(markerName, { create: true });
+              const markerWritable = await markerHandle.createWritable();
+              await markerWritable.write(String(headerSize));
+              await markerWritable.close();
+              expectedFromMarker = headerSize;
+            } else {
+              // Size mismatch — drop cached file
+              try { await parent.removeEntry(filename); } catch {}
+            }
+          }
+        } catch { /* network/HEAD failed — treat as cache miss */ }
+      }
+
+      if (
+        expectedFromMarker !== null
+        && fileObj.size === expectedFromMarker
+        && (file.expectedSize === undefined || file.expectedSize === fileObj.size)
+      ) {
+        cacheHitSize = fileObj.size;
+        onProgress?.({
+          fileProgress: 1,
+          totalProgress: totalExpected > 0 ? (totalLoaded + fileObj.size) / totalExpected : 1,
+          status: 'cached',
+          currentFile: file.path,
+          loaded: fileObj.size,
+          total: fileObj.size,
+        });
+        totalLoaded += fileObj.size;
+        result.set(file.path, fileObj.stream() as ReadableStream<Uint8Array>);
+      }
+    } catch {
+      // Cache miss — fall through to download
+    }
+
+    if (cacheHitSize !== null) continue;
+
+    // ── Network download (retry on 503) ─────────────────────────────
+    const url = `https://huggingface.co/${repo}/resolve/main/${file.path}`;
+    let response: Response | null = null;
+    for (let attempt = 0; attempt < 3; attempt++) {
+      response = await fetch(url);
+      if (response.ok) break;
+      if (response.status === 503 && attempt < 2) {
+        const wait = (attempt + 1) * 5000;
+        onProgress?.({
+          fileProgress: 0,
+          totalProgress: totalExpected > 0 ? totalLoaded / totalExpected : 0,
+          status: 'downloading',
+          currentFile: file.path,
+          loaded: 0,
+          total: file.expectedSize ?? 0,
+        });
+        await new Promise((r) => setTimeout(r, wait));
+        continue;
+      }
+      throw new Error(`Download failed for ${file.path}: ${response.status} ${response.statusText}`);
+    }
+    if (!response || !response.ok) throw new Error(`Download failed for ${file.path} after retries`);
+    if (!response.body) throw new Error(`Response body is null for ${file.path}`);
+
+    const headerTotal = Number(response.headers.get('content-length'));
+    const total = Number.isFinite(headerTotal) && headerTotal > 0
+      ? headerTotal
+      : (file.expectedSize ?? 0);
+
+    const [streamForConsumer, streamForCache] = response.body.tee();
+
+    // Background OPFS cache (fire-and-forget). Marker is written AFTER close
+    // succeeds, so a crashed tab will leave a file without a marker, which is
+    // detected as "invalid cache" on the next load.
+    (async () => {
+      try {
+        const handle = await parent.getFileHandle(filename, { create: true });
+        const writable = await handle.createWritable();
+        await streamForCache.pipeTo(writable);
+        const markerHandle = await parent.getFileHandle(markerName, { create: true });
+        const markerWritable = await markerHandle.createWritable();
+        await markerWritable.write(String(total));
+        await markerWritable.close();
+      } catch {
+        try { await parent.removeEntry(filename); } catch {}
+        try { await parent.removeEntry(markerName); } catch {}
+      }
+    })();
+
+    // Capture outer variables for the transform stream closure
+    const filePath = file.path;
+    const baselineTotalLoaded = totalLoaded;
+    const totalExpectedLocal = totalExpected;
+    let loaded = 0;
+    const progressTransform = new TransformStream<Uint8Array, Uint8Array>({
+      transform(chunk, controller) {
+        loaded += chunk.length;
+        const denom = totalExpectedLocal || total || 1;
+        onProgress?.({
+          fileProgress: total > 0 ? loaded / total : 0,
+          totalProgress: (baselineTotalLoaded + loaded) / denom,
+          status: 'downloading',
+          currentFile: filePath,
+          loaded,
+          total,
+        });
+        controller.enqueue(chunk);
+      },
+      flush() {
+        // No-op: totalLoaded is advanced eagerly below so that subsequent
+        // files reflect the contribution of the current file even if the
+        // consumer is still draining the stream.
+      },
+    });
+
+    // Eagerly advance the baseline for subsequent files: by the time we
+    // process the next file we assume this one will complete (or fail — in
+    // which case progress is moot anyway).
+    totalLoaded += total;
+
+    result.set(file.path, streamForConsumer.pipeThrough(progressTransform));
+  }
+
+  return result;
+}
+
+/**
+ * Remove every cached file for a given repo. Silently no-ops if the repo
+ * directory does not exist.
+ */
+export async function clearModelCache(repo: string): Promise<void> {
+  try {
+    const root = await navigator.storage.getDirectory();
+    const modelsDir = await root.getDirectoryHandle('webmcp-models', { create: false });
+    const repoKey = sanitizeRepoKey(repo);
+    await modelsDir.removeEntry(repoKey, { recursive: true });
+  } catch {
+    // Nothing to clear
+  }
+}