@webmcp-auto-ui/agent 2.5.25 → 2.5.27

package/src/prompts/tool-refs.ts

@@ -0,0 +1,196 @@
+// Collect per-layer prompt references (tool names formatted for the LLM).
+
+import type { ProviderTool } from '../types.js';
+import {
+  buildToolsFromLayers,
+  protocolToken,
+  resolveCanonicalTools,
+  sanitizeServerName,
+  type McpLayer,
+  type ProviderKind,
+  type ToolLayer,
+  type WebMcpLayer,
+} from '../tool-layers.js';
+import { formatGemmaToolDeclaration } from './gemma4-prompt-builder.js';
+
+/** Short descriptions for discovery pseudo-tools (used in inline Gemma decls). */
+const shortSearchToolsDesc = (serverName: string) =>
+  `Search tools by keyword on the ${serverName} server.`;
+const shortListToolsDesc = (serverName: string) =>
+  `List ALL tools on the ${serverName} server.`;
+
+/** First parameter name from a tool's input schema (for arg hints). */
+function firstParamName(tool?: ProviderTool, fallback = 'query'): string {
+  if (!tool?.input_schema) return fallback;
+  const schema = tool.input_schema as Record<string, unknown>;
+  const required = (schema.required as string[] | undefined) ?? [];
+  if (required.length > 0) return required[0];
+  const props = (schema.properties as Record<string, unknown> | undefined) ?? {};
+  return Object.keys(props)[0] ?? fallback;
+}
+
+/** Format a tool reference for the system prompt (generic or inline Gemma decl). */
+function fmtToolRef(
+  prefixedName: string,
+  args: string[] = [],
+  kind: ProviderKind = 'generic',
+  tool?: ProviderTool,
+): string {
+  if (kind === 'gemma' && tool) {
+    return formatGemmaToolDeclaration({ ...tool, name: prefixedName });
+  }
+  if (kind === 'gemma' || kind === 'qwen' || kind === 'mistral') {
+    return args.length ? `\`${prefixedName}(${args.join(', ')})\`` : `\`${prefixedName}\``;
+  }
+  return args.length ? `${prefixedName}(${args.join(', ')})` : `${prefixedName}()`;
+}
+
+export interface PromptRefs {
+  searchRecipes: string[];
+  listRecipes: string[];
+  getRecipes: string[];
+  searchTools: string[];
+  listTools: string[];
+  actionTools: string[];
+  listRecipesByCat: { data: string[]; display: string[] };
+  searchRecipesByCat: { data: string[]; display: string[] };
+  getRecipesByCat: { data: string[]; display: string[] };
+  aliasMap: Map<string, string>;
+}
+
+export function collectPromptRefs(
+  layers: ToolLayer[],
+  providerKind: ProviderKind,
+): PromptRefs {
+  const kind = providerKind;
+  const mcpLayers = layers.filter((l): l is McpLayer => l.protocol === 'mcp');
+  const webmcpLayers = layers.filter((l): l is WebMcpLayer => l.protocol === 'webmcp');
+
+  const displayLayers = webmcpLayers.filter(l => l.tools.some(t => t.name === 'widget_display'));
+  const dataLayers = layers.filter(l => !displayLayers.includes(l as WebMcpLayer));
+
+  const providerToolsByName = new Map<string, ProviderTool>(
+    buildToolsFromLayers(layers, { sanitize: true }).tools.map(t => [t.name, t]),
+  );
+
+  const aliasMap = new Map<string, string>();
+  const searchRecipes: string[] = [];
+  const listRecipes: string[] = [];
+  const getRecipes: string[] = [];
+  const searchTools: string[] = [];
+  const listTools: string[] = [];
+
+  for (const l of webmcpLayers) {
+    const prefix = `${sanitizeServerName(l.serverName)}_ui_`;
+    for (const t of l.tools) {
+      if (t.name === 'search_recipes') {
+        const name = `${prefix}search_recipes`;
+        const toolDef = providerToolsByName.get(name);
+        searchRecipes.push(fmtToolRef(name, [firstParamName(toolDef, 'query')], kind, toolDef));
+      }
+      if (t.name === 'list_recipes') {
+        const name = `${prefix}list_recipes`;
+        const toolDef = providerToolsByName.get(name);
+        listRecipes.push(fmtToolRef(name, [], kind, toolDef));
+      }
+      if (t.name === 'get_recipe') {
+        const name = `${prefix}get_recipe`;
+        const toolDef = providerToolsByName.get(name);
+        getRecipes.push(fmtToolRef(name, [firstParamName(toolDef, 'id')], kind, toolDef));
+      }
+    }
+    const searchToolsPseudo: ProviderTool = {
+      name: `${prefix}search_tools`,
+      description: shortSearchToolsDesc(l.serverName),
+      input_schema: { type: 'object', properties: { query: { type: 'string', description: 'Keyword to search for.' } }, required: ['query'] },
+    };
+    searchTools.push(fmtToolRef(`${prefix}search_tools`, ['query'], kind, searchToolsPseudo));
+    const listToolsPseudo: ProviderTool = {
+      name: `${prefix}list_tools`,
+      description: shortListToolsDesc(l.serverName),
+      input_schema: { type: 'object', properties: {} },
+    };
+    listTools.push(fmtToolRef(`${prefix}list_tools`, [], kind, listToolsPseudo));
+  }
+
+  for (const l of mcpLayers) {
+    const prefix = `${sanitizeServerName(l.serverName)}_data_`;
+    const matches = resolveCanonicalTools(l.tools);
+
+    for (const m of matches) {
+      const canonicalPrefixed = `${prefix}${m.role}`;
+      const realPrefixed = `${prefix}${m.realToolName}`;
+
+      if (m.role !== m.realToolName) {
+        aliasMap.set(canonicalPrefixed, realPrefixed);
+      }
+
+      const realToolDef = providerToolsByName.get(realPrefixed);
+
+      if (m.role === 'search_recipes') {
+        searchRecipes.push(fmtToolRef(canonicalPrefixed, [firstParamName(realToolDef, 'query')], kind, realToolDef));
+      }
+      if (m.role === 'list_recipes') {
+        listRecipes.push(fmtToolRef(canonicalPrefixed, [], kind, realToolDef));
+      }
+      if (m.role === 'get_recipe') {
+        getRecipes.push(fmtToolRef(canonicalPrefixed, [firstParamName(realToolDef, 'id')], kind, realToolDef));
+      }
+    }
+
+    const searchToolsPseudo: ProviderTool = {
+      name: `${prefix}search_tools`,
+      description: shortSearchToolsDesc(l.serverName),
+      input_schema: { type: 'object', properties: { query: { type: 'string', description: 'Keyword to search for.' } }, required: ['query'] },
+    };
+    searchTools.push(fmtToolRef(`${prefix}search_tools`, ['query'], kind, searchToolsPseudo));
+    const listToolsPseudo: ProviderTool = {
+      name: `${prefix}list_tools`,
+      description: shortListToolsDesc(l.serverName),
+      input_schema: { type: 'object', properties: {} },
+    };
+    listTools.push(fmtToolRef(`${prefix}list_tools`, [], kind, listToolsPseudo));
+  }
+
+  const actionTools: string[] = [];
+  const ACTION_NAMES = ['widget_display', 'canvas', 'recall'];
+  for (const l of webmcpLayers) {
+    const prefix = `${sanitizeServerName(l.serverName)}_ui_`;
+    for (const actionName of ACTION_NAMES) {
+      if (l.tools.some(t => t.name === actionName)) {
+        const prefixedName = `${prefix}${actionName}`;
+        const toolDef = providerToolsByName.get(prefixedName);
+        const args = actionName === 'widget_display' ? ['name', 'params'] : [];
+        actionTools.push(fmtToolRef(prefixedName, args, kind, toolDef));
+      }
+    }
+  }
+
+  const dataPrefixes = new Set(dataLayers.map(l => `${sanitizeServerName(l.serverName)}_${protocolToken(l.protocol)}_`));
+  const displayPrefixes = new Set(displayLayers.map(l => `${sanitizeServerName(l.serverName)}_ui_`));
+  void dataPrefixes;
+
+  function splitByCategory(refs: string[]): { data: string[]; display: string[] } {
+    const data: string[] = [];
+    const display: string[] = [];
+    for (const ref of refs) {
+      const isDisplay = [...displayPrefixes].some(p => ref.includes(p));
+      if (isDisplay) display.push(ref);
+      else data.push(ref);
+    }
+    return { data, display };
+  }
+
+  return {
+    searchRecipes,
+    listRecipes,
+    getRecipes,
+    searchTools,
+    listTools,
+    actionTools,
+    listRecipesByCat: splitByCategory(listRecipes),
+    searchRecipesByCat: splitByCategory(searchRecipes),
+    getRecipesByCat: splitByCategory(getRecipes),
+    aliasMap,
+  };
+}

package/src/providers/factory.ts

@@ -2,15 +2,27 @@ import type { LLMProvider, RemoteModelId, WasmModelId } from '../types.js';
 import { RemoteLLMProvider } from './remote.js';
 import { WasmProvider } from './wasm.js';
 import { LocalLLMProvider, type LocalBackend } from './local.js';
+import { TransformersProvider } from './transformers.js';
 
 export type LLMConfig =
-  | { type: 'remote'; model?: RemoteModelId; proxyUrl?: string; apiKey?: string }
-  | { type: 'wasm';   model?: WasmModelId;   onProgress?: (loaded: number, total: number) => void }
-  | { type: 'local';  model: string;         baseUrl: string; backend?: LocalBackend };
+  | { type: 'remote';       model?: RemoteModelId; proxyUrl?: string; apiKey?: string }
+  | { type: 'wasm';         model?: WasmModelId;   onProgress?: (loaded: number, total: number) => void }
+  | { type: 'transformers'; model: string;         onProgress?: (loaded: number, total: number) => void }
+  | { type: 'local';        model: string;         baseUrl: string; backend?: LocalBackend };
 
 export function createProvider(config: LLMConfig): LLMProvider {
   const base = typeof window !== 'undefined' ? (document.querySelector('base') as HTMLBaseElement | null)?.href ?? '' : '';
 
+  // Prefix-based dispatch: a `transformers-*` model routes to TransformersProvider
+  // regardless of the declared type (defensive).
+  if ('model' in config && typeof config.model === 'string' && config.model.startsWith('transformers-')) {
+    const onProgress = (config as { onProgress?: (loaded: number, total: number) => void }).onProgress;
+    return new TransformersProvider({
+      model: config.model,
+      onProgress: onProgress ? (_progress, _status, loaded, total) => onProgress(loaded ?? 0, total ?? 0) : undefined,
+    });
+  }
+
   switch (config.type) {
     case 'remote':
       return new RemoteLLMProvider({
@@ -23,6 +35,11 @@ export function createProvider(config: LLMConfig): LLMProvider {
         model: config.model,
         onProgress: config.onProgress ? (progress, _status, loaded, total) => config.onProgress!(loaded ?? 0, total ?? 0) : undefined,
       });
+    case 'transformers':
+      return new TransformersProvider({
+        model: config.model,
+        onProgress: config.onProgress ? (_progress, _status, loaded, total) => config.onProgress!(loaded ?? 0, total ?? 0) : undefined,
+      });
     case 'local':
       return new LocalLLMProvider({
         baseUrl: config.baseUrl,

package/src/providers/transformers-models.ts

@@ -0,0 +1,143 @@
+// Shared catalog of in-browser models served via transformers.js (ONNX + WebGPU).
+// Read by TransformersProvider (agent A) and the <LLMSelector> UI (agent D).
+//
+// Each entry pins:
+//   - `repo`: HuggingFace repository ID to load via from_pretrained
+//   - `dtype`: mixed per-component quantization (embed_tokens / decoder / vision)
+//   - `family`: prompt builder family — drives which {gemma,qwen,mistral}-prompt-builder is used
+//   - `toolFormat`: output tool-call syntax — drives which parser strategy is used
+//   - `contextLength`: native context budget (for clipping heuristics)
+//   - `vision`: true if the model accepts images via RawImage + AutoProcessor
+//   - `modelClass`: optional specialized transformers.js class name (for VLMs);
+//                   defaults to AutoModelForCausalLM when omitted
+
+export type TransformersFamily = 'gemma4' | 'qwen3' | 'mistral';
+export type ToolCallFormat = 'gemma-native' | 'qwen-json' | 'mistral-toolcalls';
+
+export type DType = 'q4' | 'q4f16' | 'q8' | 'fp16' | 'fp32';
+
+export interface TransformersModelEntry {
+  repo: string;
+  dtype: DType | {
+    embed_tokens?: DType;
+    decoder_model_merged?: DType;
+    vision_encoder?: DType;
+    audio_encoder?: DType;
+  };
+  family: TransformersFamily;
+  toolFormat: ToolCallFormat;
+  contextLength: number;
+  vision: boolean;
+  modelClass?: string;
+  /** Approximate download size in bytes (for progress UI). */
+  size: number;
+  /** Human-readable label for the model selector. */
+  label: string;
+}
+
+export type TransformersModelId =
+  | 'transformers-gemma-4-e2b'
+  | 'transformers-gemma-4-e4b'
+  | 'transformers-qwen-3-4b'
+  | 'transformers-qwen-3.5-2b'
+  | 'transformers-qwen-3.5-4b'
+  | 'transformers-ministral-3-3b';
+
+export const TRANSFORMERS_MODELS: Record<TransformersModelId, TransformersModelEntry> = {
+  'transformers-gemma-4-e2b': {
+    repo: 'onnx-community/gemma-4-E2B-it-ONNX',
+    modelClass: 'Gemma4ForConditionalGeneration',
+    dtype: {
+      audio_encoder: 'q4',
+      vision_encoder: 'q4',
+      embed_tokens: 'q4',
+      decoder_model_merged: 'q4f16',
+    },
+    family: 'gemma4',
+    toolFormat: 'gemma-native',
+    contextLength: 32768,
+    vision: true,
+    size: 2_000_000_000,
+    label: 'Gemma 4 E2B (Vision)',
+  },
+  'transformers-gemma-4-e4b': {
+    repo: 'onnx-community/gemma-4-E4B-it-ONNX',
+    modelClass: 'Gemma4ForConditionalGeneration',
+    dtype: {
+      audio_encoder: 'q4',
+      vision_encoder: 'q4',
+      embed_tokens: 'q4',
+      decoder_model_merged: 'q4f16',
+    },
+    family: 'gemma4',
+    toolFormat: 'gemma-native',
+    contextLength: 32768,
+    vision: true,
+    size: 3_000_000_000,
+    label: 'Gemma 4 E4B (Vision)',
+  },
+  'transformers-qwen-3-4b': {
+    // onnx-community/Qwen3-4B-ONNX ships a monolithic model_q4f16.onnx
+    // (not split into embed_tokens + decoder_model_merged), so transformers.js
+    // expects a scalar dtype string to resolve onnx/model_<dtype>.onnx.
+    repo: 'onnx-community/Qwen3-4B-ONNX',
+    dtype: 'q4f16',
+    family: 'qwen3',
+    toolFormat: 'qwen-json',
+    contextLength: 32768,
+    vision: false,
+    size: 3_050_000_000,
+    label: 'Qwen 3 4B',
+  },
+  'transformers-qwen-3.5-2b': {
+    repo: 'onnx-community/Qwen3.5-2B-ONNX',
+    dtype: { embed_tokens: 'q4', vision_encoder: 'fp16', decoder_model_merged: 'q4' },
+    family: 'qwen3',
+    toolFormat: 'qwen-json',
+    contextLength: 32768,
+    vision: true,
+    modelClass: 'Qwen3_5ForConditionalGeneration',
+    size: 1_600_000_000,
+    label: 'Qwen 3.5 2B',
+  },
+  'transformers-qwen-3.5-4b': {
+    repo: 'onnx-community/Qwen3.5-4B-ONNX',
+    dtype: { embed_tokens: 'q4', vision_encoder: 'fp16', decoder_model_merged: 'q4' },
+    family: 'qwen3',
+    toolFormat: 'qwen-json',
+    contextLength: 32768,
+    vision: true,
+    modelClass: 'Qwen3_5ForConditionalGeneration',
+    size: 3_000_000_000,
+    label: 'Qwen 3.5 4B',
+  },
+  'transformers-ministral-3-3b': {
+    repo: 'mistralai/Ministral-3-3B-Instruct-2512-ONNX',
+    // Mistral3ForConditionalGeneration is registered internally but not
+    // re-exported from transformers.js 4.1.0 — use the Auto wrapper, which
+    // routes via the registered name (this is what the official demo does).
+    modelClass: 'AutoModelForImageTextToText',
+    dtype: {
+      embed_tokens: 'fp16',
+      vision_encoder: 'q4',
+      decoder_model_merged: 'q4f16',
+    },
+    family: 'mistral',
+    toolFormat: 'mistral-toolcalls',
+    contextLength: 32768,
+    vision: true,
+    size: 2_200_000_000,
+    label: 'Ministral 3 3B (Vision)',
+  },
+};
+
+export function getTransformersModel(id: TransformersModelId): TransformersModelEntry {
+  return TRANSFORMERS_MODELS[id];
+}
+
+export function listTransformersModels(): Array<{ id: TransformersModelId; entry: TransformersModelEntry }> {
+  return Object.entries(TRANSFORMERS_MODELS).map(([id, entry]) => ({
+    id: id as TransformersModelId,
+    entry,
+  }));
+}

package/src/providers/transformers-serialize.ts

@@ -0,0 +1,81 @@
+/**
+ * Pure serializer: ChatMessage[] → flat [{role, content}] array suitable for
+ * tokenizer.apply_chat_template in the transformers.js worker.
+ *
+ * Extracted from TransformersProvider so it can be unit-tested without
+ * spinning up a Web Worker. Tool calls and tool results are rendered as
+ * textual spans using the wire format each model family expects:
+ *
+ *   - Qwen (ChatML):   <tool_call>{json}</tool_call>
+ *                      <tool_response>{json}</tool_response>
+ *   - Mistral:         [TOOL_CALLS][{json}]
+ *                      [TOOL_RESULTS] {json} [/TOOL_RESULTS]
+ *   - Gemma:           legacy path (this serializer is not used — see
+ *                      buildGemmaPrompt in prompts/gemma4-prompt-builder.ts).
+ *
+ * The actual role tags (<|im_start|>user\n…<|im_end|>, [INST]…[/INST]) are
+ * added by apply_chat_template from the model's baked-in chat_template.
+ */
+import type { ChatMessage, ContentBlock } from '../types.js';
+import { formatToolCall, formatToolResponse } from '../prompts/gemma4-prompt-builder.js';
+
+export type PromptKind = 'gemma' | 'qwen' | 'mistral';
+
+export function serializeMessagesForTemplate(
+  messages: ChatMessage[],
+  promptKind: PromptKind,
+): Array<{ role: string; content: string }> {
+  const out: Array<{ role: string; content: string }> = [];
+  for (const msg of messages) {
+    const role = msg.role; // 'user' | 'assistant' | 'system'
+    if (typeof msg.content === 'string') {
+      out.push({ role, content: msg.content });
+      continue;
+    }
+    const segments: string[] = [];
+    let toolResultBuf: string[] = [];
+    const flushToolResults = () => {
+      if (toolResultBuf.length === 0) return;
+      if (promptKind === 'qwen') {
+        for (const tr of toolResultBuf) {
+          segments.push(`<tool_response>\n${tr}\n</tool_response>`);
+        }
+      } else if (promptKind === 'mistral') {
+        for (const tr of toolResultBuf) {
+          segments.push(`[TOOL_RESULTS] ${tr} [/TOOL_RESULTS]`);
+        }
+      } else {
+        // Gemma path — kept for defensive completeness; main code uses
+        // buildGemmaPrompt instead.
+        for (const tr of toolResultBuf) segments.push(formatToolResponse(tr));
+      }
+      toolResultBuf = [];
+    };
+    for (const block of msg.content as ContentBlock[]) {
+      if (block.type === 'text') {
+        segments.push(block.text);
+      } else if (block.type === 'tool_use') {
+        if (promptKind === 'qwen') {
+          segments.push(`<tool_call>\n${JSON.stringify({ name: block.name, arguments: block.input })}\n</tool_call>`);
+        } else if (promptKind === 'mistral') {
+          segments.push(`[TOOL_CALLS][${JSON.stringify({ name: block.name, arguments: block.input })}]`);
+        } else {
+          segments.push(formatToolCall(block.name, block.input));
+        }
+      } else if (block.type === 'tool_result') {
+        toolResultBuf.push(block.content);
+      }
+      // 'image' blocks are not reachable here: vision turns go through the
+      // legacy `prompt` path in TransformersProvider.chat().
+    }
+    flushToolResults();
+    // Promote pure-tool-result turns: Qwen uses the 'tool' role, Mistral
+    // keeps 'user' (the template wraps tool results inside a user turn).
+    const onlyToolResult = (msg.content as ContentBlock[]).every(b => b.type === 'tool_result');
+    const effectiveRole = onlyToolResult && role === 'user'
+      ? (promptKind === 'qwen' ? 'tool' : 'user')
+      : role;
+    out.push({ role: effectiveRole, content: segments.join('\n') });
+  }
+  return out;
+}