@webmcp-auto-ui/agent 2.5.26 → 2.5.28

package/src/providers/hawk.ts

@@ -0,0 +1,181 @@
+import type { LLMProvider, LLMResponse, ChatMessage, ProviderTool, ContentBlock } from '../types.js';
+
+export interface HawkLLMProviderOptions {
+  proxyUrl: string;   // SvelteKit proxy endpoint, e.g. '/api/hawk'
+  model: string;      // e.g. 'qwen35-2b' (ID Hawk sans préfixe)
+}
+
+// ── OpenAI-compatible types ─────────────────────────────────────────
+
+interface OaiTool {
+  type: 'function';
+  function: { name: string; description: string; parameters: Record<string, unknown> };
+}
+
+interface OaiMessage {
+  role: 'system' | 'user' | 'assistant' | 'tool';
+  content?: string | null;
+  tool_calls?: { id: string; type: 'function'; function: { name: string; arguments: string } }[];
+  tool_call_id?: string;
+}
+
+interface OaiChoice {
+  message: {
+    content?: string | null;
+    tool_calls?: { id: string; type: 'function'; function: { name: string; arguments: string } }[];
+  };
+  finish_reason: string;
+}
+
+// ── Helpers ─────────────────────────────────────────────────────────
+
+let _counter = 0;
+function hawkId(): string {
+  return 'hawk_' + (++_counter).toString(36) + '_' + Date.now().toString(36);
+}
+
+function toOaiTools(tools: ProviderTool[]): OaiTool[] {
+  return tools.map(t => ({
+    type: 'function' as const,
+    function: {
+      name: t.name,
+      description: t.description,
+      parameters: t.input_schema,
+    },
+  }));
+}
+
+function toOaiMessages(messages: ChatMessage[], system?: string): OaiMessage[] {
+  const out: OaiMessage[] = [];
+
+  if (system) out.push({ role: 'system', content: system });
+
+  for (const msg of messages) {
+    if (typeof msg.content === 'string') {
+      out.push({ role: msg.role === 'assistant' ? 'assistant' : 'user', content: msg.content });
+      continue;
+    }
+
+    const blocks = msg.content as ContentBlock[];
+    const textParts = blocks.filter(b => b.type === 'text').map(b => (b as { type: 'text'; text: string }).text);
+    const toolUses = blocks.filter(b => b.type === 'tool_use') as { type: 'tool_use'; id: string; name: string; input: Record<string, unknown> }[];
+    const toolResults = blocks.filter(b => b.type === 'tool_result') as { type: 'tool_result'; tool_use_id: string; content: string }[];
+
+    if (msg.role === 'assistant') {
+      const oai: OaiMessage = { role: 'assistant', content: textParts.join('\n') || null };
+      if (toolUses.length > 0) {
+        oai.tool_calls = toolUses.map(tu => ({
+          id: tu.id,
+          type: 'function' as const,
+          function: { name: tu.name, arguments: JSON.stringify(tu.input) },
+        }));
+      }
+      out.push(oai);
+    } else {
+      // User turn — may contain tool_result blocks (sent back after assistant tool_use)
+      for (const tr of toolResults) {
+        out.push({ role: 'tool', tool_call_id: tr.tool_use_id, content: tr.content });
+      }
+      if (textParts.length > 0) {
+        out.push({ role: 'user', content: textParts.join('\n') });
+      }
+      // If only tool_results and no text, we've already pushed them
+      if (toolResults.length === 0 && textParts.length === 0) {
+        out.push({ role: 'user', content: '' });
+      }
+    }
+  }
+  return out;
+}
+
+function parseArguments(raw: string): Record<string, unknown> {
+  try { return JSON.parse(raw); } catch { return { _raw: raw }; }
+}
+
+// ── Provider ────────────────────────────────────────────────────────
+
+export class HawkProvider implements LLMProvider {
+  readonly name = 'hawk';
+  readonly model: string;
+  private proxyUrl: string;
+
+  constructor(options: HawkLLMProviderOptions) {
+    this.model = options.model;
+    this.proxyUrl = options.proxyUrl;
+  }
+
+  async chat(
+    messages: ChatMessage[],
+    tools: ProviderTool[],
+    options?: { signal?: AbortSignal; system?: string; maxTokens?: number; temperature?: number },
+  ): Promise<LLMResponse> {
+    const oaiMessages = toOaiMessages(messages, options?.system);
+    const oaiTools = tools.length > 0 ? toOaiTools(tools) : undefined;
+
+    // NOTE: `model` is NOT sent in the body — the server proxy injects it
+    // from the X-Model header into the upstream Hawk request.
+    const body: Record<string, unknown> = {
+      messages: oaiMessages,
+      stream: false,
+    };
+    if (oaiTools) body.tools = oaiTools;
+    if (options?.maxTokens) body.max_tokens = options.maxTokens;
+    if (options?.temperature != null) body.temperature = options.temperature;
+
+    const response = await fetch(this.proxyUrl, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'X-Model': this.model,
+      },
+      body: JSON.stringify(body),
+      signal: options?.signal,
+    });
+
+    if (!response.ok) {
+      const txt = await response.text().catch(() => '');
+      throw new Error(`Hawk LLM ${response.status}${txt ? ': ' + txt.slice(0, 200) : ''}`);
+    }
+
+    const data = await response.json() as { choices?: OaiChoice[]; usage?: { prompt_tokens?: number; completion_tokens?: number } };
+    const choice = data.choices?.[0];
+    if (!choice) throw new Error('Hawk LLM returned no choices');
+
+    const content: ContentBlock[] = [];
+    const toolCalls = choice.message.tool_calls;
+
+    if (choice.message.content) {
+      content.push({ type: 'text', text: choice.message.content });
+    }
+
+    if (toolCalls && toolCalls.length > 0) {
+      for (const tc of toolCalls) {
+        content.push({
+          type: 'tool_use',
+          id: tc.id || hawkId(),
+          name: tc.function.name,
+          input: parseArguments(tc.function.arguments),
+        });
+      }
+    }
+
+    // Ensure at least one block
+    if (content.length === 0) {
+      content.push({ type: 'text', text: '' });
+    }
+
+    const hasToolUse = content.some(b => b.type === 'tool_use');
+    const stopReason = hasToolUse ? 'tool_use'
+      : choice.finish_reason === 'tool_calls' ? 'tool_use'
+      : 'end_turn';
+
+    return {
+      content,
+      stopReason,
+      usage: data.usage ? {
+        input_tokens: data.usage.prompt_tokens ?? 0,
+        output_tokens: data.usage.completion_tokens ?? 0,
+      } : undefined,
+    };
+  }
+}

package/src/providers/transformers-models.ts

@@ -0,0 +1,143 @@
+// Shared catalog of in-browser models served via transformers.js (ONNX + WebGPU).
+// Read by TransformersProvider (agent A) and the <LLMSelector> UI (agent D).
+//
+// Each entry pins:
+//   - `repo`: HuggingFace repository ID to load via from_pretrained
+//   - `dtype`: mixed per-component quantization (embed_tokens / decoder / vision)
+//   - `family`: prompt builder family — drives which {gemma,qwen,mistral}-prompt-builder is used
+//   - `toolFormat`: output tool-call syntax — drives which parser strategy is used
+//   - `contextLength`: native context budget (for clipping heuristics)
+//   - `vision`: true if the model accepts images via RawImage + AutoProcessor
+//   - `modelClass`: optional specialized transformers.js class name (for VLMs);
+//                   defaults to AutoModelForCausalLM when omitted
+
+export type TransformersFamily = 'gemma4' | 'qwen3' | 'mistral';
+export type ToolCallFormat = 'gemma-native' | 'qwen-json' | 'mistral-toolcalls';
+
+export type DType = 'q4' | 'q4f16' | 'q8' | 'fp16' | 'fp32';
+
+export interface TransformersModelEntry {
+  repo: string;
+  dtype: DType | {
+    embed_tokens?: DType;
+    decoder_model_merged?: DType;
+    vision_encoder?: DType;
+    audio_encoder?: DType;
+  };
+  family: TransformersFamily;
+  toolFormat: ToolCallFormat;
+  contextLength: number;
+  vision: boolean;
+  modelClass?: string;
+  /** Approximate download size in bytes (for progress UI). */
+  size: number;
+  /** Human-readable label for the model selector. */
+  label: string;
+}
+
+export type TransformersModelId =
+  | 'transformers-gemma-4-e2b'
+  | 'transformers-gemma-4-e4b'
+  | 'transformers-qwen-3-4b'
+  | 'transformers-qwen-3.5-2b'
+  | 'transformers-qwen-3.5-4b'
+  | 'transformers-ministral-3-3b';
+
+export const TRANSFORMERS_MODELS: Record<TransformersModelId, TransformersModelEntry> = {
+  'transformers-gemma-4-e2b': {
+    repo: 'onnx-community/gemma-4-E2B-it-ONNX',
+    modelClass: 'Gemma4ForConditionalGeneration',
+    dtype: {
+      audio_encoder: 'q4',
+      vision_encoder: 'q4',
+      embed_tokens: 'q4',
+      decoder_model_merged: 'q4f16',
+    },
+    family: 'gemma4',
+    toolFormat: 'gemma-native',
+    contextLength: 32768,
+    vision: true,
+    size: 2_000_000_000,
+    label: 'Gemma 4 E2B (Vision)',
+  },
+  'transformers-gemma-4-e4b': {
+    repo: 'onnx-community/gemma-4-E4B-it-ONNX',
+    modelClass: 'Gemma4ForConditionalGeneration',
+    dtype: {
+      audio_encoder: 'q4',
+      vision_encoder: 'q4',
+      embed_tokens: 'q4',
+      decoder_model_merged: 'q4f16',
+    },
+    family: 'gemma4',
+    toolFormat: 'gemma-native',
+    contextLength: 32768,
+    vision: true,
+    size: 3_000_000_000,
+    label: 'Gemma 4 E4B (Vision)',
+  },
+  'transformers-qwen-3-4b': {
+    // onnx-community/Qwen3-4B-ONNX ships a monolithic model_q4f16.onnx
+    // (not split into embed_tokens + decoder_model_merged), so transformers.js
+    // expects a scalar dtype string to resolve onnx/model_<dtype>.onnx.
+    repo: 'onnx-community/Qwen3-4B-ONNX',
+    dtype: 'q4f16',
+    family: 'qwen3',
+    toolFormat: 'qwen-json',
+    contextLength: 32768,
+    vision: false,
+    size: 3_050_000_000,
+    label: 'Qwen 3 4B',
+  },
+  'transformers-qwen-3.5-2b': {
+    repo: 'onnx-community/Qwen3.5-2B-ONNX',
+    dtype: { embed_tokens: 'q4', vision_encoder: 'fp16', decoder_model_merged: 'q4' },
+    family: 'qwen3',
+    toolFormat: 'qwen-json',
+    contextLength: 32768,
+    vision: true,
+    modelClass: 'Qwen3_5ForConditionalGeneration',
+    size: 1_600_000_000,
+    label: 'Qwen 3.5 2B',
+  },
+  'transformers-qwen-3.5-4b': {
+    repo: 'onnx-community/Qwen3.5-4B-ONNX',
+    dtype: { embed_tokens: 'q4', vision_encoder: 'fp16', decoder_model_merged: 'q4' },
+    family: 'qwen3',
+    toolFormat: 'qwen-json',
+    contextLength: 32768,
+    vision: true,
+    modelClass: 'Qwen3_5ForConditionalGeneration',
+    size: 3_000_000_000,
+    label: 'Qwen 3.5 4B',
+  },
+  'transformers-ministral-3-3b': {
+    repo: 'mistralai/Ministral-3-3B-Instruct-2512-ONNX',
+    // Mistral3ForConditionalGeneration is registered internally but not
+    // re-exported from transformers.js 4.1.0 — use the Auto wrapper, which
+    // routes via the registered name (this is what the official demo does).
+    modelClass: 'AutoModelForImageTextToText',
+    dtype: {
+      embed_tokens: 'fp16',
+      vision_encoder: 'q4',
+      decoder_model_merged: 'q4f16',
+    },
+    family: 'mistral',
+    toolFormat: 'mistral-toolcalls',
+    contextLength: 32768,
+    vision: true,
+    size: 2_200_000_000,
+    label: 'Ministral 3 3B (Vision)',
+  },
+};
+
+export function getTransformersModel(id: TransformersModelId): TransformersModelEntry {
+  return TRANSFORMERS_MODELS[id];
+}
+
+export function listTransformersModels(): Array<{ id: TransformersModelId; entry: TransformersModelEntry }> {
+  return Object.entries(TRANSFORMERS_MODELS).map(([id, entry]) => ({
+    id: id as TransformersModelId,
+    entry,
+  }));
+}

package/src/providers/transformers-serialize.ts

@@ -0,0 +1,81 @@
+/**
+ * Pure serializer: ChatMessage[] → flat [{role, content}] array suitable for
+ * tokenizer.apply_chat_template in the transformers.js worker.
+ *
+ * Extracted from TransformersProvider so it can be unit-tested without
+ * spinning up a Web Worker. Tool calls and tool results are rendered as
+ * textual spans using the wire format each model family expects:
+ *
+ *   - Qwen (ChatML):   <tool_call>{json}</tool_call>
+ *                      <tool_response>{json}</tool_response>
+ *   - Mistral:         [TOOL_CALLS][{json}]
+ *                      [TOOL_RESULTS] {json} [/TOOL_RESULTS]
+ *   - Gemma:           legacy path (this serializer is not used — see
+ *                      buildGemmaPrompt in prompts/gemma4-prompt-builder.ts).
+ *
+ * The actual role tags (<|im_start|>user\n…<|im_end|>, [INST]…[/INST]) are
+ * added by apply_chat_template from the model's baked-in chat_template.
+ */
+import type { ChatMessage, ContentBlock } from '../types.js';
+import { formatToolCall, formatToolResponse } from '../prompts/gemma4-prompt-builder.js';
+
+export type PromptKind = 'gemma' | 'qwen' | 'mistral';
+
+export function serializeMessagesForTemplate(
+  messages: ChatMessage[],
+  promptKind: PromptKind,
+): Array<{ role: string; content: string }> {
+  const out: Array<{ role: string; content: string }> = [];
+  for (const msg of messages) {
+    const role = msg.role; // 'user' | 'assistant' | 'system'
+    if (typeof msg.content === 'string') {
+      out.push({ role, content: msg.content });
+      continue;
+    }
+    const segments: string[] = [];
+    let toolResultBuf: string[] = [];
+    const flushToolResults = () => {
+      if (toolResultBuf.length === 0) return;
+      if (promptKind === 'qwen') {
+        for (const tr of toolResultBuf) {
+          segments.push(`<tool_response>\n${tr}\n</tool_response>`);
+        }
+      } else if (promptKind === 'mistral') {
+        for (const tr of toolResultBuf) {
+          segments.push(`[TOOL_RESULTS] ${tr} [/TOOL_RESULTS]`);
+        }
+      } else {
+        // Gemma path — kept for defensive completeness; main code uses
+        // buildGemmaPrompt instead.
+        for (const tr of toolResultBuf) segments.push(formatToolResponse(tr));
+      }
+      toolResultBuf = [];
+    };
+    for (const block of msg.content as ContentBlock[]) {
+      if (block.type === 'text') {
+        segments.push(block.text);
+      } else if (block.type === 'tool_use') {
+        if (promptKind === 'qwen') {
+          segments.push(`<tool_call>\n${JSON.stringify({ name: block.name, arguments: block.input })}\n</tool_call>`);
+        } else if (promptKind === 'mistral') {
+          segments.push(`[TOOL_CALLS][${JSON.stringify({ name: block.name, arguments: block.input })}]`);
+        } else {
+          segments.push(formatToolCall(block.name, block.input));
+        }
+      } else if (block.type === 'tool_result') {
+        toolResultBuf.push(block.content);
+      }
+      // 'image' blocks are not reachable here: vision turns go through the
+      // legacy `prompt` path in TransformersProvider.chat().
+    }
+    flushToolResults();
+    // Promote pure-tool-result turns: Qwen uses the 'tool' role, Mistral
+    // keeps 'user' (the template wraps tool results inside a user turn).
+    const onlyToolResult = (msg.content as ContentBlock[]).every(b => b.type === 'tool_result');
+    const effectiveRole = onlyToolResult && role === 'user'
+      ? (promptKind === 'qwen' ? 'tool' : 'user')
+      : role;
+    out.push({ role: effectiveRole, content: segments.join('\n') });
+  }
+  return out;
+}