@blank-utils/llm 0.2.7 → 0.2.9

package/dist/backends/transformers.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"transformers.d.ts","sourceRoot":"","sources":["../../src/backends/transformers.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EACV,WAAW,EACX,WAAW,EACX,eAAe,EACf,cAAc,EACd,oBAAoB,EAEpB,OAAO,EACP,MAAM,EACN,YAAY,EACb,MAAM,UAAU,CAAC;AAKlB;;;GAGG;AACH,OAAO,EAAE,0BAA0B,EAAE,mBAAmB,EAAE,KAAK,mBAAmB,EAAE,MAAM,WAAW,CAAC;AACtG,OAAO,EAAE,0BAA0B,EAAE,mBAAmB,EAAE,CAAC;AAE3D,MAAM,MAAM,sBAAsB,GAAG,mBAAmB,CAAC;AAEzD;;GAEG;AACH,eAAO,MAAM,wBAAwB,EAAE,MAAM,CAAC,sBAAsB,EAAE,MAAM,CAW3E,CAAC;AAkGF;;GAEG;AACH,MAAM,WAAW,0BAA0B;IACzC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,YAAY,CAAC;CAC7B;AAED;;GAEG;AACH,qBAAa,oBAAqB,YAAW,WAAW;IACtD,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAkB;IAE3C,OAAO,CAAC,QAAQ,CAAuC;IACvD,OAAO,CAAC,YAAY,CAAuB;IAC3C,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,YAAY,CAAe;gBAEvB,MAAM,GAAE,0BAA+B;IAKnD,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,IAAI,OAAO,IAAI,MAAM,GAAG,IAAI,CAE3B;IAEK,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,IAAI,CAAC;IAkDvE,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,MAAM,CAAC;IAoBzE,MAAM,CACV,QAAQ,EAAE,WAAW,EAAE,EACvB,OAAO,EAAE,cAAc,EACvB,OAAO,CAAC,EAAE,eAAe,GACxB,OAAO,CAAC,MAAM,CAAC;IAgCZ,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;CAI9B;AAED;;GAEG;AACH,wBAAgB,0BAA0B,CAAC,MAAM,CAAC,EAAE,0BAA0B,GAAG,oBAAoB,CAEpG"}
+{"version":3,"file":"transformers.d.ts","sourceRoot":"","sources":["../../src/backends/transformers.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EACV,WAAW,EACX,WAAW,EACX,eAAe,EACf,cAAc,EACd,oBAAoB,EAEpB,OAAO,EACP,MAAM,EACN,YAAY,EACb,MAAM,UAAU,CAAC;AAKlB;;;GAGG;AACH,OAAO,EAAE,0BAA0B,EAAE,mBAAmB,EAAE,KAAK,mBAAmB,EAAE,MAAM,WAAW,CAAC;AACtG,OAAO,EAAE,0BAA0B,EAAE,mBAAmB,EAAE,CAAC;AAE3D,MAAM,MAAM,sBAAsB,GAAG,mBAAmB,CAAC;AAEzD;;GAEG;AACH,eAAO,MAAM,wBAAwB,EAAE,MAAM,CAAC,sBAAsB,EAAE,MAAM,CAc3E,CAAC;AAuBF;;GAEG;AACH,MAAM,WAAW,0BAA0B;IACzC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,YAAY,CAAC,EAAE,YAAY,CAAC;CAC7B;AAED;;GAEG;AACH,qBAAa,oBAAqB,YAAW,WAAW;IACtD,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAkB;IAE3C,OAAO,CAAC,QAAQ,CAAa;IAC7B,OAAO,CAAC,YAAY,CAAuB;IAC3C,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,YAAY,CAAe;gBAEvB,MAAM,GAAE,0BAA+B;IAKnD,IAAI,OAAO,IAAI,OAAO,CAErB;IAED,IAAI,OAAO,IAAI,MAAM,GAAG,IAAI,CAE3B;IAEK,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,IAAI,CAAC;IAoDvE,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,MAAM,CAAC;IAkBzE,MAAM,CACV,QAAQ,EAAE,WAAW,EAAE,EACvB,OAAO,EAAE,cAAc,EACvB,OAAO,CAAC,EAAE,eAAe,GACxB,OAAO,CAAC,MAAM,CAAC;IA+BZ,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;CAI9B;AAED;;GAEG;AACH,wBAAgB,0BAA0B,CAAC,MAAM,CAAC,EAAE,0BAA0B,GAAG,oBAAoB,CAEpG"}

package/dist/index.js

@@ -56629,6 +56629,9 @@ var TRANSFORMERS_MODELS = {
   "qwen-2.5-1.5b": "onnx-community/Qwen2.5-1.5B-Instruct",
   "qwen-2.5-coder-0.5b": "onnx-community/Qwen2.5-Coder-0.5B-Instruct",
   "qwen-2.5-coder-1.5b": "onnx-community/Qwen2.5-Coder-1.5B-Instruct",
+  "qwen-2.5-vl-3b": "onnx-community/Qwen2.5-VL-3B-Instruct",
+  moondream2: "Xenova/moondream2",
+  "phi-3.5-vision": "onnx-community/Phi-3.5-vision-instruct",
   "qwen-3-0.6b": "onnx-community/Qwen3-0.6B-ONNX",
   "smollm2-135m": "HuggingFaceTB/SmolLM2-135M-Instruct",
   "smollm2-360m": "HuggingFaceTB/SmolLM2-360M-Instruct",
@@ -56725,86 +56728,9 @@ function createWebLLMProvider() {
   return new WebLLMProvider;
 }
 // src/backends/transformers.ts
-function detectChatFormat(modelId) {
+function isVisionModel(modelId) {
   const lower = modelId.toLowerCase();
-  if (lower.includes("qwen") || lower.includes("smollm")) {
-    return "chatml";
-  }
-  if (lower.includes("llama") || lower.includes("tinyllama")) {
-    return "llama";
-  }
-  if (lower.includes("phi")) {
-    return "phi";
-  }
-  return "generic";
-}
-function formatPrompt(messages, modelId) {
-  const format = detectChatFormat(modelId);
-  switch (format) {
-    case "chatml": {
-      let prompt = "";
-      for (const msg of messages) {
-        prompt += `<|im_start|>${msg.role}
-${msg.content}<|im_end|>
-`;
-      }
-      prompt += `<|im_start|>assistant
-`;
-      return prompt;
-    }
-    case "llama": {
-      let prompt = "";
-      for (const msg of messages) {
-        if (msg.role === "system") {
-          prompt += `<s>[INST] <<SYS>>
-${msg.content}
-<</SYS>>
-
-`;
-        } else if (msg.role === "user") {
-          if (!prompt.includes("[INST]")) {
-            prompt += `<s>[INST] ${msg.content} [/INST]`;
-          } else {
-            prompt += `<s>[INST] ${msg.content} [/INST]`;
-          }
-        } else if (msg.role === "assistant") {
-          prompt += ` ${msg.content} </s>`;
-        }
-      }
-      return prompt;
-    }
-    case "phi": {
-      let prompt = "";
-      for (const msg of messages) {
-        if (msg.role === "system") {
-          prompt += `<|system|>
-${msg.content}<|end|>
-`;
-        } else if (msg.role === "user") {
-          prompt += `<|user|>
-${msg.content}<|end|>
-`;
-        } else if (msg.role === "assistant") {
-          prompt += `<|assistant|>
-${msg.content}<|end|>
-`;
-        }
-      }
-      prompt += `<|assistant|>
-`;
-      return prompt;
-    }
-    case "generic":
-    default: {
-      let prompt = "";
-      for (const msg of messages) {
-        prompt += `${msg.role}: ${msg.content}
-`;
-      }
-      prompt += "assistant: ";
-      return prompt;
-    }
-  }
+  return lower.includes("vl") || lower.includes("vision") || lower.includes("moondream");
 }
 function mapQuantization(quantization) {
   const map = {
@@ -56849,8 +56775,9 @@ class TransformersProvider {
         } catch {}
       }
     }
+    const task = isVisionModel(resolvedModel) ? "image-text-to-text" : "text-generation";
     const dtype = mapQuantization(this.quantization);
-    this.pipeline = await pipeline("text-generation", resolvedModel, {
+    this.pipeline = await pipeline(task, resolvedModel, {
       dtype,
       device: deviceOption,
       progress_callback: (progress) => {
@@ -56869,8 +56796,7 @@ class TransformersProvider {
     if (!this.pipeline || !this.currentModel) {
       throw new Error("Model not loaded. Call load() first.");
     }
-    const prompt = formatPrompt(messages, this.currentModel);
-    const result = await this.pipeline(prompt, {
+    const result = await this.pipeline(messages, {
       max_new_tokens: options?.maxTokens ?? 512,
       temperature: options?.temperature ?? 0.7,
       top_p: options?.topP ?? 0.95,
@@ -56884,17 +56810,17 @@ class TransformersProvider {
     if (!this.pipeline || !this.currentModel) {
       throw new Error("Model not loaded. Call load() first.");
     }
-    const prompt = formatPrompt(messages, this.currentModel);
     const { TextStreamer } = await Promise.resolve().then(() => (init_transformers_web(), exports_transformers_web));
     let fullText = "";
     const streamer = new TextStreamer(this.pipeline.tokenizer, {
       skip_prompt: true,
+      skip_special_tokens: true,
       callback_function: (token) => {
         fullText += token;
         onToken(token, fullText);
       }
     });
-    await this.pipeline(prompt, {
+    await this.pipeline(messages, {
       max_new_tokens: options?.maxTokens ?? 512,
       temperature: options?.temperature ?? 0.7,
       top_p: options?.topP ?? 0.95,
@@ -58107,9 +58033,37 @@ function Chat2({
       apiMessages.push({ role: "system", content: systemPrompt });
     }
     currentMessages.forEach((m) => {
-      apiMessages.push({ role: m.role, content: m.content });
+      let content = m.content;
+      if (m.role === "user" && m.images && m.images.length > 0) {
+        if (llm.backend === "webllm") {
+          content = [
+            { type: "text", text: m.content },
+            ...m.images.map((img) => ({ type: "image_url", image_url: { url: img.dataUrl } }))
+          ];
+        } else {
+          content = [
+            ...m.images.map((img) => ({ type: "image", image: img.dataUrl })),
+            { type: "text", text: m.content }
+          ];
+        }
+      }
+      apiMessages.push({ role: m.role, content });
     });
-    apiMessages.push({ role: "user", content: userContent });
+    let finalUserContent = userContent;
+    if (attachedImages.length > 0) {
+      if (llm.backend === "webllm") {
+        finalUserContent = [
+          { type: "text", text: userContent },
+          ...attachedImages.map((img) => ({ type: "image_url", image_url: { url: img.dataUrl } }))
+        ];
+      } else {
+        finalUserContent = [
+          ...attachedImages.map((img) => ({ type: "image", image: img.dataUrl })),
+          { type: "text", text: userContent }
+        ];
+      }
+    }
+    apiMessages.push({ role: "user", content: finalUserContent });
     try {
       const response = await llm.stream(apiMessages, (_token, fullText) => {
         if (abortRef.current)
@@ -58520,7 +58474,8 @@ function useChat(options = {}) {
     if (!lastUserMessage)
       return "";
     setMessages(previousMessages);
-    return send(lastUserMessage.content);
+    const textContent = Array.isArray(lastUserMessage.content) ? lastUserMessage.content.map((c) => c.text || "").join("") : lastUserMessage.content;
+    return send(textContent);
   }, [messages, send]);
   return {
     messages,

package/dist/models.d.ts

@@ -58,6 +58,9 @@ export declare const TRANSFORMERS_MODELS: {
     readonly 'qwen-2.5-1.5b': "onnx-community/Qwen2.5-1.5B-Instruct";
     readonly 'qwen-2.5-coder-0.5b': "onnx-community/Qwen2.5-Coder-0.5B-Instruct";
     readonly 'qwen-2.5-coder-1.5b': "onnx-community/Qwen2.5-Coder-1.5B-Instruct";
+    readonly 'qwen-2.5-vl-3b': "onnx-community/Qwen2.5-VL-3B-Instruct";
+    readonly moondream2: "Xenova/moondream2";
+    readonly 'phi-3.5-vision': "onnx-community/Phi-3.5-vision-instruct";
     readonly 'qwen-3-0.6b': "onnx-community/Qwen3-0.6B-ONNX";
     readonly 'smollm2-135m': "HuggingFaceTB/SmolLM2-135M-Instruct";
     readonly 'smollm2-360m': "HuggingFaceTB/SmolLM2-360M-Instruct";

package/dist/models.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"models.d.ts","sourceRoot":"","sources":["../src/models.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;;GAGG;AACH,eAAO,MAAM,oBAAoB,sCAAsC,CAAC;AAExE;;;;;GAKG;AACH,eAAO,MAAM,aAAa;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAgDhB,CAAC;AAGX;;;GAGG;AACH,eAAO,MAAM,0BAA0B,yCAAyC,CAAC;AAEjF;;;;;GAKG;AACH,eAAO,MAAM,mBAAmB;;;;;;;;;;;CAoBtB,CAAC;AAEX,MAAM,MAAM,aAAa,GAAG,MAAM,OAAO,aAAa,CAAC;AACvD,MAAM,MAAM,mBAAmB,GAAG,MAAM,OAAO,mBAAmB,CAAC;AAEnE;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,aAAa,GAAG,mBAAmB,GAAG,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC"}
+{"version":3,"file":"models.d.ts","sourceRoot":"","sources":["../src/models.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;;GAGG;AACH,eAAO,MAAM,oBAAoB,sCAAsC,CAAC;AAExE;;;;;GAKG;AACH,eAAO,MAAM,aAAa;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAgDhB,CAAC;AAGX;;;GAGG;AACH,eAAO,MAAM,0BAA0B,yCAAyC,CAAC;AAEjF;;;;;GAKG;AACH,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;CA0BtB,CAAC;AAEX,MAAM,MAAM,aAAa,GAAG,MAAM,OAAO,aAAa,CAAC;AACvD,MAAM,MAAM,mBAAmB,GAAG,MAAM,OAAO,mBAAmB,CAAC;AAEnE;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,aAAa,GAAG,mBAAmB,GAAG,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC"}

package/dist/react/components.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"components.d.ts","sourceRoot":"","sources":["../../src/react/components.tsx"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAE/B,OAAO,EAAU,KAAK,cAAc,EAAsC,MAAM,SAAS,CAAC;AAE1F,OAAO,EAAsC,KAAK,cAAc,EAAE,MAAM,WAAW,CAAC;AAepF,MAAM,WAAW,SAAS;IACxB,yCAAyC;IACzC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,qCAAqC;IACrC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY;IACZ,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IACzB,uDAAuD;IACvD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,6DAA6D;IAC7D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,2CAA2C;IAC3C,WAAW,CAAC,EAAE,IAAI,CAAC,cAAc,EAAE,cAAc,CAAC,CAAC;IACnD,mDAAmD;IACnD,YAAY,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC;IAC/B,oCAAoC;IACpC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IACnC,yCAAyC;IACzC,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;IACxC,sBAAsB;IACtB,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;IACjC,2DAA2D;IAC3D,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,sDAAsD;IACtD,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,gDAAgD;IAChD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,+DAA+D;IAC/D,aAAa,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CAC3C;AAED,MAAM,WAAW,YAAa,SAAQ,SAAS;IAC7C,qCAAqC;IACrC,YAAY,CAAC,EAAE,cAAc,CAAC;IAC9B,6BAA6B;IAC7B,cAAc,CAAC,EAAE,QAAQ,GAAG,cAAc,GAAG,MAAM,CAAC;IACpD,8CAA8C;IAC9C,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAsbD,iBAAS,IAAI,CAAC,EACZ,YAAoC,EACpC,WAAiC,EACjC,KAAc,EACd,SAAS,EACT,SAAmB,EACnB,YAAY,EACZ,MAAM,EAAE,UAAU,EAClB,UAAU,EACV,OAAO,EAAE,WAAW,EACpB,UAAiB,EACjB,YAAmB,EACnB,cAAkC,EAClC,aAAa,GACd,EAAE,SAAS,2CAqPX;AAMD,iBAAS,OAAO,CAAC,EACf,YAA8B,EAC9B,cAAuB,EACvB,QAAe,EACf,aAAa,EACb,GAAG,SAAS,EACb,EAAE,YAAY,2CAqBd;AAED,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC"}
+{"version":3,"file":"components.d.ts","sourceRoot":"","sources":["../../src/react/components.tsx"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAE/B,OAAO,EAAU,KAAK,cAAc,EAAsC,MAAM,SAAS,CAAC;AAE1F,OAAO,EAAsC,KAAK,cAAc,EAAE,MAAM,WAAW,CAAC;AAepF,MAAM,WAAW,SAAS;IACxB,yCAAyC;IACzC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,qCAAqC;IACrC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,YAAY;IACZ,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;IACzB,uDAAuD;IACvD,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,6DAA6D;IAC7D,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,2CAA2C;IAC3C,WAAW,CAAC,EAAE,IAAI,CAAC,cAAc,EAAE,cAAc,CAAC,CAAC;IACnD,mDAAmD;IACnD,YAAY,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC;IAC/B,oCAAoC;IACpC,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;IACnC,yCAAyC;IACzC,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;IACxC,sBAAsB;IACtB,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;IACjC,2DAA2D;IAC3D,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,sDAAsD;IACtD,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,gDAAgD;IAChD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,+DAA+D;IAC/D,aAAa,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CAC3C;AAED,MAAM,WAAW,YAAa,SAAQ,SAAS;IAC7C,qCAAqC;IACrC,YAAY,CAAC,EAAE,cAAc,CAAC;IAC9B,6BAA6B;IAC7B,cAAc,CAAC,EAAE,QAAQ,GAAG,cAAc,GAAG,MAAM,CAAC;IACpD,8CAA8C;IAC9C,QAAQ,CAAC,EAAE,OAAO,CAAC;CACpB;AAsbD,iBAAS,IAAI,CAAC,EACZ,YAAoC,EACpC,WAAiC,EACjC,KAAc,EACd,SAAS,EACT,SAAmB,EACnB,YAAY,EACZ,MAAM,EAAE,UAAU,EAClB,UAAU,EACV,OAAO,EAAE,WAAW,EACpB,UAAiB,EACjB,YAAmB,EACnB,cAAkC,EAClC,aAAa,GACd,EAAE,SAAS,2CAmRX;AAMD,iBAAS,OAAO,CAAC,EACf,YAA8B,EAC9B,cAAuB,EACvB,QAAe,EACf,aAAa,EACb,GAAG,SAAS,EACb,EAAE,YAAY,2CAqBd;AAED,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC"}

package/dist/react/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/react/index.tsx"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAW/B,OAAO,KAAK,EACV,SAAS,EACT,WAAW,EACX,eAAe,EACf,YAAY,EACZ,OAAO,EACR,MAAM,UAAU,CAAC;AAElB,OAAO,EAAa,KAAK,QAAQ,EAAE,MAAM,SAAS,CAAC;AAQnD,MAAM,WAAW,eAAe;IAC9B,4CAA4C;IAC5C,GAAG,EAAE,QAAQ,GAAG,IAAI,CAAC;IAErB,6CAA6C;IAC7C,SAAS,EAAE,OAAO,CAAC;IAEnB,+CAA+C;IAC/C,OAAO,EAAE,OAAO,CAAC;IAEjB,+BAA+B;IAC/B,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;IAElC,8BAA8B;IAC9B,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC;IAEpB,uBAAuB;IACvB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAEvB,yBAAyB;IACzB,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IAExB,gCAAgC;IAChC,MAAM,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAE5B,uBAAuB;IACvB,MAAM,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC7B;AAQD,MAAM,WAAW,gBAAiB,SAAQ,IAAI,CAAC,SAAS,EAAE,gBAAgB,CAAC;IACzE,QAAQ,EAAE,KAAK,CAAC,SAAS,CAAC;IAE1B;;;OAGG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IAEnB;;OAEG;IACH,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,QAAQ,KAAK,IAAI,CAAC;IAEjC;;OAEG;IACH,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,KAAK,IAAI,CAAC;IAE9C;;OAEG;IACH,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;CAClC;AAED;;;;;;;;;;;;;GAaG;AACH,iBAAS,WAAW,CAAC,EACnB,QAAQ,EACR,QAAe,EACf,MAAM,EACN,UAAU,EACV,OAAO,EACP,GAAG,MAAM,EACV,EAAE,gBAAgB,2CAsFlB;AAMD;;;;;;;;;GASG;AACH,iBAAS,MAAM,IAAI,eAAe,CAQjC;AAMD,MAAM,WAAW,cAAc;IAC7B,uBAAuB;IACvB,eAAe,CAAC,EAAE,WAAW,EAAE,CAAC;IAEhC,oBAAoB;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB,yBAAyB;IACzB,eAAe,CAAC,EAAE,eAAe,CAAC;IAElC;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAE5B,oCAAoC;IACpC,OAAO,CAAC,EAAE,MAAM,IAAI,CAAC;IAErB,uCAAuC;IACvC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;IAEpD,uCAAuC;IACvC,QAAQ,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;IAEtC,sBAAsB;IACtB,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;CAClC;AAED,MAAM,WAAW,aAAa;IAC5B,uCAAuC;IACvC,QAAQ,EAAE,WAAW,EAAE,CAAC;IAExB,iDAAiD;IACjD,KAAK,EAAE,MAAM,CAAC;IAEd,0BAA0B;IAC1B,QAAQ,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAElC,8CAA8C;IAC9C,YAAY,EAAE,OAAO,CAAC;IAEtB,4DAA4D;IAC5D,SAAS,EAAE,OAAO,CAAC;IAEnB,gDAAgD;IAChD,aAAa,EAAE,MAAM,CAAC;IAEtB,wCAAwC;IACxC,IAAI,EAAE,CAAC,OAAO,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IAE5C,kCAAkC;IAClC,IAAI,EAAE,MAAM,IAAI,CAAC;IAEjB,yBAAyB;IACzB,KAAK,EAAE,MAAM,IAAI,CAAC;IAElB,kDAAkD;IAClD,MAAM,EAAE,CAAC,OAAO,EAAE,WAAW,KAAK,IAAI,CAAC;IAEvC,mDAAmD;IACnD,MAAM,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;CAC/B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkDG;AACH,iBAAS,OAAO,CAAC,OAAO,GAAE,cAAmB,GAAG,aAAa,CA6L5D;AAMD,MAAM,WAAW,gBAAgB;IAC/B,yBAAyB;IACzB,eAAe,CAAC,EAAE,eAAe,CAAC;IAElC,2BAA2B;IAC3B,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;IAEpD,2BAA2B;IAC3B,QAAQ,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;IAEtC,sBAAsB;IACtB,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,4BAA4B;IAC5B,IAAI,EAAE,MAAM,CAAC;IAEb,kCAAkC;IAClC,WAAW,EAAE,OAAO,CAAC;IAErB,iCAAiC;IACjC,MAAM,EAAE,CAAC,QAAQ,EAAE,WAAW,EAAE,GAAG,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IAE9D,qBAAqB;IACrB,IAAI,EAAE,MAAM,IAAI,CAAC;IAEjB,qBAAqB;IACrB,KAAK,EAAE,MAAM,IAAI,CAAC;CACnB;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,iBAAS,SAAS,CAAC,OAAO,GAAE,gBAAqB,GAAG,eAAe,CA2DlE;AAMD,MAAM,WAAW,oBAAoB;IACnC,yBAAyB;IACzB,eAAe,CAAC,EAAE,eAAe,CAAC;CACnC;AAED,MAAM,WAAW,mBAAmB;IAClC,8BAA8B;IAC9B,UAAU,EAAE,MAAM,CAAC;IAEnB,mCAAmC;IACnC,SAAS,EAAE,OAAO,CAAC;IAEnB,4CAA4C;IAC5C,QAAQ,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IAE9C,2BAA2B;IAC3B,KAAK,EAAE,MAAM,IAAI,CAAC;CACnB;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,iBAAS,aAAa,CACpB,OAAO,GAAE,oBAAyB,GACjC,mBAAmB,CAuCrB;AAMD,MAAM,WAAW,eAAe;IAC9B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC;IAE3B,iCAAiC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;;;GASG;AACH,iBAAS,UAAU,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,EAAE,eAAe,kDAe3D;AAMD,MAAM,WAAW,aAAa;IAC5B,iCAAiC;IACjC,QAAQ,EAAE,KAAK,CAAC,SAAS,CAAC;IAE1B,oCAAoC;IACpC,QAAQ,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC;CAC5B;AAED;;;;;;;;;GASG;AACH,iBAAS,QAAQ,CAAC,EAAE,QAAQ,EAAE,QAAe,EAAE,EAAE,aAAa,2CAQ7D;AAGD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,SAAS,EAAE,KAAK,YAAY,EAAE,MAAM,cAAc,CAAC;AAChF,OAAO,EAAE,SAAS,EAAE,KAAK,cAAc,EAAE,MAAM,cAAc,CAAC;AAE9D,OAAO,EAEL,WAAW,EACX,MAAM,EAGN,OAAO,EACP,SAAS,EACT,aAAa,EAGb,UAAU,EACV,QAAQ,EAGR,IAAI,EACJ,OAAO,EACP,SAAS,EAGT,KAAK,SAAS,EACd,KAAK,YAAY,EACjB,KAAK,cAAc,GACpB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/react/index.tsx"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,OAAO,KAAK,KAAK,MAAM,OAAO,CAAC;AAW/B,OAAO,KAAK,EACV,SAAS,EACT,WAAW,EACX,eAAe,EACf,YAAY,EACZ,OAAO,EACR,MAAM,UAAU,CAAC;AAElB,OAAO,EAAa,KAAK,QAAQ,EAAE,MAAM,SAAS,CAAC;AAQnD,MAAM,WAAW,eAAe;IAC9B,4CAA4C;IAC5C,GAAG,EAAE,QAAQ,GAAG,IAAI,CAAC;IAErB,6CAA6C;IAC7C,SAAS,EAAE,OAAO,CAAC;IAEnB,+CAA+C;IAC/C,OAAO,EAAE,OAAO,CAAC;IAEjB,+BAA+B;IAC/B,YAAY,EAAE,YAAY,GAAG,IAAI,CAAC;IAElC,8BAA8B;IAC9B,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC;IAEpB,uBAAuB;IACvB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAEvB,yBAAyB;IACzB,OAAO,EAAE,OAAO,GAAG,IAAI,CAAC;IAExB,gCAAgC;IAChC,MAAM,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;IAE5B,uBAAuB;IACvB,MAAM,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC7B;AAQD,MAAM,WAAW,gBAAiB,SAAQ,IAAI,CAAC,SAAS,EAAE,gBAAgB,CAAC;IACzE,QAAQ,EAAE,KAAK,CAAC,SAAS,CAAC;IAE1B;;;OAGG;IACH,QAAQ,CAAC,EAAE,OAAO,CAAC;IAEnB;;OAEG;IACH,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,QAAQ,KAAK,IAAI,CAAC;IAEjC;;OAEG;IACH,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,YAAY,KAAK,IAAI,CAAC;IAE9C;;OAEG;IACH,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;CAClC;AAED;;;;;;;;;;;;;GAaG;AACH,iBAAS,WAAW,CAAC,EACnB,QAAQ,EACR,QAAe,EACf,MAAM,EACN,UAAU,EACV,OAAO,EACP,GAAG,MAAM,EACV,EAAE,gBAAgB,2CAsFlB;AAMD;;;;;;;;;GASG;AACH,iBAAS,MAAM,IAAI,eAAe,CAQjC;AAMD,MAAM,WAAW,cAAc;IAC7B,uBAAuB;IACvB,eAAe,CAAC,EAAE,WAAW,EAAE,CAAC;IAEhC,oBAAoB;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB,yBAAyB;IACzB,eAAe,CAAC,EAAE,eAAe,CAAC;IAElC;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAE5B,oCAAoC;IACpC,OAAO,CAAC,EAAE,MAAM,IAAI,CAAC;IAErB,uCAAuC;IACvC,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;IAEpD,uCAAuC;IACvC,QAAQ,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;IAEtC,sBAAsB;IACtB,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;CAClC;AAED,MAAM,WAAW,aAAa;IAC5B,uCAAuC;IACvC,QAAQ,EAAE,WAAW,EAAE,CAAC;IAExB,iDAAiD;IACjD,KAAK,EAAE,MAAM,CAAC;IAEd,0BAA0B;IAC1B,QAAQ,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAElC,8CAA8C;IAC9C,YAAY,EAAE,OAAO,CAAC;IAEtB,4DAA4D;IAC5D,SAAS,EAAE,OAAO,CAAC;IAEnB,gDAAgD;IAChD,aAAa,EAAE,MAAM,CAAC;IAEtB,wCAAwC;IACxC,IAAI,EAAE,CAAC,OAAO,CAAC,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IAE5C,kCAAkC;IAClC,IAAI,EAAE,MAAM,IAAI,CAAC;IAEjB,yBAAyB;IACzB,KAAK,EAAE,MAAM,IAAI,CAAC;IAElB,kDAAkD;IAClD,MAAM,EAAE,CAAC,OAAO,EAAE,WAAW,KAAK,IAAI,CAAC;IAEvC,mDAAmD;IACnD,MAAM,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,CAAC;CAC/B;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAkDG;AACH,iBAAS,OAAO,CAAC,OAAO,GAAE,cAAmB,GAAG,aAAa,CAiM5D;AAMD,MAAM,WAAW,gBAAgB;IAC/B,yBAAyB;IACzB,eAAe,CAAC,EAAE,eAAe,CAAC;IAElC,2BAA2B;IAC3B,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;IAEpD,2BAA2B;IAC3B,QAAQ,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;IAEtC,sBAAsB;IACtB,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;CAClC;AAED,MAAM,WAAW,eAAe;IAC9B,4BAA4B;IAC5B,IAAI,EAAE,MAAM,CAAC;IAEb,kCAAkC;IAClC,WAAW,EAAE,OAAO,CAAC;IAErB,iCAAiC;IACjC,MAAM,EAAE,CAAC,QAAQ,EAAE,WAAW,EAAE,GAAG,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IAE9D,qBAAqB;IACrB,IAAI,EAAE,MAAM,IAAI,CAAC;IAEjB,qBAAqB;IACrB,KAAK,EAAE,MAAM,IAAI,CAAC;CACnB;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,iBAAS,SAAS,CAAC,OAAO,GAAE,gBAAqB,GAAG,eAAe,CA2DlE;AAMD,MAAM,WAAW,oBAAoB;IACnC,yBAAyB;IACzB,eAAe,CAAC,EAAE,eAAe,CAAC;CACnC;AAED,MAAM,WAAW,mBAAmB;IAClC,8BAA8B;IAC9B,UAAU,EAAE,MAAM,CAAC;IAEnB,mCAAmC;IACnC,SAAS,EAAE,OAAO,CAAC;IAEnB,4CAA4C;IAC5C,QAAQ,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;IAE9C,2BAA2B;IAC3B,KAAK,EAAE,MAAM,IAAI,CAAC;CACnB;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,iBAAS,aAAa,CACpB,OAAO,GAAE,oBAAyB,GACjC,mBAAmB,CAuCrB;AAMD,MAAM,WAAW,eAAe;IAC9B,wBAAwB;IACxB,QAAQ,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC;IAE3B,iCAAiC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED;;;;;;;;;GASG;AACH,iBAAS,UAAU,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,EAAE,eAAe,kDAe3D;AAMD,MAAM,WAAW,aAAa;IAC5B,iCAAiC;IACjC,QAAQ,EAAE,KAAK,CAAC,SAAS,CAAC;IAE1B,oCAAoC;IACpC,QAAQ,CAAC,EAAE,KAAK,CAAC,SAAS,CAAC;CAC5B;AAED;;;;;;;;;GASG;AACH,iBAAS,QAAQ,CAAC,EAAE,QAAQ,EAAE,QAAe,EAAE,EAAE,aAAa,2CAQ7D;AAGD,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,SAAS,EAAE,KAAK,YAAY,EAAE,MAAM,cAAc,CAAC;AAChF,OAAO,EAAE,SAAS,EAAE,KAAK,cAAc,EAAE,MAAM,cAAc,CAAC;AAE9D,OAAO,EAEL,WAAW,EACX,MAAM,EAGN,OAAO,EACP,SAAS,EACT,aAAa,EAGb,UAAU,EACV,QAAQ,EAGR,IAAI,EACJ,OAAO,EACP,SAAS,EAGT,KAAK,SAAS,EACd,KAAK,YAAY,EACjB,KAAK,cAAc,GACpB,CAAC"}

package/dist/react/index.js

@@ -56632,6 +56632,9 @@ var TRANSFORMERS_MODELS = {
   "qwen-2.5-1.5b": "onnx-community/Qwen2.5-1.5B-Instruct",
   "qwen-2.5-coder-0.5b": "onnx-community/Qwen2.5-Coder-0.5B-Instruct",
   "qwen-2.5-coder-1.5b": "onnx-community/Qwen2.5-Coder-1.5B-Instruct",
+  "qwen-2.5-vl-3b": "onnx-community/Qwen2.5-VL-3B-Instruct",
+  moondream2: "Xenova/moondream2",
+  "phi-3.5-vision": "onnx-community/Phi-3.5-vision-instruct",
   "qwen-3-0.6b": "onnx-community/Qwen3-0.6B-ONNX",
   "smollm2-135m": "HuggingFaceTB/SmolLM2-135M-Instruct",
   "smollm2-360m": "HuggingFaceTB/SmolLM2-360M-Instruct",
@@ -56726,86 +56729,9 @@ class WebLLMProvider {
 }
 
 // src/backends/transformers.ts
-function detectChatFormat(modelId) {
+function isVisionModel(modelId) {
   const lower = modelId.toLowerCase();
-  if (lower.includes("qwen") || lower.includes("smollm")) {
-    return "chatml";
-  }
-  if (lower.includes("llama") || lower.includes("tinyllama")) {
-    return "llama";
-  }
-  if (lower.includes("phi")) {
-    return "phi";
-  }
-  return "generic";
-}
-function formatPrompt(messages, modelId) {
-  const format = detectChatFormat(modelId);
-  switch (format) {
-    case "chatml": {
-      let prompt = "";
-      for (const msg of messages) {
-        prompt += `<|im_start|>${msg.role}
-${msg.content}<|im_end|>
-`;
-      }
-      prompt += `<|im_start|>assistant
-`;
-      return prompt;
-    }
-    case "llama": {
-      let prompt = "";
-      for (const msg of messages) {
-        if (msg.role === "system") {
-          prompt += `<s>[INST] <<SYS>>
-${msg.content}
-<</SYS>>
-
-`;
-        } else if (msg.role === "user") {
-          if (!prompt.includes("[INST]")) {
-            prompt += `<s>[INST] ${msg.content} [/INST]`;
-          } else {
-            prompt += `<s>[INST] ${msg.content} [/INST]`;
-          }
-        } else if (msg.role === "assistant") {
-          prompt += ` ${msg.content} </s>`;
-        }
-      }
-      return prompt;
-    }
-    case "phi": {
-      let prompt = "";
-      for (const msg of messages) {
-        if (msg.role === "system") {
-          prompt += `<|system|>
-${msg.content}<|end|>
-`;
-        } else if (msg.role === "user") {
-          prompt += `<|user|>
-${msg.content}<|end|>
-`;
-        } else if (msg.role === "assistant") {
-          prompt += `<|assistant|>
-${msg.content}<|end|>
-`;
-        }
-      }
-      prompt += `<|assistant|>
-`;
-      return prompt;
-    }
-    case "generic":
-    default: {
-      let prompt = "";
-      for (const msg of messages) {
-        prompt += `${msg.role}: ${msg.content}
-`;
-      }
-      prompt += "assistant: ";
-      return prompt;
-    }
-  }
+  return lower.includes("vl") || lower.includes("vision") || lower.includes("moondream");
 }
 function mapQuantization(quantization) {
   const map = {
@@ -56850,8 +56776,9 @@ class TransformersProvider {
         } catch {}
       }
     }
+    const task = isVisionModel(resolvedModel) ? "image-text-to-text" : "text-generation";
     const dtype = mapQuantization(this.quantization);
-    this.pipeline = await pipeline("text-generation", resolvedModel, {
+    this.pipeline = await pipeline(task, resolvedModel, {
       dtype,
       device: deviceOption,
       progress_callback: (progress) => {
@@ -56870,8 +56797,7 @@ class TransformersProvider {
     if (!this.pipeline || !this.currentModel) {
       throw new Error("Model not loaded. Call load() first.");
     }
-    const prompt = formatPrompt(messages, this.currentModel);
-    const result = await this.pipeline(prompt, {
+    const result = await this.pipeline(messages, {
       max_new_tokens: options?.maxTokens ?? 512,
       temperature: options?.temperature ?? 0.7,
       top_p: options?.topP ?? 0.95,
@@ -56885,17 +56811,17 @@ class TransformersProvider {
     if (!this.pipeline || !this.currentModel) {
       throw new Error("Model not loaded. Call load() first.");
     }
-    const prompt = formatPrompt(messages, this.currentModel);
     const { TextStreamer } = await Promise.resolve().then(() => (init_transformers_web(), exports_transformers_web));
     let fullText = "";
     const streamer = new TextStreamer(this.pipeline.tokenizer, {
       skip_prompt: true,
+      skip_special_tokens: true,
       callback_function: (token) => {
         fullText += token;
         onToken(token, fullText);
       }
     });
-    await this.pipeline(prompt, {
+    await this.pipeline(messages, {
       max_new_tokens: options?.maxTokens ?? 512,
       temperature: options?.temperature ?? 0.7,
       top_p: options?.topP ?? 0.95,
@@ -58035,9 +57961,37 @@ function Chat2({
       apiMessages.push({ role: "system", content: systemPrompt });
     }
     currentMessages.forEach((m) => {
-      apiMessages.push({ role: m.role, content: m.content });
+      let content = m.content;
+      if (m.role === "user" && m.images && m.images.length > 0) {
+        if (llm.backend === "webllm") {
+          content = [
+            { type: "text", text: m.content },
+            ...m.images.map((img) => ({ type: "image_url", image_url: { url: img.dataUrl } }))
+          ];
+        } else {
+          content = [
+            ...m.images.map((img) => ({ type: "image", image: img.dataUrl })),
+            { type: "text", text: m.content }
+          ];
+        }
+      }
+      apiMessages.push({ role: m.role, content });
     });
-    apiMessages.push({ role: "user", content: userContent });
+    let finalUserContent = userContent;
+    if (attachedImages.length > 0) {
+      if (llm.backend === "webllm") {
+        finalUserContent = [
+          { type: "text", text: userContent },
+          ...attachedImages.map((img) => ({ type: "image_url", image_url: { url: img.dataUrl } }))
+        ];
+      } else {
+        finalUserContent = [
+          ...attachedImages.map((img) => ({ type: "image", image: img.dataUrl })),
+          { type: "text", text: userContent }
+        ];
+      }
+    }
+    apiMessages.push({ role: "user", content: finalUserContent });
     try {
       const response = await llm.stream(apiMessages, (_token, fullText) => {
         if (abortRef.current)
@@ -58470,7 +58424,8 @@ function useChat(options = {}) {
     if (!lastUserMessage)
       return "";
     setMessages(previousMessages);
-    return send(lastUserMessage.content);
+    const textContent = Array.isArray(lastUserMessage.content) ? lastUserMessage.content.map((c) => c.text || "").join("") : lastUserMessage.content;
+    return send(textContent);
   }, [messages, send]);
   return {
     messages,

package/dist/types.d.ts

@@ -59,7 +59,7 @@ export type MessageRole = 'system' | 'user' | 'assistant';
  */
 export interface ChatMessage {
     role: MessageRole;
-    content: string;
+    content: string | any[];
 }
 /**
  * Callback for streaming token output

package/dist/types.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAMH;;GAEG;AACH,MAAM,MAAM,OAAO,GAAG,QAAQ,GAAG,cAAc,GAAG,MAAM,CAAC;AAEzD;;GAEG;AACH,MAAM,MAAM,MAAM,GAAG,QAAQ,GAAG,MAAM,GAAG,MAAM,CAAC;AAEhD;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,IAAI,GAAG,IAAI,GAAG,MAAM,GAAG,MAAM,CAAC;AAEzD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAC/C,YAAY,EAAE,cAAc,EAAE,CAAC;AAE/B;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB;;;;OAIG;IACH,KAAK,CAAC,EAAE,cAAc,CAAC;IAEvB;;;OAGG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB;;OAEG;IACH,cAAc,CAAC,EAAE,oBAAoB,CAAC;CACvC;AAMD;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG,QAAQ,GAAG,MAAM,GAAG,WAAW,CAAC;AAE1D;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,WAAW,CAAC;IAClB,OAAO,EAAE,MAAM,CAAC;CACjB;AAMD;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;AAEvE;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,kCAAkC;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,0BAA0B;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,kCAAkC;IAClC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,iCAAiC;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAAG,CAAC,QAAQ,EAAE,YAAY,KAAK,IAAI,CAAC;AAMpE;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;CAC1B;AAMD;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAE1B;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAE1B;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAEhC;;OAEG;IACH,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAExE;;OAEG;IACH,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAE1E;;OAEG;IACH,MAAM,CACJ,QAAQ,EAAE,WAAW,EAAE,EACvB,OAAO,EAAE,cAAc,EACvB,OAAO,CAAC,EAAE,eAAe,GACxB,OAAO,CAAC,MAAM,CAAC,CAAC;IAEnB;;OAEG;IACH,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACzB;AAMD;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;;OAGG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;IAEtB;;;OAGG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;IAEtB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAMD;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,0BAA0B;IAC1B,MAAM,EAAE,OAAO,CAAC;IAChB,+BAA+B;IAC/B,IAAI,EAAE,OAAO,CAAC;IACd,gDAAgD;IAChD,kBAAkB,EAAE,OAAO,CAAC;IAC5B,+CAA+C;IAC/C,iBAAiB,EAAE,MAAM,CAAC;CAC3B"}
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAMH;;GAEG;AACH,MAAM,MAAM,OAAO,GAAG,QAAQ,GAAG,cAAc,GAAG,MAAM,CAAC;AAEzD;;GAEG;AACH,MAAM,MAAM,MAAM,GAAG,QAAQ,GAAG,MAAM,GAAG,MAAM,CAAC;AAEhD;;GAEG;AACH,MAAM,MAAM,YAAY,GAAG,IAAI,GAAG,IAAI,GAAG,MAAM,GAAG,MAAM,CAAC;AAEzD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAC/C,YAAY,EAAE,cAAc,EAAE,CAAC;AAE/B;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB;;;;OAIG;IACH,KAAK,CAAC,EAAE,cAAc,CAAC;IAEvB;;;OAGG;IACH,OAAO,CAAC,EAAE,OAAO,CAAC;IAElB;;;OAGG;IACH,MAAM,CAAC,EAAE,MAAM,CAAC;IAEhB;;;OAGG;IACH,YAAY,CAAC,EAAE,YAAY,CAAC;IAE5B;;OAEG;IACH,YAAY,CAAC,EAAE,MAAM,CAAC;IAEtB;;OAEG;IACH,cAAc,CAAC,EAAE,oBAAoB,CAAC;CACvC;AAMD;;GAEG;AACH,MAAM,MAAM,WAAW,GAAG,QAAQ,GAAG,MAAM,GAAG,WAAW,CAAC;AAE1D;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,WAAW,CAAC;IAClB,OAAO,EAAE,MAAM,GAAG,GAAG,EAAE,CAAC;CACzB;AAMD;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;AAEvE;;GAEG;AACH,MAAM,WAAW,YAAY;IAC3B,kCAAkC;IAClC,QAAQ,EAAE,MAAM,CAAC;IACjB,0BAA0B;IAC1B,MAAM,EAAE,MAAM,CAAC;IACf,kCAAkC;IAClC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,iCAAiC;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAAG,CAAC,QAAQ,EAAE,YAAY,KAAK,IAAI,CAAC;AAMpE;;GAEG;AACH,MAAM,WAAW,eAAe;IAC9B;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;IAErB;;;OAGG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IAEnB;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IAEd;;OAEG;IACH,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;CAC1B;AAMD;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAE1B;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;IAE1B;;OAEG;IACH,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAEhC;;OAEG;IACH,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,oBAAoB,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAExE;;OAEG;IACH,IAAI,CAAC,QAAQ,EAAE,WAAW,EAAE,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAE1E;;OAEG;IACH,MAAM,CACJ,QAAQ,EAAE,WAAW,EAAE,EACvB,OAAO,EAAE,cAAc,EACvB,OAAO,CAAC,EAAE,eAAe,GACxB,OAAO,CAAC,MAAM,CAAC,CAAC;IAEnB;;OAEG;IACH,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACzB;AAMD;;GAEG;AACH,MAAM,WAAW,aAAa;IAC5B;;;OAGG;IACH,cAAc,CAAC,EAAE,OAAO,CAAC;IAEzB;;;OAGG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;IAEtB;;;OAGG;IACH,WAAW,CAAC,EAAE,OAAO,CAAC;IAEtB;;;OAGG;IACH,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB;AAMD;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,0BAA0B;IAC1B,MAAM,EAAE,OAAO,CAAC;IAChB,+BAA+B;IAC/B,IAAI,EAAE,OAAO,CAAC;IACd,gDAAgD;IAChD,kBAAkB,EAAE,OAAO,CAAC;IAC5B,+CAA+C;IAC/C,iBAAiB,EAAE,MAAM,CAAC;CAC3B"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@blank-utils/llm",
-  "version": "0.2.7",
+  "version": "0.2.9",
   "description": "Run LLMs directly in your browser with WebGPU acceleration. Supports React hooks and eager background loading.",
   "type": "module",
   "main": "./dist/index.js",
@@ -22,8 +22,12 @@
   },
   "typesVersions": {
     "*": {
-      ".": ["./dist/index.d.ts"],
-      "react": ["./dist/react/index.d.ts"]
+      ".": [
+        "./dist/index.d.ts"
+      ],
+      "react": [
+        "./dist/react/index.d.ts"
+      ]
     }
   },
   "files": [