@matatbread/matbot-provider-webllm 0.0.0-stage → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +205 -2
- package/package.json +48 -3
- package/src/adapter.ts +316 -0
- package/src/format.ts +434 -0
- package/src/index.ts +13 -0
- package/src/models.ts +138 -0
- package/src/webllm.ts +204 -0
- package/src/window.ts +161 -0
package/src/format.ts
ADDED
|
@@ -0,0 +1,434 @@
|
|
|
1
|
+
import type { Message, Tool } from '@matatbread/matbot-plugin-api';
|
|
2
|
+
import { textDocument } from '@matatbread/matbot-core/providers-base';
|
|
3
|
+
import type { ChatMessage, ResponseFormat } from './webllm.js';
|
|
4
|
+
|
|
5
|
+
// Tool calling is done here rather than through WebLLM's `tools`. That path is hard-wired to two 2024
|
|
6
|
+
// Hermes models, refuses any system message of the caller's (matbot's system context could not be sent),
|
|
7
|
+
// and forces the WHOLE reply to be a JSON array of calls, so the model could never simply answer. Instead
|
|
8
|
+
// the tools are described in the format Qwen (and Hermes) models are trained on — `<tools>` in the system
|
|
9
|
+
// prompt, `<tool_call>` in the reply — and a structural tag constrains what follows each `<tool_call>`.
|
|
10
|
+
|
|
11
|
+
const TOOLS_PREAMBLE = '# Tools\n\nYou may call one or more functions to assist with the user query.\n\n'
|
|
12
|
+
+ 'You are provided with function signatures within <tools></tools> XML tags:\n<tools>';
|
|
13
|
+
const TOOLS_POSTAMBLE = '</tools>\n\nFor each function call, return a json object with function name and arguments '
|
|
14
|
+
+ 'within <tool_call></tool_call> XML tags:\n<tool_call>\n{"name": <function-name>, "arguments": <args-json-object>}\n</tool_call>\n'
|
|
15
|
+
// Repeated AFTER the example on purpose. Measured on Qwen3-1.7B: shown the object between the tags, it
|
|
16
|
+
// copies the object verbatim — spacing and all — and drops the two XML lines around it, so the reply is
|
|
17
|
+
// `{"name": "tool_search", "arguments": {…}}` and nothing else. A small model imitates what it read
|
|
18
|
+
// last, and the last line of the tools block is the one it is most likely to follow.
|
|
19
|
+
+ 'The <tool_call> and </tool_call> tags are required: an object on its own is not a call.';
|
|
20
|
+
|
|
21
|
+
// What this adapter WRITES — into the tools' prompt, and into replayed history. The parser does not use
|
|
22
|
+
// these: a model writes tags as text, and not always in this exact form (see CALL_TAG below).
|
|
23
|
+
const OPEN_CALL = '<tool_call>';
|
|
24
|
+
const CLOSE_CALL = '</tool_call>';
|
|
25
|
+
|
|
26
|
+
function objectSchema(schema: unknown): object {
|
|
27
|
+
return { ...(schema as object), type: 'object' };
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function toolsPrompt(tools: readonly Tool[]): string {
|
|
31
|
+
const lines = tools.map(t => JSON.stringify({
|
|
32
|
+
type: 'function',
|
|
33
|
+
function: { name: t.name, description: t.description, parameters: objectSchema(t.inputSchema) },
|
|
34
|
+
}));
|
|
35
|
+
return `${TOOLS_PREAMBLE}\n${lines.join('\n')}\n${TOOLS_POSTAMBLE}`;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
// Each tool is its own tag whose opening text already names it, so once the model writes `<tool_call>` it
|
|
39
|
+
// can only go on to name a tool that exists, and its arguments must satisfy that tool's schema. This
|
|
40
|
+
// enforces SHAPE — keys, types, enums, required fields — not the content of a free string.
|
|
41
|
+
export function toolCallFormat(tools: readonly Tool[]): ResponseFormat {
|
|
42
|
+
return {
|
|
43
|
+
type: 'structural_tag',
|
|
44
|
+
structural_tag: {
|
|
45
|
+
type: 'structural_tag',
|
|
46
|
+
format: {
|
|
47
|
+
type: 'triggered_tags',
|
|
48
|
+
triggers: [OPEN_CALL],
|
|
49
|
+
tags: tools.map(t => ({
|
|
50
|
+
type: 'tag',
|
|
51
|
+
begin: `${OPEN_CALL}\n{"name": ${JSON.stringify(t.name)}, "arguments": `,
|
|
52
|
+
content: { type: 'json_schema', json_schema: objectSchema(t.inputSchema) },
|
|
53
|
+
end: `}\n${CLOSE_CALL}`,
|
|
54
|
+
})),
|
|
55
|
+
},
|
|
56
|
+
},
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
function jsonText(v: unknown): string {
|
|
61
|
+
try { return JSON.stringify(v) ?? 'null'; } catch { return String(v); }
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function renderCall(name: string, input: unknown): string {
|
|
65
|
+
return `${OPEN_CALL}\n{"name": ${JSON.stringify(name)}, "arguments": ${jsonText(input ?? {})}}\n${CLOSE_CALL}`;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* matbot's history as WebLLM chat messages, rendered the way Qwen's own chat template renders them: an
|
|
70
|
+
* assistant's calls as `<tool_call>` blocks after its text, and tool results as `<tool_response>` blocks in
|
|
71
|
+
* a user turn. Deterministic on purpose — WebLLM reuses its prefill when a request extends the previous
|
|
72
|
+
* conversation exactly, so every call rendering the same history the same way is what makes a tool round
|
|
73
|
+
* or a new turn prefill only what is new.
|
|
74
|
+
*/
|
|
75
|
+
export function toChatMessages(messages: Message[], tools: readonly Tool[]): ChatMessage[] {
|
|
76
|
+
const system = [
|
|
77
|
+
...messages.filter(m => m.role === 'system').flatMap(m => m.content).flatMap(c => (c.type === 'text' && c.text ? [c.text] : [])),
|
|
78
|
+
...(tools.length > 0 ? [toolsPrompt(tools)] : []),
|
|
79
|
+
].join('\n\n');
|
|
80
|
+
|
|
81
|
+
const out: ChatMessage[] = [];
|
|
82
|
+
const push = (role: ChatMessage['role'], content: string): void => {
|
|
83
|
+
if (!content) return;
|
|
84
|
+
const last = out.at(-1);
|
|
85
|
+
// Adjacent turns of one role are one turn to the template; merged, so the result is the same whether
|
|
86
|
+
// they arrived as one matbot message or several.
|
|
87
|
+
if (last && last.role === role) last.content += `\n${content}`;
|
|
88
|
+
else out.push({ role, content });
|
|
89
|
+
};
|
|
90
|
+
|
|
91
|
+
for (const msg of messages) {
|
|
92
|
+
switch (msg.role) {
|
|
93
|
+
case 'system':
|
|
94
|
+
case 'marker':
|
|
95
|
+
continue;
|
|
96
|
+
case 'tool':
|
|
97
|
+
push('user', msg.content.flatMap(c => c.type === 'tool-result'
|
|
98
|
+
? [`<tool_response>\n${typeof c.result === 'string' ? c.result : jsonText(c.result)}\n</tool_response>`]
|
|
99
|
+
: []).join('\n'));
|
|
100
|
+
continue;
|
|
101
|
+
case 'user':
|
|
102
|
+
case 'assistant': {
|
|
103
|
+
const parts: string[] = [];
|
|
104
|
+
for (const c of msg.content) {
|
|
105
|
+
switch (c.type) {
|
|
106
|
+
case 'text': if (c.text) parts.push(c.text); break;
|
|
107
|
+
case 'tool-call': parts.push(renderCall(c.name, c.input)); break;
|
|
108
|
+
case 'document': parts.push(textDocument(c) ?? `[Attached file: ${c.name ?? c.mimeType} — not readable by this model]`); break;
|
|
109
|
+
case 'file-ref': parts.push(`[Attached file: ${c.name}]`); break;
|
|
110
|
+
case 'image':
|
|
111
|
+
case 'image-url': parts.push('[Image omitted: this model does not accept images]'); break;
|
|
112
|
+
case 'audio': parts.push('[Audio omitted: this model does not accept audio]'); break;
|
|
113
|
+
// A template drops earlier turns' reasoning from the prompt, so it is not replayed.
|
|
114
|
+
case 'thinking':
|
|
115
|
+
case 'redacted-thinking':
|
|
116
|
+
case 'reasoning':
|
|
117
|
+
case 'refusal':
|
|
118
|
+
case 'tool-result':
|
|
119
|
+
case 'form':
|
|
120
|
+
case 'form-response':
|
|
121
|
+
case 'marker':
|
|
122
|
+
case 'unknown-content':
|
|
123
|
+
break;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
push(msg.role, parts.join('\n'));
|
|
127
|
+
continue;
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
return system ? [{ role: 'system', content: system }, ...out] : out;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
export type Piece =
|
|
135
|
+
| { kind: 'text'; text: string }
|
|
136
|
+
| { kind: 'thinking'; text: string }
|
|
137
|
+
| { kind: 'call'; name: string; arguments: unknown }
|
|
138
|
+
// A `<tool_call>` that closed, whose body is not a call the arguments can be read from. Never executed:
|
|
139
|
+
// running the tool with `{}` in its place would turn a formatting slip into a real side effect. The body
|
|
140
|
+
// is shown instead (see adapter.ts), so the reader sees what the model wrote and the model meets its own
|
|
141
|
+
// attempt again in history, where it can correct it.
|
|
142
|
+
| { kind: 'unparsed-call'; name: string | undefined; body: string }
|
|
143
|
+
// The reply ended inside a call — cut off by the token limit. Its arguments are unrecoverable.
|
|
144
|
+
| { kind: 'cut-call'; name: string; bytes: number };
|
|
145
|
+
|
|
146
|
+
type Mode = 'text' | 'think' | 'call';
|
|
147
|
+
|
|
148
|
+
// The tags as a model writes them, not only as this adapter writes them: whitespace before the bracket
|
|
149
|
+
// (`<tool_call >`), and `<toolcall>` with no underscore. The structural tag only ever produces the
|
|
150
|
+
// canonical form, so this is for the reply the grammar never got to constrain — which is exactly the
|
|
151
|
+
// reply that goes wrong.
|
|
152
|
+
const CALL_TAG = { re: /<tool_?call\s*>/i, enter: 'call' } as const;
|
|
153
|
+
const END_CALL_TAG = { re: /<\/tool_?call\s*>/i, enter: 'text' } as const;
|
|
154
|
+
const THINK_TAG = { re: /<think\s*>/i, enter: 'think' } as const;
|
|
155
|
+
const END_THINK_TAG = { re: /<\/think\s*>/i, enter: 'text' } as const;
|
|
156
|
+
|
|
157
|
+
type Tag = { at: number; length: number; enter: Mode };
|
|
158
|
+
type TagPattern = { readonly re: RegExp; readonly enter: Mode };
|
|
159
|
+
|
|
160
|
+
function firstTag(s: string, tags: readonly TagPattern[]): Tag | undefined {
|
|
161
|
+
let best: Tag | undefined;
|
|
162
|
+
for (const { re, enter } of tags) {
|
|
163
|
+
const m = re.exec(s);
|
|
164
|
+
if (m && (!best || m.index < best.at)) best = { at: m.index, length: m[0].length, enter };
|
|
165
|
+
}
|
|
166
|
+
return best;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
// The tag names a model may write, for recognising the tail of one split across two chunks.
|
|
170
|
+
const TAG_NAMES = ['tool_call', 'toolcall', 'think'];
|
|
171
|
+
|
|
172
|
+
function isTagPrefix(tail: string): boolean {
|
|
173
|
+
if (!tail.startsWith('<')) return false;
|
|
174
|
+
const rest = tail.slice(tail.startsWith('</') ? 2 : 1);
|
|
175
|
+
if (rest.includes('>')) return false;
|
|
176
|
+
return TAG_NAMES.some(n => n.startsWith(rest.trim()));
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
// A suffix that could still become a tag, however the stream split it: `<`, `</`, `<to`, `<tool_call`,
|
|
180
|
+
// `<tool_call `.
|
|
181
|
+
function tagPrefixLen(s: string): number {
|
|
182
|
+
const i = s.lastIndexOf('<');
|
|
183
|
+
return i >= 0 && isTagPrefix(s.slice(i)) ? s.length - i : 0;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
// The index just past the `}`/`]` that closes the value opening at `start`, or -1 while it is still open.
|
|
187
|
+
// Strings are skipped, so a brace inside an argument value cannot close it early.
|
|
188
|
+
function balancedEnd(s: string, start: number): number {
|
|
189
|
+
let depth = 0;
|
|
190
|
+
let inString = false;
|
|
191
|
+
let escape = false;
|
|
192
|
+
for (let i = start; i < s.length; i++) {
|
|
193
|
+
const c = s[i];
|
|
194
|
+
if (inString) {
|
|
195
|
+
if (escape) escape = false;
|
|
196
|
+
else if (c === '\\') escape = true;
|
|
197
|
+
else if (c === '"') inString = false;
|
|
198
|
+
continue;
|
|
199
|
+
}
|
|
200
|
+
if (c === '"') inString = true;
|
|
201
|
+
else if (c === '{' || c === '[') depth++;
|
|
202
|
+
else if (c === '}' || c === ']') { depth--; if (depth === 0) return i + 1; }
|
|
203
|
+
}
|
|
204
|
+
return -1;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
// `arguments` given as a JSON *string* as well as an object: Qwen's own template accepts both, and the
|
|
208
|
+
// smaller Qwen3 builds use them for the same call. Normalised here, so a tool always receives the object
|
|
209
|
+
// its schema declares rather than a string it cannot read.
|
|
210
|
+
function normaliseCall(v: unknown): { name: string; arguments: unknown } | undefined {
|
|
211
|
+
if (typeof v !== 'object' || v === null || Array.isArray(v)) return undefined;
|
|
212
|
+
const { name, arguments: args } = v as { name?: unknown; arguments?: unknown };
|
|
213
|
+
if (typeof name !== 'string' || !name) return undefined;
|
|
214
|
+
let input = args ?? {};
|
|
215
|
+
if (typeof input === 'string') {
|
|
216
|
+
try { input = JSON.parse(input) ?? {}; } catch { /* as written: the tool reports what it got */ }
|
|
217
|
+
}
|
|
218
|
+
return { name, arguments: input };
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
// Worth repairing in a call's body, and only tried once a plain parse has failed: a trailing comma before
|
|
222
|
+
// a closing bracket, which small models write constantly. Never applied to a body that parses, so an
|
|
223
|
+
// argument value that legitimately contains `,}` is not rewritten.
|
|
224
|
+
function parseCallObject(body: string): { name: string; arguments: unknown } | undefined {
|
|
225
|
+
for (const text of [body, body.replace(/,(\s*[}\]])/g, '$1')]) {
|
|
226
|
+
try {
|
|
227
|
+
const call = normaliseCall(JSON.parse(text.trim()));
|
|
228
|
+
if (call) return call;
|
|
229
|
+
} catch { /* next form */ }
|
|
230
|
+
}
|
|
231
|
+
return undefined;
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
function readCall(body: string): Piece | undefined {
|
|
235
|
+
const call = parseCallObject(body);
|
|
236
|
+
return call ? { kind: 'call', ...call } : undefined;
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
// Parallel calls may be written as a list of the same objects.
|
|
240
|
+
function readCalls(body: string): Piece[] | undefined {
|
|
241
|
+
const one = readCall(body);
|
|
242
|
+
if (one) return [one];
|
|
243
|
+
try {
|
|
244
|
+
const list: unknown = JSON.parse(body.trim().replace(/,(\s*[}\]])/g, '$1'));
|
|
245
|
+
if (!Array.isArray(list) || list.length === 0) return undefined;
|
|
246
|
+
const pieces: Piece[] = [];
|
|
247
|
+
for (const item of list) {
|
|
248
|
+
const call = normaliseCall(item);
|
|
249
|
+
if (!call) return undefined;
|
|
250
|
+
pieces.push({ kind: 'call', ...call });
|
|
251
|
+
}
|
|
252
|
+
return pieces;
|
|
253
|
+
} catch { return undefined; }
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
function nameIn(body: string): string | undefined {
|
|
257
|
+
return /"name"\s*:\s*"((?:[^"\\]|\\.)*)"/.exec(body)?.[1];
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
function cutCall(body: string): Piece {
|
|
261
|
+
return { kind: 'cut-call', name: nameIn(body) ?? 'unknown', bytes: body.length };
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
// A `[` immediately before `i` (whitespace between) belongs to the call: parallel calls are written as a
|
|
265
|
+
// list, and the bracket is part of the call rather than prose. Returns the index to treat as its start.
|
|
266
|
+
function listStart(s: string, i: number): number {
|
|
267
|
+
const m = /\[\s*$/.exec(s.slice(0, i));
|
|
268
|
+
return m ? m.index : i;
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
// Where a call written with no `<tool_call>` wrapper could begin, or -1: a `{` that has only a partial
|
|
272
|
+
// `"name"` after it, or that opened as `{"name":` and has not closed yet. Anything else with a `{` in it
|
|
273
|
+
// is prose, and is emitted as prose.
|
|
274
|
+
function bareStart(s: string): number {
|
|
275
|
+
// A list of calls begins with `[`, and that bracket is the call's own: held as soon as it is the last
|
|
276
|
+
// thing in the buffer, so it cannot be emitted as prose before its calls arrive.
|
|
277
|
+
const bracket = /\[\s*$/.exec(s);
|
|
278
|
+
if (bracket) return bracket.index;
|
|
279
|
+
// The EARLIEST `{` that could begin one, since the object's own start is what must be held: a nested
|
|
280
|
+
// `{` (`"arguments": {`) would otherwise hold only the inner object and let the call's header out.
|
|
281
|
+
for (let i = s.indexOf('{'); i >= 0; i = s.indexOf('{', i + 1)) {
|
|
282
|
+
const tail = s.slice(i);
|
|
283
|
+
// Still being written: `{`, `{"`, `{"n`, `{"name`, `{"name"`, `{"name":`.
|
|
284
|
+
if (/^\{\s*(?:"(?:n(?:a(?:m(?:e(?:"\s*:?)?)?)?)?)?)?\s*$/.test(tail)) return listStart(s, i);
|
|
285
|
+
// Opened as a call and not closed yet — or one of a list that has not closed. A closed call is not
|
|
286
|
+
// held here: `findBareCall` reads it, or it is prose.
|
|
287
|
+
if (/^\{\s*"name"\s*:/.test(tail)) {
|
|
288
|
+
const from = listStart(s, i);
|
|
289
|
+
if (balancedEnd(s, from) < 0) return from;
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
return -1;
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
function barePrefixLen(s: string): number {
|
|
296
|
+
const i = bareStart(s);
|
|
297
|
+
return i >= 0 ? s.length - i : 0;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
// A call the model wrote with no `<tool_call>` around it. Measured on Qwen3-1.7B, whose whole reply was an
|
|
301
|
+
// empty `<think>` block followed by `{"name": "tool_search", "arguments": {…}}`: it copies the call object
|
|
302
|
+
// its prompt shows it and drops the two tags around it, so the structural tag never fires, the parser finds
|
|
303
|
+
// no marker, and a complete, well-formed call arrives as JSON in a text bubble.
|
|
304
|
+
//
|
|
305
|
+
// Returns where it starts and ends as well as what it says, so any prose in front of it is emitted first.
|
|
306
|
+
function findBareCall(s: string): { at: number; end: number; pieces: Piece[] } | undefined {
|
|
307
|
+
for (let i = s.indexOf('{'); i >= 0; i = s.indexOf('{', i + 1)) {
|
|
308
|
+
if (!/^\{\s*"name"\s*:/.test(s.slice(i))) continue;
|
|
309
|
+
const end = balancedEnd(s, i);
|
|
310
|
+
if (end < 0) return undefined;
|
|
311
|
+
const body = s.slice(i, end);
|
|
312
|
+
// A call names its arguments as well as itself. Without that, `{"name": "…"}` is a JSON example in
|
|
313
|
+
// prose — and running a tool from one would be a call the model never made.
|
|
314
|
+
if (!/"arguments"\s*:/.test(body)) continue;
|
|
315
|
+
const call = readCall(body);
|
|
316
|
+
if (!call) continue;
|
|
317
|
+
// A list of calls is written `[{…}, {…}]`, and the `[` is part of the call rather than prose. The
|
|
318
|
+
// whole list is read at once, so its `]` is consumed with it.
|
|
319
|
+
const from = listStart(s, i);
|
|
320
|
+
if (from !== i) {
|
|
321
|
+
const listEnd = balancedEnd(s, from);
|
|
322
|
+
if (listEnd < 0) return undefined; // still arriving; `bareStart` holds it
|
|
323
|
+
const list = readCalls(s.slice(from, listEnd));
|
|
324
|
+
if (list) return { at: from, end: listEnd, pieces: list };
|
|
325
|
+
}
|
|
326
|
+
return { at: i, end, pieces: [call] };
|
|
327
|
+
}
|
|
328
|
+
return undefined;
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
export interface ParserOptions {
|
|
332
|
+
/**
|
|
333
|
+
* Read a call written without its `<tool_call>` tags as a call (see `findBareCall`). On by default;
|
|
334
|
+
* a profile turns it off with `parameters.lenientCalls: false`.
|
|
335
|
+
*/
|
|
336
|
+
bareCalls?: boolean;
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
/**
|
|
340
|
+
* Splits a streamed reply into prose, `<think>` reasoning, and `<tool_call>` blocks, as it arrives. A chunk
|
|
341
|
+
* can end part-way through a marker, so a suffix that could still become one is held back until the next
|
|
342
|
+
* chunk decides it. Whitespace is held the same way and trimmed at every marker, since the blank lines a
|
|
343
|
+
* model writes around a call or its reasoning are layout, not something it said.
|
|
344
|
+
*
|
|
345
|
+
* A call that will not parse is not a call with no arguments — it is unreadable, and reported as such
|
|
346
|
+
* rather than executed.
|
|
347
|
+
*/
|
|
348
|
+
export class ReplyParser {
|
|
349
|
+
private buf = '';
|
|
350
|
+
private mode: Mode = 'text';
|
|
351
|
+
// Set at every marker: the whitespace a model writes around one is layout, not content.
|
|
352
|
+
private trimNext = false;
|
|
353
|
+
private readonly lenient: boolean;
|
|
354
|
+
|
|
355
|
+
constructor(options: ParserOptions = {}) {
|
|
356
|
+
this.lenient = options.bareCalls !== false;
|
|
357
|
+
}
|
|
358
|
+
|
|
359
|
+
push(chunk: string): Piece[] {
|
|
360
|
+
this.buf += chunk;
|
|
361
|
+
return this.drain(false);
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
end(): Piece[] {
|
|
365
|
+
return this.drain(true);
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
private drain(final: boolean): Piece[] {
|
|
369
|
+
const out: Piece[] = [];
|
|
370
|
+
for (;;) {
|
|
371
|
+
if (this.mode === 'call') {
|
|
372
|
+
const close = firstTag(this.buf, [END_CALL_TAG]);
|
|
373
|
+
if (!close) {
|
|
374
|
+
if (!final) return out;
|
|
375
|
+
const body = this.buf;
|
|
376
|
+
this.buf = '';
|
|
377
|
+
if (!body.trim()) return out;
|
|
378
|
+
// Ended inside a call. A body that still parses is a call whose closing tag is what got cut;
|
|
379
|
+
// anything else was severed mid-argument, and its arguments are unrecoverable.
|
|
380
|
+
out.push(readCall(body) ?? cutCall(body));
|
|
381
|
+
return out;
|
|
382
|
+
}
|
|
383
|
+
const body = this.buf.slice(0, close.at);
|
|
384
|
+
// A closed tag whose body will not parse is NOT a call with empty arguments: shown, not run.
|
|
385
|
+
out.push(readCall(body) ?? { kind: 'unparsed-call', name: nameIn(body), body });
|
|
386
|
+
this.buf = this.buf.slice(close.at + close.length);
|
|
387
|
+
this.mode = 'text';
|
|
388
|
+
this.trimNext = true;
|
|
389
|
+
continue;
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
const kind: Piece['kind'] = this.mode === 'think' ? 'thinking' : 'text';
|
|
393
|
+
if (this.trimNext) {
|
|
394
|
+
this.buf = this.buf.trimStart();
|
|
395
|
+
if (this.buf) this.trimNext = false;
|
|
396
|
+
}
|
|
397
|
+
const tag = firstTag(this.buf, this.mode === 'think'
|
|
398
|
+
// A call ends reasoning. Qwen3 opens ` thinking` and, once it decides to call a tool, writes the
|
|
399
|
+
// call without closing the block — vLLM's own Qwen3 parser treats `<tool_call>` as the implicit end
|
|
400
|
+
// of reasoning for that reason. Without this the whole call arrives as thinking: invisible when
|
|
401
|
+
// thinking is off, and "reasoned, then forgot to call" when it is on.
|
|
402
|
+
? [END_THINK_TAG, CALL_TAG]
|
|
403
|
+
: [CALL_TAG, THINK_TAG]);
|
|
404
|
+
const bare = this.lenient ? findBareCall(this.buf) : undefined;
|
|
405
|
+
const at = tag && (!bare || tag.at <= bare.at) ? tag.at : bare?.at ?? -1;
|
|
406
|
+
|
|
407
|
+
if (at < 0) {
|
|
408
|
+
const held = final ? this.buf.length - this.buf.trimEnd().length : this.hold();
|
|
409
|
+
const text = this.buf.slice(0, this.buf.length - held);
|
|
410
|
+
if (text) out.push({ kind, text });
|
|
411
|
+
this.buf = final ? '' : this.buf.slice(this.buf.length - held);
|
|
412
|
+
return out;
|
|
413
|
+
}
|
|
414
|
+
const text = this.buf.slice(0, at).trimEnd();
|
|
415
|
+
if (text) out.push({ kind, text });
|
|
416
|
+
if (tag && at === tag.at) {
|
|
417
|
+
this.buf = this.buf.slice(at + tag.length);
|
|
418
|
+
this.mode = tag.enter;
|
|
419
|
+
} else {
|
|
420
|
+
out.push(...bare!.pieces);
|
|
421
|
+
this.buf = this.buf.slice(bare!.end);
|
|
422
|
+
// A bare call ends reasoning too: a model that never closed `<think>` was not going to.
|
|
423
|
+
this.mode = 'text';
|
|
424
|
+
}
|
|
425
|
+
this.trimNext = true;
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
private hold(): number {
|
|
430
|
+
let keep = Math.max(tagPrefixLen(this.buf), this.lenient ? barePrefixLen(this.buf) : 0);
|
|
431
|
+
while (keep < this.buf.length && /\s/.test(this.buf[this.buf.length - keep - 1] ?? '')) keep++;
|
|
432
|
+
return keep;
|
|
433
|
+
}
|
|
434
|
+
}
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
export { WebLLMAdapter, DEFAULT_MODEL } from './adapter.js';
|
|
2
|
+
export { webllmModelsTool, type WebLLMModelList, type WebLLMModelSummary } from './models.js';
|
|
3
|
+
|
|
4
|
+
import { PLUGIN_API_VERSION } from '@matatbread/matbot-plugin-api';
|
|
5
|
+
import type { MatbotPluginSpec } from '@matatbread/matbot-plugin-api';
|
|
6
|
+
import { WebLLMAdapter } from './adapter.js';
|
|
7
|
+
import { webllmModelsTool } from './models.js';
|
|
8
|
+
|
|
9
|
+
export const plugin: MatbotPluginSpec = {
|
|
10
|
+
apiVersion: PLUGIN_API_VERSION,
|
|
11
|
+
provider: (_config) => new WebLLMAdapter(),
|
|
12
|
+
tools: [webllmModelsTool],
|
|
13
|
+
};
|
package/src/models.ts
ADDED
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
import type { Tool, ToolContract, ToolResultOf } from '@matatbread/matbot-plugin-api';
|
|
2
|
+
import { resident } from './adapter.js';
|
|
3
|
+
import { appConfig, closestModels, gpuFeatures, library, offeredModels } from './webllm.js';
|
|
4
|
+
|
|
5
|
+
export interface WebLLMModelSummary {
|
|
6
|
+
id: string;
|
|
7
|
+
/** GPU memory the model needs, in MB, as WebLLM records it — at its default context window. */
|
|
8
|
+
vramMB?: number;
|
|
9
|
+
/** WebLLM's mark for a model that suits phones and low-memory GPUs. */
|
|
10
|
+
lowResource?: true;
|
|
11
|
+
/** The context window WebLLM sets for it, where it overrides the model's own. */
|
|
12
|
+
contextWindow?: number;
|
|
13
|
+
/** GPU features it needs. Absent when it needs none. */
|
|
14
|
+
requires?: string[];
|
|
15
|
+
/** Whether this page's GPU has every feature it needs. Absent when there is no GPU adapter to ask. */
|
|
16
|
+
runsHere?: boolean;
|
|
17
|
+
/** Already downloaded into this origin's cache, so using it costs no download. */
|
|
18
|
+
downloaded: boolean;
|
|
19
|
+
/** The model this provider has loaded (or is loading) right now. */
|
|
20
|
+
loaded?: true;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
// What `list` returns. Detail costs context, and a small model's window is the scarcest thing here — every
|
|
24
|
+
// model as a full row is most of an 8k window — so full rows are given only for what a call selected, and
|
|
25
|
+
// otherwise the bare ids, which already carry family, size and quantisation.
|
|
26
|
+
export interface WebLLMModelList {
|
|
27
|
+
/** Full rows for the models the call selected (`query`, `downloaded`). Absent when it selected nothing. */
|
|
28
|
+
matched?: WebLLMModelSummary[];
|
|
29
|
+
/** Every model's id, smallest first: given when the call selected nothing, or its selection matched nothing. */
|
|
30
|
+
others?: string[];
|
|
31
|
+
total: number;
|
|
32
|
+
/** What happened and how to narrow — for the model reading this, not for a program. */
|
|
33
|
+
hint: string;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
declare module '@matatbread/matbot-plugin-api' {
|
|
37
|
+
interface ToolContracts {
|
|
38
|
+
webllm_models:
|
|
39
|
+
| ToolContract<WebLLMModelList, { action: 'list'; query?: string; downloaded?: boolean }>
|
|
40
|
+
| ToolContract<{ model: string; deleted: true }, { action: 'delete'; model: string }>;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
type Input =
|
|
45
|
+
| { action: 'list'; query?: string; downloaded?: boolean }
|
|
46
|
+
| { action: 'delete'; model: string };
|
|
47
|
+
|
|
48
|
+
export const webllmModelsTool: Tool<ToolResultOf<'webllm_models'>> = {
|
|
49
|
+
name: 'webllm_models',
|
|
50
|
+
description:
|
|
51
|
+
'The models the WebLLM provider can run in this browser that can call tools: the Qwen3, Qwen2.5 and Hermes '
|
|
52
|
+
+ 'families. Others would answer in prose instead of using tools, so they are not listed. Every model runs '
|
|
53
|
+
+ 'on this device\'s GPU, and downloads (once, often gigabytes) the first time it is used. A model\'s id '
|
|
54
|
+
+ 'names its family, size and quantisation, e.g. "Qwen3-4B-q4f16_1-MLC" or "Hermes-3-Llama-3.2-3B-q4f16_1-MLC".\n\n'
|
|
55
|
+
+ '"list" with no other input returns every id, smallest first. Both inputs below are optional; give either '
|
|
56
|
+
+ 'to get full details for the models they select: "vramMB" (GPU memory needed), whether it "runsHere" '
|
|
57
|
+
+ '(this GPU has the features it needs), and whether it is "downloaded" or "loaded".\n'
|
|
58
|
+
+ '- "query": words that all appear in the model id, e.g. "qwen3", "4b" or "qwen3 4b". It searches ids only, '
|
|
59
|
+
+ 'so words like "webllm" or "model" match nothing. If nothing matches, every id is returned instead.\n'
|
|
60
|
+
+ '- "downloaded": true selects the models already on this device.\n'
|
|
61
|
+
+ '"q4f16" builds are smaller and faster where the GPU supports them, "q4f32" the fallback. To switch a '
|
|
62
|
+
+ 'profile to a model, use the provider tool\'s "update" with the id.\n\n'
|
|
63
|
+
+ '"delete" removes a downloaded model from this device to free its storage; it downloads again if used.',
|
|
64
|
+
inputSchema: {
|
|
65
|
+
type: 'object',
|
|
66
|
+
required: ['action'],
|
|
67
|
+
properties: {
|
|
68
|
+
action: { type: 'string', enum: ['list', 'delete'] },
|
|
69
|
+
query: { type: 'string', description: 'list, optional: words that must all appear in the model id, case-insensitively.' },
|
|
70
|
+
downloaded: { type: 'boolean', description: 'list, optional: true to select just the models already downloaded.' },
|
|
71
|
+
model: { type: 'string', description: 'delete only: the model id.' },
|
|
72
|
+
},
|
|
73
|
+
},
|
|
74
|
+
executor: {
|
|
75
|
+
async *execute(rawInput) {
|
|
76
|
+
const input = rawInput as Input;
|
|
77
|
+
const now = resident();
|
|
78
|
+
const lib = await library(now.library);
|
|
79
|
+
const config = appConfig(lib, now.cache);
|
|
80
|
+
const models = offeredModels(lib);
|
|
81
|
+
const total = models.length;
|
|
82
|
+
|
|
83
|
+
switch (input.action) {
|
|
84
|
+
case 'list': {
|
|
85
|
+
const words = (input.query ?? '').toLowerCase().split(/\s+/).filter(Boolean);
|
|
86
|
+
const onlyDownloaded = input.downloaded === true;
|
|
87
|
+
const ids = models.map(r => r.model_id);
|
|
88
|
+
if (words.length === 0 && !onlyDownloaded) {
|
|
89
|
+
yield { type: 'result', value: { others: ids, total,
|
|
90
|
+
hint: 'Ids only. Call again with "query" (words from an id, e.g. "qwen3 4b") for the details of those models.' } };
|
|
91
|
+
return;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
const byName = models.filter(r => words.every(w => r.model_id.toLowerCase().includes(w)));
|
|
95
|
+
const cached = await Promise.all(byName.map(r => lib.hasModelInCache(r.model_id, config).catch(() => false)));
|
|
96
|
+
const picked = byName.map((r, i) => ({ r, downloaded: cached[i] === true })).filter(x => !onlyDownloaded || x.downloaded);
|
|
97
|
+
const asked = [words.length > 0 ? `id containing ${words.map(w => `"${w}"`).join(' and ')}` : '', onlyDownloaded ? 'already downloaded' : '']
|
|
98
|
+
.filter(Boolean).join(', ');
|
|
99
|
+
if (picked.length === 0) {
|
|
100
|
+
yield { type: 'result', value: { matched: [], others: ids, total,
|
|
101
|
+
hint: `No model matched (${asked}), so "others" lists every id. Query with words that appear in one, e.g. a family and size like "qwen3 4b".` } };
|
|
102
|
+
return;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
const have = await gpuFeatures();
|
|
106
|
+
const matched = picked.map(({ r, downloaded }): WebLLMModelSummary => {
|
|
107
|
+
const requires = r.required_features ?? [];
|
|
108
|
+
return {
|
|
109
|
+
id: r.model_id,
|
|
110
|
+
...(r.vram_required_MB !== undefined ? { vramMB: Math.round(r.vram_required_MB) } : {}),
|
|
111
|
+
...(r.low_resource_required ? { lowResource: true as const } : {}),
|
|
112
|
+
...(r.overrides?.context_window_size !== undefined ? { contextWindow: r.overrides.context_window_size } : {}),
|
|
113
|
+
...(requires.length > 0 ? { requires } : {}),
|
|
114
|
+
...(have ? { runsHere: requires.every(f => have.has(f)) } : {}),
|
|
115
|
+
downloaded,
|
|
116
|
+
...(r.model_id === now.model ? { loaded: true as const } : {}),
|
|
117
|
+
};
|
|
118
|
+
});
|
|
119
|
+
yield { type: 'result', value: { matched, total, hint: `${matched.length} of ${total} models matched (${asked}).` } };
|
|
120
|
+
return;
|
|
121
|
+
}
|
|
122
|
+
case 'delete': {
|
|
123
|
+
if (!input.model) { yield { type: 'error', message: 'action "delete" requires "model".' }; return; }
|
|
124
|
+
if (!models.some(r => r.model_id === input.model)) {
|
|
125
|
+
const near = closestModels(input.model, models);
|
|
126
|
+
yield { type: 'error', message: `"${input.model}" is not a WebLLM model id.${near.length > 0 ? ` Closest: ${near.join(', ')}.` : ''}` };
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
129
|
+
await lib.deleteModelAllInfoInCache(input.model, config);
|
|
130
|
+
yield { type: 'result', value: { model: input.model, deleted: true } };
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
default:
|
|
134
|
+
yield { type: 'error', message: `Unknown action "${(input as { action?: unknown }).action}". Use "list" or "delete".` };
|
|
135
|
+
}
|
|
136
|
+
},
|
|
137
|
+
},
|
|
138
|
+
};
|