@matatbread/matbot-provider-webllm 0.0.0-stage → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +205 -2
- package/package.json +48 -3
- package/src/adapter.ts +316 -0
- package/src/format.ts +434 -0
- package/src/index.ts +13 -0
- package/src/models.ts +138 -0
- package/src/webllm.ts +204 -0
- package/src/window.ts +161 -0
package/src/adapter.ts
ADDED
|
@@ -0,0 +1,316 @@
|
|
|
1
|
+
import type { ProviderAdapter, ProviderConfig, Message, Tool, CompletionEvent, HealthStatus } from '@matatbread/matbot-plugin-api';
|
|
2
|
+
import { withFailure } from '@matatbread/matbot-plugin-api';
|
|
3
|
+
import { ReplyParser, toChatMessages, toolCallFormat, type Piece } from './format.js';
|
|
4
|
+
import { CHARS_PER_TOKEN, fitToWindow, forgetFit, isWindowOverflow, omittedNote } from './window.js';
|
|
5
|
+
import { DEFAULT_LIBRARY, cacheBackend, checkModel, library, loadEngine, offeredModels, type CacheBackend, type ChatRequest, type InitProgressReport, type Loaded } from './webllm.js';
|
|
6
|
+
|
|
7
|
+
export const DEFAULT_MODEL = 'Qwen3-4B-q4f16_1-MLC';
|
|
8
|
+
// WebLLM's prebuilt Qwen3 records cap the window at 4096, which matbot's system context and a tool list
|
|
9
|
+
// alone can fill. 8192 costs roughly another 0.6 GB of GPU memory for a 4B model; `contextWindow` changes it.
|
|
10
|
+
const DEFAULT_CONTEXT = 8192;
|
|
11
|
+
// Room kept for the reply when `maxTokens` does not say how much.
|
|
12
|
+
const REPLY_RESERVE = 1024;
|
|
13
|
+
// Characters per token assumed on each attempt: the default, then stricter after each overflow WebLLM reports.
|
|
14
|
+
const OVERFLOW_ESTIMATES = [CHARS_PER_TOKEN, 2.25, 1.5] as const;
|
|
15
|
+
|
|
16
|
+
function noteMessage(text: string): Message {
|
|
17
|
+
return { id: 'webllm-omitted-note', role: 'system', content: [{ type: 'text', text }], createdAt: new Date(0).toISOString(), traceId: '' };
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
const NO_WEBGPU = 'webllm: this browser has no WebGPU (navigator.gpu), which WebLLM needs to run a model. '
|
|
21
|
+
+ 'Chrome and Edge have it on most desktops; elsewhere check chrome://gpu or the browser\'s WebGPU flag.';
|
|
22
|
+
|
|
23
|
+
interface Slot {
|
|
24
|
+
key: string;
|
|
25
|
+
loaded: Promise<Loaded>;
|
|
26
|
+
holders: number;
|
|
27
|
+
retired: boolean;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
// One model resident at a time, shared by every provider profile using this adapter: two engines would
|
|
31
|
+
// hold two models' weights in GPU memory, which is the resource this provider is shortest of. A different
|
|
32
|
+
// model or window replaces it, but the old engine is unloaded only once no call is still using it.
|
|
33
|
+
let slot: Slot | undefined;
|
|
34
|
+
|
|
35
|
+
// What the provider last loaded, for `webllm_models`: the library and cache it must read through to see the
|
|
36
|
+
// same models and downloads, and the model resident now (absent once replaced). Remembered past an unload,
|
|
37
|
+
// since a profile's own `library`/`cache` still applies to its next call.
|
|
38
|
+
let lastLoaded: { library: string; cache: CacheBackend | undefined } | undefined;
|
|
39
|
+
|
|
40
|
+
export function resident(): { library: string; cache: CacheBackend | undefined; model?: string } {
|
|
41
|
+
const model = slot && !slot.retired ? (JSON.parse(slot.key) as unknown[])[1] : undefined;
|
|
42
|
+
return {
|
|
43
|
+
...(lastLoaded ?? { library: DEFAULT_LIBRARY, cache: cacheBackend(undefined) }),
|
|
44
|
+
...(typeof model === 'string' ? { model } : {}),
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function unload(s: Slot): void {
|
|
49
|
+
void s.loaded.then(({ engine, worker }) => engine.unload().finally(() => worker?.terminate()), () => {});
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function release(s: Slot): void {
|
|
53
|
+
s.holders--;
|
|
54
|
+
if (s.retired && s.holders === 0) unload(s);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function gpu(): { requestAdapter(): Promise<unknown> } | undefined {
|
|
58
|
+
return typeof navigator === 'undefined' ? undefined
|
|
59
|
+
: (navigator as { gpu?: { requestAdapter(): Promise<unknown> } }).gpu;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Open-weight models run in the page on the GPU through WebLLM — Qwen3 by default — loaded from a CDN on
|
|
64
|
+
* first use, so the library costs nothing to an install that never picks this provider. No endpoint, no
|
|
65
|
+
* key, no network after the model is cached.
|
|
66
|
+
*
|
|
67
|
+
* `model` is a WebLLM model id (`Qwen3-8B-q4f16_1-MLC`, `Llama-3.2-3B-Instruct-q4f16_1-MLC`, …). Tool calling
|
|
68
|
+
* is this adapter's own, in Qwen's `<tool_call>` format (see format.ts), so it is reliable only on models
|
|
69
|
+
* trained on that format: Qwen and Hermes.
|
|
70
|
+
*/
|
|
71
|
+
export class WebLLMAdapter implements ProviderAdapter {
|
|
72
|
+
readonly name = 'webllm';
|
|
73
|
+
|
|
74
|
+
complete(
|
|
75
|
+
messages: Message[],
|
|
76
|
+
config: ProviderConfig,
|
|
77
|
+
tools: readonly Tool[],
|
|
78
|
+
signal: AbortSignal,
|
|
79
|
+
): AsyncIterable<CompletionEvent> {
|
|
80
|
+
return this.stream(messages, config, tools, signal);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
private async *stream(
|
|
84
|
+
messages: Message[],
|
|
85
|
+
config: ProviderConfig,
|
|
86
|
+
tools: readonly Tool[],
|
|
87
|
+
signal: AbortSignal,
|
|
88
|
+
): AsyncIterable<CompletionEvent> {
|
|
89
|
+
if (!gpu()) throw new Error(NO_WEBGPU);
|
|
90
|
+
const params = config.parameters ?? {};
|
|
91
|
+
const model = config.model || DEFAULT_MODEL;
|
|
92
|
+
const libraryUrl = typeof params['library'] === 'string' ? params['library'] : DEFAULT_LIBRARY;
|
|
93
|
+
const contextWindow = typeof params['contextWindow'] === 'number' ? params['contextWindow'] : DEFAULT_CONTEXT;
|
|
94
|
+
const offered = params['tools'] === false ? [] : tools;
|
|
95
|
+
const cache = cacheBackend(params['cache']);
|
|
96
|
+
|
|
97
|
+
// What the reply may use is taken off the window first: WebLLM decodes into the same window, so a
|
|
98
|
+
// prompt that only just fits leaves the answer nowhere to go.
|
|
99
|
+
const reserve = typeof params.maxTokens === 'number' ? params.maxTokens : REPLY_RESERVE;
|
|
100
|
+
const budget = Math.max(0, contextWindow - reserve);
|
|
101
|
+
const requestFor = (charsPerToken: number): { request: ChatRequest; omitted: number; shrunk: number } => {
|
|
102
|
+
const fitted = fitToWindow(messages, offered, budget, charsPerToken);
|
|
103
|
+
const kept = fitted.omitted > 0 ? [...fitted.messages, noteMessage(omittedNote(fitted.omitted))] : fitted.messages;
|
|
104
|
+
const chat = toChatMessages(kept, offered);
|
|
105
|
+
if (chat.at(-1)?.role !== 'user')
|
|
106
|
+
throw withFailure(new Error('webllm: the conversation must end with a user turn or a tool result — WebLLM cannot continue an assistant message.'), 'invalid');
|
|
107
|
+
return {
|
|
108
|
+
omitted: fitted.omitted,
|
|
109
|
+
shrunk: fitted.shrunk,
|
|
110
|
+
request: {
|
|
111
|
+
messages: chat,
|
|
112
|
+
stream: true,
|
|
113
|
+
stream_options: { include_usage: true },
|
|
114
|
+
...(typeof params.maxTokens === 'number' ? { max_tokens: params.maxTokens } : {}),
|
|
115
|
+
...(typeof params.temperature === 'number' ? { temperature: params.temperature } : {}),
|
|
116
|
+
...(typeof params.topP === 'number' ? { top_p: params.topP } : {}),
|
|
117
|
+
...(params.stopSequences ? { stop: params.stopSequences } : {}),
|
|
118
|
+
...(offered.length > 0 ? { response_format: toolCallFormat(offered) } : {}),
|
|
119
|
+
// Qwen3 reasons before every answer unless told not to, which on a laptop GPU is most of the wait.
|
|
120
|
+
extra_body: { enable_thinking: params['thinking'] === true },
|
|
121
|
+
},
|
|
122
|
+
};
|
|
123
|
+
};
|
|
124
|
+
// Fail before loading a model if the call cannot be sent at all.
|
|
125
|
+
requestFor(CHARS_PER_TOKEN);
|
|
126
|
+
|
|
127
|
+
const held = yield* this.acquire(libraryUrl, model, contextWindow, cache, offered.length > 0, signal);
|
|
128
|
+
let onAbort: (() => void) | undefined;
|
|
129
|
+
try {
|
|
130
|
+
const { engine } = await held.loaded;
|
|
131
|
+
onAbort = (): void => void engine.interruptGenerate();
|
|
132
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
133
|
+
|
|
134
|
+
// The estimate is only an estimate: where WebLLM still refuses the prompt — at prefill, before any
|
|
135
|
+
// output — the call is re-fitted with a stricter one rather than failed.
|
|
136
|
+
for (const [attempt, charsPerToken] of OVERFLOW_ESTIMATES.entries()) {
|
|
137
|
+
const { request, omitted, shrunk } = requestFor(charsPerToken);
|
|
138
|
+
if (omitted > 0 || shrunk > 0)
|
|
139
|
+
console.warn(`[webllm] to fit the ${contextWindow}-token context window: `
|
|
140
|
+
+ [shrunk > 0 ? `${shrunk} earlier tool result${shrunk === 1 ? '' : 's'} shortened` : '',
|
|
141
|
+
omitted > 0 ? `${omitted} earliest turn${omitted === 1 ? '' : 's'} left out (the model is told)` : '']
|
|
142
|
+
.filter(Boolean).join('; ') + '. The session is unchanged.');
|
|
143
|
+
const raw: string[] = [];
|
|
144
|
+
let finish: string | undefined;
|
|
145
|
+
let usage: Record<string, unknown> | undefined;
|
|
146
|
+
let emitted = false;
|
|
147
|
+
// `lenientCalls` reads a call the model wrote without its `<tool_call>` tags as a call. Off
|
|
148
|
+
// only where a model is found to write JSON examples as prose (see ReplyParser).
|
|
149
|
+
const parser = new ReplyParser({ bareCalls: params['lenientCalls'] !== false });
|
|
150
|
+
let reasoning = '';
|
|
151
|
+
const events = function* (pieces: Piece[]): Generator<CompletionEvent> {
|
|
152
|
+
for (const p of pieces) {
|
|
153
|
+
emitted = true;
|
|
154
|
+
switch (p.kind) {
|
|
155
|
+
case 'text': yield { type: 'text-delta', delta: p.text }; break;
|
|
156
|
+
case 'thinking': reasoning += p.text; yield { type: 'thinking', delta: p.text }; break;
|
|
157
|
+
case 'call': yield { type: 'tool-call', id: `call_${crypto.randomUUID()}`, name: p.name, input: p.arguments }; break;
|
|
158
|
+
case 'cut-call':
|
|
159
|
+
yield { type: 'tool-call', id: `call_${crypto.randomUUID()}`, name: p.name, input: {},
|
|
160
|
+
truncated: { bytes: p.bytes, ...(finish ? { stopReason: finish } : {}) } };
|
|
161
|
+
break;
|
|
162
|
+
case 'unparsed-call':
|
|
163
|
+
// The tool is deliberately NOT called. Running it with `{}` in place of arguments it never
|
|
164
|
+
// had turns a formatting slip into a real side effect — and that is what a call the model
|
|
165
|
+
// got wrong used to look like from outside: a tool that ran on nothing and returned its
|
|
166
|
+
// own error, or a dump of its defaults, which then read as the model's result.
|
|
167
|
+
// What the model actually wrote is shown instead: the reader sees the JSON, and the model
|
|
168
|
+
// meets its own attempt in history and can correct it on the next round.
|
|
169
|
+
console.warn('[webllm] unreadable tool call — not run', { name: p.name, body: p.body, finish });
|
|
170
|
+
yield { type: 'text-delta', delta: p.body };
|
|
171
|
+
break;
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
};
|
|
175
|
+
|
|
176
|
+
try {
|
|
177
|
+
for await (const chunk of await engine.chat.completions.create(request)) {
|
|
178
|
+
const choice = chunk.choices[0];
|
|
179
|
+
const delta = choice?.delta.content;
|
|
180
|
+
if (delta) { raw.push(delta); yield* events(parser.push(delta)); }
|
|
181
|
+
if (choice?.finish_reason) finish = choice.finish_reason;
|
|
182
|
+
if (chunk.usage) usage = chunk.usage;
|
|
183
|
+
}
|
|
184
|
+
} catch (e) {
|
|
185
|
+
if (isWindowOverflow(e) && !emitted && attempt < OVERFLOW_ESTIMATES.length - 1 && !signal.aborted) {
|
|
186
|
+
forgetFit(messages);
|
|
187
|
+
continue;
|
|
188
|
+
}
|
|
189
|
+
if (isWindowOverflow(e))
|
|
190
|
+
throw withFailure(new Error(`webllm: the conversation does not fit the ${contextWindow}-token context window even with its `
|
|
191
|
+
+ 'earlier turns left out — the system prompt, the tool list and the current turn alone are too long. '
|
|
192
|
+
+ 'Raise `contextWindow` (it costs GPU memory), offer fewer tools (the tool-router plugin, or '
|
|
193
|
+
+ '`parameters.tools: false`), or start a new conversation.', { cause: e }), 'invalid');
|
|
194
|
+
throw e;
|
|
195
|
+
} finally {
|
|
196
|
+
// What the model was sent and what it wrote, verbatim, at the level DevTools hides unless "Verbose".
|
|
197
|
+
console.debug('[webllm] exchange', { model, attempt, omitted, shrunk, request, reply: raw.join(''), finish, usage });
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
signal.throwIfAborted();
|
|
201
|
+
yield* events(parser.end());
|
|
202
|
+
if (reasoning) yield { type: 'reasoning-block', reasoning };
|
|
203
|
+
if (usage) {
|
|
204
|
+
yield {
|
|
205
|
+
type: 'usage',
|
|
206
|
+
inputTokens: typeof usage['prompt_tokens'] === 'number' ? usage['prompt_tokens'] : 0,
|
|
207
|
+
outputTokens: typeof usage['completion_tokens'] === 'number' ? usage['completion_tokens'] : 0,
|
|
208
|
+
reported: { ...usage },
|
|
209
|
+
};
|
|
210
|
+
}
|
|
211
|
+
if (finish === 'length') yield { type: 'truncated', reason: 'max-tokens', raw: finish };
|
|
212
|
+
yield { type: 'done' };
|
|
213
|
+
return;
|
|
214
|
+
}
|
|
215
|
+
} finally {
|
|
216
|
+
if (onAbort) signal.removeEventListener('abort', onAbort);
|
|
217
|
+
release(held);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
// The caller owns one `release`, here on success and inside on any failure — including abandoning the
|
|
222
|
+
// iteration during a progress event, which runs `finally` but no `catch`. Progress goes to the call that
|
|
223
|
+
// started the load; any other waits on the same promise.
|
|
224
|
+
private async *acquire(
|
|
225
|
+
libraryUrl: string,
|
|
226
|
+
model: string,
|
|
227
|
+
contextWindow: number,
|
|
228
|
+
cache: CacheBackend | undefined,
|
|
229
|
+
withTools: boolean,
|
|
230
|
+
signal: AbortSignal,
|
|
231
|
+
): AsyncGenerator<CompletionEvent, Slot> {
|
|
232
|
+
const key = JSON.stringify([libraryUrl, model, contextWindow, cache]);
|
|
233
|
+
if (slot && slot.key === key) {
|
|
234
|
+
const s = slot;
|
|
235
|
+
s.holders++;
|
|
236
|
+
let ok = false;
|
|
237
|
+
try {
|
|
238
|
+
await untilAborted(s.loaded, signal);
|
|
239
|
+
ok = true;
|
|
240
|
+
return s;
|
|
241
|
+
} finally {
|
|
242
|
+
if (!ok) release(s);
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
// Before the slot changes hands: a request naming a model that cannot run must not unload the one that can.
|
|
247
|
+
await checkModel(await library(libraryUrl), model, withTools);
|
|
248
|
+
lastLoaded = { library: libraryUrl, cache };
|
|
249
|
+
const reports: InitProgressReport[] = [];
|
|
250
|
+
let wake = (): void => {};
|
|
251
|
+
const loaded = loadEngine(libraryUrl, model, contextWindow, cache, r => { reports.push(r); wake(); });
|
|
252
|
+
const mine: Slot = { key, loaded, holders: 1, retired: false };
|
|
253
|
+
const previous = slot;
|
|
254
|
+
slot = mine;
|
|
255
|
+
if (previous) {
|
|
256
|
+
previous.retired = true;
|
|
257
|
+
if (previous.holders === 0) unload(previous);
|
|
258
|
+
}
|
|
259
|
+
loaded.catch(() => { mine.retired = true; if (slot === mine) slot = undefined; });
|
|
260
|
+
|
|
261
|
+
let settled = false;
|
|
262
|
+
loaded.then(() => { settled = true; wake(); }, () => { settled = true; wake(); });
|
|
263
|
+
const onAbort = (): void => wake();
|
|
264
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
265
|
+
let ok = false;
|
|
266
|
+
try {
|
|
267
|
+
while (!settled && !signal.aborted) {
|
|
268
|
+
await new Promise<void>(r => { wake = r; if (settled || reports.length > 0 || signal.aborted) r(); });
|
|
269
|
+
const report = reports.splice(0).at(-1);
|
|
270
|
+
if (report) {
|
|
271
|
+
console.debug('[webllm] load', report.text);
|
|
272
|
+
yield {
|
|
273
|
+
type: 'progress',
|
|
274
|
+
pct: Math.round(report.progress * 100),
|
|
275
|
+
message: `${report.text.startsWith('Fetching') ? 'Downloading' : 'Loading'} ${model}`,
|
|
276
|
+
};
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
signal.throwIfAborted();
|
|
280
|
+
await loaded;
|
|
281
|
+
ok = true;
|
|
282
|
+
return mine;
|
|
283
|
+
} finally {
|
|
284
|
+
signal.removeEventListener('abort', onAbort);
|
|
285
|
+
if (!ok) release(mine);
|
|
286
|
+
}
|
|
287
|
+
}
|
|
288
|
+
|
|
289
|
+
async health(): Promise<HealthStatus> {
|
|
290
|
+
const g = gpu();
|
|
291
|
+
if (!g) return { status: 'down', reason: NO_WEBGPU };
|
|
292
|
+
try {
|
|
293
|
+
return (await g.requestAdapter()) ? { status: 'ok' } : { status: 'down', reason: 'WebGPU is present but offered no GPU adapter.' };
|
|
294
|
+
} catch (e) {
|
|
295
|
+
return { status: 'down', reason: e instanceof Error ? e.message : String(e) };
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
async models(config: ProviderConfig): Promise<string[]> {
|
|
300
|
+
const params = config.parameters ?? {};
|
|
301
|
+
const lib = await library(typeof params['library'] === 'string' ? params['library'] : DEFAULT_LIBRARY);
|
|
302
|
+
return offeredModels(lib).map(r => r.model_id);
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
function untilAborted<T>(p: Promise<T>, signal: AbortSignal): Promise<T> {
|
|
307
|
+
return new Promise<T>((resolve, reject) => {
|
|
308
|
+
if (signal.aborted) { reject(signal.reason); return; }
|
|
309
|
+
const onAbort = (): void => reject(signal.reason);
|
|
310
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
311
|
+
p.then(
|
|
312
|
+
v => { signal.removeEventListener('abort', onAbort); resolve(v); },
|
|
313
|
+
e => { signal.removeEventListener('abort', onAbort); reject(e); },
|
|
314
|
+
);
|
|
315
|
+
});
|
|
316
|
+
}
|