@matatbread/matbot-provider-webllm 0.0.0-stage → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +205 -2
- package/package.json +48 -3
- package/src/adapter.ts +316 -0
- package/src/format.ts +434 -0
- package/src/index.ts +13 -0
- package/src/models.ts +138 -0
- package/src/webllm.ts +204 -0
- package/src/window.ts +161 -0
package/src/webllm.ts
ADDED
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
// The slice of WebLLM (https://github.com/mlc-ai/web-llm) this adapter uses. Declared locally rather than
|
|
2
|
+
// taken from the package because the package is never installed: the library is several MB, matters only
|
|
3
|
+
// to someone who chooses this provider, and is imported from a CDN at the moment it is first needed.
|
|
4
|
+
|
|
5
|
+
import { closestIds, withFailure } from '@matatbread/matbot-plugin-api';
|
|
6
|
+
|
|
7
|
+
export interface InitProgressReport { progress: number; timeElapsed: number; text: string }
|
|
8
|
+
|
|
9
|
+
export interface ChatMessage { role: 'system' | 'user' | 'assistant'; content: string }
|
|
10
|
+
|
|
11
|
+
export interface ChatCompletionChunk {
|
|
12
|
+
choices: Array<{
|
|
13
|
+
delta: { content?: string | null };
|
|
14
|
+
finish_reason: 'stop' | 'length' | 'tool_calls' | 'abort' | null;
|
|
15
|
+
}>;
|
|
16
|
+
usage?: { prompt_tokens: number; completion_tokens: number; [key: string]: unknown };
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export interface ResponseFormat { type: 'structural_tag'; structural_tag: object }
|
|
20
|
+
|
|
21
|
+
export interface ChatRequest {
|
|
22
|
+
messages: ChatMessage[];
|
|
23
|
+
stream: true;
|
|
24
|
+
stream_options: { include_usage: true };
|
|
25
|
+
max_tokens?: number;
|
|
26
|
+
temperature?: number;
|
|
27
|
+
top_p?: number;
|
|
28
|
+
stop?: string[];
|
|
29
|
+
response_format?: ResponseFormat;
|
|
30
|
+
extra_body?: { enable_thinking?: boolean };
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export interface Engine {
|
|
34
|
+
chat: { completions: { create(request: ChatRequest): Promise<AsyncIterable<ChatCompletionChunk>> } };
|
|
35
|
+
interruptGenerate(): void | Promise<void>;
|
|
36
|
+
unload(): Promise<void>;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export type CacheBackend = 'cache' | 'indexeddb' | 'opfs' | 'cross-origin';
|
|
40
|
+
|
|
41
|
+
interface ChatOptions { context_window_size?: number }
|
|
42
|
+
|
|
43
|
+
export interface ModelRecord {
|
|
44
|
+
model: string;
|
|
45
|
+
model_id: string;
|
|
46
|
+
model_lib: string;
|
|
47
|
+
overrides?: ChatOptions;
|
|
48
|
+
vram_required_MB?: number;
|
|
49
|
+
low_resource_required?: boolean;
|
|
50
|
+
required_features?: string[];
|
|
51
|
+
// WebLLM's `ModelType`: 0 an LLM, 1 an embedding model, 2 a vision-language model.
|
|
52
|
+
model_type?: number;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export interface AppConfig { model_list: ModelRecord[]; cacheBackend?: CacheBackend }
|
|
56
|
+
interface EngineConfig { initProgressCallback?: (report: InitProgressReport) => void; appConfig?: AppConfig }
|
|
57
|
+
|
|
58
|
+
export interface WebLLM {
|
|
59
|
+
prebuiltAppConfig: AppConfig;
|
|
60
|
+
CreateWebWorkerMLCEngine(worker: Worker, modelId: string, config?: EngineConfig, chatOpts?: ChatOptions): Promise<Engine>;
|
|
61
|
+
CreateMLCEngine(modelId: string, config?: EngineConfig, chatOpts?: ChatOptions): Promise<Engine>;
|
|
62
|
+
hasModelInCache(modelId: string, appConfig?: AppConfig): Promise<boolean>;
|
|
63
|
+
deleteModelAllInfoInCache(modelId: string, appConfig?: AppConfig): Promise<void>;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const CACHE_BACKENDS: ReadonlySet<string> = new Set<CacheBackend>(['cache', 'indexeddb', 'opfs', 'cross-origin']);
|
|
67
|
+
|
|
68
|
+
// WebLLM keeps weights in Cache Storage by default, which a page opened from `file://` — how the bundle is
|
|
69
|
+
// often run — cannot use; IndexedDB works there. Configured wins.
|
|
70
|
+
export function cacheBackend(configured: unknown): CacheBackend | undefined {
|
|
71
|
+
if (typeof configured === 'string' && CACHE_BACKENDS.has(configured)) return configured as CacheBackend;
|
|
72
|
+
return typeof location !== 'undefined' && location.protocol === 'file:' ? 'indexeddb' : undefined;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// The app config every call into the library must agree on, or a model downloaded into one cache is
|
|
76
|
+
// looked for in another.
|
|
77
|
+
export function appConfig(lib: WebLLM, cache: CacheBackend | undefined): AppConfig {
|
|
78
|
+
return cache ? { ...lib.prebuiltAppConfig, cacheBackend: cache } : lib.prebuiltAppConfig;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// 0.2.85 (2026-09-08) rather than the newest: pinned, so a release cannot change behaviour underneath an
|
|
82
|
+
// install, and not the first to try a fresh one. `parameters.library` points elsewhere.
|
|
83
|
+
export const DEFAULT_LIBRARY = 'https://cdn.jsdelivr.net/npm/@mlc-ai/web-llm@0.2.85/+esm';
|
|
84
|
+
|
|
85
|
+
const libraries = new Map<string, Promise<WebLLM>>();
|
|
86
|
+
|
|
87
|
+
export function library(url: string): Promise<WebLLM> {
|
|
88
|
+
let lib = libraries.get(url);
|
|
89
|
+
if (!lib) {
|
|
90
|
+
lib = import(/* @vite-ignore */ url) as Promise<WebLLM>;
|
|
91
|
+
libraries.set(url, lib);
|
|
92
|
+
lib.catch(() => libraries.delete(url));
|
|
93
|
+
}
|
|
94
|
+
return lib;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
export interface Loaded { engine: Engine; worker: Worker | undefined }
|
|
98
|
+
|
|
99
|
+
// In a worker where the page has one, so tokenising and sampling never stall the UI: a module worker built
|
|
100
|
+
// from a blob, since a single-file bundle has no script URL to give it, importing the same library URL.
|
|
101
|
+
// Without `Worker` (a test, an unusual host) the engine runs in-page.
|
|
102
|
+
export async function loadEngine(
|
|
103
|
+
libraryUrl: string,
|
|
104
|
+
model: string,
|
|
105
|
+
contextWindow: number,
|
|
106
|
+
cacheBackend: CacheBackend | undefined,
|
|
107
|
+
onProgress: (report: InitProgressReport) => void,
|
|
108
|
+
): Promise<Loaded> {
|
|
109
|
+
const lib = await library(libraryUrl);
|
|
110
|
+
const opts: EngineConfig = { initProgressCallback: onProgress, appConfig: appConfig(lib, cacheBackend) };
|
|
111
|
+
const chat = { context_window_size: contextWindow };
|
|
112
|
+
if (typeof Worker === 'undefined') return { engine: await lib.CreateMLCEngine(model, opts, chat), worker: undefined };
|
|
113
|
+
|
|
114
|
+
const src = `import { WebWorkerMLCEngineHandler } from ${JSON.stringify(libraryUrl)};\n`
|
|
115
|
+
+ 'const handler = new WebWorkerMLCEngineHandler();\n'
|
|
116
|
+
+ 'self.onmessage = (msg) => handler.onmessage(msg);\n';
|
|
117
|
+
const url = URL.createObjectURL(new Blob([src], { type: 'text/javascript' }));
|
|
118
|
+
const worker = new Worker(url, { type: 'module' });
|
|
119
|
+
try {
|
|
120
|
+
return { engine: await lib.CreateWebWorkerMLCEngine(worker, model, opts, chat), worker };
|
|
121
|
+
} catch (e) {
|
|
122
|
+
worker.terminate();
|
|
123
|
+
throw e;
|
|
124
|
+
} finally {
|
|
125
|
+
URL.revokeObjectURL(url);
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
let features: Promise<ReadonlySet<string> | undefined> | undefined;
|
|
130
|
+
|
|
131
|
+
// What this page's GPU offers, asked once: the one capability WebLLM's records name is `shader-f16`, which
|
|
132
|
+
// every q4f16 build needs. Undefined where there is no adapter to ask.
|
|
133
|
+
export function gpuFeatures(): Promise<ReadonlySet<string> | undefined> {
|
|
134
|
+
features ??= (async () => {
|
|
135
|
+
const gpu = typeof navigator === 'undefined' ? undefined
|
|
136
|
+
: (navigator as { gpu?: { requestAdapter(): Promise<{ features: ReadonlySet<string> } | null> } }).gpu;
|
|
137
|
+
const adapter = await gpu?.requestAdapter().catch(() => null);
|
|
138
|
+
return adapter ? new Set(adapter.features) : undefined;
|
|
139
|
+
})();
|
|
140
|
+
return features;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
// Every LLM the library can run, i.e. not its embedding models.
|
|
144
|
+
export function chatModels(lib: WebLLM): ModelRecord[] {
|
|
145
|
+
return lib.prebuiltAppConfig.model_list.filter(r => r.model_type !== 1);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
// The families trained on the tool format this adapter speaks — `<tool_call>{"name": …, "arguments": …}`,
|
|
149
|
+
// the Hermes format Qwen adopted — read off each family's published chat template (and, for Hermes-3, whose
|
|
150
|
+
// template omits it, its model card). WebLLM records nothing of the kind: its `functionCallingModelIds` is
|
|
151
|
+
// about its own `tools` path, which this adapter does not use. A model outside these can still be made to
|
|
152
|
+
// emit a valid call by the constraint, but only once it writes `<tool_call>`, which one never trained on
|
|
153
|
+
// the format rarely does — it answers in prose, or invents a format of its own.
|
|
154
|
+
// Left out on purpose: Qwen3.5 (it calls tools in XML, `<function=…><parameter=…>`), Qwen2.5-Math (shares
|
|
155
|
+
// Qwen2.5's template, but is a maths specialist), Qwen2 and the DeepSeek-R1 distills (no tool format), and
|
|
156
|
+
// Llama 3.x and Mistral (formats of their own).
|
|
157
|
+
const TOOL_FAMILIES = /^(Qwen3-|Qwen2\.5-(?!Math)|Hermes-2-Pro-|Hermes-2-Theta-|Hermes-3-)/;
|
|
158
|
+
|
|
159
|
+
export function callsTools(modelId: string): boolean {
|
|
160
|
+
return TOOL_FAMILIES.test(modelId);
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
// The models worth offering to matbot, which always has tools to offer.
|
|
164
|
+
export function toolModels(lib: WebLLM): ModelRecord[] {
|
|
165
|
+
return chatModels(lib).filter(r => callsTools(r.model_id));
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
// Smallest first: on a GPU in a page, memory decides what runs at all.
|
|
169
|
+
export function offeredModels(lib: WebLLM): ModelRecord[] {
|
|
170
|
+
return toolModels(lib).sort((a, b) =>
|
|
171
|
+
(a.vram_required_MB ?? Infinity) - (b.vram_required_MB ?? Infinity) || a.model_id.localeCompare(b.model_id));
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
export function closestModels(id: string, records: readonly ModelRecord[], n = 5): string[] {
|
|
175
|
+
return closestIds(id, records.map(r => r.model_id), n);
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
// Refused before anything downloads or loads: an unknown id would otherwise surface as a library error
|
|
179
|
+
// after the worker started, and an f16 build on a GPU without f16 as a shader failure after gigabytes.
|
|
180
|
+
// `withTools`: whether this call offers tools. Without them any chat model will do (`parameters.tools: false`).
|
|
181
|
+
export async function checkModel(lib: WebLLM, model: string, withTools: boolean): Promise<void> {
|
|
182
|
+
const models = chatModels(lib);
|
|
183
|
+
const record = models.find(r => r.model_id === model);
|
|
184
|
+
if (!record) {
|
|
185
|
+
const near = closestModels(model, withTools ? toolModels(lib) : models);
|
|
186
|
+
throw withFailure(new Error(`webllm: "${model}" is not a model this WebLLM build can run.`
|
|
187
|
+
+ (near.length > 0 ? ` Closest: ${near.join(', ')}.` : '')
|
|
188
|
+
+ ' The webllm_models tool lists them.'), 'invalid');
|
|
189
|
+
}
|
|
190
|
+
if (withTools && !callsTools(model)) {
|
|
191
|
+
const near = closestModels(model, toolModels(lib), 3);
|
|
192
|
+
throw withFailure(new Error(`webllm: "${model}" is not trained on the tool-call format this provider uses, so it would answer `
|
|
193
|
+
+ 'instead of calling tools. Use a Qwen3, Qwen2.5 or Hermes model'
|
|
194
|
+
+ (near.length > 0 ? ` (closest: ${near.join(', ')})` : '')
|
|
195
|
+
+ ', or set `parameters.tools: false` to use it for plain chat.'), 'invalid');
|
|
196
|
+
}
|
|
197
|
+
const have = await gpuFeatures();
|
|
198
|
+
const missing = have ? (record.required_features ?? []).filter(f => !have.has(f)) : [];
|
|
199
|
+
if (missing.length > 0) {
|
|
200
|
+
const f32 = model.replace('q4f16', 'q4f32');
|
|
201
|
+
throw withFailure(new Error(`webllm: "${model}" needs the GPU feature${missing.length > 1 ? 's' : ''} ${missing.join(', ')}, which this GPU lacks.`
|
|
202
|
+
+ (f32 !== model && models.some(r => r.model_id === f32) ? ` Use "${f32}" instead.` : '')), 'invalid');
|
|
203
|
+
}
|
|
204
|
+
}
|
package/src/window.ts
ADDED
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
import type { Message, MessageContent, Tool } from '@matatbread/matbot-plugin-api';
|
|
2
|
+
import { toChatMessages } from './format.js';
|
|
3
|
+
|
|
4
|
+
// Fitting a conversation into the model's context window. Two steps, cheapest loss first:
|
|
5
|
+
// 1. shrink large tool results in turns before the current one — the bulk a model has already read and
|
|
6
|
+
// answered from, and which a call can fetch again — keeping who asked what and what was decided;
|
|
7
|
+
// 2. if that is not enough, leave out the oldest whole turns.
|
|
8
|
+
// The session itself is never changed. See the README for what this does not do (summarise).
|
|
9
|
+
//
|
|
10
|
+
// WebLLM offers no tokenizer ahead of a request, so size is ESTIMATED from rendered length, conservatively.
|
|
11
|
+
// A wrong estimate is caught where WebLLM itself refuses (ContextWindowSizeExceededError, at prefill, before
|
|
12
|
+
// any output) and the call retries with a stricter one.
|
|
13
|
+
|
|
14
|
+
export const CHARS_PER_TOKEN = 3;
|
|
15
|
+
// Per rendered message: the role and turn markers a chat template wraps around each one.
|
|
16
|
+
const TOKENS_PER_MESSAGE = 8;
|
|
17
|
+
// Cut to well under the budget, not just under it, so the cut holds for several turns. WebLLM reuses its
|
|
18
|
+
// prefill only when a request extends the previous one exactly; a cut that moved every turn would make
|
|
19
|
+
// every turn re-read the whole window.
|
|
20
|
+
const CUT_TARGET = 0.75;
|
|
21
|
+
// A tool result longer than this, in an earlier turn, is cut to its first SHRINK_TO characters.
|
|
22
|
+
const SHRINK_OVER = 800;
|
|
23
|
+
const SHRINK_TO = 400;
|
|
24
|
+
|
|
25
|
+
export interface Fitted {
|
|
26
|
+
messages: Message[];
|
|
27
|
+
/** Turns left out, oldest first. 0 when none were. */
|
|
28
|
+
omitted: number;
|
|
29
|
+
/** Tool results shrunk in the turns that were kept. */
|
|
30
|
+
shrunk: number;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
// A turn is a user message and everything after it up to the next one, so a tool call is never parted from
|
|
34
|
+
// its result. Anything before the first user message (an assistant opener, a cut session's head) is turn 0.
|
|
35
|
+
function turns(messages: readonly Message[]): Message[][] {
|
|
36
|
+
const out: Message[][] = [];
|
|
37
|
+
for (const m of messages) {
|
|
38
|
+
if (m.role === 'user' || out.length === 0) out.push([m]);
|
|
39
|
+
else out[out.length - 1]!.push(m);
|
|
40
|
+
}
|
|
41
|
+
return out;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function resultText(result: unknown): string {
|
|
45
|
+
if (typeof result === 'string') return result;
|
|
46
|
+
try { return JSON.stringify(result) ?? 'null'; } catch { return String(result); }
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function shrinkContent(c: MessageContent): { content: MessageContent; shrunk: boolean } {
|
|
50
|
+
if (c.type !== 'tool-result') return { content: c, shrunk: false };
|
|
51
|
+
const text = resultText(c.result);
|
|
52
|
+
if (text.length <= SHRINK_OVER) return { content: c, shrunk: false };
|
|
53
|
+
return {
|
|
54
|
+
content: { ...c, result: `${text.slice(0, SHRINK_TO)}… [${(text.length - SHRINK_TO).toLocaleString('en')} more characters `
|
|
55
|
+
+ 'left out to fit the context window; call the tool again to see the whole result]' },
|
|
56
|
+
shrunk: true,
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// Every turn but the last: the current turn may be about to be answered from its latest result.
|
|
61
|
+
function shrinkOld(groups: readonly Message[][]): { groups: Message[][]; shrunk: number[] } {
|
|
62
|
+
const shrunk: number[] = [];
|
|
63
|
+
const out = groups.map((g, i) => {
|
|
64
|
+
let n = 0;
|
|
65
|
+
const kept = i === groups.length - 1 ? g : g.map(m => {
|
|
66
|
+
if (m.role !== 'tool') return m;
|
|
67
|
+
const parts = m.content.map(shrinkContent);
|
|
68
|
+
n += parts.filter(p => p.shrunk).length;
|
|
69
|
+
return parts.some(p => p.shrunk) ? { ...m, content: parts.map(p => p.content) } : m;
|
|
70
|
+
});
|
|
71
|
+
shrunk.push(n);
|
|
72
|
+
return kept;
|
|
73
|
+
});
|
|
74
|
+
return { groups: out, shrunk };
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function estimate(messages: readonly Message[], tools: readonly Tool[], charsPerToken: number): number {
|
|
78
|
+
const chat = toChatMessages([...messages], tools);
|
|
79
|
+
return chat.reduce((n, m) => n + Math.ceil(m.content.length / charsPerToken) + TOKENS_PER_MESSAGE, 0);
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// What fitting last decided for each conversation, by its first message's id (the one thing every call of
|
|
83
|
+
// a conversation shares), so the next call decides the same and what is sent keeps the same start. Once a
|
|
84
|
+
// conversation needed shrinking it stays shrunk — shrinking only ever touches earlier turns, and a turn's
|
|
85
|
+
// shrunk form never changes, so staying shrunk is what keeps the start stable. Bounded: a stale entry
|
|
86
|
+
// costs only a re-fit.
|
|
87
|
+
interface Decision { shrink: true; firstKept?: string }
|
|
88
|
+
const decisions = new Map<string, Decision>();
|
|
89
|
+
const MAX_DECISIONS = 32;
|
|
90
|
+
|
|
91
|
+
function remember(conversation: string, decision: Decision): void {
|
|
92
|
+
decisions.delete(conversation);
|
|
93
|
+
decisions.set(conversation, decision);
|
|
94
|
+
for (const key of decisions.keys()) {
|
|
95
|
+
if (decisions.size <= MAX_DECISIONS) break;
|
|
96
|
+
decisions.delete(key);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
export function forgetFit(messages: readonly Message[]): void {
|
|
101
|
+
const first = messages.find(m => m.role !== 'system');
|
|
102
|
+
if (first) decisions.delete(first.id);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* The messages to send so that the system prompt, the tools and the kept turns fit `budget` tokens. Never
|
|
107
|
+
* leaves out or shrinks the current turn: if that alone does not fit, it is sent anyway and WebLLM's own
|
|
108
|
+
* refusal says so.
|
|
109
|
+
*/
|
|
110
|
+
export function fitToWindow(
|
|
111
|
+
messages: readonly Message[],
|
|
112
|
+
tools: readonly Tool[],
|
|
113
|
+
budget: number,
|
|
114
|
+
charsPerToken: number = CHARS_PER_TOKEN,
|
|
115
|
+
): Fitted {
|
|
116
|
+
const system = messages.filter(m => m.role === 'system');
|
|
117
|
+
const groups = turns(messages.filter(m => m.role !== 'system'));
|
|
118
|
+
const first = groups[0]?.[0];
|
|
119
|
+
if (!first || groups.length === 1) return { messages: [...messages], omitted: 0, shrunk: 0 };
|
|
120
|
+
|
|
121
|
+
const cost = (gs: readonly Message[][], from: number): number =>
|
|
122
|
+
estimate([...system, ...gs.slice(from).flat()], tools, charsPerToken);
|
|
123
|
+
const previous = decisions.get(first.id);
|
|
124
|
+
|
|
125
|
+
if (!previous && cost(groups, 0) <= budget) return { messages: [...messages], omitted: 0, shrunk: 0 };
|
|
126
|
+
|
|
127
|
+
const { groups: small, shrunk } = shrinkOld(groups);
|
|
128
|
+
const keep = (from: number): Fitted => ({
|
|
129
|
+
messages: [...system, ...small.slice(from).flat()],
|
|
130
|
+
omitted: from,
|
|
131
|
+
shrunk: shrunk.slice(from).reduce((a, b) => a + b, 0),
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
if (previous?.firstKept !== undefined) {
|
|
135
|
+
const from = small.findIndex(g => g[0]?.id === previous.firstKept);
|
|
136
|
+
if (from > 0 && cost(small, from) <= budget) return keep(from);
|
|
137
|
+
}
|
|
138
|
+
if (cost(small, 0) <= budget) {
|
|
139
|
+
remember(first.id, { shrink: true });
|
|
140
|
+
return keep(0);
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
const last = small.length - 1;
|
|
144
|
+
let from = 1;
|
|
145
|
+
while (from < last && cost(small, from) > budget * CUT_TARGET) from++;
|
|
146
|
+
remember(first.id, { shrink: true, firstKept: small[from]![0]!.id });
|
|
147
|
+
return keep(from);
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
// Told to the model in the system prompt, so it does not answer as if it had seen what it has not.
|
|
151
|
+
export function omittedNote(omitted: number): string {
|
|
152
|
+
return `Note: the ${omitted === 1 ? 'earliest turn' : `earliest ${omitted} turns`} of this conversation `
|
|
153
|
+
+ 'are not shown to you, because the conversation is longer than your context window. If the user refers '
|
|
154
|
+
+ 'to something you cannot see, say so rather than guessing.';
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
// WebLLM's refusal, by message as well as name: an error crossing the worker boundary may lose its class.
|
|
158
|
+
export function isWindowOverflow(e: unknown): boolean {
|
|
159
|
+
if (!(e instanceof Error)) return false;
|
|
160
|
+
return e.name === 'ContextWindowSizeExceededError' || /exceed context window size/i.test(e.message);
|
|
161
|
+
}
|