@tanstack-ai-browser/prompt-api 0.0.0-stage → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +96 -2
- package/dist/adapter.d.ts +160 -0
- package/dist/adapter.d.ts.map +1 -0
- package/dist/adapter.js +414 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +7 -0
- package/dist/limit.d.ts +119 -0
- package/dist/limit.d.ts.map +1 -0
- package/dist/limit.js +203 -0
- package/dist/messages.d.ts +20 -0
- package/dist/messages.d.ts.map +1 -0
- package/dist/messages.js +95 -0
- package/dist/model-meta.d.ts +21 -0
- package/dist/model-meta.d.ts.map +1 -0
- package/dist/model-meta.js +18 -0
- package/dist/model.d.ts +28 -0
- package/dist/model.d.ts.map +1 -0
- package/dist/model.js +53 -0
- package/dist/reasoning.d.ts +42 -0
- package/dist/reasoning.d.ts.map +1 -0
- package/dist/reasoning.js +113 -0
- package/dist/sessions.d.ts +48 -0
- package/dist/sessions.d.ts.map +1 -0
- package/dist/sessions.js +275 -0
- package/dist/shared-load.d.ts +48 -0
- package/dist/shared-load.d.ts.map +1 -0
- package/dist/shared-load.js +103 -0
- package/dist/store.d.ts +8 -0
- package/dist/store.d.ts.map +1 -0
- package/dist/store.js +17 -0
- package/dist/tools.d.ts +50 -0
- package/dist/tools.d.ts.map +1 -0
- package/dist/tools.js +314 -0
- package/dist/types.d.ts +81 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +13 -0
- package/package.json +60 -4
- package/src/adapter.test.ts +571 -0
- package/src/adapter.ts +522 -0
- package/src/index.ts +13 -0
- package/src/messages.ts +102 -0
- package/src/model-meta.ts +35 -0
- package/src/model.ts +75 -0
- package/src/tools.test.ts +129 -0
- package/src/tools.ts +311 -0
- package/src/types.ts +110 -0
package/dist/adapter.js
ADDED
|
@@ -0,0 +1,414 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A TanStack AI text adapter for the browser's built-in language model,
|
|
3
|
+
* through the Prompt API (`LanguageModel`): Gemini Nano in Chrome,
|
|
4
|
+
* Phi-4-mini in Edge. The browser downloads and runs the model itself;
|
|
5
|
+
* nothing leaves the device. Hand it to `chat()` like any hosted model's
|
|
6
|
+
* adapter: `chat()` runs tools and the agent loop, and each `chatStream`
|
|
7
|
+
* call here is one step of the model, streamed back as AG-UI events.
|
|
8
|
+
*
|
|
9
|
+
* Each thread (`threadId`) keeps a session: a step prompts it with only the
|
|
10
|
+
* messages it hasn't seen, and a conversation that changed (an edit, a new
|
|
11
|
+
* system prompt) starts a new one. Tools go by prompt (`tools.ts`);
|
|
12
|
+
* structured output prompts the thread's session under the Prompt API's own
|
|
13
|
+
* `responseConstraint`.
|
|
14
|
+
*/
|
|
15
|
+
import { buildBaseUsage, EventType, normalizeSystemPrompts } from "@tanstack/ai";
|
|
16
|
+
import { BaseTextAdapter } from "@tanstack/ai/adapters";
|
|
17
|
+
import { contentText, mediaIn, toLanguageModelMessages } from "./messages.js";
|
|
18
|
+
import { baseOptions, createSession, DEFAULT_LANGUAGES, MEDIA_INPUTS, promptApiAvailability } from "./model.js";
|
|
19
|
+
import { toolCallReader, toolSystemPrompt } from "./tools.js";
|
|
20
|
+
/**
|
|
21
|
+
* Threads that keep a session open; the least recently used closes first. A
|
|
22
|
+
* session holds its conversation in the model's memory (about 100 MB for
|
|
23
|
+
* 6,000 tokens in Chrome 156), freed only by `destroy()`.
|
|
24
|
+
*/
|
|
25
|
+
const KEPT_SESSIONS = 4;
|
|
26
|
+
/** Closes a session once it's made; one that failed to start has nothing to close. */
|
|
27
|
+
const close = (session) => void session.then((s) => s.destroy(), () => undefined);
|
|
28
|
+
/**
|
|
29
|
+
* A message as compared with what a session holds. The model's own replies
|
|
30
|
+
* go by their text alone: the session has them as the model wrote them.
|
|
31
|
+
*/
|
|
32
|
+
const messageKey = (m) => m.role === "assistant"
|
|
33
|
+
? JSON.stringify(["assistant", contentText(m.content).trim()])
|
|
34
|
+
: JSON.stringify([m.role, m.content, m.toolCallId ?? null, m.error ?? null]);
|
|
35
|
+
/** The tools a step offers: none for `toolChoice: "none"`, only the chosen one for a named choice. */
|
|
36
|
+
function offeredTools(options) {
|
|
37
|
+
const { toolChoice, tools = [] } = options;
|
|
38
|
+
if (toolChoice === "none")
|
|
39
|
+
return [];
|
|
40
|
+
if (typeof toolChoice === "object")
|
|
41
|
+
return tools.filter((tool) => tool.name === toolChoice.name);
|
|
42
|
+
return tools;
|
|
43
|
+
}
|
|
44
|
+
/** A step's system prompt (the caller's, then the tools), and the settings its session is made with. */
|
|
45
|
+
function sessionSettings(options, tools) {
|
|
46
|
+
const system = toolSystemPrompt(normalizeSystemPrompts(options.systemPrompts)
|
|
47
|
+
.map((p) => p.content)
|
|
48
|
+
.join("\n\n"), tools);
|
|
49
|
+
const { samplingMode, temperature, topK } = options.modelOptions ?? {};
|
|
50
|
+
return { system, settings: JSON.stringify([system, samplingMode, temperature, topK, mediaIn(options.messages)]) };
|
|
51
|
+
}
|
|
52
|
+
/** Whether a conversation (its messages' keys) goes on from what a kept session holds, with its settings. */
|
|
53
|
+
const goesOnFrom = (held, settings, keys) => !held.busy && held.settings === settings && held.seen.length < keys.length && held.seen.every((key, i) => key === keys[i]);
|
|
54
|
+
let ids = 0;
|
|
55
|
+
const nextId = (prefix) => `${prefix}_${Date.now().toString(36)}_${(ids++).toString(36)}`;
|
|
56
|
+
/** The caller's signal: the legacy `abortController` first, then `request`. */
|
|
57
|
+
const signalOf = (options) => options.abortController?.signal ?? options.request?.signal ?? undefined;
|
|
58
|
+
export class PromptApiTextAdapter extends BaseTextAdapter {
|
|
59
|
+
kind = "text";
|
|
60
|
+
name = "prompt-api";
|
|
61
|
+
provider = "prompt-api";
|
|
62
|
+
api = "prompt-api";
|
|
63
|
+
inputModalities = ["text", "image", "audio"];
|
|
64
|
+
languages;
|
|
65
|
+
inputs;
|
|
66
|
+
onDownloadProgress;
|
|
67
|
+
onContextOverflow;
|
|
68
|
+
/** Each thread's session, least recently used first. */
|
|
69
|
+
kept = new Map();
|
|
70
|
+
constructor(config, model) {
|
|
71
|
+
super({}, model);
|
|
72
|
+
this.languages = config.languages ?? DEFAULT_LANGUAGES;
|
|
73
|
+
this.inputs = config.inputs ?? [];
|
|
74
|
+
this.onDownloadProgress = config.onDownloadProgress;
|
|
75
|
+
this.onContextOverflow = config.onContextOverflow;
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* The browser's own `LanguageModel.availability()` for this adapter's
|
|
79
|
+
* `languages` and `inputs`: "available" (on the device), "downloadable"
|
|
80
|
+
* or "downloading" (it downloads when a session starts, or at `create`),
|
|
81
|
+
* or "unavailable" (no Prompt API, or the device can't run it).
|
|
82
|
+
*/
|
|
83
|
+
availability() {
|
|
84
|
+
return promptApiAvailability(baseOptions(this.languages, this.inputs));
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* The kinds of input the browser's model takes here, of the
|
|
88
|
+
* `inputModalities` the Prompt API has: each kind the browser's
|
|
89
|
+
* `LanguageModel.availability()` doesn't call "unavailable" for this
|
|
90
|
+
* adapter's `languages` (images need more of the device, audio a GPU).
|
|
91
|
+
* Empty where there's no model. Sessions expect what each conversation has.
|
|
92
|
+
*/
|
|
93
|
+
async availableInputModalities() {
|
|
94
|
+
if ((await promptApiAvailability(baseOptions(this.languages))) === "unavailable")
|
|
95
|
+
return [];
|
|
96
|
+
const media = await Promise.all(MEDIA_INPUTS.map(async (kind) => (await promptApiAvailability(baseOptions(this.languages, [kind]))) === "unavailable" ? [] : [kind]));
|
|
97
|
+
return ["text", ...media.flat()];
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Downloads the browser's model if it isn't on the device and starts it,
|
|
101
|
+
* with the download's progress, 0 to 1, then 1 once it's ready. Resolves
|
|
102
|
+
* to this adapter; rejects if it can't start (fine to leave unawaited).
|
|
103
|
+
* `chat()` doesn't need it, but the browser asks for a click on the page
|
|
104
|
+
* before a download: call it from the click that turns the model on.
|
|
105
|
+
* Aborting `signal` aborts the browser's `create()`.
|
|
106
|
+
*/
|
|
107
|
+
create(options = {}) {
|
|
108
|
+
const listeners = [this.onDownloadProgress, options.onDownloadProgress].filter((l) => !!l);
|
|
109
|
+
const ready = createSession(baseOptions(this.languages), options.signal, (progress) => {
|
|
110
|
+
for (const listener of listeners)
|
|
111
|
+
listener(progress);
|
|
112
|
+
}).then((session) => {
|
|
113
|
+
session.destroy();
|
|
114
|
+
return this;
|
|
115
|
+
});
|
|
116
|
+
// Callers may leave it: an unawaited failure isn't an unhandled rejection.
|
|
117
|
+
ready.catch(() => undefined);
|
|
118
|
+
return ready;
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* Tokens a thread's session holds (the session's `contextUsage`): the
|
|
122
|
+
* system prompt, the tools and the conversation so far. The thread is
|
|
123
|
+
* `chat()`'s `threadId` (or its `promptCache.key`); left out, the thread
|
|
124
|
+
* used last. Undefined where the thread keeps no session.
|
|
125
|
+
*/
|
|
126
|
+
async contextUsage(threadId) {
|
|
127
|
+
return (await this.keptSession(threadId))?.contextUsage;
|
|
128
|
+
}
|
|
129
|
+
/**
|
|
130
|
+
* Tokens a thread's session can hold (the session's `contextWindow`), for
|
|
131
|
+
* the thread as `contextUsage` takes it. Undefined where the thread keeps
|
|
132
|
+
* no session.
|
|
133
|
+
*/
|
|
134
|
+
async contextWindow(threadId) {
|
|
135
|
+
return (await this.keptSession(threadId))?.contextWindow;
|
|
136
|
+
}
|
|
137
|
+
/** A thread's kept session, or the last used thread's; undefined if there's none, or it failed to start. */
|
|
138
|
+
keptSession(threadId) {
|
|
139
|
+
const kept = threadId === undefined ? [...this.kept.values()].at(-1) : this.kept.get(threadId);
|
|
140
|
+
return kept ? kept.session.catch(() => undefined) : Promise.resolve(undefined);
|
|
141
|
+
}
|
|
142
|
+
/** Closes every thread's session. The browser keeps the model. */
|
|
143
|
+
async destroy() {
|
|
144
|
+
for (const kept of this.kept.values())
|
|
145
|
+
close(kept.session);
|
|
146
|
+
this.kept.clear();
|
|
147
|
+
}
|
|
148
|
+
/** Keeps a thread's session, closing the least recently used past the limit. */
|
|
149
|
+
keep(threadId, kept) {
|
|
150
|
+
this.kept.delete(threadId);
|
|
151
|
+
this.kept.set(threadId, kept);
|
|
152
|
+
for (const [id, old] of this.kept) {
|
|
153
|
+
if (this.kept.size <= KEPT_SESSIONS)
|
|
154
|
+
break;
|
|
155
|
+
if (old.busy)
|
|
156
|
+
continue;
|
|
157
|
+
close(old.session);
|
|
158
|
+
this.kept.delete(id);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
/** Closes a thread's session, unless a step is using it. */
|
|
162
|
+
drop(threadId) {
|
|
163
|
+
const kept = this.kept.get(threadId);
|
|
164
|
+
if (!kept || kept.busy)
|
|
165
|
+
return;
|
|
166
|
+
close(kept.session);
|
|
167
|
+
this.kept.delete(threadId);
|
|
168
|
+
}
|
|
169
|
+
/** A session for this call, holding the system prompt, that reports an overflow. */
|
|
170
|
+
async session(options, system, signal) {
|
|
171
|
+
const { samplingMode, temperature, topK } = options.modelOptions ?? {};
|
|
172
|
+
const session = await createSession({
|
|
173
|
+
...baseOptions(this.languages, mediaIn(options.messages)),
|
|
174
|
+
...(system && { initialPrompts: [{ role: "system", content: system }] }),
|
|
175
|
+
...(samplingMode !== undefined && { samplingMode }),
|
|
176
|
+
...(temperature !== undefined && { temperature }),
|
|
177
|
+
...(topK !== undefined && { topK })
|
|
178
|
+
}, signal, this.onDownloadProgress);
|
|
179
|
+
if (this.onContextOverflow)
|
|
180
|
+
session.addEventListener("contextoverflow", this.onContextOverflow);
|
|
181
|
+
return session;
|
|
182
|
+
}
|
|
183
|
+
/**
|
|
184
|
+
* The step's session and the messages to prompt it with: the thread's
|
|
185
|
+
* kept session if the conversation goes on from what it holds (only the
|
|
186
|
+
* new messages), else a new one with the whole conversation. The session
|
|
187
|
+
* is reserved before anything is awaited, so a second step on a busy
|
|
188
|
+
* thread gets a session of its own (`kept` undefined), closed after it.
|
|
189
|
+
* Threads go by `promptCache.key`; retention "none" keeps no session.
|
|
190
|
+
*/
|
|
191
|
+
async begin(options, signal) {
|
|
192
|
+
const key = options.promptCache?.key ?? options.threadId;
|
|
193
|
+
const retain = options.promptCache?.retention !== "none";
|
|
194
|
+
const tools = offeredTools(options);
|
|
195
|
+
const { system, settings } = sessionSettings(options, tools);
|
|
196
|
+
const keys = options.messages.map(messageKey);
|
|
197
|
+
const held = retain ? this.kept.get(key) : undefined;
|
|
198
|
+
const goesOn = !!held && goesOnFrom(held, settings, keys);
|
|
199
|
+
let kept;
|
|
200
|
+
if (goesOn)
|
|
201
|
+
kept = held;
|
|
202
|
+
else if (retain && !held?.busy) {
|
|
203
|
+
this.drop(key);
|
|
204
|
+
kept = { session: this.session(options, system, signal), settings, seen: [], busy: false };
|
|
205
|
+
}
|
|
206
|
+
if (kept) {
|
|
207
|
+
kept.busy = true;
|
|
208
|
+
this.keep(key, kept);
|
|
209
|
+
}
|
|
210
|
+
const pending = kept?.session ?? this.session(options, system, signal);
|
|
211
|
+
try {
|
|
212
|
+
const session = await pending;
|
|
213
|
+
const input = await toLanguageModelMessages(options.messages, goesOn ? held.seen.length : 0);
|
|
214
|
+
return { key, session, kept, keys, input, tools, goesOn };
|
|
215
|
+
}
|
|
216
|
+
catch (error) {
|
|
217
|
+
// A session that goes on still holds the thread; a new one is dropped.
|
|
218
|
+
this.end(key, pending, kept, !goesOn);
|
|
219
|
+
throw error;
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
/** Frees a step's session: a kept one stays for the thread unless the step failed. */
|
|
223
|
+
end(key, session, kept, failed) {
|
|
224
|
+
if (!kept)
|
|
225
|
+
return close(session);
|
|
226
|
+
kept.busy = false;
|
|
227
|
+
if (failed && this.kept.get(key) === kept)
|
|
228
|
+
this.drop(key);
|
|
229
|
+
}
|
|
230
|
+
/**
|
|
231
|
+
* A step's token counts, which the Prompt API doesn't report, from the
|
|
232
|
+
* session's own: the tokens it held before the prompt, the prompt as the
|
|
233
|
+
* session measures it, and the reply measured the same way, less an empty
|
|
234
|
+
* message's tokens (the turn markers the session counts around any
|
|
235
|
+
* message). What a kept session held is cached: it isn't read again. A
|
|
236
|
+
* new session read what it holds (the system prompt) in this step.
|
|
237
|
+
* Nothing where the browser can't measure.
|
|
238
|
+
*/
|
|
239
|
+
async usage(session, held, goesOn, input, reply) {
|
|
240
|
+
try {
|
|
241
|
+
const [read, written, markers] = await Promise.all([
|
|
242
|
+
session.measureContextUsage(input),
|
|
243
|
+
session.measureContextUsage(reply),
|
|
244
|
+
session.measureContextUsage("")
|
|
245
|
+
]);
|
|
246
|
+
const counts = [held, read, written, markers];
|
|
247
|
+
if (!counts.every((count) => Number.isFinite(count)))
|
|
248
|
+
return undefined;
|
|
249
|
+
const promptTokens = held + read;
|
|
250
|
+
const completionTokens = Math.max(0, written - markers);
|
|
251
|
+
return {
|
|
252
|
+
...buildBaseUsage({ promptTokens, completionTokens, totalTokens: promptTokens + completionTokens }),
|
|
253
|
+
// A new session read all of it in this step: none was cached.
|
|
254
|
+
promptTokensDetails: { cachedTokens: goesOn ? held : 0 }
|
|
255
|
+
};
|
|
256
|
+
}
|
|
257
|
+
catch {
|
|
258
|
+
return undefined;
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
async *chatStream(options) {
|
|
262
|
+
const runId = options.runId ?? nextId("run");
|
|
263
|
+
const { threadId } = options;
|
|
264
|
+
const messageId = nextId("msg");
|
|
265
|
+
const model = this.model;
|
|
266
|
+
const signal = signalOf(options);
|
|
267
|
+
const timestamp = () => Date.now();
|
|
268
|
+
yield {
|
|
269
|
+
type: EventType.RUN_STARTED,
|
|
270
|
+
runId,
|
|
271
|
+
threadId,
|
|
272
|
+
model,
|
|
273
|
+
timestamp: timestamp(),
|
|
274
|
+
...(options.parentRunId && { parentRunId: options.parentRunId })
|
|
275
|
+
};
|
|
276
|
+
let step;
|
|
277
|
+
let failed = false;
|
|
278
|
+
// Stops the generation once a tool call is read.
|
|
279
|
+
const stop = new AbortController();
|
|
280
|
+
try {
|
|
281
|
+
step = await this.begin(options, signal);
|
|
282
|
+
const { session, kept, keys, input, tools, goesOn } = step;
|
|
283
|
+
// Without tools to offer, the reply is all text.
|
|
284
|
+
const reader = tools.length > 0 ? toolCallReader(tools.map((tool) => tool.name)) : undefined;
|
|
285
|
+
let text = "";
|
|
286
|
+
const textEvents = function* (delta) {
|
|
287
|
+
if (!delta)
|
|
288
|
+
return;
|
|
289
|
+
if (!text)
|
|
290
|
+
yield { type: EventType.TEXT_MESSAGE_START, messageId, role: "assistant", model, timestamp: timestamp() };
|
|
291
|
+
text += delta;
|
|
292
|
+
yield { type: EventType.TEXT_MESSAGE_CONTENT, messageId, delta, content: text, model, timestamp: timestamp() };
|
|
293
|
+
};
|
|
294
|
+
const generation = signal ? AbortSignal.any([signal, stop.signal]) : stop.signal;
|
|
295
|
+
// What the session holds before this prompt, and all the model writes (a tool call's text too).
|
|
296
|
+
const held = session.contextUsage;
|
|
297
|
+
let written = "";
|
|
298
|
+
try {
|
|
299
|
+
for await (const piece of session.promptStreaming(input, { signal: generation })) {
|
|
300
|
+
written += piece;
|
|
301
|
+
yield* textEvents(reader ? reader.push(piece) : piece);
|
|
302
|
+
if (reader?.call) {
|
|
303
|
+
stop.abort();
|
|
304
|
+
break;
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
catch (error) {
|
|
309
|
+
if (!stop.signal.aborted || signal?.aborted)
|
|
310
|
+
throw error;
|
|
311
|
+
}
|
|
312
|
+
const call = reader?.call;
|
|
313
|
+
if (reader) {
|
|
314
|
+
yield* textEvents(reader.end());
|
|
315
|
+
// Text after the call, in the piece that closed it, is shown too.
|
|
316
|
+
yield* textEvents(reader.trailing);
|
|
317
|
+
}
|
|
318
|
+
if (text)
|
|
319
|
+
yield { type: EventType.TEXT_MESSAGE_END, messageId, model, timestamp: timestamp() };
|
|
320
|
+
// A finished reply is in the session now. One stopped at a tool call
|
|
321
|
+
// isn't: Chrome drops a stopped turn whole, so the next step sends it.
|
|
322
|
+
if (kept && !stop.signal.aborted)
|
|
323
|
+
kept.seen = [...keys, messageKey({ role: "assistant", content: text })];
|
|
324
|
+
if (call) {
|
|
325
|
+
const toolCallId = nextId("call");
|
|
326
|
+
const args = JSON.stringify(call.arguments);
|
|
327
|
+
yield {
|
|
328
|
+
type: EventType.TOOL_CALL_START,
|
|
329
|
+
toolCallId,
|
|
330
|
+
toolCallName: call.name,
|
|
331
|
+
parentMessageId: messageId,
|
|
332
|
+
index: 0,
|
|
333
|
+
model,
|
|
334
|
+
timestamp: timestamp()
|
|
335
|
+
};
|
|
336
|
+
yield { type: EventType.TOOL_CALL_ARGS, toolCallId, delta: args, args, model, timestamp: timestamp() };
|
|
337
|
+
yield {
|
|
338
|
+
type: EventType.TOOL_CALL_END,
|
|
339
|
+
toolCallId,
|
|
340
|
+
toolCallName: call.name,
|
|
341
|
+
args,
|
|
342
|
+
input: call.arguments,
|
|
343
|
+
model,
|
|
344
|
+
timestamp: timestamp()
|
|
345
|
+
};
|
|
346
|
+
}
|
|
347
|
+
const usage = await this.usage(session, held, goesOn, input, written);
|
|
348
|
+
yield {
|
|
349
|
+
type: EventType.RUN_FINISHED,
|
|
350
|
+
runId,
|
|
351
|
+
threadId,
|
|
352
|
+
model,
|
|
353
|
+
timestamp: timestamp(),
|
|
354
|
+
finishReason: call ? "tool_calls" : "stop",
|
|
355
|
+
...(usage && { usage })
|
|
356
|
+
};
|
|
357
|
+
}
|
|
358
|
+
catch (error) {
|
|
359
|
+
// An abort keeps the thread's session: Chrome drops the stopped turn.
|
|
360
|
+
const aborted = !!signal?.aborted;
|
|
361
|
+
failed = !aborted;
|
|
362
|
+
options.logger.errors("prompt-api chatStream failed", { error, source: "prompt-api.chatStream" });
|
|
363
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
364
|
+
yield {
|
|
365
|
+
type: EventType.RUN_ERROR,
|
|
366
|
+
runId,
|
|
367
|
+
model,
|
|
368
|
+
timestamp: timestamp(),
|
|
369
|
+
message,
|
|
370
|
+
error: { message, ...(aborted && { code: "aborted" }) }
|
|
371
|
+
};
|
|
372
|
+
}
|
|
373
|
+
finally {
|
|
374
|
+
if (step)
|
|
375
|
+
this.end(step.key, Promise.resolve(step.session), step.kept, failed);
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
/**
|
|
379
|
+
* A JSON answer for `chat({ outputSchema })`: the thread's session
|
|
380
|
+
* answers the conversation under the Prompt API's `responseConstraint`,
|
|
381
|
+
* which holds the model to the schema as given. `chat()` validates the
|
|
382
|
+
* result.
|
|
383
|
+
*/
|
|
384
|
+
async structuredOutput(options) {
|
|
385
|
+
const { chatOptions, outputSchema } = options;
|
|
386
|
+
const signal = signalOf(chatOptions);
|
|
387
|
+
let step;
|
|
388
|
+
let failed = false;
|
|
389
|
+
try {
|
|
390
|
+
step = await this.begin(chatOptions, signal);
|
|
391
|
+
const { session, kept, keys, input, goesOn } = step;
|
|
392
|
+
const held = session.contextUsage;
|
|
393
|
+
const rawText = await session.prompt(input, { responseConstraint: outputSchema, signal });
|
|
394
|
+
if (kept)
|
|
395
|
+
kept.seen = [...keys, messageKey({ role: "assistant", content: rawText })];
|
|
396
|
+
const data = JSON.parse(rawText);
|
|
397
|
+
const usage = await this.usage(session, held, goesOn, input, rawText);
|
|
398
|
+
return { data, rawText, model: this.model, ...(usage && { usage }) };
|
|
399
|
+
}
|
|
400
|
+
catch (error) {
|
|
401
|
+
failed = true;
|
|
402
|
+
throw error;
|
|
403
|
+
}
|
|
404
|
+
finally {
|
|
405
|
+
if (step)
|
|
406
|
+
this.end(step.key, Promise.resolve(step.session), step.kept, failed);
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
export function promptApiText(modelOrConfig, config = {}) {
|
|
411
|
+
return typeof modelOrConfig === "string"
|
|
412
|
+
? new PromptApiTextAdapter(config, modelOrConfig)
|
|
413
|
+
: new PromptApiTextAdapter(modelOrConfig ?? {}, "text");
|
|
414
|
+
}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* TanStack AI adapter for the browser's built-in language model, through
|
|
3
|
+
* the Prompt API: Gemini Nano in Chrome, Phi-4-mini in Edge. The model
|
|
4
|
+
* catalog (model-meta.ts) is exported here too.
|
|
5
|
+
*/
|
|
6
|
+
export { PromptApiTextAdapter, promptApiText } from "./adapter.js";
|
|
7
|
+
export type { PromptApiConfig, PromptApiTextModelOptions } from "./adapter.js";
|
|
8
|
+
export type { Availability, DownloadProgressCallback } from "./types.js";
|
|
9
|
+
export { PROMPT_API_CHAT_MODELS, PROMPT_API_MODEL_META } from "./model-meta.js";
|
|
10
|
+
export type { PromptApiChatModel, PromptApiModelMeta } from "./model-meta.js";
|
|
11
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,cAAc,CAAA;AAClE,YAAY,EACV,eAAe,EACf,yBAAyB,EAC1B,MAAM,cAAc,CAAA;AACrB,YAAY,EAAE,YAAY,EAAE,wBAAwB,EAAE,MAAM,YAAY,CAAA;AACxE,OAAO,EAAE,sBAAsB,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAA;AAC/E,YAAY,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* TanStack AI adapter for the browser's built-in language model, through
|
|
3
|
+
* the Prompt API: Gemini Nano in Chrome, Phi-4-mini in Edge. The model
|
|
4
|
+
* catalog (model-meta.ts) is exported here too.
|
|
5
|
+
*/
|
|
6
|
+
export { PromptApiTextAdapter, promptApiText } from "./adapter.js";
|
|
7
|
+
export { PROMPT_API_CHAT_MODELS, PROMPT_API_MODEL_META } from "./model-meta.js";
|
package/dist/limit.d.ts
ADDED
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Output token limits, emulated. The Prompt API has no output cap (a prompt
|
|
3
|
+
* takes only `signal`, `responseConstraint` and
|
|
4
|
+
* `omitResponseConstraintInput`), so the adapter holds the reply to a limit
|
|
5
|
+
* itself, for `maxCompletionTokens`.
|
|
6
|
+
*
|
|
7
|
+
* Chrome runs one operation at a time per model, so the model can't count
|
|
8
|
+
* text while it writes it. The limit is estimated during the generation
|
|
9
|
+
* (from a ratio calibrated on the request before it starts), the generation
|
|
10
|
+
* is stopped by an internal abort a little past the limit, and the text is
|
|
11
|
+
* measured and trimmed exactly once the model is idle again.
|
|
12
|
+
*/
|
|
13
|
+
import type { LanguageModelSession } from "./types.js";
|
|
14
|
+
/** `maxCompletionTokens`, validated like the other adapters validate it. */
|
|
15
|
+
export declare function outputTokenLimit(options: {
|
|
16
|
+
maxCompletionTokens?: number;
|
|
17
|
+
} | undefined): number | undefined;
|
|
18
|
+
/** The cap's report in `usage.providerUsageDetails`, when a cap was set. */
|
|
19
|
+
export type PromptApiOutputLimitUsage = {
|
|
20
|
+
outputTokenLimit: number;
|
|
21
|
+
outputTokenMeasurement: "estimate-then-context-usage";
|
|
22
|
+
outputTokenLimitEnforcement: "best-effort";
|
|
23
|
+
outputTokenLimitReached: boolean;
|
|
24
|
+
};
|
|
25
|
+
export declare const outputLimitUsage: (limit: number | undefined, reached: boolean) => PromptApiOutputLimitUsage | undefined;
|
|
26
|
+
/** A message's text, without its images or audio. */
|
|
27
|
+
type TextBearing = {
|
|
28
|
+
content: string | ReadonlyArray<{
|
|
29
|
+
type: string;
|
|
30
|
+
value?: unknown;
|
|
31
|
+
}>;
|
|
32
|
+
};
|
|
33
|
+
/** The text of a prompt input (or of messages), without images or audio. */
|
|
34
|
+
export declare function textOfInput(input: string | ReadonlyArray<TextBearing>): string;
|
|
35
|
+
/**
|
|
36
|
+
* Live emission stops once the estimate passes this share of the limit, or
|
|
37
|
+
* once the text passes the limit in UTF-8 bytes. Text sent live can't be
|
|
38
|
+
* taken back, so it must stay within the limit whatever the estimate says:
|
|
39
|
+
* no token is shorter than a byte, so text of at most `limit` bytes holds at
|
|
40
|
+
* most `limit` tokens, however much denser the reply is than the calibration
|
|
41
|
+
* text. The share keeps longer limits from streaming up to the last token
|
|
42
|
+
* live, where a low estimate would leave nothing to trim.
|
|
43
|
+
*/
|
|
44
|
+
export declare const LIVE_FRACTION = 0.75;
|
|
45
|
+
/**
|
|
46
|
+
* Generation is stopped once the estimate passes STOP_FACTOR × limit +
|
|
47
|
+
* STOP_SLACK tokens: past the limit, so the exact trim afterwards finds the
|
|
48
|
+
* limit's worth of text unless the reply holds a fifth fewer tokens per byte
|
|
49
|
+
* than the calibration text (the estimate is then high by more than 1.25×).
|
|
50
|
+
* The slack covers short limits, where a chunk or two of noise is a large
|
|
51
|
+
* share; 8 tokens is about 0.15 s of Gemini Nano's decoding.
|
|
52
|
+
*/
|
|
53
|
+
export declare const STOP_FACTOR = 1.25;
|
|
54
|
+
export declare const STOP_SLACK = 8;
|
|
55
|
+
/**
|
|
56
|
+
* The text a limit's tokens-per-byte ratio is calibrated on: the request's
|
|
57
|
+
* own text first, then earlier messages (most recent first) while it's
|
|
58
|
+
* under 64 bytes, then a fixed reference sentence if it still is.
|
|
59
|
+
*/
|
|
60
|
+
export declare function calibrationSample(texts: ReadonlyArray<string>): string;
|
|
61
|
+
/** What a limiter kept, once the generation has stopped. */
|
|
62
|
+
export type LimitResult = {
|
|
63
|
+
/** The text within the limit: everything sent live, and `held`. */
|
|
64
|
+
text: string;
|
|
65
|
+
/** The part of `text` held back during generation, to send now. */
|
|
66
|
+
held: string;
|
|
67
|
+
/** `text`'s tokens, measured. */
|
|
68
|
+
tokens: number;
|
|
69
|
+
/**
|
|
70
|
+
* The limit ended the text: the exact count reached it, or the limiter
|
|
71
|
+
* stopped the generation (even if the estimate ran high and the count
|
|
72
|
+
* fell short: the reply was still cut for the limit).
|
|
73
|
+
*/
|
|
74
|
+
reached: boolean;
|
|
75
|
+
};
|
|
76
|
+
/** Holds streamed text to a token limit without calling the model mid-stream. */
|
|
77
|
+
export type TokenLimiter = {
|
|
78
|
+
/** All the text received so far. */
|
|
79
|
+
readonly text: string;
|
|
80
|
+
/** The estimate passed the stop line: stop the generation. */
|
|
81
|
+
readonly stopped: boolean;
|
|
82
|
+
/**
|
|
83
|
+
* Takes the next piece of text and returns what can be sent live now
|
|
84
|
+
* (all of it, or nothing once text is held back). Never calls the model.
|
|
85
|
+
*/
|
|
86
|
+
push(delta: string): string;
|
|
87
|
+
/**
|
|
88
|
+
* Once the generation has stopped (the model is idle, so measuring is
|
|
89
|
+
* instant): the exact count, trimmed to the longest prefix within the
|
|
90
|
+
* limit, never shorter than what was already sent live.
|
|
91
|
+
*/
|
|
92
|
+
finish(): Promise<LimitResult>;
|
|
93
|
+
};
|
|
94
|
+
/**
|
|
95
|
+
* A limiter for one generation. Chrome runs one operation at a time per
|
|
96
|
+
* model: `measureContextUsage` (on any session or clone) and `clone()` wait
|
|
97
|
+
* until a generation in progress finishes, so nothing can be measured while
|
|
98
|
+
* the model writes. Instead, before generating, `sample` (the request's
|
|
99
|
+
* text) is measured to calibrate a bytes-per-token ratio; while generating,
|
|
100
|
+
* tokens are estimated from UTF-8 bytes with it; once the generation has
|
|
101
|
+
* stopped, `finish()` measures exactly and trims.
|
|
102
|
+
*
|
|
103
|
+
* Bytes per token, not characters: it varies much less across scripts (a
|
|
104
|
+
* CJK character is 3 bytes and about a token; English runs about 4 bytes a
|
|
105
|
+
* token), and no token is shorter than a byte, so the ratio's floor of 1 is
|
|
106
|
+
* the hard bound tokens ≤ bytes.
|
|
107
|
+
*/
|
|
108
|
+
export declare function tokenLimiter(session: LanguageModelSession, limit: number, sample: string, signal: AbortSignal | undefined): Promise<TokenLimiter>;
|
|
109
|
+
/**
|
|
110
|
+
* An abort signal for one generation that follows the caller's (with its
|
|
111
|
+
* reason) and that `stop()` aborts on its own, never touching the caller's.
|
|
112
|
+
*/
|
|
113
|
+
export declare function internalAbort(signal: AbortSignal | undefined, reason: string): {
|
|
114
|
+
signal: AbortSignal;
|
|
115
|
+
stop: () => void;
|
|
116
|
+
dispose: () => void | undefined;
|
|
117
|
+
};
|
|
118
|
+
export {};
|
|
119
|
+
//# sourceMappingURL=limit.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"limit.d.ts","sourceRoot":"","sources":["../src/limit.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAA;AAEtD,4EAA4E;AAC5E,wBAAgB,gBAAgB,CAAC,OAAO,EAAE;IAAE,mBAAmB,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,sBAMrF;AAED,4EAA4E;AAC5E,MAAM,MAAM,yBAAyB,GAAG;IACtC,gBAAgB,EAAE,MAAM,CAAA;IACxB,sBAAsB,EAAE,6BAA6B,CAAA;IACrD,2BAA2B,EAAE,aAAa,CAAA;IAC1C,uBAAuB,EAAE,OAAO,CAAA;CACjC,CAAA;AAED,eAAO,MAAM,gBAAgB,GAC3B,OAAO,MAAM,GAAG,SAAS,EACzB,SAAS,OAAO,KACf,yBAAyB,GAAG,SAQxB,CAAA;AAiBP,qDAAqD;AACrD,KAAK,WAAW,GAAG;IAAE,OAAO,EAAE,MAAM,GAAG,aAAa,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC,CAAA;CAAE,CAAA;AAEzF,4EAA4E;AAC5E,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,aAAa,CAAC,WAAW,CAAC,GAAG,MAAM,CAY9E;AAED;;;;;;;;GAQG;AACH,eAAO,MAAM,aAAa,OAAO,CAAA;AACjC;;;;;;;GAOG;AACH,eAAO,MAAM,WAAW,OAAO,CAAA;AAC/B,eAAO,MAAM,UAAU,IAAI,CAAA;AAY3B;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC,GAAG,MAAM,CAatE;AAED,4DAA4D;AAC5D,MAAM,MAAM,WAAW,GAAG;IACxB,mEAAmE;IACnE,IAAI,EAAE,MAAM,CAAA;IACZ,mEAAmE;IACnE,IAAI,EAAE,MAAM,CAAA;IACZ,iCAAiC;IACjC,MAAM,EAAE,MAAM,CAAA;IACd;;;;OAIG;IACH,OAAO,EAAE,OAAO,CAAA;CACjB,CAAA;AAED,iFAAiF;AACjF,MAAM,MAAM,YAAY,GAAG;IACzB,oCAAoC;IACpC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,8DAA8D;IAC9D,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAA;IACzB;;;OAGG;IACH,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAA;IAC3B;;;;OAIG;IACH,MAAM,IAAI,OAAO,CAAC,WAAW,CAAC,CAAA;CAC/B,CAAA;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAsB,YAAY,CAChC,OAAO,EAAE,oBAAoB,EAC7B,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,WAAW,GAAG,SAAS,GAC9B,OAAO,CAAC,YAAY,CAAC,CAqEvB;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,WAAW,GAAG,SAAS,EAAE,MAAM,EAAE,MAAM;;;;EAU5E"}
|