@tanstack-ai-browser/prompt-api 0.0.0-stage → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +96 -2
  3. package/dist/adapter.d.ts +160 -0
  4. package/dist/adapter.d.ts.map +1 -0
  5. package/dist/adapter.js +414 -0
  6. package/dist/index.d.ts +11 -0
  7. package/dist/index.d.ts.map +1 -0
  8. package/dist/index.js +7 -0
  9. package/dist/limit.d.ts +119 -0
  10. package/dist/limit.d.ts.map +1 -0
  11. package/dist/limit.js +203 -0
  12. package/dist/messages.d.ts +20 -0
  13. package/dist/messages.d.ts.map +1 -0
  14. package/dist/messages.js +95 -0
  15. package/dist/model-meta.d.ts +21 -0
  16. package/dist/model-meta.d.ts.map +1 -0
  17. package/dist/model-meta.js +18 -0
  18. package/dist/model.d.ts +28 -0
  19. package/dist/model.d.ts.map +1 -0
  20. package/dist/model.js +53 -0
  21. package/dist/reasoning.d.ts +42 -0
  22. package/dist/reasoning.d.ts.map +1 -0
  23. package/dist/reasoning.js +113 -0
  24. package/dist/sessions.d.ts +48 -0
  25. package/dist/sessions.d.ts.map +1 -0
  26. package/dist/sessions.js +275 -0
  27. package/dist/shared-load.d.ts +48 -0
  28. package/dist/shared-load.d.ts.map +1 -0
  29. package/dist/shared-load.js +103 -0
  30. package/dist/store.d.ts +8 -0
  31. package/dist/store.d.ts.map +1 -0
  32. package/dist/store.js +17 -0
  33. package/dist/tools.d.ts +50 -0
  34. package/dist/tools.d.ts.map +1 -0
  35. package/dist/tools.js +314 -0
  36. package/dist/types.d.ts +81 -0
  37. package/dist/types.d.ts.map +1 -0
  38. package/dist/types.js +13 -0
  39. package/package.json +60 -4
  40. package/src/adapter.test.ts +571 -0
  41. package/src/adapter.ts +522 -0
  42. package/src/index.ts +13 -0
  43. package/src/messages.ts +102 -0
  44. package/src/model-meta.ts +35 -0
  45. package/src/model.ts +75 -0
  46. package/src/tools.test.ts +129 -0
  47. package/src/tools.ts +311 -0
  48. package/src/types.ts +110 -0
@@ -0,0 +1,414 @@
1
+ /**
2
+ * A TanStack AI text adapter for the browser's built-in language model,
3
+ * through the Prompt API (`LanguageModel`): Gemini Nano in Chrome,
4
+ * Phi-4-mini in Edge. The browser downloads and runs the model itself;
5
+ * nothing leaves the device. Hand it to `chat()` like any hosted model's
6
+ * adapter: `chat()` runs tools and the agent loop, and each `chatStream`
7
+ * call here is one step of the model, streamed back as AG-UI events.
8
+ *
9
+ * Each thread (`threadId`) keeps a session: a step prompts it with only the
10
+ * messages it hasn't seen, and a conversation that changed (an edit, a new
11
+ * system prompt) starts a new one. Tools go by prompt (`tools.ts`);
12
+ * structured output prompts the thread's session under the Prompt API's own
13
+ * `responseConstraint`.
14
+ */
15
+ import { buildBaseUsage, EventType, normalizeSystemPrompts } from "@tanstack/ai";
16
+ import { BaseTextAdapter } from "@tanstack/ai/adapters";
17
+ import { contentText, mediaIn, toLanguageModelMessages } from "./messages.js";
18
+ import { baseOptions, createSession, DEFAULT_LANGUAGES, MEDIA_INPUTS, promptApiAvailability } from "./model.js";
19
+ import { toolCallReader, toolSystemPrompt } from "./tools.js";
20
+ /**
21
+ * Threads that keep a session open; the least recently used closes first. A
22
+ * session holds its conversation in the model's memory (about 100 MB for
23
+ * 6,000 tokens in Chrome 156), freed only by `destroy()`.
24
+ */
25
+ const KEPT_SESSIONS = 4;
26
+ /** Closes a session once it's made; one that failed to start has nothing to close. */
27
+ const close = (session) => void session.then((s) => s.destroy(), () => undefined);
28
+ /**
29
+ * A message as compared with what a session holds. The model's own replies
30
+ * go by their text alone: the session has them as the model wrote them.
31
+ */
32
+ const messageKey = (m) => m.role === "assistant"
33
+ ? JSON.stringify(["assistant", contentText(m.content).trim()])
34
+ : JSON.stringify([m.role, m.content, m.toolCallId ?? null, m.error ?? null]);
35
+ /** The tools a step offers: none for `toolChoice: "none"`, only the chosen one for a named choice. */
36
+ function offeredTools(options) {
37
+ const { toolChoice, tools = [] } = options;
38
+ if (toolChoice === "none")
39
+ return [];
40
+ if (typeof toolChoice === "object")
41
+ return tools.filter((tool) => tool.name === toolChoice.name);
42
+ return tools;
43
+ }
44
+ /** A step's system prompt (the caller's, then the tools), and the settings its session is made with. */
45
+ function sessionSettings(options, tools) {
46
+ const system = toolSystemPrompt(normalizeSystemPrompts(options.systemPrompts)
47
+ .map((p) => p.content)
48
+ .join("\n\n"), tools);
49
+ const { samplingMode, temperature, topK } = options.modelOptions ?? {};
50
+ return { system, settings: JSON.stringify([system, samplingMode, temperature, topK, mediaIn(options.messages)]) };
51
+ }
52
+ /** Whether a conversation (its messages' keys) goes on from what a kept session holds, with its settings. */
53
+ const goesOnFrom = (held, settings, keys) => !held.busy && held.settings === settings && held.seen.length < keys.length && held.seen.every((key, i) => key === keys[i]);
54
+ let ids = 0;
55
+ const nextId = (prefix) => `${prefix}_${Date.now().toString(36)}_${(ids++).toString(36)}`;
56
+ /** The caller's signal: the legacy `abortController` first, then `request`. */
57
+ const signalOf = (options) => options.abortController?.signal ?? options.request?.signal ?? undefined;
58
+ export class PromptApiTextAdapter extends BaseTextAdapter {
59
+ kind = "text";
60
+ name = "prompt-api";
61
+ provider = "prompt-api";
62
+ api = "prompt-api";
63
+ inputModalities = ["text", "image", "audio"];
64
+ languages;
65
+ inputs;
66
+ onDownloadProgress;
67
+ onContextOverflow;
68
+ /** Each thread's session, least recently used first. */
69
+ kept = new Map();
70
+ constructor(config, model) {
71
+ super({}, model);
72
+ this.languages = config.languages ?? DEFAULT_LANGUAGES;
73
+ this.inputs = config.inputs ?? [];
74
+ this.onDownloadProgress = config.onDownloadProgress;
75
+ this.onContextOverflow = config.onContextOverflow;
76
+ }
77
+ /**
78
+ * The browser's own `LanguageModel.availability()` for this adapter's
79
+ * `languages` and `inputs`: "available" (on the device), "downloadable"
80
+ * or "downloading" (it downloads when a session starts, or at `create`),
81
+ * or "unavailable" (no Prompt API, or the device can't run it).
82
+ */
83
+ availability() {
84
+ return promptApiAvailability(baseOptions(this.languages, this.inputs));
85
+ }
86
+ /**
87
+ * The kinds of input the browser's model takes here, of the
88
+ * `inputModalities` the Prompt API has: each kind the browser's
89
+ * `LanguageModel.availability()` doesn't call "unavailable" for this
90
+ * adapter's `languages` (images need more of the device, audio a GPU).
91
+ * Empty where there's no model. Sessions expect what each conversation has.
92
+ */
93
+ async availableInputModalities() {
94
+ if ((await promptApiAvailability(baseOptions(this.languages))) === "unavailable")
95
+ return [];
96
+ const media = await Promise.all(MEDIA_INPUTS.map(async (kind) => (await promptApiAvailability(baseOptions(this.languages, [kind]))) === "unavailable" ? [] : [kind]));
97
+ return ["text", ...media.flat()];
98
+ }
99
+ /**
100
+ * Downloads the browser's model if it isn't on the device and starts it,
101
+ * with the download's progress, 0 to 1, then 1 once it's ready. Resolves
102
+ * to this adapter; rejects if it can't start (fine to leave unawaited).
103
+ * `chat()` doesn't need it, but the browser asks for a click on the page
104
+ * before a download: call it from the click that turns the model on.
105
+ * Aborting `signal` aborts the browser's `create()`.
106
+ */
107
+ create(options = {}) {
108
+ const listeners = [this.onDownloadProgress, options.onDownloadProgress].filter((l) => !!l);
109
+ const ready = createSession(baseOptions(this.languages), options.signal, (progress) => {
110
+ for (const listener of listeners)
111
+ listener(progress);
112
+ }).then((session) => {
113
+ session.destroy();
114
+ return this;
115
+ });
116
+ // Callers may leave it: an unawaited failure isn't an unhandled rejection.
117
+ ready.catch(() => undefined);
118
+ return ready;
119
+ }
120
+ /**
121
+ * Tokens a thread's session holds (the session's `contextUsage`): the
122
+ * system prompt, the tools and the conversation so far. The thread is
123
+ * `chat()`'s `threadId` (or its `promptCache.key`); left out, the thread
124
+ * used last. Undefined where the thread keeps no session.
125
+ */
126
+ async contextUsage(threadId) {
127
+ return (await this.keptSession(threadId))?.contextUsage;
128
+ }
129
+ /**
130
+ * Tokens a thread's session can hold (the session's `contextWindow`), for
131
+ * the thread as `contextUsage` takes it. Undefined where the thread keeps
132
+ * no session.
133
+ */
134
+ async contextWindow(threadId) {
135
+ return (await this.keptSession(threadId))?.contextWindow;
136
+ }
137
+ /** A thread's kept session, or the last used thread's; undefined if there's none, or it failed to start. */
138
+ keptSession(threadId) {
139
+ const kept = threadId === undefined ? [...this.kept.values()].at(-1) : this.kept.get(threadId);
140
+ return kept ? kept.session.catch(() => undefined) : Promise.resolve(undefined);
141
+ }
142
+ /** Closes every thread's session. The browser keeps the model. */
143
+ async destroy() {
144
+ for (const kept of this.kept.values())
145
+ close(kept.session);
146
+ this.kept.clear();
147
+ }
148
+ /** Keeps a thread's session, closing the least recently used past the limit. */
149
+ keep(threadId, kept) {
150
+ this.kept.delete(threadId);
151
+ this.kept.set(threadId, kept);
152
+ for (const [id, old] of this.kept) {
153
+ if (this.kept.size <= KEPT_SESSIONS)
154
+ break;
155
+ if (old.busy)
156
+ continue;
157
+ close(old.session);
158
+ this.kept.delete(id);
159
+ }
160
+ }
161
+ /** Closes a thread's session, unless a step is using it. */
162
+ drop(threadId) {
163
+ const kept = this.kept.get(threadId);
164
+ if (!kept || kept.busy)
165
+ return;
166
+ close(kept.session);
167
+ this.kept.delete(threadId);
168
+ }
169
+ /** A session for this call, holding the system prompt, that reports an overflow. */
170
+ async session(options, system, signal) {
171
+ const { samplingMode, temperature, topK } = options.modelOptions ?? {};
172
+ const session = await createSession({
173
+ ...baseOptions(this.languages, mediaIn(options.messages)),
174
+ ...(system && { initialPrompts: [{ role: "system", content: system }] }),
175
+ ...(samplingMode !== undefined && { samplingMode }),
176
+ ...(temperature !== undefined && { temperature }),
177
+ ...(topK !== undefined && { topK })
178
+ }, signal, this.onDownloadProgress);
179
+ if (this.onContextOverflow)
180
+ session.addEventListener("contextoverflow", this.onContextOverflow);
181
+ return session;
182
+ }
183
+ /**
184
+ * The step's session and the messages to prompt it with: the thread's
185
+ * kept session if the conversation goes on from what it holds (only the
186
+ * new messages), else a new one with the whole conversation. The session
187
+ * is reserved before anything is awaited, so a second step on a busy
188
+ * thread gets a session of its own (`kept` undefined), closed after it.
189
+ * Threads go by `promptCache.key`; retention "none" keeps no session.
190
+ */
191
+ async begin(options, signal) {
192
+ const key = options.promptCache?.key ?? options.threadId;
193
+ const retain = options.promptCache?.retention !== "none";
194
+ const tools = offeredTools(options);
195
+ const { system, settings } = sessionSettings(options, tools);
196
+ const keys = options.messages.map(messageKey);
197
+ const held = retain ? this.kept.get(key) : undefined;
198
+ const goesOn = !!held && goesOnFrom(held, settings, keys);
199
+ let kept;
200
+ if (goesOn)
201
+ kept = held;
202
+ else if (retain && !held?.busy) {
203
+ this.drop(key);
204
+ kept = { session: this.session(options, system, signal), settings, seen: [], busy: false };
205
+ }
206
+ if (kept) {
207
+ kept.busy = true;
208
+ this.keep(key, kept);
209
+ }
210
+ const pending = kept?.session ?? this.session(options, system, signal);
211
+ try {
212
+ const session = await pending;
213
+ const input = await toLanguageModelMessages(options.messages, goesOn ? held.seen.length : 0);
214
+ return { key, session, kept, keys, input, tools, goesOn };
215
+ }
216
+ catch (error) {
217
+ // A session that goes on still holds the thread; a new one is dropped.
218
+ this.end(key, pending, kept, !goesOn);
219
+ throw error;
220
+ }
221
+ }
222
+ /** Frees a step's session: a kept one stays for the thread unless the step failed. */
223
+ end(key, session, kept, failed) {
224
+ if (!kept)
225
+ return close(session);
226
+ kept.busy = false;
227
+ if (failed && this.kept.get(key) === kept)
228
+ this.drop(key);
229
+ }
230
+ /**
231
+ * A step's token counts, which the Prompt API doesn't report, from the
232
+ * session's own: the tokens it held before the prompt, the prompt as the
233
+ * session measures it, and the reply measured the same way, less an empty
234
+ * message's tokens (the turn markers the session counts around any
235
+ * message). What a kept session held is cached: it isn't read again. A
236
+ * new session read what it holds (the system prompt) in this step.
237
+ * Nothing where the browser can't measure.
238
+ */
239
+ async usage(session, held, goesOn, input, reply) {
240
+ try {
241
+ const [read, written, markers] = await Promise.all([
242
+ session.measureContextUsage(input),
243
+ session.measureContextUsage(reply),
244
+ session.measureContextUsage("")
245
+ ]);
246
+ const counts = [held, read, written, markers];
247
+ if (!counts.every((count) => Number.isFinite(count)))
248
+ return undefined;
249
+ const promptTokens = held + read;
250
+ const completionTokens = Math.max(0, written - markers);
251
+ return {
252
+ ...buildBaseUsage({ promptTokens, completionTokens, totalTokens: promptTokens + completionTokens }),
253
+ // A new session read all of it in this step: none was cached.
254
+ promptTokensDetails: { cachedTokens: goesOn ? held : 0 }
255
+ };
256
+ }
257
+ catch {
258
+ return undefined;
259
+ }
260
+ }
261
+ async *chatStream(options) {
262
+ const runId = options.runId ?? nextId("run");
263
+ const { threadId } = options;
264
+ const messageId = nextId("msg");
265
+ const model = this.model;
266
+ const signal = signalOf(options);
267
+ const timestamp = () => Date.now();
268
+ yield {
269
+ type: EventType.RUN_STARTED,
270
+ runId,
271
+ threadId,
272
+ model,
273
+ timestamp: timestamp(),
274
+ ...(options.parentRunId && { parentRunId: options.parentRunId })
275
+ };
276
+ let step;
277
+ let failed = false;
278
+ // Stops the generation once a tool call is read.
279
+ const stop = new AbortController();
280
+ try {
281
+ step = await this.begin(options, signal);
282
+ const { session, kept, keys, input, tools, goesOn } = step;
283
+ // Without tools to offer, the reply is all text.
284
+ const reader = tools.length > 0 ? toolCallReader(tools.map((tool) => tool.name)) : undefined;
285
+ let text = "";
286
+ const textEvents = function* (delta) {
287
+ if (!delta)
288
+ return;
289
+ if (!text)
290
+ yield { type: EventType.TEXT_MESSAGE_START, messageId, role: "assistant", model, timestamp: timestamp() };
291
+ text += delta;
292
+ yield { type: EventType.TEXT_MESSAGE_CONTENT, messageId, delta, content: text, model, timestamp: timestamp() };
293
+ };
294
+ const generation = signal ? AbortSignal.any([signal, stop.signal]) : stop.signal;
295
+ // What the session holds before this prompt, and all the model writes (a tool call's text too).
296
+ const held = session.contextUsage;
297
+ let written = "";
298
+ try {
299
+ for await (const piece of session.promptStreaming(input, { signal: generation })) {
300
+ written += piece;
301
+ yield* textEvents(reader ? reader.push(piece) : piece);
302
+ if (reader?.call) {
303
+ stop.abort();
304
+ break;
305
+ }
306
+ }
307
+ }
308
+ catch (error) {
309
+ if (!stop.signal.aborted || signal?.aborted)
310
+ throw error;
311
+ }
312
+ const call = reader?.call;
313
+ if (reader) {
314
+ yield* textEvents(reader.end());
315
+ // Text after the call, in the piece that closed it, is shown too.
316
+ yield* textEvents(reader.trailing);
317
+ }
318
+ if (text)
319
+ yield { type: EventType.TEXT_MESSAGE_END, messageId, model, timestamp: timestamp() };
320
+ // A finished reply is in the session now. One stopped at a tool call
321
+ // isn't: Chrome drops a stopped turn whole, so the next step sends it.
322
+ if (kept && !stop.signal.aborted)
323
+ kept.seen = [...keys, messageKey({ role: "assistant", content: text })];
324
+ if (call) {
325
+ const toolCallId = nextId("call");
326
+ const args = JSON.stringify(call.arguments);
327
+ yield {
328
+ type: EventType.TOOL_CALL_START,
329
+ toolCallId,
330
+ toolCallName: call.name,
331
+ parentMessageId: messageId,
332
+ index: 0,
333
+ model,
334
+ timestamp: timestamp()
335
+ };
336
+ yield { type: EventType.TOOL_CALL_ARGS, toolCallId, delta: args, args, model, timestamp: timestamp() };
337
+ yield {
338
+ type: EventType.TOOL_CALL_END,
339
+ toolCallId,
340
+ toolCallName: call.name,
341
+ args,
342
+ input: call.arguments,
343
+ model,
344
+ timestamp: timestamp()
345
+ };
346
+ }
347
+ const usage = await this.usage(session, held, goesOn, input, written);
348
+ yield {
349
+ type: EventType.RUN_FINISHED,
350
+ runId,
351
+ threadId,
352
+ model,
353
+ timestamp: timestamp(),
354
+ finishReason: call ? "tool_calls" : "stop",
355
+ ...(usage && { usage })
356
+ };
357
+ }
358
+ catch (error) {
359
+ // An abort keeps the thread's session: Chrome drops the stopped turn.
360
+ const aborted = !!signal?.aborted;
361
+ failed = !aborted;
362
+ options.logger.errors("prompt-api chatStream failed", { error, source: "prompt-api.chatStream" });
363
+ const message = error instanceof Error ? error.message : String(error);
364
+ yield {
365
+ type: EventType.RUN_ERROR,
366
+ runId,
367
+ model,
368
+ timestamp: timestamp(),
369
+ message,
370
+ error: { message, ...(aborted && { code: "aborted" }) }
371
+ };
372
+ }
373
+ finally {
374
+ if (step)
375
+ this.end(step.key, Promise.resolve(step.session), step.kept, failed);
376
+ }
377
+ }
378
+ /**
379
+ * A JSON answer for `chat({ outputSchema })`: the thread's session
380
+ * answers the conversation under the Prompt API's `responseConstraint`,
381
+ * which holds the model to the schema as given. `chat()` validates the
382
+ * result.
383
+ */
384
+ async structuredOutput(options) {
385
+ const { chatOptions, outputSchema } = options;
386
+ const signal = signalOf(chatOptions);
387
+ let step;
388
+ let failed = false;
389
+ try {
390
+ step = await this.begin(chatOptions, signal);
391
+ const { session, kept, keys, input, goesOn } = step;
392
+ const held = session.contextUsage;
393
+ const rawText = await session.prompt(input, { responseConstraint: outputSchema, signal });
394
+ if (kept)
395
+ kept.seen = [...keys, messageKey({ role: "assistant", content: rawText })];
396
+ const data = JSON.parse(rawText);
397
+ const usage = await this.usage(session, held, goesOn, input, rawText);
398
+ return { data, rawText, model: this.model, ...(usage && { usage }) };
399
+ }
400
+ catch (error) {
401
+ failed = true;
402
+ throw error;
403
+ }
404
+ finally {
405
+ if (step)
406
+ this.end(step.key, Promise.resolve(step.session), step.kept, failed);
407
+ }
408
+ }
409
+ }
410
+ export function promptApiText(modelOrConfig, config = {}) {
411
+ return typeof modelOrConfig === "string"
412
+ ? new PromptApiTextAdapter(config, modelOrConfig)
413
+ : new PromptApiTextAdapter(modelOrConfig ?? {}, "text");
414
+ }
@@ -0,0 +1,11 @@
1
+ /**
2
+ * TanStack AI adapter for the browser's built-in language model, through
3
+ * the Prompt API: Gemini Nano in Chrome, Phi-4-mini in Edge. The model
4
+ * catalog (model-meta.ts) is exported here too.
5
+ */
6
+ export { PromptApiTextAdapter, promptApiText } from "./adapter.js";
7
+ export type { PromptApiConfig, PromptApiTextModelOptions } from "./adapter.js";
8
+ export type { Availability, DownloadProgressCallback } from "./types.js";
9
+ export { PROMPT_API_CHAT_MODELS, PROMPT_API_MODEL_META } from "./model-meta.js";
10
+ export type { PromptApiChatModel, PromptApiModelMeta } from "./model-meta.js";
11
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,cAAc,CAAA;AAClE,YAAY,EACV,eAAe,EACf,yBAAyB,EAC1B,MAAM,cAAc,CAAA;AACrB,YAAY,EAAE,YAAY,EAAE,wBAAwB,EAAE,MAAM,YAAY,CAAA;AACxE,OAAO,EAAE,sBAAsB,EAAE,qBAAqB,EAAE,MAAM,iBAAiB,CAAA;AAC/E,YAAY,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA"}
package/dist/index.js ADDED
@@ -0,0 +1,7 @@
1
+ /**
2
+ * TanStack AI adapter for the browser's built-in language model, through
3
+ * the Prompt API: Gemini Nano in Chrome, Phi-4-mini in Edge. The model
4
+ * catalog (model-meta.ts) is exported here too.
5
+ */
6
+ export { PromptApiTextAdapter, promptApiText } from "./adapter.js";
7
+ export { PROMPT_API_CHAT_MODELS, PROMPT_API_MODEL_META } from "./model-meta.js";
@@ -0,0 +1,119 @@
1
+ /**
2
+ * Output token limits, emulated. The Prompt API has no output cap (a prompt
3
+ * takes only `signal`, `responseConstraint` and
4
+ * `omitResponseConstraintInput`), so the adapter holds the reply to a limit
5
+ * itself, for `maxCompletionTokens`.
6
+ *
7
+ * Chrome runs one operation at a time per model, so the model can't count
8
+ * text while it writes it. The limit is estimated during the generation
9
+ * (from a ratio calibrated on the request before it starts), the generation
10
+ * is stopped by an internal abort a little past the limit, and the text is
11
+ * measured and trimmed exactly once the model is idle again.
12
+ */
13
+ import type { LanguageModelSession } from "./types.js";
14
+ /** `maxCompletionTokens`, validated like the other adapters validate it. */
15
+ export declare function outputTokenLimit(options: {
16
+ maxCompletionTokens?: number;
17
+ } | undefined): number | undefined;
18
+ /** The cap's report in `usage.providerUsageDetails`, when a cap was set. */
19
+ export type PromptApiOutputLimitUsage = {
20
+ outputTokenLimit: number;
21
+ outputTokenMeasurement: "estimate-then-context-usage";
22
+ outputTokenLimitEnforcement: "best-effort";
23
+ outputTokenLimitReached: boolean;
24
+ };
25
+ export declare const outputLimitUsage: (limit: number | undefined, reached: boolean) => PromptApiOutputLimitUsage | undefined;
26
+ /** A message's text, without its images or audio. */
27
+ type TextBearing = {
28
+ content: string | ReadonlyArray<{
29
+ type: string;
30
+ value?: unknown;
31
+ }>;
32
+ };
33
+ /** The text of a prompt input (or of messages), without images or audio. */
34
+ export declare function textOfInput(input: string | ReadonlyArray<TextBearing>): string;
35
+ /**
36
+ * Live emission stops once the estimate passes this share of the limit, or
37
+ * once the text passes the limit in UTF-8 bytes. Text sent live can't be
38
+ * taken back, so it must stay within the limit whatever the estimate says:
39
+ * no token is shorter than a byte, so text of at most `limit` bytes holds at
40
+ * most `limit` tokens, however much denser the reply is than the calibration
41
+ * text. The share keeps longer limits from streaming up to the last token
42
+ * live, where a low estimate would leave nothing to trim.
43
+ */
44
+ export declare const LIVE_FRACTION = 0.75;
45
+ /**
46
+ * Generation is stopped once the estimate passes STOP_FACTOR × limit +
47
+ * STOP_SLACK tokens: past the limit, so the exact trim afterwards finds the
48
+ * limit's worth of text unless the reply holds a fifth fewer tokens per byte
49
+ * than the calibration text (the estimate is then high by more than 1.25×).
50
+ * The slack covers short limits, where a chunk or two of noise is a large
51
+ * share; 8 tokens is about 0.15 s of Gemini Nano's decoding.
52
+ */
53
+ export declare const STOP_FACTOR = 1.25;
54
+ export declare const STOP_SLACK = 8;
55
+ /**
56
+ * The text a limit's tokens-per-byte ratio is calibrated on: the request's
57
+ * own text first, then earlier messages (most recent first) while it's
58
+ * under 64 bytes, then a fixed reference sentence if it still is.
59
+ */
60
+ export declare function calibrationSample(texts: ReadonlyArray<string>): string;
61
+ /** What a limiter kept, once the generation has stopped. */
62
+ export type LimitResult = {
63
+ /** The text within the limit: everything sent live, and `held`. */
64
+ text: string;
65
+ /** The part of `text` held back during generation, to send now. */
66
+ held: string;
67
+ /** `text`'s tokens, measured. */
68
+ tokens: number;
69
+ /**
70
+ * The limit ended the text: the exact count reached it, or the limiter
71
+ * stopped the generation (even if the estimate ran high and the count
72
+ * fell short: the reply was still cut for the limit).
73
+ */
74
+ reached: boolean;
75
+ };
76
+ /** Holds streamed text to a token limit without calling the model mid-stream. */
77
+ export type TokenLimiter = {
78
+ /** All the text received so far. */
79
+ readonly text: string;
80
+ /** The estimate passed the stop line: stop the generation. */
81
+ readonly stopped: boolean;
82
+ /**
83
+ * Takes the next piece of text and returns what can be sent live now
84
+ * (all of it, or nothing once text is held back). Never calls the model.
85
+ */
86
+ push(delta: string): string;
87
+ /**
88
+ * Once the generation has stopped (the model is idle, so measuring is
89
+ * instant): the exact count, trimmed to the longest prefix within the
90
+ * limit, never shorter than what was already sent live.
91
+ */
92
+ finish(): Promise<LimitResult>;
93
+ };
94
+ /**
95
+ * A limiter for one generation. Chrome runs one operation at a time per
96
+ * model: `measureContextUsage` (on any session or clone) and `clone()` wait
97
+ * until a generation in progress finishes, so nothing can be measured while
98
+ * the model writes. Instead, before generating, `sample` (the request's
99
+ * text) is measured to calibrate a bytes-per-token ratio; while generating,
100
+ * tokens are estimated from UTF-8 bytes with it; once the generation has
101
+ * stopped, `finish()` measures exactly and trims.
102
+ *
103
+ * Bytes per token, not characters: it varies much less across scripts (a
104
+ * CJK character is 3 bytes and about a token; English runs about 4 bytes a
105
+ * token), and no token is shorter than a byte, so the ratio's floor of 1 is
106
+ * the hard bound tokens ≤ bytes.
107
+ */
108
+ export declare function tokenLimiter(session: LanguageModelSession, limit: number, sample: string, signal: AbortSignal | undefined): Promise<TokenLimiter>;
109
+ /**
110
+ * An abort signal for one generation that follows the caller's (with its
111
+ * reason) and that `stop()` aborts on its own, never touching the caller's.
112
+ */
113
+ export declare function internalAbort(signal: AbortSignal | undefined, reason: string): {
114
+ signal: AbortSignal;
115
+ stop: () => void;
116
+ dispose: () => void | undefined;
117
+ };
118
+ export {};
119
+ //# sourceMappingURL=limit.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"limit.d.ts","sourceRoot":"","sources":["../src/limit.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAA;AAEtD,4EAA4E;AAC5E,wBAAgB,gBAAgB,CAAC,OAAO,EAAE;IAAE,mBAAmB,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,sBAMrF;AAED,4EAA4E;AAC5E,MAAM,MAAM,yBAAyB,GAAG;IACtC,gBAAgB,EAAE,MAAM,CAAA;IACxB,sBAAsB,EAAE,6BAA6B,CAAA;IACrD,2BAA2B,EAAE,aAAa,CAAA;IAC1C,uBAAuB,EAAE,OAAO,CAAA;CACjC,CAAA;AAED,eAAO,MAAM,gBAAgB,GAC3B,OAAO,MAAM,GAAG,SAAS,EACzB,SAAS,OAAO,KACf,yBAAyB,GAAG,SAQxB,CAAA;AAiBP,qDAAqD;AACrD,KAAK,WAAW,GAAG;IAAE,OAAO,EAAE,MAAM,GAAG,aAAa,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,OAAO,CAAA;KAAE,CAAC,CAAA;CAAE,CAAA;AAEzF,4EAA4E;AAC5E,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,aAAa,CAAC,WAAW,CAAC,GAAG,MAAM,CAY9E;AAED;;;;;;;;GAQG;AACH,eAAO,MAAM,aAAa,OAAO,CAAA;AACjC;;;;;;;GAOG;AACH,eAAO,MAAM,WAAW,OAAO,CAAA;AAC/B,eAAO,MAAM,UAAU,IAAI,CAAA;AAY3B;;;;GAIG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,aAAa,CAAC,MAAM,CAAC,GAAG,MAAM,CAatE;AAED,4DAA4D;AAC5D,MAAM,MAAM,WAAW,GAAG;IACxB,mEAAmE;IACnE,IAAI,EAAE,MAAM,CAAA;IACZ,mEAAmE;IACnE,IAAI,EAAE,MAAM,CAAA;IACZ,iCAAiC;IACjC,MAAM,EAAE,MAAM,CAAA;IACd;;;;OAIG;IACH,OAAO,EAAE,OAAO,CAAA;CACjB,CAAA;AAED,iFAAiF;AACjF,MAAM,MAAM,YAAY,GAAG;IACzB,oCAAoC;IACpC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,8DAA8D;IAC9D,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAA;IACzB;;;OAGG;IACH,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAAA;IAC3B;;;;OAIG;IACH,MAAM,IAAI,OAAO,CAAC,WAAW,CAAC,CAAA;CAC/B,CAAA;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAsB,YAAY,CAChC,OAAO,EAAE,oBAAoB,EAC7B,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,MAAM,EAAE,WAAW,GAAG,SAAS,GAC9B,OAAO,CAAC,YAAY,CAAC,CAqEvB;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,WAAW,GAAG,SAAS,EAAE,MAAM,EAAE,MAAM;;;;EAU5E"}