@missionsquad/browserai 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. package/LICENSE +15 -0
  2. package/README.md +229 -0
  3. package/dist/audio/clip.d.ts +51 -0
  4. package/dist/audio/clip.d.ts.map +1 -0
  5. package/dist/audio/clip.js +71 -0
  6. package/dist/audio/clip.js.map +1 -0
  7. package/dist/audio/decode.d.ts +7 -0
  8. package/dist/audio/decode.d.ts.map +1 -0
  9. package/dist/audio/decode.js +19 -0
  10. package/dist/audio/decode.js.map +1 -0
  11. package/dist/audio/pcm.d.ts +13 -0
  12. package/dist/audio/pcm.d.ts.map +1 -0
  13. package/dist/audio/pcm.js +66 -0
  14. package/dist/audio/pcm.js.map +1 -0
  15. package/dist/audio/recorder.d.ts +40 -0
  16. package/dist/audio/recorder.d.ts.map +1 -0
  17. package/dist/audio/recorder.js +127 -0
  18. package/dist/audio/recorder.js.map +1 -0
  19. package/dist/audio/streaming-player.d.ts +29 -0
  20. package/dist/audio/streaming-player.d.ts.map +1 -0
  21. package/dist/audio/streaming-player.js +82 -0
  22. package/dist/audio/streaming-player.js.map +1 -0
  23. package/dist/client.d.ts +195 -0
  24. package/dist/client.d.ts.map +1 -0
  25. package/dist/client.js +638 -0
  26. package/dist/client.js.map +1 -0
  27. package/dist/config.d.ts +81 -0
  28. package/dist/config.d.ts.map +1 -0
  29. package/dist/config.js +19 -0
  30. package/dist/config.js.map +1 -0
  31. package/dist/errors.d.ts +52 -0
  32. package/dist/errors.d.ts.map +1 -0
  33. package/dist/errors.js +74 -0
  34. package/dist/errors.js.map +1 -0
  35. package/dist/events.d.ts +18 -0
  36. package/dist/events.d.ts.map +1 -0
  37. package/dist/events.js +43 -0
  38. package/dist/events.js.map +1 -0
  39. package/dist/generation.d.ts +12 -0
  40. package/dist/generation.d.ts.map +1 -0
  41. package/dist/generation.js +30 -0
  42. package/dist/generation.js.map +1 -0
  43. package/dist/hardware.d.ts +23 -0
  44. package/dist/hardware.d.ts.map +1 -0
  45. package/dist/hardware.js +150 -0
  46. package/dist/hardware.js.map +1 -0
  47. package/dist/history.d.ts +81 -0
  48. package/dist/history.d.ts.map +1 -0
  49. package/dist/history.js +193 -0
  50. package/dist/history.js.map +1 -0
  51. package/dist/index.d.ts +51 -0
  52. package/dist/index.d.ts.map +1 -0
  53. package/dist/index.js +50 -0
  54. package/dist/index.js.map +1 -0
  55. package/dist/inference.d.ts +89 -0
  56. package/dist/inference.d.ts.map +1 -0
  57. package/dist/inference.js +411 -0
  58. package/dist/inference.js.map +1 -0
  59. package/dist/json.d.ts +22 -0
  60. package/dist/json.d.ts.map +1 -0
  61. package/dist/json.js +145 -0
  62. package/dist/json.js.map +1 -0
  63. package/dist/metrics.d.ts +18 -0
  64. package/dist/metrics.d.ts.map +1 -0
  65. package/dist/metrics.js +59 -0
  66. package/dist/metrics.js.map +1 -0
  67. package/dist/model-cache.d.ts +49 -0
  68. package/dist/model-cache.d.ts.map +1 -0
  69. package/dist/model-cache.js +511 -0
  70. package/dist/model-cache.js.map +1 -0
  71. package/dist/model-source.d.ts +27 -0
  72. package/dist/model-source.d.ts.map +1 -0
  73. package/dist/model-source.js +110 -0
  74. package/dist/model-source.js.map +1 -0
  75. package/dist/models.d.ts +103 -0
  76. package/dist/models.d.ts.map +1 -0
  77. package/dist/models.js +1052 -0
  78. package/dist/models.js.map +1 -0
  79. package/dist/multimodal.d.ts +261 -0
  80. package/dist/multimodal.d.ts.map +1 -0
  81. package/dist/multimodal.js +506 -0
  82. package/dist/multimodal.js.map +1 -0
  83. package/dist/prompt.d.ts +13 -0
  84. package/dist/prompt.d.ts.map +1 -0
  85. package/dist/prompt.js +32 -0
  86. package/dist/prompt.js.map +1 -0
  87. package/dist/proxy-verify.d.ts +10 -0
  88. package/dist/proxy-verify.d.ts.map +1 -0
  89. package/dist/proxy-verify.js +98 -0
  90. package/dist/proxy-verify.js.map +1 -0
  91. package/dist/schema-validator.d.ts +6 -0
  92. package/dist/schema-validator.d.ts.map +1 -0
  93. package/dist/schema-validator.js +104 -0
  94. package/dist/schema-validator.js.map +1 -0
  95. package/dist/schema.d.ts +98 -0
  96. package/dist/schema.d.ts.map +1 -0
  97. package/dist/schema.js +115 -0
  98. package/dist/schema.js.map +1 -0
  99. package/dist/text-utils.d.ts +8 -0
  100. package/dist/text-utils.d.ts.map +1 -0
  101. package/dist/text-utils.js +14 -0
  102. package/dist/text-utils.js.map +1 -0
  103. package/dist/voice/helpers.d.ts +35 -0
  104. package/dist/voice/helpers.d.ts.map +1 -0
  105. package/dist/voice/helpers.js +49 -0
  106. package/dist/voice/helpers.js.map +1 -0
  107. package/dist/voice/voice-pipeline.d.ts +155 -0
  108. package/dist/voice/voice-pipeline.d.ts.map +1 -0
  109. package/dist/voice/voice-pipeline.js +407 -0
  110. package/dist/voice/voice-pipeline.js.map +1 -0
  111. package/dist/webllm.worker.d.ts +2 -0
  112. package/dist/webllm.worker.d.ts.map +1 -0
  113. package/dist/webllm.worker.js +8 -0
  114. package/dist/webllm.worker.js.map +1 -0
  115. package/package.json +62 -0
  116. package/src/audio/clip.ts +94 -0
  117. package/src/audio/decode.ts +19 -0
  118. package/src/audio/pcm.ts +62 -0
  119. package/src/audio/recorder.ts +146 -0
  120. package/src/audio/streaming-player.ts +99 -0
  121. package/src/client.ts +784 -0
  122. package/src/config.ts +104 -0
  123. package/src/errors.ts +81 -0
  124. package/src/events.ts +53 -0
  125. package/src/generation.ts +39 -0
  126. package/src/hardware.ts +193 -0
  127. package/src/history.ts +257 -0
  128. package/src/index.ts +203 -0
  129. package/src/inference.ts +579 -0
  130. package/src/json.ts +169 -0
  131. package/src/metrics.ts +63 -0
  132. package/src/model-cache.ts +637 -0
  133. package/src/model-source.ts +142 -0
  134. package/src/models.ts +1171 -0
  135. package/src/multimodal.ts +697 -0
  136. package/src/prompt.ts +33 -0
  137. package/src/proxy-verify.ts +113 -0
  138. package/src/schema-validator.ts +117 -0
  139. package/src/schema.ts +118 -0
  140. package/src/text-utils.ts +12 -0
  141. package/src/voice/helpers.ts +59 -0
  142. package/src/voice/voice-pipeline.ts +513 -0
  143. package/src/webllm.worker.ts +9 -0
  144. package/worker-template/README.md +39 -0
  145. package/worker-template/cloudflare-worker.ts +389 -0
  146. package/worker-template/wrangler.jsonc +19 -0
@@ -0,0 +1,579 @@
1
+ import type { MLCEngineInterface } from "@mlc-ai/web-llm";
2
+ import type { ModelBackend, ModelPreset } from "./models.js";
3
+ import type { RuntimeParameters } from "./generation.js";
4
+ import { buildResponseFormat } from "./generation.js";
5
+ import { formatError } from "./errors.js";
6
+
7
+ export type ChatMessage = { role: "system" | "user" | "assistant"; content: string };
8
+
9
+ export type LatencyBreakdown = Record<string, number[]>;
10
+
11
+ export type WebLLMUsageExtra = {
12
+ e2e_latency_s?: number;
13
+ prefill_tokens_per_s?: number;
14
+ decode_tokens_per_s?: number;
15
+ time_to_first_token_s?: number;
16
+ time_per_output_token_s?: number;
17
+ grammar_init_s?: number;
18
+ grammar_per_token_s?: number;
19
+ latencyBreakdown?: LatencyBreakdown;
20
+ token_count_source?: string;
21
+ [key: string]: unknown;
22
+ };
23
+
24
+ export type RuntimeStats = {
25
+ rawText: string;
26
+ model: string;
27
+ backend: ModelBackend | "unknown";
28
+ finishReason: string;
29
+ measuredElapsedMs: number;
30
+ promptTokens?: number;
31
+ completionTokens?: number;
32
+ totalTokens?: number;
33
+ measuredCompletionTokensPerSecond?: number;
34
+ extra?: WebLLMUsageExtra;
35
+ latencyBreakdown?: LatencyBreakdown;
36
+ legacyStats?: string;
37
+ error?: string;
38
+ };
39
+
40
+ /* ------------------------------------------------------------------ */
41
+ /* Transformers.js structural types (the package ships loose typings). */
42
+ /* ------------------------------------------------------------------ */
43
+
44
+ export type TransformersTokenizer = {
45
+ apply_chat_template?: (messages: unknown, options?: Record<string, unknown>) => unknown;
46
+ encode?: (text: string, options?: Record<string, unknown>) => unknown;
47
+ decode?: (tokens: unknown, options?: Record<string, unknown>) => string;
48
+ batch_decode?: (tokens: unknown, options?: Record<string, unknown>) => string[];
49
+ all_special_ids?: unknown[];
50
+ (input: unknown, options?: Record<string, unknown>): unknown;
51
+ };
52
+
53
+ export type TransformersTextGenerationPipeline = ((input: unknown, options?: Record<string, unknown>) => Promise<unknown>) & {
54
+ tokenizer?: TransformersTokenizer;
55
+ model?: { dispose?: () => unknown };
56
+ dispose?: () => unknown;
57
+ };
58
+
59
+ export type TransformersModule = {
60
+ env?: Record<string, unknown>;
61
+ pipeline: (task: string, model: string, options?: Record<string, unknown>) => Promise<TransformersTextGenerationPipeline>;
62
+ TextStreamer?: new (tokenizer: TransformersTokenizer, options?: Record<string, unknown>) => unknown;
63
+ };
64
+
65
+ type TransformersGenerationTelemetry = {
66
+ promptTokens?: number;
67
+ completionTokens?: number;
68
+ firstTokenOffsetMs?: number;
69
+ tokenCallbackIntervalSeconds: number[];
70
+ tokenCallbackCount: number;
71
+ tokenCountSource: "streamer" | "tokenizer" | "unavailable";
72
+ };
73
+
74
+ /**
75
+ * Live engine references owned by the client and passed into the stateless generation helpers.
76
+ */
77
+ export type EngineHandles = {
78
+ engine: MLCEngineInterface | null;
79
+ transformersGenerator: TransformersTextGenerationPipeline | null;
80
+ transformersTokenizer: TransformersTokenizer | null;
81
+ transformersModule: TransformersModule | null;
82
+ backend: ModelBackend | null;
83
+ modelId: string | null;
84
+ };
85
+
86
+ export type GenerationResult = {
87
+ text: string;
88
+ stats: RuntimeStats;
89
+ };
90
+
91
+ export type GenerateOptions = {
92
+ /** Structured cases pass a schema + json mode for WebLLM constrained decoding. */
93
+ schema?: unknown;
94
+ /** Stream tokens to this callback (receives the full accumulated text so far). */
95
+ onDelta?: (fullText: string) => void;
96
+ /** Reset the WebLLM KV cache before generating. Default true (stateless). Set false for multi-turn chat that relies on passed history — history is still passed explicitly, so reset stays safe. */
97
+ resetChat?: boolean;
98
+ };
99
+
100
+ /** Sentinel substituted for blank model output; exported so callers can detect/skip it (e.g. TTS). */
101
+ export const EMPTY_RESPONSE = "[empty response]";
102
+
103
+ /**
104
+ * Run one generation against whichever backend is loaded and return the text plus normalized stats.
105
+ * Dispatches to WebLLM (streaming or blocking) or Transformers.js. Throws if no engine is loaded.
106
+ */
107
+ export async function generate(
108
+ handles: EngineHandles,
109
+ messages: ChatMessage[],
110
+ runtime: RuntimeParameters,
111
+ preset: ModelPreset,
112
+ options: GenerateOptions = {},
113
+ ): Promise<GenerationResult> {
114
+ const started = performance.now();
115
+
116
+ if (handles.backend === "transformers-js") {
117
+ if (!handles.transformersGenerator) throw new Error("Transformers.js pipeline is not loaded.");
118
+ const telemetry = await runTransformers(handles, messages, runtime, preset, options.onDelta, started);
119
+ const ended = performance.now();
120
+ const text = telemetry.text || EMPTY_RESPONSE;
121
+ const stats = collectTransformersStats(preset.id, runtime, telemetry, started, ended);
122
+ return { text, stats };
123
+ }
124
+
125
+ if (!handles.engine) throw new Error("WebLLM engine is not loaded.");
126
+
127
+ if (options.resetChat !== false) {
128
+ await handles.engine.resetChat(false);
129
+ }
130
+
131
+ const responseFormat = options.schema !== undefined ? buildResponseFormat(runtime.jsonMode, options.schema) : undefined;
132
+ const extraBody = buildExtraBody(runtime, preset);
133
+
134
+ const outcome = options.onDelta
135
+ ? await runWebLLMStream(handles.engine, messages, runtime, responseFormat, extraBody, options.onDelta)
136
+ : await runWebLLMBlocking(handles.engine, messages, runtime, responseFormat, extraBody);
137
+
138
+ const ended = performance.now();
139
+ const stats = await collectWebLLMStats(handles, outcome.usage, outcome.finishReason, started, ended, outcome.firstTokenMs);
140
+ return { text: outcome.text || EMPTY_RESPONSE, stats };
141
+ }
142
+
143
+ function buildExtraBody(runtime: RuntimeParameters, preset: ModelPreset): Record<string, unknown> | undefined {
144
+ const extraBody: Record<string, unknown> = {};
145
+ if (preset.disableThinkingSupported) {
146
+ extraBody.enable_thinking = runtime.disableThinking === false;
147
+ }
148
+ if (runtime.latencyBreakdown) {
149
+ extraBody.enable_latency_breakdown = true;
150
+ }
151
+ return Object.keys(extraBody).length > 0 ? extraBody : undefined;
152
+ }
153
+
154
+ type WebLLMOutcome = {
155
+ text: string;
156
+ finishReason: string;
157
+ usage?: WebLLMUsage;
158
+ firstTokenMs?: number;
159
+ };
160
+
161
+ type WebLLMUsage = {
162
+ prompt_tokens?: number;
163
+ completion_tokens?: number;
164
+ total_tokens?: number;
165
+ extra?: WebLLMUsageExtra;
166
+ };
167
+
168
+ async function runWebLLMBlocking(
169
+ engine: MLCEngineInterface,
170
+ messages: ChatMessage[],
171
+ runtime: RuntimeParameters,
172
+ responseFormat: ReturnType<typeof buildResponseFormat>,
173
+ extraBody: Record<string, unknown> | undefined,
174
+ ): Promise<WebLLMOutcome> {
175
+ const response = await engine.chat.completions.create({
176
+ messages,
177
+ stream: false,
178
+ n: 1,
179
+ temperature: runtime.temperature,
180
+ top_p: runtime.topP,
181
+ max_tokens: runtime.maxTokens,
182
+ repetition_penalty: runtime.repetitionPenalty,
183
+ frequency_penalty: runtime.frequencyPenalty,
184
+ presence_penalty: runtime.presencePenalty,
185
+ seed: runtime.seed ?? undefined,
186
+ response_format: responseFormat,
187
+ extra_body: extraBody,
188
+ });
189
+
190
+ return {
191
+ text: response.choices[0]?.message?.content ?? "",
192
+ finishReason: response.choices[0]?.finish_reason ?? "unknown",
193
+ usage: response.usage as WebLLMUsage | undefined,
194
+ };
195
+ }
196
+
197
+ async function runWebLLMStream(
198
+ engine: MLCEngineInterface,
199
+ messages: ChatMessage[],
200
+ runtime: RuntimeParameters,
201
+ responseFormat: ReturnType<typeof buildResponseFormat>,
202
+ extraBody: Record<string, unknown> | undefined,
203
+ onDelta: (fullText: string) => void,
204
+ ): Promise<WebLLMOutcome> {
205
+ const stream = await engine.chat.completions.create({
206
+ messages,
207
+ stream: true,
208
+ stream_options: { include_usage: true },
209
+ temperature: runtime.temperature,
210
+ top_p: runtime.topP,
211
+ max_tokens: runtime.maxTokens,
212
+ repetition_penalty: runtime.repetitionPenalty,
213
+ frequency_penalty: runtime.frequencyPenalty,
214
+ presence_penalty: runtime.presencePenalty,
215
+ seed: runtime.seed ?? undefined,
216
+ response_format: responseFormat,
217
+ extra_body: extraBody,
218
+ });
219
+
220
+ let text = "";
221
+ let finishReason = "unknown";
222
+ let usage: WebLLMUsage | undefined;
223
+ let firstTokenMs: number | undefined;
224
+
225
+ for await (const chunk of stream) {
226
+ const choice = chunk.choices[0];
227
+ const delta = choice?.delta?.content ?? "";
228
+ if (delta) {
229
+ if (firstTokenMs === undefined) firstTokenMs = performance.now();
230
+ text += delta;
231
+ onDelta(text);
232
+ }
233
+ if (choice?.finish_reason) finishReason = choice.finish_reason;
234
+ if (chunk.usage) usage = chunk.usage as WebLLMUsage;
235
+ }
236
+
237
+ return { text, finishReason, usage, firstTokenMs };
238
+ }
239
+
240
+ async function collectWebLLMStats(
241
+ handles: EngineHandles,
242
+ usage: WebLLMUsage | undefined,
243
+ finishReason: string,
244
+ started: number,
245
+ ended: number,
246
+ firstTokenMs?: number,
247
+ ): Promise<RuntimeStats> {
248
+ const elapsedMs = ended - started;
249
+ const completionTokens = usage?.completion_tokens;
250
+ const measured = typeof completionTokens === "number" && elapsedMs > 0 ? completionTokens / (elapsedMs / 1000) : undefined;
251
+ // Prefer the engine's telemetry. When it is absent (some engine/model combos omit usage.extra),
252
+ // synthesize the headline metrics from what we measured so they never blank out.
253
+ const extra: WebLLMUsageExtra | undefined = usage?.extra
254
+ ? { ...usage.extra, latencyBreakdown: normalizeLatencyBreakdown(usage.extra.latencyBreakdown) }
255
+ : measured !== undefined || firstTokenMs !== undefined
256
+ ? {
257
+ decode_tokens_per_s: measured,
258
+ e2e_latency_s: elapsedMs / 1000,
259
+ time_to_first_token_s: firstTokenMs !== undefined ? Math.max(0, (firstTokenMs - started) / 1000) : undefined,
260
+ }
261
+ : undefined;
262
+
263
+ let legacyStats: string | undefined;
264
+ if (handles.engine && handles.modelId) {
265
+ try {
266
+ const engineStats = await handles.engine.runtimeStatsText(handles.modelId);
267
+ legacyStats = engineStats.trim() || undefined;
268
+ } catch (error) {
269
+ legacyStats = `unavailable: ${formatError(error)}`;
270
+ }
271
+ }
272
+
273
+ const stats: RuntimeStats = {
274
+ rawText: "",
275
+ model: handles.modelId ?? "n/a",
276
+ backend: "webllm",
277
+ finishReason,
278
+ measuredElapsedMs: elapsedMs,
279
+ promptTokens: usage?.prompt_tokens,
280
+ completionTokens: usage?.completion_tokens,
281
+ totalTokens: usage?.total_tokens,
282
+ measuredCompletionTokensPerSecond: measured,
283
+ extra,
284
+ latencyBreakdown: extra?.latencyBreakdown,
285
+ legacyStats,
286
+ };
287
+ stats.rawText = buildRuntimeStatsText(stats);
288
+ return stats;
289
+ }
290
+
291
+ /* ------------------------------------------------------------------ */
292
+ /* Transformers.js path */
293
+ /* ------------------------------------------------------------------ */
294
+
295
+ type TransformersRunResult = TransformersGenerationTelemetry & { text: string };
296
+
297
+ async function runTransformers(
298
+ handles: EngineHandles,
299
+ messages: ChatMessage[],
300
+ runtime: RuntimeParameters,
301
+ preset: ModelPreset,
302
+ onDelta: ((fullText: string) => void) | undefined,
303
+ // Origin clock shared with generate()'s `started` so TTFT and e2e latency are measured consistently.
304
+ startedMs: number,
305
+ ): Promise<TransformersRunResult> {
306
+ const generator = handles.transformersGenerator!;
307
+ const tokenizer = handles.transformersTokenizer ?? generator.tokenizer ?? null;
308
+
309
+ const telemetry: TransformersGenerationTelemetry = {
310
+ promptTokens: countPromptTokens(tokenizer, messages),
311
+ tokenCallbackIntervalSeconds: [],
312
+ tokenCallbackCount: 0,
313
+ tokenCountSource: "unavailable",
314
+ };
315
+ const tokenTimesMs: number[] = [];
316
+
317
+ const generationOptions: Record<string, unknown> = {
318
+ max_new_tokens: runtime.maxTokens,
319
+ do_sample: runtime.temperature > 0,
320
+ top_p: runtime.topP,
321
+ repetition_penalty: runtime.repetitionPenalty,
322
+ return_full_text: false,
323
+ };
324
+ if (runtime.temperature > 0) generationOptions.temperature = runtime.temperature;
325
+ if (runtime.seed !== null) generationOptions.seed = runtime.seed;
326
+ if (preset.disableThinkingSupported) {
327
+ generationOptions.tokenizer_encode_kwargs = { enable_thinking: !runtime.disableThinking };
328
+ }
329
+
330
+ const Streamer = handles.transformersModule?.TextStreamer;
331
+ let streamedText = "";
332
+ if (tokenizer && Streamer) {
333
+ generationOptions.streamer = new Streamer(tokenizer, {
334
+ skip_prompt: true,
335
+ skip_special_tokens: true,
336
+ callback_function: (chunk: unknown) => {
337
+ if (typeof chunk === "string" && onDelta) {
338
+ streamedText += chunk;
339
+ onDelta(streamedText);
340
+ }
341
+ },
342
+ token_callback_function: (tokens: unknown) => {
343
+ const now = performance.now();
344
+ const count = Math.max(1, countTokenItems(tokens));
345
+ for (let index = 0; index < count; index += 1) tokenTimesMs.push(now);
346
+ },
347
+ });
348
+ }
349
+
350
+ const output = await generator(messages, generationOptions);
351
+ const text = extractGeneratedText(output) || EMPTY_RESPONSE;
352
+
353
+ if (tokenTimesMs.length > 0) {
354
+ telemetry.tokenCallbackCount = tokenTimesMs.length;
355
+ telemetry.tokenCountSource = "streamer";
356
+ telemetry.completionTokens = tokenTimesMs.length;
357
+ telemetry.firstTokenOffsetMs = Math.max(0, tokenTimesMs[0] - startedMs);
358
+ telemetry.tokenCallbackIntervalSeconds = tokenTimesMs.slice(1).map((time, index) => Math.max(0, (time - tokenTimesMs[index]) / 1000));
359
+ } else {
360
+ const fallback = countCompletionTokens(tokenizer, text);
361
+ if (typeof fallback === "number") {
362
+ telemetry.completionTokens = fallback;
363
+ telemetry.tokenCountSource = "tokenizer";
364
+ }
365
+ }
366
+
367
+ return { ...telemetry, text };
368
+ }
369
+
370
+ function collectTransformersStats(
371
+ model: string,
372
+ runtime: RuntimeParameters,
373
+ telemetry: TransformersRunResult,
374
+ started: number,
375
+ ended: number,
376
+ ): RuntimeStats {
377
+ const elapsedMs = ended - started;
378
+ const completionTokens = telemetry.completionTokens;
379
+ const measured = typeof completionTokens === "number" && elapsedMs > 0 ? completionTokens / (elapsedMs / 1000) : undefined;
380
+ const decodeSeconds = typeof telemetry.firstTokenOffsetMs === "number" ? Math.max(0, (elapsedMs - telemetry.firstTokenOffsetMs) / 1000) : undefined;
381
+ const decode = typeof completionTokens === "number" && typeof decodeSeconds === "number" && decodeSeconds > 0 ? completionTokens / decodeSeconds : measured;
382
+ const prefill =
383
+ typeof telemetry.promptTokens === "number" && typeof telemetry.firstTokenOffsetMs === "number" && telemetry.firstTokenOffsetMs > 0
384
+ ? telemetry.promptTokens / (telemetry.firstTokenOffsetMs / 1000)
385
+ : undefined;
386
+ const tpot =
387
+ telemetry.tokenCallbackIntervalSeconds.length > 0
388
+ ? telemetry.tokenCallbackIntervalSeconds.reduce((sum, value) => sum + value, 0) / telemetry.tokenCallbackIntervalSeconds.length
389
+ : typeof completionTokens === "number" && completionTokens > 0
390
+ ? elapsedMs / 1000 / completionTokens
391
+ : undefined;
392
+ const totalTokens =
393
+ typeof telemetry.promptTokens === "number" && typeof completionTokens === "number" ? telemetry.promptTokens + completionTokens : undefined;
394
+ const finishReason = typeof completionTokens === "number" && completionTokens >= runtime.maxTokens ? "length (estimated)" : "stop/unknown";
395
+ const latencyBreakdown =
396
+ telemetry.tokenCallbackIntervalSeconds.length > 0 ? { transformersTokenIntervalTime: telemetry.tokenCallbackIntervalSeconds } : undefined;
397
+
398
+ const stats: RuntimeStats = {
399
+ rawText: "",
400
+ model,
401
+ backend: "transformers-js",
402
+ finishReason,
403
+ measuredElapsedMs: elapsedMs,
404
+ promptTokens: telemetry.promptTokens,
405
+ completionTokens,
406
+ totalTokens,
407
+ measuredCompletionTokensPerSecond: measured,
408
+ extra: {
409
+ e2e_latency_s: elapsedMs / 1000,
410
+ prefill_tokens_per_s: prefill,
411
+ decode_tokens_per_s: decode,
412
+ time_to_first_token_s: typeof telemetry.firstTokenOffsetMs === "number" ? telemetry.firstTokenOffsetMs / 1000 : undefined,
413
+ time_per_output_token_s: tpot,
414
+ latencyBreakdown,
415
+ token_count_source: telemetry.tokenCountSource,
416
+ },
417
+ latencyBreakdown,
418
+ };
419
+ stats.rawText = buildRuntimeStatsText(stats);
420
+ return stats;
421
+ }
422
+
423
+ export function collectGenericStats(model: string, backend: ModelBackend | "unknown", started: number, ended: number, error?: string): RuntimeStats {
424
+ const stats: RuntimeStats = {
425
+ rawText: "",
426
+ model,
427
+ backend,
428
+ finishReason: "error",
429
+ measuredElapsedMs: ended - started,
430
+ error,
431
+ };
432
+ stats.rawText = buildRuntimeStatsText(stats);
433
+ return stats;
434
+ }
435
+
436
+ /* ------------------------------------------------------------------ */
437
+ /* Shared helpers */
438
+ /* ------------------------------------------------------------------ */
439
+
440
+ export function buildRuntimeStatsText(stats: RuntimeStats): string {
441
+ const backendLabel = stats.backend === "transformers-js" ? "Transformers.js" : stats.backend === "webllm" ? "WebLLM" : "unknown";
442
+ const lines = [
443
+ `model: ${stats.model}`,
444
+ `backend: ${backendLabel}`,
445
+ `finish_reason: ${stats.finishReason}`,
446
+ `measured_elapsed_ms: ${stats.measuredElapsedMs.toFixed(0)}`,
447
+ "",
448
+ "Token usage:",
449
+ ` prompt_tokens: ${stats.promptTokens ?? "n/a"}`,
450
+ ` completion_tokens: ${stats.completionTokens ?? "n/a"}`,
451
+ ` total_tokens: ${stats.totalTokens ?? "n/a"}`,
452
+ ` measured_completion_tokens_per_second: ${stats.measuredCompletionTokensPerSecond?.toFixed(2) ?? "n/a"}`,
453
+ ];
454
+
455
+ if (stats.extra) {
456
+ lines.push(
457
+ "",
458
+ stats.backend === "transformers-js" ? "Transformers.js measured telemetry:" : "WebLLM usage.extra:",
459
+ ` e2e_latency_s: ${formatMaybe(stats.extra.e2e_latency_s, 3)}`,
460
+ ` prefill_tokens_per_s: ${formatMaybe(stats.extra.prefill_tokens_per_s, 2)}`,
461
+ ` decode_tokens_per_s: ${formatMaybe(stats.extra.decode_tokens_per_s, 2)}`,
462
+ ` time_to_first_token_s: ${formatMaybe(stats.extra.time_to_first_token_s, 3)}`,
463
+ ` time_per_output_token_s: ${formatMaybe(stats.extra.time_per_output_token_s, 4)}`,
464
+ );
465
+ if (typeof stats.extra.grammar_init_s === "number") lines.push(` grammar_init_s: ${stats.extra.grammar_init_s.toFixed(3)}`);
466
+ if (typeof stats.extra.grammar_per_token_s === "number") lines.push(` grammar_per_token_s: ${stats.extra.grammar_per_token_s.toFixed(6)}`);
467
+ if (stats.backend === "transformers-js" && typeof stats.extra.token_count_source === "string") {
468
+ lines.push(` token_count_source: ${stats.extra.token_count_source}`);
469
+ }
470
+ if (stats.latencyBreakdown && Object.keys(stats.latencyBreakdown).length > 0) {
471
+ lines.push("", "Latency breakdown:", JSON.stringify(stats.latencyBreakdown, null, 2));
472
+ }
473
+ } else {
474
+ lines.push("", `${backendLabel} backend telemetry: n/a`);
475
+ }
476
+
477
+ if (stats.error) lines.push("", `error: ${stats.error}`);
478
+ if (stats.legacyStats) lines.push("", "Legacy WebLLM runtimeStatsText:", stats.legacyStats);
479
+
480
+ return lines.join("\n");
481
+ }
482
+
483
+ export function normalizeLatencyBreakdown(value: unknown): LatencyBreakdown | undefined {
484
+ if (!value || typeof value !== "object") return undefined;
485
+ const result: LatencyBreakdown = {};
486
+ for (const [key, child] of Object.entries(value)) {
487
+ if (Array.isArray(child)) {
488
+ const numbers = child.filter((item): item is number => typeof item === "number" && Number.isFinite(item));
489
+ if (numbers.length > 0) result[key] = numbers;
490
+ }
491
+ }
492
+ return Object.keys(result).length > 0 ? result : undefined;
493
+ }
494
+
495
+ function extractGeneratedText(output: unknown): string {
496
+ if (typeof output === "string") return output;
497
+ if (Array.isArray(output)) return output.map(extractGeneratedText).filter(Boolean).join("\n");
498
+ if (!output || typeof output !== "object") return "";
499
+
500
+ const record = output as Record<string, unknown>;
501
+ const generated = record.generated_text;
502
+ if (typeof generated === "string") return generated;
503
+ if (Array.isArray(generated)) {
504
+ const assistant = generated
505
+ .filter((item): item is Record<string, unknown> => Boolean(item) && typeof item === "object")
506
+ .filter((item) => item.role === "assistant" && typeof item.content === "string");
507
+ const last = assistant.at(-1);
508
+ if (typeof last?.content === "string") return last.content;
509
+ return generated.map(extractGeneratedText).filter(Boolean).join("\n");
510
+ }
511
+ for (const key of ["text", "content", "answer"] as const) {
512
+ if (typeof record[key] === "string") return record[key] as string;
513
+ }
514
+ return JSON.stringify(output, null, 2);
515
+ }
516
+
517
+ function countPromptTokens(tokenizer: TransformersTokenizer | null, messages: ChatMessage[]): number | undefined {
518
+ if (!tokenizer) return undefined;
519
+ try {
520
+ if (typeof tokenizer.apply_chat_template === "function") {
521
+ const tokenized = tokenizer.apply_chat_template(messages, { tokenize: true, add_generation_prompt: true });
522
+ const count = extractTokenCount(tokenized);
523
+ if (typeof count === "number") return count;
524
+ }
525
+ } catch {
526
+ // Fall through to plain-text tokenization.
527
+ }
528
+ try {
529
+ const prompt = messages.map((message) => `${message.role}: ${message.content}`).join("\n\n") + "\n\nassistant:";
530
+ if (typeof tokenizer.encode === "function") {
531
+ const count = extractTokenCount(tokenizer.encode(prompt, { add_special_tokens: true }));
532
+ if (typeof count === "number") return count;
533
+ }
534
+ return extractTokenCount(tokenizer(prompt, { add_special_tokens: true }));
535
+ } catch {
536
+ return undefined;
537
+ }
538
+ }
539
+
540
+ function countCompletionTokens(tokenizer: TransformersTokenizer | null, text: string): number | undefined {
541
+ if (!tokenizer || !text || text === EMPTY_RESPONSE) return undefined;
542
+ try {
543
+ if (typeof tokenizer.encode === "function") return extractTokenCount(tokenizer.encode(text, { add_special_tokens: false }));
544
+ return extractTokenCount(tokenizer(text, { add_special_tokens: false }));
545
+ } catch {
546
+ return undefined;
547
+ }
548
+ }
549
+
550
+ function countTokenItems(value: unknown): number {
551
+ return extractTokenCount(value) ?? 1;
552
+ }
553
+
554
+ function extractTokenCount(value: unknown): number | undefined {
555
+ if (value == null) return undefined;
556
+ if (typeof value === "number" || typeof value === "bigint") return 1;
557
+ if (ArrayBuffer.isView(value)) return (value as ArrayBufferView & { length?: number }).length;
558
+ if (Array.isArray(value)) {
559
+ if (value.length === 0) return 0;
560
+ if (value.every((item) => typeof item === "number" || typeof item === "bigint")) return value.length;
561
+ const childCounts = value.map(extractTokenCount).filter((count): count is number => typeof count === "number");
562
+ return childCounts.length > 0 ? childCounts.reduce((sum, count) => sum + count, 0) : value.length;
563
+ }
564
+ if (typeof value !== "object") return undefined;
565
+ const record = value as Record<string, unknown>;
566
+ for (const key of ["input_ids", "data", "tokens", "ids"] as const) {
567
+ const count = extractTokenCount(record[key]);
568
+ if (typeof count === "number") return count;
569
+ }
570
+ const dims = record.dims;
571
+ if (Array.isArray(dims) && dims.every((dimension) => typeof dimension === "number")) {
572
+ return dims.reduce((product, dimension) => product * dimension, 1);
573
+ }
574
+ return undefined;
575
+ }
576
+
577
+ function formatMaybe(value: unknown, digits: number): string {
578
+ return typeof value === "number" && Number.isFinite(value) ? value.toFixed(digits) : "n/a";
579
+ }