@runtypelabs/persona 3.34.1 → 3.36.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. package/README.md +11 -11
  2. package/dist/animations/glyph-cycle.d.cts +1 -1
  3. package/dist/animations/glyph-cycle.d.ts +1 -1
  4. package/dist/animations/{types-CthJFfNx.d.cts → types-DgYsuwXL.d.cts} +15 -14
  5. package/dist/animations/{types-CthJFfNx.d.ts → types-DgYsuwXL.d.ts} +15 -14
  6. package/dist/animations/wipe.d.cts +1 -1
  7. package/dist/animations/wipe.d.ts +1 -1
  8. package/dist/codegen.cjs +11 -11
  9. package/dist/codegen.js +10 -10
  10. package/dist/index.cjs +46 -44
  11. package/dist/index.cjs.map +1 -1
  12. package/dist/index.d.cts +514 -158
  13. package/dist/index.d.ts +514 -158
  14. package/dist/index.global.js +41 -89
  15. package/dist/index.global.js.map +1 -1
  16. package/dist/index.js +46 -44
  17. package/dist/index.js.map +1 -1
  18. package/dist/install.global.js +1 -1
  19. package/dist/install.global.js.map +1 -1
  20. package/dist/launcher.global.js +2 -2
  21. package/dist/launcher.global.js.map +1 -1
  22. package/dist/markdown-parsers.js +53 -0
  23. package/dist/plugin-kit.d.cts +4 -4
  24. package/dist/plugin-kit.d.ts +4 -4
  25. package/dist/runtype-tts.js +1 -0
  26. package/dist/smart-dom-reader.cjs +1 -1
  27. package/dist/smart-dom-reader.d.cts +352 -103
  28. package/dist/smart-dom-reader.d.ts +352 -103
  29. package/dist/smart-dom-reader.js +1 -1
  30. package/dist/theme-editor-preview.cjs +203 -0
  31. package/dist/theme-editor-preview.d.cts +5748 -0
  32. package/dist/theme-editor-preview.d.ts +5748 -0
  33. package/dist/theme-editor-preview.js +203 -0
  34. package/dist/theme-editor.cjs +14 -131
  35. package/dist/theme-editor.d.cts +362 -574
  36. package/dist/theme-editor.d.ts +362 -574
  37. package/dist/theme-editor.js +14 -131
  38. package/dist/theme-reference.cjs +1 -1
  39. package/dist/theme-reference.d.cts +1 -1
  40. package/dist/theme-reference.d.ts +1 -1
  41. package/dist/theme-reference.js +1 -1
  42. package/dist/voice-worklet-player.cjs +80 -0
  43. package/dist/voice-worklet-player.d.cts +215 -0
  44. package/dist/voice-worklet-player.d.ts +215 -0
  45. package/dist/voice-worklet-player.js +80 -0
  46. package/dist/widget.css +54 -36
  47. package/package.json +16 -2
  48. package/src/animations/glyph-cycle.ts +11 -14
  49. package/src/animations/wipe.ts +1 -1
  50. package/src/ask-user-question-tool.test.ts +1 -1
  51. package/src/ask-user-question-tool.ts +6 -8
  52. package/src/client.test.ts +56 -12
  53. package/src/client.ts +35 -31
  54. package/src/codegen.test.ts +1 -1
  55. package/src/codegen.ts +1 -1
  56. package/src/components/approval-bubble.test.ts +1 -1
  57. package/src/components/approval-bubble.ts +3 -3
  58. package/src/components/artifact-pane.ts +3 -3
  59. package/src/components/ask-user-question-bubble.test.ts +14 -14
  60. package/src/components/ask-user-question-bubble.ts +10 -10
  61. package/src/components/composer-builder.ts +17 -16
  62. package/src/components/composer-parts.test.ts +54 -0
  63. package/src/components/composer-parts.ts +95 -126
  64. package/src/components/demo-carousel.ts +1 -1
  65. package/src/components/event-stream-view.test.ts +7 -7
  66. package/src/components/event-stream-view.ts +5 -5
  67. package/src/components/header-builder.ts +33 -27
  68. package/src/components/header-layouts.ts +1 -1
  69. package/src/components/header-parts.test.ts +90 -0
  70. package/src/components/header-parts.ts +62 -93
  71. package/src/components/launcher.test.ts +11 -0
  72. package/src/components/launcher.ts +1 -1
  73. package/src/components/message-bubble.test.ts +74 -2
  74. package/src/components/message-bubble.ts +25 -11
  75. package/src/components/messages.ts +2 -2
  76. package/src/components/panel.test.ts +1 -1
  77. package/src/components/panel.ts +92 -97
  78. package/src/components/pill-composer-builder.ts +28 -24
  79. package/src/components/reasoning-bubble.ts +2 -2
  80. package/src/components/suggestions.ts +1 -1
  81. package/src/components/tool-bubble.ts +1 -1
  82. package/src/components/widget-view.test.ts +181 -0
  83. package/src/components/widget-view.ts +234 -0
  84. package/src/defaults.ts +54 -51
  85. package/src/generated/runtype-openapi-contract.ts +5 -0
  86. package/src/index-core.ts +30 -4
  87. package/src/index-global.ts +73 -6
  88. package/src/index.ts +7 -1
  89. package/src/install.test.ts +11 -11
  90. package/src/install.ts +28 -18
  91. package/src/launcher-global.ts +25 -14
  92. package/src/markdown-parsers-eager.ts +20 -0
  93. package/src/markdown-parsers-entry.ts +4 -0
  94. package/src/markdown-parsers-loader.ts +49 -0
  95. package/src/plugin-kit.test.ts +1 -1
  96. package/src/plugin-kit.ts +7 -7
  97. package/src/plugins/types.ts +5 -5
  98. package/src/postprocessors.ts +27 -14
  99. package/src/runtime/host-layout.test.ts +1 -1
  100. package/src/runtime/host-layout.ts +4 -5
  101. package/src/runtime/init.ts +2 -2
  102. package/src/session.test.ts +5 -6
  103. package/src/session.ts +333 -227
  104. package/src/session.voice.test.ts +146 -0
  105. package/src/session.webmcp.test.ts +24 -25
  106. package/src/smart-dom-reader.ts +3 -4
  107. package/src/styles/widget.css +54 -36
  108. package/src/suggest-replies-tool.test.ts +3 -3
  109. package/src/suggest-replies-tool.ts +5 -5
  110. package/src/testing/index.ts +1 -1
  111. package/src/theme-editor/color-utils.ts +2 -2
  112. package/src/theme-editor/index.ts +4 -13
  113. package/src/theme-editor/presets.ts +1 -1
  114. package/src/theme-editor/preview-utils.ts +22 -20
  115. package/src/theme-editor/preview.ts +4 -1
  116. package/src/theme-editor/role-mappings.ts +3 -3
  117. package/src/theme-editor/sections.ts +21 -20
  118. package/src/theme-editor/types.ts +1 -1
  119. package/src/theme-editor/webmcp/index.ts +1 -1
  120. package/src/theme-editor/webmcp/summary.ts +1 -2
  121. package/src/theme-editor/webmcp/tools.ts +30 -30
  122. package/src/theme-editor/webmcp/types.ts +2 -2
  123. package/src/theme-editor-preview.ts +10 -0
  124. package/src/theme-reference.ts +26 -26
  125. package/src/types.ts +368 -99
  126. package/src/ui.approval-plugin.test.ts +3 -3
  127. package/src/ui.ask-user-question-plugin.test.ts +9 -9
  128. package/src/ui.component-directive.test.ts +4 -4
  129. package/src/ui.composer-bar.test.ts +10 -11
  130. package/src/ui.composer-keyboard.test.ts +5 -5
  131. package/src/ui.header-icon-color.test.ts +59 -0
  132. package/src/ui.scroll.test.ts +5 -6
  133. package/src/ui.stop-button.test.ts +3 -4
  134. package/src/ui.suggest-replies.test.ts +2 -2
  135. package/src/ui.ts +292 -131
  136. package/src/utils/buttons.ts +1 -1
  137. package/src/utils/code-generators.ts +1 -1
  138. package/src/utils/composer-history.ts +3 -3
  139. package/src/utils/dom-context.test.ts +2 -2
  140. package/src/utils/dom-context.ts +7 -7
  141. package/src/utils/dom.test.ts +67 -0
  142. package/src/utils/dom.ts +75 -0
  143. package/src/utils/dropdown.ts +2 -2
  144. package/src/utils/event-stream-buffer.test.ts +2 -2
  145. package/src/utils/message-fingerprint.test.ts +9 -0
  146. package/src/utils/message-fingerprint.ts +5 -0
  147. package/src/utils/morph.ts +1 -1
  148. package/src/utils/sanitize.ts +32 -16
  149. package/src/utils/sequence-buffer.test.ts +8 -8
  150. package/src/utils/sequence-buffer.ts +4 -4
  151. package/src/utils/smart-dom-adapter.test.ts +1 -1
  152. package/src/utils/smart-dom-adapter.ts +4 -5
  153. package/src/utils/speech-text.test.ts +122 -0
  154. package/src/utils/speech-text.ts +101 -0
  155. package/src/utils/stream-animation.test.ts +3 -3
  156. package/src/utils/stream-animation.ts +6 -6
  157. package/src/utils/theme.test.ts +1 -1
  158. package/src/utils/throughput-tracker.test.ts +11 -11
  159. package/src/utils/throughput-tracker.ts +9 -9
  160. package/src/utils/tokens.ts +13 -13
  161. package/src/utils/virtual-scroller.ts +1 -1
  162. package/src/vendor/smart-dom-reader/README.md +4 -4
  163. package/src/vendor/smart-dom-reader/index.d.ts +1 -1
  164. package/src/vendor/smart-dom-reader/index.js +2 -2
  165. package/src/version.ts +1 -1
  166. package/src/voice/audio-playback-manager.test.ts +159 -0
  167. package/src/voice/audio-playback-manager.ts +145 -30
  168. package/src/voice/browser-speech-engine.ts +130 -0
  169. package/src/voice/fallback-speech-engine.ts +89 -0
  170. package/src/voice/index.ts +21 -0
  171. package/src/voice/read-aloud-controller.test.ts +143 -0
  172. package/src/voice/read-aloud-controller.ts +136 -0
  173. package/src/voice/runtype-speech-engine.test.ts +280 -0
  174. package/src/voice/runtype-speech-engine.ts +211 -0
  175. package/src/voice/runtype-tts-entry.ts +10 -0
  176. package/src/voice/runtype-tts-loader.test.ts +47 -0
  177. package/src/voice/runtype-tts-loader.ts +42 -0
  178. package/src/voice/runtype-voice-provider.ts +332 -537
  179. package/src/voice/voice-factory.ts +26 -4
  180. package/src/voice/voice.test.ts +321 -7
  181. package/src/voice/worklet-playback-engine.ts +272 -0
  182. package/src/voice-worklet-player.ts +39 -0
  183. package/src/webmcp-bridge.test.ts +6 -6
  184. package/src/webmcp-bridge.ts +24 -25
  185. package/src/webmcp-polyfill.ts +1 -1
  186. package/src/voice/voice-activity-detector.ts +0 -90
package/src/session.ts CHANGED
@@ -31,12 +31,19 @@ import type {
31
31
  VoiceProvider,
32
32
  VoiceStatus,
33
33
  VoiceConfig,
34
- TextToSpeechConfig
34
+ ReadAloudState,
35
+ SpeechEngine
35
36
  } from "./types";
36
37
  import {
37
38
  createVoiceProvider,
38
- isVoiceSupported
39
+ isVoiceSupported,
40
+ BrowserSpeechEngine,
41
+ pickBestVoice,
42
+ ReadAloudController,
43
+ type ReadAloudListener
39
44
  } from "./voice";
45
+ import { resolveSpeakableText } from "./utils/speech-text";
46
+ import { loadRuntypeTts } from "./voice/runtype-tts-loader";
40
47
 
41
48
  export type AgentWidgetSessionStatus =
42
49
  | "idle"
@@ -51,7 +58,7 @@ export type AgentWidgetSessionStatus =
51
58
  * instead of swapping in a fresh client and tearing down WebMCP state. Fields
52
59
  * outside this list (theme, copy, layout, suggestionChips, iterationDisplay,
53
60
  * postprocessMessage, feature display toggles, …) are display-only and safe to
54
- * apply mid-turn which is what a self-styling widget needs when a `webmcp:*`
61
+ * apply mid-turn: which is what a self-styling widget needs when a `webmcp:*`
55
62
  * theme tool re-renders the widget while the agent's turn is still streaming.
56
63
  *
57
64
  * Compared by identity (`!==`): primitives by value, functions/objects by
@@ -102,8 +109,8 @@ type SessionCallbacks = {
102
109
  /**
103
110
  * Build the user-facing content shown when a dispatch fails before any
104
111
  * assistant content streamed back. This fires on real network/server errors
105
- * (connection refused, CORS, 4xx/5xx, malformed stream) not just an
106
- * un-wired proxy so the copy stays honest about the failure and surfaces the
112
+ * (connection refused, CORS, 4xx/5xx, malformed stream): not just an
113
+ * un-wired proxy, so the copy stays honest about the failure and surfaces the
107
114
  * underlying reason to help with debugging.
108
115
  *
109
116
  * Callers can override the copy via `config.errorMessage` (a static string or
@@ -120,7 +127,7 @@ function buildDispatchErrorContent(
120
127
  if (typeof override === "function") return override(err);
121
128
 
122
129
  const base =
123
- "Sorry I couldn't reach the assistant. The chat service didn't respond. Please check that your proxy or backend is running and reachable, then try again.";
130
+ "Sorry: I couldn't reach the assistant. The chat service didn't respond. Please check that your proxy or backend is running and reachable, then try again.";
124
131
  return err.message ? `${base}\n\n_Details: ${err.message}_` : base;
125
132
  }
126
133
 
@@ -142,7 +149,7 @@ const getWebMcpErrorMessage = (
142
149
  * Tool names whose `step_await` the widget resolves automatically (no user
143
150
  * pill click): `webmcp:*` page tools and the built-in fire-and-forget
144
151
  * `suggest_replies`. These share the await-batch / dedupe / resume machinery;
145
- * `ask_user_question` is NOT one of them it blocks on the answer sheet.
152
+ * `ask_user_question` is NOT one of them: it blocks on the answer sheet.
146
153
  */
147
154
  const isAutoResolvedLocalToolName = (name: string): boolean =>
148
155
  isWebMcpToolName(name) || name === SUGGEST_REPLIES_TOOL_NAME;
@@ -164,16 +171,16 @@ export class AgentWidgetSession {
164
171
  private artifacts = new Map<string, PersonaArtifactRecord>();
165
172
  private selectedArtifactId: string | null = null;
166
173
 
167
- // WebMCP dedupe keys are `${executionId}:${toolCallId}` so they're
174
+ // WebMCP dedupe: keys are `${executionId}:${toolCallId}` so they're
168
175
  // naturally scoped to a single dispatch. A later dispatch (new executionId)
169
176
  // that happens to recycle a `toolCall.id` never collides with prior entries,
170
177
  // and a stale re-emit from an in-flight prior dispatch stays blocked because
171
178
  // its executionId is still in the set.
172
179
  //
173
- // webMcpInflightKeys currently executing; cleared on completion of
180
+ // webMcpInflightKeys: currently executing; cleared on completion of
174
181
  // EITHER /resume success OR /resume throw. Blocks
175
182
  // concurrent re-fire during the resolve round-trip.
176
- // webMcpResolvedKeys /resume HTTP returned 2xx; not cleared on a new
183
+ // webMcpResolvedKeys: /resume HTTP returned 2xx; not cleared on a new
177
184
  // dispatch (executionId scoping makes that
178
185
  // unnecessary). Blocks stale step_await re-emits
179
186
  // for the same execution.
@@ -188,7 +195,7 @@ export class AgentWidgetSession {
188
195
  // `this.abortController` is intentionally NOT used by resolveWebMcpToolCall:
189
196
  // in a CHAINED turn (tool A → /resume → tool B, where the server emits B's
190
197
  // step_await inside A's resume SSE stream) the shared controller is still
191
- // piping A's resume stream the very stream that just delivered B. Aborting
198
+ // piping A's resume stream: the very stream that just delivered B. Aborting
192
199
  // it mid-chain (the prior shared-controller pre-abort) tore that stream down,
193
200
  // so B never reached execute() and its /resume was never POSTed, pausing the
194
201
  // dispatch forever. cancel(), clearMessages(), hydrateMessages(), and
@@ -211,11 +218,11 @@ export class AgentWidgetSession {
211
218
  private webMcpApprovalSeq = 0;
212
219
  // Parallel local-tool batching (core#3878). A single model turn can emit
213
220
  // multiple `step_await(local_tool_required)` events for ONE paused
214
- // executionId including two PARALLEL calls to the SAME tool ("add SHOE-001
221
+ // executionId: including two PARALLEL calls to the SAME tool ("add SHOE-001
215
222
  // and SHOE-007"). Those collapse to an identical `toolId`/`index` and differ
216
223
  // only by the per-call `webMcpToolCallId`. We collect all awaits for an
217
224
  // executionId that arrive in the same stream tick, then post ONE `/resume`
218
- // keyed by `webMcpToolCallId` NOT one `/resume` per tool keyed by name
225
+ // keyed by `webMcpToolCallId`: NOT one `/resume` per tool keyed by name
219
226
  // (which collides for same-tool calls, and whose concurrent posts on one
220
227
  // execution raced → the second 404'd → the turn hung). Keyed by executionId;
221
228
  // `seen` dedupes duplicate step_await re-emits within a batch. Cleared on
@@ -243,7 +250,7 @@ export class AgentWidgetSession {
243
250
  this.wireDefaultWebMcpConfirm();
244
251
 
245
252
  // Hydrate artifacts from config (mirrors `initialMessages`). Restored
246
- // records are forced to `status: "complete"` a mid-stream artifact should
253
+ // records are forced to `status: "complete"`: a mid-stream artifact should
247
254
  // never reappear after a refresh with its skeleton still showing.
248
255
  for (const rec of config.initialArtifacts ?? []) {
249
256
  this.artifacts.set(rec.id, { ...rec, status: "complete" });
@@ -259,6 +266,29 @@ export class AgentWidgetSession {
259
266
  this.emitArtifactsState();
260
267
  }
261
268
  this.callbacks.onStatusChanged(this.status);
269
+ this.prefetchRuntypeTts();
270
+ }
271
+
272
+ /**
273
+ * Warm the deferred Runtype TTS engine chunk at init so it's resolved before
274
+ * the first "Read aloud" click. Module fetch only — no AudioContext (that
275
+ * still waits for the user gesture on first playback). In the IIFE/CDN build
276
+ * this fetches the standalone `runtype-tts.js`; in bundler builds the module
277
+ * is already inlined, so this is a cheap no-op. Errors are swallowed: a failed
278
+ * prefetch just means the chunk loads on first click instead (or, on a hard
279
+ * failure, the browser-voice fallback kicks in).
280
+ */
281
+ private prefetchRuntypeTts(): void {
282
+ const tts = this.config.textToSpeech;
283
+ if (tts?.provider !== "runtype" || tts.createEngine) return;
284
+ // Only warm the chunk when the engine will actually be built (same gate as
285
+ // createSpeechEngine) — an incomplete config (e.g. a proxy demo with no
286
+ // clientToken) falls back to the browser voice, so the fetch would be wasted.
287
+ const host = tts.host ?? this.config.apiUrl;
288
+ const agentId =
289
+ tts.agentId ?? this.config.voiceRecognition?.provider?.runtype?.agentId;
290
+ if (!host || !agentId || !this.config.clientToken) return;
291
+ void loadRuntypeTts().catch(() => {});
262
292
  }
263
293
 
264
294
  /**
@@ -342,7 +372,7 @@ export class AgentWidgetSession {
342
372
  return this.voiceProvider?.isBargeInActive?.() ?? false;
343
373
  }
344
374
 
345
- /** Tear down the barge-in mic pipeline "hang up" the always-on mic */
375
+ /** Tear down the barge-in mic pipeline: "hang up" the always-on mic */
346
376
  public async deactivateBargeIn(): Promise<void> {
347
377
  if (this.voiceProvider?.deactivateBargeIn) {
348
378
  await this.voiceProvider.deactivateBargeIn();
@@ -357,6 +387,84 @@ export class AgentWidgetSession {
357
387
  // so browser TTS doesn't double-speak them
358
388
  private ttsSpokenMessageIds = new Set<string>();
359
389
 
390
+ // Owns the per-message "Read aloud" action and the auto-speak path: which
391
+ // message is active, its play/pause state, and the speech engine. The engine
392
+ // is resolved lazily on first playback (inside the user gesture) so a hosted
393
+ // engine via `textToSpeech.createEngine` plugs in without changing this class.
394
+ private readAloud = new ReadAloudController(() => this.createSpeechEngine());
395
+
396
+ /**
397
+ * Resolve the speech engine behind read-aloud (and auto-speak):
398
+ * 1. An explicit `textToSpeech.createEngine` always wins (full BYO control).
399
+ * 2. `provider: 'runtype'` builds the hosted {@link RuntypeSpeechEngine}
400
+ * from the widget config (host = `apiUrl`, `agentId`, `clientToken`),
401
+ * wrapped in a browser {@link FallbackSpeechEngine} unless
402
+ * `browserFallback: false` — so a missing endpoint or transient failure
403
+ * degrades to the browser voice instead of erroring, and auto-upgrades to
404
+ * Runtype voices the day the endpoint ships.
405
+ * 3. Otherwise the browser Web Speech API engine.
406
+ */
407
+ private createSpeechEngine(): SpeechEngine | Promise<SpeechEngine> | null {
408
+ const tts = this.config.textToSpeech;
409
+ if (tts?.createEngine) return tts.createEngine();
410
+
411
+ const browser = BrowserSpeechEngine.isSupported()
412
+ ? new BrowserSpeechEngine({ pickVoice: tts?.pickVoice })
413
+ : null;
414
+
415
+ if (tts?.provider === "runtype") {
416
+ const host = tts.host ?? this.config.apiUrl;
417
+ const agentId =
418
+ tts.agentId ?? this.config.voiceRecognition?.provider?.runtype?.agentId;
419
+ const clientToken = this.config.clientToken;
420
+ const wantFallback = tts.browserFallback !== false;
421
+
422
+ if (host && agentId && clientToken) {
423
+ // Lazy-load the Runtype engine through the deferred loader: the IIFE/CDN
424
+ // build serves it from the standalone `runtype-tts.js` chunk (kept out of
425
+ // `index.global.js`), while bundler consumers get it inlined. Resolved on
426
+ // first playback (inside the user gesture); `createSpeechEngine` may
427
+ // return a Promise. Prefetched at init (see `prefetchRuntypeTts`) so the
428
+ // chunk is warm before the first click.
429
+ return loadRuntypeTts().then(
430
+ ({ RuntypeSpeechEngine, FallbackSpeechEngine }) => {
431
+ const runtype = new RuntypeSpeechEngine({
432
+ host,
433
+ agentId,
434
+ clientToken,
435
+ voice: tts.voice,
436
+ prebufferMs: tts.prebufferMs,
437
+ createPlaybackEngine: tts.createPlaybackEngine,
438
+ });
439
+ return wantFallback && browser
440
+ ? new FallbackSpeechEngine(runtype, browser, {
441
+ onFallback: (error) =>
442
+ console.warn(
443
+ `[persona] Runtype read-aloud failed; using browser voice. ${error.message}`,
444
+ ),
445
+ })
446
+ : runtype;
447
+ },
448
+ );
449
+ }
450
+
451
+ // Runtype was requested but the config is incomplete (no agentId/token —
452
+ // e.g. a proxy-backed demo). Never break the button: use the browser
453
+ // voice. Only warn when intent is clear (a clientToken is present) so a
454
+ // plain proxy demo stays quiet.
455
+ if (wantFallback && browser) {
456
+ if (clientToken) {
457
+ console.warn(
458
+ "[persona] textToSpeech.provider 'runtype' is missing an agentId; using the browser voice. Set textToSpeech.agentId (or voiceRecognition.provider.runtype.agentId).",
459
+ );
460
+ }
461
+ return browser;
462
+ }
463
+ }
464
+
465
+ return browser;
466
+ }
467
+
360
468
  /**
361
469
  * Setup voice recognition with the given configuration
362
470
  */
@@ -371,92 +479,104 @@ export class AgentWidgetSession {
371
479
 
372
480
  // Read configurable text from widget config
373
481
  const voiceRecognitionConfig = this.config.voiceRecognition ?? {};
374
- const processingText = voiceRecognitionConfig.processingText ?? '\u{1F3A4} Processing voice...';
375
482
  const processingErrorText = voiceRecognitionConfig.processingErrorText ?? 'Voice processing failed. Please try again.';
376
483
 
377
- // Phase A: When recording stops and audio is about to be sent,
378
- // inject placeholder messages and show typing indicator immediately.
379
- // Placeholders are tagged with voiceProcessing=true so consumers can
380
- // detect them in messageTransform and render custom UI.
381
- if (this.voiceProvider.onProcessingStart) {
382
- this.voiceProvider.onProcessingStart(() => {
383
- // Inject user message placeholder
384
- const userMsg = this.injectMessage({
385
- role: 'user',
386
- content: processingText,
387
- streaming: false,
388
- voiceProcessing: true
389
- });
390
- this.pendingVoiceUserMessageId = userMsg.id;
391
-
392
- // Inject empty assistant message with streaming=true for typing indicator
393
- const assistantMsg = this.injectMessage({
394
- role: 'assistant',
395
- content: '',
396
- streaming: true,
397
- voiceProcessing: true
398
- });
399
- this.pendingVoiceAssistantMessageId = assistantMsg.id;
400
-
401
- // Trigger typing indicator in the UI
402
- this.setStreaming(true);
403
- });
404
- }
405
-
406
- // Phase B: When server responds with transcript + agent response,
407
- // upsert the placeholder messages with actual content and clear voiceProcessing flag
484
+ // STT-style providers (browser + bring-your-own `custom`) deliver a final
485
+ // transcript that we send as a normal user message: the agent then runs
486
+ // via the standard SSE chat path. Only the realtime `runtype` provider is
487
+ // excluded here: it owns the whole turn and drives onTranscript below.
408
488
  this.voiceProvider.onResult((result) => {
409
- if (result.provider === 'browser') {
410
- // Browser STT: send transcript as a user message (agent runs via normal chat)
489
+ if (result.provider !== 'runtype') {
411
490
  if (result.text && result.text.trim()) {
412
491
  this.sendMessage(result.text, { viaVoice: true });
413
492
  }
414
- } else if (result.provider === 'runtype') {
415
- // Runtype provider: agent already executed server-side, audio playback
416
- // is handled by the provider itself. Update placeholders with actual content.
417
- if (this.pendingVoiceUserMessageId && result.transcript?.trim()) {
418
- this.upsertMessage({
419
- id: this.pendingVoiceUserMessageId,
420
- role: 'user',
421
- content: result.transcript.trim(),
422
- createdAt: new Date().toISOString(),
423
- streaming: false,
424
- voiceProcessing: false
425
- });
426
- } else if (result.transcript?.trim()) {
427
- this.injectUserMessage({ content: result.transcript.trim() });
428
- }
493
+ }
494
+ });
429
495
 
430
- if (this.pendingVoiceAssistantMessageId && result.text?.trim()) {
431
- this.upsertMessage({
432
- id: this.pendingVoiceAssistantMessageId,
433
- role: 'assistant',
434
- content: result.text.trim(),
435
- createdAt: new Date().toISOString(),
436
- streaming: false,
437
- voiceProcessing: false
438
- });
439
- } else if (result.text?.trim()) {
440
- this.injectAssistantMessage({ content: result.text.trim() });
441
- }
496
+ // Realtime (runtype) voice: drive the chat thread from streaming
497
+ // transcript frames. Live interim user text grows in place; the user
498
+ // message finalizes immediately on transcript_final{user}; the assistant
499
+ // reply lands (a single block, synced with audio) on its final frame.
500
+ // In-flight bubbles carry voiceProcessing=true so consumers can style
501
+ // them via messageTransform; it clears once the text is final.
502
+ if (this.voiceProvider.onTranscript) {
503
+ this.voiceProvider.onTranscript((role, text, isFinal) => {
504
+ if (role === 'user') {
505
+ if (!this.pendingVoiceUserMessageId) {
506
+ const msg = this.injectMessage({
507
+ role: 'user',
508
+ content: text,
509
+ streaming: false,
510
+ voiceProcessing: !isFinal
511
+ });
512
+ this.pendingVoiceUserMessageId = msg.id;
513
+ } else {
514
+ this.upsertMessage({
515
+ id: this.pendingVoiceUserMessageId,
516
+ role: 'user',
517
+ content: text,
518
+ createdAt: new Date().toISOString(),
519
+ streaming: false,
520
+ voiceProcessing: !isFinal
521
+ });
522
+ }
442
523
 
443
- // Mark assistant message as already spoken so browser TTS doesn't
444
- // double-speak. This covers both paths:
445
- // - Batch: audio.base64 is present in the voice_response
446
- // - Streaming: audio arrives as binary PCM chunks (no base64 here)
447
- // In either case, the Runtype provider handles TTS — browser TTS must skip.
448
- {
449
- const spokenId = this.pendingVoiceAssistantMessageId
450
- ?? [...this.messages].reverse().find(m => m.role === 'assistant')?.id;
451
- if (spokenId) this.ttsSpokenMessageIds.add(spokenId);
524
+ if (isFinal) {
525
+ // User finished: the agent is now thinking. Release the user
526
+ // bubble (a new interim starts a fresh turn) and show a typing
527
+ // indicator in a fresh assistant placeholder.
528
+ this.pendingVoiceUserMessageId = null;
529
+ const assistantMsg = this.injectMessage({
530
+ role: 'assistant',
531
+ content: '',
532
+ streaming: true,
533
+ voiceProcessing: true
534
+ });
535
+ this.pendingVoiceAssistantMessageId = assistantMsg.id;
536
+ this.setStreaming(true);
537
+ }
538
+ } else {
539
+ // assistant: runtype sends a single final; the isFinal=false path
540
+ // is reserved for delta-streaming providers (future BYO).
541
+ if (this.pendingVoiceAssistantMessageId) {
542
+ this.upsertMessage({
543
+ id: this.pendingVoiceAssistantMessageId,
544
+ role: 'assistant',
545
+ content: text,
546
+ createdAt: new Date().toISOString(),
547
+ streaming: !isFinal,
548
+ voiceProcessing: !isFinal
549
+ });
550
+ } else {
551
+ const msg = this.injectMessage({
552
+ role: 'assistant',
553
+ content: text,
554
+ streaming: !isFinal,
555
+ voiceProcessing: !isFinal
556
+ });
557
+ this.pendingVoiceAssistantMessageId = msg.id;
558
+ }
559
+
560
+ if (isFinal) {
561
+ // The provider plays this reply's audio: mark it spoken so
562
+ // browser TTS doesn't double-speak when streaming ends. Must run
563
+ // BEFORE setStreaming(false), which triggers the TTS check.
564
+ if (this.pendingVoiceAssistantMessageId) {
565
+ this.ttsSpokenMessageIds.add(this.pendingVoiceAssistantMessageId);
566
+ }
567
+ this.setStreaming(false);
568
+ this.pendingVoiceAssistantMessageId = null;
569
+ }
452
570
  }
571
+ });
572
+ }
453
573
 
454
- // Clear streaming state and pending IDs
455
- this.setStreaming(false);
456
- this.pendingVoiceUserMessageId = null;
457
- this.pendingVoiceAssistantMessageId = null;
458
- }
459
- });
574
+ // Surface per-turn latency metrics to the optional config hook.
575
+ if (this.voiceProvider.onMetrics) {
576
+ this.voiceProvider.onMetrics((metrics) => {
577
+ this.config.voiceRecognition?.onMetrics?.(metrics);
578
+ });
579
+ }
460
580
 
461
581
  this.voiceProvider.onError((error) => {
462
582
  console.error('Voice error:', error);
@@ -540,11 +660,12 @@ export class AgentWidgetSession {
540
660
  type: 'runtype',
541
661
  runtype: {
542
662
  agentId: providerConfig.runtype?.agentId || '',
543
- clientToken: providerConfig.runtype?.clientToken || '',
544
- host: providerConfig.runtype?.host,
663
+ // Default credentials/endpoint from the widget config so the minimum
664
+ // voice config collapses to just `agentId`.
665
+ clientToken: providerConfig.runtype?.clientToken ?? this.config.clientToken,
666
+ host: providerConfig.runtype?.host ?? this.config.apiUrl,
545
667
  voiceId: providerConfig.runtype?.voiceId,
546
- pauseDuration: providerConfig.runtype?.pauseDuration,
547
- silenceThreshold: providerConfig.runtype?.silenceThreshold
668
+ createPlaybackEngine: providerConfig.runtype?.createPlaybackEngine
548
669
  }
549
670
  };
550
671
 
@@ -556,7 +677,15 @@ export class AgentWidgetSession {
556
677
  continuous: providerConfig.browser?.continuous
557
678
  }
558
679
  };
559
-
680
+
681
+ case 'custom':
682
+ // Bring-your-own provider: pass the instance/factory straight through
683
+ // to the factory, which resolves and validates it.
684
+ return {
685
+ type: 'custom',
686
+ custom: providerConfig.custom
687
+ };
688
+
560
689
  default:
561
690
  return undefined;
562
691
  }
@@ -690,7 +819,7 @@ export class AgentWidgetSession {
690
819
  // parallel-await batch, and pending approval bubble tied to the OLD client/
691
820
  // session. Tear them down BEFORE the swap (the new client has no session
692
821
  // yet) so a deferred batch flush or a parked confirm can't fire against the
693
- // fresh client in client-token mode that would POST /resume without a
822
+ // fresh client: in client-token mode that would POST /resume without a
694
823
  // valid sessionId and strand the paused turn. Mirrors clearMessages' WebMCP
695
824
  // reset; the client swap already abandons any in-flight stream regardless.
696
825
  this.abortWebMcpResolves();
@@ -900,7 +1029,7 @@ export class AgentWidgetSession {
900
1029
 
901
1030
  /**
902
1031
  * Convenience method for injecting a registered component directive as
903
- * an assistant message the same shape Persona produces from a streamed
1032
+ * an assistant message: the same shape Persona produces from a streamed
904
1033
  * `{ "text": "...", "component": "...", "props": {...} }` payload.
905
1034
  *
906
1035
  * Sets `content` to `text`, `rawContent` to the JSON directive (so
@@ -1193,7 +1322,7 @@ export class AgentWidgetSession {
1193
1322
  * awaits this Promise before executing the page tool.
1194
1323
  */
1195
1324
  public requestWebMcpApproval(info: WebMcpConfirmInfo): Promise<boolean> {
1196
- // Per-tool policy hook auto-allow opted-out tools without any UI. A
1325
+ // Per-tool policy hook: auto-allow opted-out tools without any UI. A
1197
1326
  // throwing predicate must not block the call, so fall through to an
1198
1327
  // explicit gate on error.
1199
1328
  try {
@@ -1280,7 +1409,7 @@ export class AgentWidgetSession {
1280
1409
  };
1281
1410
  // Anchor the bubble where the agent paused for permission. An approval is a
1282
1411
  // timeline checkpoint, not a "now" event, so resolving it must preserve the
1283
- // original message's createdAt/sequence otherwise sortMessages (which
1412
+ // original message's createdAt/sequence: otherwise sortMessages (which
1284
1413
  // orders by createdAt first) would re-stamp it to now and float it past any
1285
1414
  // message created later (e.g. a long-pending approval resolved after more
1286
1415
  // conversation, or restored/replayed transcripts).
@@ -1352,7 +1481,7 @@ export class AgentWidgetSession {
1352
1481
  await this.connectStream(stream, { allowReentry: true });
1353
1482
  } else {
1354
1483
  if (decision === 'denied') {
1355
- // No stream body for denied inject a denial message
1484
+ // No stream body for denied: inject a denial message
1356
1485
  this.appendMessage({
1357
1486
  id: `denial-${approval.id}`,
1358
1487
  role: "assistant",
@@ -1362,13 +1491,13 @@ export class AgentWidgetSession {
1362
1491
  sequence: this.nextSequence(),
1363
1492
  });
1364
1493
  }
1365
- // No body to pipe drop the pre-set streaming flag so the indicator
1494
+ // No body to pipe: drop the pre-set streaming flag so the indicator
1366
1495
  // doesn't linger forever.
1367
1496
  this.setStreaming(false);
1368
1497
  this.abortController = null;
1369
1498
  }
1370
1499
  } else {
1371
- // onDecision returned void / no response drop the pre-set flag.
1500
+ // onDecision returned void / no response: drop the pre-set flag.
1372
1501
  this.setStreaming(false);
1373
1502
  this.abortController = null;
1374
1503
  }
@@ -1430,7 +1559,7 @@ export class AgentWidgetSession {
1430
1559
  * Flip an `ask_user_question` tool message from awaiting → answered so
1431
1560
  * render passes stop re-mounting its answer-pill sheet. Idempotent.
1432
1561
  * When `answers` is provided, persists the full structured answer Record
1433
- * atomically with the answered flag guarding against later events that
1562
+ * atomically with the answered flag: guarding against later events that
1434
1563
  * could re-emit the tool message and clobber the per-pick persisted
1435
1564
  * answers via top-level merge.
1436
1565
  */
@@ -1455,7 +1584,7 @@ export class AgentWidgetSession {
1455
1584
  toolMessage: AgentWidgetMessage,
1456
1585
  answer: string | Record<string, string | string[]>
1457
1586
  ): Promise<void> {
1458
- // Idempotent guards against rapid double-clicks on answer pills before
1587
+ // Idempotent: guards against rapid double-clicks on answer pills before
1459
1588
  // the re-render swaps the card to its collapsed/answered state.
1460
1589
  const live = this.messages.find((m) => m.id === toolMessage.id);
1461
1590
  if (live?.agentMetadata?.askUserQuestionAnswered === true) return;
@@ -1475,7 +1604,7 @@ export class AgentWidgetSession {
1475
1604
  // avoiding the race between removeAskUserQuestionSheet's 180ms slide-out
1476
1605
  // timer and the renders that fire as the resume stream lands. Pass the
1477
1606
  // structured answer Record (when present) so it's atomically persisted
1478
- // alongside the flag the answered-state review card depends on
1607
+ // alongside the flag: the answered-state review card depends on
1479
1608
  // `agentMetadata.askUserQuestionAnswers` being populated at render time.
1480
1609
  //
1481
1610
  // For single-question payloads, callers (built-in pick handler, plugins)
@@ -1498,7 +1627,7 @@ export class AgentWidgetSession {
1498
1627
  }
1499
1628
  this.markAskUserQuestionResolved(toolMessage, structuredAnswers);
1500
1629
 
1501
- // Show the standalone typing indicator immediately the network round-trip
1630
+ // Show the standalone typing indicator immediately: the network round-trip
1502
1631
  // to /resume is otherwise silent, which reads as broken. The render
1503
1632
  // condition in ui.ts already shows the indicator once streaming flips true
1504
1633
  // and the last message is a user bubble (the answer we inject below).
@@ -1507,8 +1636,8 @@ export class AgentWidgetSession {
1507
1636
  this.abortController = new AbortController();
1508
1637
  this.setStreaming(true);
1509
1638
 
1510
- // Inject Q→A pair messages one assistant bubble per question, one user
1511
- // bubble per answer so the transcript reads like a normal conversation.
1639
+ // Inject Q→A pair messages: one assistant bubble per question, one user
1640
+ // bubble per answer, so the transcript reads like a normal conversation.
1512
1641
  // The original ask_user_question tool message is suppressed by the
1513
1642
  // renderer once `askUserQuestionAnswered` is true. Skipped questions get
1514
1643
  // a muted italic `*Skipped*` user bubble (rendered through the standard
@@ -1580,14 +1709,14 @@ export class AgentWidgetSession {
1580
1709
  if (response.body) {
1581
1710
  await this.connectStream(response.body, { allowReentry: true });
1582
1711
  } else {
1583
- // No body to pipe drop the pre-set streaming flag so the indicator
1712
+ // No body to pipe: drop the pre-set streaming flag so the indicator
1584
1713
  // doesn't linger forever.
1585
1714
  this.setStreaming(false);
1586
1715
  this.abortController = null;
1587
1716
  }
1588
1717
  } catch (error) {
1589
1718
  // Mirror sendMessage: a cancel() during the await aborts the controller
1590
- // and surfaces an AbortError don't treat that as a real failure.
1719
+ // and surfaces an AbortError: don't treat that as a real failure.
1591
1720
  const isAbortError =
1592
1721
  error instanceof Error &&
1593
1722
  (error.name === 'AbortError' ||
@@ -1620,7 +1749,7 @@ export class AgentWidgetSession {
1620
1749
  * before a resolve grabs it.
1621
1750
  *
1622
1751
  * Awaits without an `executionId` or `toolCall.id` can't be batched (no key)
1623
- * route them straight to the single-call path, which surfaces the malformed
1752
+ *: route them straight to the single-call path, which surfaces the malformed
1624
1753
  * wire shape via `onError` / an `isError` resume.
1625
1754
  */
1626
1755
  private enqueueWebMcpAwait(toolMessage: AgentWidgetMessage): void {
@@ -1640,7 +1769,7 @@ export class AgentWidgetSession {
1640
1769
  batch = { snapshots: [], seen: new Set() };
1641
1770
  this.webMcpAwaitBatches.set(executionId, batch);
1642
1771
  }
1643
- // Duplicate step_await re-emit for a call already in this batch ignore.
1772
+ // Duplicate step_await re-emit for a call already in this batch: ignore.
1644
1773
  if (batch.seen.has(callId)) return;
1645
1774
  batch.seen.add(callId);
1646
1775
  batch.snapshots.push(toolMessage);
@@ -1654,11 +1783,11 @@ export class AgentWidgetSession {
1654
1783
 
1655
1784
  /**
1656
1785
  * Flush every buffered local-tool await batch, one `/resume` per executionId.
1657
- * Called once a stream ends (`status: idle` / `error`) by then all parallel
1786
+ * Called once a stream ends (`status: idle` / `error`): by then all parallel
1658
1787
  * `step_await`s the stream carried have been collected, even if split across
1659
1788
  * SSE chunks. Deferred via `queueMicrotask` (epoch-guarded) so the idle
1660
1789
  * handler returns first and the stream's end-of-stream teardown (streaming /
1661
- * abortController) settles before a resolve grabs them the same ordering the
1790
+ * abortController) settles before a resolve grabs them: the same ordering the
1662
1791
  * single-call resolve always relied on.
1663
1792
  */
1664
1793
  private scheduleWebMcpBatchFlush(): void {
@@ -1712,7 +1841,7 @@ export class AgentWidgetSession {
1712
1841
  * Persisted-resolution guard for `suggest_replies`. The in-memory dedupe
1713
1842
  * sets (`webMcpInflightKeys` / `webMcpResolvedKeys`) are cleared by
1714
1843
  * hydrateMessages/clearMessages/cancel, but `suggestRepliesResolved`
1715
- * survives on the stored message so a stale `step_await` re-emit after a
1844
+ * survives on the stored message, so a stale `step_await` re-emit after a
1716
1845
  * hydration must not re-POST `/resume` for an already-resolved call (the
1717
1846
  * historical double-resume failure mode the batching work exists to avoid).
1718
1847
  * Checks the LIVE message first; the handleEvent snapshot is a fresh wire
@@ -1791,7 +1920,7 @@ export class AgentWidgetSession {
1791
1920
  /**
1792
1921
  * Resolve TWO OR MORE parallel local-tool awaits sharing one paused
1793
1922
  * executionId with a SINGLE `/resume` (core#3878). Each call is executed
1794
- * against the page registry concurrently every gated call renders its own
1923
+ * against the page registry concurrently: every gated call renders its own
1795
1924
  * native approval bubble, and a sibling's confirm Promise never blocks
1796
1925
  * another's execution. Outputs are keyed by per-call `webMcpToolCallId`
1797
1926
  * (server prefers it over tool name; name-keying remains the fallback for
@@ -1822,7 +1951,7 @@ export class AgentWidgetSession {
1822
1951
  this.webMcpResolveControllers.add(resumeController);
1823
1952
  this.setStreaming(true);
1824
1953
 
1825
- // Phase 1 execute every pending call concurrently. A null result means
1954
+ // Phase 1: execute every pending call concurrently. A null result means
1826
1955
  // the call was deduped, aborted, or threw; it's omitted from the resume and
1827
1956
  // (per the tolerant server) re-pauses for retry.
1828
1957
  const executed = await Promise.all(
@@ -1853,7 +1982,7 @@ export class AgentWidgetSession {
1853
1982
  toolMessage.agentMetadata?.webMcpToolCallId ?? wireToolName;
1854
1983
 
1855
1984
  // Built-in fire-and-forget tool: no bridge, no confirm gate, no
1856
- // browser-side execution the chips render from the message list and
1985
+ // browser-side execution: the chips render from the message list and
1857
1986
  // the canned output joins the batch's single /resume.
1858
1987
  if (wireToolName === SUGGEST_REPLIES_TOOL_NAME) {
1859
1988
  return {
@@ -1935,14 +2064,13 @@ export class AgentWidgetSession {
1935
2064
  let ready: ExecutedWebMcpTool[] = [];
1936
2065
  try {
1937
2066
  ready = executed.filter((r): r is ExecutedWebMcpTool => r !== null);
1938
- // Everything deduped/aborted/failed nothing to post.
2067
+ // Everything deduped/aborted/failed: nothing to post.
1939
2068
  if (ready.length === 0) return;
1940
2069
 
1941
2070
  const toolOutputs: Record<string, unknown> = {};
1942
2071
  for (const r of ready) {
1943
2072
  // Two omitted-on-collision safety: if two calls somehow resolve to the
1944
- // same key (only possible on a legacy name fallback), last write wins
1945
- // the server re-pauses the unrepresented call for retry.
2073
+ // same key (only possible on a legacy name fallback), last write wins: // the server re-pauses the unrepresented call for retry.
1946
2074
  toolOutputs[r.resumeKey] = r.output;
1947
2075
  }
1948
2076
 
@@ -1953,7 +2081,7 @@ export class AgentWidgetSession {
1953
2081
  const errorData = await response.json().catch(() => null);
1954
2082
  throw new Error(errorData?.error ?? `Resume failed: ${response.status}`);
1955
2083
  }
1956
- // Server accepted the batch mark every included call resolved so stale
2084
+ // Server accepted the batch: mark every included call resolved so stale
1957
2085
  // re-emits don't re-execute the page tool, then complete each bubble.
1958
2086
  // Do this only after /resume HTTP success; if /resume fails, the server
1959
2087
  // may still be paused and the retry path must not show a final result.
@@ -2013,13 +2141,13 @@ export class AgentWidgetSession {
2013
2141
  * list, not from this resolve).
2014
2142
  *
2015
2143
  * Triggered automatically from `handleEvent` when a `step_await`-derived
2016
- * message arrives for such a tool the user does not click a pill; the
2144
+ * message arrives for such a tool: the user does not click a pill; the
2017
2145
  * bridge's confirm-bubble gate (WebMCP only) is the only interactive
2018
2146
  * surface.
2019
2147
  *
2020
2148
  * Idempotent on the message's `toolCall.id`: re-emits of the same step_await
2021
2149
  * (e.g. from message coalescing) won't double-fire `tool.execute`. Failure
2022
- * modes declined, timed out, throw, unknown tool all resolve into a
2150
+ * modes, declined, timed out, throw, unknown tool, all resolve into a
2023
2151
  * `{ isError: true, content: [...] }` payload that resumes the dispatch
2024
2152
  * cleanly so the agent can recover.
2025
2153
  */
@@ -2034,8 +2162,8 @@ export class AgentWidgetSession {
2034
2162
  // server-side dispatch. Three failure modes:
2035
2163
  // - no executionId: no /resume target exists; surface to the host
2036
2164
  // via onError so an operator can react. This is a server-side
2037
- // wire-shape bug Persona can't recover it from the client.
2038
- // - no wireToolName: defensive guard handleEvent only calls us
2165
+ // wire-shape bug: Persona can't recover it from the client.
2166
+ // - no wireToolName: defensive guard: handleEvent only calls us
2039
2167
  // for an auto-resolving local tool name (`webmcp:*` or
2040
2168
  // `suggest_replies`), so this path indicates a direct caller
2041
2169
  // misuse. Silent return.
@@ -2045,7 +2173,7 @@ export class AgentWidgetSession {
2045
2173
  if (!executionId) {
2046
2174
  this.callbacks.onError?.(
2047
2175
  new Error(
2048
- "WebMCP step_await missing executionId dispatch left paused.",
2176
+ "WebMCP step_await missing executionId: dispatch left paused.",
2049
2177
  ),
2050
2178
  );
2051
2179
  return;
@@ -2069,7 +2197,7 @@ export class AgentWidgetSession {
2069
2197
  content: [
2070
2198
  {
2071
2199
  type: "text",
2072
- text: "WebMCP step_await missing toolCall.id cannot execute the page tool.",
2200
+ text: "WebMCP step_await missing toolCall.id: cannot execute the page tool.",
2073
2201
  },
2074
2202
  ],
2075
2203
  });
@@ -2084,7 +2212,7 @@ export class AgentWidgetSession {
2084
2212
  return;
2085
2213
  }
2086
2214
 
2087
- // Dedupe key scoped by executionId see `webMcpInflightKeys` doc comment
2215
+ // Dedupe key scoped by executionId: see `webMcpInflightKeys` doc comment
2088
2216
  // for the failure-recovery + cross-dispatch rationale. The persisted
2089
2217
  // `suggestRepliesResolved` guard backs the in-memory sets across
2090
2218
  // hydrations.
@@ -2099,20 +2227,19 @@ export class AgentWidgetSession {
2099
2227
  this.webMcpInflightKeys.add(dedupeKey);
2100
2228
 
2101
2229
  // Mark resolved on the message so the UI's local-tool sheet (if any
2102
- // generic one ever lands) does not show this is a fully-automatic
2230
+ // generic one ever lands) does not show: this is a fully-automatic
2103
2231
  // tool from the user's perspective, modulo the confirm bubble. Keep the
2104
2232
  // tool bubble running until the browser-side promise resolves; the
2105
2233
  // initial step_await was only the server pause, not tool completion.
2106
2234
  const startedAt = this.markWebMcpToolRunning(toolMessage);
2107
2235
 
2108
2236
  // Per-resolve AbortController, NOT the shared `this.abortController`.
2109
- // A single turn can produce multiple `webmcp:*` step_await messages
2110
- // both PARALLEL (two awaits in one stream) and, more commonly, CHAINED
2237
+ // A single turn can produce multiple `webmcp:*` step_await messages: // both PARALLEL (two awaits in one stream) and, more commonly, CHAINED
2111
2238
  // (tool A → /resume → tool B, where B's step_await arrives inside A's
2112
2239
  // resume SSE stream). The old code pre-aborted `this.abortController`
2113
2240
  // here to mirror the sibling resolve paths; in the chained case that
2114
2241
  // aborted the stream still delivering B, so B never executed and its
2115
- // /resume was never POSTed the dispatch hung forever. Using a dedicated
2242
+ // /resume was never POSTed: the dispatch hung forever. Using a dedicated
2116
2243
  // per-resolve controller leaves the in-flight resume stream untouched.
2117
2244
  // cancel()/clearMessages()/hydrateMessages()/sendMessage() iterate
2118
2245
  // `webMcpResolveControllers` to tear these down on a real stop / new turn.
@@ -2122,13 +2249,13 @@ export class AgentWidgetSession {
2122
2249
  this.setStreaming(true);
2123
2250
 
2124
2251
  // Built-in fire-and-forget tool: no bridge, no confirm gate, no
2125
- // browser-side execution the chips render from the message list and the
2252
+ // browser-side execution: the chips render from the message list and the
2126
2253
  // canned output resumes the execution immediately. Branch BEFORE any
2127
2254
  // bridge access so the missing-bridge error path can never fire for it.
2128
2255
  const isSuggestReplies = wireToolName === SUGGEST_REPLIES_TOOL_NAME;
2129
2256
 
2130
2257
  const args = toolMessage.toolCall?.args;
2131
- // Thread the signal INTO the bridge short-circuits the confirm bubble
2258
+ // Thread the signal INTO the bridge: short-circuits the confirm bubble
2132
2259
  // and the execute() race on cancel(), so a late confirm-approval after
2133
2260
  // cancel() cannot fire a host-page side effect with no matching /resume.
2134
2261
  const execPromise = isSuggestReplies
@@ -2155,7 +2282,7 @@ export class AgentWidgetSession {
2155
2282
  }
2156
2283
  completedAt = Date.now();
2157
2284
  // If cancel() fired during execute, the bridge returned an aborted
2158
- // result don't post it. The server's SSE has been torn down; a
2285
+ // result: don't post it. The server's SSE has been torn down; a
2159
2286
  // /resume now would just produce an orphan dispatch on the server.
2160
2287
  // Streaming/teardown is handled by the shared `finally` below (gated on
2161
2288
  // the resolve set) so we don't clobber a sibling resolve or a live
@@ -2168,7 +2295,7 @@ export class AgentWidgetSession {
2168
2295
  );
2169
2296
  return;
2170
2297
  }
2171
- // Mark resolved as soon as the HTTP /resume returns OK not after the
2298
+ // Mark resolved as soon as the HTTP /resume returns OK: not after the
2172
2299
  // SSE stream finishes. `connectStream` swallows downstream SSE errors
2173
2300
  // (they surface via onError, not by rethrowing), so awaiting it doesn't
2174
2301
  // tell us whether the server actually processed the resume. Marking
@@ -2177,7 +2304,7 @@ export class AgentWidgetSession {
2177
2304
  // toolCall.id are stale and must not re-execute the page tool.
2178
2305
  // Key the resume by the per-call id (core#3878) when present; the server
2179
2306
  // prefers it over tool name. Falls back to the wire tool name for legacy
2180
- // servers the original name-keyed contract, still correct for a single
2307
+ // servers: the original name-keyed contract, still correct for a single
2181
2308
  // call (only same-tool PARALLEL calls could collide on the name).
2182
2309
  const resumeKey =
2183
2310
  toolMessage.agentMetadata?.webMcpToolCallId ?? wireToolName;
@@ -2202,7 +2329,7 @@ export class AgentWidgetSession {
2202
2329
  error.message.includes("aborted") ||
2203
2330
  error.message.includes("abort"));
2204
2331
  // Streaming/teardown handled by the shared `finally` (gated on the
2205
- // resolve set) do NOT null the shared `this.abortController` here; it
2332
+ // resolve set): do NOT null the shared `this.abortController` here; it
2206
2333
  // may belong to a live dispatch or sibling resolve, not to us.
2207
2334
  if (phase === "execute" || isAbortError || signal.aborted) {
2208
2335
  this.markWebMcpToolComplete(
@@ -2218,7 +2345,7 @@ export class AgentWidgetSession {
2218
2345
  if (!isAbortError) {
2219
2346
  // The bridge normalizes tool errors into result objects, so reaching
2220
2347
  // here means a network failure during `/resume` itself, OR a stream
2221
- // hookup error. Surface to onError, but DO NOT mark resolved a
2348
+ // hookup error. Surface to onError, but DO NOT mark resolved: a
2222
2349
  // later step_await re-emit should be allowed to retry the resume.
2223
2350
  this.callbacks.onError?.(
2224
2351
  error instanceof Error ? error : new Error(String(error)),
@@ -2230,7 +2357,7 @@ export class AgentWidgetSession {
2230
2357
  // Only flip streaming off when this was the last in-flight resolve AND
2231
2358
  // no shared dispatch is live. Otherwise a finishing resolve would hide
2232
2359
  // the typing indicator while a sibling (parallel) or successor (chained)
2233
- // resolve or a live dispatch is still running.
2360
+ // resolve, or a live dispatch, is still running.
2234
2361
  if (this.webMcpResolveControllers.size === 0 && !this.abortController) {
2235
2362
  this.setStreaming(false);
2236
2363
  }
@@ -2269,7 +2396,7 @@ export class AgentWidgetSession {
2269
2396
  await this.connectStream(response.body, { allowReentry: true });
2270
2397
  } else if (this.webMcpResolveControllers.size === 0) {
2271
2398
  // No stream to pipe. Clear streaming only when no WebMCP resolve is in
2272
- // flight for a WebMCP caller the current resolve's controller is still
2399
+ // flight: for a WebMCP caller the current resolve's controller is still
2273
2400
  // in the set, so its own `finally` (gated on the set draining) owns the
2274
2401
  // teardown. Non-WebMCP callers (ask_user_question) keep the old behavior.
2275
2402
  this.setStreaming(false);
@@ -2282,8 +2409,7 @@ export class AgentWidgetSession {
2282
2409
  * resolve owns a dedicated AbortController (chained/parallel resolves don't
2283
2410
  * share one), so we abort them individually; the aborts propagate into the
2284
2411
  * bridge's execute race and into each `/resume` fetch signal. Bumping
2285
- * `webMcpEpoch` strands any resolve still deferred in a queued microtask
2286
- * it captured the prior epoch and bails before installing a fresh
2412
+ * `webMcpEpoch` strands any resolve still deferred in a queued microtask: * it captured the prior epoch and bails before installing a fresh
2287
2413
  * controller, so it can't escape this teardown. Called from every stop /
2288
2414
  * new-turn boundary (cancel, clearMessages, hydrateMessages, sendMessage).
2289
2415
  */
@@ -2294,7 +2420,7 @@ export class AgentWidgetSession {
2294
2420
  this.webMcpResolveControllers.clear();
2295
2421
  // Settle every approval bubble still awaiting a click. The bridge parks a
2296
2422
  // resolve on `await requestConfirm(...)` (→ requestWebMcpApproval) and only
2297
- // re-checks `signal.aborted` AFTER that await returns aborting the
2423
+ // re-checks `signal.aborted` AFTER that await returns: aborting the
2298
2424
  // controller above does NOT unblock it. Left unsettled, the bridge's
2299
2425
  // execute(), its `/resume`, and the resolve's `finally` would all hang
2300
2426
  // forever (and the resolver map would leak across teardowns). Route through
@@ -2302,12 +2428,12 @@ export class AgentWidgetSession {
2302
2428
  // `false` AND its bubble message flips out of `pending` (no stale "Approve/
2303
2429
  // Deny" left clickable). The bridge then returns cleanly and its
2304
2430
  // post-confirm `signal.aborted` guard bails before any host-page side effect
2305
- // or stale `/resume`. Snapshot the keys first resolveWebMcpApproval
2431
+ // or stale `/resume`. Snapshot the keys first: resolveWebMcpApproval
2306
2432
  // mutates the map as it deletes each resolver.
2307
2433
  for (const approvalMessageId of [...this.webMcpApprovalResolvers.keys()]) {
2308
2434
  this.resolveWebMcpApproval(approvalMessageId, "denied");
2309
2435
  }
2310
- // Drop any awaits buffered for a not-yet-flushed batch their messages are
2436
+ // Drop any awaits buffered for a not-yet-flushed batch: their messages are
2311
2437
  // being torn down, and a microtask-deferred flush must not survive. The
2312
2438
  // epoch bump below also strands an already-scheduled flush.
2313
2439
  this.webMcpAwaitBatches.clear();
@@ -2322,7 +2448,7 @@ export class AgentWidgetSession {
2322
2448
  // are possible if the user re-issues the same step_await context.
2323
2449
  this.abortWebMcpResolves();
2324
2450
  this.webMcpInflightKeys.clear();
2325
- // Stop any in-progress audio too when the user hits "stop", they want
2451
+ // Stop any in-progress audio too: when the user hits "stop", they want
2326
2452
  // the assistant to actually stop talking, not just stop generating tokens.
2327
2453
  // Both helpers are safe no-ops when audio isn't configured.
2328
2454
  this.stopSpeaking();
@@ -2335,19 +2461,19 @@ export class AgentWidgetSession {
2335
2461
  this.stopSpeaking();
2336
2462
  this.abortController?.abort();
2337
2463
  this.abortController = null;
2338
- // Tear down every in-flight WebMCP resolve too their messages are about
2464
+ // Tear down every in-flight WebMCP resolve too: their messages are about
2339
2465
  // to be wiped, and a microtask-deferred resolve must not survive the clear.
2340
2466
  this.abortWebMcpResolves();
2341
2467
  this.messages = [];
2342
2468
  this.agentExecution = null;
2343
2469
  this.clearArtifactState();
2344
- // Clearing messages also wipes the WebMCP dedupe state a fresh
2470
+ // Clearing messages also wipes the WebMCP dedupe state: a fresh
2345
2471
  // conversation should not refuse to call a webmcp:* tool just because
2346
2472
  // a tool with the same key resolved in the prior conversation.
2347
2473
  this.webMcpInflightKeys.clear();
2348
2474
  this.webMcpResolvedKeys.clear();
2349
2475
  // A fresh conversation must resend the full WebMCP tool list on its next
2350
- // turn drop the diff-only fingerprint cache (server keys by recordId, so
2476
+ // turn: drop the diff-only fingerprint cache (server keys by recordId, so
2351
2477
  // a new conversation has no stored set to match).
2352
2478
  this.client.resetClientToolsFingerprint();
2353
2479
  this.setStreaming(false);
@@ -2474,10 +2600,10 @@ export class AgentWidgetSession {
2474
2600
  public hydrateMessages(messages: AgentWidgetMessage[]) {
2475
2601
  this.abortController?.abort();
2476
2602
  this.abortController = null;
2477
- // Hydration replaces the conversation abort and forget every in-flight
2603
+ // Hydration replaces the conversation: abort and forget every in-flight
2478
2604
  // WebMCP resolve; their messages are about to be replaced.
2479
2605
  this.abortWebMcpResolves();
2480
- // Wipe the WebMCP dedupe state alongside the message restore the
2606
+ // Wipe the WebMCP dedupe state alongside the message restore: the
2481
2607
  // incoming snapshot is treated as a fresh conversation context.
2482
2608
  this.webMcpInflightKeys.clear();
2483
2609
  this.webMcpResolvedKeys.clear();
@@ -2510,8 +2636,8 @@ export class AgentWidgetSession {
2510
2636
  this.upsertMessage(event.message);
2511
2637
 
2512
2638
  // Local-tool auto-resolve: when a step_await emits a tool-variant
2513
- // message for a `webmcp:*` tool or the built-in fire-and-forget
2514
- // `suggest_replies` resolve it and post the result to /resume.
2639
+ // message for a `webmcp:*` tool, or the built-in fire-and-forget
2640
+ // `suggest_replies`: resolve it and post the result to /resume.
2515
2641
  // Unlike ask_user_question, no user pill click is required; for WebMCP
2516
2642
  // the bridge's confirm bubble is the only interactive surface, and
2517
2643
  // suggest_replies resumes with a canned "shown" result while the chips
@@ -2527,11 +2653,11 @@ export class AgentWidgetSession {
2527
2653
  //
2528
2654
  // ALWAYS resolve when the wire name carries the `webmcp:` prefix, even
2529
2655
  // if the bridge is non-operational. Otherwise the dispatch stays paused
2530
- // indefinitely `resolveWebMcpToolCall` translates the missing-bridge
2656
+ // indefinitely: `resolveWebMcpToolCall` translates the missing-bridge
2531
2657
  // case into an isError result that resumes the flow cleanly.
2532
2658
  // `suggest_replies`, by contrast, is gated on its feature flag: when
2533
2659
  // `features.suggestReplies.enabled === false` the widget neither
2534
- // renders chips nor resumes the same parked-execution posture as a
2660
+ // renders chips nor resumes: the same parked-execution posture as a
2535
2661
  // server-declared ask_user_question with its sheet disabled.
2536
2662
  const tc = event.message.toolCall;
2537
2663
  const autoResolvable =
@@ -2578,12 +2704,12 @@ export class AgentWidgetSession {
2578
2704
  this.setStreaming(false);
2579
2705
  this.abortController = null;
2580
2706
  }
2581
- // Mark agent execution as complete when streaming ends UNLESS local
2707
+ // Mark agent execution as complete when streaming ends: UNLESS local
2582
2708
  // tools are still outstanding. A batched WebMCP resume is deferred to
2583
2709
  // the microtask below (so `webMcpResolveControllers` is still empty
2584
2710
  // here) and a chained resolve may be mid-flight; marking the run
2585
2711
  // 'complete' now would make isAgentRunning() report a finished run while
2586
- // page tools are still executing. Stay 'running' the resume stream's
2712
+ // page tools are still executing. Stay 'running': the resume stream's
2587
2713
  // own idle (with batches drained and resolves settled) marks it done.
2588
2714
  const webMcpPending =
2589
2715
  this.webMcpAwaitBatches.size > 0 ||
@@ -2597,7 +2723,7 @@ export class AgentWidgetSession {
2597
2723
  }
2598
2724
  // The stream that delivered any local-tool `step_await`s has now ended,
2599
2725
  // so every parallel await it carried is collected. Flush them as ONE
2600
- // batched `/resume` per executionId (deferred see
2726
+ // batched `/resume` per executionId (deferred: see
2601
2727
  // scheduleWebMcpBatchFlush). Runs AFTER the teardown above so a resolve
2602
2728
  // doesn't fight the end-of-stream streaming/abortController reset.
2603
2729
  this.scheduleWebMcpBatchFlush();
@@ -2605,7 +2731,7 @@ export class AgentWidgetSession {
2605
2731
  } else if (event.type === "error") {
2606
2732
  this.setStatus("error");
2607
2733
  // Mirror the idle/status handler: don't tear down streaming while a
2608
- // WebMCP resolve is still confirming/executing on another stream an
2734
+ // WebMCP resolve is still confirming/executing on another stream: an
2609
2735
  // error on one chained resume stream must not hide the typing indicator
2610
2736
  // (or null a controller) for a sibling/successor resolve still in flight.
2611
2737
  // The resolve's own `finally` flips streaming off once the set drains.
@@ -2675,84 +2801,65 @@ export class AgentWidgetSession {
2675
2801
  return;
2676
2802
  }
2677
2803
 
2678
- const text = lastAssistant.content;
2804
+ const text = resolveSpeakableText(lastAssistant.content);
2679
2805
  if (!text.trim()) return;
2680
2806
 
2681
- this.speak(text, ttsConfig);
2807
+ // Route auto-speak through the same controller as the "Read aloud" button
2808
+ // so there's a single owner of the speech engine and the button reflects
2809
+ // playback state (and a single message can't be spoken by two paths at once).
2810
+ void this.readAloud.play(lastAssistant.id, {
2811
+ text,
2812
+ voice: ttsConfig.voice,
2813
+ rate: ttsConfig.rate,
2814
+ pitch: ttsConfig.pitch,
2815
+ });
2682
2816
  }
2683
2817
 
2684
2818
  /**
2685
- * Speak text using the Web Speech API.
2686
- * Cancels any in-progress speech before starting.
2819
+ * Pick the best available English voice from a list of SpeechSynthesisVoices.
2820
+ * Prefers high-quality remote/natural voices, then enhanced local voices,
2821
+ * then standard local voices. Retained for backwards compatibility; delegates
2822
+ * to the browser speech engine's picker.
2687
2823
  */
2688
- private speak(text: string, config: TextToSpeechConfig) {
2689
- if (typeof window === 'undefined' || !('speechSynthesis' in window)) return;
2690
-
2691
- const synth = window.speechSynthesis;
2692
- synth.cancel();
2693
-
2694
- const utterance = new SpeechSynthesisUtterance(text);
2695
- const voices = synth.getVoices();
2696
-
2697
- if (config.voice) {
2698
- const match = voices.find(v => v.name === config.voice);
2699
- if (match) utterance.voice = match;
2700
- } else if (voices.length > 0) {
2701
- // Use custom picker if provided, otherwise auto-detect
2702
- utterance.voice = config.pickVoice
2703
- ? config.pickVoice(voices)
2704
- : AgentWidgetSession.pickBestVoice(voices);
2705
- }
2706
-
2707
- if (config.rate !== undefined) utterance.rate = config.rate;
2708
- if (config.pitch !== undefined) utterance.pitch = config.pitch;
2709
-
2710
- // Chrome bug: cancel() immediately followed by speak() can ignore
2711
- // rate/pitch. A microtask delay lets the engine reset properly.
2712
- setTimeout(() => synth.speak(utterance), 50);
2824
+ static pickBestVoice(voices: SpeechSynthesisVoice[]): SpeechSynthesisVoice {
2825
+ return pickBestVoice(voices);
2713
2826
  }
2714
2827
 
2715
2828
  /**
2716
- * Pick the best available English voice from a list of SpeechSynthesisVoices.
2717
- * Prefers high-quality remote/natural voices, then enhanced local voices,
2718
- * then standard local voices.
2829
+ * Toggle the per-message "Read aloud" action: play pause resume (or
2830
+ * play stop when the engine can't pause). Speaks the assistant message's
2831
+ * text via the configured speech engine (browser Web Speech API by default,
2832
+ * or a hosted engine from `textToSpeech.createEngine`).
2719
2833
  */
2720
- static pickBestVoice(voices: SpeechSynthesisVoice[]): SpeechSynthesisVoice {
2721
- // Priority list: high-quality voices across browsers/platforms
2722
- const preferred = [
2723
- // Edge Online Natural (highest quality)
2724
- 'Microsoft Jenny Online (Natural) - English (United States)',
2725
- 'Microsoft Aria Online (Natural) - English (United States)',
2726
- 'Microsoft Guy Online (Natural) - English (United States)',
2727
- // Google remote (good quality, cross-platform in Chrome)
2728
- 'Google US English',
2729
- 'Google UK English Female',
2730
- // Apple premium/enhanced (macOS)
2731
- 'Ava (Premium)',
2732
- 'Evan (Enhanced)',
2733
- 'Samantha (Enhanced)',
2734
- // Apple standard (macOS/iOS)
2735
- 'Samantha',
2736
- 'Daniel',
2737
- 'Karen',
2738
- // Windows SAPI
2739
- 'Microsoft David Desktop - English (United States)',
2740
- 'Microsoft Zira Desktop - English (United States)',
2741
- ];
2834
+ public toggleReadAloud(messageId: string): void {
2835
+ const message = this.messages.find(m => m.id === messageId);
2836
+ if (!message || message.role !== 'assistant') return;
2837
+ const text = resolveSpeakableText(message.content || '');
2838
+ if (!text.trim()) return;
2839
+ const tts = this.config.textToSpeech;
2840
+ this.readAloud.toggle(messageId, {
2841
+ text,
2842
+ voice: tts?.voice,
2843
+ rate: tts?.rate,
2844
+ pitch: tts?.pitch,
2845
+ });
2846
+ }
2742
2847
 
2743
- for (const name of preferred) {
2744
- const match = voices.find(v => v.name === name);
2745
- if (match) return match;
2746
- }
2848
+ /** Current read-aloud playback state for a message (`idle` unless active). */
2849
+ public getReadAloudState(messageId: string): ReadAloudState {
2850
+ return this.readAloud.stateFor(messageId);
2851
+ }
2747
2852
 
2748
- // Fallback: any English voice, then first available
2749
- return voices.find(v => v.lang.startsWith('en')) ?? voices[0];
2853
+ /** Subscribe to read-aloud state changes. Returns an unsubscribe function. */
2854
+ public onReadAloudChange(listener: ReadAloudListener): () => void {
2855
+ return this.readAloud.onChange(listener);
2750
2856
  }
2751
2857
 
2752
2858
  /**
2753
- * Stop any in-progress text-to-speech playback.
2859
+ * Stop any in-progress text-to-speech / read-aloud playback.
2754
2860
  */
2755
2861
  public stopSpeaking() {
2862
+ this.readAloud.stop();
2756
2863
  if (typeof window !== 'undefined' && 'speechSynthesis' in window) {
2757
2864
  window.speechSynthesis.cancel();
2758
2865
  }
@@ -2776,8 +2883,7 @@ export class AgentWidgetSession {
2776
2883
  if (idx !== index) return existing;
2777
2884
  const merged = { ...existing, ...withSequence };
2778
2885
  // Preserve `ask_user_question` answered state across re-emissions.
2779
- // Top-level merge would otherwise replace `agentMetadata` wholesale
2780
- // post-resume events (e.g. `tool_complete` re-emitted from a stale
2886
+ // Top-level merge would otherwise replace `agentMetadata` wholesale: // post-resume events (e.g. `tool_complete` re-emitted from a stale
2781
2887
  // client-side cache) would wipe `askUserQuestionAnswered` and
2782
2888
  // `askUserQuestionAnswers`, causing the answered review card to
2783
2889
  // lose its answers and revert to "(skipped)" placeholders.
@@ -2794,7 +2900,7 @@ export class AgentWidgetSession {
2794
2900
  existing.agentMetadata.askUserQuestionAnswers,
2795
2901
  }
2796
2902
  : {}),
2797
- // Keep awaiting flag false once resolved never let a stale
2903
+ // Keep awaiting flag false once resolved: never let a stale
2798
2904
  // re-emit flip us back to awaiting.
2799
2905
  awaitingLocalTool: false,
2800
2906
  };
@@ -2815,7 +2921,7 @@ export class AgentWidgetSession {
2815
2921
  };
2816
2922
  }
2817
2923
  // Approval equivalent: `agent_approval_complete` carries only the
2818
- // resolution (approvalId, decision, resolvedBy) the runtime does not
2924
+ // resolution (approvalId, decision, resolvedBy): the runtime does not
2819
2925
  // re-send toolName/description/toolType/reason/parameters, so client.ts
2820
2926
  // rebuilds the approval with empty required fields and the optional
2821
2927
  // ones absent. A wholesale `approval` replacement would wipe that