@runtypelabs/persona 3.34.1 → 3.36.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -11
- package/dist/animations/glyph-cycle.d.cts +1 -1
- package/dist/animations/glyph-cycle.d.ts +1 -1
- package/dist/animations/{types-CthJFfNx.d.cts → types-DgYsuwXL.d.cts} +15 -14
- package/dist/animations/{types-CthJFfNx.d.ts → types-DgYsuwXL.d.ts} +15 -14
- package/dist/animations/wipe.d.cts +1 -1
- package/dist/animations/wipe.d.ts +1 -1
- package/dist/codegen.cjs +11 -11
- package/dist/codegen.js +10 -10
- package/dist/index.cjs +46 -44
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +514 -158
- package/dist/index.d.ts +514 -158
- package/dist/index.global.js +41 -89
- package/dist/index.global.js.map +1 -1
- package/dist/index.js +46 -44
- package/dist/index.js.map +1 -1
- package/dist/install.global.js +1 -1
- package/dist/install.global.js.map +1 -1
- package/dist/launcher.global.js +2 -2
- package/dist/launcher.global.js.map +1 -1
- package/dist/markdown-parsers.js +53 -0
- package/dist/plugin-kit.d.cts +4 -4
- package/dist/plugin-kit.d.ts +4 -4
- package/dist/runtype-tts.js +1 -0
- package/dist/smart-dom-reader.cjs +1 -1
- package/dist/smart-dom-reader.d.cts +352 -103
- package/dist/smart-dom-reader.d.ts +352 -103
- package/dist/smart-dom-reader.js +1 -1
- package/dist/theme-editor-preview.cjs +203 -0
- package/dist/theme-editor-preview.d.cts +5748 -0
- package/dist/theme-editor-preview.d.ts +5748 -0
- package/dist/theme-editor-preview.js +203 -0
- package/dist/theme-editor.cjs +14 -131
- package/dist/theme-editor.d.cts +362 -574
- package/dist/theme-editor.d.ts +362 -574
- package/dist/theme-editor.js +14 -131
- package/dist/theme-reference.cjs +1 -1
- package/dist/theme-reference.d.cts +1 -1
- package/dist/theme-reference.d.ts +1 -1
- package/dist/theme-reference.js +1 -1
- package/dist/voice-worklet-player.cjs +80 -0
- package/dist/voice-worklet-player.d.cts +215 -0
- package/dist/voice-worklet-player.d.ts +215 -0
- package/dist/voice-worklet-player.js +80 -0
- package/dist/widget.css +54 -36
- package/package.json +16 -2
- package/src/animations/glyph-cycle.ts +11 -14
- package/src/animations/wipe.ts +1 -1
- package/src/ask-user-question-tool.test.ts +1 -1
- package/src/ask-user-question-tool.ts +6 -8
- package/src/client.test.ts +56 -12
- package/src/client.ts +35 -31
- package/src/codegen.test.ts +1 -1
- package/src/codegen.ts +1 -1
- package/src/components/approval-bubble.test.ts +1 -1
- package/src/components/approval-bubble.ts +3 -3
- package/src/components/artifact-pane.ts +3 -3
- package/src/components/ask-user-question-bubble.test.ts +14 -14
- package/src/components/ask-user-question-bubble.ts +10 -10
- package/src/components/composer-builder.ts +17 -16
- package/src/components/composer-parts.test.ts +54 -0
- package/src/components/composer-parts.ts +95 -126
- package/src/components/demo-carousel.ts +1 -1
- package/src/components/event-stream-view.test.ts +7 -7
- package/src/components/event-stream-view.ts +5 -5
- package/src/components/header-builder.ts +33 -27
- package/src/components/header-layouts.ts +1 -1
- package/src/components/header-parts.test.ts +90 -0
- package/src/components/header-parts.ts +62 -93
- package/src/components/launcher.test.ts +11 -0
- package/src/components/launcher.ts +1 -1
- package/src/components/message-bubble.test.ts +74 -2
- package/src/components/message-bubble.ts +25 -11
- package/src/components/messages.ts +2 -2
- package/src/components/panel.test.ts +1 -1
- package/src/components/panel.ts +92 -97
- package/src/components/pill-composer-builder.ts +28 -24
- package/src/components/reasoning-bubble.ts +2 -2
- package/src/components/suggestions.ts +1 -1
- package/src/components/tool-bubble.ts +1 -1
- package/src/components/widget-view.test.ts +181 -0
- package/src/components/widget-view.ts +234 -0
- package/src/defaults.ts +54 -51
- package/src/generated/runtype-openapi-contract.ts +5 -0
- package/src/index-core.ts +30 -4
- package/src/index-global.ts +73 -6
- package/src/index.ts +7 -1
- package/src/install.test.ts +11 -11
- package/src/install.ts +28 -18
- package/src/launcher-global.ts +25 -14
- package/src/markdown-parsers-eager.ts +20 -0
- package/src/markdown-parsers-entry.ts +4 -0
- package/src/markdown-parsers-loader.ts +49 -0
- package/src/plugin-kit.test.ts +1 -1
- package/src/plugin-kit.ts +7 -7
- package/src/plugins/types.ts +5 -5
- package/src/postprocessors.ts +27 -14
- package/src/runtime/host-layout.test.ts +1 -1
- package/src/runtime/host-layout.ts +4 -5
- package/src/runtime/init.ts +2 -2
- package/src/session.test.ts +5 -6
- package/src/session.ts +333 -227
- package/src/session.voice.test.ts +146 -0
- package/src/session.webmcp.test.ts +24 -25
- package/src/smart-dom-reader.ts +3 -4
- package/src/styles/widget.css +54 -36
- package/src/suggest-replies-tool.test.ts +3 -3
- package/src/suggest-replies-tool.ts +5 -5
- package/src/testing/index.ts +1 -1
- package/src/theme-editor/color-utils.ts +2 -2
- package/src/theme-editor/index.ts +4 -13
- package/src/theme-editor/presets.ts +1 -1
- package/src/theme-editor/preview-utils.ts +22 -20
- package/src/theme-editor/preview.ts +4 -1
- package/src/theme-editor/role-mappings.ts +3 -3
- package/src/theme-editor/sections.ts +21 -20
- package/src/theme-editor/types.ts +1 -1
- package/src/theme-editor/webmcp/index.ts +1 -1
- package/src/theme-editor/webmcp/summary.ts +1 -2
- package/src/theme-editor/webmcp/tools.ts +30 -30
- package/src/theme-editor/webmcp/types.ts +2 -2
- package/src/theme-editor-preview.ts +10 -0
- package/src/theme-reference.ts +26 -26
- package/src/types.ts +368 -99
- package/src/ui.approval-plugin.test.ts +3 -3
- package/src/ui.ask-user-question-plugin.test.ts +9 -9
- package/src/ui.component-directive.test.ts +4 -4
- package/src/ui.composer-bar.test.ts +10 -11
- package/src/ui.composer-keyboard.test.ts +5 -5
- package/src/ui.header-icon-color.test.ts +59 -0
- package/src/ui.scroll.test.ts +5 -6
- package/src/ui.stop-button.test.ts +3 -4
- package/src/ui.suggest-replies.test.ts +2 -2
- package/src/ui.ts +292 -131
- package/src/utils/buttons.ts +1 -1
- package/src/utils/code-generators.ts +1 -1
- package/src/utils/composer-history.ts +3 -3
- package/src/utils/dom-context.test.ts +2 -2
- package/src/utils/dom-context.ts +7 -7
- package/src/utils/dom.test.ts +67 -0
- package/src/utils/dom.ts +75 -0
- package/src/utils/dropdown.ts +2 -2
- package/src/utils/event-stream-buffer.test.ts +2 -2
- package/src/utils/message-fingerprint.test.ts +9 -0
- package/src/utils/message-fingerprint.ts +5 -0
- package/src/utils/morph.ts +1 -1
- package/src/utils/sanitize.ts +32 -16
- package/src/utils/sequence-buffer.test.ts +8 -8
- package/src/utils/sequence-buffer.ts +4 -4
- package/src/utils/smart-dom-adapter.test.ts +1 -1
- package/src/utils/smart-dom-adapter.ts +4 -5
- package/src/utils/speech-text.test.ts +122 -0
- package/src/utils/speech-text.ts +101 -0
- package/src/utils/stream-animation.test.ts +3 -3
- package/src/utils/stream-animation.ts +6 -6
- package/src/utils/theme.test.ts +1 -1
- package/src/utils/throughput-tracker.test.ts +11 -11
- package/src/utils/throughput-tracker.ts +9 -9
- package/src/utils/tokens.ts +13 -13
- package/src/utils/virtual-scroller.ts +1 -1
- package/src/vendor/smart-dom-reader/README.md +4 -4
- package/src/vendor/smart-dom-reader/index.d.ts +1 -1
- package/src/vendor/smart-dom-reader/index.js +2 -2
- package/src/version.ts +1 -1
- package/src/voice/audio-playback-manager.test.ts +159 -0
- package/src/voice/audio-playback-manager.ts +145 -30
- package/src/voice/browser-speech-engine.ts +130 -0
- package/src/voice/fallback-speech-engine.ts +89 -0
- package/src/voice/index.ts +21 -0
- package/src/voice/read-aloud-controller.test.ts +143 -0
- package/src/voice/read-aloud-controller.ts +136 -0
- package/src/voice/runtype-speech-engine.test.ts +280 -0
- package/src/voice/runtype-speech-engine.ts +211 -0
- package/src/voice/runtype-tts-entry.ts +10 -0
- package/src/voice/runtype-tts-loader.test.ts +47 -0
- package/src/voice/runtype-tts-loader.ts +42 -0
- package/src/voice/runtype-voice-provider.ts +332 -537
- package/src/voice/voice-factory.ts +26 -4
- package/src/voice/voice.test.ts +321 -7
- package/src/voice/worklet-playback-engine.ts +272 -0
- package/src/voice-worklet-player.ts +39 -0
- package/src/webmcp-bridge.test.ts +6 -6
- package/src/webmcp-bridge.ts +24 -25
- package/src/webmcp-polyfill.ts +1 -1
- package/src/voice/voice-activity-detector.ts +0 -90
package/src/session.ts
CHANGED
|
@@ -31,12 +31,19 @@ import type {
|
|
|
31
31
|
VoiceProvider,
|
|
32
32
|
VoiceStatus,
|
|
33
33
|
VoiceConfig,
|
|
34
|
-
|
|
34
|
+
ReadAloudState,
|
|
35
|
+
SpeechEngine
|
|
35
36
|
} from "./types";
|
|
36
37
|
import {
|
|
37
38
|
createVoiceProvider,
|
|
38
|
-
isVoiceSupported
|
|
39
|
+
isVoiceSupported,
|
|
40
|
+
BrowserSpeechEngine,
|
|
41
|
+
pickBestVoice,
|
|
42
|
+
ReadAloudController,
|
|
43
|
+
type ReadAloudListener
|
|
39
44
|
} from "./voice";
|
|
45
|
+
import { resolveSpeakableText } from "./utils/speech-text";
|
|
46
|
+
import { loadRuntypeTts } from "./voice/runtype-tts-loader";
|
|
40
47
|
|
|
41
48
|
export type AgentWidgetSessionStatus =
|
|
42
49
|
| "idle"
|
|
@@ -51,7 +58,7 @@ export type AgentWidgetSessionStatus =
|
|
|
51
58
|
* instead of swapping in a fresh client and tearing down WebMCP state. Fields
|
|
52
59
|
* outside this list (theme, copy, layout, suggestionChips, iterationDisplay,
|
|
53
60
|
* postprocessMessage, feature display toggles, …) are display-only and safe to
|
|
54
|
-
* apply mid-turn
|
|
61
|
+
* apply mid-turn: which is what a self-styling widget needs when a `webmcp:*`
|
|
55
62
|
* theme tool re-renders the widget while the agent's turn is still streaming.
|
|
56
63
|
*
|
|
57
64
|
* Compared by identity (`!==`): primitives by value, functions/objects by
|
|
@@ -102,8 +109,8 @@ type SessionCallbacks = {
|
|
|
102
109
|
/**
|
|
103
110
|
* Build the user-facing content shown when a dispatch fails before any
|
|
104
111
|
* assistant content streamed back. This fires on real network/server errors
|
|
105
|
-
* (connection refused, CORS, 4xx/5xx, malformed stream)
|
|
106
|
-
* un-wired proxy
|
|
112
|
+
* (connection refused, CORS, 4xx/5xx, malformed stream): not just an
|
|
113
|
+
* un-wired proxy, so the copy stays honest about the failure and surfaces the
|
|
107
114
|
* underlying reason to help with debugging.
|
|
108
115
|
*
|
|
109
116
|
* Callers can override the copy via `config.errorMessage` (a static string or
|
|
@@ -120,7 +127,7 @@ function buildDispatchErrorContent(
|
|
|
120
127
|
if (typeof override === "function") return override(err);
|
|
121
128
|
|
|
122
129
|
const base =
|
|
123
|
-
"Sorry
|
|
130
|
+
"Sorry: I couldn't reach the assistant. The chat service didn't respond. Please check that your proxy or backend is running and reachable, then try again.";
|
|
124
131
|
return err.message ? `${base}\n\n_Details: ${err.message}_` : base;
|
|
125
132
|
}
|
|
126
133
|
|
|
@@ -142,7 +149,7 @@ const getWebMcpErrorMessage = (
|
|
|
142
149
|
* Tool names whose `step_await` the widget resolves automatically (no user
|
|
143
150
|
* pill click): `webmcp:*` page tools and the built-in fire-and-forget
|
|
144
151
|
* `suggest_replies`. These share the await-batch / dedupe / resume machinery;
|
|
145
|
-
* `ask_user_question` is NOT one of them
|
|
152
|
+
* `ask_user_question` is NOT one of them: it blocks on the answer sheet.
|
|
146
153
|
*/
|
|
147
154
|
const isAutoResolvedLocalToolName = (name: string): boolean =>
|
|
148
155
|
isWebMcpToolName(name) || name === SUGGEST_REPLIES_TOOL_NAME;
|
|
@@ -164,16 +171,16 @@ export class AgentWidgetSession {
|
|
|
164
171
|
private artifacts = new Map<string, PersonaArtifactRecord>();
|
|
165
172
|
private selectedArtifactId: string | null = null;
|
|
166
173
|
|
|
167
|
-
// WebMCP dedupe
|
|
174
|
+
// WebMCP dedupe: keys are `${executionId}:${toolCallId}` so they're
|
|
168
175
|
// naturally scoped to a single dispatch. A later dispatch (new executionId)
|
|
169
176
|
// that happens to recycle a `toolCall.id` never collides with prior entries,
|
|
170
177
|
// and a stale re-emit from an in-flight prior dispatch stays blocked because
|
|
171
178
|
// its executionId is still in the set.
|
|
172
179
|
//
|
|
173
|
-
// webMcpInflightKeys
|
|
180
|
+
// webMcpInflightKeys: currently executing; cleared on completion of
|
|
174
181
|
// EITHER /resume success OR /resume throw. Blocks
|
|
175
182
|
// concurrent re-fire during the resolve round-trip.
|
|
176
|
-
// webMcpResolvedKeys
|
|
183
|
+
// webMcpResolvedKeys: /resume HTTP returned 2xx; not cleared on a new
|
|
177
184
|
// dispatch (executionId scoping makes that
|
|
178
185
|
// unnecessary). Blocks stale step_await re-emits
|
|
179
186
|
// for the same execution.
|
|
@@ -188,7 +195,7 @@ export class AgentWidgetSession {
|
|
|
188
195
|
// `this.abortController` is intentionally NOT used by resolveWebMcpToolCall:
|
|
189
196
|
// in a CHAINED turn (tool A → /resume → tool B, where the server emits B's
|
|
190
197
|
// step_await inside A's resume SSE stream) the shared controller is still
|
|
191
|
-
// piping A's resume stream
|
|
198
|
+
// piping A's resume stream: the very stream that just delivered B. Aborting
|
|
192
199
|
// it mid-chain (the prior shared-controller pre-abort) tore that stream down,
|
|
193
200
|
// so B never reached execute() and its /resume was never POSTed, pausing the
|
|
194
201
|
// dispatch forever. cancel(), clearMessages(), hydrateMessages(), and
|
|
@@ -211,11 +218,11 @@ export class AgentWidgetSession {
|
|
|
211
218
|
private webMcpApprovalSeq = 0;
|
|
212
219
|
// Parallel local-tool batching (core#3878). A single model turn can emit
|
|
213
220
|
// multiple `step_await(local_tool_required)` events for ONE paused
|
|
214
|
-
// executionId
|
|
221
|
+
// executionId: including two PARALLEL calls to the SAME tool ("add SHOE-001
|
|
215
222
|
// and SHOE-007"). Those collapse to an identical `toolId`/`index` and differ
|
|
216
223
|
// only by the per-call `webMcpToolCallId`. We collect all awaits for an
|
|
217
224
|
// executionId that arrive in the same stream tick, then post ONE `/resume`
|
|
218
|
-
// keyed by `webMcpToolCallId
|
|
225
|
+
// keyed by `webMcpToolCallId`: NOT one `/resume` per tool keyed by name
|
|
219
226
|
// (which collides for same-tool calls, and whose concurrent posts on one
|
|
220
227
|
// execution raced → the second 404'd → the turn hung). Keyed by executionId;
|
|
221
228
|
// `seen` dedupes duplicate step_await re-emits within a batch. Cleared on
|
|
@@ -243,7 +250,7 @@ export class AgentWidgetSession {
|
|
|
243
250
|
this.wireDefaultWebMcpConfirm();
|
|
244
251
|
|
|
245
252
|
// Hydrate artifacts from config (mirrors `initialMessages`). Restored
|
|
246
|
-
// records are forced to `status: "complete"
|
|
253
|
+
// records are forced to `status: "complete"`: a mid-stream artifact should
|
|
247
254
|
// never reappear after a refresh with its skeleton still showing.
|
|
248
255
|
for (const rec of config.initialArtifacts ?? []) {
|
|
249
256
|
this.artifacts.set(rec.id, { ...rec, status: "complete" });
|
|
@@ -259,6 +266,29 @@ export class AgentWidgetSession {
|
|
|
259
266
|
this.emitArtifactsState();
|
|
260
267
|
}
|
|
261
268
|
this.callbacks.onStatusChanged(this.status);
|
|
269
|
+
this.prefetchRuntypeTts();
|
|
270
|
+
}
|
|
271
|
+
|
|
272
|
+
/**
|
|
273
|
+
* Warm the deferred Runtype TTS engine chunk at init so it's resolved before
|
|
274
|
+
* the first "Read aloud" click. Module fetch only — no AudioContext (that
|
|
275
|
+
* still waits for the user gesture on first playback). In the IIFE/CDN build
|
|
276
|
+
* this fetches the standalone `runtype-tts.js`; in bundler builds the module
|
|
277
|
+
* is already inlined, so this is a cheap no-op. Errors are swallowed: a failed
|
|
278
|
+
* prefetch just means the chunk loads on first click instead (or, on a hard
|
|
279
|
+
* failure, the browser-voice fallback kicks in).
|
|
280
|
+
*/
|
|
281
|
+
private prefetchRuntypeTts(): void {
|
|
282
|
+
const tts = this.config.textToSpeech;
|
|
283
|
+
if (tts?.provider !== "runtype" || tts.createEngine) return;
|
|
284
|
+
// Only warm the chunk when the engine will actually be built (same gate as
|
|
285
|
+
// createSpeechEngine) — an incomplete config (e.g. a proxy demo with no
|
|
286
|
+
// clientToken) falls back to the browser voice, so the fetch would be wasted.
|
|
287
|
+
const host = tts.host ?? this.config.apiUrl;
|
|
288
|
+
const agentId =
|
|
289
|
+
tts.agentId ?? this.config.voiceRecognition?.provider?.runtype?.agentId;
|
|
290
|
+
if (!host || !agentId || !this.config.clientToken) return;
|
|
291
|
+
void loadRuntypeTts().catch(() => {});
|
|
262
292
|
}
|
|
263
293
|
|
|
264
294
|
/**
|
|
@@ -342,7 +372,7 @@ export class AgentWidgetSession {
|
|
|
342
372
|
return this.voiceProvider?.isBargeInActive?.() ?? false;
|
|
343
373
|
}
|
|
344
374
|
|
|
345
|
-
/** Tear down the barge-in mic pipeline
|
|
375
|
+
/** Tear down the barge-in mic pipeline: "hang up" the always-on mic */
|
|
346
376
|
public async deactivateBargeIn(): Promise<void> {
|
|
347
377
|
if (this.voiceProvider?.deactivateBargeIn) {
|
|
348
378
|
await this.voiceProvider.deactivateBargeIn();
|
|
@@ -357,6 +387,84 @@ export class AgentWidgetSession {
|
|
|
357
387
|
// so browser TTS doesn't double-speak them
|
|
358
388
|
private ttsSpokenMessageIds = new Set<string>();
|
|
359
389
|
|
|
390
|
+
// Owns the per-message "Read aloud" action and the auto-speak path: which
|
|
391
|
+
// message is active, its play/pause state, and the speech engine. The engine
|
|
392
|
+
// is resolved lazily on first playback (inside the user gesture) so a hosted
|
|
393
|
+
// engine via `textToSpeech.createEngine` plugs in without changing this class.
|
|
394
|
+
private readAloud = new ReadAloudController(() => this.createSpeechEngine());
|
|
395
|
+
|
|
396
|
+
/**
|
|
397
|
+
* Resolve the speech engine behind read-aloud (and auto-speak):
|
|
398
|
+
* 1. An explicit `textToSpeech.createEngine` always wins (full BYO control).
|
|
399
|
+
* 2. `provider: 'runtype'` builds the hosted {@link RuntypeSpeechEngine}
|
|
400
|
+
* from the widget config (host = `apiUrl`, `agentId`, `clientToken`),
|
|
401
|
+
* wrapped in a browser {@link FallbackSpeechEngine} unless
|
|
402
|
+
* `browserFallback: false` — so a missing endpoint or transient failure
|
|
403
|
+
* degrades to the browser voice instead of erroring, and auto-upgrades to
|
|
404
|
+
* Runtype voices the day the endpoint ships.
|
|
405
|
+
* 3. Otherwise the browser Web Speech API engine.
|
|
406
|
+
*/
|
|
407
|
+
private createSpeechEngine(): SpeechEngine | Promise<SpeechEngine> | null {
|
|
408
|
+
const tts = this.config.textToSpeech;
|
|
409
|
+
if (tts?.createEngine) return tts.createEngine();
|
|
410
|
+
|
|
411
|
+
const browser = BrowserSpeechEngine.isSupported()
|
|
412
|
+
? new BrowserSpeechEngine({ pickVoice: tts?.pickVoice })
|
|
413
|
+
: null;
|
|
414
|
+
|
|
415
|
+
if (tts?.provider === "runtype") {
|
|
416
|
+
const host = tts.host ?? this.config.apiUrl;
|
|
417
|
+
const agentId =
|
|
418
|
+
tts.agentId ?? this.config.voiceRecognition?.provider?.runtype?.agentId;
|
|
419
|
+
const clientToken = this.config.clientToken;
|
|
420
|
+
const wantFallback = tts.browserFallback !== false;
|
|
421
|
+
|
|
422
|
+
if (host && agentId && clientToken) {
|
|
423
|
+
// Lazy-load the Runtype engine through the deferred loader: the IIFE/CDN
|
|
424
|
+
// build serves it from the standalone `runtype-tts.js` chunk (kept out of
|
|
425
|
+
// `index.global.js`), while bundler consumers get it inlined. Resolved on
|
|
426
|
+
// first playback (inside the user gesture); `createSpeechEngine` may
|
|
427
|
+
// return a Promise. Prefetched at init (see `prefetchRuntypeTts`) so the
|
|
428
|
+
// chunk is warm before the first click.
|
|
429
|
+
return loadRuntypeTts().then(
|
|
430
|
+
({ RuntypeSpeechEngine, FallbackSpeechEngine }) => {
|
|
431
|
+
const runtype = new RuntypeSpeechEngine({
|
|
432
|
+
host,
|
|
433
|
+
agentId,
|
|
434
|
+
clientToken,
|
|
435
|
+
voice: tts.voice,
|
|
436
|
+
prebufferMs: tts.prebufferMs,
|
|
437
|
+
createPlaybackEngine: tts.createPlaybackEngine,
|
|
438
|
+
});
|
|
439
|
+
return wantFallback && browser
|
|
440
|
+
? new FallbackSpeechEngine(runtype, browser, {
|
|
441
|
+
onFallback: (error) =>
|
|
442
|
+
console.warn(
|
|
443
|
+
`[persona] Runtype read-aloud failed; using browser voice. ${error.message}`,
|
|
444
|
+
),
|
|
445
|
+
})
|
|
446
|
+
: runtype;
|
|
447
|
+
},
|
|
448
|
+
);
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
// Runtype was requested but the config is incomplete (no agentId/token —
|
|
452
|
+
// e.g. a proxy-backed demo). Never break the button: use the browser
|
|
453
|
+
// voice. Only warn when intent is clear (a clientToken is present) so a
|
|
454
|
+
// plain proxy demo stays quiet.
|
|
455
|
+
if (wantFallback && browser) {
|
|
456
|
+
if (clientToken) {
|
|
457
|
+
console.warn(
|
|
458
|
+
"[persona] textToSpeech.provider 'runtype' is missing an agentId; using the browser voice. Set textToSpeech.agentId (or voiceRecognition.provider.runtype.agentId).",
|
|
459
|
+
);
|
|
460
|
+
}
|
|
461
|
+
return browser;
|
|
462
|
+
}
|
|
463
|
+
}
|
|
464
|
+
|
|
465
|
+
return browser;
|
|
466
|
+
}
|
|
467
|
+
|
|
360
468
|
/**
|
|
361
469
|
* Setup voice recognition with the given configuration
|
|
362
470
|
*/
|
|
@@ -371,92 +479,104 @@ export class AgentWidgetSession {
|
|
|
371
479
|
|
|
372
480
|
// Read configurable text from widget config
|
|
373
481
|
const voiceRecognitionConfig = this.config.voiceRecognition ?? {};
|
|
374
|
-
const processingText = voiceRecognitionConfig.processingText ?? '\u{1F3A4} Processing voice...';
|
|
375
482
|
const processingErrorText = voiceRecognitionConfig.processingErrorText ?? 'Voice processing failed. Please try again.';
|
|
376
483
|
|
|
377
|
-
//
|
|
378
|
-
//
|
|
379
|
-
//
|
|
380
|
-
//
|
|
381
|
-
if (this.voiceProvider.onProcessingStart) {
|
|
382
|
-
this.voiceProvider.onProcessingStart(() => {
|
|
383
|
-
// Inject user message placeholder
|
|
384
|
-
const userMsg = this.injectMessage({
|
|
385
|
-
role: 'user',
|
|
386
|
-
content: processingText,
|
|
387
|
-
streaming: false,
|
|
388
|
-
voiceProcessing: true
|
|
389
|
-
});
|
|
390
|
-
this.pendingVoiceUserMessageId = userMsg.id;
|
|
391
|
-
|
|
392
|
-
// Inject empty assistant message with streaming=true for typing indicator
|
|
393
|
-
const assistantMsg = this.injectMessage({
|
|
394
|
-
role: 'assistant',
|
|
395
|
-
content: '',
|
|
396
|
-
streaming: true,
|
|
397
|
-
voiceProcessing: true
|
|
398
|
-
});
|
|
399
|
-
this.pendingVoiceAssistantMessageId = assistantMsg.id;
|
|
400
|
-
|
|
401
|
-
// Trigger typing indicator in the UI
|
|
402
|
-
this.setStreaming(true);
|
|
403
|
-
});
|
|
404
|
-
}
|
|
405
|
-
|
|
406
|
-
// Phase B: When server responds with transcript + agent response,
|
|
407
|
-
// upsert the placeholder messages with actual content and clear voiceProcessing flag
|
|
484
|
+
// STT-style providers (browser + bring-your-own `custom`) deliver a final
|
|
485
|
+
// transcript that we send as a normal user message: the agent then runs
|
|
486
|
+
// via the standard SSE chat path. Only the realtime `runtype` provider is
|
|
487
|
+
// excluded here: it owns the whole turn and drives onTranscript below.
|
|
408
488
|
this.voiceProvider.onResult((result) => {
|
|
409
|
-
if (result.provider
|
|
410
|
-
// Browser STT: send transcript as a user message (agent runs via normal chat)
|
|
489
|
+
if (result.provider !== 'runtype') {
|
|
411
490
|
if (result.text && result.text.trim()) {
|
|
412
491
|
this.sendMessage(result.text, { viaVoice: true });
|
|
413
492
|
}
|
|
414
|
-
}
|
|
415
|
-
|
|
416
|
-
// is handled by the provider itself. Update placeholders with actual content.
|
|
417
|
-
if (this.pendingVoiceUserMessageId && result.transcript?.trim()) {
|
|
418
|
-
this.upsertMessage({
|
|
419
|
-
id: this.pendingVoiceUserMessageId,
|
|
420
|
-
role: 'user',
|
|
421
|
-
content: result.transcript.trim(),
|
|
422
|
-
createdAt: new Date().toISOString(),
|
|
423
|
-
streaming: false,
|
|
424
|
-
voiceProcessing: false
|
|
425
|
-
});
|
|
426
|
-
} else if (result.transcript?.trim()) {
|
|
427
|
-
this.injectUserMessage({ content: result.transcript.trim() });
|
|
428
|
-
}
|
|
493
|
+
}
|
|
494
|
+
});
|
|
429
495
|
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
496
|
+
// Realtime (runtype) voice: drive the chat thread from streaming
|
|
497
|
+
// transcript frames. Live interim user text grows in place; the user
|
|
498
|
+
// message finalizes immediately on transcript_final{user}; the assistant
|
|
499
|
+
// reply lands (a single block, synced with audio) on its final frame.
|
|
500
|
+
// In-flight bubbles carry voiceProcessing=true so consumers can style
|
|
501
|
+
// them via messageTransform; it clears once the text is final.
|
|
502
|
+
if (this.voiceProvider.onTranscript) {
|
|
503
|
+
this.voiceProvider.onTranscript((role, text, isFinal) => {
|
|
504
|
+
if (role === 'user') {
|
|
505
|
+
if (!this.pendingVoiceUserMessageId) {
|
|
506
|
+
const msg = this.injectMessage({
|
|
507
|
+
role: 'user',
|
|
508
|
+
content: text,
|
|
509
|
+
streaming: false,
|
|
510
|
+
voiceProcessing: !isFinal
|
|
511
|
+
});
|
|
512
|
+
this.pendingVoiceUserMessageId = msg.id;
|
|
513
|
+
} else {
|
|
514
|
+
this.upsertMessage({
|
|
515
|
+
id: this.pendingVoiceUserMessageId,
|
|
516
|
+
role: 'user',
|
|
517
|
+
content: text,
|
|
518
|
+
createdAt: new Date().toISOString(),
|
|
519
|
+
streaming: false,
|
|
520
|
+
voiceProcessing: !isFinal
|
|
521
|
+
});
|
|
522
|
+
}
|
|
442
523
|
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
524
|
+
if (isFinal) {
|
|
525
|
+
// User finished: the agent is now thinking. Release the user
|
|
526
|
+
// bubble (a new interim starts a fresh turn) and show a typing
|
|
527
|
+
// indicator in a fresh assistant placeholder.
|
|
528
|
+
this.pendingVoiceUserMessageId = null;
|
|
529
|
+
const assistantMsg = this.injectMessage({
|
|
530
|
+
role: 'assistant',
|
|
531
|
+
content: '',
|
|
532
|
+
streaming: true,
|
|
533
|
+
voiceProcessing: true
|
|
534
|
+
});
|
|
535
|
+
this.pendingVoiceAssistantMessageId = assistantMsg.id;
|
|
536
|
+
this.setStreaming(true);
|
|
537
|
+
}
|
|
538
|
+
} else {
|
|
539
|
+
// assistant: runtype sends a single final; the isFinal=false path
|
|
540
|
+
// is reserved for delta-streaming providers (future BYO).
|
|
541
|
+
if (this.pendingVoiceAssistantMessageId) {
|
|
542
|
+
this.upsertMessage({
|
|
543
|
+
id: this.pendingVoiceAssistantMessageId,
|
|
544
|
+
role: 'assistant',
|
|
545
|
+
content: text,
|
|
546
|
+
createdAt: new Date().toISOString(),
|
|
547
|
+
streaming: !isFinal,
|
|
548
|
+
voiceProcessing: !isFinal
|
|
549
|
+
});
|
|
550
|
+
} else {
|
|
551
|
+
const msg = this.injectMessage({
|
|
552
|
+
role: 'assistant',
|
|
553
|
+
content: text,
|
|
554
|
+
streaming: !isFinal,
|
|
555
|
+
voiceProcessing: !isFinal
|
|
556
|
+
});
|
|
557
|
+
this.pendingVoiceAssistantMessageId = msg.id;
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
if (isFinal) {
|
|
561
|
+
// The provider plays this reply's audio: mark it spoken so
|
|
562
|
+
// browser TTS doesn't double-speak when streaming ends. Must run
|
|
563
|
+
// BEFORE setStreaming(false), which triggers the TTS check.
|
|
564
|
+
if (this.pendingVoiceAssistantMessageId) {
|
|
565
|
+
this.ttsSpokenMessageIds.add(this.pendingVoiceAssistantMessageId);
|
|
566
|
+
}
|
|
567
|
+
this.setStreaming(false);
|
|
568
|
+
this.pendingVoiceAssistantMessageId = null;
|
|
569
|
+
}
|
|
452
570
|
}
|
|
571
|
+
});
|
|
572
|
+
}
|
|
453
573
|
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
this.
|
|
458
|
-
}
|
|
459
|
-
}
|
|
574
|
+
// Surface per-turn latency metrics to the optional config hook.
|
|
575
|
+
if (this.voiceProvider.onMetrics) {
|
|
576
|
+
this.voiceProvider.onMetrics((metrics) => {
|
|
577
|
+
this.config.voiceRecognition?.onMetrics?.(metrics);
|
|
578
|
+
});
|
|
579
|
+
}
|
|
460
580
|
|
|
461
581
|
this.voiceProvider.onError((error) => {
|
|
462
582
|
console.error('Voice error:', error);
|
|
@@ -540,11 +660,12 @@ export class AgentWidgetSession {
|
|
|
540
660
|
type: 'runtype',
|
|
541
661
|
runtype: {
|
|
542
662
|
agentId: providerConfig.runtype?.agentId || '',
|
|
543
|
-
|
|
544
|
-
|
|
663
|
+
// Default credentials/endpoint from the widget config so the minimum
|
|
664
|
+
// voice config collapses to just `agentId`.
|
|
665
|
+
clientToken: providerConfig.runtype?.clientToken ?? this.config.clientToken,
|
|
666
|
+
host: providerConfig.runtype?.host ?? this.config.apiUrl,
|
|
545
667
|
voiceId: providerConfig.runtype?.voiceId,
|
|
546
|
-
|
|
547
|
-
silenceThreshold: providerConfig.runtype?.silenceThreshold
|
|
668
|
+
createPlaybackEngine: providerConfig.runtype?.createPlaybackEngine
|
|
548
669
|
}
|
|
549
670
|
};
|
|
550
671
|
|
|
@@ -556,7 +677,15 @@ export class AgentWidgetSession {
|
|
|
556
677
|
continuous: providerConfig.browser?.continuous
|
|
557
678
|
}
|
|
558
679
|
};
|
|
559
|
-
|
|
680
|
+
|
|
681
|
+
case 'custom':
|
|
682
|
+
// Bring-your-own provider: pass the instance/factory straight through
|
|
683
|
+
// to the factory, which resolves and validates it.
|
|
684
|
+
return {
|
|
685
|
+
type: 'custom',
|
|
686
|
+
custom: providerConfig.custom
|
|
687
|
+
};
|
|
688
|
+
|
|
560
689
|
default:
|
|
561
690
|
return undefined;
|
|
562
691
|
}
|
|
@@ -690,7 +819,7 @@ export class AgentWidgetSession {
|
|
|
690
819
|
// parallel-await batch, and pending approval bubble tied to the OLD client/
|
|
691
820
|
// session. Tear them down BEFORE the swap (the new client has no session
|
|
692
821
|
// yet) so a deferred batch flush or a parked confirm can't fire against the
|
|
693
|
-
// fresh client
|
|
822
|
+
// fresh client: in client-token mode that would POST /resume without a
|
|
694
823
|
// valid sessionId and strand the paused turn. Mirrors clearMessages' WebMCP
|
|
695
824
|
// reset; the client swap already abandons any in-flight stream regardless.
|
|
696
825
|
this.abortWebMcpResolves();
|
|
@@ -900,7 +1029,7 @@ export class AgentWidgetSession {
|
|
|
900
1029
|
|
|
901
1030
|
/**
|
|
902
1031
|
* Convenience method for injecting a registered component directive as
|
|
903
|
-
* an assistant message
|
|
1032
|
+
* an assistant message: the same shape Persona produces from a streamed
|
|
904
1033
|
* `{ "text": "...", "component": "...", "props": {...} }` payload.
|
|
905
1034
|
*
|
|
906
1035
|
* Sets `content` to `text`, `rawContent` to the JSON directive (so
|
|
@@ -1193,7 +1322,7 @@ export class AgentWidgetSession {
|
|
|
1193
1322
|
* awaits this Promise before executing the page tool.
|
|
1194
1323
|
*/
|
|
1195
1324
|
public requestWebMcpApproval(info: WebMcpConfirmInfo): Promise<boolean> {
|
|
1196
|
-
// Per-tool policy hook
|
|
1325
|
+
// Per-tool policy hook: auto-allow opted-out tools without any UI. A
|
|
1197
1326
|
// throwing predicate must not block the call, so fall through to an
|
|
1198
1327
|
// explicit gate on error.
|
|
1199
1328
|
try {
|
|
@@ -1280,7 +1409,7 @@ export class AgentWidgetSession {
|
|
|
1280
1409
|
};
|
|
1281
1410
|
// Anchor the bubble where the agent paused for permission. An approval is a
|
|
1282
1411
|
// timeline checkpoint, not a "now" event, so resolving it must preserve the
|
|
1283
|
-
// original message's createdAt/sequence
|
|
1412
|
+
// original message's createdAt/sequence: otherwise sortMessages (which
|
|
1284
1413
|
// orders by createdAt first) would re-stamp it to now and float it past any
|
|
1285
1414
|
// message created later (e.g. a long-pending approval resolved after more
|
|
1286
1415
|
// conversation, or restored/replayed transcripts).
|
|
@@ -1352,7 +1481,7 @@ export class AgentWidgetSession {
|
|
|
1352
1481
|
await this.connectStream(stream, { allowReentry: true });
|
|
1353
1482
|
} else {
|
|
1354
1483
|
if (decision === 'denied') {
|
|
1355
|
-
// No stream body for denied
|
|
1484
|
+
// No stream body for denied: inject a denial message
|
|
1356
1485
|
this.appendMessage({
|
|
1357
1486
|
id: `denial-${approval.id}`,
|
|
1358
1487
|
role: "assistant",
|
|
@@ -1362,13 +1491,13 @@ export class AgentWidgetSession {
|
|
|
1362
1491
|
sequence: this.nextSequence(),
|
|
1363
1492
|
});
|
|
1364
1493
|
}
|
|
1365
|
-
// No body to pipe
|
|
1494
|
+
// No body to pipe: drop the pre-set streaming flag so the indicator
|
|
1366
1495
|
// doesn't linger forever.
|
|
1367
1496
|
this.setStreaming(false);
|
|
1368
1497
|
this.abortController = null;
|
|
1369
1498
|
}
|
|
1370
1499
|
} else {
|
|
1371
|
-
// onDecision returned void / no response
|
|
1500
|
+
// onDecision returned void / no response: drop the pre-set flag.
|
|
1372
1501
|
this.setStreaming(false);
|
|
1373
1502
|
this.abortController = null;
|
|
1374
1503
|
}
|
|
@@ -1430,7 +1559,7 @@ export class AgentWidgetSession {
|
|
|
1430
1559
|
* Flip an `ask_user_question` tool message from awaiting → answered so
|
|
1431
1560
|
* render passes stop re-mounting its answer-pill sheet. Idempotent.
|
|
1432
1561
|
* When `answers` is provided, persists the full structured answer Record
|
|
1433
|
-
* atomically with the answered flag
|
|
1562
|
+
* atomically with the answered flag: guarding against later events that
|
|
1434
1563
|
* could re-emit the tool message and clobber the per-pick persisted
|
|
1435
1564
|
* answers via top-level merge.
|
|
1436
1565
|
*/
|
|
@@ -1455,7 +1584,7 @@ export class AgentWidgetSession {
|
|
|
1455
1584
|
toolMessage: AgentWidgetMessage,
|
|
1456
1585
|
answer: string | Record<string, string | string[]>
|
|
1457
1586
|
): Promise<void> {
|
|
1458
|
-
// Idempotent
|
|
1587
|
+
// Idempotent: guards against rapid double-clicks on answer pills before
|
|
1459
1588
|
// the re-render swaps the card to its collapsed/answered state.
|
|
1460
1589
|
const live = this.messages.find((m) => m.id === toolMessage.id);
|
|
1461
1590
|
if (live?.agentMetadata?.askUserQuestionAnswered === true) return;
|
|
@@ -1475,7 +1604,7 @@ export class AgentWidgetSession {
|
|
|
1475
1604
|
// avoiding the race between removeAskUserQuestionSheet's 180ms slide-out
|
|
1476
1605
|
// timer and the renders that fire as the resume stream lands. Pass the
|
|
1477
1606
|
// structured answer Record (when present) so it's atomically persisted
|
|
1478
|
-
// alongside the flag
|
|
1607
|
+
// alongside the flag: the answered-state review card depends on
|
|
1479
1608
|
// `agentMetadata.askUserQuestionAnswers` being populated at render time.
|
|
1480
1609
|
//
|
|
1481
1610
|
// For single-question payloads, callers (built-in pick handler, plugins)
|
|
@@ -1498,7 +1627,7 @@ export class AgentWidgetSession {
|
|
|
1498
1627
|
}
|
|
1499
1628
|
this.markAskUserQuestionResolved(toolMessage, structuredAnswers);
|
|
1500
1629
|
|
|
1501
|
-
// Show the standalone typing indicator immediately
|
|
1630
|
+
// Show the standalone typing indicator immediately: the network round-trip
|
|
1502
1631
|
// to /resume is otherwise silent, which reads as broken. The render
|
|
1503
1632
|
// condition in ui.ts already shows the indicator once streaming flips true
|
|
1504
1633
|
// and the last message is a user bubble (the answer we inject below).
|
|
@@ -1507,8 +1636,8 @@ export class AgentWidgetSession {
|
|
|
1507
1636
|
this.abortController = new AbortController();
|
|
1508
1637
|
this.setStreaming(true);
|
|
1509
1638
|
|
|
1510
|
-
// Inject Q→A pair messages
|
|
1511
|
-
// bubble per answer
|
|
1639
|
+
// Inject Q→A pair messages: one assistant bubble per question, one user
|
|
1640
|
+
// bubble per answer, so the transcript reads like a normal conversation.
|
|
1512
1641
|
// The original ask_user_question tool message is suppressed by the
|
|
1513
1642
|
// renderer once `askUserQuestionAnswered` is true. Skipped questions get
|
|
1514
1643
|
// a muted italic `*Skipped*` user bubble (rendered through the standard
|
|
@@ -1580,14 +1709,14 @@ export class AgentWidgetSession {
|
|
|
1580
1709
|
if (response.body) {
|
|
1581
1710
|
await this.connectStream(response.body, { allowReentry: true });
|
|
1582
1711
|
} else {
|
|
1583
|
-
// No body to pipe
|
|
1712
|
+
// No body to pipe: drop the pre-set streaming flag so the indicator
|
|
1584
1713
|
// doesn't linger forever.
|
|
1585
1714
|
this.setStreaming(false);
|
|
1586
1715
|
this.abortController = null;
|
|
1587
1716
|
}
|
|
1588
1717
|
} catch (error) {
|
|
1589
1718
|
// Mirror sendMessage: a cancel() during the await aborts the controller
|
|
1590
|
-
// and surfaces an AbortError
|
|
1719
|
+
// and surfaces an AbortError: don't treat that as a real failure.
|
|
1591
1720
|
const isAbortError =
|
|
1592
1721
|
error instanceof Error &&
|
|
1593
1722
|
(error.name === 'AbortError' ||
|
|
@@ -1620,7 +1749,7 @@ export class AgentWidgetSession {
|
|
|
1620
1749
|
* before a resolve grabs it.
|
|
1621
1750
|
*
|
|
1622
1751
|
* Awaits without an `executionId` or `toolCall.id` can't be batched (no key)
|
|
1623
|
-
|
|
1752
|
+
*: route them straight to the single-call path, which surfaces the malformed
|
|
1624
1753
|
* wire shape via `onError` / an `isError` resume.
|
|
1625
1754
|
*/
|
|
1626
1755
|
private enqueueWebMcpAwait(toolMessage: AgentWidgetMessage): void {
|
|
@@ -1640,7 +1769,7 @@ export class AgentWidgetSession {
|
|
|
1640
1769
|
batch = { snapshots: [], seen: new Set() };
|
|
1641
1770
|
this.webMcpAwaitBatches.set(executionId, batch);
|
|
1642
1771
|
}
|
|
1643
|
-
// Duplicate step_await re-emit for a call already in this batch
|
|
1772
|
+
// Duplicate step_await re-emit for a call already in this batch: ignore.
|
|
1644
1773
|
if (batch.seen.has(callId)) return;
|
|
1645
1774
|
batch.seen.add(callId);
|
|
1646
1775
|
batch.snapshots.push(toolMessage);
|
|
@@ -1654,11 +1783,11 @@ export class AgentWidgetSession {
|
|
|
1654
1783
|
|
|
1655
1784
|
/**
|
|
1656
1785
|
* Flush every buffered local-tool await batch, one `/resume` per executionId.
|
|
1657
|
-
* Called once a stream ends (`status: idle` / `error`)
|
|
1786
|
+
* Called once a stream ends (`status: idle` / `error`): by then all parallel
|
|
1658
1787
|
* `step_await`s the stream carried have been collected, even if split across
|
|
1659
1788
|
* SSE chunks. Deferred via `queueMicrotask` (epoch-guarded) so the idle
|
|
1660
1789
|
* handler returns first and the stream's end-of-stream teardown (streaming /
|
|
1661
|
-
* abortController) settles before a resolve grabs them
|
|
1790
|
+
* abortController) settles before a resolve grabs them: the same ordering the
|
|
1662
1791
|
* single-call resolve always relied on.
|
|
1663
1792
|
*/
|
|
1664
1793
|
private scheduleWebMcpBatchFlush(): void {
|
|
@@ -1712,7 +1841,7 @@ export class AgentWidgetSession {
|
|
|
1712
1841
|
* Persisted-resolution guard for `suggest_replies`. The in-memory dedupe
|
|
1713
1842
|
* sets (`webMcpInflightKeys` / `webMcpResolvedKeys`) are cleared by
|
|
1714
1843
|
* hydrateMessages/clearMessages/cancel, but `suggestRepliesResolved`
|
|
1715
|
-
* survives on the stored message
|
|
1844
|
+
* survives on the stored message, so a stale `step_await` re-emit after a
|
|
1716
1845
|
* hydration must not re-POST `/resume` for an already-resolved call (the
|
|
1717
1846
|
* historical double-resume failure mode the batching work exists to avoid).
|
|
1718
1847
|
* Checks the LIVE message first; the handleEvent snapshot is a fresh wire
|
|
@@ -1791,7 +1920,7 @@ export class AgentWidgetSession {
|
|
|
1791
1920
|
/**
|
|
1792
1921
|
* Resolve TWO OR MORE parallel local-tool awaits sharing one paused
|
|
1793
1922
|
* executionId with a SINGLE `/resume` (core#3878). Each call is executed
|
|
1794
|
-
* against the page registry concurrently
|
|
1923
|
+
* against the page registry concurrently: every gated call renders its own
|
|
1795
1924
|
* native approval bubble, and a sibling's confirm Promise never blocks
|
|
1796
1925
|
* another's execution. Outputs are keyed by per-call `webMcpToolCallId`
|
|
1797
1926
|
* (server prefers it over tool name; name-keying remains the fallback for
|
|
@@ -1822,7 +1951,7 @@ export class AgentWidgetSession {
|
|
|
1822
1951
|
this.webMcpResolveControllers.add(resumeController);
|
|
1823
1952
|
this.setStreaming(true);
|
|
1824
1953
|
|
|
1825
|
-
// Phase 1
|
|
1954
|
+
// Phase 1: execute every pending call concurrently. A null result means
|
|
1826
1955
|
// the call was deduped, aborted, or threw; it's omitted from the resume and
|
|
1827
1956
|
// (per the tolerant server) re-pauses for retry.
|
|
1828
1957
|
const executed = await Promise.all(
|
|
@@ -1853,7 +1982,7 @@ export class AgentWidgetSession {
|
|
|
1853
1982
|
toolMessage.agentMetadata?.webMcpToolCallId ?? wireToolName;
|
|
1854
1983
|
|
|
1855
1984
|
// Built-in fire-and-forget tool: no bridge, no confirm gate, no
|
|
1856
|
-
// browser-side execution
|
|
1985
|
+
// browser-side execution: the chips render from the message list and
|
|
1857
1986
|
// the canned output joins the batch's single /resume.
|
|
1858
1987
|
if (wireToolName === SUGGEST_REPLIES_TOOL_NAME) {
|
|
1859
1988
|
return {
|
|
@@ -1935,14 +2064,13 @@ export class AgentWidgetSession {
|
|
|
1935
2064
|
let ready: ExecutedWebMcpTool[] = [];
|
|
1936
2065
|
try {
|
|
1937
2066
|
ready = executed.filter((r): r is ExecutedWebMcpTool => r !== null);
|
|
1938
|
-
// Everything deduped/aborted/failed
|
|
2067
|
+
// Everything deduped/aborted/failed: nothing to post.
|
|
1939
2068
|
if (ready.length === 0) return;
|
|
1940
2069
|
|
|
1941
2070
|
const toolOutputs: Record<string, unknown> = {};
|
|
1942
2071
|
for (const r of ready) {
|
|
1943
2072
|
// Two omitted-on-collision safety: if two calls somehow resolve to the
|
|
1944
|
-
// same key (only possible on a legacy name fallback), last write wins
|
|
1945
|
-
// the server re-pauses the unrepresented call for retry.
|
|
2073
|
+
// same key (only possible on a legacy name fallback), last write wins: // the server re-pauses the unrepresented call for retry.
|
|
1946
2074
|
toolOutputs[r.resumeKey] = r.output;
|
|
1947
2075
|
}
|
|
1948
2076
|
|
|
@@ -1953,7 +2081,7 @@ export class AgentWidgetSession {
|
|
|
1953
2081
|
const errorData = await response.json().catch(() => null);
|
|
1954
2082
|
throw new Error(errorData?.error ?? `Resume failed: ${response.status}`);
|
|
1955
2083
|
}
|
|
1956
|
-
// Server accepted the batch
|
|
2084
|
+
// Server accepted the batch: mark every included call resolved so stale
|
|
1957
2085
|
// re-emits don't re-execute the page tool, then complete each bubble.
|
|
1958
2086
|
// Do this only after /resume HTTP success; if /resume fails, the server
|
|
1959
2087
|
// may still be paused and the retry path must not show a final result.
|
|
@@ -2013,13 +2141,13 @@ export class AgentWidgetSession {
|
|
|
2013
2141
|
* list, not from this resolve).
|
|
2014
2142
|
*
|
|
2015
2143
|
* Triggered automatically from `handleEvent` when a `step_await`-derived
|
|
2016
|
-
* message arrives for such a tool
|
|
2144
|
+
* message arrives for such a tool: the user does not click a pill; the
|
|
2017
2145
|
* bridge's confirm-bubble gate (WebMCP only) is the only interactive
|
|
2018
2146
|
* surface.
|
|
2019
2147
|
*
|
|
2020
2148
|
* Idempotent on the message's `toolCall.id`: re-emits of the same step_await
|
|
2021
2149
|
* (e.g. from message coalescing) won't double-fire `tool.execute`. Failure
|
|
2022
|
-
* modes
|
|
2150
|
+
* modes, declined, timed out, throw, unknown tool, all resolve into a
|
|
2023
2151
|
* `{ isError: true, content: [...] }` payload that resumes the dispatch
|
|
2024
2152
|
* cleanly so the agent can recover.
|
|
2025
2153
|
*/
|
|
@@ -2034,8 +2162,8 @@ export class AgentWidgetSession {
|
|
|
2034
2162
|
// server-side dispatch. Three failure modes:
|
|
2035
2163
|
// - no executionId: no /resume target exists; surface to the host
|
|
2036
2164
|
// via onError so an operator can react. This is a server-side
|
|
2037
|
-
// wire-shape bug
|
|
2038
|
-
// - no wireToolName: defensive guard
|
|
2165
|
+
// wire-shape bug: Persona can't recover it from the client.
|
|
2166
|
+
// - no wireToolName: defensive guard: handleEvent only calls us
|
|
2039
2167
|
// for an auto-resolving local tool name (`webmcp:*` or
|
|
2040
2168
|
// `suggest_replies`), so this path indicates a direct caller
|
|
2041
2169
|
// misuse. Silent return.
|
|
@@ -2045,7 +2173,7 @@ export class AgentWidgetSession {
|
|
|
2045
2173
|
if (!executionId) {
|
|
2046
2174
|
this.callbacks.onError?.(
|
|
2047
2175
|
new Error(
|
|
2048
|
-
"WebMCP step_await missing executionId
|
|
2176
|
+
"WebMCP step_await missing executionId: dispatch left paused.",
|
|
2049
2177
|
),
|
|
2050
2178
|
);
|
|
2051
2179
|
return;
|
|
@@ -2069,7 +2197,7 @@ export class AgentWidgetSession {
|
|
|
2069
2197
|
content: [
|
|
2070
2198
|
{
|
|
2071
2199
|
type: "text",
|
|
2072
|
-
text: "WebMCP step_await missing toolCall.id
|
|
2200
|
+
text: "WebMCP step_await missing toolCall.id: cannot execute the page tool.",
|
|
2073
2201
|
},
|
|
2074
2202
|
],
|
|
2075
2203
|
});
|
|
@@ -2084,7 +2212,7 @@ export class AgentWidgetSession {
|
|
|
2084
2212
|
return;
|
|
2085
2213
|
}
|
|
2086
2214
|
|
|
2087
|
-
// Dedupe key scoped by executionId
|
|
2215
|
+
// Dedupe key scoped by executionId: see `webMcpInflightKeys` doc comment
|
|
2088
2216
|
// for the failure-recovery + cross-dispatch rationale. The persisted
|
|
2089
2217
|
// `suggestRepliesResolved` guard backs the in-memory sets across
|
|
2090
2218
|
// hydrations.
|
|
@@ -2099,20 +2227,19 @@ export class AgentWidgetSession {
|
|
|
2099
2227
|
this.webMcpInflightKeys.add(dedupeKey);
|
|
2100
2228
|
|
|
2101
2229
|
// Mark resolved on the message so the UI's local-tool sheet (if any
|
|
2102
|
-
// generic one ever lands) does not show
|
|
2230
|
+
// generic one ever lands) does not show: this is a fully-automatic
|
|
2103
2231
|
// tool from the user's perspective, modulo the confirm bubble. Keep the
|
|
2104
2232
|
// tool bubble running until the browser-side promise resolves; the
|
|
2105
2233
|
// initial step_await was only the server pause, not tool completion.
|
|
2106
2234
|
const startedAt = this.markWebMcpToolRunning(toolMessage);
|
|
2107
2235
|
|
|
2108
2236
|
// Per-resolve AbortController, NOT the shared `this.abortController`.
|
|
2109
|
-
// A single turn can produce multiple `webmcp:*` step_await messages
|
|
2110
|
-
// both PARALLEL (two awaits in one stream) and, more commonly, CHAINED
|
|
2237
|
+
// A single turn can produce multiple `webmcp:*` step_await messages: // both PARALLEL (two awaits in one stream) and, more commonly, CHAINED
|
|
2111
2238
|
// (tool A → /resume → tool B, where B's step_await arrives inside A's
|
|
2112
2239
|
// resume SSE stream). The old code pre-aborted `this.abortController`
|
|
2113
2240
|
// here to mirror the sibling resolve paths; in the chained case that
|
|
2114
2241
|
// aborted the stream still delivering B, so B never executed and its
|
|
2115
|
-
// /resume was never POSTed
|
|
2242
|
+
// /resume was never POSTed: the dispatch hung forever. Using a dedicated
|
|
2116
2243
|
// per-resolve controller leaves the in-flight resume stream untouched.
|
|
2117
2244
|
// cancel()/clearMessages()/hydrateMessages()/sendMessage() iterate
|
|
2118
2245
|
// `webMcpResolveControllers` to tear these down on a real stop / new turn.
|
|
@@ -2122,13 +2249,13 @@ export class AgentWidgetSession {
|
|
|
2122
2249
|
this.setStreaming(true);
|
|
2123
2250
|
|
|
2124
2251
|
// Built-in fire-and-forget tool: no bridge, no confirm gate, no
|
|
2125
|
-
// browser-side execution
|
|
2252
|
+
// browser-side execution: the chips render from the message list and the
|
|
2126
2253
|
// canned output resumes the execution immediately. Branch BEFORE any
|
|
2127
2254
|
// bridge access so the missing-bridge error path can never fire for it.
|
|
2128
2255
|
const isSuggestReplies = wireToolName === SUGGEST_REPLIES_TOOL_NAME;
|
|
2129
2256
|
|
|
2130
2257
|
const args = toolMessage.toolCall?.args;
|
|
2131
|
-
// Thread the signal INTO the bridge
|
|
2258
|
+
// Thread the signal INTO the bridge: short-circuits the confirm bubble
|
|
2132
2259
|
// and the execute() race on cancel(), so a late confirm-approval after
|
|
2133
2260
|
// cancel() cannot fire a host-page side effect with no matching /resume.
|
|
2134
2261
|
const execPromise = isSuggestReplies
|
|
@@ -2155,7 +2282,7 @@ export class AgentWidgetSession {
|
|
|
2155
2282
|
}
|
|
2156
2283
|
completedAt = Date.now();
|
|
2157
2284
|
// If cancel() fired during execute, the bridge returned an aborted
|
|
2158
|
-
// result
|
|
2285
|
+
// result: don't post it. The server's SSE has been torn down; a
|
|
2159
2286
|
// /resume now would just produce an orphan dispatch on the server.
|
|
2160
2287
|
// Streaming/teardown is handled by the shared `finally` below (gated on
|
|
2161
2288
|
// the resolve set) so we don't clobber a sibling resolve or a live
|
|
@@ -2168,7 +2295,7 @@ export class AgentWidgetSession {
|
|
|
2168
2295
|
);
|
|
2169
2296
|
return;
|
|
2170
2297
|
}
|
|
2171
|
-
// Mark resolved as soon as the HTTP /resume returns OK
|
|
2298
|
+
// Mark resolved as soon as the HTTP /resume returns OK: not after the
|
|
2172
2299
|
// SSE stream finishes. `connectStream` swallows downstream SSE errors
|
|
2173
2300
|
// (they surface via onError, not by rethrowing), so awaiting it doesn't
|
|
2174
2301
|
// tell us whether the server actually processed the resume. Marking
|
|
@@ -2177,7 +2304,7 @@ export class AgentWidgetSession {
|
|
|
2177
2304
|
// toolCall.id are stale and must not re-execute the page tool.
|
|
2178
2305
|
// Key the resume by the per-call id (core#3878) when present; the server
|
|
2179
2306
|
// prefers it over tool name. Falls back to the wire tool name for legacy
|
|
2180
|
-
// servers
|
|
2307
|
+
// servers: the original name-keyed contract, still correct for a single
|
|
2181
2308
|
// call (only same-tool PARALLEL calls could collide on the name).
|
|
2182
2309
|
const resumeKey =
|
|
2183
2310
|
toolMessage.agentMetadata?.webMcpToolCallId ?? wireToolName;
|
|
@@ -2202,7 +2329,7 @@ export class AgentWidgetSession {
|
|
|
2202
2329
|
error.message.includes("aborted") ||
|
|
2203
2330
|
error.message.includes("abort"));
|
|
2204
2331
|
// Streaming/teardown handled by the shared `finally` (gated on the
|
|
2205
|
-
// resolve set)
|
|
2332
|
+
// resolve set): do NOT null the shared `this.abortController` here; it
|
|
2206
2333
|
// may belong to a live dispatch or sibling resolve, not to us.
|
|
2207
2334
|
if (phase === "execute" || isAbortError || signal.aborted) {
|
|
2208
2335
|
this.markWebMcpToolComplete(
|
|
@@ -2218,7 +2345,7 @@ export class AgentWidgetSession {
|
|
|
2218
2345
|
if (!isAbortError) {
|
|
2219
2346
|
// The bridge normalizes tool errors into result objects, so reaching
|
|
2220
2347
|
// here means a network failure during `/resume` itself, OR a stream
|
|
2221
|
-
// hookup error. Surface to onError, but DO NOT mark resolved
|
|
2348
|
+
// hookup error. Surface to onError, but DO NOT mark resolved: a
|
|
2222
2349
|
// later step_await re-emit should be allowed to retry the resume.
|
|
2223
2350
|
this.callbacks.onError?.(
|
|
2224
2351
|
error instanceof Error ? error : new Error(String(error)),
|
|
@@ -2230,7 +2357,7 @@ export class AgentWidgetSession {
|
|
|
2230
2357
|
// Only flip streaming off when this was the last in-flight resolve AND
|
|
2231
2358
|
// no shared dispatch is live. Otherwise a finishing resolve would hide
|
|
2232
2359
|
// the typing indicator while a sibling (parallel) or successor (chained)
|
|
2233
|
-
// resolve
|
|
2360
|
+
// resolve, or a live dispatch, is still running.
|
|
2234
2361
|
if (this.webMcpResolveControllers.size === 0 && !this.abortController) {
|
|
2235
2362
|
this.setStreaming(false);
|
|
2236
2363
|
}
|
|
@@ -2269,7 +2396,7 @@ export class AgentWidgetSession {
|
|
|
2269
2396
|
await this.connectStream(response.body, { allowReentry: true });
|
|
2270
2397
|
} else if (this.webMcpResolveControllers.size === 0) {
|
|
2271
2398
|
// No stream to pipe. Clear streaming only when no WebMCP resolve is in
|
|
2272
|
-
// flight
|
|
2399
|
+
// flight: for a WebMCP caller the current resolve's controller is still
|
|
2273
2400
|
// in the set, so its own `finally` (gated on the set draining) owns the
|
|
2274
2401
|
// teardown. Non-WebMCP callers (ask_user_question) keep the old behavior.
|
|
2275
2402
|
this.setStreaming(false);
|
|
@@ -2282,8 +2409,7 @@ export class AgentWidgetSession {
|
|
|
2282
2409
|
* resolve owns a dedicated AbortController (chained/parallel resolves don't
|
|
2283
2410
|
* share one), so we abort them individually; the aborts propagate into the
|
|
2284
2411
|
* bridge's execute race and into each `/resume` fetch signal. Bumping
|
|
2285
|
-
* `webMcpEpoch` strands any resolve still deferred in a queued microtask
|
|
2286
|
-
* it captured the prior epoch and bails before installing a fresh
|
|
2412
|
+
* `webMcpEpoch` strands any resolve still deferred in a queued microtask: * it captured the prior epoch and bails before installing a fresh
|
|
2287
2413
|
* controller, so it can't escape this teardown. Called from every stop /
|
|
2288
2414
|
* new-turn boundary (cancel, clearMessages, hydrateMessages, sendMessage).
|
|
2289
2415
|
*/
|
|
@@ -2294,7 +2420,7 @@ export class AgentWidgetSession {
|
|
|
2294
2420
|
this.webMcpResolveControllers.clear();
|
|
2295
2421
|
// Settle every approval bubble still awaiting a click. The bridge parks a
|
|
2296
2422
|
// resolve on `await requestConfirm(...)` (→ requestWebMcpApproval) and only
|
|
2297
|
-
// re-checks `signal.aborted` AFTER that await returns
|
|
2423
|
+
// re-checks `signal.aborted` AFTER that await returns: aborting the
|
|
2298
2424
|
// controller above does NOT unblock it. Left unsettled, the bridge's
|
|
2299
2425
|
// execute(), its `/resume`, and the resolve's `finally` would all hang
|
|
2300
2426
|
// forever (and the resolver map would leak across teardowns). Route through
|
|
@@ -2302,12 +2428,12 @@ export class AgentWidgetSession {
|
|
|
2302
2428
|
// `false` AND its bubble message flips out of `pending` (no stale "Approve/
|
|
2303
2429
|
// Deny" left clickable). The bridge then returns cleanly and its
|
|
2304
2430
|
// post-confirm `signal.aborted` guard bails before any host-page side effect
|
|
2305
|
-
// or stale `/resume`. Snapshot the keys first
|
|
2431
|
+
// or stale `/resume`. Snapshot the keys first: resolveWebMcpApproval
|
|
2306
2432
|
// mutates the map as it deletes each resolver.
|
|
2307
2433
|
for (const approvalMessageId of [...this.webMcpApprovalResolvers.keys()]) {
|
|
2308
2434
|
this.resolveWebMcpApproval(approvalMessageId, "denied");
|
|
2309
2435
|
}
|
|
2310
|
-
// Drop any awaits buffered for a not-yet-flushed batch
|
|
2436
|
+
// Drop any awaits buffered for a not-yet-flushed batch: their messages are
|
|
2311
2437
|
// being torn down, and a microtask-deferred flush must not survive. The
|
|
2312
2438
|
// epoch bump below also strands an already-scheduled flush.
|
|
2313
2439
|
this.webMcpAwaitBatches.clear();
|
|
@@ -2322,7 +2448,7 @@ export class AgentWidgetSession {
|
|
|
2322
2448
|
// are possible if the user re-issues the same step_await context.
|
|
2323
2449
|
this.abortWebMcpResolves();
|
|
2324
2450
|
this.webMcpInflightKeys.clear();
|
|
2325
|
-
// Stop any in-progress audio too
|
|
2451
|
+
// Stop any in-progress audio too: when the user hits "stop", they want
|
|
2326
2452
|
// the assistant to actually stop talking, not just stop generating tokens.
|
|
2327
2453
|
// Both helpers are safe no-ops when audio isn't configured.
|
|
2328
2454
|
this.stopSpeaking();
|
|
@@ -2335,19 +2461,19 @@ export class AgentWidgetSession {
|
|
|
2335
2461
|
this.stopSpeaking();
|
|
2336
2462
|
this.abortController?.abort();
|
|
2337
2463
|
this.abortController = null;
|
|
2338
|
-
// Tear down every in-flight WebMCP resolve too
|
|
2464
|
+
// Tear down every in-flight WebMCP resolve too: their messages are about
|
|
2339
2465
|
// to be wiped, and a microtask-deferred resolve must not survive the clear.
|
|
2340
2466
|
this.abortWebMcpResolves();
|
|
2341
2467
|
this.messages = [];
|
|
2342
2468
|
this.agentExecution = null;
|
|
2343
2469
|
this.clearArtifactState();
|
|
2344
|
-
// Clearing messages also wipes the WebMCP dedupe state
|
|
2470
|
+
// Clearing messages also wipes the WebMCP dedupe state: a fresh
|
|
2345
2471
|
// conversation should not refuse to call a webmcp:* tool just because
|
|
2346
2472
|
// a tool with the same key resolved in the prior conversation.
|
|
2347
2473
|
this.webMcpInflightKeys.clear();
|
|
2348
2474
|
this.webMcpResolvedKeys.clear();
|
|
2349
2475
|
// A fresh conversation must resend the full WebMCP tool list on its next
|
|
2350
|
-
// turn
|
|
2476
|
+
// turn: drop the diff-only fingerprint cache (server keys by recordId, so
|
|
2351
2477
|
// a new conversation has no stored set to match).
|
|
2352
2478
|
this.client.resetClientToolsFingerprint();
|
|
2353
2479
|
this.setStreaming(false);
|
|
@@ -2474,10 +2600,10 @@ export class AgentWidgetSession {
|
|
|
2474
2600
|
public hydrateMessages(messages: AgentWidgetMessage[]) {
|
|
2475
2601
|
this.abortController?.abort();
|
|
2476
2602
|
this.abortController = null;
|
|
2477
|
-
// Hydration replaces the conversation
|
|
2603
|
+
// Hydration replaces the conversation: abort and forget every in-flight
|
|
2478
2604
|
// WebMCP resolve; their messages are about to be replaced.
|
|
2479
2605
|
this.abortWebMcpResolves();
|
|
2480
|
-
// Wipe the WebMCP dedupe state alongside the message restore
|
|
2606
|
+
// Wipe the WebMCP dedupe state alongside the message restore: the
|
|
2481
2607
|
// incoming snapshot is treated as a fresh conversation context.
|
|
2482
2608
|
this.webMcpInflightKeys.clear();
|
|
2483
2609
|
this.webMcpResolvedKeys.clear();
|
|
@@ -2510,8 +2636,8 @@ export class AgentWidgetSession {
|
|
|
2510
2636
|
this.upsertMessage(event.message);
|
|
2511
2637
|
|
|
2512
2638
|
// Local-tool auto-resolve: when a step_await emits a tool-variant
|
|
2513
|
-
// message for a `webmcp:*` tool
|
|
2514
|
-
// `suggest_replies
|
|
2639
|
+
// message for a `webmcp:*` tool, or the built-in fire-and-forget
|
|
2640
|
+
// `suggest_replies`: resolve it and post the result to /resume.
|
|
2515
2641
|
// Unlike ask_user_question, no user pill click is required; for WebMCP
|
|
2516
2642
|
// the bridge's confirm bubble is the only interactive surface, and
|
|
2517
2643
|
// suggest_replies resumes with a canned "shown" result while the chips
|
|
@@ -2527,11 +2653,11 @@ export class AgentWidgetSession {
|
|
|
2527
2653
|
//
|
|
2528
2654
|
// ALWAYS resolve when the wire name carries the `webmcp:` prefix, even
|
|
2529
2655
|
// if the bridge is non-operational. Otherwise the dispatch stays paused
|
|
2530
|
-
// indefinitely
|
|
2656
|
+
// indefinitely: `resolveWebMcpToolCall` translates the missing-bridge
|
|
2531
2657
|
// case into an isError result that resumes the flow cleanly.
|
|
2532
2658
|
// `suggest_replies`, by contrast, is gated on its feature flag: when
|
|
2533
2659
|
// `features.suggestReplies.enabled === false` the widget neither
|
|
2534
|
-
// renders chips nor resumes
|
|
2660
|
+
// renders chips nor resumes: the same parked-execution posture as a
|
|
2535
2661
|
// server-declared ask_user_question with its sheet disabled.
|
|
2536
2662
|
const tc = event.message.toolCall;
|
|
2537
2663
|
const autoResolvable =
|
|
@@ -2578,12 +2704,12 @@ export class AgentWidgetSession {
|
|
|
2578
2704
|
this.setStreaming(false);
|
|
2579
2705
|
this.abortController = null;
|
|
2580
2706
|
}
|
|
2581
|
-
// Mark agent execution as complete when streaming ends
|
|
2707
|
+
// Mark agent execution as complete when streaming ends: UNLESS local
|
|
2582
2708
|
// tools are still outstanding. A batched WebMCP resume is deferred to
|
|
2583
2709
|
// the microtask below (so `webMcpResolveControllers` is still empty
|
|
2584
2710
|
// here) and a chained resolve may be mid-flight; marking the run
|
|
2585
2711
|
// 'complete' now would make isAgentRunning() report a finished run while
|
|
2586
|
-
// page tools are still executing. Stay 'running'
|
|
2712
|
+
// page tools are still executing. Stay 'running': the resume stream's
|
|
2587
2713
|
// own idle (with batches drained and resolves settled) marks it done.
|
|
2588
2714
|
const webMcpPending =
|
|
2589
2715
|
this.webMcpAwaitBatches.size > 0 ||
|
|
@@ -2597,7 +2723,7 @@ export class AgentWidgetSession {
|
|
|
2597
2723
|
}
|
|
2598
2724
|
// The stream that delivered any local-tool `step_await`s has now ended,
|
|
2599
2725
|
// so every parallel await it carried is collected. Flush them as ONE
|
|
2600
|
-
// batched `/resume` per executionId (deferred
|
|
2726
|
+
// batched `/resume` per executionId (deferred: see
|
|
2601
2727
|
// scheduleWebMcpBatchFlush). Runs AFTER the teardown above so a resolve
|
|
2602
2728
|
// doesn't fight the end-of-stream streaming/abortController reset.
|
|
2603
2729
|
this.scheduleWebMcpBatchFlush();
|
|
@@ -2605,7 +2731,7 @@ export class AgentWidgetSession {
|
|
|
2605
2731
|
} else if (event.type === "error") {
|
|
2606
2732
|
this.setStatus("error");
|
|
2607
2733
|
// Mirror the idle/status handler: don't tear down streaming while a
|
|
2608
|
-
// WebMCP resolve is still confirming/executing on another stream
|
|
2734
|
+
// WebMCP resolve is still confirming/executing on another stream: an
|
|
2609
2735
|
// error on one chained resume stream must not hide the typing indicator
|
|
2610
2736
|
// (or null a controller) for a sibling/successor resolve still in flight.
|
|
2611
2737
|
// The resolve's own `finally` flips streaming off once the set drains.
|
|
@@ -2675,84 +2801,65 @@ export class AgentWidgetSession {
|
|
|
2675
2801
|
return;
|
|
2676
2802
|
}
|
|
2677
2803
|
|
|
2678
|
-
const text = lastAssistant.content;
|
|
2804
|
+
const text = resolveSpeakableText(lastAssistant.content);
|
|
2679
2805
|
if (!text.trim()) return;
|
|
2680
2806
|
|
|
2681
|
-
|
|
2807
|
+
// Route auto-speak through the same controller as the "Read aloud" button
|
|
2808
|
+
// so there's a single owner of the speech engine and the button reflects
|
|
2809
|
+
// playback state (and a single message can't be spoken by two paths at once).
|
|
2810
|
+
void this.readAloud.play(lastAssistant.id, {
|
|
2811
|
+
text,
|
|
2812
|
+
voice: ttsConfig.voice,
|
|
2813
|
+
rate: ttsConfig.rate,
|
|
2814
|
+
pitch: ttsConfig.pitch,
|
|
2815
|
+
});
|
|
2682
2816
|
}
|
|
2683
2817
|
|
|
2684
2818
|
/**
|
|
2685
|
-
*
|
|
2686
|
-
*
|
|
2819
|
+
* Pick the best available English voice from a list of SpeechSynthesisVoices.
|
|
2820
|
+
* Prefers high-quality remote/natural voices, then enhanced local voices,
|
|
2821
|
+
* then standard local voices. Retained for backwards compatibility; delegates
|
|
2822
|
+
* to the browser speech engine's picker.
|
|
2687
2823
|
*/
|
|
2688
|
-
|
|
2689
|
-
|
|
2690
|
-
|
|
2691
|
-
const synth = window.speechSynthesis;
|
|
2692
|
-
synth.cancel();
|
|
2693
|
-
|
|
2694
|
-
const utterance = new SpeechSynthesisUtterance(text);
|
|
2695
|
-
const voices = synth.getVoices();
|
|
2696
|
-
|
|
2697
|
-
if (config.voice) {
|
|
2698
|
-
const match = voices.find(v => v.name === config.voice);
|
|
2699
|
-
if (match) utterance.voice = match;
|
|
2700
|
-
} else if (voices.length > 0) {
|
|
2701
|
-
// Use custom picker if provided, otherwise auto-detect
|
|
2702
|
-
utterance.voice = config.pickVoice
|
|
2703
|
-
? config.pickVoice(voices)
|
|
2704
|
-
: AgentWidgetSession.pickBestVoice(voices);
|
|
2705
|
-
}
|
|
2706
|
-
|
|
2707
|
-
if (config.rate !== undefined) utterance.rate = config.rate;
|
|
2708
|
-
if (config.pitch !== undefined) utterance.pitch = config.pitch;
|
|
2709
|
-
|
|
2710
|
-
// Chrome bug: cancel() immediately followed by speak() can ignore
|
|
2711
|
-
// rate/pitch. A microtask delay lets the engine reset properly.
|
|
2712
|
-
setTimeout(() => synth.speak(utterance), 50);
|
|
2824
|
+
static pickBestVoice(voices: SpeechSynthesisVoice[]): SpeechSynthesisVoice {
|
|
2825
|
+
return pickBestVoice(voices);
|
|
2713
2826
|
}
|
|
2714
2827
|
|
|
2715
2828
|
/**
|
|
2716
|
-
*
|
|
2717
|
-
*
|
|
2718
|
-
*
|
|
2829
|
+
* Toggle the per-message "Read aloud" action: play → pause → resume (or
|
|
2830
|
+
* play → stop when the engine can't pause). Speaks the assistant message's
|
|
2831
|
+
* text via the configured speech engine (browser Web Speech API by default,
|
|
2832
|
+
* or a hosted engine from `textToSpeech.createEngine`).
|
|
2719
2833
|
*/
|
|
2720
|
-
|
|
2721
|
-
|
|
2722
|
-
|
|
2723
|
-
|
|
2724
|
-
|
|
2725
|
-
|
|
2726
|
-
|
|
2727
|
-
|
|
2728
|
-
|
|
2729
|
-
|
|
2730
|
-
|
|
2731
|
-
|
|
2732
|
-
|
|
2733
|
-
'Samantha (Enhanced)',
|
|
2734
|
-
// Apple standard (macOS/iOS)
|
|
2735
|
-
'Samantha',
|
|
2736
|
-
'Daniel',
|
|
2737
|
-
'Karen',
|
|
2738
|
-
// Windows SAPI
|
|
2739
|
-
'Microsoft David Desktop - English (United States)',
|
|
2740
|
-
'Microsoft Zira Desktop - English (United States)',
|
|
2741
|
-
];
|
|
2834
|
+
public toggleReadAloud(messageId: string): void {
|
|
2835
|
+
const message = this.messages.find(m => m.id === messageId);
|
|
2836
|
+
if (!message || message.role !== 'assistant') return;
|
|
2837
|
+
const text = resolveSpeakableText(message.content || '');
|
|
2838
|
+
if (!text.trim()) return;
|
|
2839
|
+
const tts = this.config.textToSpeech;
|
|
2840
|
+
this.readAloud.toggle(messageId, {
|
|
2841
|
+
text,
|
|
2842
|
+
voice: tts?.voice,
|
|
2843
|
+
rate: tts?.rate,
|
|
2844
|
+
pitch: tts?.pitch,
|
|
2845
|
+
});
|
|
2846
|
+
}
|
|
2742
2847
|
|
|
2743
|
-
|
|
2744
|
-
|
|
2745
|
-
|
|
2746
|
-
|
|
2848
|
+
/** Current read-aloud playback state for a message (`idle` unless active). */
|
|
2849
|
+
public getReadAloudState(messageId: string): ReadAloudState {
|
|
2850
|
+
return this.readAloud.stateFor(messageId);
|
|
2851
|
+
}
|
|
2747
2852
|
|
|
2748
|
-
|
|
2749
|
-
|
|
2853
|
+
/** Subscribe to read-aloud state changes. Returns an unsubscribe function. */
|
|
2854
|
+
public onReadAloudChange(listener: ReadAloudListener): () => void {
|
|
2855
|
+
return this.readAloud.onChange(listener);
|
|
2750
2856
|
}
|
|
2751
2857
|
|
|
2752
2858
|
/**
|
|
2753
|
-
* Stop any in-progress text-to-speech playback.
|
|
2859
|
+
* Stop any in-progress text-to-speech / read-aloud playback.
|
|
2754
2860
|
*/
|
|
2755
2861
|
public stopSpeaking() {
|
|
2862
|
+
this.readAloud.stop();
|
|
2756
2863
|
if (typeof window !== 'undefined' && 'speechSynthesis' in window) {
|
|
2757
2864
|
window.speechSynthesis.cancel();
|
|
2758
2865
|
}
|
|
@@ -2776,8 +2883,7 @@ export class AgentWidgetSession {
|
|
|
2776
2883
|
if (idx !== index) return existing;
|
|
2777
2884
|
const merged = { ...existing, ...withSequence };
|
|
2778
2885
|
// Preserve `ask_user_question` answered state across re-emissions.
|
|
2779
|
-
// Top-level merge would otherwise replace `agentMetadata` wholesale
|
|
2780
|
-
// post-resume events (e.g. `tool_complete` re-emitted from a stale
|
|
2886
|
+
// Top-level merge would otherwise replace `agentMetadata` wholesale: // post-resume events (e.g. `tool_complete` re-emitted from a stale
|
|
2781
2887
|
// client-side cache) would wipe `askUserQuestionAnswered` and
|
|
2782
2888
|
// `askUserQuestionAnswers`, causing the answered review card to
|
|
2783
2889
|
// lose its answers and revert to "(skipped)" placeholders.
|
|
@@ -2794,7 +2900,7 @@ export class AgentWidgetSession {
|
|
|
2794
2900
|
existing.agentMetadata.askUserQuestionAnswers,
|
|
2795
2901
|
}
|
|
2796
2902
|
: {}),
|
|
2797
|
-
// Keep awaiting flag false once resolved
|
|
2903
|
+
// Keep awaiting flag false once resolved: never let a stale
|
|
2798
2904
|
// re-emit flip us back to awaiting.
|
|
2799
2905
|
awaitingLocalTool: false,
|
|
2800
2906
|
};
|
|
@@ -2815,7 +2921,7 @@ export class AgentWidgetSession {
|
|
|
2815
2921
|
};
|
|
2816
2922
|
}
|
|
2817
2923
|
// Approval equivalent: `agent_approval_complete` carries only the
|
|
2818
|
-
// resolution (approvalId, decision, resolvedBy)
|
|
2924
|
+
// resolution (approvalId, decision, resolvedBy): the runtime does not
|
|
2819
2925
|
// re-send toolName/description/toolType/reason/parameters, so client.ts
|
|
2820
2926
|
// rebuilds the approval with empty required fields and the optional
|
|
2821
2927
|
// ones absent. A wholesale `approval` replacement would wipe that
|