@runtypelabs/persona 4.28.2 → 4.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/animations/glyph-cycle.d.cts +1 -1
- package/dist/animations/glyph-cycle.d.ts +1 -1
- package/dist/animations/wipe.d.cts +1 -1
- package/dist/animations/wipe.d.ts +1 -1
- package/dist/approval-ui.d.cts +1 -1
- package/dist/approval-ui.d.ts +1 -1
- package/dist/artifacts-ui.d.cts +2 -2
- package/dist/artifacts-ui.d.ts +2 -2
- package/dist/chunk-BkW8s3fj.js +2 -0
- package/dist/chunk-CIVsNcrk.js +1 -0
- package/dist/chunk-CKdlNyzM.js +9 -0
- package/dist/{chunk-DrYAfpvk.js → chunk-DjOvgFsn.js} +1 -1
- package/dist/chunk-lA5CJCgK.js +1 -0
- package/dist/client-history.js +1 -1
- package/dist/codegen.cjs +1 -1
- package/dist/codegen.js +1 -1
- package/dist/{composer-input-C39npJaz.d.cts → composer-input-D3sB8_P_.d.cts} +1 -1
- package/dist/{composer-input-C39npJaz.d.ts → composer-input-D3sB8_P_.d.ts} +1 -1
- package/dist/{context-mention-manager-DLCB1l9E.d.cts → context-mention-manager-D2qxE5op.d.cts} +1 -1
- package/dist/{context-mention-manager-DLCB1l9E.d.ts → context-mention-manager-D2qxE5op.d.ts} +1 -1
- package/dist/context-mentions-inline.d.cts +2 -2
- package/dist/context-mentions-inline.d.ts +2 -2
- package/dist/context-mentions.d.cts +3 -3
- package/dist/context-mentions.d.ts +3 -3
- package/dist/{defaults-DiAgWHXy.d.cts → defaults-Dsw66XqE.d.cts} +1 -1
- package/dist/{defaults-DiAgWHXy.d.ts → defaults-Dsw66XqE.d.ts} +1 -1
- package/dist/event-stream-view.d.cts +1 -1
- package/dist/event-stream-view.d.ts +1 -1
- package/dist/{fallback-speech-engine-C5_PUZjH.d.cts → fallback-speech-engine-DJpJcWAk.d.cts} +1 -1
- package/dist/{fallback-speech-engine-C5_PUZjH.d.ts → fallback-speech-engine-DJpJcWAk.d.ts} +1 -1
- package/dist/forms-ui.d.cts +2 -2
- package/dist/forms-ui.d.ts +2 -2
- package/dist/history-view.d.cts +1 -1
- package/dist/history-view.d.ts +1 -1
- package/dist/index.cjs +45 -44
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +7 -7
- package/dist/index.d.ts +7 -7
- package/dist/index.global.js +1 -1
- package/dist/index.global.js.map +1 -1
- package/dist/index.js +34 -33
- package/dist/index.js.map +1 -1
- package/dist/{markdown-parsers-loader-BBdyIEU7.d.cts → markdown-parsers-loader-CCY7T4Up.d.cts} +1 -1
- package/dist/{markdown-parsers-loader-BBdyIEU7.d.ts → markdown-parsers-loader-CCY7T4Up.d.ts} +1 -1
- package/dist/{preview-Chjb0bmI.d.cts → preview-DKKpfTo7.d.cts} +2 -2
- package/dist/{preview-Chjb0bmI.d.ts → preview-DKKpfTo7.d.ts} +2 -2
- package/dist/{session-a_qIp4xp.d.ts → session-DleMr9dZ.d.cts} +13 -7
- package/dist/{session-a_qIp4xp.d.cts → session-DleMr9dZ.d.ts} +13 -7
- package/dist/session-reconnect.d.cts +2 -2
- package/dist/session-reconnect.d.ts +2 -2
- package/dist/smart-dom-reader.d.cts +1 -1
- package/dist/smart-dom-reader.d.ts +1 -1
- package/dist/theme-editor-preview.cjs +41 -40
- package/dist/theme-editor-preview.d.cts +1 -1
- package/dist/theme-editor-preview.d.ts +1 -1
- package/dist/theme-editor-preview.js +20 -20
- package/dist/theme-editor.d.cts +2 -2
- package/dist/theme-editor.d.ts +2 -2
- package/dist/theme-reference.d.cts +2 -2
- package/dist/theme-reference.d.ts +2 -2
- package/dist/{types-BdO1jNRm.d.cts → types-oAZ1VReS.d.cts} +166 -21
- package/dist/{types-BdO1jNRm.d.ts → types-oAZ1VReS.d.ts} +166 -21
- package/dist/voice-client-tools.cjs +1 -0
- package/dist/voice-client-tools.d.cts +13 -0
- package/dist/voice-client-tools.d.ts +13 -0
- package/dist/voice-client-tools.js +1 -0
- package/dist/voice-runtime.cjs +3 -3
- package/dist/voice-runtime.d.cts +7 -3
- package/dist/voice-runtime.d.ts +7 -3
- package/dist/voice-runtime.js +4 -4
- package/dist/voice-worklet-player.d.cts +2 -2
- package/dist/voice-worklet-player.d.ts +2 -2
- package/dist/{webmcp-runtime-entry-B6bpVWSa.d.ts → webmcp-runtime-entry-Deg0NfmB.d.cts} +26 -5
- package/dist/{webmcp-runtime-entry-B6bpVWSa.d.cts → webmcp-runtime-entry-Deg0NfmB.d.ts} +26 -5
- package/dist/webmcp-runtime.cjs +2 -1
- package/dist/webmcp-runtime.d.cts +1 -1
- package/dist/webmcp-runtime.d.ts +1 -1
- package/dist/webmcp-runtime.js +2 -1
- package/package.json +6 -1
- package/src/client.ts +37 -0
- package/src/client.voice-tools.test.ts +65 -0
- package/src/generated/runtype-openapi-contract.ts +38 -0
- package/src/index-core.ts +11 -0
- package/src/index-global.ts +6 -0
- package/src/session.test.ts +26 -0
- package/src/session.ts +8 -1
- package/src/session.voice.test.ts +43 -0
- package/src/types.ts +141 -20
- package/src/voice/runtype-voice-provider.ts +85 -3
- package/src/voice/session-voice-wiring.ts +26 -1
- package/src/voice/voice-client-tools-session.ts +116 -0
- package/src/voice/voice-delegation.ts +9 -0
- package/src/voice/voice.test.ts +360 -0
- package/src/voice-client-tools-bundle.test.ts +37 -0
- package/src/voice-client-tools-loader.ts +24 -0
- package/src/voice-client-tools.ts +9 -0
- package/src/webmcp-bridge.test.ts +152 -1
- package/src/webmcp-bridge.ts +5 -6
- package/src/webmcp-polyfill.integration.test.ts +5 -1
- package/src/webmcp-runtime-entry.ts +151 -26
- package/dist/chunk-CztSjKHK.js +0 -9
- package/dist/chunk-HUOUif-P.js +0 -1
- package/dist/chunk-UlcxdHef.js +0 -1
|
@@ -35,6 +35,7 @@ export type RuntypeExecutionStreamEvent = ({
|
|
|
35
35
|
finalOutput?: string;
|
|
36
36
|
iterations?: number;
|
|
37
37
|
kind: "agent" | "flow";
|
|
38
|
+
outcome?: "completed" | "error" | "aborted";
|
|
38
39
|
seq: number;
|
|
39
40
|
stepErrorCount?: number;
|
|
40
41
|
stopReason?: string;
|
|
@@ -60,6 +61,7 @@ export type RuntypeExecutionStreamEvent = ({
|
|
|
60
61
|
executionId: string;
|
|
61
62
|
finalOutput?: string;
|
|
62
63
|
kind: "agent" | "flow";
|
|
64
|
+
outcome?: "completed" | "error" | "aborted";
|
|
63
65
|
seq: number;
|
|
64
66
|
stopReason?: string;
|
|
65
67
|
type: "execution_error";
|
|
@@ -379,6 +381,15 @@ export type RuntypeExecutionStreamEvent = ({
|
|
|
379
381
|
seq: number;
|
|
380
382
|
type: "state_delta";
|
|
381
383
|
}) | ({
|
|
384
|
+
annotations?: {
|
|
385
|
+
consequentialHint?: boolean;
|
|
386
|
+
debugging?: boolean;
|
|
387
|
+
destructiveHint?: boolean;
|
|
388
|
+
idempotentHint?: boolean;
|
|
389
|
+
openWorldHint?: boolean;
|
|
390
|
+
readOnlyHint?: boolean;
|
|
391
|
+
untrustedContentHint?: boolean;
|
|
392
|
+
};
|
|
382
393
|
executionId: string;
|
|
383
394
|
hiddenParameterNames?: Array<string>;
|
|
384
395
|
iteration?: number;
|
|
@@ -477,6 +488,15 @@ export type RuntypeExecutionStreamEvent = ({
|
|
|
477
488
|
seq: number;
|
|
478
489
|
type: "approval_complete";
|
|
479
490
|
}) | ({
|
|
491
|
+
annotations?: {
|
|
492
|
+
consequentialHint?: boolean;
|
|
493
|
+
debugging?: boolean;
|
|
494
|
+
destructiveHint?: boolean;
|
|
495
|
+
idempotentHint?: boolean;
|
|
496
|
+
openWorldHint?: boolean;
|
|
497
|
+
readOnlyHint?: boolean;
|
|
498
|
+
untrustedContentHint?: boolean;
|
|
499
|
+
};
|
|
480
500
|
awaitReason?: string;
|
|
481
501
|
awaitedAt?: string;
|
|
482
502
|
crawlId?: string;
|
|
@@ -641,6 +661,15 @@ export type RuntypeClientInitResponse = {
|
|
|
641
661
|
export type RuntypeClientChatRequest = {
|
|
642
662
|
assistantMessageId?: string;
|
|
643
663
|
clientTools?: Array<{
|
|
664
|
+
annotations?: {
|
|
665
|
+
consequentialHint?: boolean;
|
|
666
|
+
debugging?: boolean;
|
|
667
|
+
destructiveHint?: boolean;
|
|
668
|
+
idempotentHint?: boolean;
|
|
669
|
+
openWorldHint?: boolean;
|
|
670
|
+
readOnlyHint?: boolean;
|
|
671
|
+
untrustedContentHint?: boolean;
|
|
672
|
+
};
|
|
644
673
|
description: string;
|
|
645
674
|
name: string;
|
|
646
675
|
origin?: "webmcp" | "sdk";
|
|
@@ -694,6 +723,15 @@ export type RuntypeClientResumeRequest = {
|
|
|
694
723
|
after?: string;
|
|
695
724
|
assistantMessageId?: string;
|
|
696
725
|
clientTools?: Array<{
|
|
726
|
+
annotations?: {
|
|
727
|
+
consequentialHint?: boolean;
|
|
728
|
+
debugging?: boolean;
|
|
729
|
+
destructiveHint?: boolean;
|
|
730
|
+
idempotentHint?: boolean;
|
|
731
|
+
openWorldHint?: boolean;
|
|
732
|
+
readOnlyHint?: boolean;
|
|
733
|
+
untrustedContentHint?: boolean;
|
|
734
|
+
};
|
|
697
735
|
description: string;
|
|
698
736
|
name: string;
|
|
699
737
|
origin?: "webmcp" | "sdk";
|
package/src/index-core.ts
CHANGED
|
@@ -174,6 +174,7 @@ export type {
|
|
|
174
174
|
// WebMCP: page-discovered tool consumption
|
|
175
175
|
AgentWidgetWebMcpConfig,
|
|
176
176
|
ClientToolDefinition,
|
|
177
|
+
ToolAnnotations,
|
|
177
178
|
WebMcpConfirmHandler,
|
|
178
179
|
WebMcpConfirmInfo,
|
|
179
180
|
WebMcpToolResult,
|
|
@@ -659,6 +660,16 @@ export type {
|
|
|
659
660
|
VoiceDelegationStatus,
|
|
660
661
|
VoiceDelegationFollowUp,
|
|
661
662
|
VoiceSessionBridge,
|
|
663
|
+
VoiceClientTools,
|
|
664
|
+
VoiceClientToolsSnapshot,
|
|
665
|
+
VoiceClientToolsChannel,
|
|
666
|
+
VoiceClientToolsSession,
|
|
667
|
+
VoiceClientToolOutcome,
|
|
668
|
+
VoiceClientToolStatus,
|
|
669
|
+
VoiceDirectTurn,
|
|
670
|
+
ClientToolsFrame,
|
|
671
|
+
ClientToolCallFrame,
|
|
672
|
+
ClientToolResultFrame,
|
|
662
673
|
PcmStreamPlayer,
|
|
663
674
|
AgentWidgetVoiceStatusEvent,
|
|
664
675
|
ReadAloudState,
|
package/src/index-global.ts
CHANGED
|
@@ -221,6 +221,12 @@ import { setVoiceRuntimeLoader } from "./voice-runtime-loader";
|
|
|
221
221
|
|
|
222
222
|
setVoiceRuntimeLoader(siblingChunk("voice-runtime.js"));
|
|
223
223
|
|
|
224
|
+
// The voice fast tier (`client_tools`), fetched by client.ts only when a voice
|
|
225
|
+
// call negotiates it. Same sibling-URL scheme.
|
|
226
|
+
import { setVoiceClientToolsLoader } from "./voice-client-tools-loader";
|
|
227
|
+
|
|
228
|
+
setVoiceClientToolsLoader(siblingChunk("voice-client-tools.js"));
|
|
229
|
+
|
|
224
230
|
// ---------------------------------------------------------------------------
|
|
225
231
|
// Deferred artifacts-ui loading.
|
|
226
232
|
//
|
package/src/session.test.ts
CHANGED
|
@@ -1259,4 +1259,30 @@ describe('AgentWidgetSession - WebMCP native approval gate', () => {
|
|
|
1259
1259
|
getMessages().find((m) => m.variant === 'approval')?.approval?.status
|
|
1260
1260
|
).toBe('approved');
|
|
1261
1261
|
});
|
|
1262
|
+
|
|
1263
|
+
it('closes the bubble and resolves false when the call signal aborts, and ignores a later click', async () => {
|
|
1264
|
+
const { session, getMessages } = makeSession();
|
|
1265
|
+
const abort = new AbortController();
|
|
1266
|
+
const promise = session.requestWebMcpApproval({
|
|
1267
|
+
toolName: 'edit_slide',
|
|
1268
|
+
args: {},
|
|
1269
|
+
reason: 'gate',
|
|
1270
|
+
signal: abort.signal,
|
|
1271
|
+
});
|
|
1272
|
+
const bubble = getMessages().find((m) => m.variant === 'approval')!;
|
|
1273
|
+
expect(bubble.approval?.status).toBe('pending');
|
|
1274
|
+
abort.abort();
|
|
1275
|
+
await expect(promise).resolves.toBe(false);
|
|
1276
|
+
expect(getMessages().find((m) => m.variant === 'approval')?.approval?.status).toBe('timeout');
|
|
1277
|
+
session.resolveWebMcpApproval(bubble.id, 'approved');
|
|
1278
|
+
expect(getMessages().find((m) => m.variant === 'approval')?.approval?.status).toBe('timeout');
|
|
1279
|
+
});
|
|
1280
|
+
|
|
1281
|
+
it('shows no bubble for a call whose signal already aborted', async () => {
|
|
1282
|
+
const { session, getMessages } = makeSession();
|
|
1283
|
+
await expect(
|
|
1284
|
+
session.requestWebMcpApproval({ toolName: 'edit_slide', args: {}, reason: 'gate', signal: AbortSignal.abort() })
|
|
1285
|
+
).resolves.toBe(false);
|
|
1286
|
+
expect(getMessages().some((m) => m.variant === 'approval')).toBe(false);
|
|
1287
|
+
});
|
|
1262
1288
|
});
|
package/src/session.ts
CHANGED
|
@@ -3076,6 +3076,7 @@ export class AgentWidgetSession {
|
|
|
3076
3076
|
} catch {
|
|
3077
3077
|
// fall through to explicit approval
|
|
3078
3078
|
}
|
|
3079
|
+
if (info.signal?.aborted) return Promise.resolve(false);
|
|
3079
3080
|
|
|
3080
3081
|
const approval: AgentWidgetApproval = {
|
|
3081
3082
|
id: `webmcp-${++this.#webMcpApprovalSeq}`,
|
|
@@ -3102,6 +3103,12 @@ export class AgentWidgetSession {
|
|
|
3102
3103
|
|
|
3103
3104
|
return new Promise<boolean>((resolve) => {
|
|
3104
3105
|
this.webMcpApprovalResolvers.set(approvalMessageId, resolve);
|
|
3106
|
+
// The call gave up (deadline or cancel): close the card so a late click cannot look like it ran the tool.
|
|
3107
|
+
info.signal?.addEventListener(
|
|
3108
|
+
"abort",
|
|
3109
|
+
() => this.resolveWebMcpApproval(approvalMessageId, "timeout"),
|
|
3110
|
+
{ once: true }
|
|
3111
|
+
);
|
|
3105
3112
|
});
|
|
3106
3113
|
}
|
|
3107
3114
|
|
|
@@ -3113,7 +3120,7 @@ export class AgentWidgetSession {
|
|
|
3113
3120
|
*/
|
|
3114
3121
|
public resolveWebMcpApproval(
|
|
3115
3122
|
approvalMessageId: string,
|
|
3116
|
-
decision: "approved" | "denied"
|
|
3123
|
+
decision: "approved" | "denied" | "timeout"
|
|
3117
3124
|
): void {
|
|
3118
3125
|
const resolve = this.webMcpApprovalResolvers.get(approvalMessageId);
|
|
3119
3126
|
if (!resolve) return;
|
|
@@ -623,6 +623,49 @@ describe('AgentWidgetSession - voice client delegation bridge', () => {
|
|
|
623
623
|
]);
|
|
624
624
|
});
|
|
625
625
|
|
|
626
|
+
it('offers no client tools when WebMCP is off', () => {
|
|
627
|
+
expect(h.state.bridge!.clientTools!()).toBeNull();
|
|
628
|
+
});
|
|
629
|
+
|
|
630
|
+
it('records a server-answered (direct) turn: the caption becomes the user turn, the readout an answer', async () => {
|
|
631
|
+
const caption = (role: 'user' | 'assistant', text: string, turnId: string) =>
|
|
632
|
+
h.state.transcriptCb!(role, text, true, { turnId, caption: true });
|
|
633
|
+
caption('user', 'Next slide', 'u1');
|
|
634
|
+
h.state.bridge!.recordDirectTurn!({
|
|
635
|
+
delegationId: 'd1',
|
|
636
|
+
userText: 'Next slide',
|
|
637
|
+
userUtteranceIds: ['u1'],
|
|
638
|
+
assistantText: 'Slide 2: Agenda.',
|
|
639
|
+
});
|
|
640
|
+
expect(view()).toEqual([
|
|
641
|
+
['assistant', 'Welcome! How can I help?'],
|
|
642
|
+
['user', 'Next slide'],
|
|
643
|
+
['assistant', 'Slide 2: Agenda.'],
|
|
644
|
+
]);
|
|
645
|
+
expect(messages.find((m) => m.content === 'Next slide')!.voiceCaption).toBeUndefined();
|
|
646
|
+
expect(spoken('voice-direct-d1')).toBe(true);
|
|
647
|
+
expect(dispatch).not.toHaveBeenCalled();
|
|
648
|
+
|
|
649
|
+
// The next typed turn carries both as conversation.
|
|
650
|
+
dispatch.mockImplementation(async (_options, onEvent) => reply(onEvent, 'Sure.', 'assistant-r2'));
|
|
651
|
+
await session.sendMessage('and the one after?');
|
|
652
|
+
const sent = dispatch.mock.calls[0][0].messages.filter((m) => !m.voiceCaption);
|
|
653
|
+
expect(sent.map((m) => m.content)).toEqual([
|
|
654
|
+
'Welcome! How can I help?',
|
|
655
|
+
'Next slide',
|
|
656
|
+
'Slide 2: Agenda.',
|
|
657
|
+
'and the one after?',
|
|
658
|
+
]);
|
|
659
|
+
});
|
|
660
|
+
|
|
661
|
+
it('records a direct turn with no caption and no readout as just the request', () => {
|
|
662
|
+
h.state.bridge!.recordDirectTurn!({ delegationId: 'd2', userText: 'Next slide', userUtteranceIds: [], assistantText: '' });
|
|
663
|
+
expect(view()).toEqual([
|
|
664
|
+
['assistant', 'Welcome! How can I help?'],
|
|
665
|
+
['user', 'Next slide'],
|
|
666
|
+
]);
|
|
667
|
+
});
|
|
668
|
+
|
|
626
669
|
it('trims GPT-Live\'s leading space from bubbles and the submitted request', async () => {
|
|
627
670
|
h.state.transcriptCb!('user', ' What are your opening hours', true, { turnId: 'u1', caption: true });
|
|
628
671
|
dispatch.mockImplementation(async (_options, onEvent) => reply(onEvent, 'Nine to five.'));
|
package/src/types.ts
CHANGED
|
@@ -818,14 +818,18 @@ export type AgentWidgetAgentRequestPayload = {
|
|
|
818
818
|
// WebMCP Types (page-discovered tools shipped per dispatch)
|
|
819
819
|
// ============================================================================
|
|
820
820
|
|
|
821
|
-
/**
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
821
|
+
/** Advisory WebMCP and MCP hints. These do not grant approval or safe retry. */
|
|
822
|
+
export type ToolAnnotations = {
|
|
823
|
+
readOnlyHint?: boolean;
|
|
824
|
+
destructiveHint?: boolean;
|
|
825
|
+
consequentialHint?: boolean;
|
|
826
|
+
untrustedContentHint?: boolean;
|
|
827
|
+
idempotentHint?: boolean;
|
|
828
|
+
openWorldHint?: boolean;
|
|
829
|
+
debugging?: boolean;
|
|
830
|
+
};
|
|
831
|
+
|
|
832
|
+
/** Wire metadata for a client-side tool; its execute function stays in the browser. */
|
|
829
833
|
export type ClientToolDefinition = {
|
|
830
834
|
/** Bare tool name; the server prepends `webmcp:` on the wire. */
|
|
831
835
|
name: string;
|
|
@@ -842,20 +846,16 @@ export type ClientToolDefinition = {
|
|
|
842
846
|
/** Origin of the page that registered the tool: for server-side audit. */
|
|
843
847
|
pageOrigin?: string;
|
|
844
848
|
/**
|
|
845
|
-
* WebMCP
|
|
846
|
-
*
|
|
847
|
-
* `readOnlyHint` / `untrustedContentHint` on tool-call records.
|
|
849
|
+
* WebMCP hints and MCP compatibility hints forwarded to runtime tool metadata
|
|
850
|
+
* and confirmation callbacks. No hint skips the default confirmation gate.
|
|
848
851
|
*/
|
|
849
|
-
annotations?:
|
|
850
|
-
readOnlyHint?: boolean;
|
|
851
|
-
untrustedContentHint?: boolean;
|
|
852
|
-
};
|
|
852
|
+
annotations?: ToolAnnotations;
|
|
853
853
|
/**
|
|
854
854
|
* Hoisted copy of `annotations.untrustedContentHint`. This is the field the
|
|
855
855
|
* server actually acts on (`InlineClientToolSchema`): it spotlights the
|
|
856
856
|
* tool's output before the model reads it, so instructions embedded in
|
|
857
857
|
* page-provided content (reviews, comments, emails) are not treated as
|
|
858
|
-
* instructions.
|
|
858
|
+
* instructions. A nested true hint also opts into output protection.
|
|
859
859
|
*/
|
|
860
860
|
untrustedContentHint?: boolean;
|
|
861
861
|
};
|
|
@@ -875,16 +875,19 @@ export type WebMcpConfirmInfo = {
|
|
|
875
875
|
* didn't declare one.
|
|
876
876
|
*/
|
|
877
877
|
title?: string;
|
|
878
|
-
annotations?:
|
|
879
|
-
readOnlyHint?: boolean;
|
|
880
|
-
untrustedContentHint?: boolean;
|
|
881
|
-
};
|
|
878
|
+
annotations?: ToolAnnotations;
|
|
882
879
|
/**
|
|
883
880
|
* Why the confirm was requested. Currently always `'gate'`: the default
|
|
884
881
|
* confirm-by-default gate that fires before every `webmcp:*` call. Tool
|
|
885
882
|
* callbacks on v6 receive an AbortSignal, not a nested confirmation API.
|
|
886
883
|
*/
|
|
887
884
|
reason: 'gate';
|
|
885
|
+
/**
|
|
886
|
+
* Aborts when the call no longer needs an answer (its deadline passed, or it
|
|
887
|
+
* was cancelled). A confirm UI should close and resolve `false`; the tool
|
|
888
|
+
* never runs after it aborts either way.
|
|
889
|
+
*/
|
|
890
|
+
signal?: AbortSignal;
|
|
888
891
|
};
|
|
889
892
|
|
|
890
893
|
/**
|
|
@@ -5355,8 +5358,126 @@ export interface VoiceSessionBridge {
|
|
|
5355
5358
|
* declined too, as the approval TTL would.
|
|
5356
5359
|
*/
|
|
5357
5360
|
dropDelegation?(delegationId: string, expired?: boolean): void;
|
|
5361
|
+
/**
|
|
5362
|
+
* The page tools a voice call may offer the server's fast tier
|
|
5363
|
+
* (`client_tools` / `client_tool_call`). Returns `null` when WebMCP is not
|
|
5364
|
+
* enabled on this widget; the provider then does not declare `client_tools`.
|
|
5365
|
+
*/
|
|
5366
|
+
clientTools?(): VoiceClientTools | null;
|
|
5367
|
+
/**
|
|
5368
|
+
* The server answered a delegation itself (`delegation_completed.route ===
|
|
5369
|
+
* 'direct'`): record the visitor's turn and the spoken answer in the chat,
|
|
5370
|
+
* as a delegated turn would have. `assistantText` may be empty (no readout).
|
|
5371
|
+
*/
|
|
5372
|
+
recordDirectTurn?(turn: VoiceDirectTurn): void;
|
|
5358
5373
|
}
|
|
5359
5374
|
|
|
5375
|
+
/** A delegation the server answered on its fast tier, for the chat history. */
|
|
5376
|
+
export type VoiceDirectTurn = {
|
|
5377
|
+
delegationId: string;
|
|
5378
|
+
userText: string;
|
|
5379
|
+
userUtteranceIds: string[];
|
|
5380
|
+
assistantText: string;
|
|
5381
|
+
};
|
|
5382
|
+
|
|
5383
|
+
/** `client_tool_result.status` (open enum on the wire). */
|
|
5384
|
+
export type VoiceClientToolStatus =
|
|
5385
|
+
| 'completed'
|
|
5386
|
+
| 'failed'
|
|
5387
|
+
| 'denied'
|
|
5388
|
+
| 'stale'
|
|
5389
|
+
| 'timeout'
|
|
5390
|
+
| (string & {});
|
|
5391
|
+
|
|
5392
|
+
/** What running one `client_tool_call` produced. */
|
|
5393
|
+
export type VoiceClientToolOutcome = {
|
|
5394
|
+
status: VoiceClientToolStatus;
|
|
5395
|
+
/** The page tool's return value (any JSON), on `completed`. */
|
|
5396
|
+
result?: unknown;
|
|
5397
|
+
error?: { message: string };
|
|
5398
|
+
};
|
|
5399
|
+
|
|
5400
|
+
/** The page's tool set as of now, plus its content fingerprint. */
|
|
5401
|
+
export type VoiceClientToolsSnapshot = {
|
|
5402
|
+
/** Exactly what `/v1/client/chat` would send as `clientTools[]` (bare names). */
|
|
5403
|
+
tools: ClientToolDefinition[];
|
|
5404
|
+
fingerprint: string;
|
|
5405
|
+
};
|
|
5406
|
+
|
|
5407
|
+
/** WebMCP access for a voice call (see {@link VoiceSessionBridge.clientTools}). */
|
|
5408
|
+
export interface VoiceClientTools {
|
|
5409
|
+
snapshot(): Promise<VoiceClientToolsSnapshot>;
|
|
5410
|
+
/** Subscribe to registry changes; returns an unsubscribe. */
|
|
5411
|
+
onChange(listener: () => void): () => void;
|
|
5412
|
+
/**
|
|
5413
|
+
* Run a page tool. `autoConfirmReadOnly`: skip the confirm gate when the
|
|
5414
|
+
* tool's registry entry has `annotations.readOnlyHint === true`.
|
|
5415
|
+
*/
|
|
5416
|
+
execute(
|
|
5417
|
+
name: string,
|
|
5418
|
+
args: Record<string, unknown>,
|
|
5419
|
+
options: { signal: AbortSignal; autoConfirmReadOnly: boolean },
|
|
5420
|
+
): Promise<VoiceClientToolOutcome>;
|
|
5421
|
+
/**
|
|
5422
|
+
* Start the call's fast-tier session (loads its lazy chunk): it sends
|
|
5423
|
+
* `client_tools` through `channel` now and on every change, and answers
|
|
5424
|
+
* the `client_tool_call` frames handed to it.
|
|
5425
|
+
*/
|
|
5426
|
+
open(channel: VoiceClientToolsChannel): Promise<VoiceClientToolsSession>;
|
|
5427
|
+
}
|
|
5428
|
+
|
|
5429
|
+
/** The call socket, as the fast-tier session sees it. */
|
|
5430
|
+
export type VoiceClientToolsChannel = {
|
|
5431
|
+
/** Send a frame; `false` when the call (socket) is gone. */
|
|
5432
|
+
send(frame: ClientToolsFrame | ClientToolResultFrame): boolean;
|
|
5433
|
+
/** The server cancelled this delegation: its tool results are not sent. */
|
|
5434
|
+
cancelled(delegationId: string): boolean;
|
|
5435
|
+
};
|
|
5436
|
+
|
|
5437
|
+
/** A call's fast-tier session (see {@link VoiceClientTools.open}). */
|
|
5438
|
+
export interface VoiceClientToolsSession {
|
|
5439
|
+
call(frame: Partial<ClientToolCallFrame>): void;
|
|
5440
|
+
/** Abort running tool calls (of one delegation, or all); they send nothing. */
|
|
5441
|
+
abort(delegationId?: string): void;
|
|
5442
|
+
close(): void;
|
|
5443
|
+
}
|
|
5444
|
+
|
|
5445
|
+
/**
|
|
5446
|
+
* Client → server (`runtype-browser-v1`, `client_tools` capability): the page's
|
|
5447
|
+
* tool set, after `session_config` and whenever its fingerprint changes. Names
|
|
5448
|
+
* are the FULL names the server calls back with: `webmcp:` prefixed for
|
|
5449
|
+
* `origin: 'webmcp'`, bare for `sdk`.
|
|
5450
|
+
*/
|
|
5451
|
+
export type ClientToolsFrame = {
|
|
5452
|
+
type: 'client_tools';
|
|
5453
|
+
/** The content fingerprint the chat path computes for the same `clientTools[]`. */
|
|
5454
|
+
fingerprint: string;
|
|
5455
|
+
tools: ClientToolDefinition[];
|
|
5456
|
+
};
|
|
5457
|
+
|
|
5458
|
+
/** Server → client: the fast tier picked a page tool for this delegation. */
|
|
5459
|
+
export type ClientToolCallFrame = {
|
|
5460
|
+
type: 'client_tool_call';
|
|
5461
|
+
delegationId: string;
|
|
5462
|
+
toolCallId: string;
|
|
5463
|
+
/** `webmcp:<name>` or bare; the client accepts both. */
|
|
5464
|
+
name: string;
|
|
5465
|
+
arguments: Record<string, unknown>;
|
|
5466
|
+
/** The `client_tools` fingerprint the call was chosen from. */
|
|
5467
|
+
fingerprint: string;
|
|
5468
|
+
timeoutMs: number;
|
|
5469
|
+
};
|
|
5470
|
+
|
|
5471
|
+
/** Client → server: the answer to one {@link ClientToolCallFrame}. */
|
|
5472
|
+
export type ClientToolResultFrame = {
|
|
5473
|
+
type: 'client_tool_result';
|
|
5474
|
+
delegationId: string;
|
|
5475
|
+
toolCallId: string;
|
|
5476
|
+
status: VoiceClientToolStatus;
|
|
5477
|
+
result?: unknown;
|
|
5478
|
+
error?: { message: string };
|
|
5479
|
+
};
|
|
5480
|
+
|
|
5360
5481
|
/**
|
|
5361
5482
|
* Extra context for an approval decision, surfaced to `onDecision` and to the
|
|
5362
5483
|
* `approve`/`deny` callbacks passed to the `renderApproval` plugin hook.
|
|
@@ -83,8 +83,26 @@
|
|
|
83
83
|
// completed), and that phase's completion doesn't fold another utterance.
|
|
84
84
|
// Delegated sends carry `voice: { spoken: true }` on the chat request, so the
|
|
85
85
|
// agent answers in speakable sentences.
|
|
86
|
+
//
|
|
87
|
+
// Fast tier (`client_tools`, declared only with client delegation and WebMCP
|
|
88
|
+
// enabled; the session ships in the lazy voice-client-tools chunk): after
|
|
89
|
+
// `session_config` the provider sends `client_tools{fingerprint, tools}`, the
|
|
90
|
+
// same WebMCP snapshot and fingerprint the chat path sends as `clientTools[]`
|
|
91
|
+
// but with each page tool's FULL name (`webmcp:<name>`; sdk tools stay bare),
|
|
92
|
+
// and again whenever the fingerprint changes. The server may then answer a
|
|
93
|
+
// delegation itself, with no `delegation_started`: it sends
|
|
94
|
+
// `client_tool_call{delegationId, toolCallId, name, arguments, fingerprint,
|
|
95
|
+
// timeoutMs}`, which runs through the WebMCP execute path (the confirm gate is
|
|
96
|
+
// skipped only for a tool whose registry entry says `readOnlyHint: true`) and
|
|
97
|
+
// is answered by one `client_tool_result{status}`: `completed`, `denied` (gate
|
|
98
|
+
// declined), `stale` (unknown tool, or a fingerprint other than the last one
|
|
99
|
+
// sent), `failed`, or `timeout` (past `timeoutMs`, which bounds the gate too).
|
|
100
|
+
// Then `delegation_completed{route:'direct', text}` for a delegation that never
|
|
101
|
+
// started here writes the heard user turn and `text` into the chat. When the
|
|
102
|
+
// server falls through, the delegation arrives as today's `delegation_started`.
|
|
86
103
|
|
|
87
104
|
import type {
|
|
105
|
+
VoiceClientToolsSession,
|
|
88
106
|
VoiceProvider,
|
|
89
107
|
VoiceResult,
|
|
90
108
|
VoiceStatus,
|
|
@@ -327,6 +345,13 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
327
345
|
#foldReadback = false;
|
|
328
346
|
#assistantTurns = new Set<string>();
|
|
329
347
|
#foldedTurns = new Set<string>();
|
|
348
|
+
// Fast tier (see header): the call's session, from the lazy voice-client-tools chunk.
|
|
349
|
+
#fastTier: Promise<VoiceClientToolsSession | null> | null = null;
|
|
350
|
+
// Delegations this call saw start, and the last final user utterance (a direct turn's request).
|
|
351
|
+
#started = new Set<string>();
|
|
352
|
+
#heard: { id: string; text: string } | null = null;
|
|
353
|
+
// Direct turns in flight: the utterance each answers, captured at its first client_tool_call.
|
|
354
|
+
#direct = new Map<string, { id: string; text: string } | null>();
|
|
330
355
|
|
|
331
356
|
#resultCallbacks: ((result: VoiceResult) => void)[] = [];
|
|
332
357
|
#errorCallbacks: ((error: Error) => void)[] = [];
|
|
@@ -495,8 +520,10 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
495
520
|
*/
|
|
496
521
|
#voiceSocketUrl(host: string, agentId: string, attach?: URLSearchParams): string {
|
|
497
522
|
const capabilities = ["partial_transcript", "context"];
|
|
498
|
-
if (this.bridge && this.config?.clientDelegation !== false)
|
|
523
|
+
if (this.bridge && this.config?.clientDelegation !== false) {
|
|
499
524
|
capabilities.push("client_delegation", "delegation_update", "delegation_read_back", "delegation_stream");
|
|
525
|
+
if (this.bridge.clientTools?.()) capabilities.push("client_tools");
|
|
526
|
+
}
|
|
500
527
|
if (attach) capabilities.unshift("attach");
|
|
501
528
|
const params = new URLSearchParams({
|
|
502
529
|
voiceProtocol: VOICE_PROTOCOL,
|
|
@@ -849,8 +876,34 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
849
876
|
this.#contextSent = true;
|
|
850
877
|
this.#pendingContext = this.#buildContextText();
|
|
851
878
|
}
|
|
879
|
+
if (this.#clientDelegation && this.#capabilities.has("client_tools") && !this.#fastTier) {
|
|
880
|
+
this.#fastTier =
|
|
881
|
+
this.bridge
|
|
882
|
+
?.clientTools?.()
|
|
883
|
+
?.open({
|
|
884
|
+
send: (frame) => {
|
|
885
|
+
const ws = this.#ws;
|
|
886
|
+
if (generation !== this.#callGeneration || ws?.readyState !== WebSocket.OPEN) return false;
|
|
887
|
+
ws.send(JSON.stringify(frame));
|
|
888
|
+
return true;
|
|
889
|
+
},
|
|
890
|
+
cancelled: (id) => this.#cancelledDelegations.has(id),
|
|
891
|
+
})
|
|
892
|
+
.catch(() => null) ?? null;
|
|
893
|
+
}
|
|
852
894
|
break;
|
|
853
895
|
|
|
896
|
+
case "client_tool_call": {
|
|
897
|
+
// A direct turn: keep the request it answers (speech may follow) and fold its read-back.
|
|
898
|
+
const id = String(msg.delegationId);
|
|
899
|
+
if (this.#fastTier && !this.#direct.has(id)) {
|
|
900
|
+
this.#direct.set(id, this.#heard);
|
|
901
|
+
this.#answered.add(id);
|
|
902
|
+
}
|
|
903
|
+
void this.#fastTier?.then((session) => session?.call(msg));
|
|
904
|
+
break;
|
|
905
|
+
}
|
|
906
|
+
|
|
854
907
|
case "transcript_update": {
|
|
855
908
|
const role = msg.role === "assistant" ? "assistant" : "user";
|
|
856
909
|
// A reply the client cancelled keeps streaming until the server clears it.
|
|
@@ -860,7 +913,10 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
860
913
|
if (role === "user") {
|
|
861
914
|
// Release on a final user transcript, not a partial: a mid-utterance
|
|
862
915
|
// context append makes the voice model answer early (or not delegate).
|
|
863
|
-
if (msg.final === true)
|
|
916
|
+
if (msg.final === true) {
|
|
917
|
+
this.#flushCallContext(generation);
|
|
918
|
+
this.#heard = { id: turnId, text: String(msg.text ?? "") };
|
|
919
|
+
}
|
|
864
920
|
this.#foldReadback = false;
|
|
865
921
|
} else {
|
|
866
922
|
// Read-back of a delegated result the chat already renders. A server
|
|
@@ -902,6 +958,10 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
902
958
|
const delegationId = String(msg.delegationId ?? msg.turnId);
|
|
903
959
|
if (msg.type === "delegation_started") {
|
|
904
960
|
this.#flushCallContext(generation);
|
|
961
|
+
this.#started.add(delegationId);
|
|
962
|
+
// A direct attempt that fell through: its answer is the chat turn's now, and its tool must not act late.
|
|
963
|
+
this.#answered.delete(delegationId);
|
|
964
|
+
if (this.#direct.delete(delegationId)) void this.#fastTier?.then((session) => session?.abort(delegationId));
|
|
905
965
|
// `input`: run this turn through the chat pipeline (client delegation).
|
|
906
966
|
const input = msg.input;
|
|
907
967
|
if (this.#clientDelegation && input) {
|
|
@@ -918,7 +978,21 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
918
978
|
generation,
|
|
919
979
|
);
|
|
920
980
|
}
|
|
921
|
-
} else if (msg.
|
|
981
|
+
} else if (msg.route === "direct" && this.#fastTier && !this.#started.has(delegationId)) {
|
|
982
|
+
// The server answered it itself: write the request it answered and its
|
|
983
|
+
// answer into the chat, as the chat pipeline would have.
|
|
984
|
+
const heard = this.#direct.get(delegationId) ?? this.#heard;
|
|
985
|
+
this.#direct.delete(delegationId);
|
|
986
|
+
this.#started.add(delegationId);
|
|
987
|
+
this.#answered.add(delegationId);
|
|
988
|
+
this.bridge?.recordDirectTurn?.({
|
|
989
|
+
delegationId,
|
|
990
|
+
userText: heard?.text ?? "",
|
|
991
|
+
userUtteranceIds: heard ? [heard.id] : [],
|
|
992
|
+
assistantText: typeof msg.text === "string" ? msg.text : "",
|
|
993
|
+
});
|
|
994
|
+
}
|
|
995
|
+
if (msg.type === "delegation_completed" && msg.speak !== false && this.#answered.has(delegationId)) {
|
|
922
996
|
// Every spoken phase (the approval ask, the late result) of an answer
|
|
923
997
|
// the chat shows is read back. A refusal of one that never started
|
|
924
998
|
// here has no chat answer: it renders. `final` may never come. A
|
|
@@ -1086,6 +1160,9 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
1086
1160
|
// unrelated utterance: disarm it.
|
|
1087
1161
|
if (this.#foldArmedBy === delegationId) this.#foldReadback = false;
|
|
1088
1162
|
this.#progressed.delete(delegationId);
|
|
1163
|
+
// A direct answer that never completed has no chat message to fold onto.
|
|
1164
|
+
if (!this.#started.has(delegationId)) this.#answered.delete(delegationId);
|
|
1165
|
+
void this.#fastTier?.then((session) => session?.abort(delegationId));
|
|
1089
1166
|
this.bridge?.dropDelegation?.(delegationId, expired);
|
|
1090
1167
|
this.#followUps.get(delegationId)?.abort();
|
|
1091
1168
|
}
|
|
@@ -1255,6 +1332,11 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
1255
1332
|
this.#foldReadback = false;
|
|
1256
1333
|
this.#assistantTurns.clear();
|
|
1257
1334
|
this.#foldedTurns.clear();
|
|
1335
|
+
void this.#fastTier?.then((session) => session?.close());
|
|
1336
|
+
this.#fastTier = null;
|
|
1337
|
+
this.#started.clear();
|
|
1338
|
+
this.#direct.clear();
|
|
1339
|
+
this.#heard = null;
|
|
1258
1340
|
this.#finishedArmed = false;
|
|
1259
1341
|
this.#cancelledUntil = 0;
|
|
1260
1342
|
this.#playbackEndsAt = 0;
|
|
@@ -16,7 +16,7 @@ import type { VoiceTurnTracker } from "./voice-turn-tracker";
|
|
|
16
16
|
/** The session members the wiring touches (some are TS-private on the class). */
|
|
17
17
|
export type VoiceWiringSession = Pick<
|
|
18
18
|
AgentWidgetSession,
|
|
19
|
-
"injectMessage" | "sendMessage" | "resolveWebMcpApproval" | "resolveApproval"
|
|
19
|
+
"injectMessage" | "sendMessage" | "resolveWebMcpApproval" | "resolveApproval" | "getClient"
|
|
20
20
|
> & {
|
|
21
21
|
messages: AgentWidgetMessage[];
|
|
22
22
|
callbacks: AgentWidgetSession["callbacks"];
|
|
@@ -92,6 +92,31 @@ export const wireSessionVoice = (
|
|
|
92
92
|
// now. A stream still running reads it when it ends, once.
|
|
93
93
|
if (!s.streaming) s.speakLatestAssistantMessage(ids);
|
|
94
94
|
},
|
|
95
|
+
clientTools: () => s.getClient().getVoiceClientTools(),
|
|
96
|
+
recordDirect: ({ delegationId, userText, userUtteranceIds, assistantText }) => {
|
|
97
|
+
// The heard transcript bubble becomes the conversation turn, as a
|
|
98
|
+
// delegated send would make it; without one, the request text is.
|
|
99
|
+
const bubbleId = p.keyed?.claimUserTurn(userText, userUtteranceIds);
|
|
100
|
+
let claimed = false;
|
|
101
|
+
// upsertMessage merges, so it cannot drop the caption flag: replace the bubble.
|
|
102
|
+
s.messages = s.messages.map((m) => {
|
|
103
|
+
if (m.id !== bubbleId || m.role !== 'user') return m;
|
|
104
|
+
claimed = true;
|
|
105
|
+
const { voiceCaption: _caption, ...turn } = m;
|
|
106
|
+
return { ...turn, viaVoice: true };
|
|
107
|
+
});
|
|
108
|
+
if (claimed) {
|
|
109
|
+
s.callbacks.onMessagesChanged([...s.messages]);
|
|
110
|
+
} else if (userText.trim()) {
|
|
111
|
+
s.injectMessage({ role: 'user', content: userText.trim() });
|
|
112
|
+
}
|
|
113
|
+
const answer = assistantText.trim();
|
|
114
|
+
if (!answer) return;
|
|
115
|
+
// The voice model speaks this answer: browser TTS must not read it again.
|
|
116
|
+
const id = `voice-direct-${delegationId}`;
|
|
117
|
+
s.ttsSpokenMessageIds.add(id);
|
|
118
|
+
s.injectMessage({ id, role: 'assistant', content: answer });
|
|
119
|
+
},
|
|
95
120
|
decide: (id) => {
|
|
96
121
|
if (s.webMcpApprovalResolvers.has(id)) return s.resolveWebMcpApproval(id, 'denied');
|
|
97
122
|
const approval = s.messages.find((m) => m.id === id)?.approval;
|