@runtypelabs/persona 4.25.0 → 4.26.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/animations/glyph-cycle.d.cts +1 -1
- package/dist/animations/glyph-cycle.d.ts +1 -1
- package/dist/animations/{types-Db61aI9J.d.cts → types-BwKTZhg9.d.cts} +14 -0
- package/dist/animations/{types-Db61aI9J.d.ts → types-BwKTZhg9.d.ts} +14 -0
- package/dist/animations/wipe.d.cts +1 -1
- package/dist/animations/wipe.d.ts +1 -1
- package/dist/animations-extra.d.cts +14 -0
- package/dist/animations-extra.d.ts +14 -0
- package/dist/approval-ui-QAKWNFNM.js +1 -0
- package/dist/approval-ui.cjs +2 -2
- package/dist/approval-ui.d.cts +186 -8
- package/dist/approval-ui.d.ts +186 -8
- package/dist/approval-ui.js +2 -2
- package/dist/{artifacts-ui-IGB5YKLO.js → artifacts-ui-NJGMM27I.js} +1 -1
- package/dist/artifacts-ui.cjs +4 -4
- package/dist/artifacts-ui.d.cts +186 -8
- package/dist/artifacts-ui.d.ts +186 -8
- package/dist/artifacts-ui.js +5 -5
- package/dist/chunk-2ANTRKTV.js +1 -0
- package/dist/chunk-4HPTPUC6.js +1 -0
- package/dist/chunk-6V3MNEKB.js +1 -0
- package/dist/chunk-GKPCIQPB.js +1 -0
- package/dist/chunk-HGUCGM3K.js +9 -0
- package/dist/chunk-IO5VVUKP.js +3 -0
- package/dist/chunk-J4J3Y2P2.js +1 -0
- package/dist/chunk-K6HKIP5K.js +1 -0
- package/dist/chunk-K7NA7G7J.js +1 -0
- package/dist/chunk-MXUTS3LB.js +1 -0
- package/dist/chunk-PDPVORUF.js +1 -0
- package/dist/chunk-QRKGQPU7.js +1 -0
- package/dist/chunk-S6PBWE75.js +1 -0
- package/dist/chunk-T7QM7YMF.js +1 -0
- package/dist/chunk-TR2H475E.js +1 -0
- package/dist/chunk-UPO4GUFC.js +1 -0
- package/dist/chunk-ZC66NAIR.js +1 -0
- package/dist/client-history-GOZWBDA5.js +1 -0
- package/dist/client-history.js +1 -0
- package/dist/client-stream-42FOI3EC.js +4 -0
- package/dist/client-stream.js +5 -0
- package/dist/codegen.cjs +1 -1
- package/dist/codegen.js +1 -1
- package/dist/context-mentions-E7HJYWO6.js +161 -0
- package/dist/{context-mentions-inline-I2RDEBAK.js → context-mentions-inline-QVESLW55.js} +1 -1
- package/dist/context-mentions.cjs +7 -7
- package/dist/context-mentions.d.cts +186 -8
- package/dist/context-mentions.d.ts +186 -8
- package/dist/context-mentions.js +8 -8
- package/dist/event-stream-view-FOIJHFQO.js +1 -0
- package/dist/event-stream-view.cjs +1 -1
- package/dist/event-stream-view.d.cts +189 -31
- package/dist/event-stream-view.d.ts +189 -31
- package/dist/event-stream-view.js +1 -1
- package/dist/forms-ui.d.cts +227 -667
- package/dist/forms-ui.d.ts +227 -667
- package/dist/history-shell-QDQ44PSE.js +3 -0
- package/dist/history-shell.js +3 -0
- package/dist/{history-view-BD6OXLNA.js → history-view-NCFBI5GR.js} +2 -2
- package/dist/history-view.cjs +2 -2
- package/dist/history-view.d.cts +35 -1
- package/dist/history-view.d.ts +35 -1
- package/dist/history-view.js +2 -2
- package/dist/index.cjs +80 -74
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +233 -738
- package/dist/index.d.ts +233 -738
- package/dist/index.global.js +3 -215
- package/dist/index.global.js.map +1 -1
- package/dist/index.js +81 -75
- package/dist/index.js.map +1 -1
- package/dist/install.global.js +1 -1
- package/dist/install.global.js.map +1 -1
- package/dist/launcher.global.js +3 -67
- package/dist/launcher.global.js.map +1 -1
- package/dist/markdown-parsers.js +22 -22
- package/dist/runtype-tts-entry-NFBVWR7N.js +1 -0
- package/dist/runtype-tts.js +1 -1
- package/dist/session-actions-ISSKSI66.js +1 -0
- package/dist/session-actions.js +1 -0
- package/dist/session-reconnect.d.cts +186 -8
- package/dist/session-reconnect.d.ts +186 -8
- package/dist/smart-dom-reader.cjs +13 -13
- package/dist/smart-dom-reader.d.cts +186 -8
- package/dist/smart-dom-reader.d.ts +186 -8
- package/dist/smart-dom-reader.js +11 -11
- package/dist/theme-editor-preview.cjs +116 -98
- package/dist/theme-editor-preview.d.cts +188 -8
- package/dist/theme-editor-preview.d.ts +188 -8
- package/dist/theme-editor-preview.js +49 -54
- package/dist/theme-editor.cjs +2 -2
- package/dist/theme-editor.d.cts +187 -23
- package/dist/theme-editor.d.ts +187 -23
- package/dist/theme-editor.js +2 -2
- package/dist/theme-reference.d.cts +14 -0
- package/dist/theme-reference.d.ts +14 -0
- package/dist/ui-extras-entry-WAYFKVNW.js +3 -0
- package/dist/ui-extras.js +72 -0
- package/dist/voice-runtime-XLU6YK2N.js +17 -0
- package/dist/voice-runtime.cjs +17 -1
- package/dist/voice-runtime.d.cts +11656 -232
- package/dist/voice-runtime.d.ts +11656 -232
- package/dist/voice-runtime.js +17 -1
- package/dist/voice-worklet-player.cjs +19 -4
- package/dist/voice-worklet-player.d.cts +9 -10
- package/dist/voice-worklet-player.d.ts +9 -10
- package/dist/voice-worklet-player.js +19 -4
- package/dist/webmcp-polyfill.js +1 -1
- package/dist/webmcp-runtime-V6ZWW3PI.js +6 -0
- package/dist/webmcp-runtime.cjs +2 -2
- package/dist/webmcp-runtime.d.cts +1 -33
- package/dist/webmcp-runtime.d.ts +1 -33
- package/dist/webmcp-runtime.js +2 -2
- package/dist/widget.css +1 -1
- package/package.json +11 -5
- package/src/approval-paused-tool-settle.test.ts +325 -0
- package/src/client-approve.test.ts +311 -0
- package/src/client-history-eager.ts +9 -0
- package/src/client-history-loader.ts +23 -0
- package/src/client-history.ts +579 -0
- package/src/client-resume.ts +177 -0
- package/src/client-stream-eager.ts +12 -0
- package/src/client-stream-loader.ts +25 -0
- package/src/client-stream.ts +1880 -0
- package/src/client.test.ts +31 -1
- package/src/client.ts +293 -2607
- package/src/components/approval-actions.test.ts +27 -0
- package/src/components/approval-actions.ts +27 -3
- package/src/components/message-bubble.test.ts +19 -0
- package/src/components/message-bubble.ts +2 -0
- package/src/components/registry.ts +6 -6
- package/src/components/tool-bubble.test.ts +9 -0
- package/src/components/tool-bubble.ts +1 -0
- package/src/history-shell-bundle.test.ts +54 -0
- package/src/history-shell-eager.ts +12 -0
- package/src/history-shell-loader.ts +27 -0
- package/src/history-shell.ts +2362 -0
- package/src/history-view-loader.ts +2 -0
- package/src/history-view.ts +3 -0
- package/src/index-core.ts +6 -0
- package/src/index-global.ts +90 -248
- package/src/index.ts +13 -0
- package/src/install.ts +6 -4
- package/src/internal/runtype-history-provider.test.ts +1 -0
- package/src/internal/runtype-history-provider.ts +28 -13
- package/src/plugins/registry.ts +8 -8
- package/src/session-actions-chunk.test.ts +55 -0
- package/src/session-actions-eager.ts +11 -0
- package/src/session-actions-loader.ts +37 -0
- package/src/session-actions.ts +909 -0
- package/src/session-history.test.ts +3 -1
- package/src/session.ts +895 -1538
- package/src/session.voice.test.ts +1171 -5
- package/src/styles/widget.css +58 -0
- package/src/theme-editor/state.ts +91 -91
- package/src/types.ts +204 -5
- package/src/ui-ask-user-sheet.ts +382 -0
- package/src/ui-extras-eager.ts +10 -0
- package/src/ui-extras-entry.ts +9 -0
- package/src/ui-extras-loader.ts +26 -0
- package/src/ui.artifact-rail-resize.test.ts +2 -1
- package/src/ui.ask-user-chunk.test.ts +93 -0
- package/src/ui.history-chunk.test.ts +2 -1
- package/src/ui.history-config-provider.test.ts +2 -1
- package/src/ui.history-confirm-fallback.test.ts +142 -0
- package/src/ui.history-controller.test.ts +2 -1
- package/src/ui.history-lifecycle.test.ts +2 -1
- package/src/ui.history-render-hooks.test.ts +2 -1
- package/src/ui.history-shell.test.ts +2 -1
- package/src/ui.rail-overlay.test.ts +2 -1
- package/src/ui.rail-resize.test.ts +2 -1
- package/src/ui.rail-trigger-pending.test.ts +2 -1
- package/src/ui.ts +340 -2770
- package/src/utils/attachment-manager.ts +67 -67
- package/src/utils/constants.ts +3 -0
- package/src/utils/context-mention-controller.ts +323 -323
- package/src/utils/context-mention-manager.ts +40 -40
- package/src/utils/context-mention-orchestrator.ts +28 -0
- package/src/utils/event-stream-buffer.ts +50 -50
- package/src/utils/event-stream-store.ts +43 -43
- package/src/utils/throughput-tracker.ts +34 -34
- package/src/utils/virtual-scroller.ts +67 -67
- package/src/utils/visitor-store.test.ts +50 -0
- package/src/utils/visitor-store.ts +23 -5
- package/src/voice/__fixtures__/voice-wire/README.md +1 -0
- package/src/voice/__fixtures__/voice-wire/client/cancel.json +3 -0
- package/src/voice/__fixtures__/voice-wire/client/context.json +4 -0
- package/src/voice/__fixtures__/voice-wire/client/delegation_result.json +6 -0
- package/src/voice/__fixtures__/voice-wire/client/delegation_update.json +6 -0
- package/src/voice/__fixtures__/voice-wire/client/ping.json +3 -0
- package/src/voice/__fixtures__/voice-wire/client/playback_progress.json +5 -0
- package/src/voice/__fixtures__/voice-wire/client/start.json +3 -0
- package/src/voice/__fixtures__/voice-wire/server/artifact.json +11 -0
- package/src/voice/__fixtures__/voice-wire/server/attached.json +4 -0
- package/src/voice/__fixtures__/voice-wire/server/audio_clear.json +4 -0
- package/src/voice/__fixtures__/voice-wire/server/audio_end.json +3 -0
- package/src/voice/__fixtures__/voice-wire/server/delegation_cancelled.json +5 -0
- package/src/voice/__fixtures__/voice-wire/server/delegation_completed.json +8 -0
- package/src/voice/__fixtures__/voice-wire/server/delegation_started.json +15 -0
- package/src/voice/__fixtures__/voice-wire/server/error.json +6 -0
- package/src/voice/__fixtures__/voice-wire/server/metrics.json +10 -0
- package/src/voice/__fixtures__/voice-wire/server/pong.json +3 -0
- package/src/voice/__fixtures__/voice-wire/server/session_config.json +16 -0
- package/src/voice/__fixtures__/voice-wire/server/transcript_final.json +7 -0
- package/src/voice/__fixtures__/voice-wire/server/transcript_interim.json +4 -0
- package/src/voice/__fixtures__/voice-wire/server/transcript_partial.json +6 -0
- package/src/voice/__fixtures__/voice-wire/server/transcript_update.json +10 -0
- package/src/voice/__fixtures__/voice-wire/server/warning.json +5 -0
- package/src/voice/audio-playback-manager.test.ts +57 -0
- package/src/voice/audio-playback-manager.ts +120 -98
- package/src/voice/browser-voice-provider.ts +37 -37
- package/src/voice/fallback-speech-engine.ts +22 -16
- package/src/voice/keyed-voice-transcript.ts +314 -0
- package/src/voice/mic-state-styles.ts +182 -0
- package/src/voice/read-aloud-controller.ts +46 -44
- package/src/voice/runtype-speech-engine.ts +35 -33
- package/src/voice/runtype-voice-provider.ts +680 -189
- package/src/voice/session-voice-wiring.ts +266 -0
- package/src/voice/voice-delegation.test.ts +71 -0
- package/src/voice/voice-delegation.ts +364 -0
- package/src/voice/voice-turn-tracker.ts +6 -6
- package/src/voice/voice.test.ts +1060 -8
- package/src/voice/worklet-playback-engine.test.ts +81 -0
- package/src/voice/worklet-playback-engine.ts +21 -3
- package/src/voice-runtime-loader.ts +4 -1
- package/src/voice-runtime.ts +7 -0
- package/src/webmcp-runtime-entry.ts +41 -39
- package/dist/approval-ui-3NMYNXBD.js +0 -1
- package/dist/chunk-5EIIHQLQ.js +0 -1
- package/dist/chunk-7CO4T5KB.js +0 -1
- package/dist/chunk-EFJBUH5F.js +0 -1
- package/dist/chunk-R2TMIMDI.js +0 -3
- package/dist/chunk-UTVLOTWB.js +0 -9
- package/dist/context-mentions-45MODWOY.js +0 -161
- package/dist/event-stream-view-GL3ADRY2.js +0 -1
- package/dist/runtype-tts-entry-UJAEF7NZ.js +0 -1
- package/dist/voice-runtime-7DA7MJH4.js +0 -1
- package/dist/webmcp-runtime-JI7T4QYF.js +0 -6
|
@@ -4,17 +4,39 @@
|
|
|
4
4
|
// mock the voice factory to a fake provider and drive its onTranscript/onMetrics
|
|
5
5
|
// callbacks to verify how session.setupVoice() feeds the chat thread.
|
|
6
6
|
import { describe, it, expect, beforeEach, vi } from 'vitest';
|
|
7
|
-
import type {
|
|
7
|
+
import type {
|
|
8
|
+
AgentWidgetEvent,
|
|
9
|
+
AgentWidgetMessage,
|
|
10
|
+
VoiceMetrics,
|
|
11
|
+
VoiceDelegationRequest,
|
|
12
|
+
VoiceSessionBridge,
|
|
13
|
+
VoiceTranscriptMetadata,
|
|
14
|
+
} from './types';
|
|
15
|
+
|
|
16
|
+
/** A delegation request; utterance ids and messages default to empty. */
|
|
17
|
+
const req = (request: Partial<VoiceDelegationRequest> & { delegationId: string; userText: string }) => ({
|
|
18
|
+
userUtteranceIds: [],
|
|
19
|
+
messages: [],
|
|
20
|
+
...request,
|
|
21
|
+
});
|
|
8
22
|
|
|
9
23
|
// vi.mock is hoisted above module init, so the shared fake must be hoisted too.
|
|
10
24
|
const h = vi.hoisted(() => {
|
|
11
25
|
const state: {
|
|
12
26
|
transcriptCb:
|
|
13
|
-
| ((
|
|
27
|
+
| ((
|
|
28
|
+
role: 'user' | 'assistant',
|
|
29
|
+
text: string,
|
|
30
|
+
isFinal: boolean,
|
|
31
|
+
metadata?: VoiceTranscriptMetadata,
|
|
32
|
+
) => void)
|
|
14
33
|
| null;
|
|
15
34
|
metricsCb: ((m: VoiceMetrics) => void) | null;
|
|
35
|
+
statusCb: ((s: string) => void) | null;
|
|
36
|
+
errorCb: ((e: Error) => void) | null;
|
|
37
|
+
bridge: VoiceSessionBridge | null;
|
|
16
38
|
prewarms: number;
|
|
17
|
-
} = { transcriptCb: null, metricsCb: null, prewarms: 0 };
|
|
39
|
+
} = { transcriptCb: null, metricsCb: null, statusCb: null, errorCb: null, bridge: null, prewarms: 0 };
|
|
18
40
|
|
|
19
41
|
const fakeProvider = {
|
|
20
42
|
type: 'runtype' as const,
|
|
@@ -22,9 +44,14 @@ const h = vi.hoisted(() => {
|
|
|
22
44
|
disconnect: async () => {},
|
|
23
45
|
startListening: async () => {},
|
|
24
46
|
stopListening: async () => {},
|
|
47
|
+
stopPlayback: () => {},
|
|
25
48
|
onResult: () => {},
|
|
26
|
-
onError: () => {
|
|
27
|
-
|
|
49
|
+
onError: (cb: typeof state.errorCb) => {
|
|
50
|
+
state.errorCb = cb;
|
|
51
|
+
},
|
|
52
|
+
onStatusChange: (cb: typeof state.statusCb) => {
|
|
53
|
+
state.statusCb = cb;
|
|
54
|
+
},
|
|
28
55
|
onTranscript: (cb: typeof state.transcriptCb) => {
|
|
29
56
|
state.transcriptCb = cb;
|
|
30
57
|
},
|
|
@@ -34,6 +61,9 @@ const h = vi.hoisted(() => {
|
|
|
34
61
|
prewarm: () => {
|
|
35
62
|
state.prewarms += 1;
|
|
36
63
|
},
|
|
64
|
+
setSessionBridge: (bridge: VoiceSessionBridge) => {
|
|
65
|
+
state.bridge = bridge;
|
|
66
|
+
},
|
|
37
67
|
};
|
|
38
68
|
|
|
39
69
|
return { state, fakeProvider };
|
|
@@ -43,14 +73,22 @@ const h = vi.hoisted(() => {
|
|
|
43
73
|
// chunk loader; mock the loader so setupVoice adopts the fake provider.
|
|
44
74
|
vi.mock('./voice-runtime-loader', () => ({
|
|
45
75
|
setVoiceRuntimeLoader: () => {},
|
|
76
|
+
// The body runs after module init, so the statically imported class is bound.
|
|
46
77
|
loadVoiceRuntime: () =>
|
|
47
78
|
Promise.resolve({
|
|
48
79
|
createVoiceProvider: () => h.fakeProvider,
|
|
49
80
|
createBestAvailableVoiceProvider: () => h.fakeProvider,
|
|
50
81
|
isVoiceSupported: () => true,
|
|
82
|
+
KeyedVoiceTranscript,
|
|
83
|
+
createVoiceSessionBridge,
|
|
84
|
+
wireSessionVoice,
|
|
51
85
|
}),
|
|
52
86
|
}));
|
|
53
87
|
|
|
88
|
+
import { KeyedVoiceTranscript } from './voice/keyed-voice-transcript';
|
|
89
|
+
import { createVoiceSessionBridge } from './voice/voice-delegation';
|
|
90
|
+
import { wireSessionVoice } from './voice/session-voice-wiring';
|
|
91
|
+
|
|
54
92
|
import { AgentWidgetSession } from './session';
|
|
55
93
|
import { setRuntypeTtsLoader } from './voice/runtype-tts-loader';
|
|
56
94
|
|
|
@@ -153,6 +191,1134 @@ describe('AgentWidgetSession - realtime voice onTranscript (Option B)', () => {
|
|
|
153
191
|
});
|
|
154
192
|
});
|
|
155
193
|
|
|
194
|
+
describe('AgentWidgetSession - turn-keyed (full-duplex) voice transcripts', () => {
|
|
195
|
+
let session: AgentWidgetSession;
|
|
196
|
+
let messages: AgentWidgetMessage[] = [];
|
|
197
|
+
let streaming = false;
|
|
198
|
+
|
|
199
|
+
const drive = (
|
|
200
|
+
role: 'user' | 'assistant',
|
|
201
|
+
text: string,
|
|
202
|
+
isFinal: boolean,
|
|
203
|
+
turnId: string,
|
|
204
|
+
) => h.state.transcriptCb!(role, text, isFinal, { turnId });
|
|
205
|
+
const view = () => messages.map((m) => [m.role, m.content]);
|
|
206
|
+
const byContent = (content: string) => messages.find((m) => m.content === content)!;
|
|
207
|
+
const spoken = (id: string) =>
|
|
208
|
+
(session as unknown as { ttsSpokenMessageIds: Set<string> }).ttsSpokenMessageIds.has(id);
|
|
209
|
+
|
|
210
|
+
beforeEach(async () => {
|
|
211
|
+
h.state.transcriptCb = null;
|
|
212
|
+
h.state.statusCb = null;
|
|
213
|
+
h.state.errorCb = null;
|
|
214
|
+
messages = [];
|
|
215
|
+
streaming = false;
|
|
216
|
+
session = new AgentWidgetSession(
|
|
217
|
+
{
|
|
218
|
+
apiUrl: 'http://localhost:8000',
|
|
219
|
+
voiceRecognition: {
|
|
220
|
+
enabled: true,
|
|
221
|
+
provider: { type: 'runtype', runtype: { agentId: 'a1' } },
|
|
222
|
+
},
|
|
223
|
+
},
|
|
224
|
+
{
|
|
225
|
+
onMessagesChanged: (m) => {
|
|
226
|
+
messages = m;
|
|
227
|
+
},
|
|
228
|
+
onStatusChanged: () => {},
|
|
229
|
+
onStreamingChanged: (s) => {
|
|
230
|
+
streaming = s;
|
|
231
|
+
},
|
|
232
|
+
onError: () => {},
|
|
233
|
+
},
|
|
234
|
+
);
|
|
235
|
+
session.setupVoice();
|
|
236
|
+
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
237
|
+
});
|
|
238
|
+
|
|
239
|
+
it('upserts one bubble per (turnId, role) and waits without an empty placeholder', () => {
|
|
240
|
+
drive('user', 'what are', false, 'u1');
|
|
241
|
+
drive('user', 'what are your hours?', true, 'u1');
|
|
242
|
+
expect(view()).toEqual([['user', 'what are your hours?']]);
|
|
243
|
+
expect(messages[0].voiceProcessing).toBe(false);
|
|
244
|
+
expect(streaming).toBe(true); // standalone typing indicator, no placeholder bubble
|
|
245
|
+
|
|
246
|
+
drive('assistant', 'We open', false, 'a1');
|
|
247
|
+
drive('assistant', 'We open at nine.', true, 'a1');
|
|
248
|
+
expect(view()).toEqual([
|
|
249
|
+
['user', 'what are your hours?'],
|
|
250
|
+
['assistant', 'We open at nine.'],
|
|
251
|
+
]);
|
|
252
|
+
expect(messages[1]).toMatchObject({ streaming: false, voiceProcessing: false });
|
|
253
|
+
expect(spoken(messages[1].id)).toBe(true);
|
|
254
|
+
expect(streaming).toBe(false);
|
|
255
|
+
});
|
|
256
|
+
|
|
257
|
+
it('keeps an overlapping next user turn separate from the reply still streaming', () => {
|
|
258
|
+
drive('user', 'first question', true, 'u1');
|
|
259
|
+
drive('assistant', 'Here is', false, 'a1');
|
|
260
|
+
drive('user', 'and also', false, 'u2'); // user N+1 interim while assistant N streams
|
|
261
|
+
drive('assistant', 'Here is the full answer.', false, 'a1');
|
|
262
|
+
expect(view()).toEqual([
|
|
263
|
+
['user', 'first question'],
|
|
264
|
+
['assistant', 'Here is the full answer.'],
|
|
265
|
+
['user', 'and also'],
|
|
266
|
+
]);
|
|
267
|
+
expect(byContent('Here is the full answer.').streaming).toBe(true);
|
|
268
|
+
expect(byContent('and also').voiceProcessing).toBe(true);
|
|
269
|
+
|
|
270
|
+
drive('assistant', 'Here is the full answer.', true, 'a1');
|
|
271
|
+
drive('user', 'and also the price?', true, 'u2');
|
|
272
|
+
expect(view()).toEqual([
|
|
273
|
+
['user', 'first question'],
|
|
274
|
+
['assistant', 'Here is the full answer.'],
|
|
275
|
+
['user', 'and also the price?'],
|
|
276
|
+
]);
|
|
277
|
+
expect(streaming).toBe(true); // u2 awaits its reply
|
|
278
|
+
|
|
279
|
+
drive('assistant', 'It costs $5.', true, 'a2');
|
|
280
|
+
expect(view().at(-1)).toEqual(['assistant', 'It costs $5.']);
|
|
281
|
+
expect(streaming).toBe(false);
|
|
282
|
+
});
|
|
283
|
+
|
|
284
|
+
it('separates overlapping turns that share one turnId per user/assistant pair', () => {
|
|
285
|
+
drive('user', 'one', true, 't1');
|
|
286
|
+
drive('assistant', 'Reply', false, 't1');
|
|
287
|
+
drive('user', 'two', false, 't2');
|
|
288
|
+
drive('assistant', 'Reply one.', true, 't1');
|
|
289
|
+
drive('user', 'two?', true, 't2');
|
|
290
|
+
drive('assistant', 'Reply two.', true, 't2');
|
|
291
|
+
expect(view()).toEqual([
|
|
292
|
+
['user', 'one'],
|
|
293
|
+
['assistant', 'Reply one.'],
|
|
294
|
+
['user', 'two?'],
|
|
295
|
+
['assistant', 'Reply two.'],
|
|
296
|
+
]);
|
|
297
|
+
});
|
|
298
|
+
|
|
299
|
+
it('renders a late user transcript above its own turn reply', () => {
|
|
300
|
+
drive('assistant', 'Sure, I can', false, 't1');
|
|
301
|
+
drive('user', 'next thing', false, 't2');
|
|
302
|
+
drive('user', 'Can you help?', true, 't1');
|
|
303
|
+
expect(view()).toEqual([
|
|
304
|
+
['user', 'Can you help?'],
|
|
305
|
+
['assistant', 'Sure, I can'],
|
|
306
|
+
['user', 'next thing'],
|
|
307
|
+
]);
|
|
308
|
+
// Later frames keep the reordered bubble in place.
|
|
309
|
+
drive('user', 'Can you help me?', true, 't1');
|
|
310
|
+
drive('assistant', 'Sure, I can help.', true, 't1');
|
|
311
|
+
expect(view()).toEqual([
|
|
312
|
+
['user', 'Can you help me?'],
|
|
313
|
+
['assistant', 'Sure, I can help.'],
|
|
314
|
+
['user', 'next thing'],
|
|
315
|
+
]);
|
|
316
|
+
});
|
|
317
|
+
|
|
318
|
+
it('stopping playback discards the rest of the in-flight reply only', () => {
|
|
319
|
+
drive('user', 'tell me a story', true, 't1');
|
|
320
|
+
drive('assistant', 'Once upon', false, 't1');
|
|
321
|
+
session.stopVoicePlayback();
|
|
322
|
+
expect(byContent('Once upon')).toMatchObject({ streaming: false, voiceProcessing: false });
|
|
323
|
+
expect(streaming).toBe(false);
|
|
324
|
+
|
|
325
|
+
drive('assistant', 'Once upon a time there was', true, 't1');
|
|
326
|
+
drive('user', 'something else', true, 't2');
|
|
327
|
+
drive('assistant', 'Sure.', true, 't2');
|
|
328
|
+
expect(view()).toEqual([
|
|
329
|
+
['user', 'tell me a story'],
|
|
330
|
+
['assistant', 'Once upon'],
|
|
331
|
+
['user', 'something else'],
|
|
332
|
+
['assistant', 'Sure.'],
|
|
333
|
+
]);
|
|
334
|
+
});
|
|
335
|
+
|
|
336
|
+
it('stopping while a turn awaits its reply still renders a later reply under a new turnId', () => {
|
|
337
|
+
// The provider drops the cancelled reply until the server acknowledges the
|
|
338
|
+
// stop; anything after that (e.g. a delegation result the stop didn't
|
|
339
|
+
// cancel) is audible, so it must get a bubble.
|
|
340
|
+
drive('user', 'old question', true, 'u1');
|
|
341
|
+
session.stopVoicePlayback();
|
|
342
|
+
expect(streaming).toBe(false);
|
|
343
|
+
drive('assistant', 'The answer you asked for.', true, 'a1');
|
|
344
|
+
expect(view()).toEqual([
|
|
345
|
+
['user', 'old question'],
|
|
346
|
+
['assistant', 'The answer you asked for.'],
|
|
347
|
+
]);
|
|
348
|
+
expect(spoken(byContent('The answer you asked for.').id)).toBe(true);
|
|
349
|
+
});
|
|
350
|
+
|
|
351
|
+
it('keeps late bubbles placed ahead of the same reply in startMs order', () => {
|
|
352
|
+
h.state.transcriptCb!('assistant', 'reply', true, { turnId: 'a1', startMs: 2000 });
|
|
353
|
+
h.state.transcriptCb!('user', 'at one second', true, { turnId: 'u1', startMs: 1000 });
|
|
354
|
+
h.state.transcriptCb!('user', 'at 1.5 seconds', true, { turnId: 'u2', startMs: 1500 });
|
|
355
|
+
expect(view().map((v) => v[1])).toEqual(['at one second', 'at 1.5 seconds', 'reply']);
|
|
356
|
+
const seq = (content: string) => byContent(content).sequence!;
|
|
357
|
+
expect(seq('at one second')).toBeLessThan(seq('at 1.5 seconds'));
|
|
358
|
+
expect(seq('at 1.5 seconds')).toBeLessThan(seq('reply'));
|
|
359
|
+
expect(byContent('at one second').createdAt).toBe(byContent('reply').createdAt);
|
|
360
|
+
});
|
|
361
|
+
|
|
362
|
+
it('re-positions a bubble when its startMs arrives on a later update', () => {
|
|
363
|
+
drive('user', 'first', true, 'u1');
|
|
364
|
+
h.state.transcriptCb!('assistant', 'reply', true, { turnId: 'a1', startMs: 2000 });
|
|
365
|
+
h.state.transcriptCb!('user', 'spoke ear', false, { turnId: 'u2' }); // no timestamp yet
|
|
366
|
+
expect(view().map((v) => v[1])).toEqual(['first', 'reply', 'spoke ear']);
|
|
367
|
+
h.state.transcriptCb!('user', 'spoke early', true, { turnId: 'u2', startMs: 1000 });
|
|
368
|
+
expect(view().map((v) => v[1])).toEqual(['first', 'spoke early', 'reply']);
|
|
369
|
+
});
|
|
370
|
+
|
|
371
|
+
it('orders a late-arriving utterance by its startMs among the call\'s bubbles', () => {
|
|
372
|
+
const at = (role: 'user' | 'assistant', text: string, turnId: string, startMs?: number) =>
|
|
373
|
+
h.state.transcriptCb!(role, text, true, { turnId, ...(startMs !== undefined && { startMs }) });
|
|
374
|
+
at('user', 'first', 'u1', 0);
|
|
375
|
+
at('assistant', 'reply one', 'a1', 2000);
|
|
376
|
+
at('assistant', 'reply two', 'a2', 5000);
|
|
377
|
+
at('user', 'spoke at one second', 'u2', 1000); // transcribed late
|
|
378
|
+
at('user', 'spoke at four seconds', 'u3', 4000);
|
|
379
|
+
at('user', 'no timestamp', 'u4');
|
|
380
|
+
expect(view()).toEqual([
|
|
381
|
+
['user', 'first'],
|
|
382
|
+
['user', 'spoke at one second'],
|
|
383
|
+
['assistant', 'reply one'],
|
|
384
|
+
['user', 'spoke at four seconds'],
|
|
385
|
+
['assistant', 'reply two'],
|
|
386
|
+
['user', 'no timestamp'],
|
|
387
|
+
]);
|
|
388
|
+
});
|
|
389
|
+
|
|
390
|
+
it('survives listening status mid-call and settles everything when the call ends', () => {
|
|
391
|
+
drive('user', 'hello', true, 'u1');
|
|
392
|
+
drive('assistant', 'Hi the', false, 'a1');
|
|
393
|
+
h.state.statusCb!('listening');
|
|
394
|
+
expect(byContent('Hi the').streaming).toBe(true);
|
|
395
|
+
drive('assistant', 'Hi there', false, 'a1');
|
|
396
|
+
drive('user', 'bye', false, 'u2');
|
|
397
|
+
h.state.statusCb!('idle');
|
|
398
|
+
expect(messages.some((m) => m.streaming || m.voiceProcessing)).toBe(false);
|
|
399
|
+
expect(view()).toEqual([
|
|
400
|
+
['user', 'hello'],
|
|
401
|
+
['assistant', 'Hi there'],
|
|
402
|
+
['user', 'bye'],
|
|
403
|
+
]);
|
|
404
|
+
expect(streaming).toBe(false);
|
|
405
|
+
expect(spoken(byContent('Hi there').id)).toBe(true);
|
|
406
|
+
});
|
|
407
|
+
|
|
408
|
+
it('shows the processing error when the awaited reply fails', () => {
|
|
409
|
+
const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {});
|
|
410
|
+
drive('user', 'hello', true, 'u1');
|
|
411
|
+
h.state.errorCb!(new Error('boom'));
|
|
412
|
+
consoleError.mockRestore();
|
|
413
|
+
expect(view()).toEqual([
|
|
414
|
+
['user', 'hello'],
|
|
415
|
+
['assistant', 'Voice processing failed. Please try again.'],
|
|
416
|
+
]);
|
|
417
|
+
expect(streaming).toBe(false);
|
|
418
|
+
});
|
|
419
|
+
|
|
420
|
+
it('closes a partially streamed reply on a voice error so the composer unlocks', () => {
|
|
421
|
+
const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {});
|
|
422
|
+
drive('user', 'hello', true, 'u1');
|
|
423
|
+
drive('assistant', 'Hi, I was saying', false, 'a1');
|
|
424
|
+
expect(streaming).toBe(true);
|
|
425
|
+
h.state.errorCb!(new Error('boom'));
|
|
426
|
+
consoleError.mockRestore();
|
|
427
|
+
expect(view()).toEqual([
|
|
428
|
+
['user', 'hello'],
|
|
429
|
+
['assistant', 'Hi, I was saying'],
|
|
430
|
+
['assistant', 'Voice processing failed. Please try again.'],
|
|
431
|
+
]);
|
|
432
|
+
expect(messages.some((m) => m.streaming || m.voiceProcessing)).toBe(false);
|
|
433
|
+
expect(streaming).toBe(false);
|
|
434
|
+
expect(session.isStreaming()).toBe(false);
|
|
435
|
+
expect(spoken(byContent('Hi, I was saying').id)).toBe(true);
|
|
436
|
+
});
|
|
437
|
+
|
|
438
|
+
it('keeps the untagged alternating path for transcripts without a turnId', () => {
|
|
439
|
+
h.state.transcriptCb!('user', 'hi', true);
|
|
440
|
+
expect(messages.map((m) => [m.role, m.content, m.streaming])).toEqual([
|
|
441
|
+
['user', 'hi', false],
|
|
442
|
+
['assistant', '', true], // legacy placeholder is still injected
|
|
443
|
+
]);
|
|
444
|
+
});
|
|
445
|
+
});
|
|
446
|
+
|
|
447
|
+
describe('AgentWidgetSession - voice client delegation bridge', () => {
|
|
448
|
+
type Dispatch = (
|
|
449
|
+
options: { messages: AgentWidgetMessage[]; signal?: AbortSignal },
|
|
450
|
+
onEvent: (event: AgentWidgetEvent) => void,
|
|
451
|
+
) => Promise<void>;
|
|
452
|
+
|
|
453
|
+
let session: AgentWidgetSession;
|
|
454
|
+
let messages: AgentWidgetMessage[] = [];
|
|
455
|
+
let dispatch: ReturnType<typeof vi.fn<Dispatch>>;
|
|
456
|
+
|
|
457
|
+
const drive = (role: 'user' | 'assistant', text: string, isFinal: boolean, turnId: string) =>
|
|
458
|
+
h.state.transcriptCb!(role, text, isFinal, { turnId });
|
|
459
|
+
const view = () => messages.map((m) => [m.role, m.content]);
|
|
460
|
+
const spoken = (id: string) =>
|
|
461
|
+
(session as unknown as { ttsSpokenMessageIds: Set<string> }).ttsSpokenMessageIds.has(id);
|
|
462
|
+
const reply = (onEvent: (event: AgentWidgetEvent) => void, text: string, id = 'assistant-r1') => {
|
|
463
|
+
const base = { id, role: 'assistant' as const, createdAt: new Date().toISOString() };
|
|
464
|
+
onEvent({ type: 'status', status: 'connecting' });
|
|
465
|
+
onEvent({ type: 'message', message: { ...base, content: text.slice(0, 4), streaming: true } });
|
|
466
|
+
onEvent({ type: 'message', message: { ...base, content: text, streaming: false } });
|
|
467
|
+
onEvent({ type: 'status', status: 'idle' });
|
|
468
|
+
};
|
|
469
|
+
|
|
470
|
+
beforeEach(async () => {
|
|
471
|
+
h.state.transcriptCb = null;
|
|
472
|
+
h.state.bridge = null;
|
|
473
|
+
messages = [];
|
|
474
|
+
session = new AgentWidgetSession(
|
|
475
|
+
{
|
|
476
|
+
apiUrl: 'http://localhost:8000',
|
|
477
|
+
initialMessages: [
|
|
478
|
+
{ id: 'm0', role: 'assistant', content: 'Welcome! How can I help?', createdAt: '2026-01-01T00:00:00.000Z' },
|
|
479
|
+
],
|
|
480
|
+
voiceRecognition: {
|
|
481
|
+
enabled: true,
|
|
482
|
+
provider: { type: 'runtype', runtype: { agentId: 'a1' } },
|
|
483
|
+
},
|
|
484
|
+
},
|
|
485
|
+
{
|
|
486
|
+
onMessagesChanged: (m) => {
|
|
487
|
+
messages = m;
|
|
488
|
+
},
|
|
489
|
+
onStatusChanged: () => {},
|
|
490
|
+
onStreamingChanged: () => {},
|
|
491
|
+
onError: () => {},
|
|
492
|
+
},
|
|
493
|
+
);
|
|
494
|
+
dispatch = vi.fn<Dispatch>();
|
|
495
|
+
(session as unknown as { client: { dispatch: Dispatch } }).client.dispatch = dispatch;
|
|
496
|
+
session.setupVoice();
|
|
497
|
+
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
498
|
+
});
|
|
499
|
+
|
|
500
|
+
it('passes clientDelegation and callContext through to the runtype provider config', () => {
|
|
501
|
+
const config = (
|
|
502
|
+
session as unknown as { getVoiceConfigFromConfig(): { runtype?: Record<string, unknown> } }
|
|
503
|
+
);
|
|
504
|
+
const callContext = () => 'ctx';
|
|
505
|
+
session.updateConfig({
|
|
506
|
+
apiUrl: 'http://localhost:8000',
|
|
507
|
+
voiceRecognition: {
|
|
508
|
+
enabled: true,
|
|
509
|
+
provider: { type: 'runtype', runtype: { agentId: 'a1', clientDelegation: false, callContext } },
|
|
510
|
+
},
|
|
511
|
+
});
|
|
512
|
+
expect(config.getVoiceConfigFromConfig().runtype).toMatchObject({ clientDelegation: false, callContext });
|
|
513
|
+
});
|
|
514
|
+
|
|
515
|
+
it('hands the provider a bridge whose history is the visible settled messages', () => {
|
|
516
|
+
drive('user', 'still talk', false, 'u1'); // interim: not part of the context
|
|
517
|
+
// An earlier call's captions (filler, small talk) aren't conversation either.
|
|
518
|
+
h.state.transcriptCb!('assistant', 'Let me check.', true, { turnId: 'f0', caption: true });
|
|
519
|
+
h.state.transcriptCb!('user', 'hi there', true, { turnId: 'u0', caption: true });
|
|
520
|
+
expect(h.state.bridge!.getHistory()).toEqual([
|
|
521
|
+
{ role: 'assistant', content: 'Welcome! How can I help?' },
|
|
522
|
+
]);
|
|
523
|
+
});
|
|
524
|
+
|
|
525
|
+
it('submits the transcript bubble as the user message once and answers with the reply', async () => {
|
|
526
|
+
drive('user', 'whats the weather in paris', true, 'u1');
|
|
527
|
+
drive('assistant', 'Let me check that.', true, 'f1'); // filler stays visible
|
|
528
|
+
dispatch.mockImplementation(async (_options, onEvent) => reply(onEvent, 'It is **sunny** in Paris.'));
|
|
529
|
+
|
|
530
|
+
const result = await h.state.bridge!.runDelegatedTurn(req({
|
|
531
|
+
delegationId: 'd1',
|
|
532
|
+
userText: "What's the weather in Paris?" }));
|
|
533
|
+
|
|
534
|
+
expect(result).toEqual({ status: 'completed', text: 'It is **sunny** in Paris.' });
|
|
535
|
+
const sent = dispatch.mock.calls[0][0].messages;
|
|
536
|
+
const userTurns = sent.filter((m) => m.role === 'user');
|
|
537
|
+
expect(userTurns).toHaveLength(1);
|
|
538
|
+
expect(userTurns[0]).toMatchObject({
|
|
539
|
+
content: 'whats the weather in paris',
|
|
540
|
+
llmContent: "What's the weather in Paris?",
|
|
541
|
+
viaVoice: true,
|
|
542
|
+
});
|
|
543
|
+
expect(view()).toEqual([
|
|
544
|
+
['assistant', 'Welcome! How can I help?'],
|
|
545
|
+
['user', 'whats the weather in paris'],
|
|
546
|
+
['assistant', 'Let me check that.'],
|
|
547
|
+
['assistant', 'It is **sunny** in Paris.'],
|
|
548
|
+
]);
|
|
549
|
+
expect(spoken('assistant-r1')).toBe(true);
|
|
550
|
+
expect(session.isStreaming()).toBe(false);
|
|
551
|
+
});
|
|
552
|
+
|
|
553
|
+
it('captions speech in a delegated call and ends the request on the submitted bubble', async () => {
|
|
554
|
+
const caption = (role: 'user' | 'assistant', text: string, isFinal: boolean, turnId: string) =>
|
|
555
|
+
h.state.transcriptCb!(role, text, isFinal, { turnId, caption: true });
|
|
556
|
+
caption('user', 'hi there', true, 'u0'); // never delegated
|
|
557
|
+
caption('user', 'What are your opening hours', true, 'u1');
|
|
558
|
+
caption('assistant', "Yeah, I'll", false, 'f1'); // filler, still streaming
|
|
559
|
+
dispatch.mockImplementation(async (_options, onEvent) => reply(onEvent, 'We open at nine.'));
|
|
560
|
+
|
|
561
|
+
await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'What are your opening hours' }));
|
|
562
|
+
|
|
563
|
+
const sent = dispatch.mock.calls[0][0].messages;
|
|
564
|
+
const conversation = sent.filter((m) => !m.voiceCaption);
|
|
565
|
+
expect(conversation.map((m) => [m.role, m.content])).toEqual([
|
|
566
|
+
['assistant', 'Welcome! How can I help?'],
|
|
567
|
+
['user', 'What are your opening hours'],
|
|
568
|
+
]);
|
|
569
|
+
expect(sent[sent.length - 1].content).toBe('What are your opening hours');
|
|
570
|
+
// The display keeps every caption, in spoken order.
|
|
571
|
+
expect(view()).toEqual([
|
|
572
|
+
['assistant', 'Welcome! How can I help?'],
|
|
573
|
+
['user', 'hi there'],
|
|
574
|
+
['user', 'What are your opening hours'],
|
|
575
|
+
['assistant', "Yeah, I'll"],
|
|
576
|
+
['assistant', 'We open at nine.'],
|
|
577
|
+
]);
|
|
578
|
+
expect(messages.find((m) => m.content === 'What are your opening hours')!.voiceCaption).toBeUndefined();
|
|
579
|
+
expect(messages.find((m) => m.content === "Yeah, I'll")!.voiceCaption).toBe(true);
|
|
580
|
+
|
|
581
|
+
// A later typed turn doesn't send the captions either.
|
|
582
|
+
dispatch.mockImplementation(async (_options, onEvent) => reply(onEvent, 'Yes.', 'assistant-r2'));
|
|
583
|
+
await session.sendMessage('and on sundays?');
|
|
584
|
+
const typed = dispatch.mock.calls[1][0].messages.filter((m) => !m.voiceCaption);
|
|
585
|
+
expect(typed.map((m) => m.content)).toEqual([
|
|
586
|
+
'Welcome! How can I help?',
|
|
587
|
+
'What are your opening hours',
|
|
588
|
+
'We open at nine.',
|
|
589
|
+
'and on sundays?',
|
|
590
|
+
]);
|
|
591
|
+
});
|
|
592
|
+
|
|
593
|
+
it('trims GPT-Live\'s leading space from bubbles and the submitted request', async () => {
|
|
594
|
+
h.state.transcriptCb!('user', ' What are your opening hours', true, { turnId: 'u1', caption: true });
|
|
595
|
+
dispatch.mockImplementation(async (_options, onEvent) => reply(onEvent, 'Nine to five.'));
|
|
596
|
+
await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: ' What are your opening hours' }));
|
|
597
|
+
const user = dispatch.mock.calls[0][0].messages.find((m) => m.role === 'user')!;
|
|
598
|
+
expect(user.content).toBe('What are your opening hours');
|
|
599
|
+
expect(user.llmContent).toBeUndefined();
|
|
600
|
+
});
|
|
601
|
+
|
|
602
|
+
it('keeps a filler finalized after delegation completes, beside the rendered result', async () => {
|
|
603
|
+
// Core rotates the assistant transcript id at delegation_completed: the
|
|
604
|
+
// filler's late final keeps its old id (and its bubble), while the
|
|
605
|
+
// read-back under the new id is folded by the provider before it gets here.
|
|
606
|
+
const caption = (role: 'user' | 'assistant', text: string, isFinal: boolean, turnId: string) =>
|
|
607
|
+
h.state.transcriptCb!(role, text, isFinal, { turnId, caption: true });
|
|
608
|
+
caption('user', 'What are your opening hours', true, 'u1');
|
|
609
|
+
caption('assistant', "Yeah, I'll", false, 'f1');
|
|
610
|
+
dispatch.mockImplementation(async (_options, onEvent) => reply(onEvent, 'We open at nine.'));
|
|
611
|
+
await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'What are your opening hours' }));
|
|
612
|
+
caption('assistant', "Yeah, I'll check.", true, 'f1'); // late final, after delegation_completed
|
|
613
|
+
|
|
614
|
+
const filler = messages.find((m) => m.content === "Yeah, I'll check.")!;
|
|
615
|
+
expect(filler).toMatchObject({ streaming: false, voiceProcessing: false, voiceCaption: true });
|
|
616
|
+
expect(view()).toEqual([
|
|
617
|
+
['assistant', 'Welcome! How can I help?'],
|
|
618
|
+
['user', 'What are your opening hours'],
|
|
619
|
+
['assistant', "Yeah, I'll check."],
|
|
620
|
+
['assistant', 'We open at nine.'],
|
|
621
|
+
]);
|
|
622
|
+
expect(session.isStreaming()).toBe(false);
|
|
623
|
+
});
|
|
624
|
+
|
|
625
|
+
it('ends the request on the submitted bubble even when a typed turn finished after it', async () => {
|
|
626
|
+
drive('user', 'voice question', true, 'u1');
|
|
627
|
+
dispatch.mockImplementationOnce(async (_options, onEvent) => reply(onEvent, 'Typed answer.', 'assistant-typed'));
|
|
628
|
+
await session.sendMessage('typed question');
|
|
629
|
+
dispatch.mockImplementationOnce(async (_options, onEvent) => reply(onEvent, 'Voice answer.'));
|
|
630
|
+
await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'voice question' }));
|
|
631
|
+
const sent = dispatch.mock.calls[1][0].messages;
|
|
632
|
+
expect(sent[sent.length - 1]).toMatchObject({ role: 'user', content: 'voice question' });
|
|
633
|
+
expect(sent.filter((m) => m.content === 'voice question')).toHaveLength(1);
|
|
634
|
+
});
|
|
635
|
+
|
|
636
|
+
it('keeps the transcript text as-is when it matches the request', async () => {
|
|
637
|
+
drive('user', 'Book a table.', true, 'u1');
|
|
638
|
+
dispatch.mockImplementation(async (_options, onEvent) => reply(onEvent, 'Booked.'));
|
|
639
|
+
await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'Book a table.' }));
|
|
640
|
+
const user = dispatch.mock.calls[0][0].messages.find((m) => m.role === 'user')!;
|
|
641
|
+
expect(user.content).toBe('Book a table.');
|
|
642
|
+
expect(user.llmContent).toBeUndefined();
|
|
643
|
+
});
|
|
644
|
+
|
|
645
|
+
it('appends a user message when no transcript bubble is available', async () => {
|
|
646
|
+
dispatch.mockImplementation(async (_options, onEvent) => reply(onEvent, 'Done.'));
|
|
647
|
+
const result = await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'Turn on dark mode' }));
|
|
648
|
+
expect(result).toEqual({ status: 'completed', text: 'Done.' });
|
|
649
|
+
expect(view()).toEqual([
|
|
650
|
+
['assistant', 'Welcome! How can I help?'],
|
|
651
|
+
['user', 'Turn on dark mode'],
|
|
652
|
+
['assistant', 'Done.'],
|
|
653
|
+
]);
|
|
654
|
+
expect(messages[1].viaVoice).toBe(true);
|
|
655
|
+
});
|
|
656
|
+
|
|
657
|
+
it('answers ok:false when the turn errors', async () => {
|
|
658
|
+
const consoleError = vi.spyOn(console, 'error').mockImplementation(() => {});
|
|
659
|
+
drive('user', 'do the thing', true, 'u1');
|
|
660
|
+
dispatch.mockRejectedValue(new Error('upstream exploded'));
|
|
661
|
+
const result = await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'do the thing' }));
|
|
662
|
+
consoleError.mockRestore();
|
|
663
|
+
expect(result.status).toBe('failed');
|
|
664
|
+
// The visible user message is still exactly the transcript bubble.
|
|
665
|
+
expect(messages.filter((m) => m.role === 'user')).toHaveLength(1);
|
|
666
|
+
});
|
|
667
|
+
|
|
668
|
+
it('answers ok:false when the visitor stops the turn', async () => {
|
|
669
|
+
drive('user', 'long task', true, 'u1');
|
|
670
|
+
dispatch.mockImplementation(
|
|
671
|
+
(options) =>
|
|
672
|
+
new Promise((_resolve, reject) => {
|
|
673
|
+
options.signal?.addEventListener('abort', () =>
|
|
674
|
+
reject(Object.assign(new Error('aborted'), { name: 'AbortError' })),
|
|
675
|
+
);
|
|
676
|
+
}),
|
|
677
|
+
);
|
|
678
|
+
const pending = h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'long task' }));
|
|
679
|
+
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
680
|
+
session.cancel();
|
|
681
|
+
expect(await pending).toEqual({ status: 'failed', text: '' });
|
|
682
|
+
});
|
|
683
|
+
|
|
684
|
+
const parkOnApproval = (approval: Partial<NonNullable<AgentWidgetMessage['approval']>> = {}, key = 'ap1') =>
|
|
685
|
+
dispatch.mockImplementationOnce(async (_options, onEvent) => {
|
|
686
|
+
const createdAt = new Date().toISOString();
|
|
687
|
+
onEvent({ type: 'status', status: 'connecting' });
|
|
688
|
+
onEvent({ type: 'message', message: { id: `r-${key}`, role: 'assistant', content: 'I can do that.', createdAt } });
|
|
689
|
+
onEvent({
|
|
690
|
+
type: 'message',
|
|
691
|
+
message: {
|
|
692
|
+
id: `approval-${key}`,
|
|
693
|
+
role: 'assistant',
|
|
694
|
+
content: '',
|
|
695
|
+
createdAt,
|
|
696
|
+
variant: 'approval',
|
|
697
|
+
approval: {
|
|
698
|
+
id: key,
|
|
699
|
+
status: 'pending',
|
|
700
|
+
agentId: 'a1',
|
|
701
|
+
executionId: 'e1',
|
|
702
|
+
toolName: 'place_pickup_order',
|
|
703
|
+
description: 'Place a pickup order at the bakery',
|
|
704
|
+
parameters: {
|
|
705
|
+
items: [
|
|
706
|
+
{ name: 'almond croissants', quantity: 2 },
|
|
707
|
+
{ name: 'sourdough loaf', quantity: 1 },
|
|
708
|
+
],
|
|
709
|
+
pickupTime: 'today 4pm',
|
|
710
|
+
customerName: 'Nathan',
|
|
711
|
+
_approvalReason: 'hidden',
|
|
712
|
+
},
|
|
713
|
+
reason: 'The visitor asked for a pickup order',
|
|
714
|
+
...approval,
|
|
715
|
+
},
|
|
716
|
+
},
|
|
717
|
+
});
|
|
718
|
+
onEvent({ type: 'status', status: 'idle' });
|
|
719
|
+
});
|
|
720
|
+
const internals = () =>
|
|
721
|
+
session as unknown as {
|
|
722
|
+
client: { resolveApproval: (...args: unknown[]) => Promise<unknown> };
|
|
723
|
+
handleEvent: (event: AgentWidgetEvent) => void;
|
|
724
|
+
abortController: AbortController | null;
|
|
725
|
+
};
|
|
726
|
+
/** The approval round-trip resumes the turn with `text` (or no body). */
|
|
727
|
+
const resumeWith = (text: string | null) => {
|
|
728
|
+
internals().client.resolveApproval = async () => (text === null ? undefined : new ReadableStream());
|
|
729
|
+
vi.spyOn(session, 'connectStream').mockImplementation(async () => {
|
|
730
|
+
reply(internals().handleEvent, text!, 'assistant-after');
|
|
731
|
+
internals().abortController = null;
|
|
732
|
+
});
|
|
733
|
+
};
|
|
734
|
+
const approvalOf = (key = 'ap1') => messages.find((m) => m.id === `approval-${key}`)!.approval!;
|
|
735
|
+
|
|
736
|
+
it('asks for a parked approval in words the voice model can say', async () => {
|
|
737
|
+
drive('user', 'order two croissants', true, 'u1');
|
|
738
|
+
parkOnApproval();
|
|
739
|
+
const result = await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'order two croissants' }));
|
|
740
|
+
expect(result.status).not.toBe('failed');
|
|
741
|
+
expect(result.text).toBe(
|
|
742
|
+
'I can do that.\n\n' +
|
|
743
|
+
"This action needs the user's approval in the chat before it happens:\n" +
|
|
744
|
+
'- place pickup order (Place a pickup order at the bakery) with items: 2 almond croissants, 1 sourdough loaf; ' +
|
|
745
|
+
'pickup time: today 4pm; customer name: Nathan because: The visitor asked for a pickup order\n\n' +
|
|
746
|
+
"Briefly tell the user what you're about to do and ask them to approve or decline it in the chat. Don't claim it's done.",
|
|
747
|
+
);
|
|
748
|
+
expect(result.followUp).toBeTypeOf('function');
|
|
749
|
+
});
|
|
750
|
+
|
|
751
|
+
it('follows up with the answer once the visitor approves, kept off browser TTS', async () => {
|
|
752
|
+
drive('user', 'order two croissants', true, 'u1');
|
|
753
|
+
parkOnApproval();
|
|
754
|
+
const result = await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'order two croissants' }));
|
|
755
|
+
const followUp = result.followUp!({ signal: new AbortController().signal });
|
|
756
|
+
resumeWith('Your order is in: pickup today at 4pm.');
|
|
757
|
+
await session.resolveApproval(approvalOf(), 'approved');
|
|
758
|
+
expect(await followUp).toEqual({ status: 'completed', text: 'Your order is in: pickup today at 4pm.' });
|
|
759
|
+
expect(spoken('assistant-after')).toBe(true);
|
|
760
|
+
});
|
|
761
|
+
|
|
762
|
+
it("follows up once its own approval is decided, not another turn's", async () => {
|
|
763
|
+
drive('user', 'order two croissants', true, 'u1');
|
|
764
|
+
parkOnApproval();
|
|
765
|
+
const first = await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'order two croissants' }));
|
|
766
|
+
const followUp = first.followUp!({ signal: new AbortController().signal });
|
|
767
|
+
drive('user', 'and a cake', true, 'u2');
|
|
768
|
+
parkOnApproval({ toolName: 'order_cake' }, 'ap2');
|
|
769
|
+
await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd2', userText: 'and a cake' }));
|
|
770
|
+
expect(approvalOf('ap2').status).toBe('pending');
|
|
771
|
+
|
|
772
|
+
resumeWith('Croissants ordered.');
|
|
773
|
+
await session.resolveApproval(approvalOf(), 'approved');
|
|
774
|
+
expect(await followUp).toEqual({ status: 'completed', text: 'Croissants ordered.' });
|
|
775
|
+
expect(approvalOf('ap2').status).toBe('pending');
|
|
776
|
+
});
|
|
777
|
+
|
|
778
|
+
describe('Amendment 4: voice decline, supersede, expiry', () => {
|
|
779
|
+
const park = async (key = 'ap1', toolName = 'place_pickup_order') => {
|
|
780
|
+
parkOnApproval({ toolName }, key);
|
|
781
|
+
const result = await h.state.bridge!.runDelegatedTurn(req({ delegationId: `d-${key}`, userText: 'order two croissants' }));
|
|
782
|
+
return { result, followUp: result.followUp!({ signal: new AbortController().signal }) };
|
|
783
|
+
};
|
|
784
|
+
|
|
785
|
+
it('declines the one pending voice approval on an unambiguous "cancel that", silently for its follow-up', async () => {
|
|
786
|
+
const first = await park();
|
|
787
|
+
expect(first.result.status).toBe('pending_approval');
|
|
788
|
+
resumeWith(null);
|
|
789
|
+
const decline = await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd2', userText: ' No, cancel that.' }));
|
|
790
|
+
expect(decline).toEqual({
|
|
791
|
+
status: 'denied',
|
|
792
|
+
text: 'Okay, I cancelled the place pickup order request. Nothing was done.',
|
|
793
|
+
inChat: false, // only spoken: its read-back stays as a caption
|
|
794
|
+
});
|
|
795
|
+
expect(dispatch).toHaveBeenCalledTimes(1); // no chat turn for the decline
|
|
796
|
+
expect(approvalOf().status).toBe('denied');
|
|
797
|
+
// The parked delegation still gets its one terminal result, with nothing to say.
|
|
798
|
+
expect(await first.followUp).toEqual({ status: 'denied', text: '' });
|
|
799
|
+
});
|
|
800
|
+
|
|
801
|
+
it.each(['no wait, make it three', 'yes', 'sure, go ahead', 'cancel the cake and add bread'])(
|
|
802
|
+
'never decides by voice on "%s": the turn runs normally',
|
|
803
|
+
async (userText) => {
|
|
804
|
+
await park();
|
|
805
|
+
dispatch.mockImplementationOnce(async (_options, onEvent) => reply(onEvent, 'Okay.', 'assistant-next'));
|
|
806
|
+
const result = await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd2', userText }));
|
|
807
|
+
expect(result.status).toBe('completed');
|
|
808
|
+
expect(dispatch).toHaveBeenCalledTimes(2);
|
|
809
|
+
expect(approvalOf().status).toBe('pending');
|
|
810
|
+
},
|
|
811
|
+
);
|
|
812
|
+
|
|
813
|
+
it('runs "cancel" normally when more than one voice approval waits', async () => {
|
|
814
|
+
await park('ap1');
|
|
815
|
+
await park('ap2', 'order_cake');
|
|
816
|
+
dispatch.mockImplementationOnce(async (_options, onEvent) => reply(onEvent, 'Which one?', 'assistant-next'));
|
|
817
|
+
const result = await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd3', userText: 'cancel it' }));
|
|
818
|
+
expect(result.status).toBe('completed');
|
|
819
|
+
expect(approvalOf('ap1').status).toBe('pending');
|
|
820
|
+
expect(approvalOf('ap2').status).toBe('pending');
|
|
821
|
+
});
|
|
822
|
+
|
|
823
|
+
it('supersedes an earlier pending approval for the same tool, and leaves other tools alone', async () => {
|
|
824
|
+
const first = await park('ap1');
|
|
825
|
+
const cake = await park('ap2', 'order_cake');
|
|
826
|
+
resumeWith(null);
|
|
827
|
+
const second = await park('ap3');
|
|
828
|
+
expect(second.result.status).toBe('pending_approval');
|
|
829
|
+
expect(approvalOf('ap1').status).toBe('denied');
|
|
830
|
+
expect(approvalOf('ap2').status).toBe('pending');
|
|
831
|
+
expect(await first.followUp).toEqual({
|
|
832
|
+
status: 'cancelled',
|
|
833
|
+
text: 'The earlier place pickup order request was replaced by the new one; it was not done.',
|
|
834
|
+
});
|
|
835
|
+
void cake;
|
|
836
|
+
});
|
|
837
|
+
|
|
838
|
+
it('expires an unanswered voice approval after approvalTimeoutMs', async () => {
|
|
839
|
+
parkOnApproval();
|
|
840
|
+
const result = await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'order two croissants' }));
|
|
841
|
+
resumeWith(null);
|
|
842
|
+
const followUp = await result.followUp!({ signal: new AbortController().signal, approvalTimeoutMs: 60 });
|
|
843
|
+
expect(followUp).toEqual({ status: 'timeout', text: 'That place pickup order request expired, so nothing was done.' });
|
|
844
|
+
expect(approvalOf().status).toBe('denied');
|
|
845
|
+
});
|
|
846
|
+
|
|
847
|
+
it('leaves the approval pending on hang-up (no auto-decline)', async () => {
|
|
848
|
+
parkOnApproval();
|
|
849
|
+
const result = await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'order two croissants' }));
|
|
850
|
+
const call = new AbortController();
|
|
851
|
+
const followUp = result.followUp!({ signal: call.signal, approvalTimeoutMs: 60 });
|
|
852
|
+
call.abort();
|
|
853
|
+
expect(await followUp).toBeNull();
|
|
854
|
+
await new Promise((resolve) => setTimeout(resolve, 120));
|
|
855
|
+
expect(approvalOf().status).toBe('pending');
|
|
856
|
+
});
|
|
857
|
+
|
|
858
|
+
/** One turn that parks on two approvals: a pickup order and a cake. */
|
|
859
|
+
const parkTwo = () =>
|
|
860
|
+
dispatch.mockImplementationOnce(async (_options, onEvent) => {
|
|
861
|
+
const createdAt = new Date().toISOString();
|
|
862
|
+
onEvent({ type: 'status', status: 'connecting' });
|
|
863
|
+
for (const [key, toolName] of [['ap1', 'place_pickup_order'], ['ap2', 'order_cake']]) {
|
|
864
|
+
onEvent({
|
|
865
|
+
type: 'message',
|
|
866
|
+
message: {
|
|
867
|
+
id: `approval-${key}`,
|
|
868
|
+
role: 'assistant',
|
|
869
|
+
content: '',
|
|
870
|
+
createdAt,
|
|
871
|
+
variant: 'approval',
|
|
872
|
+
approval: { id: key, status: 'pending', agentId: 'a1', executionId: 'e1', toolName, description: '' },
|
|
873
|
+
},
|
|
874
|
+
});
|
|
875
|
+
}
|
|
876
|
+
onEvent({ type: 'status', status: 'idle' });
|
|
877
|
+
});
|
|
878
|
+
/** Like resumeWith, but the resumed stream arrives after a network-like delay. */
|
|
879
|
+
const resumeLater = (text: string) => {
|
|
880
|
+
internals().client.resolveApproval = async () => new ReadableStream();
|
|
881
|
+
vi.spyOn(session, 'connectStream').mockImplementation(async () => {
|
|
882
|
+
await new Promise((resolve) => setTimeout(resolve, 120));
|
|
883
|
+
reply(internals().handleEvent, text, `assistant-${text.length}`);
|
|
884
|
+
internals().abortController = null;
|
|
885
|
+
});
|
|
886
|
+
};
|
|
887
|
+
|
|
888
|
+
it('replaces only the same-tool approval of an earlier turn, keeping its other approvals', async () => {
|
|
889
|
+
parkTwo();
|
|
890
|
+
const first = await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'croissants and a cake' }));
|
|
891
|
+
const followUp = first.followUp!({ signal: new AbortController().signal });
|
|
892
|
+
resumeWith(null);
|
|
893
|
+
parkOnApproval({}, 'ap3');
|
|
894
|
+
await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd2', userText: 'make it three croissants' }));
|
|
895
|
+
expect(approvalOf('ap1').status).toBe('denied');
|
|
896
|
+
expect(approvalOf('ap2').status).toBe('pending'); // the cake still waits
|
|
897
|
+
resumeLater('Cake ordered.');
|
|
898
|
+
await session.resolveApproval(approvalOf('ap2'), 'approved');
|
|
899
|
+
expect(await followUp).toEqual({
|
|
900
|
+
status: 'completed',
|
|
901
|
+
text: 'Cake ordered.\n\nThe earlier place pickup order request was replaced by the new one; it was not done.',
|
|
902
|
+
});
|
|
903
|
+
});
|
|
904
|
+
|
|
905
|
+
it('reads back an allowed action and states the declined one when a turn gets both decisions', async () => {
|
|
906
|
+
parkTwo();
|
|
907
|
+
const result = await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'croissants and a cake' }));
|
|
908
|
+
const followUp = result.followUp!({ signal: new AbortController().signal });
|
|
909
|
+
resumeLater('Croissants ordered: JB-1234.');
|
|
910
|
+
await session.resolveApproval(approvalOf('ap1'), 'approved');
|
|
911
|
+
resumeWith(null);
|
|
912
|
+
await session.resolveApproval(approvalOf('ap2'), 'denied');
|
|
913
|
+
expect(await followUp).toEqual({
|
|
914
|
+
status: 'completed',
|
|
915
|
+
text: 'Croissants ordered: JB-1234.\n\nThe user declined the order cake request in the chat, so that part was not done.',
|
|
916
|
+
});
|
|
917
|
+
});
|
|
918
|
+
|
|
919
|
+
it('expires an approval even when nothing will be read back (server without followUpFrames)', async () => {
|
|
920
|
+
parkOnApproval();
|
|
921
|
+
const result = await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'order two croissants' }));
|
|
922
|
+
resumeWith(null);
|
|
923
|
+
const followUp = await result.followUp!({
|
|
924
|
+
signal: new AbortController().signal,
|
|
925
|
+
approvalTimeoutMs: 60,
|
|
926
|
+
readBack: false,
|
|
927
|
+
});
|
|
928
|
+
expect(followUp?.status).toBe('timeout');
|
|
929
|
+
expect(approvalOf().status).toBe('denied');
|
|
930
|
+
});
|
|
931
|
+
|
|
932
|
+
it('hands a dropped delegation\'s answer back to browser TTS, now and for the rest of its stream', async () => {
|
|
933
|
+
drive('user', 'what are your hours', true, 'u1');
|
|
934
|
+
dispatch.mockImplementation(async (_options, onEvent) => reply(onEvent, 'Nine to five.', 'assistant-hours'));
|
|
935
|
+
await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'what are your hours' }));
|
|
936
|
+
expect(spoken('assistant-hours')).toBe(true);
|
|
937
|
+
const speak = vi.spyOn(session as unknown as { speakLatestAssistantMessage(): void }, 'speakLatestAssistantMessage');
|
|
938
|
+
h.state.bridge!.dropDelegation!('d1');
|
|
939
|
+
expect(spoken('assistant-hours')).toBe(false);
|
|
940
|
+
// Its stream already ended, so browser TTS (if on) reads it now, once.
|
|
941
|
+
expect(speak).toHaveBeenCalledTimes(1);
|
|
942
|
+
expect(speak).toHaveBeenCalledWith([expect.stringMatching(/./)]);
|
|
943
|
+
});
|
|
944
|
+
|
|
945
|
+
it('keeps an already-spoken approval ask away from browser TTS when its delegation is dropped', async () => {
|
|
946
|
+
parkOnApproval();
|
|
947
|
+
const result = await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'order croissants' }));
|
|
948
|
+
void result.followUp!({ signal: new AbortController().signal });
|
|
949
|
+
const speak = vi.spyOn(session as unknown as { speakLatestAssistantMessage(): void }, 'speakLatestAssistantMessage');
|
|
950
|
+
h.state.bridge!.dropDelegation!('d1');
|
|
951
|
+
expect(spoken('r-ap1')).toBe(true); // the ask's reply went out with delegation_update
|
|
952
|
+
expect(speak).not.toHaveBeenCalled();
|
|
953
|
+
});
|
|
954
|
+
|
|
955
|
+
it('hands a parked turn\'s reply to browser TTS when the delegation is dropped before its ask went out', async () => {
|
|
956
|
+
parkOnApproval();
|
|
957
|
+
await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'order croissants' }));
|
|
958
|
+
// No followUp call: the provider never sent the ask (cancelled first).
|
|
959
|
+
const internalsTts = session as unknown as {
|
|
960
|
+
config: { textToSpeech?: unknown };
|
|
961
|
+
readAloud: { play: (id: string, request: unknown) => Promise<void> };
|
|
962
|
+
};
|
|
963
|
+
internalsTts.config.textToSpeech = { enabled: true, provider: 'browser' };
|
|
964
|
+
const play = vi.spyOn(internalsTts.readAloud, 'play').mockResolvedValue(undefined);
|
|
965
|
+
h.state.bridge!.dropDelegation!('d1');
|
|
966
|
+
expect(spoken('r-ap1')).toBe(false);
|
|
967
|
+
// Browser TTS reads the reply now, since its stream already ended.
|
|
968
|
+
expect(play).toHaveBeenCalledWith('r-ap1', expect.objectContaining({ text: 'I can do that.' }));
|
|
969
|
+
});
|
|
970
|
+
|
|
971
|
+
it('declines a parked approval whose delegation passed the server deadline, as the TTL would', async () => {
|
|
972
|
+
parkOnApproval();
|
|
973
|
+
const result = await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'order croissants' }));
|
|
974
|
+
const call = new AbortController();
|
|
975
|
+
const followUp = result.followUp!({ signal: call.signal });
|
|
976
|
+
resumeWith(null);
|
|
977
|
+
h.state.bridge!.dropDelegation!('d1', true);
|
|
978
|
+
call.abort(); // the provider stops the follow-up: nothing is sent for a cancelled id
|
|
979
|
+
expect(approvalOf().status).toBe('denied');
|
|
980
|
+
expect(await followUp).toBeNull();
|
|
981
|
+
});
|
|
982
|
+
|
|
983
|
+
it.each([
|
|
984
|
+
[true, 'denied'],
|
|
985
|
+
[false, 'pending'],
|
|
986
|
+
])('applies a cancel that came while the turn still ran once it parks (expired: %s)', async (expired, status) => {
|
|
987
|
+
dispatch.mockImplementationOnce(async (options, onEvent) => {
|
|
988
|
+
// The server cancels the delegation mid-turn, before the approval appears.
|
|
989
|
+
h.state.bridge!.dropDelegation!('d1', expired);
|
|
990
|
+
const createdAt = new Date().toISOString();
|
|
991
|
+
onEvent({ type: 'status', status: 'connecting' });
|
|
992
|
+
onEvent({
|
|
993
|
+
type: 'message',
|
|
994
|
+
message: {
|
|
995
|
+
id: 'approval-ap1',
|
|
996
|
+
role: 'assistant',
|
|
997
|
+
content: '',
|
|
998
|
+
createdAt,
|
|
999
|
+
variant: 'approval',
|
|
1000
|
+
approval: { id: 'ap1', status: 'pending', agentId: 'a1', executionId: 'e1', toolName: 'place_pickup_order', description: '' },
|
|
1001
|
+
},
|
|
1002
|
+
});
|
|
1003
|
+
onEvent({ type: 'status', status: 'idle' });
|
|
1004
|
+
void options;
|
|
1005
|
+
});
|
|
1006
|
+
resumeWith(null);
|
|
1007
|
+
const result = await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'order croissants' }));
|
|
1008
|
+
expect(result.status).toBe('pending_approval'); // the provider sends nothing for a cancelled id
|
|
1009
|
+
expect(approvalOf().status).toBe(status);
|
|
1010
|
+
});
|
|
1011
|
+
|
|
1012
|
+
it('leaves a parked approval usable when its delegation is dropped as the call ends', async () => {
|
|
1013
|
+
parkOnApproval();
|
|
1014
|
+
await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'order croissants' }));
|
|
1015
|
+
h.state.bridge!.dropDelegation!('d1', false);
|
|
1016
|
+
expect(approvalOf().status).toBe('pending');
|
|
1017
|
+
});
|
|
1018
|
+
|
|
1019
|
+
it('asks again for a second gated tool the resumed turn stops on, and waits for it', async () => {
|
|
1020
|
+
parkOnApproval({}, 'ap1');
|
|
1021
|
+
const result = await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'croissants, then a cake' }));
|
|
1022
|
+
const updates: string[] = [];
|
|
1023
|
+
const followUp = result.followUp!({ signal: new AbortController().signal, onUpdate: (text) => updates.push(text) });
|
|
1024
|
+
|
|
1025
|
+
// Approving the first resumes the turn, which stops on a cake approval.
|
|
1026
|
+
internals().client.resolveApproval = async () => new ReadableStream();
|
|
1027
|
+
vi.spyOn(session, 'connectStream').mockImplementationOnce(async () => {
|
|
1028
|
+
await new Promise((resolve) => setTimeout(resolve, 60));
|
|
1029
|
+
internals().handleEvent({
|
|
1030
|
+
type: 'message',
|
|
1031
|
+
message: {
|
|
1032
|
+
id: 'approval-ap2',
|
|
1033
|
+
role: 'assistant',
|
|
1034
|
+
content: '',
|
|
1035
|
+
createdAt: new Date().toISOString(),
|
|
1036
|
+
variant: 'approval',
|
|
1037
|
+
approval: { id: 'ap2', status: 'pending', agentId: 'a1', executionId: 'e1', toolName: 'order_cake', description: '' },
|
|
1038
|
+
},
|
|
1039
|
+
});
|
|
1040
|
+
internals().abortController = null;
|
|
1041
|
+
});
|
|
1042
|
+
await session.resolveApproval(approvalOf('ap1'), 'approved');
|
|
1043
|
+
await vi.waitFor(() => expect(updates).toHaveLength(1), { timeout: 2_000 });
|
|
1044
|
+
expect(updates[0]).toContain('- order cake');
|
|
1045
|
+
|
|
1046
|
+
resumeLater('Croissants and cake ordered.');
|
|
1047
|
+
await session.resolveApproval(approvalOf('ap2'), 'approved');
|
|
1048
|
+
expect(await followUp).toEqual({ status: 'completed', text: 'Croissants and cake ordered.' });
|
|
1049
|
+
});
|
|
1050
|
+
|
|
1051
|
+
it('reads each parked turn back with its own answer when both wait at once', async () => {
|
|
1052
|
+
parkOnApproval({}, 'ap1');
|
|
1053
|
+
const first = await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'order croissants' }));
|
|
1054
|
+
const firstFollowUp = first.followUp!({ signal: new AbortController().signal });
|
|
1055
|
+
parkOnApproval({ toolName: 'order_cake' }, 'ap2');
|
|
1056
|
+
const second = await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd2', userText: 'and a cake' }));
|
|
1057
|
+
const secondFollowUp = second.followUp!({ signal: new AbortController().signal });
|
|
1058
|
+
|
|
1059
|
+
resumeLater('Croissants ordered.');
|
|
1060
|
+
await session.resolveApproval(approvalOf('ap1'), 'approved');
|
|
1061
|
+
expect(await firstFollowUp).toEqual({ status: 'completed', text: 'Croissants ordered.' });
|
|
1062
|
+
resumeLater('Cake ordered!');
|
|
1063
|
+
await session.resolveApproval(approvalOf('ap2'), 'approved');
|
|
1064
|
+
expect(await secondFollowUp).toEqual({ status: 'completed', text: 'Cake ordered!' });
|
|
1065
|
+
});
|
|
1066
|
+
|
|
1067
|
+
it('a new request replaces a parked voice WebMCP approval instead of waiting on it', async () => {
|
|
1068
|
+
const internals = session as unknown as { webMcpResolveControllers: Set<AbortController> };
|
|
1069
|
+
dispatch.mockImplementationOnce(async (_options, onEvent) => {
|
|
1070
|
+
reply(onEvent, 'Adding it now.');
|
|
1071
|
+
internals.webMcpResolveControllers.add(new AbortController());
|
|
1072
|
+
void session.requestWebMcpApproval({ toolName: 'add_to_cart', args: { sku: 'AB-1' }, reason: 'gate' });
|
|
1073
|
+
});
|
|
1074
|
+
const first = await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'add it to my cart' }));
|
|
1075
|
+
expect(first.status).toBe('pending_approval');
|
|
1076
|
+
const followUp = first.followUp!({ signal: new AbortController().signal });
|
|
1077
|
+
|
|
1078
|
+
dispatch.mockImplementationOnce(async (_options, onEvent) => reply(onEvent, 'We open at nine.', 'assistant-hours'));
|
|
1079
|
+
const second = await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd2', userText: 'when do you open' }));
|
|
1080
|
+
expect(second).toEqual({ status: 'completed', text: 'We open at nine.' });
|
|
1081
|
+
expect(messages.find((m) => m.variant === 'approval')!.approval!.status).toBe('denied');
|
|
1082
|
+
expect(await followUp).toEqual({
|
|
1083
|
+
status: 'cancelled',
|
|
1084
|
+
text: 'The earlier add to cart request was replaced by the new one; it was not done.',
|
|
1085
|
+
});
|
|
1086
|
+
});
|
|
1087
|
+
});
|
|
1088
|
+
|
|
1089
|
+
it('states a decline plainly, even when the agent replies by re-asking', async () => {
|
|
1090
|
+
drive('user', 'order two croissants', true, 'u1');
|
|
1091
|
+
parkOnApproval();
|
|
1092
|
+
const result = await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'order two croissants' }));
|
|
1093
|
+
const followUp = result.followUp!({ signal: new AbortController().signal });
|
|
1094
|
+
resumeWith('Just to confirm: two almond croissants for 4pm, right?');
|
|
1095
|
+
await session.resolveApproval(approvalOf(), 'denied');
|
|
1096
|
+
expect(await followUp).toEqual({
|
|
1097
|
+
status: 'denied',
|
|
1098
|
+
text: 'The user declined the place pickup order request in the chat, so nothing was done.',
|
|
1099
|
+
});
|
|
1100
|
+
});
|
|
1101
|
+
|
|
1102
|
+
it('follows up with the decline when a denied approval brings no reply', async () => {
|
|
1103
|
+
drive('user', 'order two croissants', true, 'u1');
|
|
1104
|
+
parkOnApproval();
|
|
1105
|
+
const result = await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'order two croissants' }));
|
|
1106
|
+
const followUp = result.followUp!({ signal: new AbortController().signal });
|
|
1107
|
+
resumeWith(null);
|
|
1108
|
+
await session.resolveApproval(approvalOf(), 'denied');
|
|
1109
|
+
expect(await followUp).toEqual({
|
|
1110
|
+
status: 'denied',
|
|
1111
|
+
text: 'The user declined the place pickup order request in the chat, so nothing was done.',
|
|
1112
|
+
});
|
|
1113
|
+
});
|
|
1114
|
+
|
|
1115
|
+
it('drops the follow-up when the call ends before the visitor decides', async () => {
|
|
1116
|
+
drive('user', 'order two croissants', true, 'u1');
|
|
1117
|
+
parkOnApproval();
|
|
1118
|
+
const result = await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'order two croissants' }));
|
|
1119
|
+
const call = new AbortController();
|
|
1120
|
+
const followUp = result.followUp!({ signal: call.signal });
|
|
1121
|
+
call.abort();
|
|
1122
|
+
expect(await followUp).toBeNull();
|
|
1123
|
+
expect(approvalOf().status).toBe('pending');
|
|
1124
|
+
});
|
|
1125
|
+
|
|
1126
|
+
it('answers with the approval script when the turn stops for an approval', async () => {
|
|
1127
|
+
drive('user', 'delete my account', true, 'u1');
|
|
1128
|
+
dispatch.mockImplementation(async (_options, onEvent) => {
|
|
1129
|
+
const createdAt = new Date().toISOString();
|
|
1130
|
+
onEvent({ type: 'status', status: 'connecting' });
|
|
1131
|
+
onEvent({ type: 'message', message: { id: 'r1', role: 'assistant', content: 'I can do that.', createdAt } });
|
|
1132
|
+
onEvent({
|
|
1133
|
+
type: 'message',
|
|
1134
|
+
message: {
|
|
1135
|
+
id: 'ap1',
|
|
1136
|
+
role: 'assistant',
|
|
1137
|
+
content: '',
|
|
1138
|
+
createdAt,
|
|
1139
|
+
variant: 'approval',
|
|
1140
|
+
approval: {
|
|
1141
|
+
id: 'ap1',
|
|
1142
|
+
status: 'pending',
|
|
1143
|
+
agentId: 'a1',
|
|
1144
|
+
executionId: 'e1',
|
|
1145
|
+
toolName: 'delete_account',
|
|
1146
|
+
description: 'Delete the account',
|
|
1147
|
+
},
|
|
1148
|
+
},
|
|
1149
|
+
});
|
|
1150
|
+
onEvent({ type: 'status', status: 'idle' });
|
|
1151
|
+
});
|
|
1152
|
+
const result = await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'delete my account' }));
|
|
1153
|
+
expect(result.text).toBe(
|
|
1154
|
+
'I can do that.\n\n' +
|
|
1155
|
+
"This action needs the user's approval in the chat before it happens:\n" +
|
|
1156
|
+
'- delete account (Delete the account)\n\n' +
|
|
1157
|
+
"Briefly tell the user what you're about to do and ask them to approve or decline it in the chat. Don't claim it's done.",
|
|
1158
|
+
);
|
|
1159
|
+
});
|
|
1160
|
+
|
|
1161
|
+
it('answers right away when a WebMCP tool parks the turn on the visitor\'s approval', async () => {
|
|
1162
|
+
drive('user', 'add it to my cart', true, 'u1');
|
|
1163
|
+
const internals = session as unknown as { webMcpResolveControllers: Set<AbortController> };
|
|
1164
|
+
const resolve = new AbortController();
|
|
1165
|
+
dispatch.mockImplementation(async (_options, onEvent) => {
|
|
1166
|
+
reply(onEvent, 'Adding it now.');
|
|
1167
|
+
// The page tool's resolve is in flight and waiting on the confirm bubble.
|
|
1168
|
+
internals.webMcpResolveControllers.add(resolve);
|
|
1169
|
+
void session.requestWebMcpApproval({ toolName: 'add_to_cart', args: {}, reason: 'gate' });
|
|
1170
|
+
});
|
|
1171
|
+
const result = await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'add it to my cart' }));
|
|
1172
|
+
expect(result.status).not.toBe('failed');
|
|
1173
|
+
expect(result.text).toMatch(/^Adding it now\.\n\nThis action needs the user's approval in the chat before it happens:\n- add to cart \(/);
|
|
1174
|
+
|
|
1175
|
+
// Approved: the page tool runs, and the resumed turn's answer follows up.
|
|
1176
|
+
const followUp = result.followUp!({ signal: new AbortController().signal });
|
|
1177
|
+
const approval = messages.find((m) => m.variant === 'approval')!;
|
|
1178
|
+
session.resolveWebMcpApproval(approval.id, 'approved');
|
|
1179
|
+
(session as unknown as { handleEvent: (event: AgentWidgetEvent) => void }).handleEvent({
|
|
1180
|
+
type: 'message',
|
|
1181
|
+
message: { id: 'after', role: 'assistant', content: 'Added to your cart.', createdAt: new Date().toISOString() },
|
|
1182
|
+
});
|
|
1183
|
+
internals.webMcpResolveControllers.delete(resolve);
|
|
1184
|
+
expect(await followUp).toEqual({ status: 'completed', text: 'Added to your cart.' });
|
|
1185
|
+
});
|
|
1186
|
+
|
|
1187
|
+
it('fails a delegated turn that a typed send replaces, and leaves the typed reply alone', async () => {
|
|
1188
|
+
drive('user', 'long voice task', true, 'u1');
|
|
1189
|
+
dispatch.mockImplementationOnce(
|
|
1190
|
+
(options) =>
|
|
1191
|
+
new Promise((_resolve, reject) => {
|
|
1192
|
+
options.signal?.addEventListener('abort', () =>
|
|
1193
|
+
reject(Object.assign(new Error('aborted'), { name: 'AbortError' })),
|
|
1194
|
+
);
|
|
1195
|
+
}),
|
|
1196
|
+
);
|
|
1197
|
+
dispatch.mockImplementationOnce(async (_options, onEvent) => reply(onEvent, 'Typed answer.', 'assistant-typed'));
|
|
1198
|
+
const delegated = h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'long voice task' }));
|
|
1199
|
+
await new Promise((resolve) => setTimeout(resolve, 0));
|
|
1200
|
+
await session.sendMessage('never mind, typed instead');
|
|
1201
|
+
expect(await delegated).toEqual({ status: 'failed', text: '' });
|
|
1202
|
+
expect(spoken('assistant-typed')).toBe(false);
|
|
1203
|
+
});
|
|
1204
|
+
|
|
1205
|
+
it('claims the bubble named by userTurnId over a text match', async () => {
|
|
1206
|
+
drive('user', 'book a table', true, 'u1');
|
|
1207
|
+
drive('user', 'book a table', true, 'u2');
|
|
1208
|
+
dispatch.mockImplementation(async (_options, onEvent) => reply(onEvent, 'Booked.'));
|
|
1209
|
+
await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'book a table', userUtteranceIds: ['u1'] }));
|
|
1210
|
+
const sent = dispatch.mock.calls[0][0].messages;
|
|
1211
|
+
expect(sent[sent.length - 1].id).toBe(messages.find((m) => m.role === 'user')!.id); // the first bubble
|
|
1212
|
+
});
|
|
1213
|
+
|
|
1214
|
+
it('creates the bubble for a userTurnId not transcribed yet, which its transcript then fills', async () => {
|
|
1215
|
+
dispatch.mockImplementation(async (_options, onEvent) => reply(onEvent, 'Nine.'));
|
|
1216
|
+
await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'opening hours', userUtteranceIds: ['u1'] }));
|
|
1217
|
+
h.state.transcriptCb!('user', 'Opening hours?', true, { turnId: 'u1', caption: true });
|
|
1218
|
+
expect(view()).toEqual([
|
|
1219
|
+
['assistant', 'Welcome! How can I help?'],
|
|
1220
|
+
['user', 'Opening hours?'],
|
|
1221
|
+
['assistant', 'Nine.'],
|
|
1222
|
+
]);
|
|
1223
|
+
expect(messages[1].voiceCaption).toBeUndefined();
|
|
1224
|
+
});
|
|
1225
|
+
|
|
1226
|
+
it('submits a split request once: the last utterance carries it, the earlier ones stay captions', async () => {
|
|
1227
|
+
const caption = (text: string, turnId: string) =>
|
|
1228
|
+
h.state.transcriptCb!('user', text, true, { turnId, caption: true });
|
|
1229
|
+
caption('What are your', 'u1');
|
|
1230
|
+
caption('opening hours', 'u2');
|
|
1231
|
+
dispatch.mockImplementation(async (_options, onEvent) => reply(onEvent, 'Nine to five.'));
|
|
1232
|
+
await h.state.bridge!.runDelegatedTurn(req({
|
|
1233
|
+
delegationId: 'd1',
|
|
1234
|
+
userText: 'What are your opening hours', userUtteranceIds: ['u1', 'u2'] }));
|
|
1235
|
+
|
|
1236
|
+
const sent = dispatch.mock.calls[0][0].messages.filter((m) => !m.voiceCaption);
|
|
1237
|
+
expect(sent.filter((m) => m.role === 'user')).toEqual([
|
|
1238
|
+
expect.objectContaining({ content: 'opening hours', llmContent: 'What are your opening hours' }),
|
|
1239
|
+
]);
|
|
1240
|
+
expect(sent[sent.length - 1].content).toBe('opening hours');
|
|
1241
|
+
expect(view()).toEqual([
|
|
1242
|
+
['assistant', 'Welcome! How can I help?'],
|
|
1243
|
+
['user', 'What are your'],
|
|
1244
|
+
['user', 'opening hours'],
|
|
1245
|
+
['assistant', 'Nine to five.'],
|
|
1246
|
+
]);
|
|
1247
|
+
expect(messages.find((m) => m.content === 'What are your')!.voiceCaption).toBe(true);
|
|
1248
|
+
|
|
1249
|
+
// The joined utterance is never claimed again by a later request.
|
|
1250
|
+
dispatch.mockImplementation(async (_options, onEvent) => reply(onEvent, 'Sure.', 'assistant-r2'));
|
|
1251
|
+
await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd2', userText: 'What are your' }));
|
|
1252
|
+
expect(messages.filter((m) => m.content === 'What are your')).toHaveLength(2);
|
|
1253
|
+
expect(messages.find((m) => m.content === 'What are your' && !m.voiceCaption)).toBeDefined();
|
|
1254
|
+
});
|
|
1255
|
+
|
|
1256
|
+
it('reserves a listed utterance not transcribed yet, so it later renders as a caption', async () => {
|
|
1257
|
+
h.state.transcriptCb!('user', 'and on sundays', true, { turnId: 'u2', caption: true });
|
|
1258
|
+
dispatch.mockImplementation(async (_options, onEvent) => reply(onEvent, 'Closed on Sundays.'));
|
|
1259
|
+
await h.state.bridge!.runDelegatedTurn(req({
|
|
1260
|
+
delegationId: 'd1',
|
|
1261
|
+
userText: 'Your hours and on sundays',
|
|
1262
|
+
// The last listed utterance is the request's.
|
|
1263
|
+
userUtteranceIds: ['u1', 'u2'],
|
|
1264
|
+
}));
|
|
1265
|
+
h.state.transcriptCb!('user', 'Your hours', true, { turnId: 'u1', caption: true }); // late
|
|
1266
|
+
const late = messages.find((m) => m.content === 'Your hours')!;
|
|
1267
|
+
expect(late.voiceCaption).toBe(true);
|
|
1268
|
+
expect(messages.find((m) => m.content === 'and on sundays')).toMatchObject({
|
|
1269
|
+
llmContent: 'Your hours and on sundays',
|
|
1270
|
+
});
|
|
1271
|
+
expect(messages.filter((m) => m.role === 'user')).toHaveLength(2);
|
|
1272
|
+
});
|
|
1273
|
+
|
|
1274
|
+
it('claims a prefix match when the request beats the final transcript', async () => {
|
|
1275
|
+
drive('user', 'What are your opening', false, 'u1');
|
|
1276
|
+
dispatch.mockImplementation(async (_options, onEvent) => reply(onEvent, 'Nine.'));
|
|
1277
|
+
await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'What are your opening hours?' }));
|
|
1278
|
+
expect(messages.filter((m) => m.role === 'user')).toHaveLength(1);
|
|
1279
|
+
expect(dispatch.mock.calls[0][0].messages.filter((m) => m.role === 'user')[0]).toMatchObject({
|
|
1280
|
+
content: 'What are your opening',
|
|
1281
|
+
llmContent: 'What are your opening hours?',
|
|
1282
|
+
});
|
|
1283
|
+
});
|
|
1284
|
+
|
|
1285
|
+
it('claims nothing rather than another utterance\'s bubble', async () => {
|
|
1286
|
+
drive('user', 'how is the weather', true, 'u1');
|
|
1287
|
+
dispatch.mockImplementation(async (_options, onEvent) => reply(onEvent, 'Done.'));
|
|
1288
|
+
await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'turn on dark mode' }));
|
|
1289
|
+
const users = messages.filter((m) => m.role === 'user').map((m) => m.content);
|
|
1290
|
+
expect(users).toEqual(['how is the weather', 'turn on dark mode']);
|
|
1291
|
+
const sent = dispatch.mock.calls[0][0].messages;
|
|
1292
|
+
expect(sent[sent.length - 1].content).toBe('turn on dark mode');
|
|
1293
|
+
});
|
|
1294
|
+
|
|
1295
|
+
it('waits for a chat turn already in flight instead of aborting it', async () => {
|
|
1296
|
+
let finishTyped!: () => void;
|
|
1297
|
+
dispatch.mockImplementationOnce(
|
|
1298
|
+
(_options, onEvent) =>
|
|
1299
|
+
new Promise<void>((resolve) => {
|
|
1300
|
+
finishTyped = () => {
|
|
1301
|
+
reply(onEvent, 'Typed answer.', 'assistant-typed');
|
|
1302
|
+
resolve();
|
|
1303
|
+
};
|
|
1304
|
+
}),
|
|
1305
|
+
);
|
|
1306
|
+
dispatch.mockImplementationOnce(async (_options, onEvent) => reply(onEvent, 'Voice answer.'));
|
|
1307
|
+
const typed = session.sendMessage('typed question');
|
|
1308
|
+
drive('user', 'voice question', true, 'u1');
|
|
1309
|
+
const delegated = h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'voice question' }));
|
|
1310
|
+
await new Promise((resolve) => setTimeout(resolve, 120));
|
|
1311
|
+
expect(dispatch).toHaveBeenCalledTimes(1);
|
|
1312
|
+
|
|
1313
|
+
finishTyped();
|
|
1314
|
+
await typed;
|
|
1315
|
+
expect(await delegated).toEqual({ status: 'completed', text: 'Voice answer.' });
|
|
1316
|
+
expect(dispatch).toHaveBeenCalledTimes(2);
|
|
1317
|
+
expect(dispatch.mock.calls[0][0].signal?.aborted).toBe(false);
|
|
1318
|
+
expect(messages.find((m) => m.id === 'assistant-typed')?.content).toBe('Typed answer.');
|
|
1319
|
+
});
|
|
1320
|
+
});
|
|
1321
|
+
|
|
156
1322
|
describe('AgentWidgetSession - Runtype TTS config', () => {
|
|
157
1323
|
it('uses top-level agentId as the default Runtype TTS agent', async () => {
|
|
158
1324
|
let capturedOptions: { agentId?: string; clientToken?: string; host?: string } | null = null;
|