@runtypelabs/persona 4.25.0 → 4.26.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/animations/glyph-cycle.d.cts +1 -1
- package/dist/animations/glyph-cycle.d.ts +1 -1
- package/dist/animations/{types-Db61aI9J.d.cts → types-BwKTZhg9.d.cts} +14 -0
- package/dist/animations/{types-Db61aI9J.d.ts → types-BwKTZhg9.d.ts} +14 -0
- package/dist/animations/wipe.d.cts +1 -1
- package/dist/animations/wipe.d.ts +1 -1
- package/dist/animations-extra.d.cts +14 -0
- package/dist/animations-extra.d.ts +14 -0
- package/dist/approval-ui-QAKWNFNM.js +1 -0
- package/dist/approval-ui.cjs +2 -2
- package/dist/approval-ui.d.cts +186 -8
- package/dist/approval-ui.d.ts +186 -8
- package/dist/approval-ui.js +2 -2
- package/dist/{artifacts-ui-IGB5YKLO.js → artifacts-ui-NJGMM27I.js} +1 -1
- package/dist/artifacts-ui.cjs +4 -4
- package/dist/artifacts-ui.d.cts +186 -8
- package/dist/artifacts-ui.d.ts +186 -8
- package/dist/artifacts-ui.js +5 -5
- package/dist/chunk-2ANTRKTV.js +1 -0
- package/dist/chunk-4HPTPUC6.js +1 -0
- package/dist/chunk-6V3MNEKB.js +1 -0
- package/dist/chunk-GKPCIQPB.js +1 -0
- package/dist/chunk-HGUCGM3K.js +9 -0
- package/dist/chunk-IO5VVUKP.js +3 -0
- package/dist/chunk-J4J3Y2P2.js +1 -0
- package/dist/chunk-K6HKIP5K.js +1 -0
- package/dist/chunk-K7NA7G7J.js +1 -0
- package/dist/chunk-MXUTS3LB.js +1 -0
- package/dist/chunk-PDPVORUF.js +1 -0
- package/dist/chunk-QRKGQPU7.js +1 -0
- package/dist/chunk-S6PBWE75.js +1 -0
- package/dist/chunk-T7QM7YMF.js +1 -0
- package/dist/chunk-TR2H475E.js +1 -0
- package/dist/chunk-UPO4GUFC.js +1 -0
- package/dist/chunk-ZC66NAIR.js +1 -0
- package/dist/client-history-GOZWBDA5.js +1 -0
- package/dist/client-history.js +1 -0
- package/dist/client-stream-42FOI3EC.js +4 -0
- package/dist/client-stream.js +5 -0
- package/dist/codegen.cjs +1 -1
- package/dist/codegen.js +1 -1
- package/dist/context-mentions-E7HJYWO6.js +161 -0
- package/dist/{context-mentions-inline-I2RDEBAK.js → context-mentions-inline-QVESLW55.js} +1 -1
- package/dist/context-mentions.cjs +7 -7
- package/dist/context-mentions.d.cts +186 -8
- package/dist/context-mentions.d.ts +186 -8
- package/dist/context-mentions.js +8 -8
- package/dist/event-stream-view-FOIJHFQO.js +1 -0
- package/dist/event-stream-view.cjs +1 -1
- package/dist/event-stream-view.d.cts +189 -31
- package/dist/event-stream-view.d.ts +189 -31
- package/dist/event-stream-view.js +1 -1
- package/dist/forms-ui.d.cts +227 -667
- package/dist/forms-ui.d.ts +227 -667
- package/dist/history-shell-QDQ44PSE.js +3 -0
- package/dist/history-shell.js +3 -0
- package/dist/{history-view-BD6OXLNA.js → history-view-NCFBI5GR.js} +2 -2
- package/dist/history-view.cjs +2 -2
- package/dist/history-view.d.cts +35 -1
- package/dist/history-view.d.ts +35 -1
- package/dist/history-view.js +2 -2
- package/dist/index.cjs +80 -74
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +233 -738
- package/dist/index.d.ts +233 -738
- package/dist/index.global.js +3 -215
- package/dist/index.global.js.map +1 -1
- package/dist/index.js +81 -75
- package/dist/index.js.map +1 -1
- package/dist/install.global.js +1 -1
- package/dist/install.global.js.map +1 -1
- package/dist/launcher.global.js +3 -67
- package/dist/launcher.global.js.map +1 -1
- package/dist/markdown-parsers.js +22 -22
- package/dist/runtype-tts-entry-NFBVWR7N.js +1 -0
- package/dist/runtype-tts.js +1 -1
- package/dist/session-actions-ISSKSI66.js +1 -0
- package/dist/session-actions.js +1 -0
- package/dist/session-reconnect.d.cts +186 -8
- package/dist/session-reconnect.d.ts +186 -8
- package/dist/smart-dom-reader.cjs +13 -13
- package/dist/smart-dom-reader.d.cts +186 -8
- package/dist/smart-dom-reader.d.ts +186 -8
- package/dist/smart-dom-reader.js +11 -11
- package/dist/theme-editor-preview.cjs +116 -98
- package/dist/theme-editor-preview.d.cts +188 -8
- package/dist/theme-editor-preview.d.ts +188 -8
- package/dist/theme-editor-preview.js +49 -54
- package/dist/theme-editor.cjs +2 -2
- package/dist/theme-editor.d.cts +187 -23
- package/dist/theme-editor.d.ts +187 -23
- package/dist/theme-editor.js +2 -2
- package/dist/theme-reference.d.cts +14 -0
- package/dist/theme-reference.d.ts +14 -0
- package/dist/ui-extras-entry-WAYFKVNW.js +3 -0
- package/dist/ui-extras.js +72 -0
- package/dist/voice-runtime-XLU6YK2N.js +17 -0
- package/dist/voice-runtime.cjs +17 -1
- package/dist/voice-runtime.d.cts +11656 -232
- package/dist/voice-runtime.d.ts +11656 -232
- package/dist/voice-runtime.js +17 -1
- package/dist/voice-worklet-player.cjs +19 -4
- package/dist/voice-worklet-player.d.cts +9 -10
- package/dist/voice-worklet-player.d.ts +9 -10
- package/dist/voice-worklet-player.js +19 -4
- package/dist/webmcp-polyfill.js +1 -1
- package/dist/webmcp-runtime-V6ZWW3PI.js +6 -0
- package/dist/webmcp-runtime.cjs +2 -2
- package/dist/webmcp-runtime.d.cts +1 -33
- package/dist/webmcp-runtime.d.ts +1 -33
- package/dist/webmcp-runtime.js +2 -2
- package/dist/widget.css +1 -1
- package/package.json +11 -5
- package/src/approval-paused-tool-settle.test.ts +325 -0
- package/src/client-approve.test.ts +311 -0
- package/src/client-history-eager.ts +9 -0
- package/src/client-history-loader.ts +23 -0
- package/src/client-history.ts +579 -0
- package/src/client-resume.ts +177 -0
- package/src/client-stream-eager.ts +12 -0
- package/src/client-stream-loader.ts +25 -0
- package/src/client-stream.ts +1880 -0
- package/src/client.test.ts +31 -1
- package/src/client.ts +293 -2607
- package/src/components/approval-actions.test.ts +27 -0
- package/src/components/approval-actions.ts +27 -3
- package/src/components/message-bubble.test.ts +19 -0
- package/src/components/message-bubble.ts +2 -0
- package/src/components/registry.ts +6 -6
- package/src/components/tool-bubble.test.ts +9 -0
- package/src/components/tool-bubble.ts +1 -0
- package/src/history-shell-bundle.test.ts +54 -0
- package/src/history-shell-eager.ts +12 -0
- package/src/history-shell-loader.ts +27 -0
- package/src/history-shell.ts +2362 -0
- package/src/history-view-loader.ts +2 -0
- package/src/history-view.ts +3 -0
- package/src/index-core.ts +6 -0
- package/src/index-global.ts +90 -248
- package/src/index.ts +13 -0
- package/src/install.ts +6 -4
- package/src/internal/runtype-history-provider.test.ts +1 -0
- package/src/internal/runtype-history-provider.ts +28 -13
- package/src/plugins/registry.ts +8 -8
- package/src/session-actions-chunk.test.ts +55 -0
- package/src/session-actions-eager.ts +11 -0
- package/src/session-actions-loader.ts +37 -0
- package/src/session-actions.ts +909 -0
- package/src/session-history.test.ts +3 -1
- package/src/session.ts +895 -1538
- package/src/session.voice.test.ts +1171 -5
- package/src/styles/widget.css +58 -0
- package/src/theme-editor/state.ts +91 -91
- package/src/types.ts +204 -5
- package/src/ui-ask-user-sheet.ts +382 -0
- package/src/ui-extras-eager.ts +10 -0
- package/src/ui-extras-entry.ts +9 -0
- package/src/ui-extras-loader.ts +26 -0
- package/src/ui.artifact-rail-resize.test.ts +2 -1
- package/src/ui.ask-user-chunk.test.ts +93 -0
- package/src/ui.history-chunk.test.ts +2 -1
- package/src/ui.history-config-provider.test.ts +2 -1
- package/src/ui.history-confirm-fallback.test.ts +142 -0
- package/src/ui.history-controller.test.ts +2 -1
- package/src/ui.history-lifecycle.test.ts +2 -1
- package/src/ui.history-render-hooks.test.ts +2 -1
- package/src/ui.history-shell.test.ts +2 -1
- package/src/ui.rail-overlay.test.ts +2 -1
- package/src/ui.rail-resize.test.ts +2 -1
- package/src/ui.rail-trigger-pending.test.ts +2 -1
- package/src/ui.ts +340 -2770
- package/src/utils/attachment-manager.ts +67 -67
- package/src/utils/constants.ts +3 -0
- package/src/utils/context-mention-controller.ts +323 -323
- package/src/utils/context-mention-manager.ts +40 -40
- package/src/utils/context-mention-orchestrator.ts +28 -0
- package/src/utils/event-stream-buffer.ts +50 -50
- package/src/utils/event-stream-store.ts +43 -43
- package/src/utils/throughput-tracker.ts +34 -34
- package/src/utils/virtual-scroller.ts +67 -67
- package/src/utils/visitor-store.test.ts +50 -0
- package/src/utils/visitor-store.ts +23 -5
- package/src/voice/__fixtures__/voice-wire/README.md +1 -0
- package/src/voice/__fixtures__/voice-wire/client/cancel.json +3 -0
- package/src/voice/__fixtures__/voice-wire/client/context.json +4 -0
- package/src/voice/__fixtures__/voice-wire/client/delegation_result.json +6 -0
- package/src/voice/__fixtures__/voice-wire/client/delegation_update.json +6 -0
- package/src/voice/__fixtures__/voice-wire/client/ping.json +3 -0
- package/src/voice/__fixtures__/voice-wire/client/playback_progress.json +5 -0
- package/src/voice/__fixtures__/voice-wire/client/start.json +3 -0
- package/src/voice/__fixtures__/voice-wire/server/artifact.json +11 -0
- package/src/voice/__fixtures__/voice-wire/server/attached.json +4 -0
- package/src/voice/__fixtures__/voice-wire/server/audio_clear.json +4 -0
- package/src/voice/__fixtures__/voice-wire/server/audio_end.json +3 -0
- package/src/voice/__fixtures__/voice-wire/server/delegation_cancelled.json +5 -0
- package/src/voice/__fixtures__/voice-wire/server/delegation_completed.json +8 -0
- package/src/voice/__fixtures__/voice-wire/server/delegation_started.json +15 -0
- package/src/voice/__fixtures__/voice-wire/server/error.json +6 -0
- package/src/voice/__fixtures__/voice-wire/server/metrics.json +10 -0
- package/src/voice/__fixtures__/voice-wire/server/pong.json +3 -0
- package/src/voice/__fixtures__/voice-wire/server/session_config.json +16 -0
- package/src/voice/__fixtures__/voice-wire/server/transcript_final.json +7 -0
- package/src/voice/__fixtures__/voice-wire/server/transcript_interim.json +4 -0
- package/src/voice/__fixtures__/voice-wire/server/transcript_partial.json +6 -0
- package/src/voice/__fixtures__/voice-wire/server/transcript_update.json +10 -0
- package/src/voice/__fixtures__/voice-wire/server/warning.json +5 -0
- package/src/voice/audio-playback-manager.test.ts +57 -0
- package/src/voice/audio-playback-manager.ts +120 -98
- package/src/voice/browser-voice-provider.ts +37 -37
- package/src/voice/fallback-speech-engine.ts +22 -16
- package/src/voice/keyed-voice-transcript.ts +314 -0
- package/src/voice/mic-state-styles.ts +182 -0
- package/src/voice/read-aloud-controller.ts +46 -44
- package/src/voice/runtype-speech-engine.ts +35 -33
- package/src/voice/runtype-voice-provider.ts +680 -189
- package/src/voice/session-voice-wiring.ts +266 -0
- package/src/voice/voice-delegation.test.ts +71 -0
- package/src/voice/voice-delegation.ts +364 -0
- package/src/voice/voice-turn-tracker.ts +6 -6
- package/src/voice/voice.test.ts +1060 -8
- package/src/voice/worklet-playback-engine.test.ts +81 -0
- package/src/voice/worklet-playback-engine.ts +21 -3
- package/src/voice-runtime-loader.ts +4 -1
- package/src/voice-runtime.ts +7 -0
- package/src/webmcp-runtime-entry.ts +41 -39
- package/dist/approval-ui-3NMYNXBD.js +0 -1
- package/dist/chunk-5EIIHQLQ.js +0 -1
- package/dist/chunk-7CO4T5KB.js +0 -1
- package/dist/chunk-EFJBUH5F.js +0 -1
- package/dist/chunk-R2TMIMDI.js +0 -3
- package/dist/chunk-UTVLOTWB.js +0 -9
- package/dist/context-mentions-45MODWOY.js +0 -161
- package/dist/event-stream-view-GL3ADRY2.js +0 -1
- package/dist/runtype-tts-entry-UJAEF7NZ.js +0 -1
- package/dist/voice-runtime-7DA7MJH4.js +0 -1
- package/dist/webmcp-runtime-JI7T4QYF.js +0 -6
package/src/voice/voice.test.ts
CHANGED
|
@@ -1,9 +1,39 @@
|
|
|
1
1
|
// Voice SDK Tests
|
|
2
2
|
import { describe, it, expect, beforeAll, afterAll, beforeEach, afterEach, vi } from 'vitest';
|
|
3
|
-
import type {
|
|
4
|
-
|
|
3
|
+
import type {
|
|
4
|
+
VoiceConfig,
|
|
5
|
+
VoiceDelegationFollowUp,
|
|
6
|
+
VoiceDelegationRequest,
|
|
7
|
+
VoiceDelegationResult,
|
|
8
|
+
VoiceSessionBridge,
|
|
9
|
+
} from '../types';
|
|
10
|
+
|
|
11
|
+
import { RuntypeVoiceProvider, buildCallContext } from './runtype-voice-provider';
|
|
12
|
+
import { VERSION } from '../version';
|
|
5
13
|
import { BrowserVoiceProvider } from './browser-voice-provider';
|
|
6
14
|
import { createVoiceProvider, createBestAvailableVoiceProvider, isVoiceSupported } from './voice-factory';
|
|
15
|
+
import { readdirSync, readFileSync } from 'node:fs';
|
|
16
|
+
|
|
17
|
+
const ALL_CAPS = ['client_delegation', 'context', 'delegation_update'];
|
|
18
|
+
|
|
19
|
+
/** The voice socket URL v1 builds (Amendment 5): the protocol, client version and capability params. */
|
|
20
|
+
const voiceUrl = (base: string, agent: string, capabilities: string[], extra: Record<string, string> = {}) =>
|
|
21
|
+
`${base}/ws/agents/${agent}/voice?${new URLSearchParams({
|
|
22
|
+
voiceProtocol: 'runtype-browser-v1',
|
|
23
|
+
clientVersion: `persona/${VERSION}`,
|
|
24
|
+
voiceCapabilities: 'full-duplex-v1',
|
|
25
|
+
clientCapabilities: capabilities.join(','),
|
|
26
|
+
...extra,
|
|
27
|
+
})}`;
|
|
28
|
+
const BASE_CAPS = ['partial_transcript', 'context'];
|
|
29
|
+
|
|
30
|
+
/** Wire v1 fixtures (core #9522): one frame per file, by direction. */
|
|
31
|
+
const wireFixtures = (dir: 'client' | 'server') => {
|
|
32
|
+
const base = new URL(`./__fixtures__/voice-wire/${dir}/`, import.meta.url);
|
|
33
|
+
return readdirSync(base).flatMap((file) =>
|
|
34
|
+
file.endsWith('.json') ? [JSON.parse(readFileSync(new URL(file, base), 'utf8')) as Record<string, unknown>] : [],
|
|
35
|
+
);
|
|
36
|
+
};
|
|
7
37
|
|
|
8
38
|
// Mock window object for browser tests
|
|
9
39
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
@@ -317,6 +347,8 @@ function makeFakeEngine() {
|
|
|
317
347
|
flushed: false,
|
|
318
348
|
destroyed: false,
|
|
319
349
|
finishedCb: null as null | (() => void),
|
|
350
|
+
continuous: undefined as boolean | undefined,
|
|
351
|
+
setContinuousMode(enabled: boolean) { engine.continuous = enabled; },
|
|
320
352
|
enqueue(p: Uint8Array) { engine.enqueued.push(p); },
|
|
321
353
|
markStreamEnd() { engine.streamEnded = true; },
|
|
322
354
|
flush() { engine.flushed = true; },
|
|
@@ -371,7 +403,7 @@ describe('RuntypeVoiceProvider (realtime streaming)', () => {
|
|
|
371
403
|
await provider.startListening();
|
|
372
404
|
|
|
373
405
|
const ws = lastWs();
|
|
374
|
-
expect(ws.url).toBe('wss://api.example.com
|
|
406
|
+
expect(ws.url).toBe(voiceUrl('wss://api.example.com', 'a1', BASE_CAPS));
|
|
375
407
|
expect(ws.protocols).toEqual(['runtype.bearer', 'ct_secret']);
|
|
376
408
|
expect(ws.binaryType).toBe('arraybuffer');
|
|
377
409
|
expect(ws.url).not.toContain('token=');
|
|
@@ -386,7 +418,20 @@ describe('RuntypeVoiceProvider (realtime streaming)', () => {
|
|
|
386
418
|
])('derives ws base %s -> %s', async (host, expected) => {
|
|
387
419
|
const provider = new RuntypeVoiceProvider({ ...baseConfig(), host });
|
|
388
420
|
await provider.startListening();
|
|
389
|
-
expect(lastWs().url).toBe(
|
|
421
|
+
expect(lastWs().url).toBe(voiceUrl(expected, 'a1', BASE_CAPS));
|
|
422
|
+
});
|
|
423
|
+
|
|
424
|
+
it('always sends the v1 protocol, its version, full duplex, and snake_case capabilities', async () => {
|
|
425
|
+
// core #9165 removed the 503 that once made Persona omit voiceProtocol.
|
|
426
|
+
const provider = new RuntypeVoiceProvider(baseConfig());
|
|
427
|
+
await provider.startListening();
|
|
428
|
+
const url = new URL(lastWs().url);
|
|
429
|
+
expect(Object.fromEntries(url.searchParams)).toEqual({
|
|
430
|
+
voiceProtocol: 'runtype-browser-v1',
|
|
431
|
+
clientVersion: `persona/${VERSION}`,
|
|
432
|
+
voiceCapabilities: 'full-duplex-v1',
|
|
433
|
+
clientCapabilities: 'partial_transcript,context',
|
|
434
|
+
});
|
|
390
435
|
});
|
|
391
436
|
|
|
392
437
|
it('publishes a 0..1 capture level from the buffer it already sends', async () => {
|
|
@@ -427,6 +472,66 @@ describe('RuntypeVoiceProvider (realtime streaming)', () => {
|
|
|
427
472
|
expect(ws.sent.length).toBe(before + 1);
|
|
428
473
|
});
|
|
429
474
|
|
|
475
|
+
describe('audio captured before the socket opens', () => {
|
|
476
|
+
/** The first PCM16 sample of a sent binary frame. */
|
|
477
|
+
const firstSample = (frame: ArrayBuffer | string) => new Int16Array(frame as ArrayBuffer)[0];
|
|
478
|
+
const binary = (ws: MockWebSocket) => ws.sent.filter((f) => typeof f !== 'string');
|
|
479
|
+
|
|
480
|
+
it('captures as soon as the mic resolves and flushes in order on open, before live frames', async () => {
|
|
481
|
+
const levels: number[] = [];
|
|
482
|
+
const provider = new RuntypeVoiceProvider(baseConfig());
|
|
483
|
+
provider.onLevel((level) => levels.push(level));
|
|
484
|
+
await provider.startListening();
|
|
485
|
+
const ws = lastWs();
|
|
486
|
+
|
|
487
|
+
pumpCapture([0.1]);
|
|
488
|
+
pumpCapture([0.2]);
|
|
489
|
+
expect(ws.sent).toEqual([]);
|
|
490
|
+
expect(levels).toHaveLength(2); // the mic is live: the level animates already
|
|
491
|
+
|
|
492
|
+
ws.triggerOpen();
|
|
493
|
+
pumpCapture([0.3]);
|
|
494
|
+
expect(binary(ws).map(firstSample)).toEqual([0.1, 0.2, 0.3].map((v) => Math.trunc(v * 0x7fff)));
|
|
495
|
+
});
|
|
496
|
+
|
|
497
|
+
it('keeps at most 8 s (256 KB), dropping the oldest frames', async () => {
|
|
498
|
+
const provider = new RuntypeVoiceProvider(baseConfig());
|
|
499
|
+
await provider.startListening();
|
|
500
|
+
const ws = lastWs();
|
|
501
|
+
|
|
502
|
+
// 40 frames of 4096 samples (8 KB each); 31 fit under 256,000 bytes.
|
|
503
|
+
for (let i = 0; i < 40; i++) pumpCapture(Array.from({ length: 4096 }, () => (i + 1) / 100));
|
|
504
|
+
ws.triggerOpen();
|
|
505
|
+
|
|
506
|
+
const sent = binary(ws);
|
|
507
|
+
expect(sent).toHaveLength(31);
|
|
508
|
+
expect(sent.reduce((n, f) => n + (f as ArrayBuffer).byteLength, 0)).toBeLessThanOrEqual(256_000);
|
|
509
|
+
expect(firstSample(sent[0])).toBe(Math.trunc(0.1 * 0x7fff)); // frame 10 of 40
|
|
510
|
+
expect(firstSample(sent[30])).toBe(Math.trunc(0.4 * 0x7fff)); // frame 40
|
|
511
|
+
});
|
|
512
|
+
|
|
513
|
+
it.each([
|
|
514
|
+
['hang-up', async (provider: RuntypeVoiceProvider) => provider.stopListening()],
|
|
515
|
+
['socket error', async (_provider: RuntypeVoiceProvider, ws: MockWebSocket) => ws.triggerError()],
|
|
516
|
+
['close before open', async (_provider: RuntypeVoiceProvider, ws: MockWebSocket) => ws.triggerClose(1006)],
|
|
517
|
+
])('drops the buffer on %s, so the next call starts clean', async (_name, end) => {
|
|
518
|
+
const provider = new RuntypeVoiceProvider(baseConfig());
|
|
519
|
+
provider.onError(() => {});
|
|
520
|
+
await provider.startListening();
|
|
521
|
+
const first = lastWs();
|
|
522
|
+
pumpCapture([0.5]);
|
|
523
|
+
await end(provider, first);
|
|
524
|
+
|
|
525
|
+
await provider.startListening();
|
|
526
|
+
const second = lastWs();
|
|
527
|
+
expect(second).not.toBe(first);
|
|
528
|
+
second.triggerOpen();
|
|
529
|
+
pumpCapture([0.25]);
|
|
530
|
+
expect(binary(second).map(firstSample)).toEqual([Math.trunc(0.25 * 0x7fff)]);
|
|
531
|
+
expect(binary(first)).toEqual([]);
|
|
532
|
+
});
|
|
533
|
+
});
|
|
534
|
+
|
|
430
535
|
it('drives status + onTranscript from control frames', async () => {
|
|
431
536
|
const statuses: string[] = [];
|
|
432
537
|
const transcripts: Array<[string, string, boolean]> = [];
|
|
@@ -538,6 +643,901 @@ describe('RuntypeVoiceProvider (realtime streaming)', () => {
|
|
|
538
643
|
engine.finishedCb?.(); // playback drained
|
|
539
644
|
expect(statuses).toEqual(['listening']);
|
|
540
645
|
});
|
|
646
|
+
|
|
647
|
+
it('re-arms the one-shot drain callback so later replies also return to listening', async () => {
|
|
648
|
+
const engine = makeFakeEngine();
|
|
649
|
+
const statuses: string[] = [];
|
|
650
|
+
const provider = new RuntypeVoiceProvider({ ...baseConfig(), createPlaybackEngine: () => engine });
|
|
651
|
+
provider.onStatusChange((s) => statuses.push(s));
|
|
652
|
+
await provider.startListening();
|
|
653
|
+
lastWs().triggerOpen();
|
|
654
|
+
|
|
655
|
+
lastWs().triggerMessage(makeWavFrame([1, 2]));
|
|
656
|
+
const firstDrain = engine.finishedCb;
|
|
657
|
+
engine.finishedCb = null;
|
|
658
|
+
firstDrain?.(); // engines clear their callbacks once they fire
|
|
659
|
+
lastWs().triggerMessage(makeWavFrame([3, 4]));
|
|
660
|
+
const rearmed = engine.finishedCb as (() => void) | null;
|
|
661
|
+
expect(rearmed).toBeTypeOf('function');
|
|
662
|
+
statuses.length = 0;
|
|
663
|
+
rearmed?.();
|
|
664
|
+
expect(statuses).toEqual(['listening']);
|
|
665
|
+
});
|
|
666
|
+
|
|
667
|
+
it('passes no turn metadata on the legacy transcript frames', async () => {
|
|
668
|
+
const calls: unknown[][] = [];
|
|
669
|
+
const provider = new RuntypeVoiceProvider(baseConfig());
|
|
670
|
+
provider.onTranscript((...args) => calls.push(args));
|
|
671
|
+
await provider.startListening();
|
|
672
|
+
lastWs().triggerOpen();
|
|
673
|
+
lastWs().triggerMessage(JSON.stringify({ type: 'transcript_final', role: 'user', text: 'hi' }));
|
|
674
|
+
expect(calls).toHaveLength(1);
|
|
675
|
+
expect(calls[0].slice(0, 3)).toEqual(['user', 'hi', true]);
|
|
676
|
+
expect(calls[0][3]).toBeUndefined();
|
|
677
|
+
});
|
|
678
|
+
|
|
679
|
+
describe('full duplex (speech-to-speech)', () => {
|
|
680
|
+
const PCM_100MS = 4800; // 2400 samples @ 24 kHz, 16-bit
|
|
681
|
+
|
|
682
|
+
beforeEach(() => {
|
|
683
|
+
vi.useFakeTimers();
|
|
684
|
+
});
|
|
685
|
+
|
|
686
|
+
afterEach(() => {
|
|
687
|
+
vi.useRealTimers();
|
|
688
|
+
});
|
|
689
|
+
|
|
690
|
+
async function startFullDuplexCall() {
|
|
691
|
+
const engine = makeFakeEngine();
|
|
692
|
+
const statuses: string[] = [];
|
|
693
|
+
const transcripts: unknown[][] = [];
|
|
694
|
+
const provider = new RuntypeVoiceProvider({ ...baseConfig(), createPlaybackEngine: () => engine });
|
|
695
|
+
provider.onStatusChange((s) => statuses.push(s));
|
|
696
|
+
provider.onTranscript((...args) => transcripts.push(args));
|
|
697
|
+
await provider.startListening();
|
|
698
|
+
const ws = lastWs();
|
|
699
|
+
ws.triggerOpen();
|
|
700
|
+
ws.triggerMessage(
|
|
701
|
+
JSON.stringify({ type: 'session_config', interruptionMode: 'barge-in', speechMode: 'speech_to_speech' }),
|
|
702
|
+
);
|
|
703
|
+
statuses.length = 0;
|
|
704
|
+
return { engine, statuses, transcripts, provider, ws };
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
const pcm = (bytes: number) => new Uint8Array(bytes).buffer;
|
|
708
|
+
const sentJson = (ws: MockWebSocket) =>
|
|
709
|
+
(ws.sent as unknown[]).filter((d): d is string => typeof d === 'string').map((d) => JSON.parse(d));
|
|
710
|
+
|
|
711
|
+
it('puts the engine in continuous mode and keeps it across the follow-up session_config', async () => {
|
|
712
|
+
const { engine, provider, ws } = await startFullDuplexCall();
|
|
713
|
+
expect(engine.continuous).toBe(true);
|
|
714
|
+
ws.triggerMessage(JSON.stringify({ type: 'session_config', interruptionMode: 'barge-in' }));
|
|
715
|
+
expect(engine.continuous).toBe(true);
|
|
716
|
+
provider.stopPlayback(); // still speech-to-speech: the stop is sent to the server
|
|
717
|
+
expect(sentJson(ws)).toEqual([{ type: 'cancel' }]);
|
|
718
|
+
});
|
|
719
|
+
|
|
720
|
+
it('forwards transcript_update with its turnId and ignores frames without one', async () => {
|
|
721
|
+
const { transcripts, ws } = await startFullDuplexCall();
|
|
722
|
+
ws.triggerMessage(JSON.stringify({ type: 'transcript_update', role: 'user', text: 'hel', turnId: 'u1', final: false }));
|
|
723
|
+
ws.triggerMessage(JSON.stringify({ type: 'transcript_update', role: 'assistant', text: 'Hi', turnId: 'a1', final: true }));
|
|
724
|
+
ws.triggerMessage(JSON.stringify({ type: 'transcript_update', role: 'user', text: 'x' }));
|
|
725
|
+
expect(transcripts).toEqual([
|
|
726
|
+
['user', 'hel', false, { turnId: 'u1' }],
|
|
727
|
+
['assistant', 'Hi', true, { turnId: 'a1' }],
|
|
728
|
+
]);
|
|
729
|
+
});
|
|
730
|
+
|
|
731
|
+
it('plays continuously: releases speaking after the queued audio, never ends the stream', async () => {
|
|
732
|
+
const { engine, statuses, ws } = await startFullDuplexCall();
|
|
733
|
+
ws.triggerMessage(pcm(PCM_100MS));
|
|
734
|
+
ws.triggerMessage(pcm(PCM_100MS));
|
|
735
|
+
expect(statuses).toEqual(['speaking']);
|
|
736
|
+
expect(engine.enqueued).toHaveLength(2);
|
|
737
|
+
|
|
738
|
+
vi.advanceTimersByTime(450); // 200ms of audio + 300ms grace not yet elapsed
|
|
739
|
+
expect(statuses).toEqual(['speaking']);
|
|
740
|
+
vi.advanceTimersByTime(100);
|
|
741
|
+
expect(statuses).toEqual(['speaking', 'listening']);
|
|
742
|
+
expect(engine.streamEnded).toBe(false);
|
|
743
|
+
|
|
744
|
+
// The next reply's audio plays through the same, never-ended stream.
|
|
745
|
+
ws.triggerMessage(pcm(PCM_100MS));
|
|
746
|
+
expect(engine.enqueued).toHaveLength(3);
|
|
747
|
+
expect(statuses).toEqual(['speaking', 'listening', 'speaking']);
|
|
748
|
+
});
|
|
749
|
+
|
|
750
|
+
it('shows processing during a delegation and returns to listening after it', async () => {
|
|
751
|
+
const { statuses, ws } = await startFullDuplexCall();
|
|
752
|
+
ws.triggerMessage(JSON.stringify({ type: 'delegation_started', turnId: 'd1' }));
|
|
753
|
+
expect(statuses).toEqual(['processing']);
|
|
754
|
+
ws.triggerMessage(JSON.stringify({ type: 'delegation_completed', turnId: 'd1' }));
|
|
755
|
+
expect(statuses).toEqual(['processing', 'listening']);
|
|
756
|
+
});
|
|
757
|
+
|
|
758
|
+
it('keeps speaking through a delegation that starts mid-reply, then shows processing', async () => {
|
|
759
|
+
const { statuses, ws } = await startFullDuplexCall();
|
|
760
|
+
ws.triggerMessage(pcm(PCM_100MS));
|
|
761
|
+
ws.triggerMessage(JSON.stringify({ type: 'delegation_started', turnId: 'd1' }));
|
|
762
|
+
expect(statuses).toEqual(['speaking']);
|
|
763
|
+
vi.advanceTimersByTime(1000);
|
|
764
|
+
expect(statuses).toEqual(['speaking', 'processing']);
|
|
765
|
+
ws.triggerMessage(JSON.stringify({ type: 'delegation_completed', turnId: 'd1' }));
|
|
766
|
+
expect(statuses).toEqual(['speaking', 'processing', 'listening']);
|
|
767
|
+
});
|
|
768
|
+
|
|
769
|
+
it('flushes playback immediately on audio_clear and returns to listening', async () => {
|
|
770
|
+
const { engine, statuses, ws } = await startFullDuplexCall();
|
|
771
|
+
ws.triggerMessage(pcm(PCM_100MS * 10));
|
|
772
|
+
ws.triggerMessage(JSON.stringify({ type: 'audio_clear' }));
|
|
773
|
+
expect(engine.flushed).toBe(true);
|
|
774
|
+
expect(statuses).toEqual(['speaking', 'listening']);
|
|
775
|
+
vi.advanceTimersByTime(5000); // the cleared reply's drain timer is gone
|
|
776
|
+
expect(statuses).toEqual(['speaking', 'listening']);
|
|
777
|
+
ws.triggerMessage(pcm(PCM_100MS));
|
|
778
|
+
expect(statuses).toEqual(['speaking', 'listening', 'speaking']);
|
|
779
|
+
});
|
|
780
|
+
|
|
781
|
+
it('stopPlayback cancels server-side and drops the reply until audio_clear', async () => {
|
|
782
|
+
const { engine, statuses, transcripts, provider, ws } = await startFullDuplexCall();
|
|
783
|
+
ws.triggerMessage(pcm(PCM_100MS));
|
|
784
|
+
provider.stopPlayback();
|
|
785
|
+
expect(sentJson(ws)).toEqual([{ type: 'cancel' }]);
|
|
786
|
+
expect(engine.flushed).toBe(true);
|
|
787
|
+
expect(statuses).toEqual(['speaking', 'listening']);
|
|
788
|
+
|
|
789
|
+
const enqueued = engine.enqueued.length;
|
|
790
|
+
ws.triggerMessage(pcm(PCM_100MS));
|
|
791
|
+
ws.triggerMessage(JSON.stringify({ type: 'transcript_update', role: 'assistant', text: 'late', turnId: 'a1', final: true }));
|
|
792
|
+
ws.triggerMessage(JSON.stringify({ type: 'transcript_update', role: 'user', text: 'next', turnId: 'u2', final: false }));
|
|
793
|
+
expect(engine.enqueued).toHaveLength(enqueued);
|
|
794
|
+
expect(transcripts).toEqual([['user', 'next', false, { turnId: 'u2' }]]);
|
|
795
|
+
|
|
796
|
+
ws.triggerMessage(JSON.stringify({ type: 'audio_clear' }));
|
|
797
|
+
ws.triggerMessage(pcm(PCM_100MS));
|
|
798
|
+
expect(engine.enqueued).toHaveLength(enqueued + 1);
|
|
799
|
+
});
|
|
800
|
+
|
|
801
|
+
it('accepts audio again if the cancel acknowledgement never arrives', async () => {
|
|
802
|
+
const { engine, provider, ws } = await startFullDuplexCall();
|
|
803
|
+
provider.stopPlayback();
|
|
804
|
+
vi.advanceTimersByTime(2000);
|
|
805
|
+
ws.triggerMessage(pcm(PCM_100MS));
|
|
806
|
+
expect(engine.enqueued).toHaveLength(1);
|
|
807
|
+
});
|
|
808
|
+
|
|
809
|
+
it('leaves the legacy path without a drain timer or a server cancel', async () => {
|
|
810
|
+
const engine = makeFakeEngine();
|
|
811
|
+
const provider = new RuntypeVoiceProvider({ ...baseConfig(), createPlaybackEngine: () => engine });
|
|
812
|
+
await provider.startListening();
|
|
813
|
+
const ws = lastWs();
|
|
814
|
+
ws.triggerOpen();
|
|
815
|
+
ws.triggerMessage(pcm(PCM_100MS));
|
|
816
|
+
expect(vi.getTimerCount()).toBe(0);
|
|
817
|
+
provider.stopPlayback();
|
|
818
|
+
expect(sentJson(ws)).toEqual([]);
|
|
819
|
+
expect(engine.continuous).toBeUndefined();
|
|
820
|
+
});
|
|
821
|
+
|
|
822
|
+
it('forgets full-duplex state on hang-up', async () => {
|
|
823
|
+
const { provider, ws } = await startFullDuplexCall();
|
|
824
|
+
ws.triggerMessage(pcm(PCM_100MS));
|
|
825
|
+
await provider.stopListening();
|
|
826
|
+
expect(vi.getTimerCount()).toBe(0);
|
|
827
|
+
|
|
828
|
+
await provider.startListening(); // next call starts in the classic mode
|
|
829
|
+
const next = lastWs();
|
|
830
|
+
next.triggerOpen();
|
|
831
|
+
provider.stopPlayback();
|
|
832
|
+
expect(sentJson(next)).toEqual([]);
|
|
833
|
+
});
|
|
834
|
+
|
|
835
|
+
it('passes transcript startMs/endMs through as metadata', async () => {
|
|
836
|
+
const { transcripts, ws } = await startFullDuplexCall();
|
|
837
|
+
ws.triggerMessage(
|
|
838
|
+
JSON.stringify({ type: 'transcript_update', role: 'user', text: 'hi', turnId: 'u1', final: true, startMs: 1200, endMs: 1800 }),
|
|
839
|
+
);
|
|
840
|
+
expect(transcripts).toEqual([['user', 'hi', true, { turnId: 'u1', startMs: 1200, endMs: 1800 }]]);
|
|
841
|
+
});
|
|
842
|
+
|
|
843
|
+
describe('client delegation', () => {
|
|
844
|
+
const flush = async () => {
|
|
845
|
+
for (let i = 0; i < 10; i++) await Promise.resolve();
|
|
846
|
+
};
|
|
847
|
+
|
|
848
|
+
function makeBridge(history: Array<{ role: 'user' | 'assistant'; content: string }> = []) {
|
|
849
|
+
const calls: VoiceDelegationRequest[] = [];
|
|
850
|
+
const pending: Array<(r: VoiceDelegationResult) => void> = [];
|
|
851
|
+
const bridge: VoiceSessionBridge = {
|
|
852
|
+
getHistory: () => history,
|
|
853
|
+
runDelegatedTurn: (request) => {
|
|
854
|
+
calls.push(request);
|
|
855
|
+
return new Promise((resolve) => pending.push(resolve));
|
|
856
|
+
},
|
|
857
|
+
};
|
|
858
|
+
return { bridge, calls, pending };
|
|
859
|
+
}
|
|
860
|
+
|
|
861
|
+
async function startDelegatedCall(
|
|
862
|
+
extra: Partial<NonNullable<VoiceConfig['runtype']>> = {},
|
|
863
|
+
sessionConfig: Record<string, unknown> = { capabilities: ALL_CAPS },
|
|
864
|
+
history: Array<{ role: 'user' | 'assistant'; content: string }> = [],
|
|
865
|
+
) {
|
|
866
|
+
const engine = makeFakeEngine();
|
|
867
|
+
const transcripts: unknown[][] = [];
|
|
868
|
+
const { bridge, calls, pending } = makeBridge(history);
|
|
869
|
+
const provider = new RuntypeVoiceProvider({ ...baseConfig(), ...extra, createPlaybackEngine: () => engine });
|
|
870
|
+
provider.setSessionBridge(bridge);
|
|
871
|
+
provider.onTranscript((...args) => transcripts.push(args));
|
|
872
|
+
await provider.startListening();
|
|
873
|
+
const ws = lastWs();
|
|
874
|
+
ws.triggerOpen();
|
|
875
|
+
ws.triggerMessage(
|
|
876
|
+
JSON.stringify({ type: 'session_config', interruptionMode: 'barge-in', speechMode: 'speech_to_speech', ...sessionConfig }),
|
|
877
|
+
);
|
|
878
|
+
await flush();
|
|
879
|
+
return { provider, ws, transcripts, calls, pending };
|
|
880
|
+
}
|
|
881
|
+
|
|
882
|
+
const update = (role: string, text: string, turnId: string) =>
|
|
883
|
+
JSON.stringify({ type: 'transcript_update', role, text, utteranceId: turnId, final: true });
|
|
884
|
+
/** `delegation_started` asking this client to run the turn (client delegation). */
|
|
885
|
+
const delegate = (ws: MockWebSocket, delegationId: string, text: string, input: Record<string, unknown> = {}) =>
|
|
886
|
+
ws.triggerMessage(
|
|
887
|
+
JSON.stringify({
|
|
888
|
+
type: 'delegation_started',
|
|
889
|
+
delegationId,
|
|
890
|
+
turnId: delegationId,
|
|
891
|
+
input: { text, userUtteranceIds: [], messages: [], ...input },
|
|
892
|
+
}),
|
|
893
|
+
);
|
|
894
|
+
const completed = (ws: MockWebSocket, delegationId: string, final = true) =>
|
|
895
|
+
ws.triggerMessage(JSON.stringify({ type: 'delegation_completed', delegationId, speak: true, text: 'x', final }));
|
|
896
|
+
|
|
897
|
+
it('declares client_delegation (never the kebab form) only with a bridge and the switch on', async () => {
|
|
898
|
+
const { ws } = await startDelegatedCall();
|
|
899
|
+
expect(ws.url).toBe(voiceUrl('wss://api.example.com', 'a1', [...BASE_CAPS, 'client_delegation', 'delegation_update', 'delegation_read_back']));
|
|
900
|
+
expect(ws.url).not.toContain('client-delegation');
|
|
901
|
+
await startDelegatedCall({ clientDelegation: false });
|
|
902
|
+
expect(lastWs().url).toBe(voiceUrl('wss://api.example.com', 'a1', BASE_CAPS));
|
|
903
|
+
const plain = new RuntypeVoiceProvider(baseConfig()); // no session bridge
|
|
904
|
+
await plain.startListening();
|
|
905
|
+
expect(lastWs().url).toBe(voiceUrl('wss://api.example.com', 'a1', BASE_CAPS));
|
|
906
|
+
});
|
|
907
|
+
|
|
908
|
+
it('holds the context frame until the visitor\'s first final transcript, then sends it once', async () => {
|
|
909
|
+
const history: Array<{ role: 'user' | 'assistant'; content: string }> = [
|
|
910
|
+
{ role: 'user', content: 'Hi there' },
|
|
911
|
+
{ role: 'assistant', content: 'Hello!\n\nHow can I help?' },
|
|
912
|
+
];
|
|
913
|
+
const { ws } = await startDelegatedCall(
|
|
914
|
+
{ callContext: async () => 'Visitor is on /pricing.' },
|
|
915
|
+
{ capabilities: ALL_CAPS },
|
|
916
|
+
history,
|
|
917
|
+
);
|
|
918
|
+
expect(sentJson(ws)).toEqual([]); // nothing at call start
|
|
919
|
+
ws.triggerMessage(update('assistant', 'Hi! What can I do?', 'g1')); // a greeting doesn't release it
|
|
920
|
+
await flush();
|
|
921
|
+
expect(sentJson(ws)).toEqual([]);
|
|
922
|
+
|
|
923
|
+
// A partial doesn't release it: a mid-utterance append makes the model answer early.
|
|
924
|
+
history.push({ role: 'user', content: 'what are' });
|
|
925
|
+
ws.triggerMessage(JSON.stringify({ type: 'transcript_update', role: 'user', text: 'what are', turnId: 'u1', final: false }));
|
|
926
|
+
await flush();
|
|
927
|
+
expect(sentJson(ws)).toEqual([]);
|
|
928
|
+
|
|
929
|
+
// The final does; the frame still holds only the history from before the call.
|
|
930
|
+
ws.triggerMessage(update('user', 'what are your hours', 'u1'));
|
|
931
|
+
await flush();
|
|
932
|
+
expect(sentJson(ws)).toEqual([
|
|
933
|
+
{
|
|
934
|
+
type: 'context',
|
|
935
|
+
text: 'Conversation so far:\nUser: Hi there\nAssistant: Hello! How can I help?\n\nVisitor is on /pricing.',
|
|
936
|
+
},
|
|
937
|
+
]);
|
|
938
|
+
ws.triggerMessage(JSON.stringify({ type: 'session_config', interruptionMode: 'barge-in', contextFrames: true }));
|
|
939
|
+
ws.triggerMessage(update('user', 'and sundays', 'u2'));
|
|
940
|
+
await flush();
|
|
941
|
+
expect(sentJson(ws)).toHaveLength(1);
|
|
942
|
+
});
|
|
943
|
+
|
|
944
|
+
it('sends the held context on a delegation that comes first, ahead of its result', async () => {
|
|
945
|
+
const { ws, pending } = await startDelegatedCall({ callContext: 'On /docs.' });
|
|
946
|
+
delegate(ws, 'd1', 'q');
|
|
947
|
+
await flush();
|
|
948
|
+
pending[0]({ status: 'completed', text: 'Answer.' });
|
|
949
|
+
await flush();
|
|
950
|
+
expect(sentJson(ws).map((f) => f.type)).toEqual(['context', 'delegation_result']);
|
|
951
|
+
});
|
|
952
|
+
|
|
953
|
+
it('holds a delegation_result until a slow callContext has gone out', async () => {
|
|
954
|
+
let resolveContext!: (text: string) => void;
|
|
955
|
+
const { ws, pending } = await startDelegatedCall({
|
|
956
|
+
callContext: () => new Promise<string>((resolve) => (resolveContext = resolve)),
|
|
957
|
+
});
|
|
958
|
+
delegate(ws, 'd1', 'q');
|
|
959
|
+
await flush();
|
|
960
|
+
pending[0]({ status: 'completed', text: 'Answer.' });
|
|
961
|
+
await flush();
|
|
962
|
+
expect(sentJson(ws)).toEqual([]); // the result waits for the context
|
|
963
|
+
resolveContext('On /docs.');
|
|
964
|
+
await flush();
|
|
965
|
+
expect(sentJson(ws).map((f) => f.type)).toEqual(['context', 'delegation_result']);
|
|
966
|
+
});
|
|
967
|
+
|
|
968
|
+
it('sends the result after 2 s and drops a context whose callContext hangs', async () => {
|
|
969
|
+
let resolveContext!: (text: string) => void;
|
|
970
|
+
const { ws, pending } = await startDelegatedCall({
|
|
971
|
+
callContext: () => new Promise<string>((resolve) => (resolveContext = resolve)),
|
|
972
|
+
});
|
|
973
|
+
delegate(ws, 'd1', 'q');
|
|
974
|
+
await flush();
|
|
975
|
+
pending[0]({ status: 'completed', text: 'Answer.' });
|
|
976
|
+
await flush();
|
|
977
|
+
await vi.advanceTimersByTimeAsync(1999);
|
|
978
|
+
expect(sentJson(ws)).toEqual([]);
|
|
979
|
+
await vi.advanceTimersByTimeAsync(1);
|
|
980
|
+
expect(sentJson(ws).map((f) => f.type)).toEqual(['delegation_result']);
|
|
981
|
+
resolveContext('too late');
|
|
982
|
+
await flush();
|
|
983
|
+
expect(sentJson(ws).map((f) => f.type)).toEqual(['delegation_result']);
|
|
984
|
+
});
|
|
985
|
+
|
|
986
|
+
it('sends context on contextFrames alone, with delegation off', async () => {
|
|
987
|
+
const { ws, calls } = await startDelegatedCall(
|
|
988
|
+
{ clientDelegation: false, callContext: 'On /docs.' },
|
|
989
|
+
{ capabilities: ['context'] },
|
|
990
|
+
[{ role: 'user', content: 'Hi' }],
|
|
991
|
+
);
|
|
992
|
+
expect(new URL(lastWs().url).searchParams.get('clientCapabilities')).toBe('partial_transcript,context');
|
|
993
|
+
ws.triggerMessage(update('user', 'hello', 'u1'));
|
|
994
|
+
await flush();
|
|
995
|
+
expect(sentJson(ws)).toEqual([{ type: 'context', text: 'Conversation so far:\nUser: Hi\n\nOn /docs.' }]);
|
|
996
|
+
delegate(ws, 'd1', 'x');
|
|
997
|
+
await flush();
|
|
998
|
+
expect(calls).toEqual([]);
|
|
999
|
+
expect(sentJson(ws)).toHaveLength(1);
|
|
1000
|
+
});
|
|
1001
|
+
|
|
1002
|
+
it('sends no context when the server does not announce contextFrames', async () => {
|
|
1003
|
+
const { ws } = await startDelegatedCall({ callContext: 'On /docs.' }, { capabilities: ['client_delegation', 'delegation_update'] }, [
|
|
1004
|
+
{ role: 'user', content: 'Hi' },
|
|
1005
|
+
]);
|
|
1006
|
+
ws.triggerMessage(update('user', 'hello', 'u1'));
|
|
1007
|
+
await flush();
|
|
1008
|
+
expect(sentJson(ws)).toEqual([]);
|
|
1009
|
+
});
|
|
1010
|
+
|
|
1011
|
+
it('drops a held context when the call ends before the visitor speaks', async () => {
|
|
1012
|
+
const { ws, provider } = await startDelegatedCall({ callContext: 'On /docs.' });
|
|
1013
|
+
await provider.stopListening();
|
|
1014
|
+
ws.triggerMessage(update('user', 'hello', 'u1'));
|
|
1015
|
+
await flush();
|
|
1016
|
+
expect(sentJson(ws)).toEqual([]);
|
|
1017
|
+
});
|
|
1018
|
+
|
|
1019
|
+
it('skips the context frame when there is nothing to say', async () => {
|
|
1020
|
+
const { ws } = await startDelegatedCall();
|
|
1021
|
+
ws.triggerMessage(update('user', 'hello', 'u1'));
|
|
1022
|
+
await flush();
|
|
1023
|
+
expect(sentJson(ws)).toEqual([]);
|
|
1024
|
+
});
|
|
1025
|
+
|
|
1026
|
+
it('keeps today\'s behavior when the server does not confirm delegation', async () => {
|
|
1027
|
+
const { ws, calls, transcripts } = await startDelegatedCall(
|
|
1028
|
+
{ callContext: 'page context' },
|
|
1029
|
+
{},
|
|
1030
|
+
[{ role: 'user', content: 'Hi' }],
|
|
1031
|
+
);
|
|
1032
|
+
delegate(ws, 'd1', 'x');
|
|
1033
|
+
ws.triggerMessage(JSON.stringify({ type: 'delegation_completed', turnId: 'd1', speak: true, text: 'Result' }));
|
|
1034
|
+
ws.triggerMessage(update('assistant', 'Result', 'a9'));
|
|
1035
|
+
await flush();
|
|
1036
|
+
expect(calls).toEqual([]);
|
|
1037
|
+
expect(sentJson(ws)).toEqual([]); // no context, no delegation_result
|
|
1038
|
+
expect(transcripts).toEqual([['assistant', 'Result', true, { turnId: 'a9' }]]); // no caption flag
|
|
1039
|
+
});
|
|
1040
|
+
|
|
1041
|
+
it('runs a delegation through the bridge and answers delegation_result', async () => {
|
|
1042
|
+
const { ws, calls, pending } = await startDelegatedCall();
|
|
1043
|
+
delegate(ws, 'd1', 'Weather?', { userUtteranceIds: ['u1'], messages: [{ role: 'user', content: 'Weather?' }] });
|
|
1044
|
+
await flush();
|
|
1045
|
+
expect(calls).toEqual([
|
|
1046
|
+
{ delegationId: 'd1', userText: 'Weather?', userUtteranceIds: ['u1'], messages: [{ role: 'user', content: 'Weather?' }] },
|
|
1047
|
+
]);
|
|
1048
|
+
pending[0]({ status: 'completed', text: 'It is **sunny**.' });
|
|
1049
|
+
await flush();
|
|
1050
|
+
expect(sentJson(ws)).toEqual([{ type: 'delegation_result', delegationId: 'd1', status: 'completed', text: 'It is **sunny**.' }]);
|
|
1051
|
+
});
|
|
1052
|
+
|
|
1053
|
+
it('answers ok:false when the bridge fails', async () => {
|
|
1054
|
+
const { ws, provider } = await startDelegatedCall();
|
|
1055
|
+
provider.setSessionBridge({
|
|
1056
|
+
getHistory: () => [],
|
|
1057
|
+
runDelegatedTurn: () => Promise.reject(new Error('nope')),
|
|
1058
|
+
});
|
|
1059
|
+
delegate(ws, 'd1', 'x');
|
|
1060
|
+
await flush();
|
|
1061
|
+
expect(sentJson(ws)).toEqual([{ type: 'delegation_result', delegationId: 'd1', status: 'failed', text: '' }]);
|
|
1062
|
+
});
|
|
1063
|
+
|
|
1064
|
+
it('runs delegations one at a time', async () => {
|
|
1065
|
+
const { ws, calls, pending } = await startDelegatedCall();
|
|
1066
|
+
delegate(ws, 'd1', 'one');
|
|
1067
|
+
delegate(ws, 'd2', 'two');
|
|
1068
|
+
await flush();
|
|
1069
|
+
expect(calls.map((c) => c.delegationId)).toEqual(['d1']);
|
|
1070
|
+
pending[0]({ status: 'completed', text: 'first' });
|
|
1071
|
+
await flush();
|
|
1072
|
+
expect(calls.map((c) => c.delegationId)).toEqual(['d1', 'd2']);
|
|
1073
|
+
});
|
|
1074
|
+
|
|
1075
|
+
it('marks transcripts as captions only when delegation is negotiated', async () => {
|
|
1076
|
+
const { ws, transcripts } = await startDelegatedCall();
|
|
1077
|
+
ws.triggerMessage(update('assistant', 'One moment.', 'f1'));
|
|
1078
|
+
expect(transcripts).toEqual([['assistant', 'One moment.', true, { turnId: 'f1', caption: true }]]);
|
|
1079
|
+
const legacy = await startDelegatedCall({}, {});
|
|
1080
|
+
legacy.ws.triggerMessage(update('assistant', 'One moment.', 'f1'));
|
|
1081
|
+
expect(legacy.transcripts).toEqual([['assistant', 'One moment.', true, { turnId: 'f1' }]]);
|
|
1082
|
+
});
|
|
1083
|
+
|
|
1084
|
+
it('folds the spoken read-back of a rendered result until the next user utterance', async () => {
|
|
1085
|
+
const { ws, transcripts, pending } = await startDelegatedCall();
|
|
1086
|
+
ws.triggerMessage(update('assistant', 'Let me check.', 'f1'));
|
|
1087
|
+
delegate(ws, 'd1', 'q');
|
|
1088
|
+
await flush();
|
|
1089
|
+
pending[0]({ status: 'completed', text: 'It is sunny.' });
|
|
1090
|
+
await flush();
|
|
1091
|
+
completed(ws, 'd1');
|
|
1092
|
+
ws.triggerMessage(update('assistant', 'Let me check. Okay.', 'f1')); // filler already showing: keeps updating
|
|
1093
|
+
ws.triggerMessage(update('assistant', 'It is sunny.', 'r1')); // read-back: folded
|
|
1094
|
+
ws.triggerMessage(update('user', 'thanks', 'u2'));
|
|
1095
|
+
ws.triggerMessage(update('assistant', 'You are welcome.', 'a2'));
|
|
1096
|
+
expect(transcripts.map((t) => t[1])).toEqual(['Let me check.', 'Let me check. Okay.', 'thanks', 'You are welcome.']);
|
|
1097
|
+
});
|
|
1098
|
+
|
|
1099
|
+
it('keeps a late final for the pre-completion filler id and folds the rotated read-back', async () => {
|
|
1100
|
+
const { ws, transcripts, pending } = await startDelegatedCall();
|
|
1101
|
+
const partial = (role: string, text: string, turnId: string, final: boolean) =>
|
|
1102
|
+
JSON.stringify({ type: 'transcript_update', role, text, turnId, final });
|
|
1103
|
+
ws.triggerMessage(partial('assistant', " Yeah, I'll", 'f1', false));
|
|
1104
|
+
delegate(ws, 'd1', 'q');
|
|
1105
|
+
await flush();
|
|
1106
|
+
pending[0]({ status: 'completed', text: 'We open at nine.' });
|
|
1107
|
+
await flush();
|
|
1108
|
+
completed(ws, 'd1');
|
|
1109
|
+
ws.triggerMessage(partial('assistant', " Yeah, I'll check.", 'f1', true)); // late final, old id
|
|
1110
|
+
ws.triggerMessage(partial('assistant', ' We open', 'r1', false)); // rotated id: read-back
|
|
1111
|
+
ws.triggerMessage(partial('assistant', ' We open at nine.', 'r1', true));
|
|
1112
|
+
expect(transcripts.map((t) => [t[1], t[2]])).toEqual([
|
|
1113
|
+
[" Yeah, I'll", false],
|
|
1114
|
+
[" Yeah, I'll check.", true],
|
|
1115
|
+
]);
|
|
1116
|
+
});
|
|
1117
|
+
|
|
1118
|
+
it('folds only the first new utterance after completion, so later speech still renders', async () => {
|
|
1119
|
+
const { ws, transcripts, pending } = await startDelegatedCall();
|
|
1120
|
+
delegate(ws, 'd1', 'q');
|
|
1121
|
+
await flush();
|
|
1122
|
+
pending[0]({ status: 'completed', text: 'It is sunny.' });
|
|
1123
|
+
await flush();
|
|
1124
|
+
completed(ws, 'd1');
|
|
1125
|
+
ws.triggerMessage(update('assistant', 'It is sunny.', 'r1')); // read-back
|
|
1126
|
+
ws.triggerMessage(update('assistant', 'Anything else?', 'a2')); // a separate utterance
|
|
1127
|
+
ws.triggerMessage(update('assistant', 'Anything else I can do?', 'a2'));
|
|
1128
|
+
expect(transcripts.map((t) => t[1])).toEqual(['Anything else?', 'Anything else I can do?']);
|
|
1129
|
+
});
|
|
1130
|
+
|
|
1131
|
+
it('forwards userUtteranceIds (strings only) to the bridge, reading the deprecated turnId as a fallback id', async () => {
|
|
1132
|
+
const { ws, calls } = await startDelegatedCall();
|
|
1133
|
+
ws.triggerMessage(
|
|
1134
|
+
JSON.stringify({ type: 'delegation_started', turnId: 'd1', input: { text: 'a b', userUtteranceIds: ['u1', 7, 'u2'] } }),
|
|
1135
|
+
);
|
|
1136
|
+
await flush();
|
|
1137
|
+
expect(calls).toEqual([{ delegationId: 'd1', userText: 'a b', userUtteranceIds: ['u1', 'u2'], messages: [] }]);
|
|
1138
|
+
});
|
|
1139
|
+
|
|
1140
|
+
it('runs nothing for a delegation_started without input (server-side delegation)', async () => {
|
|
1141
|
+
const { ws, calls } = await startDelegatedCall();
|
|
1142
|
+
ws.triggerMessage(JSON.stringify({ type: 'delegation_started', delegationId: 'd1' }));
|
|
1143
|
+
await flush();
|
|
1144
|
+
expect(calls).toEqual([]);
|
|
1145
|
+
});
|
|
1146
|
+
|
|
1147
|
+
describe('with delegation_read_back negotiated', () => {
|
|
1148
|
+
const TAG_CAPS = [...ALL_CAPS, 'delegation_read_back'];
|
|
1149
|
+
const tagged = (role: string, text: string, turnId: string, delegationId?: string) =>
|
|
1150
|
+
JSON.stringify({ type: 'transcript_update', role, text, utteranceId: turnId, final: true, delegationId });
|
|
1151
|
+
async function answered(text = 'Three are overdue.') {
|
|
1152
|
+
const call = await startDelegatedCall({}, { capabilities: TAG_CAPS });
|
|
1153
|
+
delegate(call.ws, 'd1', 'q');
|
|
1154
|
+
await flush();
|
|
1155
|
+
call.pending[0]({ status: 'completed', text });
|
|
1156
|
+
await flush();
|
|
1157
|
+
completed(call.ws, 'd1');
|
|
1158
|
+
return call;
|
|
1159
|
+
}
|
|
1160
|
+
|
|
1161
|
+
it('folds the tagged read-back, not a cut filler resuming first (production trace)', async () => {
|
|
1162
|
+
const { ws, transcripts } = await answered();
|
|
1163
|
+
ws.triggerMessage(tagged('assistant', "I'm pulling up the support queue.", 'f2'));
|
|
1164
|
+
ws.triggerMessage(tagged('assistant', 'Three are overdue.', 'r1', 'd1'));
|
|
1165
|
+
expect(transcripts.map((t) => t[1])).toEqual(["I'm pulling up the support queue."]);
|
|
1166
|
+
});
|
|
1167
|
+
|
|
1168
|
+
it('keeps folding the tagged read-back after a late user transcript', async () => {
|
|
1169
|
+
const { ws, transcripts } = await answered();
|
|
1170
|
+
ws.triggerMessage(tagged('user', 'Who is overdue?', 'u1'));
|
|
1171
|
+
ws.triggerMessage(tagged('assistant', 'Three are overdue.', 'r1', 'd1'));
|
|
1172
|
+
expect(transcripts.map((t) => t[1])).toEqual(['Who is overdue?']);
|
|
1173
|
+
});
|
|
1174
|
+
|
|
1175
|
+
it('renders untagged speech and a read-back tagged for a delegation the chat does not show', async () => {
|
|
1176
|
+
const { ws, transcripts } = await answered();
|
|
1177
|
+
ws.triggerMessage(tagged('assistant', 'Anything else?', 'a2'));
|
|
1178
|
+
ws.triggerMessage(tagged('assistant', 'I could not do that.', 'r9', 'd9'));
|
|
1179
|
+
expect(transcripts.map((t) => t[1])).toEqual(['Anything else?', 'I could not do that.']);
|
|
1180
|
+
});
|
|
1181
|
+
|
|
1182
|
+
it('keeps the tagged read-back of an answer that is not in the chat (a spoken decline)', async () => {
|
|
1183
|
+
const { ws, transcripts, pending } = await startDelegatedCall({}, { capabilities: TAG_CAPS });
|
|
1184
|
+
delegate(ws, 'd1', 'cancel that');
|
|
1185
|
+
await flush();
|
|
1186
|
+
pending[0]({ status: 'denied', text: 'Okay, I cancelled the request.', inChat: false });
|
|
1187
|
+
await flush();
|
|
1188
|
+
completed(ws, 'd1');
|
|
1189
|
+
ws.triggerMessage(tagged('assistant', 'Okay, I cancelled the request.', 'r1', 'd1'));
|
|
1190
|
+
expect(transcripts.map((t) => t[1])).toEqual(['Okay, I cancelled the request.']);
|
|
1191
|
+
});
|
|
1192
|
+
|
|
1193
|
+
it('folds every frame of a read-back tagged from its first partial', async () => {
|
|
1194
|
+
const { ws, transcripts } = await answered();
|
|
1195
|
+
const frame = (text: string, final: boolean, delegationId?: string) =>
|
|
1196
|
+
JSON.stringify({ type: 'transcript_update', role: 'assistant', text, utteranceId: 'r1', final, delegationId });
|
|
1197
|
+
ws.triggerMessage(frame('Three are', false, 'd1'));
|
|
1198
|
+
ws.triggerMessage(frame('Three are overdue.', true, 'd1'));
|
|
1199
|
+
expect(transcripts).toEqual([]);
|
|
1200
|
+
});
|
|
1201
|
+
|
|
1202
|
+
it('keeps showing an utterance whose tag arrives after an untagged partial, so no caption is left half-done', async () => {
|
|
1203
|
+
const { ws, transcripts } = await answered();
|
|
1204
|
+
const frame = (text: string, final: boolean, delegationId?: string) =>
|
|
1205
|
+
JSON.stringify({ type: 'transcript_update', role: 'assistant', text, utteranceId: 'r1', final, delegationId });
|
|
1206
|
+
ws.triggerMessage(frame('Three are', false));
|
|
1207
|
+
ws.triggerMessage(frame('Three are overdue.', true, 'd1'));
|
|
1208
|
+
expect(transcripts.map((t) => [t[1], t[2]])).toEqual([
|
|
1209
|
+
['Three are', false],
|
|
1210
|
+
['Three are overdue.', true],
|
|
1211
|
+
]);
|
|
1212
|
+
});
|
|
1213
|
+
|
|
1214
|
+
it('does not fold the tagged read-back of a failed delegation', async () => {
|
|
1215
|
+
const { ws, transcripts, pending } = await startDelegatedCall({}, { capabilities: TAG_CAPS });
|
|
1216
|
+
delegate(ws, 'd1', 'q');
|
|
1217
|
+
await flush();
|
|
1218
|
+
pending[0]({ status: 'failed', text: '' });
|
|
1219
|
+
await flush();
|
|
1220
|
+
completed(ws, 'd1');
|
|
1221
|
+
ws.triggerMessage(tagged('assistant', "Sorry, I couldn't complete that.", 'r1', 'd1'));
|
|
1222
|
+
expect(transcripts).toHaveLength(1);
|
|
1223
|
+
});
|
|
1224
|
+
});
|
|
1225
|
+
|
|
1226
|
+
it('does not fold after a failed delegation', async () => {
|
|
1227
|
+
const { ws, transcripts, pending } = await startDelegatedCall();
|
|
1228
|
+
delegate(ws, 'd1', 'q');
|
|
1229
|
+
await flush();
|
|
1230
|
+
pending[0]({ status: 'failed', text: '' });
|
|
1231
|
+
await flush();
|
|
1232
|
+
completed(ws, 'd1');
|
|
1233
|
+
ws.triggerMessage(update('assistant', "Sorry, I couldn't complete that.", 'r1'));
|
|
1234
|
+
expect(transcripts).toHaveLength(1);
|
|
1235
|
+
});
|
|
1236
|
+
|
|
1237
|
+
it('offers the AI disclosure once a speech-to-speech call is live, unless disabled', async () => {
|
|
1238
|
+
const statuses: string[] = [];
|
|
1239
|
+
const { provider, ws } = await startDelegatedCall();
|
|
1240
|
+
expect(provider.getDisclosure()).toBe("You're talking to an AI assistant. Voice is processed by OpenAI.");
|
|
1241
|
+
await provider.stopListening();
|
|
1242
|
+
expect(provider.getDisclosure()).toBeNull();
|
|
1243
|
+
void ws;
|
|
1244
|
+
|
|
1245
|
+
const custom = await startDelegatedCall({ disclosureText: 'AI voice by OpenAI.' });
|
|
1246
|
+
expect(custom.provider.getDisclosure()).toBe('AI voice by OpenAI.');
|
|
1247
|
+
const hidden = await startDelegatedCall({ disclosureText: false });
|
|
1248
|
+
expect(hidden.provider.getDisclosure()).toBeNull();
|
|
1249
|
+
|
|
1250
|
+
// session_config re-announces "listening" so the UI can read the disclosure.
|
|
1251
|
+
const late = new RuntypeVoiceProvider({ ...baseConfig(), createPlaybackEngine: () => makeFakeEngine() });
|
|
1252
|
+
late.onStatusChange((s) => statuses.push(s));
|
|
1253
|
+
await late.startListening();
|
|
1254
|
+
lastWs().triggerOpen();
|
|
1255
|
+
expect(late.getDisclosure()).toBeNull(); // not known to be speech-to-speech yet
|
|
1256
|
+
lastWs().triggerMessage(JSON.stringify({ type: 'session_config', speechMode: 'speech_to_speech' }));
|
|
1257
|
+
expect(statuses.filter((s) => s === 'listening')).toHaveLength(2);
|
|
1258
|
+
expect(late.getDisclosure()).not.toBeNull();
|
|
1259
|
+
});
|
|
1260
|
+
|
|
1261
|
+
it('drops a result that finishes after hang-up', async () => {
|
|
1262
|
+
const { ws, provider, pending } = await startDelegatedCall();
|
|
1263
|
+
delegate(ws, 'd1', 'q');
|
|
1264
|
+
await flush();
|
|
1265
|
+
await provider.stopListening();
|
|
1266
|
+
pending[0]({ status: 'completed', text: 'late' });
|
|
1267
|
+
await flush();
|
|
1268
|
+
expect(sentJson(ws)).toEqual([]);
|
|
1269
|
+
});
|
|
1270
|
+
|
|
1271
|
+
describe('approval lifecycle (delegation_update, then one terminal delegation_result)', () => {
|
|
1272
|
+
/** A parked result whose follow-up the test settles, recording the options it got. */
|
|
1273
|
+
const parked = () => {
|
|
1274
|
+
let settle!: (followUp: VoiceDelegationFollowUp | null) => void;
|
|
1275
|
+
let options:
|
|
1276
|
+
| { signal: AbortSignal; approvalTimeoutMs?: number; readBack?: boolean; onUpdate?: (text: string) => void }
|
|
1277
|
+
| undefined;
|
|
1278
|
+
const result: VoiceDelegationResult = {
|
|
1279
|
+
status: 'pending_approval',
|
|
1280
|
+
text: 'Approve it in the chat.',
|
|
1281
|
+
followUp: (o) => {
|
|
1282
|
+
options = o;
|
|
1283
|
+
return new Promise((resolve) => (settle = resolve));
|
|
1284
|
+
},
|
|
1285
|
+
};
|
|
1286
|
+
return {
|
|
1287
|
+
result,
|
|
1288
|
+
settle: (followUp: VoiceDelegationFollowUp | null) => settle(followUp),
|
|
1289
|
+
options: () => options,
|
|
1290
|
+
};
|
|
1291
|
+
};
|
|
1292
|
+
const parkOn = async (ws: MockWebSocket, pending: Array<(r: VoiceDelegationResult) => void>) => {
|
|
1293
|
+
const park = parked();
|
|
1294
|
+
delegate(ws, 'd1', 'order croissants');
|
|
1295
|
+
await flush();
|
|
1296
|
+
pending[0](park.result);
|
|
1297
|
+
await flush();
|
|
1298
|
+
return park;
|
|
1299
|
+
};
|
|
1300
|
+
|
|
1301
|
+
it('sends the full wire sequence: update, then the terminal result, folding both read-backs', async () => {
|
|
1302
|
+
const { ws, transcripts, pending } = await startDelegatedCall({ approvalTimeoutMs: 1_500 });
|
|
1303
|
+
const park = await parkOn(ws, pending);
|
|
1304
|
+
expect(sentJson(ws)).toEqual([
|
|
1305
|
+
{ type: 'delegation_update', delegationId: 'd1', status: 'pending_approval', text: 'Approve it in the chat.' },
|
|
1306
|
+
]);
|
|
1307
|
+
expect(park.options()).toMatchObject({ approvalTimeoutMs: 1_500, readBack: true });
|
|
1308
|
+
// Clamped under core's 600 s deadline; unset leaves the default to the bridge.
|
|
1309
|
+
const long = await startDelegatedCall({ approvalTimeoutMs: 900_000 });
|
|
1310
|
+
const longPark = await parkOn(long.ws, long.pending);
|
|
1311
|
+
expect(longPark.options()!.approvalTimeoutMs).toBe(540_000);
|
|
1312
|
+
const unset = await startDelegatedCall();
|
|
1313
|
+
expect((await parkOn(unset.ws, unset.pending)).options()!.approvalTimeoutMs).toBeUndefined();
|
|
1314
|
+
completed(ws, 'd1', false);
|
|
1315
|
+
ws.triggerMessage(update('assistant', 'Please approve it in the chat.', 'r1')); // folded
|
|
1316
|
+
|
|
1317
|
+
park.settle({ status: 'completed', text: 'Your order is in: JB-1234.' });
|
|
1318
|
+
await flush();
|
|
1319
|
+
expect(sentJson(ws)).toEqual([
|
|
1320
|
+
{ type: 'delegation_update', delegationId: 'd1', status: 'pending_approval', text: 'Approve it in the chat.' },
|
|
1321
|
+
{ type: 'delegation_result', delegationId: 'd1', status: 'completed', text: 'Your order is in: JB-1234.' },
|
|
1322
|
+
]);
|
|
1323
|
+
ws.triggerMessage(update('user', 'thanks', 'u2'));
|
|
1324
|
+
completed(ws, 'd1', true);
|
|
1325
|
+
ws.triggerMessage(update('assistant', 'Your order is in.', 'r2')); // folded too
|
|
1326
|
+
expect(transcripts.map((t) => t[1])).toEqual(['thanks']);
|
|
1327
|
+
});
|
|
1328
|
+
|
|
1329
|
+
it('sends another update when the resumed turn stops on a second gated tool', async () => {
|
|
1330
|
+
const { ws, pending } = await startDelegatedCall();
|
|
1331
|
+
const park = await parkOn(ws, pending);
|
|
1332
|
+
park.options()!.onUpdate!('Approve the cake order too.');
|
|
1333
|
+
await flush();
|
|
1334
|
+
park.settle({ status: 'completed', text: 'Both done.' });
|
|
1335
|
+
await flush();
|
|
1336
|
+
expect(sentJson(ws).map((f) => `${f.type}:${f.status}:${f.text}`)).toEqual([
|
|
1337
|
+
'delegation_update:pending_approval:Approve it in the chat.',
|
|
1338
|
+
'delegation_update:pending_approval:Approve the cake order too.',
|
|
1339
|
+
'delegation_result:completed:Both done.',
|
|
1340
|
+
]);
|
|
1341
|
+
});
|
|
1342
|
+
|
|
1343
|
+
it.each(['denied', 'timeout', 'cancelled', 'failed'])('sends a %s terminal result as it comes', async (status) => {
|
|
1344
|
+
const { ws, pending } = await startDelegatedCall();
|
|
1345
|
+
const park = await parkOn(ws, pending);
|
|
1346
|
+
park.settle({ status, text: `It was ${status}.` });
|
|
1347
|
+
await flush();
|
|
1348
|
+
expect(sentJson(ws).at(-1)).toEqual({ type: 'delegation_result', delegationId: 'd1', status, text: `It was ${status}.` });
|
|
1349
|
+
});
|
|
1350
|
+
|
|
1351
|
+
it('answers with the ask as the result to a server without delegation_update, keeping the expiry bookkeeping', async () => {
|
|
1352
|
+
const { ws, pending } = await startDelegatedCall({}, { capabilities: ['client_delegation', 'context'] });
|
|
1353
|
+
const park = await parkOn(ws, pending);
|
|
1354
|
+
expect(sentJson(ws)).toEqual([
|
|
1355
|
+
{ type: 'delegation_result', delegationId: 'd1', status: 'completed', text: 'Approve it in the chat.' },
|
|
1356
|
+
]);
|
|
1357
|
+
expect(park.options()).toMatchObject({ readBack: false });
|
|
1358
|
+
park.settle({ status: 'timeout', text: 'That request expired, so nothing was done.' });
|
|
1359
|
+
await flush();
|
|
1360
|
+
expect(sentJson(ws)).toHaveLength(1);
|
|
1361
|
+
});
|
|
1362
|
+
|
|
1363
|
+
it('aborts the follow-up and sends nothing once the call ends', async () => {
|
|
1364
|
+
const { ws, provider, pending } = await startDelegatedCall();
|
|
1365
|
+
const park = await parkOn(ws, pending);
|
|
1366
|
+
await provider.stopListening();
|
|
1367
|
+
expect(park.options()!.signal.aborted).toBe(true);
|
|
1368
|
+
park.settle({ status: 'completed', text: 'too late' });
|
|
1369
|
+
await flush();
|
|
1370
|
+
expect(sentJson(ws).map((f) => f.type)).toEqual(['delegation_update']);
|
|
1371
|
+
});
|
|
1372
|
+
|
|
1373
|
+
it('stops all frames for a delegation the server cancelled, and its approval timers', async () => {
|
|
1374
|
+
const { ws, pending } = await startDelegatedCall();
|
|
1375
|
+
const park = await parkOn(ws, pending);
|
|
1376
|
+
ws.triggerMessage(JSON.stringify({ type: 'delegation_cancelled', delegationId: 'd1', reason: 'timeout' }));
|
|
1377
|
+
expect(park.options()!.signal.aborted).toBe(true);
|
|
1378
|
+
park.settle({ status: 'completed', text: 'Done.' });
|
|
1379
|
+
await flush();
|
|
1380
|
+
expect(sentJson(ws).map((f) => f.type)).toEqual(['delegation_update']);
|
|
1381
|
+
});
|
|
1382
|
+
|
|
1383
|
+
it('never starts a delegation cancelled while it waited behind another', async () => {
|
|
1384
|
+
const { ws, calls, pending } = await startDelegatedCall();
|
|
1385
|
+
delegate(ws, 'd1', 'first');
|
|
1386
|
+
delegate(ws, 'd2', 'second');
|
|
1387
|
+
await flush();
|
|
1388
|
+
ws.triggerMessage(JSON.stringify({ type: 'delegation_cancelled', delegationId: 'd2', reason: 'timeout' }));
|
|
1389
|
+
pending[0]({ status: 'completed', text: 'one' });
|
|
1390
|
+
await flush();
|
|
1391
|
+
expect(calls.map((c) => c.delegationId)).toEqual(['d1']);
|
|
1392
|
+
expect(sentJson(ws).map((f) => f.delegationId)).toEqual(['d1']);
|
|
1393
|
+
});
|
|
1394
|
+
|
|
1395
|
+
it('sends no result for a delegation cancelled while its turn still runs', async () => {
|
|
1396
|
+
const { ws, pending } = await startDelegatedCall();
|
|
1397
|
+
delegate(ws, 'd1', 'q');
|
|
1398
|
+
await flush();
|
|
1399
|
+
ws.triggerMessage(JSON.stringify({ type: 'delegation_cancelled', delegationId: 'd1', reason: 'timeout' }));
|
|
1400
|
+
pending[0]({ status: 'completed', text: 'late' });
|
|
1401
|
+
await flush();
|
|
1402
|
+
expect(sentJson(ws)).toEqual([]);
|
|
1403
|
+
});
|
|
1404
|
+
});
|
|
1405
|
+
|
|
1406
|
+
describe('capabilities and tolerance', () => {
|
|
1407
|
+
it('uses no client delegation, context or updates when session_config has no capabilities (old server)', async () => {
|
|
1408
|
+
const { ws, calls, transcripts } = await startDelegatedCall(
|
|
1409
|
+
{ callContext: 'On /docs.' },
|
|
1410
|
+
{ clientDelegation: true, contextFrames: true, followUpFrames: true },
|
|
1411
|
+
[{ role: 'user', content: 'Hi' }],
|
|
1412
|
+
);
|
|
1413
|
+
ws.triggerMessage(update('user', 'hello', 'u1'));
|
|
1414
|
+
delegate(ws, 'd1', 'x');
|
|
1415
|
+
await flush();
|
|
1416
|
+
expect(calls).toEqual([]);
|
|
1417
|
+
expect(sentJson(ws)).toEqual([]);
|
|
1418
|
+
expect(transcripts).toEqual([['user', 'hello', true, { turnId: 'u1' }]]); // not a caption
|
|
1419
|
+
});
|
|
1420
|
+
|
|
1421
|
+
it('gates each frame type on its own capability', async () => {
|
|
1422
|
+
const contextOnly = await startDelegatedCall({ callContext: 'On /docs.' }, { capabilities: ['context'] });
|
|
1423
|
+
contextOnly.ws.triggerMessage(update('user', 'hello', 'u1'));
|
|
1424
|
+
delegate(contextOnly.ws, 'd1', 'x');
|
|
1425
|
+
await flush();
|
|
1426
|
+
expect(sentJson(contextOnly.ws).map((f) => f.type)).toEqual(['context']);
|
|
1427
|
+
expect(contextOnly.calls).toEqual([]);
|
|
1428
|
+
});
|
|
1429
|
+
|
|
1430
|
+
it('logs a warning and keeps the call; an error always ends it', async () => {
|
|
1431
|
+
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
1432
|
+
const errors: Error[] = [];
|
|
1433
|
+
const statuses: string[] = [];
|
|
1434
|
+
const { ws, provider } = await startDelegatedCall({}, { capabilities: ALL_CAPS, callId: 'vc_1' });
|
|
1435
|
+
provider.onError((e) => errors.push(e));
|
|
1436
|
+
provider.onStatusChange((s) => statuses.push(s));
|
|
1437
|
+
ws.triggerMessage(JSON.stringify({ type: 'warning', code: 'UNKNOWN_FRAME', message: 'Unknown frame: x' }));
|
|
1438
|
+
expect(warn).toHaveBeenCalledWith('[Persona voice] UNKNOWN_FRAME: Unknown frame: x', 'vc_1');
|
|
1439
|
+
expect(errors).toEqual([]);
|
|
1440
|
+
expect(statuses).not.toContain('error');
|
|
1441
|
+
expect(provider.isBargeInActive()).toBe(true);
|
|
1442
|
+
// `fatal: false` on an error no longer means anything: errors end the call.
|
|
1443
|
+
ws.triggerMessage(JSON.stringify({ type: 'error', error: 'Boom', fatal: false }));
|
|
1444
|
+
expect(errors.map((e) => e.message)).toEqual(['Boom']);
|
|
1445
|
+
expect(statuses).toContain('error');
|
|
1446
|
+
warn.mockRestore();
|
|
1447
|
+
});
|
|
1448
|
+
|
|
1449
|
+
it.each(['UNKNOWN_DELEGATION', 'LATE_RESULT_LIMIT'])(
|
|
1450
|
+
'stops all frames for a delegation a %s warning names, and hands its answer back to TTS',
|
|
1451
|
+
async (code) => {
|
|
1452
|
+
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
1453
|
+
const dropped: string[] = [];
|
|
1454
|
+
const { ws, provider, pending } = await startDelegatedCall();
|
|
1455
|
+
const bridge = (provider as unknown as { bridge: VoiceSessionBridge }).bridge;
|
|
1456
|
+
bridge.dropDelegation = (id) => dropped.push(id);
|
|
1457
|
+
delegate(ws, 'd1', 'q');
|
|
1458
|
+
await flush();
|
|
1459
|
+
ws.triggerMessage(JSON.stringify({ type: 'warning', code, message: 'refused', delegationId: 'd1' }));
|
|
1460
|
+
pending[0]({ status: 'completed', text: 'late' });
|
|
1461
|
+
await flush();
|
|
1462
|
+
expect(sentJson(ws)).toEqual([]);
|
|
1463
|
+
expect(dropped).toEqual(['d1']);
|
|
1464
|
+
warn.mockRestore();
|
|
1465
|
+
},
|
|
1466
|
+
);
|
|
1467
|
+
|
|
1468
|
+
it('hands a cancelled delegation\'s answer back to TTS, expiring its card only past the deadline (or for an unknown reason)', async () => {
|
|
1469
|
+
const dropped: Array<[string, boolean | undefined]> = [];
|
|
1470
|
+
const { ws, provider } = await startDelegatedCall();
|
|
1471
|
+
(provider as unknown as { bridge: VoiceSessionBridge }).bridge.dropDelegation = (id, expired) =>
|
|
1472
|
+
dropped.push([id, expired]);
|
|
1473
|
+
delegate(ws, 'd1', 'q');
|
|
1474
|
+
for (const [id, reason] of [['d1', 'deadline'], ['d2', 'provider_cancelled'], ['d3', 'some_future_reason'], ['d4', 'session_ending']]) {
|
|
1475
|
+
ws.triggerMessage(JSON.stringify({ type: 'delegation_cancelled', delegationId: id, reason }));
|
|
1476
|
+
}
|
|
1477
|
+
expect(dropped).toEqual([['d1', true], ['d2', false], ['d3', true], ['d4', false]]);
|
|
1478
|
+
});
|
|
1479
|
+
|
|
1480
|
+
it('renders the spoken refusal of a delegation it never saw start, resolving nothing', async () => {
|
|
1481
|
+
const { ws, transcripts, calls } = await startDelegatedCall();
|
|
1482
|
+
completed(ws, 'd_backlog', true);
|
|
1483
|
+
ws.triggerMessage(update('assistant', "Sorry, I can't take that right now.", 'r1'));
|
|
1484
|
+
// No chat answer to dedupe against: the refusal is the only text, so it shows.
|
|
1485
|
+
expect(transcripts.map((t) => t[1])).toEqual(["Sorry, I can't take that right now."]);
|
|
1486
|
+
expect(calls).toEqual([]);
|
|
1487
|
+
expect(sentJson(ws)).toEqual([]);
|
|
1488
|
+
});
|
|
1489
|
+
|
|
1490
|
+
it('takes every v1 server fixture without failing; only `error` ends the call', async () => {
|
|
1491
|
+
const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
|
|
1492
|
+
const fixtures = wireFixtures('server');
|
|
1493
|
+
expect(fixtures.length).toBeGreaterThan(10);
|
|
1494
|
+
for (const frame of fixtures) {
|
|
1495
|
+
const errors: Error[] = [];
|
|
1496
|
+
const { ws, provider } = await startDelegatedCall();
|
|
1497
|
+
provider.onError((e) => errors.push(e));
|
|
1498
|
+
expect(() => ws.triggerMessage(JSON.stringify(frame))).not.toThrow();
|
|
1499
|
+
await flush();
|
|
1500
|
+
expect(errors.length, `${frame.type}`).toBe(frame.type === 'error' ? 1 : 0);
|
|
1501
|
+
}
|
|
1502
|
+
warn.mockRestore();
|
|
1503
|
+
});
|
|
1504
|
+
|
|
1505
|
+
it('sends client frames shaped like the v1 client fixtures', async () => {
|
|
1506
|
+
const shape = (frame: Record<string, unknown>) => [String(frame.type), Object.keys(frame).sort().join(',')];
|
|
1507
|
+
const fixtures = Object.fromEntries(wireFixtures('client').map(shape));
|
|
1508
|
+
const { ws, pending } = await startDelegatedCall({ callContext: 'On /docs.' });
|
|
1509
|
+
ws.triggerMessage(update('user', 'hello', 'u1')); // releases context
|
|
1510
|
+
delegate(ws, 'd1', 'q');
|
|
1511
|
+
await flush();
|
|
1512
|
+
pending[0]({ status: 'pending_approval', text: 'Approve it.', followUp: async () => ({ status: 'completed', text: 'Done.' }) });
|
|
1513
|
+
await flush();
|
|
1514
|
+
const sent = sentJson(ws).map(shape);
|
|
1515
|
+
expect(sent.map(([type]) => type)).toEqual(['context', 'delegation_update', 'delegation_result']);
|
|
1516
|
+
for (const [type, keys] of sent) expect(keys, type).toBe(fixtures[type]);
|
|
1517
|
+
});
|
|
1518
|
+
|
|
1519
|
+
it('ignores unknown frames and fields', async () => {
|
|
1520
|
+
const { ws, transcripts, calls } = await startDelegatedCall();
|
|
1521
|
+
ws.triggerMessage(JSON.stringify({ type: 'agent_state', state: 'thinking' }));
|
|
1522
|
+
ws.triggerMessage(JSON.stringify({ type: 'audio_clear', reason: 'barge_in', extra: 1 }));
|
|
1523
|
+
ws.triggerMessage(
|
|
1524
|
+
JSON.stringify({ type: 'transcript_update', role: 'assistant', text: 'Hi', utteranceId: 'a1', final: true, interrupted: true, future: 'x' }),
|
|
1525
|
+
);
|
|
1526
|
+
delegate(ws, 'd1', 'q', { future: true });
|
|
1527
|
+
await flush();
|
|
1528
|
+
expect(transcripts.map((t) => t[1])).toEqual(['Hi']);
|
|
1529
|
+
expect(calls).toHaveLength(1);
|
|
1530
|
+
});
|
|
1531
|
+
|
|
1532
|
+
it('reads utteranceId, falling back to the deprecated turnId', async () => {
|
|
1533
|
+
const { ws, transcripts } = await startDelegatedCall();
|
|
1534
|
+
ws.triggerMessage(JSON.stringify({ type: 'transcript_update', role: 'user', text: 'a', utteranceId: 'u1', turnId: 'ignored', final: true }));
|
|
1535
|
+
ws.triggerMessage(JSON.stringify({ type: 'transcript_update', role: 'user', text: 'b', turnId: 'u2', final: true }));
|
|
1536
|
+
expect(transcripts.map((t) => (t[3] as { turnId: string }).turnId)).toEqual(['u1', 'u2']);
|
|
1537
|
+
});
|
|
1538
|
+
});
|
|
1539
|
+
});
|
|
1540
|
+
});
|
|
541
1541
|
});
|
|
542
1542
|
|
|
543
1543
|
describe('RuntypeVoiceProvider prewarm', () => {
|
|
@@ -650,16 +1650,25 @@ describe('RuntypeVoiceProvider prewarm', () => {
|
|
|
650
1650
|
new RuntypeVoiceProvider(attachConfig()).prewarm();
|
|
651
1651
|
|
|
652
1652
|
const ws = lastWs();
|
|
653
|
-
expect(ws.url).toBe('wss://api.example.com
|
|
1653
|
+
expect(ws.url).toBe(voiceUrl('wss://api.example.com', 'agent%2F1', ['attach', ...BASE_CAPS]));
|
|
654
1654
|
expect(ws.protocols).toEqual(['runtype.bearer', 'runtype.attach', 'ct_secret']);
|
|
655
1655
|
expect(fetchMock).not.toHaveBeenCalled();
|
|
656
1656
|
openAs(ws, 'runtype.attach');
|
|
657
1657
|
expect(ws.sent).toEqual(['{"type":"ping"}']);
|
|
658
1658
|
});
|
|
659
1659
|
|
|
1660
|
+
it('combines attach with client_delegation when a session bridge is set', () => {
|
|
1661
|
+
const provider = new RuntypeVoiceProvider(attachConfig());
|
|
1662
|
+
provider.setSessionBridge({ getHistory: () => [], runDelegatedTurn: async () => ({ status: 'completed', text: '' }) });
|
|
1663
|
+
provider.prewarm();
|
|
1664
|
+
expect(lastWs().url).toBe(
|
|
1665
|
+
voiceUrl('wss://api.example.com', 'agent%2F1', ['attach', ...BASE_CAPS, 'client_delegation', 'delegation_update', 'delegation_read_back']),
|
|
1666
|
+
);
|
|
1667
|
+
});
|
|
1668
|
+
|
|
660
1669
|
it('sends a clamped attachIdleMs when configured', () => {
|
|
661
1670
|
new RuntypeVoiceProvider(attachConfig({ attachIdleMs: 5_000 })).prewarm();
|
|
662
|
-
expect(lastWs().url).
|
|
1671
|
+
expect(lastWs().url).toBe(voiceUrl('wss://api.example.com', 'agent%2F1', ['attach', ...BASE_CAPS], { attachIdleMs: '30000' }));
|
|
663
1672
|
});
|
|
664
1673
|
|
|
665
1674
|
it('never attaches twice concurrently', () => {
|
|
@@ -718,7 +1727,7 @@ describe('RuntypeVoiceProvider prewarm', () => {
|
|
|
718
1727
|
await provider.startListening();
|
|
719
1728
|
|
|
720
1729
|
expect(MockWebSocket.instances).toHaveLength(2);
|
|
721
|
-
expect(lastWs().url).toBe('wss://api.example.com
|
|
1730
|
+
expect(lastWs().url).toBe(voiceUrl('wss://api.example.com', 'agent%2F1', BASE_CAPS));
|
|
722
1731
|
expect(lastWs().protocols).toEqual(['runtype.bearer', 'ct_secret']);
|
|
723
1732
|
});
|
|
724
1733
|
|
|
@@ -752,6 +1761,22 @@ describe('RuntypeVoiceProvider prewarm', () => {
|
|
|
752
1761
|
expect(ws.sent).toContain('{"type":"start"}');
|
|
753
1762
|
});
|
|
754
1763
|
|
|
1764
|
+
it('keeps the audio captured while waiting on the attach, sent after start', async () => {
|
|
1765
|
+
const provider = new RuntypeVoiceProvider(attachConfig());
|
|
1766
|
+
provider.prewarm();
|
|
1767
|
+
const ws = lastWs();
|
|
1768
|
+
|
|
1769
|
+
const started = provider.startListening();
|
|
1770
|
+
await vi.advanceTimersByTimeAsync(500);
|
|
1771
|
+
pumpCapture(constantBuffer(0.1));
|
|
1772
|
+
openAs(ws, 'runtype.attach');
|
|
1773
|
+
await started;
|
|
1774
|
+
|
|
1775
|
+
expect(ws.sent.slice(0, 2)).toEqual(['{"type":"ping"}', '{"type":"start"}']);
|
|
1776
|
+
expect(ws.sent).toHaveLength(3);
|
|
1777
|
+
expect(ws.sent[2]).toBeInstanceOf(ArrayBuffer);
|
|
1778
|
+
});
|
|
1779
|
+
|
|
755
1780
|
it('closes an unused attached socket at the idle window', () => {
|
|
756
1781
|
const errors: Error[] = [];
|
|
757
1782
|
const provider = new RuntypeVoiceProvider(attachConfig());
|
|
@@ -805,4 +1830,31 @@ describe('RuntypeVoiceProvider prewarm', () => {
|
|
|
805
1830
|
expect(ws.closeCalls).toEqual([{ code: 1000, reason: 'prewarm released' }]);
|
|
806
1831
|
});
|
|
807
1832
|
});
|
|
808
|
-
});
|
|
1833
|
+
});
|
|
1834
|
+
describe('buildCallContext', () => {
|
|
1835
|
+
it('keeps the last 12 messages, newest first when over budget, and caps at 8000 chars', () => {
|
|
1836
|
+
const history = Array.from({ length: 20 }, (_, i) => ({
|
|
1837
|
+
role: (i % 2 === 0 ? 'user' : 'assistant') as 'user' | 'assistant',
|
|
1838
|
+
content: `message ${i} ${'x'.repeat(1000)}`,
|
|
1839
|
+
}));
|
|
1840
|
+
const text = buildCallContext(history, '');
|
|
1841
|
+
expect(text.length).toBeLessThanOrEqual(8000);
|
|
1842
|
+
expect(text.startsWith('Conversation so far:\n')).toBe(true);
|
|
1843
|
+
expect(text).toContain('message 19');
|
|
1844
|
+
expect(text).not.toContain('message 7 '); // outside the 12-message window
|
|
1845
|
+
expect(text).not.toContain('message 8 '); // dropped: oldest over budget
|
|
1846
|
+
});
|
|
1847
|
+
|
|
1848
|
+
it('truncates a single huge message and always fits the host context', () => {
|
|
1849
|
+
const text = buildCallContext([{ role: 'assistant', content: 'y'.repeat(50_000) }], 'host '.repeat(3000));
|
|
1850
|
+
expect(text.length).toBeLessThanOrEqual(8000);
|
|
1851
|
+
expect(text).toContain('Assistant: yyy');
|
|
1852
|
+
expect(text).toContain('…');
|
|
1853
|
+
expect(text.endsWith('host')).toBe(true); // host share capped, still present
|
|
1854
|
+
});
|
|
1855
|
+
|
|
1856
|
+
it('returns only the host context without history, and "" when both are empty', () => {
|
|
1857
|
+
expect(buildCallContext([], ' On /docs ')).toBe('On /docs');
|
|
1858
|
+
expect(buildCallContext([{ role: 'user', content: ' ' }], '')).toBe('');
|
|
1859
|
+
});
|
|
1860
|
+
});
|