@runtypelabs/persona 4.26.0 → 4.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/approval-ui.d.cts +11 -0
- package/dist/approval-ui.d.ts +11 -0
- package/dist/artifacts-ui.d.cts +11 -0
- package/dist/artifacts-ui.d.ts +11 -0
- package/dist/chunk-3255TNY7.js +1 -0
- package/dist/{client-history-YPEG45Q4.js → client-history-HTRKHBAF.js} +1 -1
- package/dist/client-history.js +1 -1
- package/dist/client-stream-N22JHTLE.js +4 -0
- package/dist/client-stream.js +3 -3
- package/dist/codegen.cjs +1 -1
- package/dist/codegen.js +1 -1
- package/dist/context-mentions.d.cts +11 -0
- package/dist/context-mentions.d.ts +11 -0
- package/dist/event-stream-view.d.cts +11 -0
- package/dist/event-stream-view.d.ts +11 -0
- package/dist/forms-ui.d.cts +13 -0
- package/dist/forms-ui.d.ts +13 -0
- package/dist/index.cjs +57 -57
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +13 -0
- package/dist/index.d.ts +13 -0
- package/dist/index.global.js +1 -1
- package/dist/index.global.js.map +1 -1
- package/dist/index.js +58 -58
- package/dist/index.js.map +1 -1
- package/dist/session-reconnect.d.cts +11 -0
- package/dist/session-reconnect.d.ts +11 -0
- package/dist/smart-dom-reader.d.cts +11 -0
- package/dist/smart-dom-reader.d.ts +11 -0
- package/dist/theme-editor-preview.cjs +59 -59
- package/dist/theme-editor-preview.d.cts +11 -0
- package/dist/theme-editor-preview.d.ts +11 -0
- package/dist/theme-editor-preview.js +31 -31
- package/dist/theme-editor.d.cts +11 -0
- package/dist/theme-editor.d.ts +11 -0
- package/dist/voice-runtime-RHRX4F3T.js +17 -0
- package/dist/voice-runtime.cjs +8 -8
- package/dist/voice-runtime.d.cts +15 -0
- package/dist/voice-runtime.d.ts +15 -0
- package/dist/voice-runtime.js +8 -8
- package/package.json +1 -1
- package/src/approval-paused-tool-settle.test.ts +108 -4
- package/src/client-stream.ts +10 -3
- package/src/client.test.ts +20 -0
- package/src/session.ts +56 -10
- package/src/session.voice.test.ts +60 -3
- package/src/types.ts +11 -0
- package/src/ui.composer-defer-one.test.ts +17 -0
- package/src/ui.ts +49 -12
- package/src/voice/__fixtures__/voice-wire/server/session_end-provider-error.json +6 -0
- package/src/voice/__fixtures__/voice-wire/server/session_end-unknown-reason.json +4 -0
- package/src/voice/__fixtures__/voice-wire/server/session_end.json +5 -0
- package/src/voice/runtype-voice-provider.ts +131 -19
- package/src/voice/voice-delegation.ts +27 -11
- package/src/voice/voice.test.ts +257 -2
- package/dist/chunk-QPPJUSZH.js +0 -1
- package/dist/client-stream-42FOI3EC.js +0 -4
- package/dist/voice-runtime-UOQOFAFY.js +0 -17
|
@@ -36,7 +36,16 @@ const h = vi.hoisted(() => {
|
|
|
36
36
|
errorCb: ((e: Error) => void) | null;
|
|
37
37
|
bridge: VoiceSessionBridge | null;
|
|
38
38
|
prewarms: number;
|
|
39
|
-
|
|
39
|
+
notice: string | null;
|
|
40
|
+
} = {
|
|
41
|
+
transcriptCb: null,
|
|
42
|
+
metricsCb: null,
|
|
43
|
+
statusCb: null,
|
|
44
|
+
errorCb: null,
|
|
45
|
+
bridge: null,
|
|
46
|
+
prewarms: 0,
|
|
47
|
+
notice: null,
|
|
48
|
+
};
|
|
40
49
|
|
|
41
50
|
const fakeProvider = {
|
|
42
51
|
type: 'runtype' as const,
|
|
@@ -64,6 +73,11 @@ const h = vi.hoisted(() => {
|
|
|
64
73
|
setSessionBridge: (bridge: VoiceSessionBridge) => {
|
|
65
74
|
state.bridge = bridge;
|
|
66
75
|
},
|
|
76
|
+
takeNotice: () => {
|
|
77
|
+
const notice = state.notice;
|
|
78
|
+
state.notice = null;
|
|
79
|
+
return notice;
|
|
80
|
+
},
|
|
67
81
|
};
|
|
68
82
|
|
|
69
83
|
return { state, fakeProvider };
|
|
@@ -513,12 +527,31 @@ describe('AgentWidgetSession - voice client delegation bridge', () => {
|
|
|
513
527
|
});
|
|
514
528
|
|
|
515
529
|
it('hands the provider a bridge whose history is the visible settled messages', () => {
|
|
516
|
-
|
|
517
|
-
//
|
|
530
|
+
// An earlier call's captions (e.g. on the previous page) are conversation
|
|
531
|
+
// the new voice session never heard, so they carry over.
|
|
518
532
|
h.state.transcriptCb!('assistant', 'Let me check.', true, { turnId: 'f0', caption: true });
|
|
519
533
|
h.state.transcriptCb!('user', 'hi there', true, { turnId: 'u0', caption: true });
|
|
534
|
+
drive('user', 'still talk', false, 'u1'); // interim: not part of the context
|
|
535
|
+
expect(h.state.bridge!.getHistory()).toEqual([
|
|
536
|
+
{ role: 'assistant', content: 'Welcome! How can I help?' },
|
|
537
|
+
{ role: 'assistant', content: 'Let me check.' },
|
|
538
|
+
{ role: 'user', content: 'hi there' },
|
|
539
|
+
]);
|
|
540
|
+
});
|
|
541
|
+
|
|
542
|
+
it("leaves a delegated turn's spoken filler out of the history, keeping the answer", async () => {
|
|
543
|
+
// Full duplex: the voice model's words arrive as captions.
|
|
544
|
+
h.state.transcriptCb!('user', 'whats the weather in paris', true, { turnId: 'u1', caption: true });
|
|
545
|
+
h.state.transcriptCb!('assistant', 'Let me check that.', true, { turnId: 'f1', caption: true });
|
|
546
|
+
dispatch.mockImplementation(async (_options, onEvent) => reply(onEvent, 'It is sunny in Paris.'));
|
|
547
|
+
await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'whats the weather in paris' }));
|
|
548
|
+
h.state.transcriptCb!('assistant', 'Anything else?', true, { turnId: 'a2', caption: true });
|
|
549
|
+
|
|
520
550
|
expect(h.state.bridge!.getHistory()).toEqual([
|
|
521
551
|
{ role: 'assistant', content: 'Welcome! How can I help?' },
|
|
552
|
+
{ role: 'user', content: 'whats the weather in paris' },
|
|
553
|
+
{ role: 'assistant', content: 'It is sunny in Paris.' },
|
|
554
|
+
{ role: 'assistant', content: 'Anything else?' },
|
|
522
555
|
]);
|
|
523
556
|
});
|
|
524
557
|
|
|
@@ -790,6 +823,7 @@ describe('AgentWidgetSession - voice client delegation bridge', () => {
|
|
|
790
823
|
expect(decline).toEqual({
|
|
791
824
|
status: 'denied',
|
|
792
825
|
text: 'Okay, I cancelled the place pickup order request. Nothing was done.',
|
|
826
|
+
inChat: false, // only spoken: its read-back stays as a caption
|
|
793
827
|
});
|
|
794
828
|
expect(dispatch).toHaveBeenCalledTimes(1); // no chat turn for the decline
|
|
795
829
|
expect(approvalOf().status).toBe('denied');
|
|
@@ -1448,3 +1482,26 @@ describe('AgentWidgetSession - prewarmVoice', () => {
|
|
|
1448
1482
|
expect(h.state.prewarms).toBe(0);
|
|
1449
1483
|
});
|
|
1450
1484
|
});
|
|
1485
|
+
|
|
1486
|
+
describe('AgentWidgetSession - voice session end (Amendment 6)', () => {
|
|
1487
|
+
const flush = () => new Promise((resolve) => setTimeout(resolve, 0));
|
|
1488
|
+
const tokenSession = () =>
|
|
1489
|
+
new AgentWidgetSession(
|
|
1490
|
+
{
|
|
1491
|
+
clientToken: 'ct_live_demo',
|
|
1492
|
+
apiUrl: 'https://api.runtype.com',
|
|
1493
|
+
voiceRecognition: { enabled: true, provider: { type: 'runtype', runtype: { agentId: 'a1' } } },
|
|
1494
|
+
},
|
|
1495
|
+
{ onMessagesChanged: () => {}, onStatusChanged: () => {}, onStreamingChanged: () => {}, onError: () => {} },
|
|
1496
|
+
);
|
|
1497
|
+
|
|
1498
|
+
it("hands the UI the provider's status-line notice once", async () => {
|
|
1499
|
+
const session = tokenSession();
|
|
1500
|
+
session.setupVoice();
|
|
1501
|
+
await flush();
|
|
1502
|
+
h.state.notice = 'This voice call reached its time limit.';
|
|
1503
|
+
expect(session.getVoiceNotice()).toBe('This voice call reached its time limit.');
|
|
1504
|
+
expect(session.getVoiceNotice()).toBeNull();
|
|
1505
|
+
});
|
|
1506
|
+
|
|
1507
|
+
});
|
package/src/types.ts
CHANGED
|
@@ -5230,6 +5230,11 @@ export interface VoiceProvider {
|
|
|
5230
5230
|
setSessionBridge?(bridge: VoiceSessionBridge): void;
|
|
5231
5231
|
/** The AI-disclosure notice for the live call, if one should show (speech-to-speech). */
|
|
5232
5232
|
getDisclosure?(): string | null;
|
|
5233
|
+
/**
|
|
5234
|
+
* Status-line text for the call, returned once: why the server ended it
|
|
5235
|
+
* (`session_end`) or that it is reconnecting. `null` when there is none.
|
|
5236
|
+
*/
|
|
5237
|
+
takeNotice?(): string | null;
|
|
5233
5238
|
}
|
|
5234
5239
|
|
|
5235
5240
|
/** Metadata on a {@link VoiceProvider.onTranscript} update. */
|
|
@@ -5294,6 +5299,12 @@ export type VoiceDelegationResult = {
|
|
|
5294
5299
|
status: VoiceDelegationStatus;
|
|
5295
5300
|
/** The assistant text (Markdown allowed). */
|
|
5296
5301
|
text: string;
|
|
5302
|
+
/**
|
|
5303
|
+
* `false` when no chat message carries this answer (it is only spoken), so
|
|
5304
|
+
* the voice model's read-back of it stays visible as a caption.
|
|
5305
|
+
* @default true
|
|
5306
|
+
*/
|
|
5307
|
+
inChat?: boolean;
|
|
5297
5308
|
/**
|
|
5298
5309
|
* Set when the turn parked on approvals (`text` then asks for the decision).
|
|
5299
5310
|
* Resolves with the terminal result once the visitor decides (or the
|
|
@@ -159,6 +159,23 @@ describe("streamingSubmitBehavior: defer-one", () => {
|
|
|
159
159
|
expect(status?.textContent).toContain("already queued");
|
|
160
160
|
});
|
|
161
161
|
|
|
162
|
+
it("keeps a notice in the status line when the chat status changes under it", async () => {
|
|
163
|
+
const { mount } = makeController();
|
|
164
|
+
await startStreaming(mount);
|
|
165
|
+
type(mount, "queued message");
|
|
166
|
+
pressEnter(mount);
|
|
167
|
+
await flush();
|
|
168
|
+
type(mount, "second attempt");
|
|
169
|
+
pressEnter(mount);
|
|
170
|
+
await flush();
|
|
171
|
+
|
|
172
|
+
stream.push(unifiedFrames());
|
|
173
|
+
stream.close();
|
|
174
|
+
await flush(50);
|
|
175
|
+
const status = mount.querySelector<HTMLElement>("[data-persona-composer-notice]");
|
|
176
|
+
expect(status?.textContent).toContain("already queued");
|
|
177
|
+
});
|
|
178
|
+
|
|
162
179
|
it("card Edit restores the snapshot into the draft and drops the card", async () => {
|
|
163
180
|
const { mount, controller } = makeController();
|
|
164
181
|
await startStreaming(mount);
|
package/src/ui.ts
CHANGED
|
@@ -353,11 +353,14 @@ import {
|
|
|
353
353
|
// Default localStorage key for chat history (automatically cleared on clear chat)
|
|
354
354
|
const DEFAULT_CHAT_HISTORY_STORAGE_KEY = "persona-chat-history";
|
|
355
355
|
const VOICE_STATE_RESTORE_WINDOW = 30 * 1000;
|
|
356
|
+
const VOICE_RESUME_NOTICE = "Tap the mic to resume your voice call";
|
|
356
357
|
// Split desktop boundary; must match widget.css artifact media queries (min-width:641px).
|
|
357
358
|
const ARTIFACT_SPLIT_DESKTOP_MIN = 641;
|
|
358
359
|
|
|
359
360
|
/** Marks the status region while a composer lock reason owns it. */
|
|
360
361
|
const COMPOSER_REASON_ATTR = "data-persona-composer-reason";
|
|
362
|
+
/** Marks the status region while a transient composer notice owns it. */
|
|
363
|
+
const COMPOSER_NOTICE_ATTR = "data-persona-composer-notice";
|
|
361
364
|
|
|
362
365
|
const IMAGE_FILE_EXTENSION_BY_MIME_TYPE: Record<string, string> = {
|
|
363
366
|
"image/png": "png",
|
|
@@ -1384,8 +1387,9 @@ export const createAgentExperience = (
|
|
|
1384
1387
|
placeStatusIndicator(el, composerForm, statusCfg);
|
|
1385
1388
|
applyStatusIndicatorState(el, statusCfg, status);
|
|
1386
1389
|
}
|
|
1387
|
-
// A composer lock reason owns the status region until the lock clears
|
|
1388
|
-
|
|
1390
|
+
// A composer lock reason owns the status region until the lock clears, and
|
|
1391
|
+
// a notice until its timer ends (which re-applies the status).
|
|
1392
|
+
if (el.hasAttribute(COMPOSER_REASON_ATTR) || el.hasAttribute(COMPOSER_NOTICE_ATTR)) return;
|
|
1389
1393
|
if (status === "idle" && statusCfg.idleLink) {
|
|
1390
1394
|
el.textContent = "";
|
|
1391
1395
|
const link = document.createElement("a");
|
|
@@ -5083,6 +5087,15 @@ export const createAgentExperience = (
|
|
|
5083
5087
|
// would otherwise not appear until the first keystroke.
|
|
5084
5088
|
syncComposerCompact();
|
|
5085
5089
|
const voiceAutoResumeMode = config.voiceRecognition?.autoResume ?? false;
|
|
5090
|
+
// A session-voice call is live while listening OR while its continuous
|
|
5091
|
+
// (barge-in) mic is hot: a full-duplex call cycles listening → processing →
|
|
5092
|
+
// speaking, and `isVoiceActive()` alone (status === listening) would record a
|
|
5093
|
+
// live call as hung up and never restore it on the next page.
|
|
5094
|
+
const isSessionVoiceLive = () => session.isVoiceActive() || session.isBargeInActive();
|
|
5095
|
+
// Set on `pagehide`: the unload tears the call down without the visitor
|
|
5096
|
+
// hanging up, so its trailing status changes must not overwrite the
|
|
5097
|
+
// persisted "call was live" record the next page restores from.
|
|
5098
|
+
let pageHiding = false;
|
|
5086
5099
|
const emitVoiceState = (source: AgentWidgetVoiceStateEvent["source"]) => {
|
|
5087
5100
|
eventBus.emit("voice:state", {
|
|
5088
5101
|
active: voiceState.active,
|
|
@@ -5112,17 +5125,37 @@ export const createAgentExperience = (
|
|
|
5112
5125
|
voiceState.manuallyDeactivated = false;
|
|
5113
5126
|
if (usesSessionVoice(config.voiceRecognition?.provider)) {
|
|
5114
5127
|
session.toggleVoice().then(() => {
|
|
5115
|
-
voiceState.active =
|
|
5128
|
+
voiceState.active = isSessionVoiceLive();
|
|
5116
5129
|
emitVoiceState("restore");
|
|
5117
|
-
if (
|
|
5130
|
+
if (voiceState.active) {
|
|
5131
|
+
applyRuntypeMicRecordingStyles();
|
|
5132
|
+
} else {
|
|
5133
|
+
// Typically the autoplay policy: this page has no user gesture
|
|
5134
|
+
// yet, so the audio graph can't start. A mic tap resumes it.
|
|
5135
|
+
persistVoiceMetadata();
|
|
5136
|
+
showComposerNotice(VOICE_RESUME_NOTICE, 10_000);
|
|
5137
|
+
}
|
|
5118
5138
|
});
|
|
5119
5139
|
} else {
|
|
5120
5140
|
startVoiceRecognition("restore");
|
|
5121
5141
|
}
|
|
5122
5142
|
}
|
|
5123
|
-
|
|
5143
|
+
// Session voice redials at once (a navigation mid-call should feel
|
|
5144
|
+
// seamless); toggleVoice() already waits on the provider setup.
|
|
5145
|
+
}, usesSessionVoice(config.voiceRecognition?.provider) ? 0 : 1000);
|
|
5124
5146
|
}
|
|
5125
5147
|
};
|
|
5148
|
+
// Re-stamp a live call as the page unloads: the restore window counts from
|
|
5149
|
+
// the navigation, not from when the (possibly long) call was started. A
|
|
5150
|
+
// back/forward-cache hide (`persisted`) isn't an unload: the page lives on.
|
|
5151
|
+
const handleVoicePageHide = (event: PageTransitionEvent) => {
|
|
5152
|
+
pageHiding = !event.persisted;
|
|
5153
|
+
if (voiceState.active) persistVoiceMetadata();
|
|
5154
|
+
};
|
|
5155
|
+
if (typeof window !== "undefined") {
|
|
5156
|
+
window.addEventListener("pagehide", handleVoicePageHide);
|
|
5157
|
+
destroyCallbacks.push(() => window.removeEventListener("pagehide", handleVoicePageHide));
|
|
5158
|
+
}
|
|
5126
5159
|
|
|
5127
5160
|
const getMessagesForPersistence = () =>
|
|
5128
5161
|
session
|
|
@@ -7916,6 +7949,7 @@ export const createAgentExperience = (
|
|
|
7916
7949
|
statusText.setAttribute("role", "status");
|
|
7917
7950
|
statusText.setAttribute("aria-live", "polite");
|
|
7918
7951
|
statusText.setAttribute(COMPOSER_REASON_ATTR, "");
|
|
7952
|
+
statusText.removeAttribute(COMPOSER_NOTICE_ATTR); // the reason outranks a notice
|
|
7919
7953
|
statusText.textContent = reason;
|
|
7920
7954
|
applyStatusIndicatorState(statusText, config.statusIndicator ?? {}, "idle");
|
|
7921
7955
|
return;
|
|
@@ -8803,6 +8837,9 @@ export const createAgentExperience = (
|
|
|
8803
8837
|
// provider; the mic-button styling below is runtype-specific.
|
|
8804
8838
|
eventBus.emit("voice:status", { status, timestamp: Date.now() });
|
|
8805
8839
|
if (status !== 'disconnected' && !usesSessionVoice(config.voiceRecognition?.provider)) return;
|
|
8840
|
+
// Why the server ended the call, or that it is reconnecting.
|
|
8841
|
+
const voiceNotice = session.getVoiceNotice();
|
|
8842
|
+
if (voiceNotice) showComposerNotice(voiceNotice, 10_000);
|
|
8806
8843
|
|
|
8807
8844
|
switch (status) {
|
|
8808
8845
|
case 'listening':
|
|
@@ -8852,7 +8889,7 @@ export const createAgentExperience = (
|
|
|
8852
8889
|
}
|
|
8853
8890
|
removeRuntypeMicStateStyles();
|
|
8854
8891
|
emitVoiceState("system");
|
|
8855
|
-
persistVoiceMetadata();
|
|
8892
|
+
if (!pageHiding) persistVoiceMetadata();
|
|
8856
8893
|
}
|
|
8857
8894
|
break;
|
|
8858
8895
|
}
|
|
@@ -9507,7 +9544,6 @@ export const createAgentExperience = (
|
|
|
9507
9544
|
resetHistoryNavigation();
|
|
9508
9545
|
};
|
|
9509
9546
|
|
|
9510
|
-
const COMPOSER_NOTICE_ATTR = "data-persona-composer-notice";
|
|
9511
9547
|
let composerNoticeTimer: ReturnType<typeof setTimeout> | null = null;
|
|
9512
9548
|
|
|
9513
9549
|
/** Transient message in the status region; a lock reason always outranks it. */
|
|
@@ -10497,11 +10533,11 @@ export const createAgentExperience = (
|
|
|
10497
10533
|
}
|
|
10498
10534
|
|
|
10499
10535
|
session.toggleVoice().then(() => {
|
|
10500
|
-
voiceState.active =
|
|
10501
|
-
voiceState.manuallyDeactivated = !
|
|
10536
|
+
voiceState.active = isSessionVoiceLive();
|
|
10537
|
+
voiceState.manuallyDeactivated = !voiceState.active;
|
|
10502
10538
|
persistVoiceMetadata();
|
|
10503
10539
|
emitVoiceState("user");
|
|
10504
|
-
if (
|
|
10540
|
+
if (voiceState.active) {
|
|
10505
10541
|
applyRuntypeMicRecordingStyles();
|
|
10506
10542
|
} else {
|
|
10507
10543
|
removeRuntypeMicStateStyles();
|
|
@@ -12956,9 +12992,10 @@ export const createAgentExperience = (
|
|
|
12956
12992
|
voiceState.manuallyDeactivated = false;
|
|
12957
12993
|
persistVoiceMetadata();
|
|
12958
12994
|
session.toggleVoice().then(() => {
|
|
12959
|
-
voiceState.active =
|
|
12995
|
+
voiceState.active = isSessionVoiceLive();
|
|
12996
|
+
persistVoiceMetadata();
|
|
12960
12997
|
emitVoiceState("user");
|
|
12961
|
-
if (
|
|
12998
|
+
if (voiceState.active) applyRuntypeMicRecordingStyles();
|
|
12962
12999
|
});
|
|
12963
13000
|
return true;
|
|
12964
13001
|
}
|
|
@@ -56,10 +56,11 @@
|
|
|
56
56
|
// replaced). A second gated tool in the same turn sends another update.
|
|
57
57
|
// `delegation_cancelled{delegationId}` (or a `warning` refusing a frame for
|
|
58
58
|
// it) stops all frames for that id; the chat turn still renders, and since
|
|
59
|
-
// nobody speaks it now, browser TTS may read it. The voice model's spoken read-back
|
|
60
|
-
//
|
|
61
|
-
//
|
|
62
|
-
//
|
|
59
|
+
// nobody speaks it now, browser TTS may read it. The voice model's spoken read-back of
|
|
60
|
+
// an answered delegation is folded: the chat already shows it. With
|
|
61
|
+
// `delegation_read_back` negotiated it is the utterance the server tags with that
|
|
62
|
+
// `delegationId`; otherwise it is the first new assistant utterance after
|
|
63
|
+
// `delegation_completed`, unless the visitor speaks first. Transcripts in such a call are display-only captions, never sent to the
|
|
63
64
|
// agent as conversation; a user bubble becomes conversation only when it is
|
|
64
65
|
// submitted. With `context` negotiated, one `context{text}` frame (chat history
|
|
65
66
|
// as of call start plus the host's `callContext`) is held until the visitor's
|
|
@@ -98,6 +99,8 @@ const LEVEL_RMS_SCALE = 4;
|
|
|
98
99
|
const RIFF_MAGIC = 0x52494646; // "RIFF"
|
|
99
100
|
/** `prewarm()` warms at most once per this window per provider instance. */
|
|
100
101
|
const PREWARM_THROTTLE_MS = 30_000;
|
|
102
|
+
/** How long a call start waits for a suspended capture context to run. */
|
|
103
|
+
const AUDIO_RESUME_TIMEOUT_MS = 1_000;
|
|
101
104
|
/** How long `startListening()` waits on an attach still handshaking. */
|
|
102
105
|
const ATTACH_HANDSHAKE_TIMEOUT_MS = 5_000;
|
|
103
106
|
/** Attached-socket idle window: server default and clamp range. */
|
|
@@ -141,6 +144,17 @@ const VOICE_PROTOCOL = "runtype-browser-v1";
|
|
|
141
144
|
/** `approvalTimeoutMs` cap: below core's delegation deadline (600 s, restarted by each update). */
|
|
142
145
|
const APPROVAL_TIMEOUT_MAX_MS = 540_000;
|
|
143
146
|
const DISCLOSURE_TEXT = "You're talking to an AI assistant. Voice is processed by OpenAI.";
|
|
147
|
+
/** Status-line text per `session_end.reason` (Amendment 6); unlisted reasons show nothing. */
|
|
148
|
+
const SESSION_END_TEXT = new Map([
|
|
149
|
+
["idle_timeout", "Voice call ended after a quiet period."],
|
|
150
|
+
["max_duration", "This voice call reached its time limit."],
|
|
151
|
+
["provider_ended", "The voice session ended."],
|
|
152
|
+
["quota", "Voice is unavailable right now."],
|
|
153
|
+
["auth_expired", "Voice session expired. Tap the mic to reconnect."],
|
|
154
|
+
]);
|
|
155
|
+
/** Reasons that get one automatic reconnect. */
|
|
156
|
+
const RECONNECT_REASONS = ["provider_error", "server_restart"];
|
|
157
|
+
const CONNECTION_LOST = "Voice connection lost.";
|
|
144
158
|
/** Call-start context frame: total cap, host share, history window, per-message cap. */
|
|
145
159
|
const CONTEXT_MAX_CHARS = 8000;
|
|
146
160
|
const CONTEXT_HOST_MAX_CHARS = 4000;
|
|
@@ -270,6 +284,14 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
270
284
|
#cancelledDelegations = new Set<string>();
|
|
271
285
|
// session_config.callId: the server's id for this call, for logs.
|
|
272
286
|
#callId: string | undefined;
|
|
287
|
+
// Amendment 6: why the server ended this call (its last frame), if it said.
|
|
288
|
+
#sessionEnd: { reason: string; message?: string } | null = null;
|
|
289
|
+
// Status-line text for the UI, read once (`takeNotice`).
|
|
290
|
+
#notice: string | null = null;
|
|
291
|
+
// One automatic reconnect per call the visitor started: pending, and spent.
|
|
292
|
+
#reconnectTimer: ReturnType<typeof setTimeout> | undefined;
|
|
293
|
+
#reconnected = false;
|
|
294
|
+
#autoReconnect = false;
|
|
273
295
|
// Call-start context, built at session_config and held until the visitor's
|
|
274
296
|
// first final transcript (sent earlier, the voice model tends to answer it).
|
|
275
297
|
#contextSent = false;
|
|
@@ -450,7 +472,8 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
450
472
|
*/
|
|
451
473
|
#voiceSocketUrl(host: string, agentId: string, attach?: URLSearchParams): string {
|
|
452
474
|
const capabilities = ["partial_transcript", "context"];
|
|
453
|
-
if (this.bridge && this.config?.clientDelegation !== false)
|
|
475
|
+
if (this.bridge && this.config?.clientDelegation !== false)
|
|
476
|
+
capabilities.push("client_delegation", "delegation_update", "delegation_read_back");
|
|
454
477
|
if (attach) capabilities.unshift("attach");
|
|
455
478
|
const params = new URLSearchParams({
|
|
456
479
|
voiceProtocol: VOICE_PROTOCOL,
|
|
@@ -465,6 +488,13 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
465
488
|
/** Start the call: acquire mic, open the WS, stream PCM until hang-up. */
|
|
466
489
|
async startListening(): Promise<void> {
|
|
467
490
|
if (this.#callLive) return; // idempotent: a call is already live
|
|
491
|
+
// A call the visitor starts gets a fresh reconnect; the automatic one doesn't.
|
|
492
|
+
const auto = this.#autoReconnect;
|
|
493
|
+
if (!auto) {
|
|
494
|
+
this.#reconnected = false;
|
|
495
|
+
this.#notice = null;
|
|
496
|
+
}
|
|
497
|
+
this.#autoReconnect = false;
|
|
468
498
|
|
|
469
499
|
const agentId = this.config?.agentId;
|
|
470
500
|
const token = this.config?.clientToken;
|
|
@@ -497,10 +527,22 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
497
527
|
const captureContext: AudioContext = new AudioCtx({
|
|
498
528
|
sampleRate: CAPTURE_SAMPLE_RATE,
|
|
499
529
|
});
|
|
530
|
+
this.#captureContext = captureContext;
|
|
500
531
|
if (captureContext.state === "suspended") {
|
|
501
|
-
|
|
532
|
+
// Outside a click (the automatic reconnect, or a call restored after a
|
|
533
|
+
// page navigation) the autoplay policy can leave audio suspended and
|
|
534
|
+
// `resume()` pending forever: fail fast rather than hang or open a dead call.
|
|
535
|
+
await Promise.race([
|
|
536
|
+
captureContext.resume().catch(() => {}),
|
|
537
|
+
new Promise((resolve) => setTimeout(resolve, auto ? 500 : AUDIO_RESUME_TIMEOUT_MS)),
|
|
538
|
+
]);
|
|
539
|
+
if (generation !== this.#callGeneration) return; // hung up meanwhile
|
|
540
|
+
if ((captureContext.state as string) !== "running") {
|
|
541
|
+
if (auto) throw new Error((this.#notice = `${CONNECTION_LOST} Tap the mic to reconnect.`));
|
|
542
|
+
// A restored call: the widget asks for a tap to resume.
|
|
543
|
+
throw new Error("Audio blocked until a tap");
|
|
544
|
+
}
|
|
502
545
|
}
|
|
503
|
-
this.#captureContext = captureContext;
|
|
504
546
|
|
|
505
547
|
const engine = this.config?.createPlaybackEngine
|
|
506
548
|
? await this.config.createPlaybackEngine()
|
|
@@ -558,7 +600,8 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
558
600
|
ws.onmessage = (event) => this.#handleMessage(event, generation);
|
|
559
601
|
|
|
560
602
|
ws.onerror = () => {
|
|
561
|
-
|
|
603
|
+
// After `session_end`, the close that follows reports the server's reason.
|
|
604
|
+
if (generation !== this.#callGeneration || this.#sessionEnd) return;
|
|
562
605
|
this.#emitError(new Error("Voice connection failed"));
|
|
563
606
|
this.#emitStatus("error");
|
|
564
607
|
this.#cleanup();
|
|
@@ -570,17 +613,68 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
570
613
|
return;
|
|
571
614
|
}
|
|
572
615
|
if (generation !== this.#callGeneration) return;
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
616
|
+
// Why the server ended it: its `session_end`, or the attach idle close.
|
|
617
|
+
const end = this.#sessionEnd ?? (evt.code === 4408 ? { reason: "idle_timeout" } : null);
|
|
618
|
+
if (!end) {
|
|
619
|
+
// No reason given (a deploy, a dropped connection): as before.
|
|
620
|
+
if (evt.code !== 1000) {
|
|
621
|
+
const codeMsg = evt.code ? ` (code ${evt.code})` : "";
|
|
622
|
+
this.#emitError(new Error(`Voice connection closed${codeMsg}`));
|
|
623
|
+
this.#emitStatus("error");
|
|
624
|
+
} else {
|
|
625
|
+
this.#emitStatus("idle");
|
|
626
|
+
}
|
|
627
|
+
this.#cleanup();
|
|
628
|
+
return;
|
|
579
629
|
}
|
|
580
|
-
this.#
|
|
630
|
+
this.#endCall(end.reason, end.message);
|
|
581
631
|
};
|
|
582
632
|
}
|
|
583
633
|
|
|
634
|
+
/**
|
|
635
|
+
* The server ended the call with a reason (`session_end`, Amendment 6): show
|
|
636
|
+
* its text, and reconnect once after a provider error or a restart. Unknown
|
|
637
|
+
* reasons are an ordinary end (`ended_by_server`).
|
|
638
|
+
*/
|
|
639
|
+
#endCall(reason: string, message?: string): void {
|
|
640
|
+
const retry = RECONNECT_REASONS.includes(reason) && !this.#reconnected;
|
|
641
|
+
this.#cleanup();
|
|
642
|
+
if (retry) {
|
|
643
|
+
// Keep the call "live" while it waits, so the mic button hangs up (and cancels).
|
|
644
|
+
this.#reconnected = true;
|
|
645
|
+
this.#callLive = true;
|
|
646
|
+
this.#notice = reason === "provider_error" ? `${CONNECTION_LOST} Reconnecting…` : "Reconnecting…";
|
|
647
|
+
// A hang-up meanwhile runs #cleanup, which clears this timer.
|
|
648
|
+
this.#reconnectTimer = setTimeout(
|
|
649
|
+
() => {
|
|
650
|
+
this.#callLive = false;
|
|
651
|
+
this.#autoReconnect = true;
|
|
652
|
+
void this.startListening().catch(() => {});
|
|
653
|
+
},
|
|
654
|
+
1000 + Math.random() * 1000,
|
|
655
|
+
);
|
|
656
|
+
this.#emitStatus("idle");
|
|
657
|
+
return;
|
|
658
|
+
}
|
|
659
|
+
if (RECONNECT_REASONS.includes(reason)) {
|
|
660
|
+
// The reconnect already ran: this end is the second failure.
|
|
661
|
+
this.#notice = CONNECTION_LOST;
|
|
662
|
+
this.#emitError(new Error(CONNECTION_LOST));
|
|
663
|
+
return this.#emitStatus("error");
|
|
664
|
+
}
|
|
665
|
+
// `error`: the preceding `error` frame already reported it.
|
|
666
|
+
if (reason === "error") return this.#emitStatus("error");
|
|
667
|
+
this.#notice = (reason === "quota" && message) || SESSION_END_TEXT.get(reason) || null;
|
|
668
|
+
this.#emitStatus("idle");
|
|
669
|
+
}
|
|
670
|
+
|
|
671
|
+
/** Status-line text for the call (why it ended, or that it is reconnecting), once. */
|
|
672
|
+
takeNotice(): string | null {
|
|
673
|
+
const notice = this.#notice;
|
|
674
|
+
this.#notice = null;
|
|
675
|
+
return notice;
|
|
676
|
+
}
|
|
677
|
+
|
|
584
678
|
/** The AI-disclosure notice for a live speech-to-speech call (`disclosureText`; `false` hides it). */
|
|
585
679
|
getDisclosure(): string | null {
|
|
586
680
|
const text = this.config?.disclosureText;
|
|
@@ -746,9 +840,17 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
746
840
|
if (msg.final === true) this.#flushCallContext(generation);
|
|
747
841
|
this.#foldReadback = false;
|
|
748
842
|
} else {
|
|
749
|
-
// Read-back of a delegated result
|
|
750
|
-
//
|
|
751
|
-
|
|
843
|
+
// Read-back of a delegated result the chat already renders. A server
|
|
844
|
+
// that negotiated `delegation_read_back` tags it with its delegation;
|
|
845
|
+
// an older one rotates the assistant id at completion, so it is the
|
|
846
|
+
// first new id after it (a guess a filler finishing late defeats).
|
|
847
|
+
if (this.#capabilities.has("delegation_read_back")) {
|
|
848
|
+
// Fold only from the first frame: hiding an utterance already shown would strand a partial caption.
|
|
849
|
+
const tag = msg.delegationId;
|
|
850
|
+
if (typeof tag === "string" && this.#answered.has(tag) && !this.#assistantTurns.has(turnId)) {
|
|
851
|
+
this.#foldedTurns.add(turnId);
|
|
852
|
+
}
|
|
853
|
+
} else if (this.#foldReadback && !this.#assistantTurns.has(turnId)) {
|
|
752
854
|
this.#foldedTurns.add(turnId);
|
|
753
855
|
this.#foldReadback = false;
|
|
754
856
|
}
|
|
@@ -862,6 +964,14 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
862
964
|
this.#emitError(new Error(msg.error || "Voice error"));
|
|
863
965
|
this.#emitStatus("error");
|
|
864
966
|
break;
|
|
967
|
+
|
|
968
|
+
case "session_end":
|
|
969
|
+
// The server's last frame before it closes: the close acts on it.
|
|
970
|
+
this.#sessionEnd = {
|
|
971
|
+
reason: String(msg.reason),
|
|
972
|
+
...(typeof msg.message === "string" && { message: msg.message }),
|
|
973
|
+
};
|
|
974
|
+
break;
|
|
865
975
|
}
|
|
866
976
|
}
|
|
867
977
|
|
|
@@ -946,7 +1056,7 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
946
1056
|
result.text,
|
|
947
1057
|
);
|
|
948
1058
|
if (!sent) return;
|
|
949
|
-
if (result.status !== "failed") this.#answered.add(delegationId);
|
|
1059
|
+
if (result.status !== "failed" && result.inChat !== false) this.#answered.add(delegationId);
|
|
950
1060
|
if (!result.followUp) return;
|
|
951
1061
|
// The approval bookkeeping (expiry, supersede) runs either way.
|
|
952
1062
|
const abort = new AbortController();
|
|
@@ -1053,6 +1163,8 @@ export class RuntypeVoiceProvider implements VoiceProvider {
|
|
|
1053
1163
|
this.#followUps.clear();
|
|
1054
1164
|
this.#cancelledDelegations.clear();
|
|
1055
1165
|
this.#callId = undefined;
|
|
1166
|
+
this.#sessionEnd = null;
|
|
1167
|
+
clearTimeout(this.#reconnectTimer);
|
|
1056
1168
|
this.#contextSent = false;
|
|
1057
1169
|
this.#pendingContext = null;
|
|
1058
1170
|
this.#contextSend = null;
|
|
@@ -164,16 +164,32 @@ export function createVoiceSessionBridge(host: VoiceDelegationHost): VoiceSessio
|
|
|
164
164
|
};
|
|
165
165
|
|
|
166
166
|
return {
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
167
|
+
// Read once at call start, before this call has any captions, so the
|
|
168
|
+
// captions here are earlier calls' (e.g. on the previous page of a
|
|
169
|
+
// multi-page site): spoken conversation the new voice session never heard.
|
|
170
|
+
// An assistant caption followed by a chat answer before the next user turn
|
|
171
|
+
// is a delegated turn's spoken filler ("Let me check."): it would only use
|
|
172
|
+
// up a slot of the context's recent-message window.
|
|
173
|
+
getHistory: () => {
|
|
174
|
+
const settled = host.messages().filter(
|
|
175
|
+
(m) =>
|
|
176
|
+
(m.role === "user" || m.role === "assistant") &&
|
|
177
|
+
!m.variant &&
|
|
178
|
+
!m.voiceProcessing &&
|
|
179
|
+
!m.streaming &&
|
|
180
|
+
m.content,
|
|
181
|
+
);
|
|
182
|
+
let answered = false;
|
|
183
|
+
const kept: Array<{ role: "user" | "assistant"; content: string }> = [];
|
|
184
|
+
for (let i = settled.length - 1; i >= 0; i -= 1) {
|
|
185
|
+
const m = settled[i];
|
|
186
|
+
if (m.role === "user") answered = false;
|
|
187
|
+
else if (!m.voiceCaption) answered = true;
|
|
188
|
+
else if (answered) continue;
|
|
189
|
+
kept.unshift({ role: m.role as "user" | "assistant", content: m.content });
|
|
190
|
+
}
|
|
191
|
+
return kept;
|
|
192
|
+
},
|
|
177
193
|
|
|
178
194
|
dropDelegation(delegationId, expired) {
|
|
179
195
|
// Remembered for a turn still running: when it parks, the same rule applies.
|
|
@@ -201,7 +217,7 @@ export function createVoiceSessionBridge(host: VoiceDelegationHost): VoiceSessio
|
|
|
201
217
|
const tool = toolOf(pendingIds[0]);
|
|
202
218
|
// The parked delegation still gets its one terminal result, silently.
|
|
203
219
|
settle(live[0], "denied", "");
|
|
204
|
-
return { status: "denied", text: `Okay, I cancelled the ${tool} request. Nothing was done
|
|
220
|
+
return { status: "denied", text: `Okay, I cancelled the ${tool} request. Nothing was done.`, inChat: false };
|
|
205
221
|
}
|
|
206
222
|
// A voice-originated WebMCP approval holds the chat turn open: this turn
|
|
207
223
|
// replaces it (a new send declines it), so don't wait on it. Anything
|