@runtypelabs/persona 4.26.0 → 4.27.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. package/dist/approval-ui.d.cts +11 -0
  2. package/dist/approval-ui.d.ts +11 -0
  3. package/dist/artifacts-ui.d.cts +11 -0
  4. package/dist/artifacts-ui.d.ts +11 -0
  5. package/dist/chunk-3255TNY7.js +1 -0
  6. package/dist/{client-history-YPEG45Q4.js → client-history-HTRKHBAF.js} +1 -1
  7. package/dist/client-history.js +1 -1
  8. package/dist/client-stream-N22JHTLE.js +4 -0
  9. package/dist/client-stream.js +3 -3
  10. package/dist/codegen.cjs +1 -1
  11. package/dist/codegen.js +1 -1
  12. package/dist/context-mentions.d.cts +11 -0
  13. package/dist/context-mentions.d.ts +11 -0
  14. package/dist/event-stream-view.d.cts +11 -0
  15. package/dist/event-stream-view.d.ts +11 -0
  16. package/dist/forms-ui.d.cts +13 -0
  17. package/dist/forms-ui.d.ts +13 -0
  18. package/dist/index.cjs +57 -57
  19. package/dist/index.cjs.map +1 -1
  20. package/dist/index.d.cts +13 -0
  21. package/dist/index.d.ts +13 -0
  22. package/dist/index.global.js +1 -1
  23. package/dist/index.global.js.map +1 -1
  24. package/dist/index.js +58 -58
  25. package/dist/index.js.map +1 -1
  26. package/dist/session-reconnect.d.cts +11 -0
  27. package/dist/session-reconnect.d.ts +11 -0
  28. package/dist/smart-dom-reader.d.cts +11 -0
  29. package/dist/smart-dom-reader.d.ts +11 -0
  30. package/dist/theme-editor-preview.cjs +59 -59
  31. package/dist/theme-editor-preview.d.cts +11 -0
  32. package/dist/theme-editor-preview.d.ts +11 -0
  33. package/dist/theme-editor-preview.js +31 -31
  34. package/dist/theme-editor.d.cts +11 -0
  35. package/dist/theme-editor.d.ts +11 -0
  36. package/dist/voice-runtime-RHRX4F3T.js +17 -0
  37. package/dist/voice-runtime.cjs +8 -8
  38. package/dist/voice-runtime.d.cts +15 -0
  39. package/dist/voice-runtime.d.ts +15 -0
  40. package/dist/voice-runtime.js +8 -8
  41. package/package.json +1 -1
  42. package/src/approval-paused-tool-settle.test.ts +108 -4
  43. package/src/client-stream.ts +10 -3
  44. package/src/client.test.ts +20 -0
  45. package/src/session.ts +56 -10
  46. package/src/session.voice.test.ts +60 -3
  47. package/src/types.ts +11 -0
  48. package/src/ui.composer-defer-one.test.ts +17 -0
  49. package/src/ui.ts +49 -12
  50. package/src/voice/__fixtures__/voice-wire/server/session_end-provider-error.json +6 -0
  51. package/src/voice/__fixtures__/voice-wire/server/session_end-unknown-reason.json +4 -0
  52. package/src/voice/__fixtures__/voice-wire/server/session_end.json +5 -0
  53. package/src/voice/runtype-voice-provider.ts +131 -19
  54. package/src/voice/voice-delegation.ts +27 -11
  55. package/src/voice/voice.test.ts +257 -2
  56. package/dist/chunk-QPPJUSZH.js +0 -1
  57. package/dist/client-stream-42FOI3EC.js +0 -4
  58. package/dist/voice-runtime-UOQOFAFY.js +0 -17
@@ -36,7 +36,16 @@ const h = vi.hoisted(() => {
36
36
  errorCb: ((e: Error) => void) | null;
37
37
  bridge: VoiceSessionBridge | null;
38
38
  prewarms: number;
39
- } = { transcriptCb: null, metricsCb: null, statusCb: null, errorCb: null, bridge: null, prewarms: 0 };
39
+ notice: string | null;
40
+ } = {
41
+ transcriptCb: null,
42
+ metricsCb: null,
43
+ statusCb: null,
44
+ errorCb: null,
45
+ bridge: null,
46
+ prewarms: 0,
47
+ notice: null,
48
+ };
40
49
 
41
50
  const fakeProvider = {
42
51
  type: 'runtype' as const,
@@ -64,6 +73,11 @@ const h = vi.hoisted(() => {
64
73
  setSessionBridge: (bridge: VoiceSessionBridge) => {
65
74
  state.bridge = bridge;
66
75
  },
76
+ takeNotice: () => {
77
+ const notice = state.notice;
78
+ state.notice = null;
79
+ return notice;
80
+ },
67
81
  };
68
82
 
69
83
  return { state, fakeProvider };
@@ -513,12 +527,31 @@ describe('AgentWidgetSession - voice client delegation bridge', () => {
513
527
  });
514
528
 
515
529
  it('hands the provider a bridge whose history is the visible settled messages', () => {
516
- drive('user', 'still talk', false, 'u1'); // interim: not part of the context
517
- // An earlier call's captions (filler, small talk) aren't conversation either.
530
+ // An earlier call's captions (e.g. on the previous page) are conversation
531
+ // the new voice session never heard, so they carry over.
518
532
  h.state.transcriptCb!('assistant', 'Let me check.', true, { turnId: 'f0', caption: true });
519
533
  h.state.transcriptCb!('user', 'hi there', true, { turnId: 'u0', caption: true });
534
+ drive('user', 'still talk', false, 'u1'); // interim: not part of the context
535
+ expect(h.state.bridge!.getHistory()).toEqual([
536
+ { role: 'assistant', content: 'Welcome! How can I help?' },
537
+ { role: 'assistant', content: 'Let me check.' },
538
+ { role: 'user', content: 'hi there' },
539
+ ]);
540
+ });
541
+
542
+ it("leaves a delegated turn's spoken filler out of the history, keeping the answer", async () => {
543
+ // Full duplex: the voice model's words arrive as captions.
544
+ h.state.transcriptCb!('user', 'whats the weather in paris', true, { turnId: 'u1', caption: true });
545
+ h.state.transcriptCb!('assistant', 'Let me check that.', true, { turnId: 'f1', caption: true });
546
+ dispatch.mockImplementation(async (_options, onEvent) => reply(onEvent, 'It is sunny in Paris.'));
547
+ await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'whats the weather in paris' }));
548
+ h.state.transcriptCb!('assistant', 'Anything else?', true, { turnId: 'a2', caption: true });
549
+
520
550
  expect(h.state.bridge!.getHistory()).toEqual([
521
551
  { role: 'assistant', content: 'Welcome! How can I help?' },
552
+ { role: 'user', content: 'whats the weather in paris' },
553
+ { role: 'assistant', content: 'It is sunny in Paris.' },
554
+ { role: 'assistant', content: 'Anything else?' },
522
555
  ]);
523
556
  });
524
557
 
@@ -790,6 +823,7 @@ describe('AgentWidgetSession - voice client delegation bridge', () => {
790
823
  expect(decline).toEqual({
791
824
  status: 'denied',
792
825
  text: 'Okay, I cancelled the place pickup order request. Nothing was done.',
826
+ inChat: false, // only spoken: its read-back stays as a caption
793
827
  });
794
828
  expect(dispatch).toHaveBeenCalledTimes(1); // no chat turn for the decline
795
829
  expect(approvalOf().status).toBe('denied');
@@ -1448,3 +1482,26 @@ describe('AgentWidgetSession - prewarmVoice', () => {
1448
1482
  expect(h.state.prewarms).toBe(0);
1449
1483
  });
1450
1484
  });
1485
+
1486
+ describe('AgentWidgetSession - voice session end (Amendment 6)', () => {
1487
+ const flush = () => new Promise((resolve) => setTimeout(resolve, 0));
1488
+ const tokenSession = () =>
1489
+ new AgentWidgetSession(
1490
+ {
1491
+ clientToken: 'ct_live_demo',
1492
+ apiUrl: 'https://api.runtype.com',
1493
+ voiceRecognition: { enabled: true, provider: { type: 'runtype', runtype: { agentId: 'a1' } } },
1494
+ },
1495
+ { onMessagesChanged: () => {}, onStatusChanged: () => {}, onStreamingChanged: () => {}, onError: () => {} },
1496
+ );
1497
+
1498
+ it("hands the UI the provider's status-line notice once", async () => {
1499
+ const session = tokenSession();
1500
+ session.setupVoice();
1501
+ await flush();
1502
+ h.state.notice = 'This voice call reached its time limit.';
1503
+ expect(session.getVoiceNotice()).toBe('This voice call reached its time limit.');
1504
+ expect(session.getVoiceNotice()).toBeNull();
1505
+ });
1506
+
1507
+ });
package/src/types.ts CHANGED
@@ -5230,6 +5230,11 @@ export interface VoiceProvider {
5230
5230
  setSessionBridge?(bridge: VoiceSessionBridge): void;
5231
5231
  /** The AI-disclosure notice for the live call, if one should show (speech-to-speech). */
5232
5232
  getDisclosure?(): string | null;
5233
+ /**
5234
+ * Status-line text for the call, returned once: why the server ended it
5235
+ * (`session_end`) or that it is reconnecting. `null` when there is none.
5236
+ */
5237
+ takeNotice?(): string | null;
5233
5238
  }
5234
5239
 
5235
5240
  /** Metadata on a {@link VoiceProvider.onTranscript} update. */
@@ -5294,6 +5299,12 @@ export type VoiceDelegationResult = {
5294
5299
  status: VoiceDelegationStatus;
5295
5300
  /** The assistant text (Markdown allowed). */
5296
5301
  text: string;
5302
+ /**
5303
+ * `false` when no chat message carries this answer (it is only spoken), so
5304
+ * the voice model's read-back of it stays visible as a caption.
5305
+ * @default true
5306
+ */
5307
+ inChat?: boolean;
5297
5308
  /**
5298
5309
  * Set when the turn parked on approvals (`text` then asks for the decision).
5299
5310
  * Resolves with the terminal result once the visitor decides (or the
@@ -159,6 +159,23 @@ describe("streamingSubmitBehavior: defer-one", () => {
159
159
  expect(status?.textContent).toContain("already queued");
160
160
  });
161
161
 
162
+ it("keeps a notice in the status line when the chat status changes under it", async () => {
163
+ const { mount } = makeController();
164
+ await startStreaming(mount);
165
+ type(mount, "queued message");
166
+ pressEnter(mount);
167
+ await flush();
168
+ type(mount, "second attempt");
169
+ pressEnter(mount);
170
+ await flush();
171
+
172
+ stream.push(unifiedFrames());
173
+ stream.close();
174
+ await flush(50);
175
+ const status = mount.querySelector<HTMLElement>("[data-persona-composer-notice]");
176
+ expect(status?.textContent).toContain("already queued");
177
+ });
178
+
162
179
  it("card Edit restores the snapshot into the draft and drops the card", async () => {
163
180
  const { mount, controller } = makeController();
164
181
  await startStreaming(mount);
package/src/ui.ts CHANGED
@@ -353,11 +353,14 @@ import {
353
353
  // Default localStorage key for chat history (automatically cleared on clear chat)
354
354
  const DEFAULT_CHAT_HISTORY_STORAGE_KEY = "persona-chat-history";
355
355
  const VOICE_STATE_RESTORE_WINDOW = 30 * 1000;
356
+ const VOICE_RESUME_NOTICE = "Tap the mic to resume your voice call";
356
357
  // Split desktop boundary; must match widget.css artifact media queries (min-width:641px).
357
358
  const ARTIFACT_SPLIT_DESKTOP_MIN = 641;
358
359
 
359
360
  /** Marks the status region while a composer lock reason owns it. */
360
361
  const COMPOSER_REASON_ATTR = "data-persona-composer-reason";
362
+ /** Marks the status region while a transient composer notice owns it. */
363
+ const COMPOSER_NOTICE_ATTR = "data-persona-composer-notice";
361
364
 
362
365
  const IMAGE_FILE_EXTENSION_BY_MIME_TYPE: Record<string, string> = {
363
366
  "image/png": "png",
@@ -1384,8 +1387,9 @@ export const createAgentExperience = (
1384
1387
  placeStatusIndicator(el, composerForm, statusCfg);
1385
1388
  applyStatusIndicatorState(el, statusCfg, status);
1386
1389
  }
1387
- // A composer lock reason owns the status region until the lock clears.
1388
- if (el.hasAttribute(COMPOSER_REASON_ATTR)) return;
1390
+ // A composer lock reason owns the status region until the lock clears, and
1391
+ // a notice until its timer ends (which re-applies the status).
1392
+ if (el.hasAttribute(COMPOSER_REASON_ATTR) || el.hasAttribute(COMPOSER_NOTICE_ATTR)) return;
1389
1393
  if (status === "idle" && statusCfg.idleLink) {
1390
1394
  el.textContent = "";
1391
1395
  const link = document.createElement("a");
@@ -5083,6 +5087,15 @@ export const createAgentExperience = (
5083
5087
  // would otherwise not appear until the first keystroke.
5084
5088
  syncComposerCompact();
5085
5089
  const voiceAutoResumeMode = config.voiceRecognition?.autoResume ?? false;
5090
+ // A session-voice call is live while listening OR while its continuous
5091
+ // (barge-in) mic is hot: a full-duplex call cycles listening → processing →
5092
+ // speaking, and `isVoiceActive()` alone (status === listening) would record a
5093
+ // live call as hung up and never restore it on the next page.
5094
+ const isSessionVoiceLive = () => session.isVoiceActive() || session.isBargeInActive();
5095
+ // Set on `pagehide`: the unload tears the call down without the visitor
5096
+ // hanging up, so its trailing status changes must not overwrite the
5097
+ // persisted "call was live" record the next page restores from.
5098
+ let pageHiding = false;
5086
5099
  const emitVoiceState = (source: AgentWidgetVoiceStateEvent["source"]) => {
5087
5100
  eventBus.emit("voice:state", {
5088
5101
  active: voiceState.active,
@@ -5112,17 +5125,37 @@ export const createAgentExperience = (
5112
5125
  voiceState.manuallyDeactivated = false;
5113
5126
  if (usesSessionVoice(config.voiceRecognition?.provider)) {
5114
5127
  session.toggleVoice().then(() => {
5115
- voiceState.active = session.isVoiceActive();
5128
+ voiceState.active = isSessionVoiceLive();
5116
5129
  emitVoiceState("restore");
5117
- if (session.isVoiceActive()) applyRuntypeMicRecordingStyles();
5130
+ if (voiceState.active) {
5131
+ applyRuntypeMicRecordingStyles();
5132
+ } else {
5133
+ // Typically the autoplay policy: this page has no user gesture
5134
+ // yet, so the audio graph can't start. A mic tap resumes it.
5135
+ persistVoiceMetadata();
5136
+ showComposerNotice(VOICE_RESUME_NOTICE, 10_000);
5137
+ }
5118
5138
  });
5119
5139
  } else {
5120
5140
  startVoiceRecognition("restore");
5121
5141
  }
5122
5142
  }
5123
- }, 1000);
5143
+ // Session voice redials at once (a navigation mid-call should feel
5144
+ // seamless); toggleVoice() already waits on the provider setup.
5145
+ }, usesSessionVoice(config.voiceRecognition?.provider) ? 0 : 1000);
5124
5146
  }
5125
5147
  };
5148
+ // Re-stamp a live call as the page unloads: the restore window counts from
5149
+ // the navigation, not from when the (possibly long) call was started. A
5150
+ // back/forward-cache hide (`persisted`) isn't an unload: the page lives on.
5151
+ const handleVoicePageHide = (event: PageTransitionEvent) => {
5152
+ pageHiding = !event.persisted;
5153
+ if (voiceState.active) persistVoiceMetadata();
5154
+ };
5155
+ if (typeof window !== "undefined") {
5156
+ window.addEventListener("pagehide", handleVoicePageHide);
5157
+ destroyCallbacks.push(() => window.removeEventListener("pagehide", handleVoicePageHide));
5158
+ }
5126
5159
 
5127
5160
  const getMessagesForPersistence = () =>
5128
5161
  session
@@ -7916,6 +7949,7 @@ export const createAgentExperience = (
7916
7949
  statusText.setAttribute("role", "status");
7917
7950
  statusText.setAttribute("aria-live", "polite");
7918
7951
  statusText.setAttribute(COMPOSER_REASON_ATTR, "");
7952
+ statusText.removeAttribute(COMPOSER_NOTICE_ATTR); // the reason outranks a notice
7919
7953
  statusText.textContent = reason;
7920
7954
  applyStatusIndicatorState(statusText, config.statusIndicator ?? {}, "idle");
7921
7955
  return;
@@ -8803,6 +8837,9 @@ export const createAgentExperience = (
8803
8837
  // provider; the mic-button styling below is runtype-specific.
8804
8838
  eventBus.emit("voice:status", { status, timestamp: Date.now() });
8805
8839
  if (status !== 'disconnected' && !usesSessionVoice(config.voiceRecognition?.provider)) return;
8840
+ // Why the server ended the call, or that it is reconnecting.
8841
+ const voiceNotice = session.getVoiceNotice();
8842
+ if (voiceNotice) showComposerNotice(voiceNotice, 10_000);
8806
8843
 
8807
8844
  switch (status) {
8808
8845
  case 'listening':
@@ -8852,7 +8889,7 @@ export const createAgentExperience = (
8852
8889
  }
8853
8890
  removeRuntypeMicStateStyles();
8854
8891
  emitVoiceState("system");
8855
- persistVoiceMetadata();
8892
+ if (!pageHiding) persistVoiceMetadata();
8856
8893
  }
8857
8894
  break;
8858
8895
  }
@@ -9507,7 +9544,6 @@ export const createAgentExperience = (
9507
9544
  resetHistoryNavigation();
9508
9545
  };
9509
9546
 
9510
- const COMPOSER_NOTICE_ATTR = "data-persona-composer-notice";
9511
9547
  let composerNoticeTimer: ReturnType<typeof setTimeout> | null = null;
9512
9548
 
9513
9549
  /** Transient message in the status region; a lock reason always outranks it. */
@@ -10497,11 +10533,11 @@ export const createAgentExperience = (
10497
10533
  }
10498
10534
 
10499
10535
  session.toggleVoice().then(() => {
10500
- voiceState.active = session.isVoiceActive();
10501
- voiceState.manuallyDeactivated = !session.isVoiceActive();
10536
+ voiceState.active = isSessionVoiceLive();
10537
+ voiceState.manuallyDeactivated = !voiceState.active;
10502
10538
  persistVoiceMetadata();
10503
10539
  emitVoiceState("user");
10504
- if (session.isVoiceActive()) {
10540
+ if (voiceState.active) {
10505
10541
  applyRuntypeMicRecordingStyles();
10506
10542
  } else {
10507
10543
  removeRuntypeMicStateStyles();
@@ -12956,9 +12992,10 @@ export const createAgentExperience = (
12956
12992
  voiceState.manuallyDeactivated = false;
12957
12993
  persistVoiceMetadata();
12958
12994
  session.toggleVoice().then(() => {
12959
- voiceState.active = session.isVoiceActive();
12995
+ voiceState.active = isSessionVoiceLive();
12996
+ persistVoiceMetadata();
12960
12997
  emitVoiceState("user");
12961
- if (session.isVoiceActive()) applyRuntypeMicRecordingStyles();
12998
+ if (voiceState.active) applyRuntypeMicRecordingStyles();
12962
12999
  });
12963
13000
  return true;
12964
13001
  }
@@ -0,0 +1,6 @@
1
+ {
2
+ "type": "session_end",
3
+ "reason": "provider_error",
4
+ "message": "The voice provider disconnected.",
5
+ "retryable": true
6
+ }
@@ -0,0 +1,4 @@
1
+ {
2
+ "type": "session_end",
3
+ "reason": "maintenance_window"
4
+ }
@@ -0,0 +1,5 @@
1
+ {
2
+ "type": "session_end",
3
+ "reason": "max_duration",
4
+ "retryable": true
5
+ }
@@ -56,10 +56,11 @@
56
56
  // replaced). A second gated tool in the same turn sends another update.
57
57
  // `delegation_cancelled{delegationId}` (or a `warning` refusing a frame for
58
58
  // it) stops all frames for that id; the chat turn still renders, and since
59
- // nobody speaks it now, browser TTS may read it. The voice model's spoken read-back after each
60
- // `delegation_completed` of an answered delegation (the first new assistant
61
- // utterance, unless the visitor speaks first) is folded: the chat already shows
62
- // it. Transcripts in such a call are display-only captions, never sent to the
59
+ // nobody speaks it now, browser TTS may read it. The voice model's spoken read-back of
60
+ // an answered delegation is folded: the chat already shows it. With
61
+ // `delegation_read_back` negotiated it is the utterance the server tags with that
62
+ // `delegationId`; otherwise it is the first new assistant utterance after
63
+ // `delegation_completed`, unless the visitor speaks first. Transcripts in such a call are display-only captions, never sent to the
63
64
  // agent as conversation; a user bubble becomes conversation only when it is
64
65
  // submitted. With `context` negotiated, one `context{text}` frame (chat history
65
66
  // as of call start plus the host's `callContext`) is held until the visitor's
@@ -98,6 +99,8 @@ const LEVEL_RMS_SCALE = 4;
98
99
  const RIFF_MAGIC = 0x52494646; // "RIFF"
99
100
  /** `prewarm()` warms at most once per this window per provider instance. */
100
101
  const PREWARM_THROTTLE_MS = 30_000;
102
+ /** How long a call start waits for a suspended capture context to run. */
103
+ const AUDIO_RESUME_TIMEOUT_MS = 1_000;
101
104
  /** How long `startListening()` waits on an attach still handshaking. */
102
105
  const ATTACH_HANDSHAKE_TIMEOUT_MS = 5_000;
103
106
  /** Attached-socket idle window: server default and clamp range. */
@@ -141,6 +144,17 @@ const VOICE_PROTOCOL = "runtype-browser-v1";
141
144
  /** `approvalTimeoutMs` cap: below core's delegation deadline (600 s, restarted by each update). */
142
145
  const APPROVAL_TIMEOUT_MAX_MS = 540_000;
143
146
  const DISCLOSURE_TEXT = "You're talking to an AI assistant. Voice is processed by OpenAI.";
147
+ /** Status-line text per `session_end.reason` (Amendment 6); unlisted reasons show nothing. */
148
+ const SESSION_END_TEXT = new Map([
149
+ ["idle_timeout", "Voice call ended after a quiet period."],
150
+ ["max_duration", "This voice call reached its time limit."],
151
+ ["provider_ended", "The voice session ended."],
152
+ ["quota", "Voice is unavailable right now."],
153
+ ["auth_expired", "Voice session expired. Tap the mic to reconnect."],
154
+ ]);
155
+ /** Reasons that get one automatic reconnect. */
156
+ const RECONNECT_REASONS = ["provider_error", "server_restart"];
157
+ const CONNECTION_LOST = "Voice connection lost.";
144
158
  /** Call-start context frame: total cap, host share, history window, per-message cap. */
145
159
  const CONTEXT_MAX_CHARS = 8000;
146
160
  const CONTEXT_HOST_MAX_CHARS = 4000;
@@ -270,6 +284,14 @@ export class RuntypeVoiceProvider implements VoiceProvider {
270
284
  #cancelledDelegations = new Set<string>();
271
285
  // session_config.callId: the server's id for this call, for logs.
272
286
  #callId: string | undefined;
287
+ // Amendment 6: why the server ended this call (its last frame), if it said.
288
+ #sessionEnd: { reason: string; message?: string } | null = null;
289
+ // Status-line text for the UI, read once (`takeNotice`).
290
+ #notice: string | null = null;
291
+ // One automatic reconnect per call the visitor started: pending, and spent.
292
+ #reconnectTimer: ReturnType<typeof setTimeout> | undefined;
293
+ #reconnected = false;
294
+ #autoReconnect = false;
273
295
  // Call-start context, built at session_config and held until the visitor's
274
296
  // first final transcript (sent earlier, the voice model tends to answer it).
275
297
  #contextSent = false;
@@ -450,7 +472,8 @@ export class RuntypeVoiceProvider implements VoiceProvider {
450
472
  */
451
473
  #voiceSocketUrl(host: string, agentId: string, attach?: URLSearchParams): string {
452
474
  const capabilities = ["partial_transcript", "context"];
453
- if (this.bridge && this.config?.clientDelegation !== false) capabilities.push("client_delegation", "delegation_update");
475
+ if (this.bridge && this.config?.clientDelegation !== false)
476
+ capabilities.push("client_delegation", "delegation_update", "delegation_read_back");
454
477
  if (attach) capabilities.unshift("attach");
455
478
  const params = new URLSearchParams({
456
479
  voiceProtocol: VOICE_PROTOCOL,
@@ -465,6 +488,13 @@ export class RuntypeVoiceProvider implements VoiceProvider {
465
488
  /** Start the call: acquire mic, open the WS, stream PCM until hang-up. */
466
489
  async startListening(): Promise<void> {
467
490
  if (this.#callLive) return; // idempotent: a call is already live
491
+ // A call the visitor starts gets a fresh reconnect; the automatic one doesn't.
492
+ const auto = this.#autoReconnect;
493
+ if (!auto) {
494
+ this.#reconnected = false;
495
+ this.#notice = null;
496
+ }
497
+ this.#autoReconnect = false;
468
498
 
469
499
  const agentId = this.config?.agentId;
470
500
  const token = this.config?.clientToken;
@@ -497,10 +527,22 @@ export class RuntypeVoiceProvider implements VoiceProvider {
497
527
  const captureContext: AudioContext = new AudioCtx({
498
528
  sampleRate: CAPTURE_SAMPLE_RATE,
499
529
  });
530
+ this.#captureContext = captureContext;
500
531
  if (captureContext.state === "suspended") {
501
- await captureContext.resume().catch(() => {});
532
+ // Outside a click (the automatic reconnect, or a call restored after a
533
+ // page navigation) the autoplay policy can leave audio suspended and
534
+ // `resume()` pending forever: fail fast rather than hang or open a dead call.
535
+ await Promise.race([
536
+ captureContext.resume().catch(() => {}),
537
+ new Promise((resolve) => setTimeout(resolve, auto ? 500 : AUDIO_RESUME_TIMEOUT_MS)),
538
+ ]);
539
+ if (generation !== this.#callGeneration) return; // hung up meanwhile
540
+ if ((captureContext.state as string) !== "running") {
541
+ if (auto) throw new Error((this.#notice = `${CONNECTION_LOST} Tap the mic to reconnect.`));
542
+ // A restored call: the widget asks for a tap to resume.
543
+ throw new Error("Audio blocked until a tap");
544
+ }
502
545
  }
503
- this.#captureContext = captureContext;
504
546
 
505
547
  const engine = this.config?.createPlaybackEngine
506
548
  ? await this.config.createPlaybackEngine()
@@ -558,7 +600,8 @@ export class RuntypeVoiceProvider implements VoiceProvider {
558
600
  ws.onmessage = (event) => this.#handleMessage(event, generation);
559
601
 
560
602
  ws.onerror = () => {
561
- if (generation !== this.#callGeneration) return;
603
+ // After `session_end`, the close that follows reports the server's reason.
604
+ if (generation !== this.#callGeneration || this.#sessionEnd) return;
562
605
  this.#emitError(new Error("Voice connection failed"));
563
606
  this.#emitStatus("error");
564
607
  this.#cleanup();
@@ -570,17 +613,68 @@ export class RuntypeVoiceProvider implements VoiceProvider {
570
613
  return;
571
614
  }
572
615
  if (generation !== this.#callGeneration) return;
573
- if (evt.code !== 1000) {
574
- const codeMsg = evt.code ? ` (code ${evt.code})` : "";
575
- this.#emitError(new Error(`Voice connection closed${codeMsg}`));
576
- this.#emitStatus("error");
577
- } else {
578
- this.#emitStatus("idle");
616
+ // Why the server ended it: its `session_end`, or the attach idle close.
617
+ const end = this.#sessionEnd ?? (evt.code === 4408 ? { reason: "idle_timeout" } : null);
618
+ if (!end) {
619
+ // No reason given (a deploy, a dropped connection): as before.
620
+ if (evt.code !== 1000) {
621
+ const codeMsg = evt.code ? ` (code ${evt.code})` : "";
622
+ this.#emitError(new Error(`Voice connection closed${codeMsg}`));
623
+ this.#emitStatus("error");
624
+ } else {
625
+ this.#emitStatus("idle");
626
+ }
627
+ this.#cleanup();
628
+ return;
579
629
  }
580
- this.#cleanup();
630
+ this.#endCall(end.reason, end.message);
581
631
  };
582
632
  }
583
633
 
634
+ /**
635
+ * The server ended the call with a reason (`session_end`, Amendment 6): show
636
+ * its text, and reconnect once after a provider error or a restart. Unknown
637
+ * reasons are an ordinary end (`ended_by_server`).
638
+ */
639
+ #endCall(reason: string, message?: string): void {
640
+ const retry = RECONNECT_REASONS.includes(reason) && !this.#reconnected;
641
+ this.#cleanup();
642
+ if (retry) {
643
+ // Keep the call "live" while it waits, so the mic button hangs up (and cancels).
644
+ this.#reconnected = true;
645
+ this.#callLive = true;
646
+ this.#notice = reason === "provider_error" ? `${CONNECTION_LOST} Reconnecting…` : "Reconnecting…";
647
+ // A hang-up meanwhile runs #cleanup, which clears this timer.
648
+ this.#reconnectTimer = setTimeout(
649
+ () => {
650
+ this.#callLive = false;
651
+ this.#autoReconnect = true;
652
+ void this.startListening().catch(() => {});
653
+ },
654
+ 1000 + Math.random() * 1000,
655
+ );
656
+ this.#emitStatus("idle");
657
+ return;
658
+ }
659
+ if (RECONNECT_REASONS.includes(reason)) {
660
+ // The reconnect already ran: this end is the second failure.
661
+ this.#notice = CONNECTION_LOST;
662
+ this.#emitError(new Error(CONNECTION_LOST));
663
+ return this.#emitStatus("error");
664
+ }
665
+ // `error`: the preceding `error` frame already reported it.
666
+ if (reason === "error") return this.#emitStatus("error");
667
+ this.#notice = (reason === "quota" && message) || SESSION_END_TEXT.get(reason) || null;
668
+ this.#emitStatus("idle");
669
+ }
670
+
671
+ /** Status-line text for the call (why it ended, or that it is reconnecting), once. */
672
+ takeNotice(): string | null {
673
+ const notice = this.#notice;
674
+ this.#notice = null;
675
+ return notice;
676
+ }
677
+
584
678
  /** The AI-disclosure notice for a live speech-to-speech call (`disclosureText`; `false` hides it). */
585
679
  getDisclosure(): string | null {
586
680
  const text = this.config?.disclosureText;
@@ -746,9 +840,17 @@ export class RuntypeVoiceProvider implements VoiceProvider {
746
840
  if (msg.final === true) this.#flushCallContext(generation);
747
841
  this.#foldReadback = false;
748
842
  } else {
749
- // Read-back of a delegated result: core rotates the assistant id at
750
- // completion, so it is the first new id after it. The chat renders it.
751
- if (this.#foldReadback && !this.#assistantTurns.has(turnId)) {
843
+ // Read-back of a delegated result the chat already renders. A server
844
+ // that negotiated `delegation_read_back` tags it with its delegation;
845
+ // an older one rotates the assistant id at completion, so it is the
846
+ // first new id after it (a guess a filler finishing late defeats).
847
+ if (this.#capabilities.has("delegation_read_back")) {
848
+ // Fold only from the first frame: hiding an utterance already shown would strand a partial caption.
849
+ const tag = msg.delegationId;
850
+ if (typeof tag === "string" && this.#answered.has(tag) && !this.#assistantTurns.has(turnId)) {
851
+ this.#foldedTurns.add(turnId);
852
+ }
853
+ } else if (this.#foldReadback && !this.#assistantTurns.has(turnId)) {
752
854
  this.#foldedTurns.add(turnId);
753
855
  this.#foldReadback = false;
754
856
  }
@@ -862,6 +964,14 @@ export class RuntypeVoiceProvider implements VoiceProvider {
862
964
  this.#emitError(new Error(msg.error || "Voice error"));
863
965
  this.#emitStatus("error");
864
966
  break;
967
+
968
+ case "session_end":
969
+ // The server's last frame before it closes: the close acts on it.
970
+ this.#sessionEnd = {
971
+ reason: String(msg.reason),
972
+ ...(typeof msg.message === "string" && { message: msg.message }),
973
+ };
974
+ break;
865
975
  }
866
976
  }
867
977
 
@@ -946,7 +1056,7 @@ export class RuntypeVoiceProvider implements VoiceProvider {
946
1056
  result.text,
947
1057
  );
948
1058
  if (!sent) return;
949
- if (result.status !== "failed") this.#answered.add(delegationId);
1059
+ if (result.status !== "failed" && result.inChat !== false) this.#answered.add(delegationId);
950
1060
  if (!result.followUp) return;
951
1061
  // The approval bookkeeping (expiry, supersede) runs either way.
952
1062
  const abort = new AbortController();
@@ -1053,6 +1163,8 @@ export class RuntypeVoiceProvider implements VoiceProvider {
1053
1163
  this.#followUps.clear();
1054
1164
  this.#cancelledDelegations.clear();
1055
1165
  this.#callId = undefined;
1166
+ this.#sessionEnd = null;
1167
+ clearTimeout(this.#reconnectTimer);
1056
1168
  this.#contextSent = false;
1057
1169
  this.#pendingContext = null;
1058
1170
  this.#contextSend = null;
@@ -164,16 +164,32 @@ export function createVoiceSessionBridge(host: VoiceDelegationHost): VoiceSessio
164
164
  };
165
165
 
166
166
  return {
167
- getHistory: () =>
168
- host.messages().flatMap((m) =>
169
- (m.role === "user" || m.role === "assistant") &&
170
- !m.variant &&
171
- !m.voiceProcessing &&
172
- !m.voiceCaption &&
173
- m.content
174
- ? [{ role: m.role, content: m.content }]
175
- : [],
176
- ),
167
+ // Read once at call start, before this call has any captions, so the
168
+ // captions here are earlier calls' (e.g. on the previous page of a
169
+ // multi-page site): spoken conversation the new voice session never heard.
170
+ // An assistant caption followed by a chat answer before the next user turn
171
+ // is a delegated turn's spoken filler ("Let me check."): it would only use
172
+ // up a slot of the context's recent-message window.
173
+ getHistory: () => {
174
+ const settled = host.messages().filter(
175
+ (m) =>
176
+ (m.role === "user" || m.role === "assistant") &&
177
+ !m.variant &&
178
+ !m.voiceProcessing &&
179
+ !m.streaming &&
180
+ m.content,
181
+ );
182
+ let answered = false;
183
+ const kept: Array<{ role: "user" | "assistant"; content: string }> = [];
184
+ for (let i = settled.length - 1; i >= 0; i -= 1) {
185
+ const m = settled[i];
186
+ if (m.role === "user") answered = false;
187
+ else if (!m.voiceCaption) answered = true;
188
+ else if (answered) continue;
189
+ kept.unshift({ role: m.role as "user" | "assistant", content: m.content });
190
+ }
191
+ return kept;
192
+ },
177
193
 
178
194
  dropDelegation(delegationId, expired) {
179
195
  // Remembered for a turn still running: when it parks, the same rule applies.
@@ -201,7 +217,7 @@ export function createVoiceSessionBridge(host: VoiceDelegationHost): VoiceSessio
201
217
  const tool = toolOf(pendingIds[0]);
202
218
  // The parked delegation still gets its one terminal result, silently.
203
219
  settle(live[0], "denied", "");
204
- return { status: "denied", text: `Okay, I cancelled the ${tool} request. Nothing was done.` };
220
+ return { status: "denied", text: `Okay, I cancelled the ${tool} request. Nothing was done.`, inChat: false };
205
221
  }
206
222
  // A voice-originated WebMCP approval holds the chat turn open: this turn
207
223
  // replaces it (a new send declines it), so don't wait on it. Anything