@runtypelabs/persona 4.26.1 → 4.28.0

package/src/session.voice.test.ts

@@ -36,7 +36,16 @@ const h = vi.hoisted(() => {
     errorCb: ((e: Error) => void) | null;
     bridge: VoiceSessionBridge | null;
     prewarms: number;
-  } = { transcriptCb: null, metricsCb: null, statusCb: null, errorCb: null, bridge: null, prewarms: 0 };
+    notice: string | null;
+  } = {
+    transcriptCb: null,
+    metricsCb: null,
+    statusCb: null,
+    errorCb: null,
+    bridge: null,
+    prewarms: 0,
+    notice: null,
+  };
 
   const fakeProvider = {
     type: 'runtype' as const,
@@ -64,6 +73,11 @@ const h = vi.hoisted(() => {
     setSessionBridge: (bridge: VoiceSessionBridge) => {
       state.bridge = bridge;
     },
+    takeNotice: () => {
+      const notice = state.notice;
+      state.notice = null;
+      return notice;
+    },
   };
 
   return { state, fakeProvider };
@@ -513,12 +527,31 @@ describe('AgentWidgetSession - voice client delegation bridge', () => {
   });
 
   it('hands the provider a bridge whose history is the visible settled messages', () => {
-    drive('user', 'still talk', false, 'u1'); // interim: not part of the context
-    // An earlier call's captions (filler, small talk) aren't conversation either.
+    // An earlier call's captions (e.g. on the previous page) are conversation
+    // the new voice session never heard, so they carry over.
     h.state.transcriptCb!('assistant', 'Let me check.', true, { turnId: 'f0', caption: true });
     h.state.transcriptCb!('user', 'hi there', true, { turnId: 'u0', caption: true });
+    drive('user', 'still talk', false, 'u1'); // interim: not part of the context
+    expect(h.state.bridge!.getHistory()).toEqual([
+      { role: 'assistant', content: 'Welcome! How can I help?' },
+      { role: 'assistant', content: 'Let me check.' },
+      { role: 'user', content: 'hi there' },
+    ]);
+  });
+
+  it("leaves a delegated turn's spoken filler out of the history, keeping the answer", async () => {
+    // Full duplex: the voice model's words arrive as captions.
+    h.state.transcriptCb!('user', 'whats the weather in paris', true, { turnId: 'u1', caption: true });
+    h.state.transcriptCb!('assistant', 'Let me check that.', true, { turnId: 'f1', caption: true });
+    dispatch.mockImplementation(async (_options, onEvent) => reply(onEvent, 'It is sunny in Paris.'));
+    await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'whats the weather in paris' }));
+    h.state.transcriptCb!('assistant', 'Anything else?', true, { turnId: 'a2', caption: true });
+
     expect(h.state.bridge!.getHistory()).toEqual([
       { role: 'assistant', content: 'Welcome! How can I help?' },
+      { role: 'user', content: 'whats the weather in paris' },
+      { role: 'assistant', content: 'It is sunny in Paris.' },
+      { role: 'assistant', content: 'Anything else?' },
     ]);
   });
 
@@ -748,6 +781,146 @@ describe('AgentWidgetSession - voice client delegation bridge', () => {
     expect(result.followUp).toBeTypeOf('function');
   });
 
+  describe('streaming the answer (delegation_stream)', () => {
+    /** Streams `chunks` into one assistant message, growing it like a text stream. */
+    const streamReply = (onEvent: (event: AgentWidgetEvent) => void, chunks: string[], id = 'assistant-s1') => {
+      const base = { id, role: 'assistant' as const, createdAt: new Date().toISOString() };
+      let content = '';
+      onEvent({ type: 'status', status: 'connecting' });
+      for (const chunk of chunks) {
+        content += chunk;
+        onEvent({ type: 'message', message: { ...base, content, streaming: true } });
+      }
+      onEvent({ type: 'message', message: { ...base, content, streaming: false } });
+    };
+
+    it('streams the answer in clause pieces while it renders, then reports streamedChars', async () => {
+      drive('user', 'what are your hours', true, 'u1');
+      const pieces: string[] = [];
+      const seenBeforeEnd: string[] = [];
+      dispatch.mockImplementation(async (_options, onEvent) => {
+        streamReply(onEvent, ['We open at ', 'eight, and close ', 'at six. On Sundays ', 'we are closed.']);
+        seenBeforeEnd.push(...pieces);
+        // A tool bubble between two answer messages never streams.
+        onEvent({
+          type: 'message',
+          message: { id: 'tool-1', role: 'assistant', content: '', createdAt: new Date().toISOString(), variant: 'tool' },
+        });
+        streamReply(onEvent, ['Anything else?'], 'assistant-s2');
+        onEvent({ type: 'status', status: 'idle' });
+      });
+      const result = await h.state.bridge!.runDelegatedTurn(
+        req({ delegationId: 'd1', userText: 'what are your hours', stream: (text) => pieces.push(text) }),
+      );
+      expect(result.text).toBe('We open at eight, and close at six. On Sundays we are closed.\n\nAnything else?');
+      // The first pieces went out while the stream was still running.
+      expect(seenBeforeEnd).toEqual(['We open at eight, ', 'and close at six. ']);
+      expect(pieces.join('')).toBe(result.text);
+      expect(result.streamedChars).toBe(result.text.length);
+    });
+
+    it('streams the answer of an approval-parked turn; the approval script is the unstreamed suffix', async () => {
+      drive('user', 'order two croissants', true, 'u1');
+      parkOnApproval();
+      const pieces: string[] = [];
+      const result = await h.state.bridge!.runDelegatedTurn(
+        req({ delegationId: 'd1', userText: 'order two croissants', stream: (text) => pieces.push(text) }),
+      );
+      expect(result.status).toBe('pending_approval');
+      expect(pieces).toEqual(['I can do that.']);
+      expect(result.streamedChars).toBe('I can do that.'.length);
+      expect(result.text.startsWith('I can do that.\n\n')).toBe(true);
+      expect(result.text.slice(result.streamedChars)).toContain("This action needs the user's approval");
+    });
+
+    it('stops streaming when the answer is rewritten, and counts only the shared prefix', async () => {
+      drive('user', 'hours', true, 'u1');
+      const pieces: string[] = [];
+      dispatch.mockImplementation(async (_options, onEvent) => {
+        const base = { id: 'assistant-rw', role: 'assistant' as const, createdAt: new Date().toISOString() };
+        onEvent({ type: 'message', message: { ...base, content: 'We open at eight. Sun', streaming: true } });
+        onEvent({ type: 'message', message: { ...base, content: 'We open at 8am. Closed Sundays.', streaming: false } });
+        onEvent({ type: 'status', status: 'idle' });
+      });
+      const result = await h.state.bridge!.runDelegatedTurn(
+        req({ delegationId: 'd1', userText: 'hours', stream: (text) => pieces.push(text) }),
+      );
+      expect(pieces).toEqual(['We open at eight. ']);
+      expect(result.text).toBe('We open at 8am. Closed Sundays.');
+      expect(result.streamedChars).toBe('We open at '.length);
+    });
+
+    it('streams a held sentence end after 200 ms of quiet, and stops on cancellation', async () => {
+      vi.useFakeTimers();
+      try {
+        drive('user', 'hours', true, 'u1');
+        const pieces: string[] = [];
+        let resume!: () => void;
+        const paused = new Promise<void>((resolve) => (resume = resolve));
+        const base = { id: 'assistant-idle', role: 'assistant' as const, createdAt: new Date().toISOString() };
+        dispatch.mockImplementation(async (_options, onEvent) => {
+          onEvent({ type: 'message', message: { ...base, content: 'We open at 8 a.m.', streaming: true } });
+          await paused;
+          onEvent({ type: 'message', message: { ...base, content: 'We open at 8 a.m. Closed Sundays.', streaming: false } });
+          onEvent({ type: 'status', status: 'idle' });
+        });
+        const turn = h.state.bridge!.runDelegatedTurn(
+          req({ delegationId: 'd1', userText: 'hours', stream: (text) => pieces.push(text) }),
+        );
+        await vi.advanceTimersByTimeAsync(199);
+        expect(pieces).toEqual([]);
+        await vi.advanceTimersByTimeAsync(1);
+        expect(pieces).toEqual(['We open at 8 a.m.']);
+        resume();
+        await vi.advanceTimersByTimeAsync(500);
+        const result = await turn;
+        expect(pieces).toEqual(['We open at 8 a.m.', ' Closed Sundays.']);
+        expect(pieces.join('')).toBe(result.text);
+        expect(result.streamedChars).toBe(result.text.length);
+
+        // Cancelled while a sentence end is held: the idle flush never fires.
+        const later: string[] = [];
+        let resume2!: () => void;
+        const paused2 = new Promise<void>((resolve) => (resume2 = resolve));
+        const base2 = { ...base, id: 'assistant-idle-2' };
+        dispatch.mockImplementation(async (_options, onEvent) => {
+          onEvent({ type: 'message', message: { ...base2, content: 'Sure thing.', streaming: true } });
+          await paused2;
+          onEvent({ type: 'message', message: { ...base2, content: 'Sure thing. Done.', streaming: false } });
+          onEvent({ type: 'status', status: 'idle' });
+        });
+        const turn2 = h.state.bridge!.runDelegatedTurn(
+          req({ delegationId: 'd2', userText: 'more', stream: (text) => later.push(text) }),
+        );
+        await vi.advanceTimersByTimeAsync(100);
+        h.state.bridge!.dropDelegation!('d2');
+        await vi.advanceTimersByTimeAsync(300);
+        expect(later).toEqual([]);
+        resume2();
+        await vi.advanceTimersByTimeAsync(500);
+        await turn2;
+        expect(later).toEqual([]);
+      } finally {
+        vi.useRealTimers();
+      }
+    });
+
+    it('streams nothing and omits streamedChars without a stream callback', async () => {
+      drive('user', 'hours', true, 'u1');
+      dispatch.mockImplementation(async (_options, onEvent) => reply(onEvent, 'We open at nine. Sundays closed.'));
+      const result = await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'hours' }));
+      expect(result).not.toHaveProperty('streamedChars');
+    });
+  });
+
+  it('marks only the delegated voice turn as spoken (voice: { spoken: true })', async () => {
+    dispatch.mockImplementation(async (_options, onEvent) => reply(onEvent, 'Nine to five.'));
+    await h.state.bridge!.runDelegatedTurn(req({ delegationId: 'd1', userText: 'what are your hours' }));
+    await session.sendMessage('typed question');
+    const options = dispatch.mock.calls.map(([o]) => (o as { voiceSpoken?: boolean }).voiceSpoken);
+    expect(options).toEqual([true, false]);
+  });
+
   it('follows up with the answer once the visitor approves, kept off browser TTS', async () => {
     drive('user', 'order two croissants', true, 'u1');
     parkOnApproval();
@@ -1449,3 +1622,26 @@ describe('AgentWidgetSession - prewarmVoice', () => {
     expect(h.state.prewarms).toBe(0);
   });
 });
+
+describe('AgentWidgetSession - voice session end (Amendment 6)', () => {
+  const flush = () => new Promise((resolve) => setTimeout(resolve, 0));
+  const tokenSession = () =>
+    new AgentWidgetSession(
+      {
+        clientToken: 'ct_live_demo',
+        apiUrl: 'https://api.runtype.com',
+        voiceRecognition: { enabled: true, provider: { type: 'runtype', runtype: { agentId: 'a1' } } },
+      },
+      { onMessagesChanged: () => {}, onStatusChanged: () => {}, onStreamingChanged: () => {}, onError: () => {} },
+    );
+
+  it("hands the UI the provider's status-line notice once", async () => {
+    const session = tokenSession();
+    session.setupVoice();
+    await flush();
+    h.state.notice = 'This voice call reached its time limit.';
+    expect(session.getVoiceNotice()).toBe('This voice call reached its time limit.');
+    expect(session.getVoiceNotice()).toBeNull();
+  });
+
+});

package/src/types.ts

@@ -5230,6 +5230,11 @@ export interface VoiceProvider {
   setSessionBridge?(bridge: VoiceSessionBridge): void;
   /** The AI-disclosure notice for the live call, if one should show (speech-to-speech). */
   getDisclosure?(): string | null;
+  /**
+   * Status-line text for the call, returned once: why the server ended it
+   * (`session_end`) or that it is reconnecting. `null` when there is none.
+   */
+  takeNotice?(): string | null;
 }
 
 /** Metadata on a {@link VoiceProvider.onTranscript} update. */
@@ -5260,6 +5265,13 @@ export type VoiceDelegationRequest = {
   userUtteranceIds: string[];
   /** The voice model's view of the conversation it delegated from. */
   messages: Array<{ role: string; content: unknown }>;
+  /**
+   * Set when the server streams delegated answers (`delegation_stream`): called
+   * with each speakable piece of the turn's answer, in order, while it renders.
+   * The pieces, concatenated, are a prefix of the result's `text`; the result
+   * says how much of it with `streamedChars`.
+   */
+  stream?: (text: string) => void;
 };
 
 /**
@@ -5300,6 +5312,12 @@ export type VoiceDelegationResult = {
    * @default true
    */
   inChat?: boolean;
+  /**
+   * How many leading characters of `text` already went out through
+   * `request.stream` (the rest, such as an approval script, is unstreamed).
+   * Omitted or 0: nothing was streamed.
+   */
+  streamedChars?: number;
   /**
    * Set when the turn parked on approvals (`text` then asks for the decision).
    * Resolves with the terminal result once the visitor decides (or the
@@ -6261,6 +6279,12 @@ export type ClientChatRequest = {
   turnId?: RuntypeClientChatRequest['turnId'];
   /** Whether this turn should interrupt a prior in-flight response (from Core public OpenAPI). */
   submitMode?: RuntypeClientChatRequest['submitMode'];
+  /**
+   * Spoken-reply hint, set on a delegated voice turn whose reply a voice model
+   * reads aloud: the server asks the agent for a speakable answer (the direct
+   * answer first, in plain sentences). Not persisted into history.
+   */
+  voice?: { spoken: true };
 };
 
 /**

package/src/ui.composer-defer-one.test.ts

@@ -159,6 +159,23 @@ describe("streamingSubmitBehavior: defer-one", () => {
     expect(status?.textContent).toContain("already queued");
   });
 
+  it("keeps a notice in the status line when the chat status changes under it", async () => {
+    const { mount } = makeController();
+    await startStreaming(mount);
+    type(mount, "queued message");
+    pressEnter(mount);
+    await flush();
+    type(mount, "second attempt");
+    pressEnter(mount);
+    await flush();
+
+    stream.push(unifiedFrames());
+    stream.close();
+    await flush(50);
+    const status = mount.querySelector<HTMLElement>("[data-persona-composer-notice]");
+    expect(status?.textContent).toContain("already queued");
+  });
+
   it("card Edit restores the snapshot into the draft and drops the card", async () => {
     const { mount, controller } = makeController();
     await startStreaming(mount);

package/src/ui.ts

@@ -353,11 +353,14 @@ import {
 // Default localStorage key for chat history (automatically cleared on clear chat)
 const DEFAULT_CHAT_HISTORY_STORAGE_KEY = "persona-chat-history";
 const VOICE_STATE_RESTORE_WINDOW = 30 * 1000;
+const VOICE_RESUME_NOTICE = "Tap the mic to resume your voice call";
 // Split desktop boundary; must match widget.css artifact media queries (min-width:641px).
 const ARTIFACT_SPLIT_DESKTOP_MIN = 641;
 
 /** Marks the status region while a composer lock reason owns it. */
 const COMPOSER_REASON_ATTR = "data-persona-composer-reason";
+/** Marks the status region while a transient composer notice owns it. */
+const COMPOSER_NOTICE_ATTR = "data-persona-composer-notice";
 
 const IMAGE_FILE_EXTENSION_BY_MIME_TYPE: Record<string, string> = {
   "image/png": "png",
@@ -1384,8 +1387,9 @@ export const createAgentExperience = (
       placeStatusIndicator(el, composerForm, statusCfg);
       applyStatusIndicatorState(el, statusCfg, status);
     }
-    // A composer lock reason owns the status region until the lock clears.
-    if (el.hasAttribute(COMPOSER_REASON_ATTR)) return;
+    // A composer lock reason owns the status region until the lock clears, and
+    // a notice until its timer ends (which re-applies the status).
+    if (el.hasAttribute(COMPOSER_REASON_ATTR) || el.hasAttribute(COMPOSER_NOTICE_ATTR)) return;
     if (status === "idle" && statusCfg.idleLink) {
       el.textContent = "";
       const link = document.createElement("a");
@@ -5083,6 +5087,15 @@ export const createAgentExperience = (
   // would otherwise not appear until the first keystroke.
   syncComposerCompact();
   const voiceAutoResumeMode = config.voiceRecognition?.autoResume ?? false;
+  // A session-voice call is live while listening OR while its continuous
+  // (barge-in) mic is hot: a full-duplex call cycles listening → processing →
+  // speaking, and `isVoiceActive()` alone (status === listening) would record a
+  // live call as hung up and never restore it on the next page.
+  const isSessionVoiceLive = () => session.isVoiceActive() || session.isBargeInActive();
+  // Set on `pagehide`: the unload tears the call down without the visitor
+  // hanging up, so its trailing status changes must not overwrite the
+  // persisted "call was live" record the next page restores from.
+  let pageHiding = false;
   const emitVoiceState = (source: AgentWidgetVoiceStateEvent["source"]) => {
     eventBus.emit("voice:state", {
       active: voiceState.active,
@@ -5112,17 +5125,37 @@ export const createAgentExperience = (
           voiceState.manuallyDeactivated = false;
           if (usesSessionVoice(config.voiceRecognition?.provider)) {
             session.toggleVoice().then(() => {
-              voiceState.active = session.isVoiceActive();
+              voiceState.active = isSessionVoiceLive();
               emitVoiceState("restore");
-              if (session.isVoiceActive()) applyRuntypeMicRecordingStyles();
+              if (voiceState.active) {
+                applyRuntypeMicRecordingStyles();
+              } else {
+                // Typically the autoplay policy: this page has no user gesture
+                // yet, so the audio graph can't start. A mic tap resumes it.
+                persistVoiceMetadata();
+                showComposerNotice(VOICE_RESUME_NOTICE, 10_000);
+              }
             });
           } else {
             startVoiceRecognition("restore");
           }
         }
-      }, 1000);
+        // Session voice redials at once (a navigation mid-call should feel
+        // seamless); toggleVoice() already waits on the provider setup.
+      }, usesSessionVoice(config.voiceRecognition?.provider) ? 0 : 1000);
     }
   };
+  // Re-stamp a live call as the page unloads: the restore window counts from
+  // the navigation, not from when the (possibly long) call was started. A
+  // back/forward-cache hide (`persisted`) isn't an unload: the page lives on.
+  const handleVoicePageHide = (event: PageTransitionEvent) => {
+    pageHiding = !event.persisted;
+    if (voiceState.active) persistVoiceMetadata();
+  };
+  if (typeof window !== "undefined") {
+    window.addEventListener("pagehide", handleVoicePageHide);
+    destroyCallbacks.push(() => window.removeEventListener("pagehide", handleVoicePageHide));
+  }
 
   const getMessagesForPersistence = () =>
     session
@@ -7916,6 +7949,7 @@ export const createAgentExperience = (
       statusText.setAttribute("role", "status");
       statusText.setAttribute("aria-live", "polite");
       statusText.setAttribute(COMPOSER_REASON_ATTR, "");
+      statusText.removeAttribute(COMPOSER_NOTICE_ATTR); // the reason outranks a notice
       statusText.textContent = reason;
       applyStatusIndicatorState(statusText, config.statusIndicator ?? {}, "idle");
       return;
@@ -8803,6 +8837,9 @@ export const createAgentExperience = (
       // provider; the mic-button styling below is runtype-specific.
       eventBus.emit("voice:status", { status, timestamp: Date.now() });
       if (status !== 'disconnected' && !usesSessionVoice(config.voiceRecognition?.provider)) return;
+      // Why the server ended the call, or that it is reconnecting.
+      const voiceNotice = session.getVoiceNotice();
+      if (voiceNotice) showComposerNotice(voiceNotice, 10_000);
 
       switch (status) {
         case 'listening':
@@ -8852,7 +8889,7 @@ export const createAgentExperience = (
             }
             removeRuntypeMicStateStyles();
             emitVoiceState("system");
-            persistVoiceMetadata();
+            if (!pageHiding) persistVoiceMetadata();
           }
           break;
       }
@@ -9507,7 +9544,6 @@ export const createAgentExperience = (
     resetHistoryNavigation();
   };
 
-  const COMPOSER_NOTICE_ATTR = "data-persona-composer-notice";
   let composerNoticeTimer: ReturnType<typeof setTimeout> | null = null;
 
   /** Transient message in the status region; a lock reason always outranks it. */
@@ -10497,11 +10533,11 @@ export const createAgentExperience = (
       }
 
       session.toggleVoice().then(() => {
-        voiceState.active = session.isVoiceActive();
-        voiceState.manuallyDeactivated = !session.isVoiceActive();
+        voiceState.active = isSessionVoiceLive();
+        voiceState.manuallyDeactivated = !voiceState.active;
         persistVoiceMetadata();
         emitVoiceState("user");
-        if (session.isVoiceActive()) {
+        if (voiceState.active) {
           applyRuntypeMicRecordingStyles();
         } else {
           removeRuntypeMicStateStyles();
@@ -12956,9 +12992,10 @@ export const createAgentExperience = (
         voiceState.manuallyDeactivated = false;
         persistVoiceMetadata();
         session.toggleVoice().then(() => {
-          voiceState.active = session.isVoiceActive();
+          voiceState.active = isSessionVoiceLive();
+          persistVoiceMetadata();
           emitVoiceState("user");
-          if (session.isVoiceActive()) applyRuntypeMicRecordingStyles();
+          if (voiceState.active) applyRuntypeMicRecordingStyles();
         });
         return true;
       }

package/src/voice/__fixtures__/voice-wire/server/session_end-provider-error.json

@@ -0,0 +1,6 @@
+{
+  "type": "session_end",
+  "reason": "provider_error",
+  "message": "The voice provider disconnected.",
+  "retryable": true
+}

package/src/voice/__fixtures__/voice-wire/server/session_end-unknown-reason.json

@@ -0,0 +1,4 @@
+{
+  "type": "session_end",
+  "reason": "maintenance_window"
+}

package/src/voice/__fixtures__/voice-wire/server/session_end.json

@@ -0,0 +1,5 @@
+{
+  "type": "session_end",
+  "reason": "max_duration",
+  "retryable": true
+}