@runtypelabs/persona 4.28.2 → 4.29.1

package/src/utils/storage.ts

@@ -30,8 +30,19 @@ const sanitizeArtifacts = (artifacts: PersonaArtifactRecord[]) =>
     status: "complete" as const
   }));
 
+const DEFAULT_PERSIST_KEY_PREFIX = "persona-";
+
+/**
+ * The widget's default transcript key, scoped by `persistState.keyPrefix` so
+ * two widgets on one origin with distinct prefixes keep separate transcripts.
+ * The default prefix yields the legacy `"persona-state"` key, so unprefixed
+ * widgets keep their stored conversation; prefixed widgets never adopt it.
+ */
+export const transcriptStorageKey = (keyPrefix?: string): string =>
+  `${keyPrefix ?? DEFAULT_PERSIST_KEY_PREFIX}state`;
+
 export const createLocalStorageAdapter = (
-  key = "persona-state"
+  key = transcriptStorageKey()
 ): AgentWidgetStorageAdapter => {
   const getStorage = () => {
     if (typeof window === "undefined" || !window.localStorage) {

package/src/utils/voice-prior-tool-results.ts

@@ -0,0 +1,37 @@
+import type { AgentWidgetMessage } from "../types";
+
+const isRecord = (value: unknown): value is Record<string, unknown> =>
+  typeof value === "object" && value !== null && !Array.isArray(value);
+
+/**
+ * Tool rows for the page tools a voice server already ran before handing the
+ * turn to the chat (`delegation_started.input.priorToolResults`). Each row is
+ * a completed call carrying `clientToolAnswer`, so proxy/agent payloads replay
+ * it to the model as an assistant `toolCalls` + `tool` `toolResults` pair under
+ * the same `toolCallId`. Nothing marks it awaiting, so it never runs again.
+ * Malformed entries and repeats of a `toolCallId` are skipped.
+ */
+export const voicePriorToolRows = (value: unknown, createdAt: string): AgentWidgetMessage[] => {
+  const seen = new Set<string>();
+  return (Array.isArray(value) ? value : []).flatMap((entry): AgentWidgetMessage[] => {
+    if (!isRecord(entry)) return [];
+    const { toolCallId, name } = entry;
+    if (typeof toolCallId !== "string" || !toolCallId || typeof name !== "string" || !name) return [];
+    if (seen.has(toolCallId)) return [];
+    seen.add(toolCallId);
+    const args = isRecord(entry.arguments) ? entry.arguments : {};
+    const result = entry.result ?? null;
+    return [
+      {
+        id: `voice-tool-${toolCallId}`,
+        role: "assistant",
+        variant: "tool",
+        content: "",
+        createdAt,
+        streaming: false,
+        toolCall: { id: toolCallId, name, status: "complete", args, result },
+        agentMetadata: { clientToolAnswer: { toolCallId, toolName: name, args, result } },
+      },
+    ];
+  });
+};

package/src/voice/runtype-voice-provider.ts

@@ -83,8 +83,29 @@
 // completed), and that phase's completion doesn't fold another utterance.
 // Delegated sends carry `voice: { spoken: true }` on the chat request, so the
 // agent answers in speakable sentences.
+//
+// Fast tier (`client_tools`, declared only with client delegation and WebMCP
+// enabled; the session ships in the lazy voice-client-tools chunk): after
+// `session_config` the provider sends `client_tools{fingerprint, tools}`, the
+// same WebMCP snapshot and fingerprint the chat path sends as `clientTools[]`
+// but with each page tool's FULL name (`webmcp:<name>`; sdk tools stay bare),
+// and again whenever the fingerprint changes. The server may then answer a
+// delegation itself, with no `delegation_started`: it sends
+// `client_tool_call{delegationId, toolCallId, name, arguments, fingerprint,
+// timeoutMs}`, which runs through the WebMCP execute path (the confirm gate is
+// skipped only for a tool whose registry entry says `readOnlyHint: true`) and
+// is answered by one `client_tool_result{status}`: `completed`, `denied` (gate
+// declined), `stale` (unknown tool, or a fingerprint other than the last one
+// sent), `failed`, or `timeout` (past `timeoutMs`, which bounds the gate too).
+// Then `delegation_completed{route:'direct', text}` for a delegation that never
+// started here writes the heard user turn and `text` into the chat. When the
+// server falls through, the delegation arrives as today's `delegation_started`;
+// its `input.priorToolResults` (page tools the server already ran; sent only
+// with `prior_tool_results`, declared beside `client_tools`) ride along to the
+// chat turn as completed calls, never run again here.
 
 import type {
+  VoiceClientToolsSession,
   VoiceProvider,
   VoiceResult,
   VoiceStatus,
@@ -327,6 +348,13 @@ export class RuntypeVoiceProvider implements VoiceProvider {
   #foldReadback = false;
   #assistantTurns = new Set<string>();
   #foldedTurns = new Set<string>();
+  // Fast tier (see header): the call's session, from the lazy voice-client-tools chunk.
+  #fastTier: Promise<VoiceClientToolsSession | null> | null = null;
+  // Delegations this call saw start, and the last final user utterance (a direct turn's request).
+  #started = new Set<string>();
+  #heard: { id: string; text: string } | null = null;
+  // Direct turns in flight: the utterance each answers, captured at its first client_tool_call.
+  #direct = new Map<string, { id: string; text: string } | null>();
 
   #resultCallbacks: ((result: VoiceResult) => void)[] = [];
   #errorCallbacks: ((error: Error) => void)[] = [];
@@ -494,15 +522,17 @@ export class RuntypeVoiceProvider implements VoiceProvider {
    * the URL, and these params are not secrets.
    */
   #voiceSocketUrl(host: string, agentId: string, attach?: URLSearchParams): string {
-    const capabilities = ["partial_transcript", "context"];
-    if (this.bridge && this.config?.clientDelegation !== false)
-      capabilities.push("client_delegation", "delegation_update", "delegation_read_back", "delegation_stream");
-    if (attach) capabilities.unshift("attach");
+    let capabilities = "partial_transcript,context";
+    if (this.bridge && this.config?.clientDelegation !== false) {
+      capabilities += ",client_delegation,delegation_update,delegation_read_back,delegation_stream";
+      if (this.bridge.clientTools?.()) capabilities += ",client_tools,prior_tool_results";
+    }
+    if (attach) capabilities = `attach,${capabilities}`;
     const params = new URLSearchParams({
       voiceProtocol: VOICE_PROTOCOL,
       clientVersion: `persona/${VERSION}`,
       voiceCapabilities: FULL_DUPLEX_CAPABILITY,
-      clientCapabilities: capabilities.join(","),
+      clientCapabilities: capabilities,
     });
     attach?.forEach((value, key) => params.set(key, value));
     return `${toWsBase(host)}/ws/agents/${encodeURIComponent(agentId)}/voice?${params}`;
@@ -849,8 +879,35 @@ export class RuntypeVoiceProvider implements VoiceProvider {
           this.#contextSent = true;
           this.#pendingContext = this.#buildContextText();
         }
+        if (this.#clientDelegation && this.#capabilities.has("client_tools") && !this.#fastTier) {
+          // INVARIANT: #clientDelegation implies a bridge.
+          this.#fastTier =
+            this.bridge!
+              .clientTools?.()
+              ?.open({
+                send: (frame) => {
+                  const ws = this.#ws;
+                  if (generation !== this.#callGeneration || ws?.readyState !== WebSocket.OPEN) return false;
+                  ws.send(JSON.stringify(frame));
+                  return true;
+                },
+                cancelled: (id) => this.#cancelledDelegations.has(id),
+              })
+              .catch(() => null) ?? null;
+        }
         break;
 
+      case "client_tool_call": {
+        // A direct turn: keep the request it answers (speech may follow) and fold its read-back.
+        const id = String(msg.delegationId);
+        if (this.#fastTier && !this.#direct.has(id)) {
+          this.#direct.set(id, this.#heard);
+          this.#answered.add(id);
+        }
+        void this.#fastTier?.then((session) => session?.call(msg));
+        break;
+      }
+
       case "transcript_update": {
         const role = msg.role === "assistant" ? "assistant" : "user";
         // A reply the client cancelled keeps streaming until the server clears it.
@@ -860,7 +917,10 @@ export class RuntypeVoiceProvider implements VoiceProvider {
         if (role === "user") {
           // Release on a final user transcript, not a partial: a mid-utterance
           // context append makes the voice model answer early (or not delegate).
-          if (msg.final === true) this.#flushCallContext(generation);
+          if (msg.final === true) {
+            this.#flushCallContext(generation);
+            this.#heard = { id: turnId, text: String(msg.text ?? "") };
+          }
           this.#foldReadback = false;
         } else {
           // Read-back of a delegated result the chat already renders. A server
@@ -902,6 +962,10 @@ export class RuntypeVoiceProvider implements VoiceProvider {
         const delegationId = String(msg.delegationId ?? msg.turnId);
         if (msg.type === "delegation_started") {
           this.#flushCallContext(generation);
+          this.#started.add(delegationId);
+          // A direct attempt that fell through: its answer is the chat turn's now, and its tool must not act late.
+          this.#answered.delete(delegationId);
+          if (this.#direct.delete(delegationId)) void this.#fastTier?.then((session) => session?.abort(delegationId));
           // `input`: run this turn through the chat pipeline (client delegation).
           const input = msg.input;
           if (this.#clientDelegation && input) {
@@ -914,11 +978,26 @@ export class RuntypeVoiceProvider implements VoiceProvider {
                   ? input.userUtteranceIds.filter((id: unknown): id is string => typeof id === "string")
                   : [],
                 messages: Array.isArray(input.messages) ? input.messages : [],
+                priorToolResults: input.priorToolResults,
               },
               generation,
             );
           }
-        } else if (msg.speak !== false && this.#answered.has(delegationId)) {
+        } else if (msg.route === "direct" && this.#fastTier && !this.#started.has(delegationId)) {
+          // The server answered it itself: write the request it answered and its
+          // answer into the chat, as the chat pipeline would have.
+          const heard = this.#direct.get(delegationId) ?? this.#heard;
+          this.#direct.delete(delegationId);
+          this.#started.add(delegationId);
+          this.#answered.add(delegationId);
+          this.bridge?.recordDirectTurn?.({
+            delegationId,
+            userText: heard?.text ?? "",
+            userUtteranceIds: heard ? [heard.id] : [],
+            assistantText: typeof msg.text === "string" ? msg.text : "",
+          });
+        }
+        if (msg.type === "delegation_completed" && msg.speak !== false && this.#answered.has(delegationId)) {
           // Every spoken phase (the approval ask, the late result) of an answer
           // the chat shows is read back. A refusal of one that never started
           // here has no chat answer: it renders. `final` may never come. A
@@ -1086,6 +1165,9 @@ export class RuntypeVoiceProvider implements VoiceProvider {
     // unrelated utterance: disarm it.
     if (this.#foldArmedBy === delegationId) this.#foldReadback = false;
     this.#progressed.delete(delegationId);
+    // A direct answer that never completed has no chat message to fold onto.
+    if (!this.#started.has(delegationId)) this.#answered.delete(delegationId);
+    void this.#fastTier?.then((session) => session?.abort(delegationId));
     this.bridge?.dropDelegation?.(delegationId, expired);
     this.#followUps.get(delegationId)?.abort();
   }
@@ -1255,6 +1337,11 @@ export class RuntypeVoiceProvider implements VoiceProvider {
     this.#foldReadback = false;
     this.#assistantTurns.clear();
     this.#foldedTurns.clear();
+    void this.#fastTier?.then((session) => session?.close());
+    this.#fastTier = null;
+    this.#started.clear();
+    this.#direct.clear();
+    this.#heard = null;
     this.#finishedArmed = false;
     this.#cancelledUntil = 0;
     this.#playbackEndsAt = 0;

package/src/voice/session-voice-wiring.ts

@@ -16,7 +16,7 @@ import type { VoiceTurnTracker } from "./voice-turn-tracker";
 /** The session members the wiring touches (some are TS-private on the class). */
 export type VoiceWiringSession = Pick<
   AgentWidgetSession,
-  "injectMessage" | "sendMessage" | "resolveWebMcpApproval" | "resolveApproval"
+  "injectMessage" | "sendMessage" | "resolveWebMcpApproval" | "resolveApproval" | "getClient"
 > & {
   messages: AgentWidgetMessage[];
   callbacks: AgentWidgetSession["callbacks"];
@@ -81,8 +81,7 @@ export const wireSessionVoice = (
           busy: () => p.busy(),
           parked: () => s.webMcpApprovalResolvers.size > 0,
           claim: (text, userUtteranceIds) => p.keyed?.claimUserTurn(text, userUtteranceIds) ?? null,
-          send: (text, userMessageId) =>
-            s.sendMessage(text, { viaVoice: true, voiceTurn: { userMessageId } }),
+          send: (text, voiceTurn) => s.sendMessage(text, { viaVoice: true, voiceTurn }),
           track: (capture) => {
             p.setDelegation(capture);
           },
@@ -92,6 +91,31 @@ export const wireSessionVoice = (
             // now. A stream still running reads it when it ends, once.
             if (!s.streaming) s.speakLatestAssistantMessage(ids);
           },
+          clientTools: () => s.getClient().getVoiceClientTools(),
+          recordDirect: ({ delegationId, userText, userUtteranceIds, assistantText }) => {
+            // The heard transcript bubble becomes the conversation turn, as a
+            // delegated send would make it; without one, the request text is.
+            const bubbleId = p.keyed?.claimUserTurn(userText, userUtteranceIds);
+            let claimed = false;
+            // upsertMessage merges, so it cannot drop the caption flag: replace the bubble.
+            s.messages = s.messages.map((m) => {
+              if (m.id !== bubbleId || m.role !== 'user') return m;
+              claimed = true;
+              const { voiceCaption: _caption, ...turn } = m;
+              return { ...turn, viaVoice: true };
+            });
+            if (claimed) {
+              s.callbacks.onMessagesChanged([...s.messages]);
+            } else if (userText.trim()) {
+              s.injectMessage({ role: 'user', content: userText.trim() });
+            }
+            const answer = assistantText.trim();
+            if (!answer) return;
+            // The voice model speaks this answer: browser TTS must not read it again.
+            const id = `voice-direct-${delegationId}`;
+            s.ttsSpokenMessageIds.add(id);
+            s.injectMessage({ id, role: 'assistant', content: answer });
+          },
           decide: (id) => {
             if (s.webMcpApprovalResolvers.has(id)) return s.resolveWebMcpApproval(id, 'denied');
             const approval = s.messages.find((m) => m.id === id)?.approval;

package/src/voice/voice-client-tools-session.ts

@@ -0,0 +1,116 @@
+import type {
+  ClientToolCallFrame,
+  ClientToolDefinition,
+  ClientToolsFrame,
+  VoiceClientToolOutcome,
+  VoiceClientTools,
+  VoiceClientToolsChannel,
+  VoiceClientToolsSession,
+} from "../types";
+
+const WEBMCP_PREFIX = "webmcp:";
+
+/** The name the server calls a client tool back with: `webmcp:` prefixed for page tools. */
+export function voiceClientToolName(tool: ClientToolDefinition): string {
+  return tool.origin === "webmcp" && !tool.name.startsWith(WEBMCP_PREFIX) ? WEBMCP_PREFIX + tool.name : tool.name;
+}
+
+/**
+ * The fast-tier half of a voice call (see runtype-voice-provider.ts): keeps the
+ * server's `client_tools` snapshot current and answers each `client_tool_call`
+ * with exactly one `client_tool_result`. Ships in the lazy
+ * `voice-client-tools` chunk, loaded only when the server negotiates it.
+ */
+export function createVoiceClientToolsSession(
+  surface: VoiceClientTools,
+  channel: VoiceClientToolsChannel,
+): VoiceClientToolsSession {
+  let closed = false;
+  let queue = Promise.resolve();
+  let sentFingerprint: string | null = null;
+  let sent = new Map<string, ClientToolDefinition>();
+  const calls = new Map<string, { delegationId: string; abort: AbortController }>();
+
+  // Serialized, so snapshots go out in order; only a changed fingerprint is resent.
+  const sync = () => {
+    queue = queue.then(async () => {
+      const snapshot = closed ? null : await surface.snapshot().catch(() => null);
+      if (!snapshot || closed || snapshot.fingerprint === sentFingerprint) return;
+      const tools = snapshot.tools.map((tool) => ({ ...tool, name: voiceClientToolName(tool) }));
+      const frame: ClientToolsFrame = { type: "client_tools", fingerprint: snapshot.fingerprint, tools };
+      if (!channel.send(frame)) return;
+      sentFingerprint = snapshot.fingerprint;
+      sent = new Map(tools.map((tool) => [tool.name, tool]));
+    });
+  };
+
+  const abort = (delegationId?: string) => {
+    for (const [id, call] of calls) {
+      if (delegationId !== undefined && call.delegationId !== delegationId) continue;
+      calls.delete(id);
+      call.abort.abort();
+    }
+  };
+
+  const stopWatch = surface.onChange(sync);
+  sync();
+
+  return {
+    abort,
+    close() {
+      closed = true;
+      stopWatch();
+      abort();
+    },
+    call(msg: Partial<ClientToolCallFrame>) {
+      const { delegationId, toolCallId } = msg;
+      if (closed || typeof delegationId !== "string" || !delegationId || typeof toolCallId !== "string" || !toolCallId)
+        return;
+      const controller = new AbortController();
+      calls.get(toolCallId)?.abort.abort();
+      calls.set(toolCallId, { delegationId, abort: controller });
+      // Exactly one answer, unless the call was aborted (cancelled, handed back, hang-up).
+      const reply = ({ status, result, error }: VoiceClientToolOutcome) => {
+        // A replacement call with the same id owns the slot now: leave it alone.
+        if (calls.get(toolCallId)?.abort !== controller) return;
+        calls.delete(toolCallId);
+        if (closed || channel.cancelled(delegationId)) return;
+        channel.send({
+          type: "client_tool_result",
+          delegationId,
+          toolCallId,
+          status,
+          ...(result !== undefined && { result }),
+          ...(error && { error: { message: error.message } }),
+        });
+      };
+
+      const name = typeof msg.name === "string" ? msg.name : "";
+      const tool = sent.get(name) ?? sent.get(WEBMCP_PREFIX + name);
+      if (!tool || msg.fingerprint !== sentFingerprint) {
+        return reply({ status: "stale", error: { message: tool ? "The page's tools changed." : `Unknown tool: ${name}` } });
+      }
+      const args = msg.arguments && typeof msg.arguments === "object" && !Array.isArray(msg.arguments) ? msg.arguments : {};
+      const timeoutMs = typeof msg.timeoutMs === "number" && msg.timeoutMs > 0 ? msg.timeoutMs : 0;
+      let timer: ReturnType<typeof setTimeout> | undefined;
+      const run = surface
+        .execute(tool.name, args, { signal: controller.signal, autoConfirmReadOnly: true })
+        .catch((error: unknown): VoiceClientToolOutcome => ({
+          status: "failed",
+          error: { message: error instanceof Error ? error.message : String(error) },
+        }));
+      // The deadline also stops a gate still waiting: a late approval must not run the tool.
+      const deadline = new Promise<VoiceClientToolOutcome>((resolve) => {
+        if (timeoutMs)
+          timer = setTimeout(() => {
+            controller.abort();
+            resolve({ status: "timeout", error: { message: `Page tool exceeded ${timeoutMs}ms.` } });
+          }, timeoutMs);
+      });
+      void Promise.race([run, deadline]).then((outcome) => {
+        clearTimeout(timer);
+        reply(outcome);
+      });
+    },
+  };
+}

package/src/voice/voice-delegation.ts

@@ -4,6 +4,8 @@ import type {
   VoiceDelegationFollowUp,
   VoiceDelegationRequest,
   VoiceDelegationResult,
+  VoiceClientTools,
+  VoiceDirectTurn,
   VoiceSessionBridge,
 } from "../types";
 
@@ -43,14 +45,21 @@ export interface VoiceDelegationHost {
   parked(): boolean;
   /** The transcript bubble the request came from (see KeyedVoiceTranscript.claimUserTurn). */
   claim(userText: string, userUtteranceIds: string[]): string | null;
-  /** sendMessage as a voice turn, submitting `userMessageId`'s bubble when given. */
-  send(userText: string, userMessageId: string | undefined): Promise<void>;
+  /**
+   * sendMessage as a voice turn, submitting `userMessageId`'s bubble when
+   * given and recording `priorToolResults` as already-run tool calls.
+   */
+  send(userText: string, voiceTurn: { userMessageId?: string | null; priorToolResults?: unknown }): Promise<void>;
   /** Route the chat stream's assistant messages and failures into `capture` (or stop). */
   track(capture: VoiceDelegationCapture | null): void;
   /** Deny a pending approval (server gate or WebMCP) by its message id. */
   decide(approvalMessageId: string): void;
   /** Let browser TTS read these messages after all (their spoken result was dropped). */
   unspoken(messageIds: string[]): void;
+  /** WebMCP access for the voice fast tier; `null` when WebMCP is off. */
+  clientTools?(): VoiceClientTools | null;
+  /** Write a server-answered (`route: 'direct'`) turn into the chat. */
+  recordDirect?(turn: VoiceDirectTurn): void;
 }
 
 const SETTLE_POLL_MS = 50;
@@ -185,6 +194,9 @@ export function createVoiceSessionBridge(host: VoiceDelegationHost): VoiceSessio
   };
 
   return {
+    clientTools: host.clientTools,
+    recordDirectTurn: host.recordDirect,
+
     // Read once at call start, before this call has any captions, so the
     // captions here are earlier calls' (e.g. on the previous page of a
     // multi-page site): spoken conversation the new voice session never heard.
@@ -231,6 +243,7 @@ export function createVoiceSessionBridge(host: VoiceDelegationHost): VoiceSessio
       userText,
       userUtteranceIds,
       stream,
+      priorToolResults,
     }: VoiceDelegationRequest): Promise<VoiceDelegationResult> {
       const live = parks.filter((park) => pendingOf(park).length);
       const pendingIds = live.flatMap(pendingOf);
@@ -297,7 +310,10 @@ export function createVoiceSessionBridge(host: VoiceDelegationHost): VoiceSessio
       active = capture;
       retrack();
       try {
-        await host.send(userText, host.claim(userText, userUtteranceIds) ?? undefined);
+        await host.send(userText, {
+          userMessageId: host.claim(userText, userUtteranceIds),
+          priorToolResults,
+        });
         // Local tools and approvals may continue the turn past the first stream.
         await settled(capture);
       } catch {