@runtypelabs/persona 4.26.1 → 4.28.0

package/src/voice/voice-clause-buffer.ts

@@ -0,0 +1,187 @@
+/**
+ * Groups a growing answer into speakable pieces for `delegation_delta`
+ * (streaming client delegation): the first piece goes out early (at a clause
+ * mark, or once ~30 characters / 6 words are in), later pieces at sentence ends
+ * or line breaks, and no piece is longer than {@link CLAUSE_MAX_CHARS}.
+ *
+ * Pieces are exact slices of the input: concatenated, they equal everything
+ * pushed so far (whitespace included), so a receiver can count them as a
+ * prefix of the final text.
+ */
+
+/** The longest piece. */
+export const CLAUSE_MAX_CHARS = 220;
+const FIRST_MIN_CHARS = 30;
+const FIRST_MIN_WORDS = 6;
+
+/** Words ending in "." that rarely end a sentence (lowercased, inner dots kept). */
+const ABBREVIATIONS = new Set([
+  "mr",
+  "mrs",
+  "ms",
+  "dr",
+  "prof",
+  "sr",
+  "jr",
+  "st",
+  "mt",
+  "vs",
+  "etc",
+  "approx",
+  "inc",
+  "ltd",
+  "co",
+  "corp",
+  "no",
+  "nos",
+  "fig",
+  "ft",
+  "dept",
+  "est",
+  "min",
+  "max",
+  "e.g",
+  "i.e",
+  "a.m",
+  "p.m",
+  "u.s",
+  "u.k",
+]);
+
+// Punctuation (then closers like quotes, brackets or Markdown emphasis) and the
+// whitespace after it, or a line break.
+const BOUNDARY = /([.!?]+|[,;:])(["'”’)\]*_]*)(\s+)|\n\s*/g;
+
+/** Whether the "." ending `before` belongs to an abbreviation, an initial, or a list number. */
+function dotIsNotSentenceEnd(before: string): boolean {
+  const token = /(\S+)$/.exec(before)?.[1];
+  if (!token) return false;
+  const word = token.replace(/^[("'“‘[*_]+/, "").toLowerCase();
+  if (ABBREVIATIONS.has(word)) return true;
+  // An initial ("J.") or a dotted abbreviation ("U.S", "e.g").
+  if (/^[a-z](?:\.[a-z])*$/.test(word)) return true;
+  // "1. " opening a line is a list marker, not a sentence.
+  if (/^\d+$/.test(token)) {
+    const start = before.length - token.length;
+    return start === 0 || /\n\s*$/.test(before.slice(0, start));
+  }
+  return false;
+}
+
+/**
+ * Ends (after the whitespace) of each boundary in `text` with content before
+ * it. A boundary whose whitespace reaches the end of `text` may still grow
+ * ("\n" then "\n- item"), so it counts only once more text follows.
+ */
+function* boundaryEnds(text: string, clauses: boolean): Generator<number> {
+  const re = new RegExp(BOUNDARY.source, "g");
+  for (let m = re.exec(text); m; m = re.exec(text)) {
+    const mark = m[1];
+    if (!text.slice(0, m.index + (mark?.length ?? 0)).trim()) continue;
+    if (mark !== undefined) {
+      if (/^[,;:]$/.test(mark) && !clauses) continue;
+      if (mark === "." && dotIsNotSentenceEnd(text.slice(0, m.index))) continue;
+    }
+    const end = m.index + m[0].length;
+    if (end < text.length) yield end;
+  }
+}
+
+/** End of the last whitespace run within `limit` that has content before it, or 0. */
+function lastWordEnd(text: string, limit: number): number {
+  const window = text.slice(0, limit);
+  let best = 0;
+  for (const m of window.matchAll(/\s+/g)) {
+    if (window.slice(0, m.index).trim()) best = m.index! + m[0].length;
+  }
+  return best;
+}
+
+/** Where to cut a piece that reached {@link CLAUSE_MAX_CHARS} with no sentence end. */
+function overflowCut(text: string): number {
+  let clause = 0;
+  for (const end of boundaryEnds(text.slice(0, CLAUSE_MAX_CHARS), true)) clause = end;
+  if (clause) return clause;
+  const word = lastWordEnd(text, CLAUSE_MAX_CHARS);
+  if (word) return word;
+  // One unbroken run: cut hard, never inside a surrogate pair.
+  const code = text.charCodeAt(CLAUSE_MAX_CHARS - 1);
+  return code >= 0xd800 && code <= 0xdbff ? CLAUSE_MAX_CHARS - 1 : CLAUSE_MAX_CHARS;
+}
+
+export class ClauseBuffer {
+  #pending = "";
+  #first = true;
+
+  /** Add text; returns the pieces now complete (possibly none). */
+  push(text: string): string[] {
+    this.#pending += text;
+    return this.#drain();
+  }
+
+  /** Everything still held, as the last pieces (whitespace-only remainder is dropped). */
+  flush(): string[] {
+    const pieces = this.#drain();
+    const rest = this.#pending;
+    this.#pending = "";
+    if (rest.trim()) {
+      for (let at = 0; at < rest.length; ) {
+        const tail = rest.slice(at);
+        const cut = tail.length > CLAUSE_MAX_CHARS ? overflowCut(tail) : tail.length;
+        pieces.push(tail.slice(0, cut));
+        at += cut;
+      }
+      this.#first = false;
+    }
+    return pieces;
+  }
+
+  /**
+   * The stream went quiet: if the held text ends in a sentence terminator
+   * (`. ! ? …`, closers allowed) or a line break, release it as if the
+   * boundary were confirmed. No abbreviation guards: a pause is enough.
+   */
+  idle(): string[] {
+    const text = this.#pending;
+    if (!text.trim() || !/(?:[.!?…]["'”’)\]*_]*|\n)\s*$/.test(text)) return [];
+    this.#pending = "";
+    this.#first = false;
+    const pieces: string[] = [];
+    for (let at = 0; at < text.length; ) {
+      const tail = text.slice(at);
+      const cut = tail.length > CLAUSE_MAX_CHARS ? overflowCut(tail) : tail.length;
+      pieces.push(tail.slice(0, cut));
+      at += cut;
+    }
+    return pieces;
+  }
+
+  #drain(): string[] {
+    const pieces: string[] = [];
+    for (let cut = this.#cut(); cut > 0; cut = this.#cut()) {
+      pieces.push(this.#pending.slice(0, cut));
+      this.#pending = this.#pending.slice(cut);
+      this.#first = false;
+    }
+    return pieces;
+  }
+
+  /** Length of the next complete piece, or 0 to keep waiting. */
+  #cut(): number {
+    const text = this.#pending;
+    if (!text.trim()) return 0;
+    const end = boundaryEnds(text, this.#first).next().value ?? 0;
+    if (end && end <= CLAUSE_MAX_CHARS) return end;
+    // Ending on sentence punctuation, the text is most likely a whole sentence
+    // waiting for the space that confirms it: don't split it early.
+    const atSentenceEnd = /[.!?]["'”’)\]*_]*$/.test(text);
+    if (this.#first && !atSentenceEnd) {
+      const words = text.match(/\S+\s+/g)?.length ?? 0;
+      if (text.trim().length >= FIRST_MIN_CHARS || words >= FIRST_MIN_WORDS) {
+        const word = lastWordEnd(text, CLAUSE_MAX_CHARS);
+        if (word) return word;
+      }
+    }
+    return text.length > CLAUSE_MAX_CHARS ? overflowCut(text) : 0;
+  }
+}

package/src/voice/voice-delegation.ts

@@ -7,11 +7,32 @@ import type {
   VoiceSessionBridge,
 } from "../types";
 
+import { ClauseBuffer } from "./voice-clause-buffer";
+
 /**
  * What a delegated turn streamed: its assistant messages, whether it failed,
  * and whether the server dropped its result (then browser TTS may read it).
+ * `onUpdate` runs after each of its assistant messages is stored (the answer
+ * grew), so a streaming bridge can read the new text.
  */
-export type VoiceDelegationCapture = { ids: string[]; failed: boolean; dropped?: boolean };
+export type VoiceDelegationCapture = {
+  ids: string[];
+  failed: boolean;
+  dropped?: boolean;
+  onUpdate?: () => void;
+  /** Stops a streaming answer (its pending idle flush included); set while it streams. */
+  stopStream?: () => void;
+};
+
+/** How long a held sentence end waits for more text before it streams anyway. */
+const STREAM_IDLE_MS = 200;
+
+/** Length of the common prefix of `a` and `b`. */
+const commonPrefix = (a: string, b: string) => {
+  let i = 0;
+  while (i < a.length && i < b.length && a[i] === b[i]) i += 1;
+  return i;
+};
 
 /** The session internals the delegation bridge drives. */
 export interface VoiceDelegationHost {
@@ -164,16 +185,32 @@ export function createVoiceSessionBridge(host: VoiceDelegationHost): VoiceSessio
   };
 
   return {
-    getHistory: () =>
-      host.messages().flatMap((m) =>
-        (m.role === "user" || m.role === "assistant") &&
-        !m.variant &&
-        !m.voiceProcessing &&
-        !m.voiceCaption &&
-        m.content
-          ? [{ role: m.role, content: m.content }]
-          : [],
-      ),
+    // Read once at call start, before this call has any captions, so the
+    // captions here are earlier calls' (e.g. on the previous page of a
+    // multi-page site): spoken conversation the new voice session never heard.
+    // An assistant caption followed by a chat answer before the next user turn
+    // is a delegated turn's spoken filler ("Let me check."): it would only use
+    // up a slot of the context's recent-message window.
+    getHistory: () => {
+      const settled = host.messages().filter(
+        (m) =>
+          (m.role === "user" || m.role === "assistant") &&
+          !m.variant &&
+          !m.voiceProcessing &&
+          !m.streaming &&
+          m.content,
+      );
+      let answered = false;
+      const kept: Array<{ role: "user" | "assistant"; content: string }> = [];
+      for (let i = settled.length - 1; i >= 0; i -= 1) {
+        const m = settled[i];
+        if (m.role === "user") answered = false;
+        else if (!m.voiceCaption) answered = true;
+        else if (answered) continue;
+        kept.unshift({ role: m.role as "user" | "assistant", content: m.content });
+      }
+      return kept;
+    },
 
     dropDelegation(delegationId, expired) {
       // Remembered for a turn still running: when it parks, the same rule applies.
@@ -183,6 +220,7 @@ export function createVoiceSessionBridge(host: VoiceDelegationHost): VoiceSessio
       const park = expired && parks.find((p) => p.delegationId === delegationId);
       if (park && !park.outcome) settle(park, "timeout", "");
       for (const capture of captures.get(delegationId) ?? []) {
+        capture.stopStream?.();
         capture.dropped = true;
         if (capture.ids.length) host.unspoken(capture.ids);
       }
@@ -192,6 +230,7 @@ export function createVoiceSessionBridge(host: VoiceDelegationHost): VoiceSessio
       delegationId,
       userText,
       userUtteranceIds,
+      stream,
     }: VoiceDelegationRequest): Promise<VoiceDelegationResult> {
       const live = parks.filter((park) => pendingOf(park).length);
       const pendingIds = live.flatMap(pendingOf);
@@ -212,6 +251,48 @@ export function createVoiceSessionBridge(host: VoiceDelegationHost): VoiceSessio
       const before = new Set(host.messages().map((m) => m.id));
       const raised = () => host.messages().filter((m) => !before.has(m.id) && m.variant === "approval");
       const capture: VoiceDelegationCapture = { ids: [], failed: false };
+      // Streaming (`delegation_stream`): the answer goes out in speakable
+      // pieces while it renders. `fed` is the answer text the buffer has seen,
+      // `sent` what went out. An answer rewritten mid-stream (not an extension
+      // of what was fed) stops the stream; the result then says how much of
+      // its text was already sent.
+      const buffer = stream ? new ClauseBuffer() : null;
+      let fed = "";
+      let sent = "";
+      let rewritten = false;
+      const emit = (pieces: string[]) => {
+        for (const piece of pieces) {
+          sent += piece;
+          stream!(piece);
+        }
+      };
+      // A sentence end is confirmed only by the text after it; when the
+      // stream pauses on one, it goes out after STREAM_IDLE_MS anyway.
+      let idleTimer: ReturnType<typeof setTimeout> | undefined;
+      if (buffer) {
+        capture.onUpdate = () => {
+          if (rewritten) return;
+          const now = answerOf(capture.ids);
+          if (!now.startsWith(fed)) {
+            rewritten = true;
+            clearTimeout(idleTimer);
+            return;
+          }
+          if (now.length === fed.length) return;
+          const delta = now.slice(fed.length);
+          fed = now;
+          emit(buffer.push(delta));
+          clearTimeout(idleTimer);
+          idleTimer = setTimeout(() => {
+            if (!rewritten) emit(buffer.idle());
+          }, STREAM_IDLE_MS);
+        };
+        capture.stopStream = () => {
+          clearTimeout(idleTimer);
+          rewritten = true; // nothing more streams
+          capture.onUpdate = undefined;
+        };
+      }
       remember(delegationId, capture);
       active = capture;
       retrack();
@@ -225,6 +306,19 @@ export function createVoiceSessionBridge(host: VoiceDelegationHost): VoiceSessio
         active = null;
         retrack();
       }
+      if (buffer) {
+        // The last of the answer goes out before the result.
+        capture.onUpdate?.();
+        clearTimeout(idleTimer);
+        if (!rewritten) emit(buffer.flush());
+        capture.onUpdate = undefined;
+        capture.stopStream = undefined;
+      }
+      /** `streamedChars` for a result whose `text` starts with the answer. */
+      const streamed = (text: string) => {
+        const chars = commonPrefix(sent, text);
+        return chars > 0 ? { streamedChars: chars } : {};
+      };
       const replies = capture.ids.flatMap((id) => host.messages().find((m) => m.id === id) ?? []);
       let text = answerOf(capture.ids);
       const pending = capture.failed ? [] : raised().filter(isPendingApproval);
@@ -256,10 +350,13 @@ export function createVoiceSessionBridge(host: VoiceDelegationHost): VoiceSessio
         if (cancelled.get(delegationId)) settle(park, "timeout", "");
         // Parked on approvals: answer now, so the voice model asks for the
         // decision, then read the outcome back once the visitor decides.
+        // The streamed answer stays the prefix; the approval script is the
+        // unstreamed suffix.
         text = `${text}\n\n${buildApprovalScript(pending.map((m) => m.approval!))}`.trim();
         return {
           status: "pending_approval",
           text,
+          ...streamed(text),
           followUp: async ({ signal, approvalTimeoutMs = APPROVAL_TTL_MS, readBack = true, onUpdate }) => {
             const follow: VoiceDelegationCapture = { ids: [], failed: false };
             // Called once the ask went out: it carried this turn's reply to the
@@ -356,9 +453,10 @@ export function createVoiceSessionBridge(host: VoiceDelegationHost): VoiceSessio
             !m.toolCall?.name?.startsWith("webmcp:"),
         )
       ) {
-        return { status: capture.failed ? "failed" : "completed", text: `${text}\n\n${WAITING_FOR_INPUT}`.trim() };
+        text = `${text}\n\n${WAITING_FOR_INPUT}`.trim();
+        return { status: capture.failed ? "failed" : "completed", text, ...streamed(text) };
       }
-      return { status: !capture.failed && text ? "completed" : "failed", text };
+      return { status: !capture.failed && text ? "completed" : "failed", text, ...streamed(text) };
     },
   };
 }