@runtypelabs/persona 4.27.0 → 4.28.1

package/src/voice/voice-clause-buffer.test.ts

@@ -0,0 +1,143 @@
+import { describe, expect, it } from "vitest";
+import { CLAUSE_MAX_CHARS, ClauseBuffer } from "./voice-clause-buffer";
+
+/** Push `chunks` one by one, then flush; returns every piece in order. */
+function run(chunks: string[]): string[] {
+  const buffer = new ClauseBuffer();
+  const pieces = chunks.flatMap((chunk) => buffer.push(chunk));
+  return [...pieces, ...buffer.flush()];
+}
+
+/** Push character by character (the worst case for boundary detection). */
+const byChar = (text: string) => run([...text]);
+
+describe("ClauseBuffer", () => {
+  it("emits the first piece at a clause mark, then whole sentences", () => {
+    expect(byChar("Sure, we open at eight. On Sundays we are closed. Anything else?")).toEqual([
+      "Sure, ",
+      "we open at eight. ",
+      "On Sundays we are closed. ",
+      "Anything else?",
+    ]);
+  });
+
+  it("does not split later pieces at clause marks", () => {
+    expect(byChar("Yes. Monday, Tuesday, and Friday work; Sunday doesn't. Done")).toEqual([
+      "Yes. ",
+      "Monday, Tuesday, and Friday work; Sunday doesn't. ",
+      "Done",
+    ]);
+  });
+
+  it("emits the first piece after ~30 characters or 6 words without punctuation", () => {
+    const buffer = new ClauseBuffer();
+    expect(buffer.push("We bake sourdough rye ")).toEqual([]);
+    expect(buffer.push("and a seeded loaf every")).toEqual(["We bake sourdough rye and a seeded loaf "]);
+    expect(buffer.push(" morning")).toEqual([]);
+    expect(buffer.flush()).toEqual(["every morning"]);
+
+    const words = new ClauseBuffer();
+    expect(words.push("a b c d e f ")).toEqual(["a b c d e f "]);
+  });
+
+  it("keeps a whole first sentence together when it arrives before its trailing space", () => {
+    const buffer = new ClauseBuffer();
+    expect(buffer.push("I can place that order for you.")).toEqual([]);
+    expect(buffer.flush()).toEqual(["I can place that order for you."]);
+    expect(run(["I can place that order for you.", " Shall I?"])).toEqual(["I can place that order for you. ", "Shall I?"]);
+  });
+
+  it("keeps abbreviations, initials, decimals and list numbers inside a sentence", () => {
+    expect(
+      byChar("Hi there. Dr. Smith and J. R. Lee arrive at 9 a.m. today, e.g. for 3.5 hours. Then we close."),
+    ).toEqual(["Hi there. ", "Dr. Smith and J. R. Lee arrive at 9 a.m. today, e.g. for 3.5 hours. ", "Then we close."]);
+    expect(byChar("Steps:\n1. Mix the flour.\n2. Bake it.")).toEqual([
+      "Steps:\n",
+      "1. Mix the flour.\n",
+      "2. Bake it.",
+    ]);
+    expect(byChar("Please call 911. It is urgent.")).toEqual(["Please call 911. ", "It is urgent."]);
+  });
+
+  it("treats line breaks (Markdown blocks and list items) as boundaries", () => {
+    expect(byChar("**Opening hours**\n\n- Monday to Friday: 8am to 6pm\n- Sunday: closed")).toEqual([
+      "**Opening hours**\n\n",
+      "- Monday to Friday: 8am to 6pm\n",
+      "- Sunday: closed",
+    ]);
+  });
+
+  it("keeps closing quotes and emphasis with their sentence", () => {
+    expect(byChar('He said "go." Then **we left.** Fine')).toEqual(['He said "go." ', "Then **we left.** ", "Fine"]);
+    expect(byChar("Really?! Yes")).toEqual(["Really?! ", "Yes"]);
+  });
+
+  it("waits for whitespace after a period before cutting", () => {
+    const buffer = new ClauseBuffer();
+    expect(buffer.push("It costs 3.")).toEqual([]);
+    expect(buffer.push("50 dollars. And")).toEqual(["It costs 3.50 dollars. "]);
+  });
+
+  it("caps pieces at the max length, cutting at a clause mark, then a word, then hard", () => {
+    const clauses = `${"word ".repeat(30)}and then, ${"more ".repeat(30)}end.`;
+    const buffer = new ClauseBuffer();
+    expect(buffer.push("Hi. ")).toEqual([]); // the whitespace may still grow
+    const [hi, ...pieces] = [...buffer.push(clauses), ...buffer.flush()];
+    expect(hi).toBe("Hi. ");
+    expect(pieces.every((p) => p.length <= CLAUSE_MAX_CHARS)).toBe(true);
+    expect(pieces[0]).toMatch(/and then, $/);
+    expect(pieces.join("")).toBe(clauses);
+
+    const words = run(["Ok. ", "abcd ".repeat(100)]);
+    expect(words.slice(1).every((p) => p.length <= CLAUSE_MAX_CHARS && /\s$/.test(p))).toBe(true);
+
+    const solid = run(["x".repeat(500)]);
+    expect(solid.map((p) => p.length)).toEqual([CLAUSE_MAX_CHARS, CLAUSE_MAX_CHARS, 60]);
+  });
+
+  it("never cuts inside a surrogate pair", () => {
+    const text = `${"x".repeat(CLAUSE_MAX_CHARS - 1)}😀${"y".repeat(10)}`;
+    const pieces = run([text]);
+    expect(pieces.join("")).toBe(text);
+    expect(pieces[0]).toBe("x".repeat(CLAUSE_MAX_CHARS - 1));
+  });
+
+  it("pieces always concatenate to the input, however it is chunked", () => {
+    const text =
+      "**Opening hours**\n\n- Monday to Friday: 8am to 6pm\n- Saturday: 9am to 4pm\n\nWe're on Main St. near the U.S. post office, e.g. by the park. Call us!  ";
+    for (const size of [1, 2, 3, 7, 13, 40, text.length]) {
+      const chunks: string[] = [];
+      for (let i = 0; i < text.length; i += size) chunks.push(text.slice(i, i + size));
+      const pieces = run(chunks);
+      const joined = pieces.join("");
+      expect(text.startsWith(joined)).toBe(true);
+      expect(joined.trimEnd()).toBe(text.trimEnd());
+      expect(pieces.every((p) => p.trim().length > 0 && p.length <= CLAUSE_MAX_CHARS)).toBe(true);
+    }
+  });
+
+  it("idle() releases held text that ends in a terminator, and nothing else", () => {
+    const buffer = new ClauseBuffer();
+    expect(buffer.push("We're open Monday to Friday from 8am to 6pm.")).toEqual([]);
+    expect(buffer.idle()).toEqual(["We're open Monday to Friday from 8am to 6pm."]);
+    expect(buffer.idle()).toEqual([]);
+    expect(buffer.push(" On Saturdays we open")).toEqual([]);
+    expect(buffer.idle()).toEqual([]); // no terminator: keep waiting
+    expect(buffer.push(" at 9 a.m.")).toEqual([]);
+    expect(buffer.idle()).toEqual([" On Saturdays we open at 9 a.m."]); // no guards on a pause
+    for (const ending of ["Really?!", 'He said "go."', "**Done.**", "Wait…", "Line\n"]) {
+      const b = new ClauseBuffer();
+      b.push(ending);
+      expect(b.idle()).toEqual([ending]);
+    }
+  });
+
+  it("emits nothing for whitespace only, and flush empties the buffer", () => {
+    const buffer = new ClauseBuffer();
+    expect(buffer.push("   ")).toEqual([]);
+    expect(buffer.flush()).toEqual([]);
+    expect(buffer.push("Hello")).toEqual([]);
+    expect(buffer.flush()).toEqual(["Hello"]);
+    expect(buffer.flush()).toEqual([]);
+  });
+});

package/src/voice/voice-clause-buffer.ts

@@ -0,0 +1,187 @@
+/**
+ * Groups a growing answer into speakable pieces for `delegation_delta`
+ * (streaming client delegation): the first piece goes out early (at a clause
+ * mark, or once ~30 characters / 6 words are in), later pieces at sentence ends
+ * or line breaks, and no piece is longer than {@link CLAUSE_MAX_CHARS}.
+ *
+ * Pieces are exact slices of the input: concatenated, they equal everything
+ * pushed so far (whitespace included), so a receiver can count them as a
+ * prefix of the final text.
+ */
+
+/** The longest piece. */
+export const CLAUSE_MAX_CHARS = 220;
+const FIRST_MIN_CHARS = 30;
+const FIRST_MIN_WORDS = 6;
+
+/** Words ending in "." that rarely end a sentence (lowercased, inner dots kept). */
+const ABBREVIATIONS = new Set([
+  "mr",
+  "mrs",
+  "ms",
+  "dr",
+  "prof",
+  "sr",
+  "jr",
+  "st",
+  "mt",
+  "vs",
+  "etc",
+  "approx",
+  "inc",
+  "ltd",
+  "co",
+  "corp",
+  "no",
+  "nos",
+  "fig",
+  "ft",
+  "dept",
+  "est",
+  "min",
+  "max",
+  "e.g",
+  "i.e",
+  "a.m",
+  "p.m",
+  "u.s",
+  "u.k",
+]);
+
+// Punctuation (then closers like quotes, brackets or Markdown emphasis) and the
+// whitespace after it, or a line break.
+const BOUNDARY = /([.!?]+|[,;:])(["'”’)\]*_]*)(\s+)|\n\s*/g;
+
+/** Whether the "." ending `before` belongs to an abbreviation, an initial, or a list number. */
+function dotIsNotSentenceEnd(before: string): boolean {
+  const token = /(\S+)$/.exec(before)?.[1];
+  if (!token) return false;
+  const word = token.replace(/^[("'“‘[*_]+/, "").toLowerCase();
+  if (ABBREVIATIONS.has(word)) return true;
+  // An initial ("J.") or a dotted abbreviation ("U.S", "e.g").
+  if (/^[a-z](?:\.[a-z])*$/.test(word)) return true;
+  // "1. " opening a line is a list marker, not a sentence.
+  if (/^\d+$/.test(token)) {
+    const start = before.length - token.length;
+    return start === 0 || /\n\s*$/.test(before.slice(0, start));
+  }
+  return false;
+}
+
+/**
+ * Ends (after the whitespace) of each boundary in `text` with content before
+ * it. A boundary whose whitespace reaches the end of `text` may still grow
+ * ("\n" then "\n- item"), so it counts only once more text follows.
+ */
+function* boundaryEnds(text: string, clauses: boolean): Generator<number> {
+  const re = new RegExp(BOUNDARY.source, "g");
+  for (let m = re.exec(text); m; m = re.exec(text)) {
+    const mark = m[1];
+    if (!text.slice(0, m.index + (mark?.length ?? 0)).trim()) continue;
+    if (mark !== undefined) {
+      if (/^[,;:]$/.test(mark) && !clauses) continue;
+      if (mark === "." && dotIsNotSentenceEnd(text.slice(0, m.index))) continue;
+    }
+    const end = m.index + m[0].length;
+    if (end < text.length) yield end;
+  }
+}
+
+/** End of the last whitespace run within `limit` that has content before it, or 0. */
+function lastWordEnd(text: string, limit: number): number {
+  const window = text.slice(0, limit);
+  let best = 0;
+  for (const m of window.matchAll(/\s+/g)) {
+    if (window.slice(0, m.index).trim()) best = m.index! + m[0].length;
+  }
+  return best;
+}
+
+/** Where to cut a piece that reached {@link CLAUSE_MAX_CHARS} with no sentence end. */
+function overflowCut(text: string): number {
+  let clause = 0;
+  for (const end of boundaryEnds(text.slice(0, CLAUSE_MAX_CHARS), true)) clause = end;
+  if (clause) return clause;
+  const word = lastWordEnd(text, CLAUSE_MAX_CHARS);
+  if (word) return word;
+  // One unbroken run: cut hard, never inside a surrogate pair.
+  const code = text.charCodeAt(CLAUSE_MAX_CHARS - 1);
+  return code >= 0xd800 && code <= 0xdbff ? CLAUSE_MAX_CHARS - 1 : CLAUSE_MAX_CHARS;
+}
+
+export class ClauseBuffer {
+  #pending = "";
+  #first = true;
+
+  /** Add text; returns the pieces now complete (possibly none). */
+  push(text: string): string[] {
+    this.#pending += text;
+    return this.#drain();
+  }
+
+  /** Everything still held, as the last pieces (whitespace-only remainder is dropped). */
+  flush(): string[] {
+    const pieces = this.#drain();
+    const rest = this.#pending;
+    this.#pending = "";
+    if (rest.trim()) {
+      for (let at = 0; at < rest.length; ) {
+        const tail = rest.slice(at);
+        const cut = tail.length > CLAUSE_MAX_CHARS ? overflowCut(tail) : tail.length;
+        pieces.push(tail.slice(0, cut));
+        at += cut;
+      }
+      this.#first = false;
+    }
+    return pieces;
+  }
+
+  /**
+   * The stream went quiet: if the held text ends in a sentence terminator
+   * (`. ! ? …`, closers allowed) or a line break, release it as if the
+   * boundary were confirmed. No abbreviation guards: a pause is enough.
+   */
+  idle(): string[] {
+    const text = this.#pending;
+    if (!text.trim() || !/(?:[.!?…]["'”’)\]*_]*|\n)\s*$/.test(text)) return [];
+    this.#pending = "";
+    this.#first = false;
+    const pieces: string[] = [];
+    for (let at = 0; at < text.length; ) {
+      const tail = text.slice(at);
+      const cut = tail.length > CLAUSE_MAX_CHARS ? overflowCut(tail) : tail.length;
+      pieces.push(tail.slice(0, cut));
+      at += cut;
+    }
+    return pieces;
+  }
+
+  #drain(): string[] {
+    const pieces: string[] = [];
+    for (let cut = this.#cut(); cut > 0; cut = this.#cut()) {
+      pieces.push(this.#pending.slice(0, cut));
+      this.#pending = this.#pending.slice(cut);
+      this.#first = false;
+    }
+    return pieces;
+  }
+
+  /** Length of the next complete piece, or 0 to keep waiting. */
+  #cut(): number {
+    const text = this.#pending;
+    if (!text.trim()) return 0;
+    const end = boundaryEnds(text, this.#first).next().value ?? 0;
+    if (end && end <= CLAUSE_MAX_CHARS) return end;
+    // Ending on sentence punctuation, the text is most likely a whole sentence
+    // waiting for the space that confirms it: don't split it early.
+    const atSentenceEnd = /[.!?]["'”’)\]*_]*$/.test(text);
+    if (this.#first && !atSentenceEnd) {
+      const words = text.match(/\S+\s+/g)?.length ?? 0;
+      if (text.trim().length >= FIRST_MIN_CHARS || words >= FIRST_MIN_WORDS) {
+        const word = lastWordEnd(text, CLAUSE_MAX_CHARS);
+        if (word) return word;
+      }
+    }
+    return text.length > CLAUSE_MAX_CHARS ? overflowCut(text) : 0;
+  }
+}

package/src/voice/voice-delegation.ts

@@ -7,11 +7,32 @@ import type {
   VoiceSessionBridge,
 } from "../types";
 
+import { ClauseBuffer } from "./voice-clause-buffer";
+
 /**
  * What a delegated turn streamed: its assistant messages, whether it failed,
  * and whether the server dropped its result (then browser TTS may read it).
+ * `onUpdate` runs after each of its assistant messages is stored (the answer
+ * grew), so a streaming bridge can read the new text.
  */
-export type VoiceDelegationCapture = { ids: string[]; failed: boolean; dropped?: boolean };
+export type VoiceDelegationCapture = {
+  ids: string[];
+  failed: boolean;
+  dropped?: boolean;
+  onUpdate?: () => void;
+  /** Stops a streaming answer (its pending idle flush included); set while it streams. */
+  stopStream?: () => void;
+};
+
+/** How long a held sentence end waits for more text before it streams anyway. */
+const STREAM_IDLE_MS = 200;
+
+/** Length of the common prefix of `a` and `b`. */
+const commonPrefix = (a: string, b: string) => {
+  let i = 0;
+  while (i < a.length && i < b.length && a[i] === b[i]) i += 1;
+  return i;
+};
 
 /** The session internals the delegation bridge drives. */
 export interface VoiceDelegationHost {
@@ -199,6 +220,7 @@ export function createVoiceSessionBridge(host: VoiceDelegationHost): VoiceSessio
       const park = expired && parks.find((p) => p.delegationId === delegationId);
       if (park && !park.outcome) settle(park, "timeout", "");
       for (const capture of captures.get(delegationId) ?? []) {
+        capture.stopStream?.();
         capture.dropped = true;
         if (capture.ids.length) host.unspoken(capture.ids);
       }
@@ -208,6 +230,7 @@ export function createVoiceSessionBridge(host: VoiceDelegationHost): VoiceSessio
       delegationId,
       userText,
       userUtteranceIds,
+      stream,
     }: VoiceDelegationRequest): Promise<VoiceDelegationResult> {
       const live = parks.filter((park) => pendingOf(park).length);
       const pendingIds = live.flatMap(pendingOf);
@@ -228,6 +251,48 @@ export function createVoiceSessionBridge(host: VoiceDelegationHost): VoiceSessio
       const before = new Set(host.messages().map((m) => m.id));
       const raised = () => host.messages().filter((m) => !before.has(m.id) && m.variant === "approval");
       const capture: VoiceDelegationCapture = { ids: [], failed: false };
+      // Streaming (`delegation_stream`): the answer goes out in speakable
+      // pieces while it renders. `fed` is the answer text the buffer has seen,
+      // `sent` what went out. An answer rewritten mid-stream (not an extension
+      // of what was fed) stops the stream; the result then says how much of
+      // its text was already sent.
+      const buffer = stream ? new ClauseBuffer() : null;
+      let fed = "";
+      let sent = "";
+      let rewritten = false;
+      const emit = (pieces: string[]) => {
+        for (const piece of pieces) {
+          sent += piece;
+          stream!(piece);
+        }
+      };
+      // A sentence end is confirmed only by the text after it; when the
+      // stream pauses on one, it goes out after STREAM_IDLE_MS anyway.
+      let idleTimer: ReturnType<typeof setTimeout> | undefined;
+      if (buffer) {
+        capture.onUpdate = () => {
+          if (rewritten) return;
+          const now = answerOf(capture.ids);
+          if (!now.startsWith(fed)) {
+            rewritten = true;
+            clearTimeout(idleTimer);
+            return;
+          }
+          if (now.length === fed.length) return;
+          const delta = now.slice(fed.length);
+          fed = now;
+          emit(buffer.push(delta));
+          clearTimeout(idleTimer);
+          idleTimer = setTimeout(() => {
+            if (!rewritten) emit(buffer.idle());
+          }, STREAM_IDLE_MS);
+        };
+        capture.stopStream = () => {
+          clearTimeout(idleTimer);
+          rewritten = true; // nothing more streams
+          capture.onUpdate = undefined;
+        };
+      }
       remember(delegationId, capture);
       active = capture;
       retrack();
@@ -241,6 +306,19 @@ export function createVoiceSessionBridge(host: VoiceDelegationHost): VoiceSessio
         active = null;
         retrack();
       }
+      if (buffer) {
+        // The last of the answer goes out before the result.
+        capture.onUpdate?.();
+        clearTimeout(idleTimer);
+        if (!rewritten) emit(buffer.flush());
+        capture.onUpdate = undefined;
+        capture.stopStream = undefined;
+      }
+      /** `streamedChars` for a result whose `text` starts with the answer. */
+      const streamed = (text: string) => {
+        const chars = commonPrefix(sent, text);
+        return chars > 0 ? { streamedChars: chars } : {};
+      };
       const replies = capture.ids.flatMap((id) => host.messages().find((m) => m.id === id) ?? []);
       let text = answerOf(capture.ids);
       const pending = capture.failed ? [] : raised().filter(isPendingApproval);
@@ -272,10 +350,13 @@ export function createVoiceSessionBridge(host: VoiceDelegationHost): VoiceSessio
         if (cancelled.get(delegationId)) settle(park, "timeout", "");
         // Parked on approvals: answer now, so the voice model asks for the
         // decision, then read the outcome back once the visitor decides.
+        // The streamed answer stays the prefix; the approval script is the
+        // unstreamed suffix.
         text = `${text}\n\n${buildApprovalScript(pending.map((m) => m.approval!))}`.trim();
         return {
           status: "pending_approval",
           text,
+          ...streamed(text),
           followUp: async ({ signal, approvalTimeoutMs = APPROVAL_TTL_MS, readBack = true, onUpdate }) => {
             const follow: VoiceDelegationCapture = { ids: [], failed: false };
             // Called once the ask went out: it carried this turn's reply to the
@@ -372,9 +453,10 @@ export function createVoiceSessionBridge(host: VoiceDelegationHost): VoiceSessio
             !m.toolCall?.name?.startsWith("webmcp:"),
         )
       ) {
-        return { status: capture.failed ? "failed" : "completed", text: `${text}\n\n${WAITING_FOR_INPUT}`.trim() };
+        text = `${text}\n\n${WAITING_FOR_INPUT}`.trim();
+        return { status: capture.failed ? "failed" : "completed", text, ...streamed(text) };
       }
-      return { status: !capture.failed && text ? "completed" : "failed", text };
+      return { status: !capture.failed && text ? "completed" : "failed", text, ...streamed(text) };
     },
   };
 }