@wix/himalaya-cli 0.889.0 → 0.891.0

package/templates/community-club-owner/tier5-src/_shared/aria-chat.ts

@@ -34,9 +34,11 @@ import {
   messageText,
   messagesOf,
   sendMessage,
+  triggerAssistantMessage,
   type ConversationMessage,
   type ConversationSummary,
 } from "@himalaya/wix-genie-chat";
+import { proseOfParts, scanEnvelopes } from "./aria-widgets.js";
 
 // ---------------------------------------------------------------------------
 // Shapes
@@ -420,20 +422,35 @@ export function plainBubble(id: string, role: string, who: string, content: stri
 }
 
 /** Server transcript → bubbles. USER text is literal; ASSISTANT replies are
- *  markdown, so they are normalized and their links lifted into slots. */
+ *  markdown, so they are normalized and their links lifted into slots.
+ *
+ *  ORDER IS LOAD-BEARING. An assistant reply may carry inline widget envelopes
+ *  (`<<{"widget":…}>>`), and they are scanned out FIRST, because `stripMarkdown`
+ *  applied to a payload eats the underscores inside its JSON keys — which is how
+ *  `SINGLE_SELECT_WIDGET` used to reach the screen as `SINGLESELECTWIDGET`, both
+ *  corrupted and rendered as raw text. Everything downstream — the stripper, the
+ *  link scan — sees the PROSE only, so a markdown link that exists inside a
+ *  payload never becomes one of the bubble's three link rows either.
+ *
+ *  The widgets themselves are dropped here; rendering them is #2605 onward. A
+ *  reply that was nothing but a widget therefore has empty prose and is filtered
+ *  out below, which is deliberate: an empty bubble is a visible defect. */
 export function toBubbles(msgs: ConversationMessage[], assistantName: string, youName = "You"): AriaBubble[] {
   return msgs
     .map((m, i) => {
       const raw = messageText(m);
       const role = m.role || "ASSISTANT";
+      // Only assistant turns carry envelopes; a user's own text is literal, and
+      // scanning it would let an owner who typed `<<…>>` erase their own message.
+      const prose = role === "USER" ? raw : proseOfParts(scanEnvelopes(raw, 0, true).parts);
       return {
         id: m.id || `m${i}`,
         role,
         who: role === "USER" ? youName : assistantName,
-        content: role === "USER" ? raw : stripMarkdown(raw),
+        content: role === "USER" ? prose : stripMarkdown(prose),
         notUser: role !== "USER",
         notAria: role === "USER",
-        ...linkFieldsOf(raw, role),
+        ...linkFieldsOf(prose, role),
       };
     })
     .filter((b) => b.content && b.content.trim());
@@ -491,6 +508,34 @@ export function countRole(bubbles: AriaBubble[], role: string): number {
   return n;
 }
 
+/** How many turns of one role the SERVER holds — the mark the turn is judged
+ *  against. Counted over RAW messages, gated on the raw text being non-empty.
+ *
+ *  Both halves are load-bearing, and they pull in opposite directions:
+ *
+ *  - The emptiness gate is why this is not a plain `.length`. The service
+ *    creates an assistant row before it has written anything into it, and an
+ *    empty placeholder must not read as the answer that ended the turn. That is
+ *    the reason `countRole` counted bubbles in the first place.
+ *  - Counting RAW messages rather than bubbles is what this function adds, and
+ *    it is a bug fix rather than a tidy-up. An assistant reply that is nothing
+ *    but a widget envelope (`<<{"widget":…}>>`) is invisible to a bubble count:
+ *    `toBubbles` lifts the envelope out, the prose left behind is empty, and the
+ *    bubble is dropped so no blank row paints. `tick` then never saw the
+ *    assistant turn arrive, so the screen sat in `sending` at the FAST cadence
+ *    indefinitely — one request a second, forever, with the typing line stuck
+ *    on. A widget-only reply is a real answer; it just has nothing to say in
+ *    prose, and its raw text is non-empty, which is exactly what separates it
+ *    from the placeholder above. */
+export function countTurns(msgs: ConversationMessage[], role: string): number {
+  let n = 0;
+  for (const m of msgs) {
+    if ((m.role || "ASSISTANT") !== role) continue;
+    if (messageText(m).trim()) n++;
+  }
+  return n;
+}
+
 // ---------------------------------------------------------------------------
 // Local recent-conversation list (himalaya.storage)
 // ---------------------------------------------------------------------------
@@ -637,14 +682,30 @@ export function createAriaChat(o: AriaChatOptions): AriaChatCore {
    *  passed in — the only place a concurrent tap is visible is here. */
   let inFlightSend: string | null = null;
 
+  /** A turn the server STORED but never got asked to answer.
+   *
+   *  Sending is two calls: persist, then trigger. When the second fails, the
+   *  owner's message is already on the server — so the obvious recovery, tapping
+   *  Send again, posts a SECOND copy of it and can have the agent answer both.
+   *  Remembering the stored id turns that retry into a trigger of the turn that
+   *  already exists.
+   *
+   *  A latch rather than state, for the same reason `inFlightSend` is one: it is
+   *  a transient recovery hint about one failed attempt, not something that
+   *  should survive a reload or be persisted with the transcript. Keyed by
+   *  conversation AND content so it can only ever re-trigger the exact turn it
+   *  was recorded for. */
+  let stalledTurn: { convId: string; content: string; userMessageId: string } | null = null;
+
   /** Refuse a send that is already on its way. State half + latch half. */
   const duplicate = (s: AriaChatState, content: string): boolean =>
     isDuplicateSend(s, content) || inFlightSend === content;
 
-  /** Fold a freshly read server transcript into the observed-count marks. */
-  const observed = (bubbles: AriaBubble[]) => ({
-    _srvUsers: countRole(bubbles, "USER"),
-    _srvAssistants: countRole(bubbles, "ASSISTANT"),
+  /** Fold a freshly read server transcript into the observed-count marks.
+   *  Takes the RAW messages, not the bubbles — see `countTurns`. */
+  const observed = (msgs: ConversationMessage[]) => ({
+    _srvUsers: countTurns(msgs, "USER"),
+    _srvAssistants: countTurns(msgs, "ASSISTANT"),
   });
 
   const greetingOnly = (): AriaBubble[] => [plainBubble(GREETING_ID, "ASSISTANT", assistant(), res(o.greeting))];
@@ -692,16 +753,22 @@ export function createAriaChat(o: AriaChatOptions): AriaChatCore {
    *  deliberately. */
   async function busyLoop(s: AriaChatState, convId: string, optimistic: AriaBubble[]): Promise<AriaChatState> {
     let server: AriaBubble[] = optimistic;
+    let seen: ConversationMessage[] = [];
     for (let n = 0; n < BUSY_LOOP_ROUNDS; n++) {
       const gm = await getConversationMessages(convId);
-      const bubbles = toBubbles(messagesOf(gm), assistant(), you());
+      const raw = messagesOf(gm);
+      const bubbles = toBubbles(raw, assistant(), you());
       if (bubbles.length) server = bubbles;
-      const last = bubbles[bubbles.length - 1];
-      if (last && last.role === "ASSISTANT") {
-        return { ...s, status: "ready", _convId: convId, messages: bubbles, prompts: [], input: "", statusText: "", error: "", _pending: "", ...observed(bubbles), ok: false };
+      seen = raw;
+      // The turn is over when the SERVER holds an assistant answer — which a
+      // widget-only reply is, even though it leaves no bubble to inspect. Asking
+      // the last BUBBLE stalled this loop on exactly that reply, the same way it
+      // stalled `tick`.
+      if (countTurns(raw, "ASSISTANT") > s._replyAfter) {
+        return { ...s, status: "ready", _convId: convId, messages: bubbles, prompts: [], input: "", statusText: "", error: "", _pending: "", ...observed(raw), ok: false };
       }
     }
-    return { ...s, status: "ready", _convId: convId, messages: server, input: "", statusText: slowText(), _pending: "", ...observed(server), ok: false };
+    return { ...s, status: "ready", _convId: convId, messages: server, input: "", statusText: slowText(), _pending: "", ...observed(seen), ok: false };
   }
 
   /** Send + wait, by whichever transport this consumer configured. */
@@ -727,13 +794,47 @@ export function createAriaChat(o: AriaChatOptions): AriaChatCore {
     // it never received.
     let convId = s._convId;
     let site = s._siteId;
+    // The stored id this attempt is RE-triggering, if it is one. Declared out
+    // here because the catch needs it: `triggerAssistantMessage` rejects with a
+    // plain transport error and no `storedUserMessageId` — only `sendMessage`
+    // attaches that, because only `sendMessage` is the call that stored the turn.
+    // Without this, a retry that failed AGAIN cleared the latch and the attempt
+    // after it fell back to a full send, duplicating the very turn the latch
+    // exists to protect.
+    let retriedId = "";
     try {
       const ensured = await ensureConversation(s);
       convId = ensured.convId;
       site = ensured.site;
       const created = ensured.created;
       if (convId) remember(s, site, convId, content);
-      await sendMessage(convId, content);
+      // A retry of a turn the server already stored triggers THAT turn rather
+      // than posting the same words twice — see `stalledTurn`. Anything else is
+      // an ordinary two-call send.
+      //
+      // The SAME origin the conversation was created under (line above and
+      // `createConversation`). Sending without it defaulted the turn to
+      // `ARIA_ORIGIN`, so a consumer that configured its own origin created the
+      // conversation under one name and posted into it under another.
+      const stalled = stalledTurn && stalledTurn.convId === convId && stalledTurn.content === content
+        ? stalledTurn
+        : null;
+      if (stalled) {
+        retriedId = stalled.userMessageId;
+        await triggerAssistantMessage(convId, stalled.userMessageId, { origin });
+      } else {
+        await sendMessage(convId, content, { origin });
+      }
+      // Delivered — whatever was stalled is answered or superseded.
+      //
+      // `retriedId` is cleared with it, and that is not redundant: the legacy
+      // `busyLoop` consumer keeps reading the transcript AFTER this point, still
+      // inside this `try`. A failure there is a failure to READ, not to deliver
+      // — the turn has been triggered — so leaving `retriedId` set let the catch
+      // below rebuild a latch for a turn already being answered, and the next
+      // identical submission triggered it a second time.
+      stalledTurn = null;
+      retriedId = "";
       // A conversation minted just now starts empty, so the marks taken above
       // from the PREVIOUS conversation's counts would be ahead of it and its
       // first reply would never read as new — the same stranded "typing…" this
@@ -752,6 +853,18 @@ export function createAriaChat(o: AriaChatOptions): AriaChatCore {
       return busyLoop({ ...sent, _convId: convId, _siteId: site }, convId, optimistic);
     } catch (e) {
       inFlightSend = null;
+      // A send that failed AFTER the turn was stored hands back the id it stored
+      // under. Remember it, so the owner tapping Send again triggers that turn
+      // instead of posting their message a second time. Any other failure clears
+      // the latch: nothing is stored, so a plain retry is the right recovery and
+      // a stale id would re-trigger a turn the owner has moved on from.
+      // Two ways this attempt can leave a turn stored and unanswered: a SEND
+      // whose trigger failed (the id arrives on the error), or a RE-TRIGGER of
+      // an already-stored turn that failed again (no id on the error, because
+      // nothing new was stored — the one we were retrying is still the answer).
+      const storedId = (e as { storedUserMessageId?: unknown })?.storedUserMessageId;
+      const keptId = typeof storedId === "string" && storedId ? storedId : retriedId;
+      stalledTurn = convId && keptId ? { convId, content, userMessageId: keptId } : null;
       return {
         ...base, status: "ready", _pending: "",
         // `_siteId` moves with the id or neither does: a conversation remembered
@@ -786,10 +899,11 @@ export function createAriaChat(o: AriaChatOptions): AriaChatCore {
       if (resumeId) {
         try {
           const gm = await getConversationMessages(resumeId);
-          const bubbles = toBubbles(messagesOf(gm), assistant(), you());
+          const raw = messagesOf(gm);
+          const bubbles = toBubbles(raw, assistant(), you());
           const resumed = bubbles.length ? bubbles : greetingOnly();
           return { ...s, status: "ready", _convId: resumeId, _siteId: site, _pending: "", statusText: "", error: "",
-            messages: resumed, ...observed(bubbles), prompts: promptsFor(resumed, promptList()) };
+            messages: resumed, ...observed(raw), prompts: promptsFor(resumed, promptList()) };
         } catch { /* expired or unreachable — fall through to a fresh one */ }
       }
       // Opening a screen runs `load` more than once — the bundle self-kicks at
@@ -884,20 +998,32 @@ export function createAriaChat(o: AriaChatOptions): AriaChatCore {
       const wasFast = s.status === "sending";
       try {
         const gm = await getConversationMessages(s._convId);
-        const server = toBubbles(messagesOf(gm), assistant(), you());
-        if (!server.length) return s;
+        const raw = messagesOf(gm);
+        const server = toBubbles(raw, assistant(), you());
+        // A transcript that renders to nothing is not necessarily an empty one:
+        // a reply made entirely of widgets leaves no bubble behind. So the bail
+        // is on the RAW transcript — bailing on the bubbles is what let such a
+        // reply go unnoticed and pinned the ladder fast forever.
+        if (!raw.length) return s;
         const { messages, pending } = reconcile(server, s._pending, s._pendingAfter, you());
+        // …but still never PAINT nothing over something. `messages` is only
+        // adopted when it has content: a transcript that renders empty leaves
+        // the screen as it was (the greeting, or the last good turn) while the
+        // counts below still register the answer.
+        const painted = messages.length ? messages : s.messages;
         // The turn is over when the server holds an assistant turn it did not
         // hold when we sent. Deliberately independent of `pending`: whether our
         // own words came back is a rendering detail, not the answer arriving.
-        const replied = countRole(server, "ASSISTANT") > s._replyAfter;
+        // Counted over RAW messages, so a widget-only answer still ends the turn
+        // instead of pinning the ladder to its fast rung forever (`countTurns`).
+        const replied = countTurns(raw, "ASSISTANT") > s._replyAfter;
         const next: AriaChatState = {
           ...s,
           status: replied ? "ready" : s.status,
-          messages,
+          messages: painted,
           _pending: pending,
-          ...observed(server),
-          prompts: promptsFor(messages, promptList()),
+          ...observed(raw),
+          prompts: promptsFor(painted, promptList()),
           statusText: replied ? "" : s.statusText,
         };
         // Slow the ladder the moment the turn is over; leave it alone otherwise.
@@ -932,6 +1058,14 @@ export interface AriaHistoryState {
   items: AriaHistoryRow[];
   /** Non-empty ONLY when the list is empty (collapses otherwise). */
   emptyText: string;
+  /** The business the rows in `items` belong to. A kept list is only safe to
+   *  keep while this still matches, so it is state rather than a closure — the
+   *  chat service tracks its own `_siteId` for the same reason. */
+  _siteId: string;
+  /** The last load could not REACH the server, as opposed to reaching it and
+   *  being told there are none. The two are different answers and the list
+   *  treats them differently; see `load`. */
+  failed: boolean;
 }
 
 /** Worker-emitted strings for the history list. Thunks for the same reason the
@@ -1033,28 +1167,59 @@ export function createAriaHistory(o: AriaHistoryOptions) {
     last30: res(o.labels?.last30 ?? STOCK_BUCKETS.last30),
     older: res(o.labels?.older ?? STOCK_BUCKETS.older),
   });
-  const initialState: AriaHistoryState = { status: "loading", items: [], emptyText: "" };
+  const initialState: AriaHistoryState = {
+    status: "loading", items: [], emptyText: "", _siteId: "", failed: false,
+  };
   return {
     initialState,
     load: async (s: AriaHistoryState): Promise<AriaHistoryState> => {
       const now = Date.now();
-      let entries: Array<{ id: string; title: string; ms: number }> = [];
+      const site = siteOf();
+      // `null` means the server was never reached. Distinct from `[]`, which is
+      // the server answering that this business has no conversations — the whole
+      // fix below turns on telling those two apart.
+      let fetched: Array<{ id: string; title: string; ms: number }> | null = null;
       try {
         const conversations = await getConversations(origin);
-        entries = fromServer(conversations.conversations ?? [], untitled());
+        fetched = fromServer(conversations.conversations ?? [], untitled());
       } catch {
-        // server unavailable — fall through to the local fallback below
+        fetched = null;
       }
-      // The server list is authoritative; the local recent list only covers the
-      // case where it is empty or unreachable. It is site-partitioned for the
-      // same reason the server list is site-scoped — but a metaSite is not an
-      // account, so an app where two people can sign in on one device turns this
-      // off entirely rather than risk showing one owner the other's titles.
+
+      // A FAILED fetch must not blank a list that is already on screen. An app
+      // that sets `localHistory: false` has the server list as its ONLY source,
+      // so rebuilding from an empty `entries` turns a network blip into
+      // "No past conversations yet." over a list the owner was just reading.
+      //
+      // Kept ONLY while the rows still belong to the same business. That clause
+      // is the point, not a detail — without it this becomes a way to show one
+      // owner the previous business's conversation titles, which is precisely
+      // the leak `localHistory: false` is set to avoid.
+      if (fetched === null && s.items.length && s._siteId === site) {
+        return { ...s, status: "ready", failed: true };
+      }
+
+      // A SUCCESSFUL empty answer is authoritative and DOES clear. The tempting
+      // shortcut is to keep the last good list on ANY empty result, failure or
+      // not — do not. This list is site-scoped, so right after a business switch
+      // the new business truthfully answers "none", and keeping the old rows
+      // would carry the previous business's titles across. A stale-looking list
+      // is not worth a leak.
+      let entries = fetched ?? [];
+      // The local recent list still covers empty-or-unreachable, and is
+      // site-partitioned for the same reason the server list is.
       if (!entries.length && o.localHistory !== false) {
-        entries = fromLocal(readHistory(scopedKey(o.historyKey, siteOf())));
+        entries = fromLocal(readHistory(scopedKey(o.historyKey, site)));
       }
       const items = bucketRows(entries, now, buckets());
-      return { ...s, status: "ready", items, emptyText: items.length ? "" : emptyText() };
+      return {
+        ...s,
+        status: "ready",
+        items,
+        emptyText: items.length ? "" : emptyText(),
+        _siteId: site,
+        failed: fetched === null,
+      };
     },
   };
 }

package/templates/community-club-owner/tier5-src/_shared/aria-widgets.ts

@@ -0,0 +1,181 @@
+// Inline widget envelopes in Aria's assistant text.
+//
+// The Wix Genie chat-service embeds interactive widgets inside the message body:
+//
+//     Here are two templates.\n\n<<{"widget":"SINGLE_SELECT_WIDGET","data":{…}}>>\n\nWhich?
+//
+// The RN Aria app pulls them out with `/<<(.*?)>>/gs`; the standalone native app
+// scans incrementally. We do the latter, for the reason in `scanEnvelopes` below.
+//
+// WHY THIS IS A MODULE AND NOT A REGEX AT THE CALL SITE
+//
+// Order. `stripMarkdown` must never see a payload: its italic rules (`__x__` -> `x`,
+// `_x_` -> `x`) run across the whole string, so a payload that passes through it
+// comes back with the underscores eaten — `SINGLE_SELECT_WIDGET` arrives as
+// `SINGLESELECTWIDGET`, and every later parser reads a corrupted key. Scanning
+// before stripping, and stripping only the text parts, is the fix.
+//
+// This module is deliberately DEPENDENCY-FREE so it can be imported by the worker
+// bundle and unit-tested without booting anything.
+
+/** One widget lifted out of the text, with its payload untouched. */
+export interface WidgetPayload {
+  /** The `widget` field, or `""` when the payload could not be read. */
+  name: string;
+  /** The `data` object re-serialized, or null when absent. Kept as a STRING so the
+   *  scanner never becomes the thing that decides a payload's shape — that is the
+   *  normalizer's job, and it can fail without taking the chat down with it. */
+  dataJson: string | null;
+  /** The `config` object re-serialized, or null when absent. */
+  configJson: string | null;
+  /** Exactly what sat between `<<` and `>>`. Preserved verbatim for diagnosis. */
+  raw: string;
+}
+
+export type WidgetPart =
+  | { kind: "text"; text: string }
+  | { kind: "widget"; payload: WidgetPayload };
+
+export interface ScanResult {
+  parts: WidgetPart[];
+  /** Where a subsequent scan of the same message should start. Equal to the text
+   *  length once the turn is complete; otherwise it may point back at an unclosed
+   *  `<<` (or a bare trailing `<`) that must be re-read when more text arrives. */
+  resumeOffset: number;
+}
+
+const OPEN = "<<";
+const CLOSE = ">>";
+
+/** Read the payload between the delimiters. Never throws: an unreadable payload
+ *  still gets lifted out of the prose, with an empty name for the normalizer to
+ *  turn into UNKNOWN. Showing raw JSON to an owner is the failure being avoided. */
+function payloadOf(raw: string): WidgetPayload {
+  let name = "";
+  let dataJson: string | null = null;
+  let configJson: string | null = null;
+  try {
+    const parsed = JSON.parse(raw) as unknown;
+    if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) {
+      const obj = parsed as Record<string, unknown>;
+      if (typeof obj.widget === "string") name = obj.widget;
+      if (obj.data !== undefined) dataJson = JSON.stringify(obj.data);
+      if (obj.config !== undefined) configJson = JSON.stringify(obj.config);
+    }
+  } catch {
+    // Deliberately silent: `name` stays "" and the caller degrades to UNKNOWN.
+  }
+  return { name, dataJson, configJson, raw };
+}
+
+/**
+ * Index of the `>>` that actually TERMINATES the envelope opened at `start`, or
+ * -1 if there is none yet.
+ *
+ * Not `indexOf`, because the envelope body is JSON and JSON strings may contain
+ * anything — including `>>`. A perfectly valid payload such as
+ * `{"widget":"CHART","config":{"title":"Q3 >> Q4"}}` was cut at the inner `>>`:
+ * `payloadOf` then got truncated, unparseable JSON (degrading the widget to
+ * UNKNOWN) AND the remainder — `Q4"}}` — was emitted as prose, printing a
+ * fragment of the payload into the chat: the very thing this module exists to
+ * prevent.
+ *
+ * So the scan tracks whether it is inside a JSON string, honouring backslash
+ * escapes. Outside a string, the first `>>` wins as before.
+ */
+function closeOf(text: string, start: number): number {
+  let inStr = false;
+  for (let i = start; i < text.length; i++) {
+    const c = text[i];
+    if (inStr) {
+      // A backslash escapes the next character, `\"` and `\\` included — so
+      // skipping one character here is what keeps `"a\\"` from reading as an
+      // unterminated string.
+      if (c === "\\") { i++; continue; }
+      if (c === '"') inStr = false;
+      continue;
+    }
+    if (c === '"') { inStr = true; continue; }
+    if (c === ">" && text[i + 1] === ">") return i;
+  }
+  // No string-aware terminator. That is EITHER a genuinely unclosed envelope, or
+  // a payload whose JSON is malformed — an unterminated string swallows the rest
+  // of the message, `>>` included. Fall back to the naive scan for the second
+  // case: a broken envelope still belongs OUT of the prose, degraded to an
+  // UNKNOWN widget, rather than printed into the chat as raw JSON. When the text
+  // really holds no `>>`, this returns -1 too and the caller's unclosed handling
+  // takes over unchanged.
+  return text.indexOf(CLOSE, start);
+}
+
+/**
+ * Split `text` into prose and widget parts, starting at `from`.
+ *
+ * `completed` says whether the assistant's turn has finished. It is the only thing
+ * that decides what to do with an unterminated `<<`:
+ *
+ *   - mid-turn  — HOLD IT BACK and point `resumeOffset` at the `<<`. A half-arrived
+ *     envelope must not flash as literal JSON and then vanish when it closes.
+ *   - complete  — it never closed, so it was never an envelope. Emit it as ordinary
+ *     text rather than swallowing content the assistant actually sent.
+ *
+ * A bare `<` at the very end is held back mid-turn for the same reason: it may be
+ * the first half of `<<`.
+ *
+ * We poll rather than stream today, so every real call is `(text, 0, true)`. The
+ * resumable shape is kept because retrofitting it later is the expensive part, and
+ * because the held-back case is invisible under polling and obvious under streaming.
+ */
+export function scanEnvelopes(text: string, from: number, completed: boolean): ScanResult {
+  const parts: WidgetPart[] = [];
+  const pushText = (s: string): void => {
+    if (s) parts.push({ kind: "text", text: s });
+  };
+
+  let cursor = Math.min(Math.max(from, 0), text.length);
+
+  for (;;) {
+    const open = text.indexOf(OPEN, cursor);
+    if (open < 0) break;
+
+    const close = closeOf(text, open + OPEN.length);
+    if (close < 0) {
+      if (completed) {
+        // Never closed and never will. It is text.
+        pushText(text.slice(cursor));
+        return { parts, resumeOffset: text.length };
+      }
+      // Still arriving: show what precedes it, and re-read from the `<<` next time.
+      pushText(text.slice(cursor, open));
+      return { parts, resumeOffset: open };
+    }
+
+    pushText(text.slice(cursor, open));
+    parts.push({ kind: "widget", payload: payloadOf(text.slice(open + OPEN.length, close)) });
+    cursor = close + CLOSE.length;
+  }
+
+  // No further `<<`. A lone trailing `<` mid-turn may be the start of one.
+  if (!completed && text.endsWith("<")) {
+    const held = text.length - 1;
+    if (cursor < held) pushText(text.slice(cursor, held));
+    return { parts, resumeOffset: held };
+  }
+
+  pushText(text.slice(cursor));
+  return { parts, resumeOffset: text.length };
+}
+
+/** The prose of a scan, with every widget removed. The string that reaches a bubble. */
+export function proseOfParts(parts: WidgetPart[]): string {
+  let out = "";
+  for (const p of parts) if (p.kind === "text") out += p.text;
+  return out;
+}
+
+/** Every widget a scan lifted out, in the order they appeared. */
+export function widgetsOfParts(parts: WidgetPart[]): WidgetPayload[] {
+  const out: WidgetPayload[] = [];
+  for (const p of parts) if (p.kind === "widget") out.push(p.payload);
+  return out;
+}