@cairnvibe/sdk 0.2.5 → 0.2.7

package/src/index.tsx

@@ -3,6 +3,8 @@
 import { useEffect, useRef, useState } from "react";
 import { usePathname, useRouter } from "next/navigation";
 import {
+  ChevronDown,
+  ChevronUp,
   Loader2,
   Mic,
   MicOff,
@@ -17,6 +19,7 @@ import {
 import type { HistoryTurn as HistoryEntry, TourStep } from "@cairnvibe/core";
 import { collectVisible } from "./context-collector";
 import { findElement, highlightElement, logMiss, type MissContext } from "./element-ladder";
+import { createLiveElementRegistry } from "./runtime-scan";
 import { executeVerbResponse } from "./verb-executor";
 
 export interface CopilotProps {
@@ -63,8 +66,24 @@ export function Copilot({
   persona = "Cairn",
 }: CopilotProps) {
   const pathname = usePathname() ?? "/";
+  // Mirrors `pathname` for use inside long-lived closures (a realtime
+  // session's handlers are all created once, when the connection opens —
+  // same staleness reason runTour tracks its own `currentRoute` locally
+  // rather than trusting its closure's `pathname` after a mid-tour
+  // navigation).
+  const pathnameRef = useRef(pathname);
+  useEffect(() => {
+    pathnameRef.current = pathname;
+    sendFreshContext(); // no-op if no realtime session is open
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [pathname]);
   const router = useRouter();
   const [open, setOpen] = useState(false);
+  // Collapsed by default so the panel only ever shows the current exchange
+  // — the full archived transcript (built up over a long conversation)
+  // stays out of the way behind an explicit toggle instead of always being
+  // visible inline, which made the panel grow uncomfortably tall.
+  const [historyExpanded, setHistoryExpanded] = useState(false);
   const [question, setQuestion] = useState("");
   const [answer, setAnswer] = useState<string | null>(null);
   const [status, setStatus] = useState<Status>("idle");
@@ -92,16 +111,19 @@ export function Copilot({
   const [rtMicMuted, setRtMicMuted] = useState(false);
   const [rtSpeakerMuted, setRtSpeakerMuted] = useState(false);
   // Set while a "tour" verb's steps are being narrated/highlighted one at a
-  // time — drives the step-progress caption and blocks the input/mic so a
-  // typed or spoken question can't interrupt mid-walkthrough.
+  // time — drives the step-progress caption and blocks the *typed* input so
+  // a typed question can't collide with the walkthrough (a voice
+  // interruption is handled separately — see touringRef/triggerBargeIn).
   const [tourStep, setTourStep] = useState<{ index: number; total: number } | null>(null);
-  const tourGenerationRef = useRef(0); // bumped to cancel an in-progress tour (e.g. widget closed) without extra flags
+  const tourGenerationRef = useRef(0); // bumped to cancel an in-progress tour (e.g. widget closed, or a voice barge-in) without extra flags
   // Mirrors whether a tour is running, for use inside the mic's
   // onaudioprocess callback (a stale closure over React state there would
   // miss a tour that started after the callback was created) — a tour
-  // reuses "rt-speaking" to hold the mic off too, but must NOT be
-  // barge-in-able the way a real conversational reply is (see the RMS
-  // check below): it's a deliberate walkthrough, not a turn to interrupt.
+  // reuses "rt-speaking" to hold the mic off between steps, but IS
+  // barge-in-able like a real conversational reply (see the RMS check
+  // below): interrupting mid-tour cancels the rest of the walkthrough,
+  // the way a real person giving a tour stops when you have a question
+  // instead of talking over you.
   const touringRef = useRef(false);
   // Resolver for "this tour step's audio has fully finished playing" when
   // narrating over an already-open realtime session (see maybeResumeListening
@@ -115,6 +137,20 @@ export function Copilot({
   // already stateful).
   const historyRef = useRef<HistoryEntry[]>([]);
 
+  // A background scanner that keeps a live inventory of what's actually
+  // clickable on screen right now (runtime-scan.ts) — running continuously
+  // via a MutationObserver so there's never a pause to "go look at the
+  // page" right when a verb needs to click something. `liveMapRef` freezes
+  // one snapshot of it per turn (set alongside every context/question send,
+  // below) so a background rescan landing mid-flight can't shift what an id
+  // resolves to between when a request went out and its response came back.
+  const liveRegistryRef = useRef(createLiveElementRegistry());
+  const liveMapRef = useRef<Map<string, HTMLElement>>(new Map());
+  useEffect(() => {
+    liveRegistryRef.current.start();
+    return () => liveRegistryRef.current.stop();
+  }, []);
+
   const mediaRecorderRef = useRef<MediaRecorder | null>(null);
   const audioChunksRef = useRef<Blob[]>([]);
   const transcribeInFlightRef = useRef(false);
@@ -210,6 +246,28 @@ export function Copilot({
     setStatus(next);
   }
 
+  /**
+   * Refreshes the server's picture of route/visible/liveElements over an
+   * already-open realtime connection. Beyond the initial connect, called
+   * on every route change and whenever the mic is about to start listening
+   * again — a real, pre-existing gap this closes as a side effect: the
+   * server's context previously updated only once, at connection open, so
+   * navigating mid-call (via a "navigate" verb, or the user clicking
+   * around) left the server answering every later turn as if the user were
+   * still on the original page. Reads pathnameRef, not the closure's
+   * `pathname`, so it's correct even called from a handler created once at
+   * connection-open time.
+   */
+  function sendFreshContext() {
+    const ws = rtSocketRef.current;
+    if (!ws || ws.readyState !== WebSocket.OPEN) return;
+    const liveScan = liveRegistryRef.current.getSnapshot();
+    liveMapRef.current = liveScan.byId;
+    ws.send(
+      JSON.stringify({ type: "context", route: pathnameRef.current, visible: collectVisible(), liveElements: liveScan.elements }),
+    );
+  }
+
   function reportMiss(context: MissContext) {
     logMiss(context);
     if (reportMissesEndpoint) {
@@ -232,6 +290,7 @@ export function Copilot({
       onDo,
       onTour: (steps) => void runTour(steps),
       registeredActions,
+      liveElements: liveMapRef.current,
     });
   }
 
@@ -242,12 +301,14 @@ export function Copilot({
    * actually showing you around instead of one paragraph naming several
    * buttons at once with nothing highlighted.
    *
-   * Always narrates via speakEndpoint (plain request/response TTS), even
-   * during a live realtime session — a tour is a distinct guided
-   * walkthrough, not a conversational turn, so it doesn't need the
-   * streaming relay's turn-taking machinery. If the widget is mid
-   * realtime call, the mic is held off (mirrors "rt-speaking") for the
-   * tour's duration so it can't pick up the tour's own narration.
+   * During a live realtime session, narration reuses the same streaming
+   * Speak connection a normal conversational reply uses (see
+   * speakOverRealtime below) instead of a separate buffered REST call —
+   * otherwise falls back to speakEndpoint. Either way, the mic is held off
+   * between steps (mirrors "rt-speaking") so it doesn't pick up the tour's
+   * own narration — but it's still listening for a real interruption:
+   * talking during a step cancels the rest of the tour via triggerBargeIn,
+   * the same as interrupting a normal spoken reply.
    */
   async function runTour(steps: TourStep[]) {
     const myGeneration = ++tourGenerationRef.current;
@@ -292,9 +353,23 @@ export function Copilot({
         }
 
         if (step.target) {
-          const el = findElement(step.target);
-          if (el) highlightElement(el);
-          else reportMiss({ attempted: step.target, route: currentRoute });
+          // A fresh scan, not the tour's starting liveMapRef snapshot — a
+          // step after a mid-tour navigation targets elements on a page
+          // that didn't exist when the tour began.
+          const liveScan = liveRegistryRef.current.getSnapshot();
+          const el = findElement(step.target, liveScan.byId);
+          if (el) {
+            highlightElement(el);
+            if (step.click) {
+              el.click();
+              // Give whatever the click reveals (a detail view, an expanded
+              // row) a moment to actually render before narrating it.
+              await new Promise((resolve) => setTimeout(resolve, 400));
+              if (tourGenerationRef.current !== myGeneration) return;
+            }
+          } else {
+            reportMiss({ attempted: step.target, route: currentRoute });
+          }
         }
 
         if (wasRealtimeListening && rtSocketRef.current?.readyState === WebSocket.OPEN) {
@@ -332,10 +407,18 @@ export function Copilot({
     setLastQuestion(q);
     setQuestion("");
     try {
+      const liveScan = liveRegistryRef.current.getSnapshot();
+      liveMapRef.current = liveScan.byId;
       const res = await fetch(endpoint, {
         method: "POST",
         headers: { "content-type": "application/json" },
-        body: JSON.stringify({ route: pathname, question: q, visible: collectVisible(), history: historyRef.current }),
+        body: JSON.stringify({
+          route: pathname,
+          question: q,
+          visible: collectVisible(),
+          history: historyRef.current,
+          liveElements: liveScan.elements,
+        }),
       });
       const data = await res.json().catch(() => null);
       handleVerb(data);
@@ -550,12 +633,20 @@ export function Copilot({
         if (ws.readyState !== WebSocket.OPEN) return;
         if (rtMicMutedRef.current) return;
 
-        // Barge-in: while the agent is speaking a real conversational
-        // reply (not touring — a tour deliberately can't be talked over),
-        // keep listening to the mic locally even though it isn't being
-        // sent yet, and cut the agent off the instant the user starts
-        // talking over it instead of making them wait for it to finish.
-        if (rtStateRef.current === "rt-speaking" && !touringRef.current) {
+        // Barge-in: while the agent is speaking a real conversational reply,
+        // still thinking about one, OR mid-tour, keep listening to the mic
+        // locally even though it isn't being sent yet, and cut the agent
+        // off the instant the user starts talking again instead of making
+        // them wait — including during a guided tour, which now cancels the
+        // rest of the walkthrough on interruption (see triggerBargeIn)
+        // instead of being talked-over-proof by design, the way a real
+        // person giving a tour stops when you have a question. The
+        // "rt-thinking" half matters just as much as "rt-speaking": an LLM
+        // turn can easily take a couple of seconds with nothing playing
+        // yet, and without this the mic was completely deaf during that
+        // whole window — found live as "not listening while speaking... no
+        // interrupting system", not just a missed nice-to-have.
+        if (rtStateRef.current === "rt-speaking" || rtStateRef.current === "rt-thinking") {
           const rms = computeRms(e.inputBuffer.getChannelData(0));
           if (rms > BARGE_IN_RMS_THRESHOLD) triggerBargeIn();
           return;
@@ -601,6 +692,7 @@ export function Copilot({
         }
         setRtStatus("rt-listening");
         setCaption("");
+        sendFreshContext(); // refresh before the user starts talking again, not after
       }
 
       function disarmThinkingWatchdog() {
@@ -642,13 +734,26 @@ export function Copilot({
         disarmThinkingWatchdog();
         stopScheduledRtAudio();
         rtAudioDoneArrivingRef.current = true;
+        if (touringRef.current) {
+          // Interrupting mid-guide cancels the whole rest of the tour, not
+          // just the current step — the way a real person giving a tour
+          // stops and answers your question instead of continuing to talk
+          // over you. Without resolving the current step's own pending
+          // promise here, runTour only notices the cancellation via its own
+          // 15s-per-step fallback timeout instead of right away.
+          tourGenerationRef.current++;
+          touringRef.current = false;
+          setTourStep(null);
+          rtTourAudioDoneRef.current?.();
+          rtTourAudioDoneRef.current = null;
+        }
         if (ws.readyState === WebSocket.OPEN) ws.send(JSON.stringify({ type: "barge_in" }));
         setRtStatus("rt-listening");
         setCaption("");
       }
 
       ws.onopen = () => {
-        ws.send(JSON.stringify({ type: "context", route: pathname, visible: collectVisible() }));
+        sendFreshContext();
         setRtStatus("rt-listening");
         rtStartingRef.current = false;
       };
@@ -718,7 +823,17 @@ export function Copilot({
           // exactly the way a silently-dropped response used to leave it.
           disarmThinkingWatchdog();
           setAnswer(msg.message ?? "Something went wrong.");
-          if (!touringRef.current) {
+          if (touringRef.current) {
+            // A tour step's own speakStreamed() failed server-side (see
+            // realtime-server.ts's "speak" handler). Without resolving this
+            // step's pending promise here, runTour's `await
+            // speakOverRealtime(step.text)` only recovers via its own 15s
+            // fallback timeout — found live as a guide that goes badly
+            // quiet for long stretches, one step at a time.
+            rtAudioDoneArrivingRef.current = true;
+            rtTourAudioDoneRef.current?.();
+            rtTourAudioDoneRef.current = null;
+          } else {
             setRtStatus("rt-listening");
             setCaption("");
           }
@@ -807,14 +922,26 @@ export function Copilot({
 
           {(transcript.length > 0 || userCaption || answer || busy) && (
             <div className="cairn-stack">
-              {transcript.map((entry) => (
-                <div
-                  className={entry.role === "user" ? "cairn-bubble cairn-bubble-user cairn-bubble-past" : "cairn-bubble cairn-bubble-agent cairn-bubble-past"}
-                  key={entry.id}
+              {transcript.length > 0 && (
+                <button
+                  type="button"
+                  className="cairn-history-toggle"
+                  onClick={() => setHistoryExpanded((v) => !v)}
+                  aria-expanded={historyExpanded}
                 >
-                  {entry.role === "agent" ? <span className="cairn-bubble-text">{entry.text}</span> : entry.text}
-                </div>
-              ))}
+                  {historyExpanded ? <ChevronUp size={12} /> : <ChevronDown size={12} />}
+                  {historyExpanded ? "Hide earlier" : `${transcript.length} earlier`}
+                </button>
+              )}
+              {historyExpanded &&
+                transcript.map((entry) => (
+                  <div
+                    className={entry.role === "user" ? "cairn-bubble cairn-bubble-user cairn-bubble-past" : "cairn-bubble cairn-bubble-agent cairn-bubble-past"}
+                    key={entry.id}
+                  >
+                    {entry.role === "agent" ? <span className="cairn-bubble-text">{entry.text}</span> : entry.text}
+                  </div>
+                ))}
               {userCaption && (
                 <div className="cairn-bubble cairn-bubble-user" key={`u-${userCaption}`}>
                   {userCaption}
@@ -1184,6 +1311,26 @@ const COPILOT_STYLES = `
   text-transform: uppercase;
   color: rgba(11, 13, 18, 0.48);
 }
+.cairn-history-toggle {
+  align-self: center;
+  display: inline-flex;
+  align-items: center;
+  gap: 3px;
+  border: none;
+  background: none;
+  padding: 2px 8px;
+  font: inherit;
+  font-size: 11px;
+  font-weight: 600;
+  color: rgba(11, 13, 18, 0.4);
+  cursor: pointer;
+  border-radius: 999px;
+  transition: background 0.15s ease, color 0.15s ease;
+}
+.cairn-history-toggle:hover {
+  background: rgba(11, 13, 18, 0.05);
+  color: rgba(11, 13, 18, 0.6);
+}
 .cairn-thinking {
   display: inline-flex;
   gap: 4px;

package/src/realtime-server.ts

@@ -27,7 +27,7 @@
 
 import http from "node:http";
 import { WebSocket, WebSocketServer } from "ws";
-import type { HistoryTurn, Manifest, VerbResponse } from "@cairnvibe/core";
+import type { HistoryTurn, LiveElement, Manifest, VerbResponse } from "@cairnvibe/core";
 import { buildSystemPrompt, createVerbLLM, resolveVerb, type CapabilityTier, type CreateCopilotHandlerOptions } from "./server";
 import { DeepgramSpeakStream } from "./tts-stream";
 
@@ -95,7 +95,7 @@ export function createRealtimeServer(options: CreateRealtimeServerOptions): http
   return httpServer;
 }
 
-interface ConnectionDeps {
+export interface ConnectionDeps {
   deepgramApiKey: string;
   sttModel: string;
   ttsVoice: string;
@@ -109,7 +109,10 @@ interface ConnectionDeps {
 const MAX_HISTORY_TURNS = 8; // 4 exchanges — enough for "the first one"/"do that instead" without growing the prompt unbounded over a long call
 
 async function handleConnection(client: WebSocket, deps: ConnectionDeps): Promise<void> {
-  let context = { route: "/", visible: [] as string[] };
+  // liveElements refreshes on every "context" resend (the client sends one
+  // on route changes and each time it's about to start listening again),
+  // so a live scan from several turns ago never lingers into a later one.
+  let context: { route: string; visible: string[]; liveElements: LiveElement[] } = { route: "/", visible: [], liveElements: [] };
   // Unlike the stateless HTTP path (which needs the client to resend
   // history every request), a realtime connection is already stateful —
   // one WebSocket per call — so this is accumulated here directly rather
@@ -202,8 +205,13 @@ async function handleConnection(client: WebSocket, deps: ConnectionDeps): Promis
     });
   }
 
+  // Accumulates Deepgram "Results" transcript segments across one utterance
+  // — see handleDeepgramMessage for why this can't just react to every
+  // is_final.
+  const turnState = { buffer: "" };
+
   dg.on("message", (data) => {
-    void handleDeepgramMessage(data.toString(), client, deps, () => context, speakStreamed, history);
+    void handleDeepgramMessage(data.toString(), client, deps, () => context, speakStreamed, history, turnState, () => generation);
   });
 
   dg.on("error", (err) => {
@@ -221,7 +229,11 @@ async function handleConnection(client: WebSocket, deps: ConnectionDeps): Promis
     try {
       const msg = JSON.parse(data.toString());
       if (msg.type === "context") {
-        context = { route: String(msg.route ?? "/"), visible: Array.isArray(msg.visible) ? msg.visible : [] };
+        context = {
+          route: String(msg.route ?? "/"),
+          visible: Array.isArray(msg.visible) ? msg.visible : [],
+          liveElements: parseLiveElements(msg.liveElements),
+        };
       } else if (msg.type === "end") {
         client.close();
       } else if (msg.type === "barge_in") {
@@ -233,7 +245,20 @@ async function handleConnection(client: WebSocket, deps: ConnectionDeps): Promis
         // client falling back to a separate buffered REST call. No STT/verb
         // resolution involved; the client already resolved the tour steps
         // itself and just needs this text spoken.
-        void speakStreamed(msg.text);
+        //
+        // Caught explicitly, unlike a normal turn's speakStreamed call (see
+        // handleDeepgramMessage) — this one isn't inside that function's own
+        // try/catch, and an uncaught rejection here previously vanished
+        // silently: the client's speakOverRealtime() promise for this step
+        // never resolves except via its own 15s fallback timeout, with
+        // nothing telling the user anything went wrong in the meantime —
+        // found live as a tour that goes badly quiet for stretches at a
+        // time. A real "error" message lets the client's tour-step handler
+        // (index.tsx's ws.onmessage) unstick itself immediately instead.
+        speakStreamed(msg.text).catch((err) => {
+          console.error("[cairn realtime] speakStreamed failed for a tour step:", err);
+          safeSend(client, { type: "error", message: "Something went wrong narrating that step." });
+        });
       }
     } catch {
       // Ignore malformed control messages — never crash the relay on bad client input.
@@ -250,13 +275,15 @@ async function handleConnection(client: WebSocket, deps: ConnectionDeps): Promis
   });
 }
 
-async function handleDeepgramMessage(
+export async function handleDeepgramMessage(
   raw: string,
   client: WebSocket,
   deps: ConnectionDeps,
-  getContext: () => { route: string; visible: string[] },
+  getContext: () => { route: string; visible: string[]; liveElements: LiveElement[] },
   speakStreamed: (text: string) => Promise<void>,
   history: HistoryTurn[],
+  turnState: { buffer: string },
+  getGeneration: () => number,
 ): Promise<void> {
   let msg: any;
   try {
@@ -265,34 +292,86 @@ async function handleDeepgramMessage(
     return;
   }
 
+  if (msg.type === "UtteranceEnd") {
+    // A second, independent "the user is truly done" signal Deepgram sends
+    // after utterance_end_ms of silence — a safety net for the rare case a
+    // Results message never carries speech_final:true, so a turn can't get
+    // permanently stuck with real transcript sitting in the buffer forever.
+    if (turnState.buffer) await finalizeTurn(turnState, client, deps, getContext, speakStreamed, history, getGeneration);
+    return;
+  }
+
   if (msg.type !== "Results") return;
   const transcript: string | undefined = msg.channel?.alternatives?.[0]?.transcript;
   if (!transcript) return;
 
   if (!msg.is_final) {
-    safeSend(client, { type: "interim", text: transcript });
+    safeSend(client, { type: "interim", text: turnState.buffer ? `${turnState.buffer} ${transcript}` : transcript });
     return;
   }
 
+  // is_final means this chunk of transcript is stable and won't be
+  // revised — it does NOT mean the user is done talking. Deepgram can (and
+  // routinely does) finalize several chunks of one continuous utterance in
+  // a row with no real pause between them. Only speech_final (endpointing
+  // actually detected a pause) means the turn is genuinely over. Found
+  // live, not theoretical: treating every is_final as a separate finished
+  // question fired two independent LLM+TTS turns for one utterance — the
+  // literal cause of both the duplicated transcript entries ("hello" /
+  // "hello" with no reply in between) and the agent audibly speaking
+  // twice, overlapping.
+  turnState.buffer = turnState.buffer ? `${turnState.buffer} ${transcript}` : transcript;
+  if (!msg.speech_final) {
+    safeSend(client, { type: "interim", text: turnState.buffer });
+    return;
+  }
+
+  await finalizeTurn(turnState, client, deps, getContext, speakStreamed, history, getGeneration);
+}
+
+/**
+ * Everything from here on (the LLM call, TTS streaming) can fail in ways
+ * that have nothing to do with a malformed message — a flaky provider
+ * call, a rate limit, a dropped upstream connection. handleDeepgramMessage
+ * is invoked fire-and-forget (`void handleDeepgramMessage(...)`), so an
+ * uncaught throw here previously vanished into an unhandled rejection: the
+ * client had already been told "final" (entering its "thinking" state) and
+ * then simply never heard from the server again for this turn — stuck
+ * indefinitely with the mic never resuming. Every path out of the try
+ * block now sends the client something that ends the turn.
+ *
+ * myGeneration is captured before the (potentially slow) LLM call and
+ * re-checked before the verb/speech actually goes out — a barge-in that
+ * happens while this turn is still "thinking" bumps the generation, and
+ * without this check the now-stale response would still land on the
+ * client after the user had already moved on to a new question.
+ */
+async function finalizeTurn(
+  turnState: { buffer: string },
+  client: WebSocket,
+  deps: ConnectionDeps,
+  getContext: () => { route: string; visible: string[]; liveElements: LiveElement[] },
+  speakStreamed: (text: string) => Promise<void>,
+  history: HistoryTurn[],
+  getGeneration: () => number,
+): Promise<void> {
+  const transcript = turnState.buffer;
+  turnState.buffer = "";
+  const myGeneration = getGeneration();
   safeSend(client, { type: "final", text: transcript });
 
-  // Everything from here on (the LLM call, TTS streaming) can fail in ways
-  // that have nothing to do with a malformed message — a flaky provider
-  // call, a rate limit, a dropped upstream connection. This whole function
-  // is invoked fire-and-forget (`void handleDeepgramMessage(...)`), so an
-  // uncaught throw here previously vanished into an unhandled rejection:
-  // the client had already been told "final" (entering its "thinking"
-  // state) and then simply never heard from the server again for this
-  // turn — stuck indefinitely with the mic never resuming. Every path out
-  // of this try block now sends the client something that ends the turn.
   try {
-    const { route, visible } = getContext();
+    const { route, visible, liveElements } = getContext();
     const verb = await resolveVerb(deps.llm, deps.systemPrompt, deps.manifest, deps.registeredActions, deps.capability, {
       route,
       question: transcript,
       visible,
+      liveElements,
       history,
     });
+
+    if (myGeneration !== getGeneration()) return; // superseded by a barge-in while this turn was resolving
+
     // Sent immediately — before speech synthesis even starts — so
     // highlight/navigate/do execute in the browser right away instead of
     // waiting on audio. The agent visibly acts while it's still about to
@@ -312,8 +391,10 @@ async function handleDeepgramMessage(
     }
   } catch (err) {
     console.error("[cairn realtime] failed to resolve/speak this turn:", err);
-    safeSend(client, { type: "error", message: "Something went wrong answering that — try again." });
-    safeSend(client, { type: "turn_complete" });
+    if (myGeneration === getGeneration()) {
+      safeSend(client, { type: "error", message: "Something went wrong answering that — try again." });
+      safeSend(client, { type: "turn_complete" });
+    }
   }
 }
 
@@ -322,6 +403,28 @@ function safeSend(client: WebSocket, message: ServerMessage): void {
   client.send(JSON.stringify(message));
 }
 
+/** Defensive parse for the client's self-reported live DOM scan — same
+ * untrusted-input treatment `visible` already gets on this control-message
+ * path (no CopilotRequestSchema here, unlike the HTTP handler), just
+ * shaped-checked so a malformed entry can't reach the LLM prompt oddly. */
+function parseLiveElements(raw: unknown): LiveElement[] {
+  if (!Array.isArray(raw)) return [];
+  const elements: LiveElement[] = [];
+  for (const entry of raw) {
+    if (
+      entry &&
+      typeof entry === "object" &&
+      typeof (entry as any).id === "string" &&
+      typeof (entry as any).role === "string" &&
+      typeof (entry as any).label === "string"
+    ) {
+      elements.push({ id: (entry as any).id, role: (entry as any).role, label: (entry as any).label });
+    }
+    if (elements.length >= 60) break;
+  }
+  return elements;
+}
+
 /** A short text form of any verb for the history log — not shown to the
  * user, just fed back to the model on later turns so it knows what it
  * already did/said. */