@cairnvibe/sdk 0.2.5 → 0.2.7

package/dist/index.js

@@ -8,11 +8,28 @@ const navigation_1 = require("next/navigation");
 const lucide_react_1 = require("lucide-react");
 const context_collector_1 = require("./context-collector");
 const element_ladder_1 = require("./element-ladder");
+const runtime_scan_1 = require("./runtime-scan");
 const verb_executor_1 = require("./verb-executor");
 function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, reportMissesEndpoint, transcribeEndpoint, speakEndpoint, realtimeUrl, persona = "Cairn", }) {
     const pathname = (0, navigation_1.usePathname)() ?? "/";
+    // Mirrors `pathname` for use inside long-lived closures (a realtime
+    // session's handlers are all created once, when the connection opens —
+    // same staleness reason runTour tracks its own `currentRoute` locally
+    // rather than trusting its closure's `pathname` after a mid-tour
+    // navigation).
+    const pathnameRef = (0, react_1.useRef)(pathname);
+    (0, react_1.useEffect)(() => {
+        pathnameRef.current = pathname;
+        sendFreshContext(); // no-op if no realtime session is open
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [pathname]);
     const router = (0, navigation_1.useRouter)();
     const [open, setOpen] = (0, react_1.useState)(false);
+    // Collapsed by default so the panel only ever shows the current exchange
+    // — the full archived transcript (built up over a long conversation)
+    // stays out of the way behind an explicit toggle instead of always being
+    // visible inline, which made the panel grow uncomfortably tall.
+    const [historyExpanded, setHistoryExpanded] = (0, react_1.useState)(false);
     const [question, setQuestion] = (0, react_1.useState)("");
     const [answer, setAnswer] = (0, react_1.useState)(null);
     const [status, setStatus] = (0, react_1.useState)("idle");
@@ -40,16 +57,19 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
     const [rtMicMuted, setRtMicMuted] = (0, react_1.useState)(false);
     const [rtSpeakerMuted, setRtSpeakerMuted] = (0, react_1.useState)(false);
     // Set while a "tour" verb's steps are being narrated/highlighted one at a
-    // time — drives the step-progress caption and blocks the input/mic so a
-    // typed or spoken question can't interrupt mid-walkthrough.
+    // time — drives the step-progress caption and blocks the *typed* input so
+    // a typed question can't collide with the walkthrough (a voice
+    // interruption is handled separately — see touringRef/triggerBargeIn).
     const [tourStep, setTourStep] = (0, react_1.useState)(null);
-    const tourGenerationRef = (0, react_1.useRef)(0); // bumped to cancel an in-progress tour (e.g. widget closed) without extra flags
+    const tourGenerationRef = (0, react_1.useRef)(0); // bumped to cancel an in-progress tour (e.g. widget closed, or a voice barge-in) without extra flags
     // Mirrors whether a tour is running, for use inside the mic's
     // onaudioprocess callback (a stale closure over React state there would
     // miss a tour that started after the callback was created) — a tour
-    // reuses "rt-speaking" to hold the mic off too, but must NOT be
-    // barge-in-able the way a real conversational reply is (see the RMS
-    // check below): it's a deliberate walkthrough, not a turn to interrupt.
+    // reuses "rt-speaking" to hold the mic off between steps, but IS
+    // barge-in-able like a real conversational reply (see the RMS check
+    // below): interrupting mid-tour cancels the rest of the walkthrough,
+    // the way a real person giving a tour stops when you have a question
+    // instead of talking over you.
     const touringRef = (0, react_1.useRef)(false);
     // Resolver for "this tour step's audio has fully finished playing" when
     // narrating over an already-open realtime session (see maybeResumeListening
@@ -62,6 +82,19 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
     // keeps its own history server-side instead, since that connection is
     // already stateful).
     const historyRef = (0, react_1.useRef)([]);
+    // A background scanner that keeps a live inventory of what's actually
+    // clickable on screen right now (runtime-scan.ts) — running continuously
+    // via a MutationObserver so there's never a pause to "go look at the
+    // page" right when a verb needs to click something. `liveMapRef` freezes
+    // one snapshot of it per turn (set alongside every context/question send,
+    // below) so a background rescan landing mid-flight can't shift what an id
+    // resolves to between when a request went out and its response came back.
+    const liveRegistryRef = (0, react_1.useRef)((0, runtime_scan_1.createLiveElementRegistry)());
+    const liveMapRef = (0, react_1.useRef)(new Map());
+    (0, react_1.useEffect)(() => {
+        liveRegistryRef.current.start();
+        return () => liveRegistryRef.current.stop();
+    }, []);
     const mediaRecorderRef = (0, react_1.useRef)(null);
     const audioChunksRef = (0, react_1.useRef)([]);
     const transcribeInFlightRef = (0, react_1.useRef)(false);
@@ -149,6 +182,26 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
         rtStateRef.current = next;
         setStatus(next);
     }
+    /**
+     * Refreshes the server's picture of route/visible/liveElements over an
+     * already-open realtime connection. Beyond the initial connect, called
+     * on every route change and whenever the mic is about to start listening
+     * again — a real, pre-existing gap this closes as a side effect: the
+     * server's context previously updated only once, at connection open, so
+     * navigating mid-call (via a "navigate" verb, or the user clicking
+     * around) left the server answering every later turn as if the user were
+     * still on the original page. Reads pathnameRef, not the closure's
+     * `pathname`, so it's correct even called from a handler created once at
+     * connection-open time.
+     */
+    function sendFreshContext() {
+        const ws = rtSocketRef.current;
+        if (!ws || ws.readyState !== WebSocket.OPEN)
+            return;
+        const liveScan = liveRegistryRef.current.getSnapshot();
+        liveMapRef.current = liveScan.byId;
+        ws.send(JSON.stringify({ type: "context", route: pathnameRef.current, visible: (0, context_collector_1.collectVisible)(), liveElements: liveScan.elements }));
+    }
     function reportMiss(context) {
         (0, element_ladder_1.logMiss)(context);
         if (reportMissesEndpoint) {
@@ -171,6 +224,7 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
             onDo,
             onTour: (steps) => void runTour(steps),
             registeredActions,
+            liveElements: liveMapRef.current,
         });
     }
     /**
@@ -180,12 +234,14 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
      * actually showing you around instead of one paragraph naming several
      * buttons at once with nothing highlighted.
      *
-     * Always narrates via speakEndpoint (plain request/response TTS), even
-     * during a live realtime session — a tour is a distinct guided
-     * walkthrough, not a conversational turn, so it doesn't need the
-     * streaming relay's turn-taking machinery. If the widget is mid
-     * realtime call, the mic is held off (mirrors "rt-speaking") for the
-     * tour's duration so it can't pick up the tour's own narration.
+     * During a live realtime session, narration reuses the same streaming
+     * Speak connection a normal conversational reply uses (see
+     * speakOverRealtime below) instead of a separate buffered REST call —
+     * otherwise falls back to speakEndpoint. Either way, the mic is held off
+     * between steps (mirrors "rt-speaking") so it doesn't pick up the tour's
+     * own narration — but it's still listening for a real interruption:
+     * talking during a step cancels the rest of the tour via triggerBargeIn,
+     * the same as interrupting a normal spoken reply.
      */
     async function runTour(steps) {
         const myGeneration = ++tourGenerationRef.current;
@@ -231,11 +287,25 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
                         return;
                 }
                 if (step.target) {
-                    const el = (0, element_ladder_1.findElement)(step.target);
-                    if (el)
+                    // A fresh scan, not the tour's starting liveMapRef snapshot — a
+                    // step after a mid-tour navigation targets elements on a page
+                    // that didn't exist when the tour began.
+                    const liveScan = liveRegistryRef.current.getSnapshot();
+                    const el = (0, element_ladder_1.findElement)(step.target, liveScan.byId);
+                    if (el) {
                         (0, element_ladder_1.highlightElement)(el);
-                    else
+                        if (step.click) {
+                            el.click();
+                            // Give whatever the click reveals (a detail view, an expanded
+                            // row) a moment to actually render before narrating it.
+                            await new Promise((resolve) => setTimeout(resolve, 400));
+                            if (tourGenerationRef.current !== myGeneration)
+                                return;
+                        }
+                    }
+                    else {
                         reportMiss({ attempted: step.target, route: currentRoute });
+                    }
                 }
                 if (wasRealtimeListening && rtSocketRef.current?.readyState === WebSocket.OPEN) {
                     // Already have a live streaming connection open — reuse it
@@ -276,10 +346,18 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
         setLastQuestion(q);
         setQuestion("");
         try {
+            const liveScan = liveRegistryRef.current.getSnapshot();
+            liveMapRef.current = liveScan.byId;
             const res = await fetch(endpoint, {
                 method: "POST",
                 headers: { "content-type": "application/json" },
-                body: JSON.stringify({ route: pathname, question: q, visible: (0, context_collector_1.collectVisible)(), history: historyRef.current }),
+                body: JSON.stringify({
+                    route: pathname,
+                    question: q,
+                    visible: (0, context_collector_1.collectVisible)(),
+                    history: historyRef.current,
+                    liveElements: liveScan.elements,
+                }),
             });
             const data = await res.json().catch(() => null);
             handleVerb(data);
@@ -503,12 +581,20 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
                     return;
                 if (rtMicMutedRef.current)
                     return;
-                // Barge-in: while the agent is speaking a real conversational
-                // reply (not touring — a tour deliberately can't be talked over),
-                // keep listening to the mic locally even though it isn't being
-                // sent yet, and cut the agent off the instant the user starts
-                // talking over it instead of making them wait for it to finish.
-                if (rtStateRef.current === "rt-speaking" && !touringRef.current) {
+                // Barge-in: while the agent is speaking a real conversational reply,
+                // still thinking about one, OR mid-tour, keep listening to the mic
+                // locally even though it isn't being sent yet, and cut the agent
+                // off the instant the user starts talking again instead of making
+                // them wait — including during a guided tour, which now cancels the
+                // rest of the walkthrough on interruption (see triggerBargeIn)
+                // instead of being talked-over-proof by design, the way a real
+                // person giving a tour stops when you have a question. The
+                // "rt-thinking" half matters just as much as "rt-speaking": an LLM
+                // turn can easily take a couple of seconds with nothing playing
+                // yet, and without this the mic was completely deaf during that
+                // whole window — found live as "not listening while speaking... no
+                // interrupting system", not just a missed nice-to-have.
+                if (rtStateRef.current === "rt-speaking" || rtStateRef.current === "rt-thinking") {
                     const rms = computeRms(e.inputBuffer.getChannelData(0));
                     if (rms > BARGE_IN_RMS_THRESHOLD)
                         triggerBargeIn();
@@ -555,6 +641,7 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
                 }
                 setRtStatus("rt-listening");
                 setCaption("");
+                sendFreshContext(); // refresh before the user starts talking again, not after
             }
             function disarmThinkingWatchdog() {
                 if (rtThinkingWatchdogRef.current) {
@@ -593,13 +680,26 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
                 disarmThinkingWatchdog();
                 stopScheduledRtAudio();
                 rtAudioDoneArrivingRef.current = true;
+                if (touringRef.current) {
+                    // Interrupting mid-guide cancels the whole rest of the tour, not
+                    // just the current step — the way a real person giving a tour
+                    // stops and answers your question instead of continuing to talk
+                    // over you. Without resolving the current step's own pending
+                    // promise here, runTour only notices the cancellation via its own
+                    // 15s-per-step fallback timeout instead of right away.
+                    tourGenerationRef.current++;
+                    touringRef.current = false;
+                    setTourStep(null);
+                    rtTourAudioDoneRef.current?.();
+                    rtTourAudioDoneRef.current = null;
+                }
                 if (ws.readyState === WebSocket.OPEN)
                     ws.send(JSON.stringify({ type: "barge_in" }));
                 setRtStatus("rt-listening");
                 setCaption("");
             }
             ws.onopen = () => {
-                ws.send(JSON.stringify({ type: "context", route: pathname, visible: (0, context_collector_1.collectVisible)() }));
+                sendFreshContext();
                 setRtStatus("rt-listening");
                 rtStartingRef.current = false;
             };
@@ -672,7 +772,18 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
                     // exactly the way a silently-dropped response used to leave it.
                     disarmThinkingWatchdog();
                     setAnswer(msg.message ?? "Something went wrong.");
-                    if (!touringRef.current) {
+                    if (touringRef.current) {
+                        // A tour step's own speakStreamed() failed server-side (see
+                        // realtime-server.ts's "speak" handler). Without resolving this
+                        // step's pending promise here, runTour's `await
+                        // speakOverRealtime(step.text)` only recovers via its own 15s
+                        // fallback timeout — found live as a guide that goes badly
+                        // quiet for long stretches, one step at a time.
+                        rtAudioDoneArrivingRef.current = true;
+                        rtTourAudioDoneRef.current?.();
+                        rtTourAudioDoneRef.current = null;
+                    }
+                    else {
                         setRtStatus("rt-listening");
                         setCaption("");
                     }
@@ -742,7 +853,8 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
         "rt-thinking": "Thinking…",
         "rt-speaking": "Speaking…",
     };
-    return ((0, jsx_runtime_1.jsxs)(jsx_runtime_1.Fragment, { children: [(0, jsx_runtime_1.jsx)("style", { suppressHydrationWarning: true, dangerouslySetInnerHTML: { __html: COPILOT_STYLES } }), (0, jsx_runtime_1.jsx)("button", { className: status === "rt-speaking" ? "cairn-fab cairn-fab-speaking" : "cairn-fab", "aria-label": open ? `Close ${persona} help` : `Open ${persona} help`, onClick: () => setOpen((v) => !v), children: open ? (0, jsx_runtime_1.jsx)(lucide_react_1.X, { size: 22 }) : (0, jsx_runtime_1.jsx)(CairnMark, {}) }), open && ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-panel", role: "dialog", "aria-label": `${persona} help panel`, ref: panelRef, children: [(transcript.length > 0 || userCaption || answer || busy) && ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-stack", children: [transcript.map((entry) => ((0, jsx_runtime_1.jsx)("div", { className: entry.role === "user" ? "cairn-bubble cairn-bubble-user cairn-bubble-past" : "cairn-bubble cairn-bubble-agent cairn-bubble-past", children: entry.role === "agent" ? (0, jsx_runtime_1.jsx)("span", { className: "cairn-bubble-text", children: entry.text }) : entry.text }, entry.id))), userCaption && ((0, jsx_runtime_1.jsx)("div", { className: "cairn-bubble cairn-bubble-user", children: userCaption }, `u-${userCaption}`)), (answer || busy) && ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-bubble cairn-bubble-agent", children: [tourChip && (0, jsx_runtime_1.jsx)("span", { className: "cairn-chip", children: tourChip }), answer ? ((0, jsx_runtime_1.jsx)("span", { className: "cairn-bubble-text", children: renderCaptionWords(answer) })) : ((0, jsx_runtime_1.jsxs)("span", { className: "cairn-thinking", "aria-label": "Thinking", children: [(0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" })] }))] }, `a-${answer ?? status}`))] })), realtimeActive ? ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-rt-bar", children: [(0, jsx_runtime_1.jsx)("span", { className: `cairn-rt-dot cairn-rt-dot-${status}` }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-rt-label", children: statusLabel[status] }), (0, jsx_runtime_1.jsxs)("div", { className: "cairn-rt-controls", children: [(0, jsx_runtime_1.jsx)("button", { type: "button", className: status === "rt-speaking" ? "cairn-icon-btn cairn-icon-btn-speaking" : "cairn-icon-btn", "aria-label": rtMicMuted ? "Unmute microphone" : "Mute microphone", onClick: toggleRtMic, children: rtMicMuted ? (0, jsx_runtime_1.jsx)(lucide_react_1.MicOff, { size: 16 }) : (0, jsx_runtime_1.jsx)(lucide_react_1.Mic, { size: 16 }) }), (0, jsx_runtime_1.jsx)("button", { type: "button", className: status === "rt-speaking" ? "cairn-icon-btn cairn-icon-btn-speaking" : "cairn-icon-btn", "aria-label": rtSpeakerMuted ? "Unmute speaker" : "Mute speaker", onClick: toggleRtSpeaker, children: rtSpeakerMuted ? (0, jsx_runtime_1.jsx)(lucide_react_1.VolumeX, { size: 16 }) : (0, jsx_runtime_1.jsx)(lucide_react_1.Volume2, { size: 16 }) }), (0, jsx_runtime_1.jsx)("button", { type: "button", className: "cairn-icon-btn cairn-icon-btn-end", "aria-label": "End conversation", onClick: endRealtime, children: (0, jsx_runtime_1.jsx)(lucide_react_1.PhoneOff, { size: 16 }) })] })] })) : ((0, jsx_runtime_1.jsx)("form", { onSubmit: (e) => {
+    return ((0, jsx_runtime_1.jsxs)(jsx_runtime_1.Fragment, { children: [(0, jsx_runtime_1.jsx)("style", { suppressHydrationWarning: true, dangerouslySetInnerHTML: { __html: COPILOT_STYLES } }), (0, jsx_runtime_1.jsx)("button", { className: status === "rt-speaking" ? "cairn-fab cairn-fab-speaking" : "cairn-fab", "aria-label": open ? `Close ${persona} help` : `Open ${persona} help`, onClick: () => setOpen((v) => !v), children: open ? (0, jsx_runtime_1.jsx)(lucide_react_1.X, { size: 22 }) : (0, jsx_runtime_1.jsx)(CairnMark, {}) }), open && ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-panel", role: "dialog", "aria-label": `${persona} help panel`, ref: panelRef, children: [(transcript.length > 0 || userCaption || answer || busy) && ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-stack", children: [transcript.length > 0 && ((0, jsx_runtime_1.jsxs)("button", { type: "button", className: "cairn-history-toggle", onClick: () => setHistoryExpanded((v) => !v), "aria-expanded": historyExpanded, children: [historyExpanded ? (0, jsx_runtime_1.jsx)(lucide_react_1.ChevronUp, { size: 12 }) : (0, jsx_runtime_1.jsx)(lucide_react_1.ChevronDown, { size: 12 }), historyExpanded ? "Hide earlier" : `${transcript.length} earlier`] })), historyExpanded &&
+                                transcript.map((entry) => ((0, jsx_runtime_1.jsx)("div", { className: entry.role === "user" ? "cairn-bubble cairn-bubble-user cairn-bubble-past" : "cairn-bubble cairn-bubble-agent cairn-bubble-past", children: entry.role === "agent" ? (0, jsx_runtime_1.jsx)("span", { className: "cairn-bubble-text", children: entry.text }) : entry.text }, entry.id))), userCaption && ((0, jsx_runtime_1.jsx)("div", { className: "cairn-bubble cairn-bubble-user", children: userCaption }, `u-${userCaption}`)), (answer || busy) && ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-bubble cairn-bubble-agent", children: [tourChip && (0, jsx_runtime_1.jsx)("span", { className: "cairn-chip", children: tourChip }), answer ? ((0, jsx_runtime_1.jsx)("span", { className: "cairn-bubble-text", children: renderCaptionWords(answer) })) : ((0, jsx_runtime_1.jsxs)("span", { className: "cairn-thinking", "aria-label": "Thinking", children: [(0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" })] }))] }, `a-${answer ?? status}`))] })), realtimeActive ? ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-rt-bar", children: [(0, jsx_runtime_1.jsx)("span", { className: `cairn-rt-dot cairn-rt-dot-${status}` }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-rt-label", children: statusLabel[status] }), (0, jsx_runtime_1.jsxs)("div", { className: "cairn-rt-controls", children: [(0, jsx_runtime_1.jsx)("button", { type: "button", className: status === "rt-speaking" ? "cairn-icon-btn cairn-icon-btn-speaking" : "cairn-icon-btn", "aria-label": rtMicMuted ? "Unmute microphone" : "Mute microphone", onClick: toggleRtMic, children: rtMicMuted ? (0, jsx_runtime_1.jsx)(lucide_react_1.MicOff, { size: 16 }) : (0, jsx_runtime_1.jsx)(lucide_react_1.Mic, { size: 16 }) }), (0, jsx_runtime_1.jsx)("button", { type: "button", className: status === "rt-speaking" ? "cairn-icon-btn cairn-icon-btn-speaking" : "cairn-icon-btn", "aria-label": rtSpeakerMuted ? "Unmute speaker" : "Mute speaker", onClick: toggleRtSpeaker, children: rtSpeakerMuted ? (0, jsx_runtime_1.jsx)(lucide_react_1.VolumeX, { size: 16 }) : (0, jsx_runtime_1.jsx)(lucide_react_1.Volume2, { size: 16 }) }), (0, jsx_runtime_1.jsx)("button", { type: "button", className: "cairn-icon-btn cairn-icon-btn-end", "aria-label": "End conversation", onClick: endRealtime, children: (0, jsx_runtime_1.jsx)(lucide_react_1.PhoneOff, { size: 16 }) })] })] })) : ((0, jsx_runtime_1.jsx)("form", { onSubmit: (e) => {
                             e.preventDefault();
                             const trimmed = question.trim();
                             if (trimmed)
@@ -994,6 +1106,26 @@ const COPILOT_STYLES = `
   text-transform: uppercase;
   color: rgba(11, 13, 18, 0.48);
 }
+.cairn-history-toggle {
+  align-self: center;
+  display: inline-flex;
+  align-items: center;
+  gap: 3px;
+  border: none;
+  background: none;
+  padding: 2px 8px;
+  font: inherit;
+  font-size: 11px;
+  font-weight: 600;
+  color: rgba(11, 13, 18, 0.4);
+  cursor: pointer;
+  border-radius: 999px;
+  transition: background 0.15s ease, color 0.15s ease;
+}
+.cairn-history-toggle:hover {
+  background: rgba(11, 13, 18, 0.05);
+  color: rgba(11, 13, 18, 0.6);
+}
 .cairn-thinking {
   display: inline-flex;
   gap: 4px;

package/dist/realtime-server.d.ts

@@ -1,6 +1,7 @@
 import http from "node:http";
-import type { Manifest } from "@cairnvibe/core";
-import { type CreateCopilotHandlerOptions } from "./server";
+import { WebSocket } from "ws";
+import type { HistoryTurn, LiveElement, Manifest } from "@cairnvibe/core";
+import { createVerbLLM, type CapabilityTier, type CreateCopilotHandlerOptions } from "./server";
 export interface CreateRealtimeServerOptions extends CreateCopilotHandlerOptions {
     manifest: Manifest;
     deepgramApiKey: string;
@@ -8,3 +9,20 @@ export interface CreateRealtimeServerOptions extends CreateCopilotHandlerOptions
     ttsVoice?: string;
 }
 export declare function createRealtimeServer(options: CreateRealtimeServerOptions): http.Server;
+export interface ConnectionDeps {
+    deepgramApiKey: string;
+    sttModel: string;
+    ttsVoice: string;
+    llm: ReturnType<typeof createVerbLLM>;
+    systemPrompt: string;
+    manifest: Manifest;
+    registeredActions: string[];
+    capability: CapabilityTier;
+}
+export declare function handleDeepgramMessage(raw: string, client: WebSocket, deps: ConnectionDeps, getContext: () => {
+    route: string;
+    visible: string[];
+    liveElements: LiveElement[];
+}, speakStreamed: (text: string) => Promise<void>, history: HistoryTurn[], turnState: {
+    buffer: string;
+}, getGeneration: () => number): Promise<void>;

package/dist/realtime-server.js

@@ -30,6 +30,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createRealtimeServer = createRealtimeServer;
+exports.handleDeepgramMessage = handleDeepgramMessage;
 const node_http_1 = __importDefault(require("node:http"));
 const ws_1 = require("ws");
 const server_1 = require("./server");
@@ -73,7 +74,10 @@ function createRealtimeServer(options) {
 }
 const MAX_HISTORY_TURNS = 8; // 4 exchanges — enough for "the first one"/"do that instead" without growing the prompt unbounded over a long call
 async function handleConnection(client, deps) {
-    let context = { route: "/", visible: [] };
+    // liveElements refreshes on every "context" resend (the client sends one
+    // on route changes and each time it's about to start listening again),
+    // so a live scan from several turns ago never lingers into a later one.
+    let context = { route: "/", visible: [], liveElements: [] };
     // Unlike the stateless HTTP path (which needs the client to resend
     // history every request), a realtime connection is already stateful —
     // one WebSocket per call — so this is accumulated here directly rather
@@ -155,8 +159,12 @@ async function handleConnection(client, deps) {
             stream.flush();
         });
     }
+    // Accumulates Deepgram "Results" transcript segments across one utterance
+    // — see handleDeepgramMessage for why this can't just react to every
+    // is_final.
+    const turnState = { buffer: "" };
     dg.on("message", (data) => {
-        void handleDeepgramMessage(data.toString(), client, deps, () => context, speakStreamed, history);
+        void handleDeepgramMessage(data.toString(), client, deps, () => context, speakStreamed, history, turnState, () => generation);
     });
     dg.on("error", (err) => {
         console.error("[cairn realtime] Deepgram STT connection error:", err);
@@ -174,7 +182,11 @@ async function handleConnection(client, deps) {
         try {
             const msg = JSON.parse(data.toString());
             if (msg.type === "context") {
-                context = { route: String(msg.route ?? "/"), visible: Array.isArray(msg.visible) ? msg.visible : [] };
+                context = {
+                    route: String(msg.route ?? "/"),
+                    visible: Array.isArray(msg.visible) ? msg.visible : [],
+                    liveElements: parseLiveElements(msg.liveElements),
+                };
             }
             else if (msg.type === "end") {
                 client.close();
@@ -189,7 +201,20 @@ async function handleConnection(client, deps) {
                 // client falling back to a separate buffered REST call. No STT/verb
                 // resolution involved; the client already resolved the tour steps
                 // itself and just needs this text spoken.
-                void speakStreamed(msg.text);
+                //
+                // Caught explicitly, unlike a normal turn's speakStreamed call (see
+                // handleDeepgramMessage) — this one isn't inside that function's own
+                // try/catch, and an uncaught rejection here previously vanished
+                // silently: the client's speakOverRealtime() promise for this step
+                // never resolves except via its own 15s fallback timeout, with
+                // nothing telling the user anything went wrong in the meantime —
+                // found live as a tour that goes badly quiet for stretches at a
+                // time. A real "error" message lets the client's tour-step handler
+                // (index.tsx's ws.onmessage) unstick itself immediately instead.
+                speakStreamed(msg.text).catch((err) => {
+                    console.error("[cairn realtime] speakStreamed failed for a tour step:", err);
+                    safeSend(client, { type: "error", message: "Something went wrong narrating that step." });
+                });
             }
         }
         catch {
@@ -206,7 +231,7 @@ async function handleConnection(client, deps) {
         speakStream?.close();
     });
 }
-async function handleDeepgramMessage(raw, client, deps, getContext, speakStreamed, history) {
+async function handleDeepgramMessage(raw, client, deps, getContext, speakStreamed, history, turnState, getGeneration) {
     let msg;
     try {
         msg = JSON.parse(raw);
@@ -214,33 +239,74 @@ async function handleDeepgramMessage(raw, client, deps, getContext, speakStreame
     catch {
         return;
     }
+    if (msg.type === "UtteranceEnd") {
+        // A second, independent "the user is truly done" signal Deepgram sends
+        // after utterance_end_ms of silence — a safety net for the rare case a
+        // Results message never carries speech_final:true, so a turn can't get
+        // permanently stuck with real transcript sitting in the buffer forever.
+        if (turnState.buffer)
+            await finalizeTurn(turnState, client, deps, getContext, speakStreamed, history, getGeneration);
+        return;
+    }
     if (msg.type !== "Results")
         return;
     const transcript = msg.channel?.alternatives?.[0]?.transcript;
     if (!transcript)
         return;
     if (!msg.is_final) {
-        safeSend(client, { type: "interim", text: transcript });
+        safeSend(client, { type: "interim", text: turnState.buffer ? `${turnState.buffer} ${transcript}` : transcript });
         return;
     }
+    // is_final means this chunk of transcript is stable and won't be
+    // revised — it does NOT mean the user is done talking. Deepgram can (and
+    // routinely does) finalize several chunks of one continuous utterance in
+    // a row with no real pause between them. Only speech_final (endpointing
+    // actually detected a pause) means the turn is genuinely over. Found
+    // live, not theoretical: treating every is_final as a separate finished
+    // question fired two independent LLM+TTS turns for one utterance — the
+    // literal cause of both the duplicated transcript entries ("hello" /
+    // "hello" with no reply in between) and the agent audibly speaking
+    // twice, overlapping.
+    turnState.buffer = turnState.buffer ? `${turnState.buffer} ${transcript}` : transcript;
+    if (!msg.speech_final) {
+        safeSend(client, { type: "interim", text: turnState.buffer });
+        return;
+    }
+    await finalizeTurn(turnState, client, deps, getContext, speakStreamed, history, getGeneration);
+}
+/**
+ * Everything from here on (the LLM call, TTS streaming) can fail in ways
+ * that have nothing to do with a malformed message — a flaky provider
+ * call, a rate limit, a dropped upstream connection. handleDeepgramMessage
+ * is invoked fire-and-forget (`void handleDeepgramMessage(...)`), so an
+ * uncaught throw here previously vanished into an unhandled rejection: the
+ * client had already been told "final" (entering its "thinking" state) and
+ * then simply never heard from the server again for this turn — stuck
+ * indefinitely with the mic never resuming. Every path out of the try
+ * block now sends the client something that ends the turn.
+ *
+ * myGeneration is captured before the (potentially slow) LLM call and
+ * re-checked before the verb/speech actually goes out — a barge-in that
+ * happens while this turn is still "thinking" bumps the generation, and
+ * without this check the now-stale response would still land on the
+ * client after the user had already moved on to a new question.
+ */
+async function finalizeTurn(turnState, client, deps, getContext, speakStreamed, history, getGeneration) {
+    const transcript = turnState.buffer;
+    turnState.buffer = "";
+    const myGeneration = getGeneration();
     safeSend(client, { type: "final", text: transcript });
-    // Everything from here on (the LLM call, TTS streaming) can fail in ways
-    // that have nothing to do with a malformed message — a flaky provider
-    // call, a rate limit, a dropped upstream connection. This whole function
-    // is invoked fire-and-forget (`void handleDeepgramMessage(...)`), so an
-    // uncaught throw here previously vanished into an unhandled rejection:
-    // the client had already been told "final" (entering its "thinking"
-    // state) and then simply never heard from the server again for this
-    // turn — stuck indefinitely with the mic never resuming. Every path out
-    // of this try block now sends the client something that ends the turn.
     try {
-        const { route, visible } = getContext();
+        const { route, visible, liveElements } = getContext();
         const verb = await (0, server_1.resolveVerb)(deps.llm, deps.systemPrompt, deps.manifest, deps.registeredActions, deps.capability, {
             route,
             question: transcript,
             visible,
+            liveElements,
             history,
         });
+        if (myGeneration !== getGeneration())
+            return; // superseded by a barge-in while this turn was resolving
         // Sent immediately — before speech synthesis even starts — so
         // highlight/navigate/do execute in the browser right away instead of
         // waiting on audio. The agent visibly acts while it's still about to
@@ -260,8 +326,10 @@ async function handleDeepgramMessage(raw, client, deps, getContext, speakStreame
     }
     catch (err) {
         console.error("[cairn realtime] failed to resolve/speak this turn:", err);
-        safeSend(client, { type: "error", message: "Something went wrong answering that — try again." });
-        safeSend(client, { type: "turn_complete" });
+        if (myGeneration === getGeneration()) {
+            safeSend(client, { type: "error", message: "Something went wrong answering that — try again." });
+            safeSend(client, { type: "turn_complete" });
+        }
     }
 }
 function safeSend(client, message) {
@@ -269,6 +337,27 @@ function safeSend(client, message) {
         return;
     client.send(JSON.stringify(message));
 }
+/** Defensive parse for the client's self-reported live DOM scan — same
+ * untrusted-input treatment `visible` already gets on this control-message
+ * path (no CopilotRequestSchema here, unlike the HTTP handler), just
+ * shaped-checked so a malformed entry can't reach the LLM prompt oddly. */
+function parseLiveElements(raw) {
+    if (!Array.isArray(raw))
+        return [];
+    const elements = [];
+    for (const entry of raw) {
+        if (entry &&
+            typeof entry === "object" &&
+            typeof entry.id === "string" &&
+            typeof entry.role === "string" &&
+            typeof entry.label === "string") {
+            elements.push({ id: entry.id, role: entry.role, label: entry.label });
+        }
+        if (elements.length >= 60)
+            break;
+    }
+    return elements;
+}
 /** A short text form of any verb for the history log — not shown to the
  * user, just fed back to the model on later turns so it knows what it
  * already did/said. */

package/dist/runtime-scan.d.ts

@@ -0,0 +1,36 @@
+import type { LiveElement } from "@cairnvibe/core";
+export interface LiveScan {
+    elements: LiveElement[];
+    byId: Map<string, HTMLElement>;
+}
+/**
+ * Scans the live DOM for interactive elements currently in the viewport.
+ * Returns both the bounded list to send to the model (`elements`, capped at
+ * MAX_ELEMENTS and MAX_LABEL_LENGTH — the actual privacy/payload backstop,
+ * mirrored server-side in CopilotRequestSchema) and the real elements it
+ * maps to, keyed by the same ids (`byId`) — resolve a verb's target by
+ * looking it up here, never by re-deriving a selector from the id string.
+ */
+export declare function scanInteractiveElements(root?: ParentNode): LiveScan;
+export interface LiveElementRegistry {
+    /** Starts continuous background scanning — call once, typically on mount. */
+    start(): void;
+    stop(): void;
+    /**
+     * Freezes the current scan for one request/response round trip. Call
+     * this once when a question is sent, and resolve that turn's verb
+     * against exactly this snapshot — not a fresh call — so a background
+     * rescan that lands mid-flight can't shift what an id resolves to
+     * between when the request went out and when the response comes back.
+     */
+    getSnapshot(): LiveScan;
+}
+/**
+ * Keeps a scan continuously fresh in the background via a debounced
+ * MutationObserver (plus scroll/resize, since viewport membership changes
+ * without any DOM mutation) instead of only scanning at the moment a
+ * question is asked — so the agent never has to pause to "go look at the
+ * page" right when it needs to click something; a sub-agent gathering
+ * context while the main conversation keeps moving.
+ */
+export declare function createLiveElementRegistry(): LiveElementRegistry;