@cairnvibe/sdk 0.2.6 → 0.2.8

package/dist/index.js

@@ -6,13 +6,32 @@ const jsx_runtime_1 = require("react/jsx-runtime");
 const react_1 = require("react");
 const navigation_1 = require("next/navigation");
 const lucide_react_1 = require("lucide-react");
+const core_1 = require("@cairnvibe/core");
 const context_collector_1 = require("./context-collector");
 const element_ladder_1 = require("./element-ladder");
+const runtime_scan_1 = require("./runtime-scan");
+const webmcp_client_1 = require("./webmcp-client");
 const verb_executor_1 = require("./verb-executor");
 function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, reportMissesEndpoint, transcribeEndpoint, speakEndpoint, realtimeUrl, persona = "Cairn", }) {
     const pathname = (0, navigation_1.usePathname)() ?? "/";
+    // Mirrors `pathname` for use inside long-lived closures (a realtime
+    // session's handlers are all created once, when the connection opens —
+    // same staleness reason runTour tracks its own `currentRoute` locally
+    // rather than trusting its closure's `pathname` after a mid-tour
+    // navigation).
+    const pathnameRef = (0, react_1.useRef)(pathname);
+    (0, react_1.useEffect)(() => {
+        pathnameRef.current = pathname;
+        void sendFreshContext(); // no-op if no realtime session is open
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [pathname]);
     const router = (0, navigation_1.useRouter)();
     const [open, setOpen] = (0, react_1.useState)(false);
+    // Collapsed by default so the panel only ever shows the current exchange
+    // — the full archived transcript (built up over a long conversation)
+    // stays out of the way behind an explicit toggle instead of always being
+    // visible inline, which made the panel grow uncomfortably tall.
+    const [historyExpanded, setHistoryExpanded] = (0, react_1.useState)(false);
     const [question, setQuestion] = (0, react_1.useState)("");
     const [answer, setAnswer] = (0, react_1.useState)(null);
     const [status, setStatus] = (0, react_1.useState)("idle");
@@ -65,6 +84,19 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
     // keeps its own history server-side instead, since that connection is
     // already stateful).
     const historyRef = (0, react_1.useRef)([]);
+    // A background scanner that keeps a live inventory of what's actually
+    // clickable on screen right now (runtime-scan.ts) — running continuously
+    // via a MutationObserver so there's never a pause to "go look at the
+    // page" right when a verb needs to click something. `liveMapRef` freezes
+    // one snapshot of it per turn (set alongside every context/question send,
+    // below) so a background rescan landing mid-flight can't shift what an id
+    // resolves to between when a request went out and its response came back.
+    const liveRegistryRef = (0, react_1.useRef)((0, runtime_scan_1.createLiveElementRegistry)());
+    const liveMapRef = (0, react_1.useRef)(new Map());
+    (0, react_1.useEffect)(() => {
+        liveRegistryRef.current.start();
+        return () => liveRegistryRef.current.stop();
+    }, []);
     const mediaRecorderRef = (0, react_1.useRef)(null);
     const audioChunksRef = (0, react_1.useRef)([]);
     const transcribeInFlightRef = (0, react_1.useRef)(false);
@@ -152,6 +184,35 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
         rtStateRef.current = next;
         setStatus(next);
     }
+    /**
+     * Refreshes the server's picture of route/visible/liveElements over an
+     * already-open realtime connection. Beyond the initial connect, called
+     * on every route change and whenever the mic is about to start listening
+     * again — a real, pre-existing gap this closes as a side effect: the
+     * server's context previously updated only once, at connection open, so
+     * navigating mid-call (via a "navigate" verb, or the user clicking
+     * around) left the server answering every later turn as if the user were
+     * still on the original page. Reads pathnameRef, not the closure's
+     * `pathname`, so it's correct even called from a handler created once at
+     * connection-open time.
+     */
+    async function sendFreshContext() {
+        const ws = rtSocketRef.current;
+        if (!ws || ws.readyState !== WebSocket.OPEN)
+            return;
+        const liveScan = liveRegistryRef.current.getSnapshot();
+        liveMapRef.current = liveScan.byId;
+        const webMcpTools = await (0, webmcp_client_1.discoverWebMcpTools)();
+        if (ws.readyState !== WebSocket.OPEN)
+            return; // may have closed while awaiting discovery
+        ws.send(JSON.stringify({
+            type: "context",
+            route: pathnameRef.current,
+            visible: (0, context_collector_1.collectVisible)(),
+            liveElements: liveScan.elements,
+            webMcpTools,
+        }));
+    }
     function reportMiss(context) {
         (0, element_ladder_1.logMiss)(context);
         if (reportMissesEndpoint) {
@@ -174,6 +235,7 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
             onDo,
             onTour: (steps) => void runTour(steps),
             registeredActions,
+            liveElements: liveMapRef.current,
         });
     }
     /**
@@ -236,11 +298,25 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
                         return;
                 }
                 if (step.target) {
-                    const el = (0, element_ladder_1.findElement)(step.target);
-                    if (el)
+                    // A fresh scan, not the tour's starting liveMapRef snapshot — a
+                    // step after a mid-tour navigation targets elements on a page
+                    // that didn't exist when the tour began.
+                    const liveScan = liveRegistryRef.current.getSnapshot();
+                    const el = (0, element_ladder_1.findElement)(step.target, liveScan.byId);
+                    if (el) {
                         (0, element_ladder_1.highlightElement)(el);
-                    else
+                        if (step.click) {
+                            el.click();
+                            // Give whatever the click reveals (a detail view, an expanded
+                            // row) a moment to actually render before narrating it.
+                            await new Promise((resolve) => setTimeout(resolve, 400));
+                            if (tourGenerationRef.current !== myGeneration)
+                                return;
+                        }
+                    }
+                    else {
                         reportMiss({ attempted: step.target, route: currentRoute });
+                    }
                 }
                 if (wasRealtimeListening && rtSocketRef.current?.readyState === WebSocket.OPEN) {
                     // Already have a live streaming connection open — reuse it
@@ -281,22 +357,7 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
         setLastQuestion(q);
         setQuestion("");
         try {
-            const res = await fetch(endpoint, {
-                method: "POST",
-                headers: { "content-type": "application/json" },
-                body: JSON.stringify({ route: pathname, question: q, visible: (0, context_collector_1.collectVisible)(), history: historyRef.current }),
-            });
-            const data = await res.json().catch(() => null);
-            handleVerb(data);
-            // Unlike the realtime relay (one persistent connection, memory lives
-            // server-side), each of these POSTs is stateless — the widget itself
-            // is what remembers, and resends it above so the model has context
-            // for "the first one" / "do that instead" on the next question.
-            historyRef.current = [
-                ...historyRef.current,
-                { role: "user", text: q },
-                { role: "assistant", text: summarizeVerbForHistory(data) },
-            ].slice(-MAX_HISTORY_TURNS);
+            await runTypedAgentLoop(q);
         }
         catch {
             setAnswer("Something went wrong reaching the help service — try again in a moment.");
@@ -305,6 +366,73 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
             setStatus("idle");
         }
     }
+    const MAX_LOOP_ITERATIONS = 6; // a hard cap, not a target — see runTypedAgentLoop
+    /**
+     * Drives the agent loop over the stateless HTTP path: ask the server,
+     * and if it comes back with a continuing step (click/fill/read/
+     * call_tool — TERMINAL_VERBS in @cairnvibe/core says which verbs end a
+     * turn), execute that step for real, fold the real result into this
+     * turn's own working history, and ask again — repeat until a terminal
+     * verb or the iteration cap, instead of the old one-call-one-answer
+     * shape. `question` stays the original ask on every call; only
+     * `history` grows with each step's real trace, so the model always
+     * still knows what it was actually asked. `historyRef` (the
+     * conversation's real memory) is only ever committed once, at the end —
+     * a turn that hits the cap mid-loop doesn't leave partial noise in it.
+     */
+    async function runTypedAgentLoop(q) {
+        let loopHistory = historyRef.current;
+        const webMcpTools = await (0, webmcp_client_1.discoverWebMcpTools)();
+        for (let i = 0; i < MAX_LOOP_ITERATIONS; i++) {
+            const liveScan = liveRegistryRef.current.getSnapshot();
+            liveMapRef.current = liveScan.byId;
+            const res = await fetch(endpoint, {
+                method: "POST",
+                headers: { "content-type": "application/json" },
+                body: JSON.stringify({
+                    route: pathname,
+                    question: q,
+                    visible: (0, context_collector_1.collectVisible)(),
+                    history: loopHistory,
+                    liveElements: liveScan.elements,
+                    webMcpTools,
+                }),
+            });
+            const data = await res.json().catch(() => null);
+            const parsed = (0, core_1.safeParseVerbResponse)(data);
+            if (!parsed || core_1.TERMINAL_VERBS.has(parsed.verb)) {
+                handleVerb(data);
+                // Unlike the realtime relay (one persistent connection, memory
+                // lives server-side), each of these POSTs is stateless — the
+                // widget itself is what remembers, and resends it above so the
+                // model has context for "the first one" / "do that instead" on
+                // the next question.
+                historyRef.current = [
+                    ...loopHistory,
+                    { role: "user", text: q },
+                    { role: "assistant", text: summarizeVerbForHistory(data) },
+                ].slice(-MAX_HISTORY_TURNS);
+                return;
+            }
+            // A continuing step — show it happening, execute it for real, and
+            // go around again with the real result instead of ending the turn.
+            setAnswer(summarizeVerbForHistory(data));
+            const stepResult = await (0, verb_executor_1.executeToolStep)(data, pathname, liveMapRef.current);
+            loopHistory = [
+                ...loopHistory,
+                {
+                    role: "assistant",
+                    text: `${summarizeVerbForHistory(data)}. Result: ${stepResult?.observation ?? "no result"}`,
+                },
+            ].slice(-MAX_HISTORY_TURNS);
+        }
+        setAnswer("I wasn't able to finish that — try asking again or breaking it into smaller steps.");
+        historyRef.current = [
+            ...loopHistory,
+            { role: "user", text: q },
+            { role: "assistant", text: "(gave up after too many steps)" },
+        ].slice(-MAX_HISTORY_TURNS);
+    }
     /**
      * The one place that starts audio playback for a spoken response — stops
      * whatever's currently playing first, so two responses (e.g. a rapid
@@ -568,6 +696,7 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
                 }
                 setRtStatus("rt-listening");
                 setCaption("");
+                void sendFreshContext(); // refresh before the user starts talking again, not after
             }
             function disarmThinkingWatchdog() {
                 if (rtThinkingWatchdogRef.current) {
@@ -625,7 +754,7 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
                 setCaption("");
             }
             ws.onopen = () => {
-                ws.send(JSON.stringify({ type: "context", route: pathname, visible: (0, context_collector_1.collectVisible)() }));
+                sendFreshContext();
                 setRtStatus("rt-listening");
                 rtStartingRef.current = false;
             };
@@ -643,6 +772,25 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
                     armThinkingWatchdog();
                 }
                 else if (msg.type === "verb") {
+                    const parsedStep = (0, core_1.safeParseVerbResponse)(msg.verb);
+                    if (parsedStep && !core_1.TERMINAL_VERBS.has(parsedStep.verb)) {
+                        // A continuing agent-loop step (click/fill/read/call_tool) —
+                        // the turn isn't over: execute it for real and report the
+                        // result back so the server can decide the next step, instead
+                        // of treating this like a normal answer (no
+                        // disarmThinkingWatchdog/handleVerb — those are for when a
+                        // turn actually ends). Shown visually so a multi-step turn
+                        // reads as visible progress, not a silent pause; never spoken
+                        // — the server's loop stays quiet between steps on purpose,
+                        // to keep it fast.
+                        setAnswer(summarizeVerbForHistory(msg.verb));
+                        void (0, verb_executor_1.executeToolStep)(msg.verb, pathnameRef.current, liveMapRef.current).then((result) => {
+                            if (ws.readyState === WebSocket.OPEN) {
+                                ws.send(JSON.stringify({ type: "tool_result", observation: result?.observation ?? "no result" }));
+                            }
+                        });
+                        return;
+                    }
                     disarmThinkingWatchdog();
                     handleVerb(msg.verb);
                 }
@@ -779,7 +927,8 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
         "rt-thinking": "Thinking…",
         "rt-speaking": "Speaking…",
     };
-    return ((0, jsx_runtime_1.jsxs)(jsx_runtime_1.Fragment, { children: [(0, jsx_runtime_1.jsx)("style", { suppressHydrationWarning: true, dangerouslySetInnerHTML: { __html: COPILOT_STYLES } }), (0, jsx_runtime_1.jsx)("button", { className: status === "rt-speaking" ? "cairn-fab cairn-fab-speaking" : "cairn-fab", "aria-label": open ? `Close ${persona} help` : `Open ${persona} help`, onClick: () => setOpen((v) => !v), children: open ? (0, jsx_runtime_1.jsx)(lucide_react_1.X, { size: 22 }) : (0, jsx_runtime_1.jsx)(CairnMark, {}) }), open && ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-panel", role: "dialog", "aria-label": `${persona} help panel`, ref: panelRef, children: [(transcript.length > 0 || userCaption || answer || busy) && ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-stack", children: [transcript.map((entry) => ((0, jsx_runtime_1.jsx)("div", { className: entry.role === "user" ? "cairn-bubble cairn-bubble-user cairn-bubble-past" : "cairn-bubble cairn-bubble-agent cairn-bubble-past", children: entry.role === "agent" ? (0, jsx_runtime_1.jsx)("span", { className: "cairn-bubble-text", children: entry.text }) : entry.text }, entry.id))), userCaption && ((0, jsx_runtime_1.jsx)("div", { className: "cairn-bubble cairn-bubble-user", children: userCaption }, `u-${userCaption}`)), (answer || busy) && ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-bubble cairn-bubble-agent", children: [tourChip && (0, jsx_runtime_1.jsx)("span", { className: "cairn-chip", children: tourChip }), answer ? ((0, jsx_runtime_1.jsx)("span", { className: "cairn-bubble-text", children: renderCaptionWords(answer) })) : ((0, jsx_runtime_1.jsxs)("span", { className: "cairn-thinking", "aria-label": "Thinking", children: [(0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" })] }))] }, `a-${answer ?? status}`))] })), realtimeActive ? ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-rt-bar", children: [(0, jsx_runtime_1.jsx)("span", { className: `cairn-rt-dot cairn-rt-dot-${status}` }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-rt-label", children: statusLabel[status] }), (0, jsx_runtime_1.jsxs)("div", { className: "cairn-rt-controls", children: [(0, jsx_runtime_1.jsx)("button", { type: "button", className: status === "rt-speaking" ? "cairn-icon-btn cairn-icon-btn-speaking" : "cairn-icon-btn", "aria-label": rtMicMuted ? "Unmute microphone" : "Mute microphone", onClick: toggleRtMic, children: rtMicMuted ? (0, jsx_runtime_1.jsx)(lucide_react_1.MicOff, { size: 16 }) : (0, jsx_runtime_1.jsx)(lucide_react_1.Mic, { size: 16 }) }), (0, jsx_runtime_1.jsx)("button", { type: "button", className: status === "rt-speaking" ? "cairn-icon-btn cairn-icon-btn-speaking" : "cairn-icon-btn", "aria-label": rtSpeakerMuted ? "Unmute speaker" : "Mute speaker", onClick: toggleRtSpeaker, children: rtSpeakerMuted ? (0, jsx_runtime_1.jsx)(lucide_react_1.VolumeX, { size: 16 }) : (0, jsx_runtime_1.jsx)(lucide_react_1.Volume2, { size: 16 }) }), (0, jsx_runtime_1.jsx)("button", { type: "button", className: "cairn-icon-btn cairn-icon-btn-end", "aria-label": "End conversation", onClick: endRealtime, children: (0, jsx_runtime_1.jsx)(lucide_react_1.PhoneOff, { size: 16 }) })] })] })) : ((0, jsx_runtime_1.jsx)("form", { onSubmit: (e) => {
+    return ((0, jsx_runtime_1.jsxs)(jsx_runtime_1.Fragment, { children: [(0, jsx_runtime_1.jsx)("style", { suppressHydrationWarning: true, dangerouslySetInnerHTML: { __html: COPILOT_STYLES } }), (0, jsx_runtime_1.jsx)("button", { className: status === "rt-speaking" ? "cairn-fab cairn-fab-speaking" : "cairn-fab", "aria-label": open ? `Close ${persona} help` : `Open ${persona} help`, onClick: () => setOpen((v) => !v), children: open ? (0, jsx_runtime_1.jsx)(lucide_react_1.X, { size: 22 }) : (0, jsx_runtime_1.jsx)(CairnMark, {}) }), open && ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-panel", role: "dialog", "aria-label": `${persona} help panel`, ref: panelRef, children: [(transcript.length > 0 || userCaption || answer || busy) && ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-stack", children: [transcript.length > 0 && ((0, jsx_runtime_1.jsxs)("button", { type: "button", className: "cairn-history-toggle", onClick: () => setHistoryExpanded((v) => !v), "aria-expanded": historyExpanded, children: [historyExpanded ? (0, jsx_runtime_1.jsx)(lucide_react_1.ChevronUp, { size: 12 }) : (0, jsx_runtime_1.jsx)(lucide_react_1.ChevronDown, { size: 12 }), historyExpanded ? "Hide earlier" : `${transcript.length} earlier`] })), historyExpanded &&
+                                transcript.map((entry) => ((0, jsx_runtime_1.jsx)("div", { className: entry.role === "user" ? "cairn-bubble cairn-bubble-user cairn-bubble-past" : "cairn-bubble cairn-bubble-agent cairn-bubble-past", children: entry.role === "agent" ? (0, jsx_runtime_1.jsx)("span", { className: "cairn-bubble-text", children: entry.text }) : entry.text }, entry.id))), userCaption && ((0, jsx_runtime_1.jsx)("div", { className: "cairn-bubble cairn-bubble-user", children: userCaption }, `u-${userCaption}`)), (answer || busy) && ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-bubble cairn-bubble-agent", children: [tourChip && (0, jsx_runtime_1.jsx)("span", { className: "cairn-chip", children: tourChip }), answer ? ((0, jsx_runtime_1.jsx)("span", { className: "cairn-bubble-text", children: renderCaptionWords(answer) })) : ((0, jsx_runtime_1.jsxs)("span", { className: "cairn-thinking", "aria-label": "Thinking", children: [(0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" })] }))] }, `a-${answer ?? status}`))] })), realtimeActive ? ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-rt-bar", children: [(0, jsx_runtime_1.jsx)("span", { className: `cairn-rt-dot cairn-rt-dot-${status}` }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-rt-label", children: statusLabel[status] }), (0, jsx_runtime_1.jsxs)("div", { className: "cairn-rt-controls", children: [(0, jsx_runtime_1.jsx)("button", { type: "button", className: status === "rt-speaking" ? "cairn-icon-btn cairn-icon-btn-speaking" : "cairn-icon-btn", "aria-label": rtMicMuted ? "Unmute microphone" : "Mute microphone", onClick: toggleRtMic, children: rtMicMuted ? (0, jsx_runtime_1.jsx)(lucide_react_1.MicOff, { size: 16 }) : (0, jsx_runtime_1.jsx)(lucide_react_1.Mic, { size: 16 }) }), (0, jsx_runtime_1.jsx)("button", { type: "button", className: status === "rt-speaking" ? "cairn-icon-btn cairn-icon-btn-speaking" : "cairn-icon-btn", "aria-label": rtSpeakerMuted ? "Unmute speaker" : "Mute speaker", onClick: toggleRtSpeaker, children: rtSpeakerMuted ? (0, jsx_runtime_1.jsx)(lucide_react_1.VolumeX, { size: 16 }) : (0, jsx_runtime_1.jsx)(lucide_react_1.Volume2, { size: 16 }) }), (0, jsx_runtime_1.jsx)("button", { type: "button", className: "cairn-icon-btn cairn-icon-btn-end", "aria-label": "End conversation", onClick: endRealtime, children: (0, jsx_runtime_1.jsx)(lucide_react_1.PhoneOff, { size: 16 }) })] })] })) : ((0, jsx_runtime_1.jsx)("form", { onSubmit: (e) => {
                             e.preventDefault();
                             const trimmed = question.trim();
                             if (trimmed)
@@ -809,6 +958,14 @@ function summarizeVerbForHistory(raw) {
             return `(ran ${String(v.action)}${v.target ? ` on ${String(v.target)}` : ""})`;
         case "tour":
             return Array.isArray(v.steps) ? v.steps.map((s) => s.text ?? "").join(" ") : "(tour)";
+        case "click":
+            return `(clicked ${String(v.target)})`;
+        case "fill":
+            return `(typed "${String(v.value)}" into ${String(v.target)})`;
+        case "read":
+            return `(read ${String(v.target)})`;
+        case "call_tool":
+            return `(called ${String(v.name)})`;
         default:
             return "(no response)";
     }
@@ -1031,6 +1188,26 @@ const COPILOT_STYLES = `
   text-transform: uppercase;
   color: rgba(11, 13, 18, 0.48);
 }
+.cairn-history-toggle {
+  align-self: center;
+  display: inline-flex;
+  align-items: center;
+  gap: 3px;
+  border: none;
+  background: none;
+  padding: 2px 8px;
+  font: inherit;
+  font-size: 11px;
+  font-weight: 600;
+  color: rgba(11, 13, 18, 0.4);
+  cursor: pointer;
+  border-radius: 999px;
+  transition: background 0.15s ease, color 0.15s ease;
+}
+.cairn-history-toggle:hover {
+  background: rgba(11, 13, 18, 0.05);
+  color: rgba(11, 13, 18, 0.6);
+}
 .cairn-thinking {
   display: inline-flex;
   gap: 4px;

package/dist/realtime-server.d.ts

@@ -1,6 +1,6 @@
 import http from "node:http";
 import { WebSocket } from "ws";
-import type { HistoryTurn, Manifest } from "@cairnvibe/core";
+import { type HistoryTurn, type LiveElement, type Manifest, type WebMcpTool } from "@cairnvibe/core";
 import { createVerbLLM, type CapabilityTier, type CreateCopilotHandlerOptions } from "./server";
 export interface CreateRealtimeServerOptions extends CreateCopilotHandlerOptions {
     manifest: Manifest;
@@ -22,6 +22,8 @@ export interface ConnectionDeps {
 export declare function handleDeepgramMessage(raw: string, client: WebSocket, deps: ConnectionDeps, getContext: () => {
     route: string;
     visible: string[];
+    liveElements: LiveElement[];
+    webMcpTools: WebMcpTool[];
 }, speakStreamed: (text: string) => Promise<void>, history: HistoryTurn[], turnState: {
     buffer: string;
-}, getGeneration: () => number): Promise<void>;
+}, getGeneration: () => number, waitForToolResult: () => Promise<string>): Promise<void>;

package/dist/realtime-server.js

@@ -33,6 +33,7 @@ exports.createRealtimeServer = createRealtimeServer;
 exports.handleDeepgramMessage = handleDeepgramMessage;
 const node_http_1 = __importDefault(require("node:http"));
 const ws_1 = require("ws");
+const core_1 = require("@cairnvibe/core");
 const server_1 = require("./server");
 const tts_stream_1 = require("./tts-stream");
 const DEEPGRAM_LIVE_URL = "wss://api.deepgram.com/v1/listen";
@@ -73,13 +74,28 @@ function createRealtimeServer(options) {
     return httpServer;
 }
 const MAX_HISTORY_TURNS = 8; // 4 exchanges — enough for "the first one"/"do that instead" without growing the prompt unbounded over a long call
+const MAX_LOOP_ITERATIONS = 6; // a hard cap on one turn's agent-loop steps, not a target — see finalizeTurn
 async function handleConnection(client, deps) {
-    let context = { route: "/", visible: [] };
+    // liveElements/webMcpTools refresh on every "context" resend (the client
+    // sends one on route changes and each time it's about to start listening
+    // again), so a live scan from several turns ago never lingers into a
+    // later one.
+    let context = {
+        route: "/",
+        visible: [],
+        liveElements: [],
+        webMcpTools: [],
+    };
     // Unlike the stateless HTTP path (which needs the client to resend
     // history every request), a realtime connection is already stateful —
     // one WebSocket per call — so this is accumulated here directly rather
     // than round-tripped through the client.
     const history = [];
+    // Resolves the agent loop's in-flight waitForToolResult() call once the
+    // client reports back what a click/fill/read/call_tool step actually
+    // did — same "a mutable pending-callback slot, resolved when the right
+    // message arrives" pattern onCurrentTurnFlushed already uses below.
+    let pendingToolResultResolve = null;
     const dgUrl = `${DEEPGRAM_LIVE_URL}?model=${encodeURIComponent(deps.sttModel)}` +
         `&encoding=linear16&sample_rate=16000&channels=1&interim_results=true&endpointing=300&utterance_end_ms=1000`;
     const dg = new ws_1.WebSocket(dgUrl, { headers: { Authorization: `Token ${deps.deepgramApiKey}` } });
@@ -156,12 +172,31 @@ async function handleConnection(client, deps) {
             stream.flush();
         });
     }
+    /**
+     * Pauses the agent loop (finalizeTurn, below) until the client reports
+     * back the real result of a click/fill/read/call_tool step it just sent
+     * out — the server can't execute a DOM action itself, so every
+     * continuing step needs a real round trip to the browser and back. A
+     * real timeout, not a hang: a client that never answers (closed tab,
+     * dropped connection) can't leave a turn stuck forever.
+     */
+    function waitForToolResult() {
+        return new Promise((resolve) => {
+            pendingToolResultResolve = resolve;
+            setTimeout(() => {
+                if (pendingToolResultResolve === resolve) {
+                    pendingToolResultResolve = null;
+                    resolve("(no result — timed out waiting for the browser)");
+                }
+            }, 15000);
+        });
+    }
     // Accumulates Deepgram "Results" transcript segments across one utterance
     // — see handleDeepgramMessage for why this can't just react to every
     // is_final.
     const turnState = { buffer: "" };
     dg.on("message", (data) => {
-        void handleDeepgramMessage(data.toString(), client, deps, () => context, speakStreamed, history, turnState, () => generation);
+        void handleDeepgramMessage(data.toString(), client, deps, () => context, speakStreamed, history, turnState, () => generation, waitForToolResult);
     });
     dg.on("error", (err) => {
         console.error("[cairn realtime] Deepgram STT connection error:", err);
@@ -179,7 +214,19 @@ async function handleConnection(client, deps) {
         try {
             const msg = JSON.parse(data.toString());
             if (msg.type === "context") {
-                context = { route: String(msg.route ?? "/"), visible: Array.isArray(msg.visible) ? msg.visible : [] };
+                context = {
+                    route: String(msg.route ?? "/"),
+                    visible: Array.isArray(msg.visible) ? msg.visible : [],
+                    liveElements: parseLiveElements(msg.liveElements),
+                    webMcpTools: parseWebMcpTools(msg.webMcpTools),
+                };
+            }
+            else if (msg.type === "tool_result" && typeof msg.observation === "string") {
+                // The client finished executing a click/fill/read/call_tool step
+                // the agent loop sent it — this is what finalizeTurn's
+                // waitForToolResult() below is paused on.
+                pendingToolResultResolve?.(msg.observation);
+                pendingToolResultResolve = null;
             }
             else if (msg.type === "end") {
                 client.close();
@@ -224,7 +271,7 @@ async function handleConnection(client, deps) {
         speakStream?.close();
     });
 }
-async function handleDeepgramMessage(raw, client, deps, getContext, speakStreamed, history, turnState, getGeneration) {
+async function handleDeepgramMessage(raw, client, deps, getContext, speakStreamed, history, turnState, getGeneration, waitForToolResult) {
     let msg;
     try {
         msg = JSON.parse(raw);
@@ -238,7 +285,7 @@ async function handleDeepgramMessage(raw, client, deps, getContext, speakStreame
         // Results message never carries speech_final:true, so a turn can't get
         // permanently stuck with real transcript sitting in the buffer forever.
         if (turnState.buffer)
-            await finalizeTurn(turnState, client, deps, getContext, speakStreamed, history, getGeneration);
+            await finalizeTurn(turnState, client, deps, getContext, speakStreamed, history, getGeneration, waitForToolResult);
         return;
     }
     if (msg.type !== "Results")
@@ -265,7 +312,7 @@ async function handleDeepgramMessage(raw, client, deps, getContext, speakStreame
         safeSend(client, { type: "interim", text: turnState.buffer });
         return;
     }
-    await finalizeTurn(turnState, client, deps, getContext, speakStreamed, history, getGeneration);
+    await finalizeTurn(turnState, client, deps, getContext, speakStreamed, history, getGeneration, waitForToolResult);
 }
 /**
  * Everything from here on (the LLM call, TTS streaming) can fail in ways
@@ -283,38 +330,74 @@ async function handleDeepgramMessage(raw, client, deps, getContext, speakStreame
  * happens while this turn is still "thinking" bumps the generation, and
  * without this check the now-stale response would still land on the
  * client after the user had already moved on to a new question.
+ *
+ * A continuing verb (click/fill/read/call_tool — TERMINAL_VERBS says which
+ * ones aren't) doesn't end the turn here: the server can't execute a DOM
+ * action itself, so it sends the step to the client, awaits its real
+ * result over waitForToolResult(), folds that into a *local* working copy
+ * of history, and calls resolveVerb again — repeat up to
+ * MAX_LOOP_ITERATIONS. The connection's real `history` only gets the
+ * user's real question plus the turn's final answer, committed once at
+ * the end — a turn that hits the cap mid-loop doesn't leave partial tool
+ * noise in the conversation's real memory, same discipline the HTTP
+ * path's runTypedAgentLoop (index.tsx) follows.
  */
-async function finalizeTurn(turnState, client, deps, getContext, speakStreamed, history, getGeneration) {
+async function finalizeTurn(turnState, client, deps, getContext, speakStreamed, history, getGeneration, waitForToolResult) {
     const transcript = turnState.buffer;
     turnState.buffer = "";
     const myGeneration = getGeneration();
     safeSend(client, { type: "final", text: transcript });
+    let loopHistory = history;
     try {
-        const { route, visible } = getContext();
-        const verb = await (0, server_1.resolveVerb)(deps.llm, deps.systemPrompt, deps.manifest, deps.registeredActions, deps.capability, {
-            route,
-            question: transcript,
-            visible,
-            history,
-        });
-        if (myGeneration !== getGeneration())
-            return; // superseded by a barge-in while this turn was resolving
-        // Sent immediately — before speech synthesis even starts — so
-        // highlight/navigate/do execute in the browser right away instead of
-        // waiting on audio. The agent visibly acts while it's still about to
-        // speak, not after.
-        safeSend(client, { type: "verb", verb });
-        history.push({ role: "user", text: transcript }, { role: "assistant", text: summarizeVerbForHistory(verb) });
-        history.splice(0, Math.max(0, history.length - MAX_HISTORY_TURNS));
-        // A verb with no spoken text (highlight/navigate/do often have none)
-        // still needs to unstick the client's "thinking" state and let the mic
-        // resume — turn_complete covers that with no audio path involved.
-        if ("text" in verb && verb.text) {
-            await speakStreamed(verb.text);
-        }
-        else {
-            safeSend(client, { type: "turn_complete" });
+        for (let i = 0; i < MAX_LOOP_ITERATIONS; i++) {
+            const { route, visible, liveElements, webMcpTools } = getContext();
+            const verb = await (0, server_1.resolveVerb)(deps.llm, deps.systemPrompt, deps.manifest, deps.registeredActions, deps.capability, {
+                route,
+                question: transcript,
+                visible,
+                liveElements,
+                webMcpTools,
+                history: loopHistory,
+            });
+            if (myGeneration !== getGeneration())
+                return; // superseded by a barge-in while this turn was resolving
+            // Sent immediately — before speech synthesis even starts — so
+            // highlight/navigate/do execute in the browser right away instead of
+            // waiting on audio. The agent visibly acts while it's still about to
+            // speak, not after.
+            safeSend(client, { type: "verb", verb });
+            if (!core_1.TERMINAL_VERBS.has(verb.verb)) {
+                // A continuing step — no speech for it (keeps the loop fast;
+                // the client still shows it visually) — wait for its real result
+                // and go around again instead of ending the turn.
+                const observation = await waitForToolResult();
+                if (myGeneration !== getGeneration())
+                    return;
+                loopHistory = [
+                    ...loopHistory,
+                    { role: "assistant", text: `${summarizeVerbForHistory(verb)}. Result: ${observation}` },
+                ].slice(-MAX_HISTORY_TURNS);
+                continue;
+            }
+            history.push({ role: "user", text: transcript }, { role: "assistant", text: summarizeVerbForHistory(verb) });
+            history.splice(0, Math.max(0, history.length - MAX_HISTORY_TURNS));
+            // A verb with no spoken text (highlight/navigate/do often have none)
+            // still needs to unstick the client's "thinking" state and let the mic
+            // resume — turn_complete covers that with no audio path involved.
+            if ("text" in verb && verb.text) {
+                await speakStreamed(verb.text);
+            }
+            else {
+                safeSend(client, { type: "turn_complete" });
+            }
+            return;
         }
+        // Iteration cap hit with no terminal verb — degrade honestly instead
+        // of leaving the client waiting forever.
+        history.push({ role: "user", text: transcript }, { role: "assistant", text: "(gave up after too many steps)" });
+        history.splice(0, Math.max(0, history.length - MAX_HISTORY_TURNS));
+        safeSend(client, { type: "verb", verb: { verb: "explain", text: "I wasn't able to finish that — try asking again or breaking it into smaller steps." } });
+        await speakStreamed("I wasn't able to finish that — try asking again or breaking it into smaller steps.");
     }
     catch (err) {
         console.error("[cairn realtime] failed to resolve/speak this turn:", err);
@@ -329,6 +412,42 @@ function safeSend(client, message) {
         return;
     client.send(JSON.stringify(message));
 }
+/** Defensive parse for the client's self-reported live DOM scan — same
+ * untrusted-input treatment `visible` already gets on this control-message
+ * path (no CopilotRequestSchema here, unlike the HTTP handler), just
+ * shaped-checked so a malformed entry can't reach the LLM prompt oddly. */
+function parseLiveElements(raw) {
+    if (!Array.isArray(raw))
+        return [];
+    const elements = [];
+    for (const entry of raw) {
+        if (entry &&
+            typeof entry === "object" &&
+            typeof entry.id === "string" &&
+            typeof entry.role === "string" &&
+            typeof entry.label === "string") {
+            elements.push({ id: entry.id, role: entry.role, label: entry.label });
+        }
+        if (elements.length >= 60)
+            break;
+    }
+    return elements;
+}
+/** Same defensive shape-check as parseLiveElements, for the client's
+ * self-reported WebMCP tool list. */
+function parseWebMcpTools(raw) {
+    if (!Array.isArray(raw))
+        return [];
+    const tools = [];
+    for (const entry of raw) {
+        if (entry && typeof entry === "object" && typeof entry.name === "string" && typeof entry.description === "string") {
+            tools.push({ name: entry.name, description: entry.description, inputSchema: entry.inputSchema });
+        }
+        if (tools.length >= 30)
+            break;
+    }
+    return tools;
+}
 /** A short text form of any verb for the history log — not shown to the
  * user, just fed back to the model on later turns so it knows what it
  * already did/said. */
@@ -345,6 +464,14 @@ function summarizeVerbForHistory(verb) {
             return `(ran ${verb.action}${verb.target ? ` on ${verb.target}` : ""})`;
         case "tour":
             return verb.steps.map((s) => s.text).join(" ");
+        case "click":
+            return `(clicked ${verb.target})`;
+        case "fill":
+            return `(typed "${verb.value}" into ${verb.target})`;
+        case "read":
+            return `(read ${verb.target})`;
+        case "call_tool":
+            return `(called ${verb.name})`;
         default:
             return "(no response)";
     }