@cairnvibe/sdk 0.2.6 → 0.2.7

package/dist/index.js

@@ -8,11 +8,28 @@ const navigation_1 = require("next/navigation");
 const lucide_react_1 = require("lucide-react");
 const context_collector_1 = require("./context-collector");
 const element_ladder_1 = require("./element-ladder");
+const runtime_scan_1 = require("./runtime-scan");
 const verb_executor_1 = require("./verb-executor");
 function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, reportMissesEndpoint, transcribeEndpoint, speakEndpoint, realtimeUrl, persona = "Cairn", }) {
     const pathname = (0, navigation_1.usePathname)() ?? "/";
+    // Mirrors `pathname` for use inside long-lived closures (a realtime
+    // session's handlers are all created once, when the connection opens —
+    // same staleness reason runTour tracks its own `currentRoute` locally
+    // rather than trusting its closure's `pathname` after a mid-tour
+    // navigation).
+    const pathnameRef = (0, react_1.useRef)(pathname);
+    (0, react_1.useEffect)(() => {
+        pathnameRef.current = pathname;
+        sendFreshContext(); // no-op if no realtime session is open
+        // eslint-disable-next-line react-hooks/exhaustive-deps
+    }, [pathname]);
     const router = (0, navigation_1.useRouter)();
     const [open, setOpen] = (0, react_1.useState)(false);
+    // Collapsed by default so the panel only ever shows the current exchange
+    // — the full archived transcript (built up over a long conversation)
+    // stays out of the way behind an explicit toggle instead of always being
+    // visible inline, which made the panel grow uncomfortably tall.
+    const [historyExpanded, setHistoryExpanded] = (0, react_1.useState)(false);
     const [question, setQuestion] = (0, react_1.useState)("");
     const [answer, setAnswer] = (0, react_1.useState)(null);
     const [status, setStatus] = (0, react_1.useState)("idle");
@@ -65,6 +82,19 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
     // keeps its own history server-side instead, since that connection is
     // already stateful).
     const historyRef = (0, react_1.useRef)([]);
+    // A background scanner that keeps a live inventory of what's actually
+    // clickable on screen right now (runtime-scan.ts) — running continuously
+    // via a MutationObserver so there's never a pause to "go look at the
+    // page" right when a verb needs to click something. `liveMapRef` freezes
+    // one snapshot of it per turn (set alongside every context/question send,
+    // below) so a background rescan landing mid-flight can't shift what an id
+    // resolves to between when a request went out and its response came back.
+    const liveRegistryRef = (0, react_1.useRef)((0, runtime_scan_1.createLiveElementRegistry)());
+    const liveMapRef = (0, react_1.useRef)(new Map());
+    (0, react_1.useEffect)(() => {
+        liveRegistryRef.current.start();
+        return () => liveRegistryRef.current.stop();
+    }, []);
     const mediaRecorderRef = (0, react_1.useRef)(null);
     const audioChunksRef = (0, react_1.useRef)([]);
     const transcribeInFlightRef = (0, react_1.useRef)(false);
@@ -152,6 +182,26 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
         rtStateRef.current = next;
         setStatus(next);
     }
+    /**
+     * Refreshes the server's picture of route/visible/liveElements over an
+     * already-open realtime connection. Beyond the initial connect, called
+     * on every route change and whenever the mic is about to start listening
+     * again — a real, pre-existing gap this closes as a side effect: the
+     * server's context previously updated only once, at connection open, so
+     * navigating mid-call (via a "navigate" verb, or the user clicking
+     * around) left the server answering every later turn as if the user were
+     * still on the original page. Reads pathnameRef, not the closure's
+     * `pathname`, so it's correct even called from a handler created once at
+     * connection-open time.
+     */
+    function sendFreshContext() {
+        const ws = rtSocketRef.current;
+        if (!ws || ws.readyState !== WebSocket.OPEN)
+            return;
+        const liveScan = liveRegistryRef.current.getSnapshot();
+        liveMapRef.current = liveScan.byId;
+        ws.send(JSON.stringify({ type: "context", route: pathnameRef.current, visible: (0, context_collector_1.collectVisible)(), liveElements: liveScan.elements }));
+    }
     function reportMiss(context) {
         (0, element_ladder_1.logMiss)(context);
         if (reportMissesEndpoint) {
@@ -174,6 +224,7 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
             onDo,
             onTour: (steps) => void runTour(steps),
             registeredActions,
+            liveElements: liveMapRef.current,
         });
     }
     /**
@@ -236,11 +287,25 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
                         return;
                 }
                 if (step.target) {
-                    const el = (0, element_ladder_1.findElement)(step.target);
-                    if (el)
+                    // A fresh scan, not the tour's starting liveMapRef snapshot — a
+                    // step after a mid-tour navigation targets elements on a page
+                    // that didn't exist when the tour began.
+                    const liveScan = liveRegistryRef.current.getSnapshot();
+                    const el = (0, element_ladder_1.findElement)(step.target, liveScan.byId);
+                    if (el) {
                         (0, element_ladder_1.highlightElement)(el);
-                    else
+                        if (step.click) {
+                            el.click();
+                            // Give whatever the click reveals (a detail view, an expanded
+                            // row) a moment to actually render before narrating it.
+                            await new Promise((resolve) => setTimeout(resolve, 400));
+                            if (tourGenerationRef.current !== myGeneration)
+                                return;
+                        }
+                    }
+                    else {
                         reportMiss({ attempted: step.target, route: currentRoute });
+                    }
                 }
                 if (wasRealtimeListening && rtSocketRef.current?.readyState === WebSocket.OPEN) {
                     // Already have a live streaming connection open — reuse it
@@ -281,10 +346,18 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
         setLastQuestion(q);
         setQuestion("");
         try {
+            const liveScan = liveRegistryRef.current.getSnapshot();
+            liveMapRef.current = liveScan.byId;
             const res = await fetch(endpoint, {
                 method: "POST",
                 headers: { "content-type": "application/json" },
-                body: JSON.stringify({ route: pathname, question: q, visible: (0, context_collector_1.collectVisible)(), history: historyRef.current }),
+                body: JSON.stringify({
+                    route: pathname,
+                    question: q,
+                    visible: (0, context_collector_1.collectVisible)(),
+                    history: historyRef.current,
+                    liveElements: liveScan.elements,
+                }),
             });
             const data = await res.json().catch(() => null);
             handleVerb(data);
@@ -568,6 +641,7 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
                 }
                 setRtStatus("rt-listening");
                 setCaption("");
+                sendFreshContext(); // refresh before the user starts talking again, not after
             }
             function disarmThinkingWatchdog() {
                 if (rtThinkingWatchdogRef.current) {
@@ -625,7 +699,7 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
                 setCaption("");
             }
             ws.onopen = () => {
-                ws.send(JSON.stringify({ type: "context", route: pathname, visible: (0, context_collector_1.collectVisible)() }));
+                sendFreshContext();
                 setRtStatus("rt-listening");
                 rtStartingRef.current = false;
             };
@@ -779,7 +853,8 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
         "rt-thinking": "Thinking…",
         "rt-speaking": "Speaking…",
     };
-    return ((0, jsx_runtime_1.jsxs)(jsx_runtime_1.Fragment, { children: [(0, jsx_runtime_1.jsx)("style", { suppressHydrationWarning: true, dangerouslySetInnerHTML: { __html: COPILOT_STYLES } }), (0, jsx_runtime_1.jsx)("button", { className: status === "rt-speaking" ? "cairn-fab cairn-fab-speaking" : "cairn-fab", "aria-label": open ? `Close ${persona} help` : `Open ${persona} help`, onClick: () => setOpen((v) => !v), children: open ? (0, jsx_runtime_1.jsx)(lucide_react_1.X, { size: 22 }) : (0, jsx_runtime_1.jsx)(CairnMark, {}) }), open && ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-panel", role: "dialog", "aria-label": `${persona} help panel`, ref: panelRef, children: [(transcript.length > 0 || userCaption || answer || busy) && ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-stack", children: [transcript.map((entry) => ((0, jsx_runtime_1.jsx)("div", { className: entry.role === "user" ? "cairn-bubble cairn-bubble-user cairn-bubble-past" : "cairn-bubble cairn-bubble-agent cairn-bubble-past", children: entry.role === "agent" ? (0, jsx_runtime_1.jsx)("span", { className: "cairn-bubble-text", children: entry.text }) : entry.text }, entry.id))), userCaption && ((0, jsx_runtime_1.jsx)("div", { className: "cairn-bubble cairn-bubble-user", children: userCaption }, `u-${userCaption}`)), (answer || busy) && ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-bubble cairn-bubble-agent", children: [tourChip && (0, jsx_runtime_1.jsx)("span", { className: "cairn-chip", children: tourChip }), answer ? ((0, jsx_runtime_1.jsx)("span", { className: "cairn-bubble-text", children: renderCaptionWords(answer) })) : ((0, jsx_runtime_1.jsxs)("span", { className: "cairn-thinking", "aria-label": "Thinking", children: [(0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" })] }))] }, `a-${answer ?? status}`))] })), realtimeActive ? ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-rt-bar", children: [(0, jsx_runtime_1.jsx)("span", { className: `cairn-rt-dot cairn-rt-dot-${status}` }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-rt-label", children: statusLabel[status] }), (0, jsx_runtime_1.jsxs)("div", { className: "cairn-rt-controls", children: [(0, jsx_runtime_1.jsx)("button", { type: "button", className: status === "rt-speaking" ? "cairn-icon-btn cairn-icon-btn-speaking" : "cairn-icon-btn", "aria-label": rtMicMuted ? "Unmute microphone" : "Mute microphone", onClick: toggleRtMic, children: rtMicMuted ? (0, jsx_runtime_1.jsx)(lucide_react_1.MicOff, { size: 16 }) : (0, jsx_runtime_1.jsx)(lucide_react_1.Mic, { size: 16 }) }), (0, jsx_runtime_1.jsx)("button", { type: "button", className: status === "rt-speaking" ? "cairn-icon-btn cairn-icon-btn-speaking" : "cairn-icon-btn", "aria-label": rtSpeakerMuted ? "Unmute speaker" : "Mute speaker", onClick: toggleRtSpeaker, children: rtSpeakerMuted ? (0, jsx_runtime_1.jsx)(lucide_react_1.VolumeX, { size: 16 }) : (0, jsx_runtime_1.jsx)(lucide_react_1.Volume2, { size: 16 }) }), (0, jsx_runtime_1.jsx)("button", { type: "button", className: "cairn-icon-btn cairn-icon-btn-end", "aria-label": "End conversation", onClick: endRealtime, children: (0, jsx_runtime_1.jsx)(lucide_react_1.PhoneOff, { size: 16 }) })] })] })) : ((0, jsx_runtime_1.jsx)("form", { onSubmit: (e) => {
+    return ((0, jsx_runtime_1.jsxs)(jsx_runtime_1.Fragment, { children: [(0, jsx_runtime_1.jsx)("style", { suppressHydrationWarning: true, dangerouslySetInnerHTML: { __html: COPILOT_STYLES } }), (0, jsx_runtime_1.jsx)("button", { className: status === "rt-speaking" ? "cairn-fab cairn-fab-speaking" : "cairn-fab", "aria-label": open ? `Close ${persona} help` : `Open ${persona} help`, onClick: () => setOpen((v) => !v), children: open ? (0, jsx_runtime_1.jsx)(lucide_react_1.X, { size: 22 }) : (0, jsx_runtime_1.jsx)(CairnMark, {}) }), open && ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-panel", role: "dialog", "aria-label": `${persona} help panel`, ref: panelRef, children: [(transcript.length > 0 || userCaption || answer || busy) && ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-stack", children: [transcript.length > 0 && ((0, jsx_runtime_1.jsxs)("button", { type: "button", className: "cairn-history-toggle", onClick: () => setHistoryExpanded((v) => !v), "aria-expanded": historyExpanded, children: [historyExpanded ? (0, jsx_runtime_1.jsx)(lucide_react_1.ChevronUp, { size: 12 }) : (0, jsx_runtime_1.jsx)(lucide_react_1.ChevronDown, { size: 12 }), historyExpanded ? "Hide earlier" : `${transcript.length} earlier`] })), historyExpanded &&
+                                transcript.map((entry) => ((0, jsx_runtime_1.jsx)("div", { className: entry.role === "user" ? "cairn-bubble cairn-bubble-user cairn-bubble-past" : "cairn-bubble cairn-bubble-agent cairn-bubble-past", children: entry.role === "agent" ? (0, jsx_runtime_1.jsx)("span", { className: "cairn-bubble-text", children: entry.text }) : entry.text }, entry.id))), userCaption && ((0, jsx_runtime_1.jsx)("div", { className: "cairn-bubble cairn-bubble-user", children: userCaption }, `u-${userCaption}`)), (answer || busy) && ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-bubble cairn-bubble-agent", children: [tourChip && (0, jsx_runtime_1.jsx)("span", { className: "cairn-chip", children: tourChip }), answer ? ((0, jsx_runtime_1.jsx)("span", { className: "cairn-bubble-text", children: renderCaptionWords(answer) })) : ((0, jsx_runtime_1.jsxs)("span", { className: "cairn-thinking", "aria-label": "Thinking", children: [(0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-thinking-dot" })] }))] }, `a-${answer ?? status}`))] })), realtimeActive ? ((0, jsx_runtime_1.jsxs)("div", { className: "cairn-rt-bar", children: [(0, jsx_runtime_1.jsx)("span", { className: `cairn-rt-dot cairn-rt-dot-${status}` }), (0, jsx_runtime_1.jsx)("span", { className: "cairn-rt-label", children: statusLabel[status] }), (0, jsx_runtime_1.jsxs)("div", { className: "cairn-rt-controls", children: [(0, jsx_runtime_1.jsx)("button", { type: "button", className: status === "rt-speaking" ? "cairn-icon-btn cairn-icon-btn-speaking" : "cairn-icon-btn", "aria-label": rtMicMuted ? "Unmute microphone" : "Mute microphone", onClick: toggleRtMic, children: rtMicMuted ? (0, jsx_runtime_1.jsx)(lucide_react_1.MicOff, { size: 16 }) : (0, jsx_runtime_1.jsx)(lucide_react_1.Mic, { size: 16 }) }), (0, jsx_runtime_1.jsx)("button", { type: "button", className: status === "rt-speaking" ? "cairn-icon-btn cairn-icon-btn-speaking" : "cairn-icon-btn", "aria-label": rtSpeakerMuted ? "Unmute speaker" : "Mute speaker", onClick: toggleRtSpeaker, children: rtSpeakerMuted ? (0, jsx_runtime_1.jsx)(lucide_react_1.VolumeX, { size: 16 }) : (0, jsx_runtime_1.jsx)(lucide_react_1.Volume2, { size: 16 }) }), (0, jsx_runtime_1.jsx)("button", { type: "button", className: "cairn-icon-btn cairn-icon-btn-end", "aria-label": "End conversation", onClick: endRealtime, children: (0, jsx_runtime_1.jsx)(lucide_react_1.PhoneOff, { size: 16 }) })] })] })) : ((0, jsx_runtime_1.jsx)("form", { onSubmit: (e) => {
                             e.preventDefault();
                             const trimmed = question.trim();
                             if (trimmed)
@@ -1031,6 +1106,26 @@ const COPILOT_STYLES = `
   text-transform: uppercase;
   color: rgba(11, 13, 18, 0.48);
 }
+.cairn-history-toggle {
+  align-self: center;
+  display: inline-flex;
+  align-items: center;
+  gap: 3px;
+  border: none;
+  background: none;
+  padding: 2px 8px;
+  font: inherit;
+  font-size: 11px;
+  font-weight: 600;
+  color: rgba(11, 13, 18, 0.4);
+  cursor: pointer;
+  border-radius: 999px;
+  transition: background 0.15s ease, color 0.15s ease;
+}
+.cairn-history-toggle:hover {
+  background: rgba(11, 13, 18, 0.05);
+  color: rgba(11, 13, 18, 0.6);
+}
 .cairn-thinking {
   display: inline-flex;
   gap: 4px;

package/dist/realtime-server.d.ts

@@ -1,6 +1,6 @@
 import http from "node:http";
 import { WebSocket } from "ws";
-import type { HistoryTurn, Manifest } from "@cairnvibe/core";
+import type { HistoryTurn, LiveElement, Manifest } from "@cairnvibe/core";
 import { createVerbLLM, type CapabilityTier, type CreateCopilotHandlerOptions } from "./server";
 export interface CreateRealtimeServerOptions extends CreateCopilotHandlerOptions {
     manifest: Manifest;
@@ -22,6 +22,7 @@ export interface ConnectionDeps {
 export declare function handleDeepgramMessage(raw: string, client: WebSocket, deps: ConnectionDeps, getContext: () => {
     route: string;
     visible: string[];
+    liveElements: LiveElement[];
 }, speakStreamed: (text: string) => Promise<void>, history: HistoryTurn[], turnState: {
     buffer: string;
 }, getGeneration: () => number): Promise<void>;

package/dist/realtime-server.js

@@ -74,7 +74,10 @@ function createRealtimeServer(options) {
 }
 const MAX_HISTORY_TURNS = 8; // 4 exchanges — enough for "the first one"/"do that instead" without growing the prompt unbounded over a long call
 async function handleConnection(client, deps) {
-    let context = { route: "/", visible: [] };
+    // liveElements refreshes on every "context" resend (the client sends one
+    // on route changes and each time it's about to start listening again),
+    // so a live scan from several turns ago never lingers into a later one.
+    let context = { route: "/", visible: [], liveElements: [] };
     // Unlike the stateless HTTP path (which needs the client to resend
     // history every request), a realtime connection is already stateful —
     // one WebSocket per call — so this is accumulated here directly rather
@@ -179,7 +182,11 @@ async function handleConnection(client, deps) {
         try {
             const msg = JSON.parse(data.toString());
             if (msg.type === "context") {
-                context = { route: String(msg.route ?? "/"), visible: Array.isArray(msg.visible) ? msg.visible : [] };
+                context = {
+                    route: String(msg.route ?? "/"),
+                    visible: Array.isArray(msg.visible) ? msg.visible : [],
+                    liveElements: parseLiveElements(msg.liveElements),
+                };
             }
             else if (msg.type === "end") {
                 client.close();
@@ -290,11 +297,12 @@ async function finalizeTurn(turnState, client, deps, getContext, speakStreamed,
     const myGeneration = getGeneration();
     safeSend(client, { type: "final", text: transcript });
     try {
-        const { route, visible } = getContext();
+        const { route, visible, liveElements } = getContext();
         const verb = await (0, server_1.resolveVerb)(deps.llm, deps.systemPrompt, deps.manifest, deps.registeredActions, deps.capability, {
             route,
             question: transcript,
             visible,
+            liveElements,
             history,
         });
         if (myGeneration !== getGeneration())
@@ -329,6 +337,27 @@ function safeSend(client, message) {
         return;
     client.send(JSON.stringify(message));
 }
+/** Defensive parse for the client's self-reported live DOM scan — same
+ * untrusted-input treatment `visible` already gets on this control-message
+ * path (no CopilotRequestSchema here, unlike the HTTP handler), just
+ * shaped-checked so a malformed entry can't reach the LLM prompt oddly. */
+function parseLiveElements(raw) {
+    if (!Array.isArray(raw))
+        return [];
+    const elements = [];
+    for (const entry of raw) {
+        if (entry &&
+            typeof entry === "object" &&
+            typeof entry.id === "string" &&
+            typeof entry.role === "string" &&
+            typeof entry.label === "string") {
+            elements.push({ id: entry.id, role: entry.role, label: entry.label });
+        }
+        if (elements.length >= 60)
+            break;
+    }
+    return elements;
+}
 /** A short text form of any verb for the history log — not shown to the
  * user, just fed back to the model on later turns so it knows what it
  * already did/said. */

package/dist/runtime-scan.d.ts

@@ -0,0 +1,36 @@
+import type { LiveElement } from "@cairnvibe/core";
+export interface LiveScan {
+    elements: LiveElement[];
+    byId: Map<string, HTMLElement>;
+}
+/**
+ * Scans the live DOM for interactive elements currently in the viewport.
+ * Returns both the bounded list to send to the model (`elements`, capped at
+ * MAX_ELEMENTS and MAX_LABEL_LENGTH — the actual privacy/payload backstop,
+ * mirrored server-side in CopilotRequestSchema) and the real elements it
+ * maps to, keyed by the same ids (`byId`) — resolve a verb's target by
+ * looking it up here, never by re-deriving a selector from the id string.
+ */
+export declare function scanInteractiveElements(root?: ParentNode): LiveScan;
+export interface LiveElementRegistry {
+    /** Starts continuous background scanning — call once, typically on mount. */
+    start(): void;
+    stop(): void;
+    /**
+     * Freezes the current scan for one request/response round trip. Call
+     * this once when a question is sent, and resolve that turn's verb
+     * against exactly this snapshot — not a fresh call — so a background
+     * rescan that lands mid-flight can't shift what an id resolves to
+     * between when the request went out and when the response comes back.
+     */
+    getSnapshot(): LiveScan;
+}
+/**
+ * Keeps a scan continuously fresh in the background via a debounced
+ * MutationObserver (plus scroll/resize, since viewport membership changes
+ * without any DOM mutation) instead of only scanning at the moment a
+ * question is asked — so the agent never has to pause to "go look at the
+ * page" right when it needs to click something; a sub-agent gathering
+ * context while the main conversation keeps moving.
+ */
+export declare function createLiveElementRegistry(): LiveElementRegistry;

package/dist/runtime-scan.js

@@ -0,0 +1,113 @@
+"use strict";
+// A live inventory of what's actually clickable on screen right now — not
+// the build-time manifest, and not limited to elements a developer
+// remembered to tag with data-ai. This is what lets the agent address a
+// dynamically-rendered row (a session card, a list item) it was never told
+// about ahead of time. Deliberately narrow in what counts as "interactive":
+// the same semantic-clickable set element-ladder.ts's own fallback search
+// already uses, plus anything carrying data-ai — a plain `<div onClick>`
+// with no semantic role is invisible to this, same limit the existing
+// ladder already has.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.scanInteractiveElements = scanInteractiveElements;
+exports.createLiveElementRegistry = createLiveElementRegistry;
+const CANDIDATE_SELECTOR = "[data-ai], button, a, [role='button'], input[type='submit'], input[type='button']";
+const MAX_ELEMENTS = 40;
+const MAX_LABEL_LENGTH = 80;
+const RESCAN_DEBOUNCE_MS = 250;
+function isInViewport(el) {
+    const rect = el.getBoundingClientRect();
+    return rect.bottom > 0 && rect.top < window.innerHeight && rect.right > 0 && rect.left < window.innerWidth;
+}
+function labelFor(el) {
+    const raw = el.getAttribute("aria-label") || el.textContent || "";
+    const trimmed = raw.replace(/\s+/g, " ").trim();
+    return trimmed.length > MAX_LABEL_LENGTH ? `${trimmed.slice(0, MAX_LABEL_LENGTH - 1)}…` : trimmed;
+}
+function roleFor(el) {
+    return el.getAttribute("role") || el.tagName.toLowerCase();
+}
+/**
+ * Scans the live DOM for interactive elements currently in the viewport.
+ * Returns both the bounded list to send to the model (`elements`, capped at
+ * MAX_ELEMENTS and MAX_LABEL_LENGTH — the actual privacy/payload backstop,
+ * mirrored server-side in CopilotRequestSchema) and the real elements it
+ * maps to, keyed by the same ids (`byId`) — resolve a verb's target by
+ * looking it up here, never by re-deriving a selector from the id string.
+ */
+function scanInteractiveElements(root = document) {
+    const elements = [];
+    const byId = new Map();
+    let counter = 0;
+    if (typeof document === "undefined")
+        return { elements, byId };
+    const candidates = root.querySelectorAll(CANDIDATE_SELECTOR);
+    for (const el of Array.from(candidates)) {
+        if (elements.length >= MAX_ELEMENTS)
+            break;
+        if (!isInViewport(el))
+            continue;
+        const dataAi = el.getAttribute("data-ai");
+        const id = dataAi ?? `live-${counter++}`;
+        if (byId.has(id))
+            continue; // a data-ai id already covered by an earlier match
+        const label = labelFor(el);
+        if (!label)
+            continue; // nothing to address it by — skip rather than send an empty label
+        byId.set(id, el);
+        elements.push({ id, role: roleFor(el), label });
+    }
+    return { elements, byId };
+}
+/**
+ * Keeps a scan continuously fresh in the background via a debounced
+ * MutationObserver (plus scroll/resize, since viewport membership changes
+ * without any DOM mutation) instead of only scanning at the moment a
+ * question is asked — so the agent never has to pause to "go look at the
+ * page" right when it needs to click something; a sub-agent gathering
+ * context while the main conversation keeps moving.
+ */
+function createLiveElementRegistry() {
+    let current = { elements: [], byId: new Map() };
+    let observer = null;
+    let debounceTimer = null;
+    function rescan() {
+        current = scanInteractiveElements();
+    }
+    function scheduleRescan() {
+        if (debounceTimer)
+            return;
+        debounceTimer = setTimeout(() => {
+            debounceTimer = null;
+            rescan();
+        }, RESCAN_DEBOUNCE_MS);
+    }
+    function start() {
+        if (typeof document === "undefined" || observer)
+            return;
+        rescan();
+        observer = new MutationObserver(scheduleRescan);
+        observer.observe(document.body, {
+            childList: true,
+            subtree: true,
+            attributes: true,
+            attributeFilter: ["data-ai", "aria-label"],
+        });
+        window.addEventListener("scroll", scheduleRescan, { passive: true });
+        window.addEventListener("resize", scheduleRescan);
+    }
+    function stop() {
+        observer?.disconnect();
+        observer = null;
+        if (debounceTimer) {
+            clearTimeout(debounceTimer);
+            debounceTimer = null;
+        }
+        window.removeEventListener("scroll", scheduleRescan);
+        window.removeEventListener("resize", scheduleRescan);
+    }
+    function getSnapshot() {
+        return current;
+    }
+    return { start, stop, getSnapshot };
+}

package/dist/server.d.ts

@@ -1,4 +1,4 @@
-import { type HistoryTurn, type Manifest, type VerbResponse } from "@cairnvibe/core";
+import { type HistoryTurn, type LiveElement, type Manifest, type VerbResponse } from "@cairnvibe/core";
 import { KeyRotator } from "./key-rotator";
 /**
  * What the agent is allowed to do, independent of which specific "do"
@@ -64,6 +64,7 @@ export declare function resolveVerb(llm: VerbLLM, systemPrompt: string, manifest
     question: string;
     visible: string[];
     history?: HistoryTurn[];
+    liveElements?: LiveElement[];
 }): Promise<VerbResponse>;
 /** Builds the provider-appropriate VerbLLM from the same options createCopilotHandler accepts — reused by the realtime relay. */
 export declare function createVerbLLM(options?: CreateCopilotHandlerOptions): VerbLLM;