@cairnvibe/sdk 0.2.7 → 0.2.9

package/dist/index.js

@@ -6,9 +6,11 @@ const jsx_runtime_1 = require("react/jsx-runtime");
 const react_1 = require("react");
 const navigation_1 = require("next/navigation");
 const lucide_react_1 = require("lucide-react");
+const core_1 = require("@cairnvibe/core");
 const context_collector_1 = require("./context-collector");
 const element_ladder_1 = require("./element-ladder");
 const runtime_scan_1 = require("./runtime-scan");
+const webmcp_client_1 = require("./webmcp-client");
 const verb_executor_1 = require("./verb-executor");
 function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, reportMissesEndpoint, transcribeEndpoint, speakEndpoint, realtimeUrl, persona = "Cairn", }) {
     const pathname = (0, navigation_1.usePathname)() ?? "/";
@@ -20,7 +22,7 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
     const pathnameRef = (0, react_1.useRef)(pathname);
     (0, react_1.useEffect)(() => {
         pathnameRef.current = pathname;
-        sendFreshContext(); // no-op if no realtime session is open
+        void sendFreshContext(); // no-op if no realtime session is open
         // eslint-disable-next-line react-hooks/exhaustive-deps
     }, [pathname]);
     const router = (0, navigation_1.useRouter)();
@@ -104,7 +106,18 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
     const rtMicMutedRef = (0, react_1.useRef)(false);
     const rtSpeakerMutedRef = (0, react_1.useRef)(false);
     const rtStartingRef = (0, react_1.useRef)(false); // closes the click-to-first-state-update gap so a rapid double-click can't open two sessions
-    const activeAudioRef = (0, react_1.useRef)(null);
+    // Progressive PCM playback for the buffered (non-realtime) speak endpoint
+    // — the same gapless AudioBufferSourceNode scheduling the realtime path
+    // uses for its audio_chunk messages (see rtPlaybackCtxRef below), just fed
+    // by a fetch() ReadableStream instead of WebSocket messages. This exists
+    // because res.blob()/res.arrayBuffer() always wait for the whole response
+    // body in every browser no matter how the server sent it — streaming the
+    // wire alone (speak-server.ts) doesn't help unless playback also starts
+    // before the full reply has arrived.
+    const typedPlaybackCtxRef = (0, react_1.useRef)(null);
+    const typedPlaybackGainRef = (0, react_1.useRef)(null);
+    const typedNextPlayTimeRef = (0, react_1.useRef)(0);
+    const typedScheduledSourcesRef = (0, react_1.useRef)([]);
     // Watchdog for the "rt-thinking" state: started on every "final" transcript,
     // cleared the moment the server responds with anything for that turn
     // (verb/speaking_start/speaking_end/turn_complete/error). If it ever
@@ -194,13 +207,22 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
      * `pathname`, so it's correct even called from a handler created once at
      * connection-open time.
      */
-    function sendFreshContext() {
+    async function sendFreshContext() {
         const ws = rtSocketRef.current;
         if (!ws || ws.readyState !== WebSocket.OPEN)
             return;
         const liveScan = liveRegistryRef.current.getSnapshot();
         liveMapRef.current = liveScan.byId;
-        ws.send(JSON.stringify({ type: "context", route: pathnameRef.current, visible: (0, context_collector_1.collectVisible)(), liveElements: liveScan.elements }));
+        const webMcpTools = await (0, webmcp_client_1.discoverWebMcpTools)();
+        if (ws.readyState !== WebSocket.OPEN)
+            return; // may have closed while awaiting discovery
+        ws.send(JSON.stringify({
+            type: "context",
+            route: pathnameRef.current,
+            visible: (0, context_collector_1.collectVisible)(),
+            liveElements: liveScan.elements,
+            webMcpTools,
+        }));
     }
     function reportMiss(context) {
         (0, element_ladder_1.logMiss)(context);
@@ -346,6 +368,33 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
         setLastQuestion(q);
         setQuestion("");
         try {
+            await runTypedAgentLoop(q);
+        }
+        catch {
+            setAnswer("Something went wrong reaching the help service — try again in a moment.");
+        }
+        finally {
+            setStatus("idle");
+        }
+    }
+    const MAX_LOOP_ITERATIONS = 6; // a hard cap, not a target — see runTypedAgentLoop
+    /**
+     * Drives the agent loop over the stateless HTTP path: ask the server,
+     * and if it comes back with a continuing step (click/fill/read/
+     * call_tool — TERMINAL_VERBS in @cairnvibe/core says which verbs end a
+     * turn), execute that step for real, fold the real result into this
+     * turn's own working history, and ask again — repeat until a terminal
+     * verb or the iteration cap, instead of the old one-call-one-answer
+     * shape. `question` stays the original ask on every call; only
+     * `history` grows with each step's real trace, so the model always
+     * still knows what it was actually asked. `historyRef` (the
+     * conversation's real memory) is only ever committed once, at the end —
+     * a turn that hits the cap mid-loop doesn't leave partial noise in it.
+     */
+    async function runTypedAgentLoop(q) {
+        let loopHistory = historyRef.current;
+        const webMcpTools = await (0, webmcp_client_1.discoverWebMcpTools)();
+        for (let i = 0; i < MAX_LOOP_ITERATIONS; i++) {
             const liveScan = liveRegistryRef.current.getSnapshot();
             liveMapRef.current = liveScan.byId;
             const res = await fetch(endpoint, {
@@ -355,53 +404,145 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
                     route: pathname,
                     question: q,
                     visible: (0, context_collector_1.collectVisible)(),
-                    history: historyRef.current,
+                    history: loopHistory,
                     liveElements: liveScan.elements,
+                    webMcpTools,
                 }),
             });
             const data = await res.json().catch(() => null);
-            handleVerb(data);
-            // Unlike the realtime relay (one persistent connection, memory lives
-            // server-side), each of these POSTs is stateless — the widget itself
-            // is what remembers, and resends it above so the model has context
-            // for "the first one" / "do that instead" on the next question.
-            historyRef.current = [
-                ...historyRef.current,
-                { role: "user", text: q },
-                { role: "assistant", text: summarizeVerbForHistory(data) },
+            const parsed = (0, core_1.safeParseVerbResponse)(data);
+            if (!parsed || core_1.TERMINAL_VERBS.has(parsed.verb)) {
+                handleVerb(data);
+                // Unlike the realtime relay (one persistent connection, memory
+                // lives server-side), each of these POSTs is stateless — the
+                // widget itself is what remembers, and resends it above so the
+                // model has context for "the first one" / "do that instead" on
+                // the next question.
+                historyRef.current = [
+                    ...loopHistory,
+                    { role: "user", text: q },
+                    { role: "assistant", text: summarizeVerbForHistory(data) },
+                ].slice(-MAX_HISTORY_TURNS);
+                return;
+            }
+            // A continuing step — show it happening, execute it for real, and
+            // go around again with the real result instead of ending the turn.
+            setAnswer(summarizeVerbForHistory(data));
+            const stepResult = await (0, verb_executor_1.executeToolStep)(data, pathname, liveMapRef.current);
+            loopHistory = [
+                ...loopHistory,
+                {
+                    role: "assistant",
+                    text: `${summarizeVerbForHistory(data)}. Result: ${stepResult?.observation ?? "no result"}`,
+                },
             ].slice(-MAX_HISTORY_TURNS);
         }
-        catch {
-            setAnswer("Something went wrong reaching the help service — try again in a moment.");
+        setAnswer("I wasn't able to finish that — try asking again or breaking it into smaller steps.");
+        historyRef.current = [
+            ...loopHistory,
+            { role: "user", text: q },
+            { role: "assistant", text: "(gave up after too many steps)" },
+        ].slice(-MAX_HISTORY_TURNS);
+    }
+    function ensureTypedPlaybackGraph() {
+        if (!typedPlaybackCtxRef.current) {
+            const ctx = new AudioContext();
+            const gain = ctx.createGain();
+            gain.connect(ctx.destination);
+            typedPlaybackCtxRef.current = ctx;
+            typedPlaybackGainRef.current = gain;
         }
-        finally {
-            setStatus("idle");
+        return { ctx: typedPlaybackCtxRef.current, gain: typedPlaybackGainRef.current };
+    }
+    /** Stops whatever's currently playing on the typed/mic path's playback
+     * graph, so two responses (e.g. a rapid double-click, or two answers
+     * resolved close together) can never be heard overlapping. */
+    function stopTypedPlayback() {
+        for (const source of typedScheduledSourcesRef.current) {
+            source.onended = null;
+            try {
+                source.stop();
+            }
+            catch {
+                // may already have finished naturally
+            }
         }
+        typedScheduledSourcesRef.current = [];
+        typedNextPlayTimeRef.current = typedPlaybackCtxRef.current?.currentTime ?? 0;
+    }
+    function concatBytes(a, b) {
+        const out = new Uint8Array(a.length + b.length);
+        out.set(a, 0);
+        out.set(b, a.length);
+        return out;
     }
     /**
-     * The one place that starts audio playback for a spoken response — stops
-     * whatever's currently playing first, so two responses (e.g. a rapid
-     * double-click on "start conversation", or two utterances resolved close
-     * together) can never be heard overlapping. Used by both the typed/mic
-     * path and the realtime path.
+     * Reads a raw linear16 PCM stream (mono, 24kHz — matches speak-server.ts)
+     * and schedules it gapless-appended into the Web Audio graph as chunks
+     * arrive — the same technique the realtime path uses for its audio_chunk
+     * messages, just driven by a fetch() reader instead of WebSocket frames.
+     * Resolves once every scheduled chunk has actually finished *playing*,
+     * not just finished arriving.
      */
-    function playResponseAudio(blob) {
-        if (activeAudioRef.current) {
-            activeAudioRef.current.pause();
-            activeAudioRef.current.currentTime = 0;
-        }
-        const url = URL.createObjectURL(blob);
-        const audio = new Audio(url);
-        activeAudioRef.current = audio;
+    function playPcmStream(stream) {
+        stopTypedPlayback();
+        const { ctx, gain } = ensureTypedPlaybackGraph();
+        void ctx.resume().catch(() => { });
         return new Promise((resolve) => {
-            const clear = () => {
-                URL.revokeObjectURL(url);
-                if (activeAudioRef.current === audio)
-                    activeAudioRef.current = null;
-                resolve();
+            let doneArriving = false;
+            let leftover = new Uint8Array(0);
+            const maybeResolve = () => {
+                if (doneArriving && typedScheduledSourcesRef.current.length === 0)
+                    resolve();
             };
-            audio.onended = clear;
-            audio.play().catch(clear);
+            const scheduleChunk = (bytes) => {
+                const sampleCount = Math.floor(bytes.length / 2);
+                if (sampleCount === 0)
+                    return;
+                const float32 = new Float32Array(sampleCount);
+                const view = new DataView(bytes.buffer, bytes.byteOffset, sampleCount * 2);
+                for (let i = 0; i < sampleCount; i++)
+                    float32[i] = view.getInt16(i * 2, true) / 32768;
+                const buffer = ctx.createBuffer(1, sampleCount, 24000);
+                buffer.copyToChannel(float32, 0);
+                const source = ctx.createBufferSource();
+                source.buffer = buffer;
+                source.connect(gain);
+                const startAt = Math.max(ctx.currentTime, typedNextPlayTimeRef.current);
+                source.start(startAt);
+                typedNextPlayTimeRef.current = startAt + buffer.duration;
+                typedScheduledSourcesRef.current.push(source);
+                source.onended = () => {
+                    typedScheduledSourcesRef.current = typedScheduledSourcesRef.current.filter((s) => s !== source);
+                    maybeResolve();
+                };
+            };
+            (async () => {
+                const reader = stream.getReader();
+                try {
+                    for (;;) {
+                        const { done, value } = await reader.read();
+                        if (done)
+                            break;
+                        if (!value || value.length === 0)
+                            continue;
+                        // PCM16 samples are 2 bytes each — a chunk boundary can split a
+                        // sample in half, so carry any odd trailing byte into the next
+                        // read instead of corrupting one sample at every chunk seam.
+                        const combined = concatBytes(leftover, value);
+                        const usableLen = combined.length - (combined.length % 2);
+                        scheduleChunk(combined.subarray(0, usableLen));
+                        leftover = combined.subarray(usableLen);
+                    }
+                }
+                catch {
+                    // Best-effort — never let a stream read failure hang the caller forever.
+                }
+                finally {
+                    doneArriving = true;
+                    maybeResolve();
+                }
+            })();
         });
     }
     async function speak(text) {
@@ -413,9 +554,9 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
                 headers: { "content-type": "application/json" },
                 body: JSON.stringify({ text }),
             });
-            if (!res.ok)
+            if (!res.ok || !res.body)
                 return;
-            void playResponseAudio(await res.blob());
+            void playPcmStream(res.body);
         }
         catch {
             // Best-effort — never let speech playback break the widget.
@@ -433,9 +574,9 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
                 headers: { "content-type": "application/json" },
                 body: JSON.stringify({ text }),
             });
-            if (!res.ok)
+            if (!res.ok || !res.body)
                 return;
-            await playResponseAudio(await res.blob());
+            await playPcmStream(res.body);
         }
         catch {
             // Best-effort — never let a synthesis failure hang the tour forever.
@@ -641,7 +782,7 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
                 }
                 setRtStatus("rt-listening");
                 setCaption("");
-                sendFreshContext(); // refresh before the user starts talking again, not after
+                void sendFreshContext(); // refresh before the user starts talking again, not after
             }
             function disarmThinkingWatchdog() {
                 if (rtThinkingWatchdogRef.current) {
@@ -717,6 +858,25 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
                     armThinkingWatchdog();
                 }
                 else if (msg.type === "verb") {
+                    const parsedStep = (0, core_1.safeParseVerbResponse)(msg.verb);
+                    if (parsedStep && !core_1.TERMINAL_VERBS.has(parsedStep.verb)) {
+                        // A continuing agent-loop step (click/fill/read/call_tool) —
+                        // the turn isn't over: execute it for real and report the
+                        // result back so the server can decide the next step, instead
+                        // of treating this like a normal answer (no
+                        // disarmThinkingWatchdog/handleVerb — those are for when a
+                        // turn actually ends). Shown visually so a multi-step turn
+                        // reads as visible progress, not a silent pause; never spoken
+                        // — the server's loop stays quiet between steps on purpose,
+                        // to keep it fast.
+                        setAnswer(summarizeVerbForHistory(msg.verb));
+                        void (0, verb_executor_1.executeToolStep)(msg.verb, pathnameRef.current, liveMapRef.current).then((result) => {
+                            if (ws.readyState === WebSocket.OPEN) {
+                                ws.send(JSON.stringify({ type: "tool_result", observation: result?.observation ?? "no result" }));
+                            }
+                        });
+                        return;
+                    }
                     disarmThinkingWatchdog();
                     handleVerb(msg.verb);
                 }
@@ -810,8 +970,7 @@ function Copilot({ endpoint = "/api/copilot", registeredActions = [], onDo, repo
             rtThinkingWatchdogRef.current = null;
         }
         rtStartingRef.current = false;
-        activeAudioRef.current?.pause();
-        activeAudioRef.current = null;
+        stopTypedPlayback();
         rtSocketRef.current?.close();
         rtSocketRef.current = null;
         rtCleanupRef.current?.();
@@ -884,6 +1043,14 @@ function summarizeVerbForHistory(raw) {
             return `(ran ${String(v.action)}${v.target ? ` on ${String(v.target)}` : ""})`;
         case "tour":
             return Array.isArray(v.steps) ? v.steps.map((s) => s.text ?? "").join(" ") : "(tour)";
+        case "click":
+            return `(clicked ${String(v.target)})`;
+        case "fill":
+            return `(typed "${String(v.value)}" into ${String(v.target)})`;
+        case "read":
+            return `(read ${String(v.target)})`;
+        case "call_tool":
+            return `(called ${String(v.name)})`;
         default:
             return "(no response)";
     }

package/dist/realtime-server.d.ts

@@ -1,6 +1,6 @@
 import http from "node:http";
 import { WebSocket } from "ws";
-import type { HistoryTurn, LiveElement, Manifest } from "@cairnvibe/core";
+import { type HistoryTurn, type LiveElement, type Manifest, type WebMcpTool } from "@cairnvibe/core";
 import { createVerbLLM, type CapabilityTier, type CreateCopilotHandlerOptions } from "./server";
 export interface CreateRealtimeServerOptions extends CreateCopilotHandlerOptions {
     manifest: Manifest;
@@ -23,6 +23,7 @@ export declare function handleDeepgramMessage(raw: string, client: WebSocket, de
     route: string;
     visible: string[];
     liveElements: LiveElement[];
+    webMcpTools: WebMcpTool[];
 }, speakStreamed: (text: string) => Promise<void>, history: HistoryTurn[], turnState: {
     buffer: string;
-}, getGeneration: () => number): Promise<void>;
+}, getGeneration: () => number, waitForToolResult: () => Promise<string>): Promise<void>;