@cairnvibe/sdk 0.2.7 → 0.2.9

package/src/index.tsx

@@ -16,11 +16,12 @@ import {
   VolumeX,
   X,
 } from "lucide-react";
-import type { HistoryTurn as HistoryEntry, TourStep } from "@cairnvibe/core";
+import { TERMINAL_VERBS, safeParseVerbResponse, type HistoryTurn as HistoryEntry, type TourStep } from "@cairnvibe/core";
 import { collectVisible } from "./context-collector";
 import { findElement, highlightElement, logMiss, type MissContext } from "./element-ladder";
 import { createLiveElementRegistry } from "./runtime-scan";
-import { executeVerbResponse } from "./verb-executor";
+import { discoverWebMcpTools } from "./webmcp-client";
+import { executeToolStep, executeVerbResponse } from "./verb-executor";
 
 export interface CopilotProps {
   /** Reserved for a future client-side manifest fetch. Not required — the server handler owns the manifest. */
@@ -74,7 +75,7 @@ export function Copilot({
   const pathnameRef = useRef(pathname);
   useEffect(() => {
     pathnameRef.current = pathname;
-    sendFreshContext(); // no-op if no realtime session is open
+    void sendFreshContext(); // no-op if no realtime session is open
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [pathname]);
   const router = useRouter();
@@ -160,7 +161,18 @@ export function Copilot({
   const rtMicMutedRef = useRef(false);
   const rtSpeakerMutedRef = useRef(false);
   const rtStartingRef = useRef(false); // closes the click-to-first-state-update gap so a rapid double-click can't open two sessions
-  const activeAudioRef = useRef<HTMLAudioElement | null>(null);
+  // Progressive PCM playback for the buffered (non-realtime) speak endpoint
+  // — the same gapless AudioBufferSourceNode scheduling the realtime path
+  // uses for its audio_chunk messages (see rtPlaybackCtxRef below), just fed
+  // by a fetch() ReadableStream instead of WebSocket messages. This exists
+  // because res.blob()/res.arrayBuffer() always wait for the whole response
+  // body in every browser no matter how the server sent it — streaming the
+  // wire alone (speak-server.ts) doesn't help unless playback also starts
+  // before the full reply has arrived.
+  const typedPlaybackCtxRef = useRef<AudioContext | null>(null);
+  const typedPlaybackGainRef = useRef<GainNode | null>(null);
+  const typedNextPlayTimeRef = useRef(0);
+  const typedScheduledSourcesRef = useRef<AudioBufferSourceNode[]>([]);
   // Watchdog for the "rt-thinking" state: started on every "final" transcript,
   // cleared the moment the server responds with anything for that turn
   // (verb/speaking_start/speaking_end/turn_complete/error). If it ever
@@ -258,13 +270,21 @@ export function Copilot({
    * `pathname`, so it's correct even called from a handler created once at
    * connection-open time.
    */
-  function sendFreshContext() {
+  async function sendFreshContext() {
     const ws = rtSocketRef.current;
     if (!ws || ws.readyState !== WebSocket.OPEN) return;
     const liveScan = liveRegistryRef.current.getSnapshot();
     liveMapRef.current = liveScan.byId;
+    const webMcpTools = await discoverWebMcpTools();
+    if (ws.readyState !== WebSocket.OPEN) return; // may have closed while awaiting discovery
     ws.send(
-      JSON.stringify({ type: "context", route: pathnameRef.current, visible: collectVisible(), liveElements: liveScan.elements }),
+      JSON.stringify({
+        type: "context",
+        route: pathnameRef.current,
+        visible: collectVisible(),
+        liveElements: liveScan.elements,
+        webMcpTools,
+      }),
     );
   }
 
@@ -407,6 +427,34 @@ export function Copilot({
     setLastQuestion(q);
     setQuestion("");
     try {
+      await runTypedAgentLoop(q);
+    } catch {
+      setAnswer("Something went wrong reaching the help service — try again in a moment.");
+    } finally {
+      setStatus("idle");
+    }
+  }
+
+  const MAX_LOOP_ITERATIONS = 6; // a hard cap, not a target — see runTypedAgentLoop
+
+  /**
+   * Drives the agent loop over the stateless HTTP path: ask the server,
+   * and if it comes back with a continuing step (click/fill/read/
+   * call_tool — TERMINAL_VERBS in @cairnvibe/core says which verbs end a
+   * turn), execute that step for real, fold the real result into this
+   * turn's own working history, and ask again — repeat until a terminal
+   * verb or the iteration cap, instead of the old one-call-one-answer
+   * shape. `question` stays the original ask on every call; only
+   * `history` grows with each step's real trace, so the model always
+   * still knows what it was actually asked. `historyRef` (the
+   * conversation's real memory) is only ever committed once, at the end —
+   * a turn that hits the cap mid-loop doesn't leave partial noise in it.
+   */
+  async function runTypedAgentLoop(q: string): Promise<void> {
+    let loopHistory = historyRef.current;
+    const webMcpTools = await discoverWebMcpTools();
+
+    for (let i = 0; i < MAX_LOOP_ITERATIONS; i++) {
       const liveScan = liveRegistryRef.current.getSnapshot();
       liveMapRef.current = liveScan.byId;
       const res = await fetch(endpoint, {
@@ -416,51 +464,152 @@ export function Copilot({
           route: pathname,
           question: q,
           visible: collectVisible(),
-          history: historyRef.current,
+          history: loopHistory,
           liveElements: liveScan.elements,
+          webMcpTools,
         }),
       });
       const data = await res.json().catch(() => null);
-      handleVerb(data);
-      // Unlike the realtime relay (one persistent connection, memory lives
-      // server-side), each of these POSTs is stateless — the widget itself
-      // is what remembers, and resends it above so the model has context
-      // for "the first one" / "do that instead" on the next question.
-      historyRef.current = [
-        ...historyRef.current,
-        { role: "user", text: q } satisfies HistoryEntry,
-        { role: "assistant", text: summarizeVerbForHistory(data) } satisfies HistoryEntry,
+      const parsed = safeParseVerbResponse(data);
+
+      if (!parsed || TERMINAL_VERBS.has(parsed.verb)) {
+        handleVerb(data);
+        // Unlike the realtime relay (one persistent connection, memory
+        // lives server-side), each of these POSTs is stateless — the
+        // widget itself is what remembers, and resends it above so the
+        // model has context for "the first one" / "do that instead" on
+        // the next question.
+        historyRef.current = [
+          ...loopHistory,
+          { role: "user", text: q } satisfies HistoryEntry,
+          { role: "assistant", text: summarizeVerbForHistory(data) } satisfies HistoryEntry,
+        ].slice(-MAX_HISTORY_TURNS);
+        return;
+      }
+
+      // A continuing step — show it happening, execute it for real, and
+      // go around again with the real result instead of ending the turn.
+      setAnswer(summarizeVerbForHistory(data));
+      const stepResult = await executeToolStep(data, pathname, liveMapRef.current);
+      loopHistory = [
+        ...loopHistory,
+        {
+          role: "assistant",
+          text: `${summarizeVerbForHistory(data)}. Result: ${stepResult?.observation ?? "no result"}`,
+        } satisfies HistoryEntry,
       ].slice(-MAX_HISTORY_TURNS);
-    } catch {
-      setAnswer("Something went wrong reaching the help service — try again in a moment.");
-    } finally {
-      setStatus("idle");
     }
+
+    setAnswer("I wasn't able to finish that — try asking again or breaking it into smaller steps.");
+    historyRef.current = [
+      ...loopHistory,
+      { role: "user", text: q } satisfies HistoryEntry,
+      { role: "assistant", text: "(gave up after too many steps)" } satisfies HistoryEntry,
+    ].slice(-MAX_HISTORY_TURNS);
+  }
+
+  function ensureTypedPlaybackGraph(): { ctx: AudioContext; gain: GainNode } {
+    if (!typedPlaybackCtxRef.current) {
+      const ctx = new AudioContext();
+      const gain = ctx.createGain();
+      gain.connect(ctx.destination);
+      typedPlaybackCtxRef.current = ctx;
+      typedPlaybackGainRef.current = gain;
+    }
+    return { ctx: typedPlaybackCtxRef.current, gain: typedPlaybackGainRef.current! };
+  }
+
+  /** Stops whatever's currently playing on the typed/mic path's playback
+   * graph, so two responses (e.g. a rapid double-click, or two answers
+   * resolved close together) can never be heard overlapping. */
+  function stopTypedPlayback() {
+    for (const source of typedScheduledSourcesRef.current) {
+      source.onended = null;
+      try {
+        source.stop();
+      } catch {
+        // may already have finished naturally
+      }
+    }
+    typedScheduledSourcesRef.current = [];
+    typedNextPlayTimeRef.current = typedPlaybackCtxRef.current?.currentTime ?? 0;
+  }
+
+  function concatBytes(a: Uint8Array, b: Uint8Array): Uint8Array {
+    const out = new Uint8Array(a.length + b.length);
+    out.set(a, 0);
+    out.set(b, a.length);
+    return out;
   }
 
   /**
-   * The one place that starts audio playback for a spoken response — stops
-   * whatever's currently playing first, so two responses (e.g. a rapid
-   * double-click on "start conversation", or two utterances resolved close
-   * together) can never be heard overlapping. Used by both the typed/mic
-   * path and the realtime path.
+   * Reads a raw linear16 PCM stream (mono, 24kHz — matches speak-server.ts)
+   * and schedules it gapless-appended into the Web Audio graph as chunks
+   * arrive — the same technique the realtime path uses for its audio_chunk
+   * messages, just driven by a fetch() reader instead of WebSocket frames.
+   * Resolves once every scheduled chunk has actually finished *playing*,
+   * not just finished arriving.
    */
-  function playResponseAudio(blob: Blob): Promise<void> {
-    if (activeAudioRef.current) {
-      activeAudioRef.current.pause();
-      activeAudioRef.current.currentTime = 0;
-    }
-    const url = URL.createObjectURL(blob);
-    const audio = new Audio(url);
-    activeAudioRef.current = audio;
+  function playPcmStream(stream: ReadableStream<Uint8Array>): Promise<void> {
+    stopTypedPlayback();
+    const { ctx, gain } = ensureTypedPlaybackGraph();
+    void ctx.resume().catch(() => {});
+
     return new Promise((resolve) => {
-      const clear = () => {
-        URL.revokeObjectURL(url);
-        if (activeAudioRef.current === audio) activeAudioRef.current = null;
-        resolve();
+      let doneArriving = false;
+      let leftover: Uint8Array<ArrayBufferLike> = new Uint8Array(0);
+
+      const maybeResolve = () => {
+        if (doneArriving && typedScheduledSourcesRef.current.length === 0) resolve();
+      };
+
+      const scheduleChunk = (bytes: Uint8Array) => {
+        const sampleCount = Math.floor(bytes.length / 2);
+        if (sampleCount === 0) return;
+        const float32 = new Float32Array(sampleCount);
+        const view = new DataView(bytes.buffer, bytes.byteOffset, sampleCount * 2);
+        for (let i = 0; i < sampleCount; i++) float32[i] = view.getInt16(i * 2, true) / 32768;
+
+        const buffer = ctx.createBuffer(1, sampleCount, 24000);
+        buffer.copyToChannel(float32, 0);
+
+        const source = ctx.createBufferSource();
+        source.buffer = buffer;
+        source.connect(gain);
+
+        const startAt = Math.max(ctx.currentTime, typedNextPlayTimeRef.current);
+        source.start(startAt);
+        typedNextPlayTimeRef.current = startAt + buffer.duration;
+
+        typedScheduledSourcesRef.current.push(source);
+        source.onended = () => {
+          typedScheduledSourcesRef.current = typedScheduledSourcesRef.current.filter((s) => s !== source);
+          maybeResolve();
+        };
       };
-      audio.onended = clear;
-      audio.play().catch(clear);
+
+      (async () => {
+        const reader = stream.getReader();
+        try {
+          for (;;) {
+            const { done, value } = await reader.read();
+            if (done) break;
+            if (!value || value.length === 0) continue;
+            // PCM16 samples are 2 bytes each — a chunk boundary can split a
+            // sample in half, so carry any odd trailing byte into the next
+            // read instead of corrupting one sample at every chunk seam.
+            const combined = concatBytes(leftover, value);
+            const usableLen = combined.length - (combined.length % 2);
+            scheduleChunk(combined.subarray(0, usableLen));
+            leftover = combined.subarray(usableLen);
+          }
+        } catch {
+          // Best-effort — never let a stream read failure hang the caller forever.
+        } finally {
+          doneArriving = true;
+          maybeResolve();
+        }
+      })();
     });
   }
 
@@ -472,8 +621,8 @@ export function Copilot({
         headers: { "content-type": "application/json" },
         body: JSON.stringify({ text }),
       });
-      if (!res.ok) return;
-      void playResponseAudio(await res.blob());
+      if (!res.ok || !res.body) return;
+      void playPcmStream(res.body);
     } catch {
       // Best-effort — never let speech playback break the widget.
     }
@@ -490,8 +639,8 @@ export function Copilot({
         headers: { "content-type": "application/json" },
         body: JSON.stringify({ text }),
       });
-      if (!res.ok) return;
-      await playResponseAudio(await res.blob());
+      if (!res.ok || !res.body) return;
+      await playPcmStream(res.body);
     } catch {
       // Best-effort — never let a synthesis failure hang the tour forever.
     }
@@ -692,7 +841,7 @@ export function Copilot({
         }
         setRtStatus("rt-listening");
         setCaption("");
-        sendFreshContext(); // refresh before the user starts talking again, not after
+        void sendFreshContext(); // refresh before the user starts talking again, not after
       }
 
       function disarmThinkingWatchdog() {
@@ -769,6 +918,25 @@ export function Copilot({
           setRtStatus("rt-thinking");
           armThinkingWatchdog();
         } else if (msg.type === "verb") {
+          const parsedStep = safeParseVerbResponse(msg.verb);
+          if (parsedStep && !TERMINAL_VERBS.has(parsedStep.verb)) {
+            // A continuing agent-loop step (click/fill/read/call_tool) —
+            // the turn isn't over: execute it for real and report the
+            // result back so the server can decide the next step, instead
+            // of treating this like a normal answer (no
+            // disarmThinkingWatchdog/handleVerb — those are for when a
+            // turn actually ends). Shown visually so a multi-step turn
+            // reads as visible progress, not a silent pause; never spoken
+            // — the server's loop stays quiet between steps on purpose,
+            // to keep it fast.
+            setAnswer(summarizeVerbForHistory(msg.verb));
+            void executeToolStep(msg.verb, pathnameRef.current, liveMapRef.current).then((result) => {
+              if (ws.readyState === WebSocket.OPEN) {
+                ws.send(JSON.stringify({ type: "tool_result", observation: result?.observation ?? "no result" }));
+              }
+            });
+            return;
+          }
           disarmThinkingWatchdog();
           handleVerb(msg.verb);
         } else if (msg.type === "speaking_start") {
@@ -860,8 +1028,7 @@ export function Copilot({
       rtThinkingWatchdogRef.current = null;
     }
     rtStartingRef.current = false;
-    activeAudioRef.current?.pause();
-    activeAudioRef.current = null;
+    stopTypedPlayback();
     rtSocketRef.current?.close();
     rtSocketRef.current = null;
     rtCleanupRef.current?.();
@@ -1072,6 +1239,14 @@ function summarizeVerbForHistory(raw: unknown): string {
       return `(ran ${String(v.action)}${v.target ? ` on ${String(v.target)}` : ""})`;
     case "tour":
       return Array.isArray(v.steps) ? v.steps.map((s: { text?: string }) => s.text ?? "").join(" ") : "(tour)";
+    case "click":
+      return `(clicked ${String(v.target)})`;
+    case "fill":
+      return `(typed "${String(v.value)}" into ${String(v.target)})`;
+    case "read":
+      return `(read ${String(v.target)})`;
+    case "call_tool":
+      return `(called ${String(v.name)})`;
     default:
       return "(no response)";
   }