@cairnvibe/sdk 0.2.7 → 0.2.9

package/src/speak-server.ts

@@ -1,12 +1,36 @@
 // Server-side text-to-speech for the Copilot widget's spoken answers (see
 // `speakEndpoint` in index.tsx). The Deepgram key must never reach the
-// client, so this is a plain fetch to Deepgram's /v1/speak REST endpoint —
-// no SDK dependency needed for one request shape.
+// client.
+//
+// This used to be one fetch to Deepgram's /v1/speak REST endpoint, buffered
+// into an ArrayBuffer with `await response.arrayBuffer()` before returning
+// anything. That's the exact same bug the realtime path already fixed once
+// (see tts-stream.ts's own comment): nothing plays until Deepgram renders
+// AND the network delivers the *entire* reply, which measured 5-8s for a
+// normal explain answer in real production logs. It also hit Deepgram's
+// REST-only 2000-character cap on longer replies with no handling at all.
+//
+// Fixed the same way the realtime path was: open the streaming Speak
+// WebSocket (tts-stream.ts's DeepgramSpeakStream, the same class the
+// realtime server already uses) and forward audio chunks to the caller as
+// they arrive, via a ReadableStream — not buffered. The route handler
+// forwards that stream straight through as the HTTP response body, and the
+// client (index.tsx) reads it progressively instead of awaiting a full
+// Blob, so playback can start on the first chunk. Splitting the text into
+// sentence-sized `Speak` messages before one `Flush` sidesteps the old
+// 2000-char REST limit entirely (it doesn't apply to the WS protocol) and
+// lets Deepgram start rendering the first sentence sooner.
+import { DeepgramSpeakStream, type DeepgramSpeakStreamOptions, type SpeakChunkCallback } from "./tts-stream";
 
-const DEEPGRAM_SPEAK_URL = "https://api.deepgram.com/v1/speak";
-// Verified against Deepgram's docs while building this — re-check if this
-// starts erroring, voice model names retire over time.
 const DEEPGRAM_DEFAULT_VOICE = "aura-2-thalia-en";
+// Matches the realtime path's own playback sample rate (index.tsx's
+// audio_chunk handling defaults to 24000 too) — keeping them identical lets
+// both paths share one raw-PCM16 decode/schedule routine on the client.
+const SAMPLE_RATE = 24000;
+// Not a protocol limit (the WS Speak protocol has none like REST's 2000
+// chars) — just keeps each queued chunk sentence-sized so Deepgram can start
+// rendering the first one quickly instead of parsing one giant message.
+const MAX_CHUNK_CHARS = 300;
 
 export interface CreateSpeakHandlerOptions {
   apiKey: string;
@@ -15,13 +39,42 @@ export interface CreateSpeakHandlerOptions {
 
 export interface SpeakResult {
   status: number;
-  /** `audio` is raw MP3 bytes on success. */
-  body: { audio: ArrayBuffer; contentType: string } | { error: string };
+  /** `stream` yields raw linear16 PCM chunks (mono, 24kHz) as Deepgram
+   * renders them — forward it directly, unbuffered; do not await it into a
+   * Blob/ArrayBuffer or the whole point of streaming is lost. */
+  body: { stream: ReadableStream<Uint8Array>; contentType: string } | { error: string };
 }
 
 export type SpeakHandler = (text: string) => Promise<SpeakResult>;
 
-export function createSpeakHandler(options: CreateSpeakHandlerOptions): SpeakHandler {
+/** Test-only seam: lets tests inject a fake stream instead of opening a real
+ * Deepgram WebSocket. Not part of CreateSpeakHandlerOptions on purpose — real
+ * call sites (the scaffolded route templates) never pass this. */
+export type SpeakStreamFactory = (
+  opts: DeepgramSpeakStreamOptions,
+  onAudioChunk: SpeakChunkCallback,
+  handlers?: { onFlushed?: (sequenceId: number) => void; onError?: (err: Error) => void },
+) => DeepgramSpeakStream;
+
+function splitIntoChunks(text: string, maxChars: number): string[] {
+  const sentences = text.match(/[^.!?]+[.!?]*\s*/g) ?? [text];
+  const chunks: string[] = [];
+  let current = "";
+  for (const sentence of sentences) {
+    if (current && current.length + sentence.length > maxChars) {
+      chunks.push(current);
+      current = "";
+    }
+    current += sentence;
+  }
+  if (current) chunks.push(current);
+  return chunks;
+}
+
+export function createSpeakHandler(
+  options: CreateSpeakHandlerOptions,
+  streamFactory: SpeakStreamFactory = (opts, onAudioChunk, handlers) => new DeepgramSpeakStream(opts, onAudioChunk, handlers),
+): SpeakHandler {
   const model = options.model ?? process.env.DEEPGRAM_VOICE ?? DEEPGRAM_DEFAULT_VOICE;
 
   return async function handleSpeak(text: string) {
@@ -29,28 +82,51 @@ export function createSpeakHandler(options: CreateSpeakHandlerOptions): SpeakHan
       return { status: 400, body: { error: "no text provided" } };
     }
 
-    let response: Response;
-    try {
-      response = await fetch(`${DEEPGRAM_SPEAK_URL}?model=${encodeURIComponent(model)}`, {
-        method: "POST",
-        headers: {
-          Authorization: `Token ${options.apiKey}`,
-          "content-type": "application/json",
+    let enqueue: ((chunk: Uint8Array) => void) | null = null;
+    let closeOut: (() => void) | null = null;
+    let failOut: ((err: Error) => void) | null = null;
+    const stream = new ReadableStream<Uint8Array>({
+      start(controller) {
+        enqueue = (chunk) => controller.enqueue(chunk);
+        closeOut = () => controller.close();
+        failOut = (err) => controller.error(err);
+      },
+    });
+
+    // True once connect() below resolves — an onError before that point is
+    // already reported through connect()'s own rejection, so it's ignored
+    // here to avoid double-handling the same failure.
+    let connected = false;
+
+    const speakStream = streamFactory(
+      { apiKey: options.apiKey, model, encoding: "linear16", sampleRate: SAMPLE_RATE },
+      (chunk) => enqueue?.(new Uint8Array(chunk)),
+      {
+        onFlushed: () => {
+          closeOut?.();
+          speakStream.close();
+        },
+        onError: (err) => {
+          if (!connected) return;
+          console.error("[cairn] speak stream error:", err);
+          failOut?.(err);
         },
-        body: JSON.stringify({ text }),
-      });
+      },
+    );
+
+    try {
+      await speakStream.connect();
     } catch (err) {
       console.error("[cairn] speak request failed:", err);
       return { status: 200, body: { error: "speech service unreachable" } };
     }
+    connected = true;
 
-    if (!response.ok) {
-      const detail = await response.text().catch(() => "");
-      console.error("[cairn] Deepgram speak returned an error:", response.status, detail);
-      return { status: 200, body: { error: "speech synthesis failed" } };
+    for (const chunk of splitIntoChunks(text, MAX_CHUNK_CHARS)) {
+      speakStream.sendText(chunk);
     }
+    speakStream.flush();
 
-    const audio = await response.arrayBuffer();
-    return { status: 200, body: { audio, contentType: response.headers.get("content-type") ?? "audio/mpeg" } };
+    return { status: 200, body: { stream, contentType: `audio/L16;rate=${SAMPLE_RATE}` } };
   };
 }

package/src/verb-executor.ts

@@ -6,7 +6,50 @@
 // enforces the same schema independently — never trust the client alone.
 
 import { VerbResponseSchema, type ApiCall, type TourStep, type VerbResponse } from "@cairnvibe/core";
-import { findElement, highlightElement, logMiss, type MissContext } from "./element-ladder";
+import { findElement, fillElement, highlightElement, logMiss, readElement, type MissContext } from "./element-ladder";
+import { executeWebMcpTool } from "./webmcp-client";
+
+/** The real result of one agent-loop step (click/fill/read/call_tool) —
+ * fed back to the model as its next turn's "observation" so it can decide
+ * what to do next instead of acting blind. The loop that drives this lives
+ * on the caller's side, not here: index.tsx's runTypedAgentLoop for the
+ * HTTP path, realtime-server.ts's finalizeTurn for the realtime one — this
+ * module only ever executes one step at a time. */
+export interface ToolStepResult {
+  verb: "click" | "fill" | "read" | "call_tool";
+  target?: string;
+  ok: boolean;
+  observation: string;
+}
+
+/**
+ * Promise wrapper around executeVerbResponse for a continuing verb
+ * (click/fill/read/call_tool) — resolves once the real action has actually
+ * finished (synchronously for click/fill/read, after a real await for
+ * call_tool) with its real observation, instead of the fire-and-forget
+ * callback shape every other verb uses. This is what a loop driver awaits
+ * before deciding whether to call the model again.
+ */
+export function executeToolStep(raw: unknown, route: string, liveElements?: Map<string, HTMLElement>): Promise<ToolStepResult | null> {
+  return new Promise((resolve) => {
+    // executeVerbResponse only ever reaches onToolStep for a genuinely
+    // continuing verb — callers are only expected to call this after
+    // already confirming (via TERMINAL_VERBS) that the parsed verb is one,
+    // so this should always fire; a real timeout (not an immediate
+    // microtask — call_tool's own real network round trip needs the time)
+    // is the safety net for the case where it somehow doesn't, so a loop
+    // driver awaiting this can never hang forever.
+    const timer = setTimeout(() => resolve(null), 15000);
+    executeVerbResponse(raw, route, {
+      onExplain: () => {},
+      liveElements,
+      onToolStep: (result) => {
+        clearTimeout(timer);
+        resolve(result);
+      },
+    });
+  });
+}
 
 export interface VerbExecutorOptions {
   onExplain: (text: string) => void;
@@ -19,6 +62,9 @@ export interface VerbExecutorOptions {
    * owns the UI (progress display) and, for voice, the TTS sequencing.
    */
   onTour?: (steps: TourStep[]) => void;
+  /** A click/fill/read/call_tool step finished — see ToolStepResult. Only
+   * called for the agent loop's continuing verbs, never the terminal ones. */
+  onToolStep?: (result: ToolStepResult) => void;
   /** Action ids the customer has actually wired up. "do" is rejected for anything else. */
   registeredActions?: string[];
   /**
@@ -126,6 +172,63 @@ function dispatchVerb(verb: VerbResponse, route: string, options: VerbExecutorOp
         options.onExplain(verb.steps.map((s) => s.text).join(" "));
       }
       return;
+
+    // The agent loop's steps (server.ts's runAgentLoop) — each executes for
+    // real and reports a real observation back via onToolStep, instead of
+    // ending the turn the way every verb above does. `target` for these
+    // always came from the manifest/currentPageElements/liveElements this
+    // exact turn showed the model — never invented, same invariant as do.
+    case "click": {
+      if (verb.text) options.onExplain(verb.text);
+      const el = findElement(verb.target, options.liveElements);
+      if (!el) {
+        (options.onMiss ?? logMiss)({ attempted: verb.target, route });
+        options.onToolStep?.({ verb: "click", target: verb.target, ok: false, observation: "Could not find that element on the page." });
+        return;
+      }
+      highlightElement(el);
+      el.click();
+      options.onToolStep?.({ verb: "click", target: verb.target, ok: true, observation: "Clicked it." });
+      return;
+    }
+
+    case "fill": {
+      if (verb.text) options.onExplain(verb.text);
+      const el = findElement(verb.target, options.liveElements);
+      if (!el || !fillElement(el, verb.value)) {
+        (options.onMiss ?? logMiss)({ attempted: verb.target, route });
+        options.onToolStep?.({
+          verb: "fill",
+          target: verb.target,
+          ok: false,
+          observation: el ? "That element isn't a real form field — can't type into it." : "Could not find that element on the page.",
+        });
+        return;
+      }
+      highlightElement(el);
+      options.onToolStep?.({ verb: "fill", target: verb.target, ok: true, observation: `Typed "${verb.value}" into it.` });
+      return;
+    }
+
+    case "read": {
+      if (verb.text) options.onExplain(verb.text);
+      const el = findElement(verb.target, options.liveElements);
+      if (!el) {
+        (options.onMiss ?? logMiss)({ attempted: verb.target, route });
+        options.onToolStep?.({ verb: "read", target: verb.target, ok: false, observation: "Could not find that element on the page." });
+        return;
+      }
+      options.onToolStep?.({ verb: "read", target: verb.target, ok: true, observation: readElement(el) });
+      return;
+    }
+
+    case "call_tool": {
+      if (verb.text) options.onExplain(verb.text);
+      void executeWebMcpTool(verb.name, verb.args).then((result) => {
+        options.onToolStep?.({ verb: "call_tool", target: verb.name, ok: result.ok, observation: result.observation });
+      });
+      return;
+    }
   }
 }
 

package/src/webmcp-client.ts

@@ -0,0 +1,79 @@
+// Discovers and calls real tools a page has registered via WebMCP
+// (https://webmachinelearning.github.io/webmcp/) — an in-progress web
+// standard letting a site expose its own functions as typed, parameterized
+// tools ("document.modelContext.registerTool(...)"), running in the page's
+// own JS with the user's real session. This is the highest-trust action
+// source there is: a real function the app's own developer wrote, with a
+// real return value — not a click simulated from static analysis. Almost
+// no site has adopted it yet, so this is deliberately a no-op (empty list,
+// nothing to call) everywhere it isn't present, not a hard dependency.
+
+import type { WebMcpTool } from "@cairnvibe/core";
+
+interface ModelContextTool {
+  name: string;
+  description?: string;
+  inputSchema?: Record<string, unknown>;
+}
+
+interface ModelContext {
+  getTools?: () => Promise<ModelContextTool[]> | ModelContextTool[];
+  executeTool?: (tool: ModelContextTool, args: unknown) => Promise<unknown>;
+}
+
+function getModelContext(): ModelContext | null {
+  if (typeof document === "undefined") return null;
+  return (document as unknown as { modelContext?: ModelContext }).modelContext ?? null;
+}
+
+/** Bounded the same way LiveElementSchema/CopilotRequestSchema bound the
+ * live DOM scan — a hard cap on both count and description length, so a
+ * page that registers an unreasonable number of tools (or one with a huge
+ * description) can't blow the request payload or the prompt. */
+const MAX_TOOLS = 30;
+const MAX_DESCRIPTION_LENGTH = 500;
+
+export async function discoverWebMcpTools(): Promise<WebMcpTool[]> {
+  const modelContext = getModelContext();
+  if (!modelContext?.getTools) return [];
+
+  try {
+    const tools = await modelContext.getTools();
+    if (!Array.isArray(tools)) return [];
+    return tools.slice(0, MAX_TOOLS).map((tool) => ({
+      name: String(tool.name),
+      description: String(tool.description ?? "").slice(0, MAX_DESCRIPTION_LENGTH),
+      inputSchema: tool.inputSchema,
+    }));
+  } catch {
+    // A page's own registerTool()/getTools() implementation throwing is
+    // that page's bug, not Cairn's — degrade to "no WebMCP tools" rather
+    // than breaking the rest of the turn.
+    return [];
+  }
+}
+
+/**
+ * Calls a real WebMCP tool by name — `name` must be one the model was
+ * actually shown this turn (server.ts only ever includes tools from this
+ * exact request's own discoverWebMcpTools() call), never invented.
+ * Returns a plain-text observation for the agent loop to reason about
+ * next, the same shape a click/fill/read result already takes.
+ */
+export async function executeWebMcpTool(name: string, args: Record<string, unknown> | undefined): Promise<{ ok: boolean; observation: string }> {
+  const modelContext = getModelContext();
+  if (!modelContext?.getTools || !modelContext.executeTool) {
+    return { ok: false, observation: "This page no longer has that tool available." };
+  }
+  try {
+    const tools = await modelContext.getTools();
+    const tool = Array.isArray(tools) ? tools.find((t) => t.name === name) : undefined;
+    if (!tool) return { ok: false, observation: `No tool named "${name}" is available on this page right now.` };
+
+    const result = await modelContext.executeTool(tool, args ?? {});
+    const observation = typeof result === "string" ? result : JSON.stringify(result ?? null);
+    return { ok: true, observation: observation.slice(0, 2000) };
+  } catch (err) {
+    return { ok: false, observation: `That tool failed: ${err instanceof Error ? err.message : "unknown error"}` };
+  }
+}