@cairnvibe/sdk 0.2.7 → 0.2.9

package/dist/speak-server.js

@@ -1,41 +1,97 @@
 "use strict";
-// Server-side text-to-speech for the Copilot widget's spoken answers (see
-// `speakEndpoint` in index.tsx). The Deepgram key must never reach the
-// client, so this is a plain fetch to Deepgram's /v1/speak REST endpoint —
-// no SDK dependency needed for one request shape.
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createSpeakHandler = createSpeakHandler;
-const DEEPGRAM_SPEAK_URL = "https://api.deepgram.com/v1/speak";
-// Verified against Deepgram's docs while building this — re-check if this
-// starts erroring, voice model names retire over time.
+// Server-side text-to-speech for the Copilot widget's spoken answers (see
+// `speakEndpoint` in index.tsx). The Deepgram key must never reach the
+// client.
+//
+// This used to be one fetch to Deepgram's /v1/speak REST endpoint, buffered
+// into an ArrayBuffer with `await response.arrayBuffer()` before returning
+// anything. That's the exact same bug the realtime path already fixed once
+// (see tts-stream.ts's own comment): nothing plays until Deepgram renders
+// AND the network delivers the *entire* reply, which measured 5-8s for a
+// normal explain answer in real production logs. It also hit Deepgram's
+// REST-only 2000-character cap on longer replies with no handling at all.
+//
+// Fixed the same way the realtime path was: open the streaming Speak
+// WebSocket (tts-stream.ts's DeepgramSpeakStream, the same class the
+// realtime server already uses) and forward audio chunks to the caller as
+// they arrive, via a ReadableStream — not buffered. The route handler
+// forwards that stream straight through as the HTTP response body, and the
+// client (index.tsx) reads it progressively instead of awaiting a full
+// Blob, so playback can start on the first chunk. Splitting the text into
+// sentence-sized `Speak` messages before one `Flush` sidesteps the old
+// 2000-char REST limit entirely (it doesn't apply to the WS protocol) and
+// lets Deepgram start rendering the first sentence sooner.
+const tts_stream_1 = require("./tts-stream");
 const DEEPGRAM_DEFAULT_VOICE = "aura-2-thalia-en";
-function createSpeakHandler(options) {
+// Matches the realtime path's own playback sample rate (index.tsx's
+// audio_chunk handling defaults to 24000 too) — keeping them identical lets
+// both paths share one raw-PCM16 decode/schedule routine on the client.
+const SAMPLE_RATE = 24000;
+// Not a protocol limit (the WS Speak protocol has none like REST's 2000
+// chars) — just keeps each queued chunk sentence-sized so Deepgram can start
+// rendering the first one quickly instead of parsing one giant message.
+const MAX_CHUNK_CHARS = 300;
+function splitIntoChunks(text, maxChars) {
+    const sentences = text.match(/[^.!?]+[.!?]*\s*/g) ?? [text];
+    const chunks = [];
+    let current = "";
+    for (const sentence of sentences) {
+        if (current && current.length + sentence.length > maxChars) {
+            chunks.push(current);
+            current = "";
+        }
+        current += sentence;
+    }
+    if (current)
+        chunks.push(current);
+    return chunks;
+}
+function createSpeakHandler(options, streamFactory = (opts, onAudioChunk, handlers) => new tts_stream_1.DeepgramSpeakStream(opts, onAudioChunk, handlers)) {
     const model = options.model ?? process.env.DEEPGRAM_VOICE ?? DEEPGRAM_DEFAULT_VOICE;
     return async function handleSpeak(text) {
         if (!text || !text.trim()) {
             return { status: 400, body: { error: "no text provided" } };
         }
-        let response;
+        let enqueue = null;
+        let closeOut = null;
+        let failOut = null;
+        const stream = new ReadableStream({
+            start(controller) {
+                enqueue = (chunk) => controller.enqueue(chunk);
+                closeOut = () => controller.close();
+                failOut = (err) => controller.error(err);
+            },
+        });
+        // True once connect() below resolves — an onError before that point is
+        // already reported through connect()'s own rejection, so it's ignored
+        // here to avoid double-handling the same failure.
+        let connected = false;
+        const speakStream = streamFactory({ apiKey: options.apiKey, model, encoding: "linear16", sampleRate: SAMPLE_RATE }, (chunk) => enqueue?.(new Uint8Array(chunk)), {
+            onFlushed: () => {
+                closeOut?.();
+                speakStream.close();
+            },
+            onError: (err) => {
+                if (!connected)
+                    return;
+                console.error("[cairn] speak stream error:", err);
+                failOut?.(err);
+            },
+        });
         try {
-            response = await fetch(`${DEEPGRAM_SPEAK_URL}?model=${encodeURIComponent(model)}`, {
-                method: "POST",
-                headers: {
-                    Authorization: `Token ${options.apiKey}`,
-                    "content-type": "application/json",
-                },
-                body: JSON.stringify({ text }),
-            });
+            await speakStream.connect();
         }
         catch (err) {
             console.error("[cairn] speak request failed:", err);
             return { status: 200, body: { error: "speech service unreachable" } };
         }
-        if (!response.ok) {
-            const detail = await response.text().catch(() => "");
-            console.error("[cairn] Deepgram speak returned an error:", response.status, detail);
-            return { status: 200, body: { error: "speech synthesis failed" } };
+        connected = true;
+        for (const chunk of splitIntoChunks(text, MAX_CHUNK_CHARS)) {
+            speakStream.sendText(chunk);
         }
-        const audio = await response.arrayBuffer();
-        return { status: 200, body: { audio, contentType: response.headers.get("content-type") ?? "audio/mpeg" } };
+        speakStream.flush();
+        return { status: 200, body: { stream, contentType: `audio/L16;rate=${SAMPLE_RATE}` } };
     };
 }

package/dist/verb-executor.d.ts

@@ -1,5 +1,26 @@
 import { type TourStep } from "@cairnvibe/core";
 import { type MissContext } from "./element-ladder";
+/** The real result of one agent-loop step (click/fill/read/call_tool) —
+ * fed back to the model as its next turn's "observation" so it can decide
+ * what to do next instead of acting blind. The loop that drives this lives
+ * on the caller's side, not here: index.tsx's runTypedAgentLoop for the
+ * HTTP path, realtime-server.ts's finalizeTurn for the realtime one — this
+ * module only ever executes one step at a time. */
+export interface ToolStepResult {
+    verb: "click" | "fill" | "read" | "call_tool";
+    target?: string;
+    ok: boolean;
+    observation: string;
+}
+/**
+ * Promise wrapper around executeVerbResponse for a continuing verb
+ * (click/fill/read/call_tool) — resolves once the real action has actually
+ * finished (synchronously for click/fill/read, after a real await for
+ * call_tool) with its real observation, instead of the fire-and-forget
+ * callback shape every other verb uses. This is what a loop driver awaits
+ * before deciding whether to call the model again.
+ */
+export declare function executeToolStep(raw: unknown, route: string, liveElements?: Map<string, HTMLElement>): Promise<ToolStepResult | null>;
 export interface VerbExecutorOptions {
     onExplain: (text: string) => void;
     onNavigate?: (route: string) => void;
@@ -11,6 +32,9 @@ export interface VerbExecutorOptions {
      * owns the UI (progress display) and, for voice, the TTS sequencing.
      */
     onTour?: (steps: TourStep[]) => void;
+    /** A click/fill/read/call_tool step finished — see ToolStepResult. Only
+     * called for the agent loop's continuing verbs, never the terminal ones. */
+    onToolStep?: (result: ToolStepResult) => void;
     /** Action ids the customer has actually wired up. "do" is rejected for anything else. */
     registeredActions?: string[];
     /**

package/dist/verb-executor.js

@@ -6,9 +6,39 @@
 // explain — never guess, never wrong-click"). The server (`server.ts`)
 // enforces the same schema independently — never trust the client alone.
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.executeToolStep = executeToolStep;
 exports.executeVerbResponse = executeVerbResponse;
 const core_1 = require("@cairnvibe/core");
 const element_ladder_1 = require("./element-ladder");
+const webmcp_client_1 = require("./webmcp-client");
+/**
+ * Promise wrapper around executeVerbResponse for a continuing verb
+ * (click/fill/read/call_tool) — resolves once the real action has actually
+ * finished (synchronously for click/fill/read, after a real await for
+ * call_tool) with its real observation, instead of the fire-and-forget
+ * callback shape every other verb uses. This is what a loop driver awaits
+ * before deciding whether to call the model again.
+ */
+function executeToolStep(raw, route, liveElements) {
+    return new Promise((resolve) => {
+        // executeVerbResponse only ever reaches onToolStep for a genuinely
+        // continuing verb — callers are only expected to call this after
+        // already confirming (via TERMINAL_VERBS) that the parsed verb is one,
+        // so this should always fire; a real timeout (not an immediate
+        // microtask — call_tool's own real network round trip needs the time)
+        // is the safety net for the case where it somehow doesn't, so a loop
+        // driver awaiting this can never hang forever.
+        const timer = setTimeout(() => resolve(null), 15000);
+        executeVerbResponse(raw, route, {
+            onExplain: () => { },
+            liveElements,
+            onToolStep: (result) => {
+                clearTimeout(timer);
+                resolve(result);
+            },
+        });
+    });
+}
 const FALLBACK_TEXT = "I'm not sure — I couldn't understand that response. Try rephrasing your question.";
 function executeVerbResponse(raw, route, options) {
     const parsed = core_1.VerbResponseSchema.safeParse(raw);
@@ -103,6 +133,63 @@ function dispatchVerb(verb, route, options) {
                 options.onExplain(verb.steps.map((s) => s.text).join(" "));
             }
             return;
+        // The agent loop's steps (server.ts's runAgentLoop) — each executes for
+        // real and reports a real observation back via onToolStep, instead of
+        // ending the turn the way every verb above does. `target` for these
+        // always came from the manifest/currentPageElements/liveElements this
+        // exact turn showed the model — never invented, same invariant as do.
+        case "click": {
+            if (verb.text)
+                options.onExplain(verb.text);
+            const el = (0, element_ladder_1.findElement)(verb.target, options.liveElements);
+            if (!el) {
+                (options.onMiss ?? element_ladder_1.logMiss)({ attempted: verb.target, route });
+                options.onToolStep?.({ verb: "click", target: verb.target, ok: false, observation: "Could not find that element on the page." });
+                return;
+            }
+            (0, element_ladder_1.highlightElement)(el);
+            el.click();
+            options.onToolStep?.({ verb: "click", target: verb.target, ok: true, observation: "Clicked it." });
+            return;
+        }
+        case "fill": {
+            if (verb.text)
+                options.onExplain(verb.text);
+            const el = (0, element_ladder_1.findElement)(verb.target, options.liveElements);
+            if (!el || !(0, element_ladder_1.fillElement)(el, verb.value)) {
+                (options.onMiss ?? element_ladder_1.logMiss)({ attempted: verb.target, route });
+                options.onToolStep?.({
+                    verb: "fill",
+                    target: verb.target,
+                    ok: false,
+                    observation: el ? "That element isn't a real form field — can't type into it." : "Could not find that element on the page.",
+                });
+                return;
+            }
+            (0, element_ladder_1.highlightElement)(el);
+            options.onToolStep?.({ verb: "fill", target: verb.target, ok: true, observation: `Typed "${verb.value}" into it.` });
+            return;
+        }
+        case "read": {
+            if (verb.text)
+                options.onExplain(verb.text);
+            const el = (0, element_ladder_1.findElement)(verb.target, options.liveElements);
+            if (!el) {
+                (options.onMiss ?? element_ladder_1.logMiss)({ attempted: verb.target, route });
+                options.onToolStep?.({ verb: "read", target: verb.target, ok: false, observation: "Could not find that element on the page." });
+                return;
+            }
+            options.onToolStep?.({ verb: "read", target: verb.target, ok: true, observation: (0, element_ladder_1.readElement)(el) });
+            return;
+        }
+        case "call_tool": {
+            if (verb.text)
+                options.onExplain(verb.text);
+            void (0, webmcp_client_1.executeWebMcpTool)(verb.name, verb.args).then((result) => {
+                options.onToolStep?.({ verb: "call_tool", target: verb.name, ok: result.ok, observation: result.observation });
+            });
+            return;
+        }
     }
 }
 /**

package/dist/webmcp-client.d.ts

@@ -0,0 +1,13 @@
+import type { WebMcpTool } from "@cairnvibe/core";
+export declare function discoverWebMcpTools(): Promise<WebMcpTool[]>;
+/**
+ * Calls a real WebMCP tool by name — `name` must be one the model was
+ * actually shown this turn (server.ts only ever includes tools from this
+ * exact request's own discoverWebMcpTools() call), never invented.
+ * Returns a plain-text observation for the agent loop to reason about
+ * next, the same shape a click/fill/read result already takes.
+ */
+export declare function executeWebMcpTool(name: string, args: Record<string, unknown> | undefined): Promise<{
+    ok: boolean;
+    observation: string;
+}>;

package/dist/webmcp-client.js

@@ -0,0 +1,70 @@
+"use strict";
+// Discovers and calls real tools a page has registered via WebMCP
+// (https://webmachinelearning.github.io/webmcp/) — an in-progress web
+// standard letting a site expose its own functions as typed, parameterized
+// tools ("document.modelContext.registerTool(...)"), running in the page's
+// own JS with the user's real session. This is the highest-trust action
+// source there is: a real function the app's own developer wrote, with a
+// real return value — not a click simulated from static analysis. Almost
+// no site has adopted it yet, so this is deliberately a no-op (empty list,
+// nothing to call) everywhere it isn't present, not a hard dependency.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.discoverWebMcpTools = discoverWebMcpTools;
+exports.executeWebMcpTool = executeWebMcpTool;
+function getModelContext() {
+    if (typeof document === "undefined")
+        return null;
+    return document.modelContext ?? null;
+}
+/** Bounded the same way LiveElementSchema/CopilotRequestSchema bound the
+ * live DOM scan — a hard cap on both count and description length, so a
+ * page that registers an unreasonable number of tools (or one with a huge
+ * description) can't blow the request payload or the prompt. */
+const MAX_TOOLS = 30;
+const MAX_DESCRIPTION_LENGTH = 500;
+async function discoverWebMcpTools() {
+    const modelContext = getModelContext();
+    if (!modelContext?.getTools)
+        return [];
+    try {
+        const tools = await modelContext.getTools();
+        if (!Array.isArray(tools))
+            return [];
+        return tools.slice(0, MAX_TOOLS).map((tool) => ({
+            name: String(tool.name),
+            description: String(tool.description ?? "").slice(0, MAX_DESCRIPTION_LENGTH),
+            inputSchema: tool.inputSchema,
+        }));
+    }
+    catch {
+        // A page's own registerTool()/getTools() implementation throwing is
+        // that page's bug, not Cairn's — degrade to "no WebMCP tools" rather
+        // than breaking the rest of the turn.
+        return [];
+    }
+}
+/**
+ * Calls a real WebMCP tool by name — `name` must be one the model was
+ * actually shown this turn (server.ts only ever includes tools from this
+ * exact request's own discoverWebMcpTools() call), never invented.
+ * Returns a plain-text observation for the agent loop to reason about
+ * next, the same shape a click/fill/read result already takes.
+ */
+async function executeWebMcpTool(name, args) {
+    const modelContext = getModelContext();
+    if (!modelContext?.getTools || !modelContext.executeTool) {
+        return { ok: false, observation: "This page no longer has that tool available." };
+    }
+    try {
+        const tools = await modelContext.getTools();
+        const tool = Array.isArray(tools) ? tools.find((t) => t.name === name) : undefined;
+        if (!tool)
+            return { ok: false, observation: `No tool named "${name}" is available on this page right now.` };
+        const result = await modelContext.executeTool(tool, args ?? {});
+        const observation = typeof result === "string" ? result : JSON.stringify(result ?? null);
+        return { ok: true, observation: observation.slice(0, 2000) };
+    }
+    catch (err) {
+        return { ok: false, observation: `That tool failed: ${err instanceof Error ? err.message : "unknown error"}` };
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cairnvibe/sdk",
-  "version": "0.2.7",
+  "version": "0.2.9",
   "description": "In-app AI copilot — <Copilot/> for React/Next.js, <cairn-widget> for any framework — plus the server handlers and realtime voice relay behind them.",
   "license": "MIT",
   "publishConfig": { "access": "public" },

package/src/element-ladder.ts

@@ -46,6 +46,47 @@ export function highlightElement(el: HTMLElement, glowMs = 4000): void {
   window.setTimeout(() => el.classList.remove("cairn-glow"), glowMs);
 }
 
+// tagName, not `instanceof HTMLInputElement` — avoids depending on those
+// classes existing as globals at all (they don't in a plain Node test
+// environment, only a real browser/jsdom), and tagName is exactly what
+// distinguishes a real form field regardless.
+function isFormField(el: HTMLElement): el is HTMLInputElement | HTMLTextAreaElement | HTMLSelectElement {
+  return el.tagName === "INPUT" || el.tagName === "TEXTAREA" || el.tagName === "SELECT";
+}
+
+/**
+ * Sets a real form field's value AND makes the framework that owns it (React,
+ * almost always, in this SDK's own target apps) actually notice — directly
+ * assigning `.value` bypasses React's tracked setter, so its own onChange
+ * never fires and the app's state silently doesn't update, a well-known
+ * React quirk. Going through the *native* prototype's value setter before
+ * dispatching a real "input" event is what makes React's synthetic event
+ * system pick it up the same way a real keystroke would.
+ */
+export function fillElement(el: HTMLElement, value: string): boolean {
+  if (!isFormField(el)) return false;
+
+  const ctorByTag: Record<string, unknown> = typeof window !== "undefined" ? { INPUT: window.HTMLInputElement, TEXTAREA: window.HTMLTextAreaElement, SELECT: window.HTMLSelectElement } : {};
+  const ctor = ctorByTag[el.tagName] as { prototype: object } | undefined;
+  const nativeSetter = ctor && (Object.getOwnPropertyDescriptor(ctor.prototype, "value")?.set as ((this: HTMLElement, v: string) => void) | undefined);
+  if (nativeSetter) nativeSetter.call(el, value);
+  else el.value = value;
+
+  el.dispatchEvent(new Event("input", { bubbles: true }));
+  el.dispatchEvent(new Event("change", { bubbles: true }));
+  return true;
+}
+
+/** The real current value/text of an element — a form field's `.value`,
+ * otherwise its trimmed visible text, bounded the same way runtime-scan.ts
+ * bounds a live element's label (this is what the agent loop "observes"
+ * after a read step, so it needs the same payload/privacy discipline). */
+export function readElement(el: HTMLElement): string {
+  const raw = isFormField(el) ? el.value : (el.textContent ?? "");
+  const trimmed = raw.replace(/\s+/g, " ").trim();
+  return trimmed.length > 500 ? `${trimmed.slice(0, 499)}…` : trimmed || "(empty)";
+}
+
 export interface MissContext {
   attempted: string;
   route: string;