@cairnvibe/sdk 0.2.6 → 0.2.8

package/src/verb-executor.ts

@@ -6,7 +6,50 @@
 // enforces the same schema independently — never trust the client alone.
 
 import { VerbResponseSchema, type ApiCall, type TourStep, type VerbResponse } from "@cairnvibe/core";
-import { findElement, highlightElement, logMiss, type MissContext } from "./element-ladder";
+import { findElement, fillElement, highlightElement, logMiss, readElement, type MissContext } from "./element-ladder";
+import { executeWebMcpTool } from "./webmcp-client";
+
+/** The real result of one agent-loop step (click/fill/read/call_tool) —
+ * fed back to the model as its next turn's "observation" so it can decide
+ * what to do next instead of acting blind. The loop that drives this lives
+ * on the caller's side, not here: index.tsx's runTypedAgentLoop for the
+ * HTTP path, realtime-server.ts's finalizeTurn for the realtime one — this
+ * module only ever executes one step at a time. */
+export interface ToolStepResult {
+  verb: "click" | "fill" | "read" | "call_tool";
+  target?: string;
+  ok: boolean;
+  observation: string;
+}
+
+/**
+ * Promise wrapper around executeVerbResponse for a continuing verb
+ * (click/fill/read/call_tool) — resolves once the real action has actually
+ * finished (synchronously for click/fill/read, after a real await for
+ * call_tool) with its real observation, instead of the fire-and-forget
+ * callback shape every other verb uses. This is what a loop driver awaits
+ * before deciding whether to call the model again.
+ */
+export function executeToolStep(raw: unknown, route: string, liveElements?: Map<string, HTMLElement>): Promise<ToolStepResult | null> {
+  return new Promise((resolve) => {
+    // executeVerbResponse only ever reaches onToolStep for a genuinely
+    // continuing verb — callers are only expected to call this after
+    // already confirming (via TERMINAL_VERBS) that the parsed verb is one,
+    // so this should always fire; a real timeout (not an immediate
+    // microtask — call_tool's own real network round trip needs the time)
+    // is the safety net for the case where it somehow doesn't, so a loop
+    // driver awaiting this can never hang forever.
+    const timer = setTimeout(() => resolve(null), 15000);
+    executeVerbResponse(raw, route, {
+      onExplain: () => {},
+      liveElements,
+      onToolStep: (result) => {
+        clearTimeout(timer);
+        resolve(result);
+      },
+    });
+  });
+}
 
 export interface VerbExecutorOptions {
   onExplain: (text: string) => void;
@@ -19,8 +62,18 @@ export interface VerbExecutorOptions {
    * owns the UI (progress display) and, for voice, the TTS sequencing.
    */
   onTour?: (steps: TourStep[]) => void;
+  /** A click/fill/read/call_tool step finished — see ToolStepResult. Only
+   * called for the agent loop's continuing verbs, never the terminal ones. */
+  onToolStep?: (result: ToolStepResult) => void;
   /** Action ids the customer has actually wired up. "do" is rejected for anything else. */
   registeredActions?: string[];
+  /**
+   * This turn's frozen runtime-scan.ts snapshot (id -> real element),
+   * checked before the static data-ai/aria-label/text ladder — lets a verb
+   * target a dynamically-rendered element (a list row) the manifest never
+   * saw. Absent entirely for a caller that hasn't wired up live scanning.
+   */
+  liveElements?: Map<string, HTMLElement>;
 }
 
 const FALLBACK_TEXT = "I'm not sure — I couldn't understand that response. Try rephrasing your question.";
@@ -43,13 +96,16 @@ function dispatchVerb(verb: VerbResponse, route: string, options: VerbExecutorOp
 
     case "highlight":
     case "open": {
-      const el = findElement(verb.target);
+      const el = findElement(verb.target, options.liveElements);
       if (!el) {
         (options.onMiss ?? logMiss)({ attempted: verb.target, route });
         options.onExplain(verb.text ?? "I know what you need, but I can't find it on this page right now.");
         return;
       }
       highlightElement(el);
+      // "open" means make the thing actually appear (a menu, a modal, a
+      // panel) — highlighting alone doesn't do that; a real click does.
+      if (verb.verb === "open") el.click();
       if (verb.text) options.onExplain(verb.text);
       return;
     }
@@ -68,22 +124,38 @@ function dispatchVerb(verb: VerbResponse, route: string, options: VerbExecutorOp
         return;
       }
 
-      if (verb.apiCall) {
-        // Auto-discovered path: a real, indexer-found handler call on this
-        // exact target element, attached server-side after looking the
-        // target up in the manifest — never something the model emitted
-        // itself (see ApiCallSchema's doc comment in @cairnvibe/core).
-        const el = verb.target ? findElement(verb.target) : null;
-        if (el) highlightElement(el);
-        else if (verb.target) (options.onMiss ?? logMiss)({ attempted: verb.target, route });
+      // Auto-discovered path: the server already verified `target` names a
+      // real element (the static manifest or this exact request's own
+      // live-DOM scan) before ever returning this verb — never something
+      // the model invented (see resolveVerb in server.ts).
+      if (verb.target || verb.apiCall) {
+        const el = verb.target ? findElement(verb.target, options.liveElements) : null;
+        if (el) {
+          // Click-first: the real element's own handler runs in full (any
+          // local state update, spinner, or non-network side effect a raw
+          // fetch would silently skip), and it's the only way to fire an
+          // action that has no fetch/axios call at all — a button that
+          // just reveals a form, e.g. — which never gets an `apiCall` in
+          // the first place. `apiCall` is only ever the fallback below,
+          // for a target that can't be resolved live right now (e.g. it's
+          // on a different page) — never fired in addition to a real
+          // click, so the action can't run twice.
+          highlightElement(el);
+          el.click();
+          if (verb.text) options.onExplain(verb.text);
+          return;
+        }
+        if (verb.target) (options.onMiss ?? logMiss)({ attempted: verb.target, route });
 
-        if (verb.text) options.onExplain(verb.text);
-        void executeApiCall(verb.apiCall).then((result) => {
-          if (!result.ok) {
-            options.onExplain("I tried to do that, but something went wrong — try again in a moment.");
-          }
-        });
-        return;
+        if (verb.apiCall) {
+          if (verb.text) options.onExplain(verb.text);
+          void executeApiCall(verb.apiCall).then((result) => {
+            if (!result.ok) {
+              options.onExplain("I tried to do that, but something went wrong — try again in a moment.");
+            }
+          });
+          return;
+        }
       }
 
       options.onExplain(verb.text ?? "That action isn't available here.");
@@ -100,6 +172,63 @@ function dispatchVerb(verb: VerbResponse, route: string, options: VerbExecutorOp
         options.onExplain(verb.steps.map((s) => s.text).join(" "));
       }
       return;
+
+    // The agent loop's steps (server.ts's runAgentLoop) — each executes for
+    // real and reports a real observation back via onToolStep, instead of
+    // ending the turn the way every verb above does. `target` for these
+    // always came from the manifest/currentPageElements/liveElements this
+    // exact turn showed the model — never invented, same invariant as do.
+    case "click": {
+      if (verb.text) options.onExplain(verb.text);
+      const el = findElement(verb.target, options.liveElements);
+      if (!el) {
+        (options.onMiss ?? logMiss)({ attempted: verb.target, route });
+        options.onToolStep?.({ verb: "click", target: verb.target, ok: false, observation: "Could not find that element on the page." });
+        return;
+      }
+      highlightElement(el);
+      el.click();
+      options.onToolStep?.({ verb: "click", target: verb.target, ok: true, observation: "Clicked it." });
+      return;
+    }
+
+    case "fill": {
+      if (verb.text) options.onExplain(verb.text);
+      const el = findElement(verb.target, options.liveElements);
+      if (!el || !fillElement(el, verb.value)) {
+        (options.onMiss ?? logMiss)({ attempted: verb.target, route });
+        options.onToolStep?.({
+          verb: "fill",
+          target: verb.target,
+          ok: false,
+          observation: el ? "That element isn't a real form field — can't type into it." : "Could not find that element on the page.",
+        });
+        return;
+      }
+      highlightElement(el);
+      options.onToolStep?.({ verb: "fill", target: verb.target, ok: true, observation: `Typed "${verb.value}" into it.` });
+      return;
+    }
+
+    case "read": {
+      if (verb.text) options.onExplain(verb.text);
+      const el = findElement(verb.target, options.liveElements);
+      if (!el) {
+        (options.onMiss ?? logMiss)({ attempted: verb.target, route });
+        options.onToolStep?.({ verb: "read", target: verb.target, ok: false, observation: "Could not find that element on the page." });
+        return;
+      }
+      options.onToolStep?.({ verb: "read", target: verb.target, ok: true, observation: readElement(el) });
+      return;
+    }
+
+    case "call_tool": {
+      if (verb.text) options.onExplain(verb.text);
+      void executeWebMcpTool(verb.name, verb.args).then((result) => {
+        options.onToolStep?.({ verb: "call_tool", target: verb.name, ok: result.ok, observation: result.observation });
+      });
+      return;
+    }
   }
 }
 

package/src/webmcp-client.ts

@@ -0,0 +1,79 @@
+// Discovers and calls real tools a page has registered via WebMCP
+// (https://webmachinelearning.github.io/webmcp/) — an in-progress web
+// standard letting a site expose its own functions as typed, parameterized
+// tools ("document.modelContext.registerTool(...)"), running in the page's
+// own JS with the user's real session. This is the highest-trust action
+// source there is: a real function the app's own developer wrote, with a
+// real return value — not a click simulated from static analysis. Almost
+// no site has adopted it yet, so this is deliberately a no-op (empty list,
+// nothing to call) everywhere it isn't present, not a hard dependency.
+
+import type { WebMcpTool } from "@cairnvibe/core";
+
+interface ModelContextTool {
+  name: string;
+  description?: string;
+  inputSchema?: Record<string, unknown>;
+}
+
+interface ModelContext {
+  getTools?: () => Promise<ModelContextTool[]> | ModelContextTool[];
+  executeTool?: (tool: ModelContextTool, args: unknown) => Promise<unknown>;
+}
+
+function getModelContext(): ModelContext | null {
+  if (typeof document === "undefined") return null;
+  return (document as unknown as { modelContext?: ModelContext }).modelContext ?? null;
+}
+
+/** Bounded the same way LiveElementSchema/CopilotRequestSchema bound the
+ * live DOM scan — a hard cap on both count and description length, so a
+ * page that registers an unreasonable number of tools (or one with a huge
+ * description) can't blow the request payload or the prompt. */
+const MAX_TOOLS = 30;
+const MAX_DESCRIPTION_LENGTH = 500;
+
+export async function discoverWebMcpTools(): Promise<WebMcpTool[]> {
+  const modelContext = getModelContext();
+  if (!modelContext?.getTools) return [];
+
+  try {
+    const tools = await modelContext.getTools();
+    if (!Array.isArray(tools)) return [];
+    return tools.slice(0, MAX_TOOLS).map((tool) => ({
+      name: String(tool.name),
+      description: String(tool.description ?? "").slice(0, MAX_DESCRIPTION_LENGTH),
+      inputSchema: tool.inputSchema,
+    }));
+  } catch {
+    // A page's own registerTool()/getTools() implementation throwing is
+    // that page's bug, not Cairn's — degrade to "no WebMCP tools" rather
+    // than breaking the rest of the turn.
+    return [];
+  }
+}
+
+/**
+ * Calls a real WebMCP tool by name — `name` must be one the model was
+ * actually shown this turn (server.ts only ever includes tools from this
+ * exact request's own discoverWebMcpTools() call), never invented.
+ * Returns a plain-text observation for the agent loop to reason about
+ * next, the same shape a click/fill/read result already takes.
+ */
+export async function executeWebMcpTool(name: string, args: Record<string, unknown> | undefined): Promise<{ ok: boolean; observation: string }> {
+  const modelContext = getModelContext();
+  if (!modelContext?.getTools || !modelContext.executeTool) {
+    return { ok: false, observation: "This page no longer has that tool available." };
+  }
+  try {
+    const tools = await modelContext.getTools();
+    const tool = Array.isArray(tools) ? tools.find((t) => t.name === name) : undefined;
+    if (!tool) return { ok: false, observation: `No tool named "${name}" is available on this page right now.` };
+
+    const result = await modelContext.executeTool(tool, args ?? {});
+    const observation = typeof result === "string" ? result : JSON.stringify(result ?? null);
+    return { ok: true, observation: observation.slice(0, 2000) };
+  } catch (err) {
+    return { ok: false, observation: `That tool failed: ${err instanceof Error ? err.message : "unknown error"}` };
+  }
+}