@testdriverai/mcp 7.10.0-canary.0 → 7.10.1-test

package/mcp-server/src/core/actions.ts

@@ -0,0 +1,912 @@
+/**
+ * TestDriver action core — framework-agnostic.
+ *
+ * This module owns the actual computer-use logic: it holds the singleton SDK,
+ * the element-reference map, and the "last screenshot" used by `check`, and
+ * exposes one async function per action. Every function returns a neutral
+ * {@link ActionResult} — no MCP types, no eve types — so it can be wrapped by:
+ *
+ *   - the MCP server (`server.ts`), which turns images into resource URIs and
+ *     calls `createToolResult`, and
+ *   - eve `defineTool()` files, which return the text/code/images as eve content.
+ *
+ * Keep MCP- and eve-specific concerns (Sentry, progress heartbeats, abort
+ * racing, image stores, UI resources) in the adapters. This file only knows
+ * about the TestDriver SDK and `generateActionCode`.
+ *
+ * State is process-global and single-session by design: one sandbox at a time,
+ * mirroring how the stdio MCP server has always behaved.
+ */
+
+import { generateActionCode } from "../codegen.js";
+import { getProvisionOptions, SessionStartInputSchema, type SessionStartInput } from "../provision-types.js";
+import { sessionManager, type SessionState } from "../session.js";
+
+// Re-export the input contract so adapters (MCP server, eve tools) share one
+// source of truth for the session_start schema instead of redeclaring it.
+export { SessionStartInputSchema, type SessionStartInput };
+
+// =============================================================================
+// Neutral result shape
+// =============================================================================
+
+export interface ActionImage {
+  kind: "screenshot" | "cropped";
+  /** Raw base64 (no `data:` prefix). */
+  base64: string;
+}
+
+export interface ActionResult {
+  /** Whether the action succeeded (assertions/finds may legitimately be false). */
+  ok: boolean;
+  /** Human/agent-readable summary line(s). */
+  text: string;
+  /** Structured fields for programmatic consumers and UIs. */
+  data: Record<string, unknown>;
+  /** Code to append to the test file, when the action produces test code. */
+  code?: string;
+  /** Images produced by the action (screenshots / cropped element shots). */
+  images?: ActionImage[];
+}
+
+/** Thrown when an action runs without an active/valid session. */
+export class NoActiveSessionError extends Error {
+  readonly code: "NO_SESSION" | "SESSION_EXPIRED";
+  readonly expiredSessionId?: string;
+  constructor(code: "NO_SESSION" | "SESSION_EXPIRED", message: string, expiredSessionId?: string) {
+    super(message);
+    this.name = "NoActiveSessionError";
+    this.code = code;
+    this.expiredSessionId = expiredSessionId;
+  }
+}
+
+// =============================================================================
+// Singleton state (single sandbox per process)
+// =============================================================================
+
+let sdk: any = null;
+let lastScreenshotBase64: string | null = null;
+
+/**
+ * Optional adapter-supplied recovery hook. When the singleton SDK is missing or
+ * stale at action time, {@link requireActiveSession} calls this to obtain the
+ * params needed to rebuild the connection from the still-alive sandbox, then
+ * reconnects before throwing NO_SESSION.
+ *
+ * Why a hook instead of mcp-core owning the handle: the durable facts (sandbox
+ * id, config) and the API key must survive a host process recycle, but this
+ * module's globals do NOT — they reset on the very recycle we're recovering
+ * from. So the *durable owner* (eve's per-session state) registers a resolver
+ * that reads its own durable store and re-resolves the key per call. Hosts with
+ * one long-lived process (stdio MCP server, CLI) register nothing: the resolver
+ * stays null and behavior is unchanged — `requireActiveSession` throws the same
+ * NO_SESSION/SESSION_EXPIRED as before.
+ *
+ * The resolver returns `null` when it has nothing to recover from (no sandbox
+ * provisioned this session), in which case we fall through to the normal throw.
+ * It is re-registered each step by the adapter (module globals reset on recycle),
+ * so a stale closure can't outlive the process it was bound to.
+ */
+export type ReconnectResolver = () => Promise<ReconnectParams | null>;
+let reconnectResolver: ReconnectResolver | null = null;
+
+/**
+ * Register (or clear, with `null`) the recovery hook. Durable adapters call this
+ * at the start of each step with a resolver bound to the current tool context.
+ */
+export function setReconnectResolver(resolver: ReconnectResolver | null): void {
+  reconnectResolver = resolver;
+}
+
+/** In-flight reconnect, so concurrent actions in one step share one rebuild. */
+let reconnecting: Promise<void> | null = null;
+
+/** Stored element instances from `find`/`findall`, addressable by ref. */
+const elementRefs = new Map<
+  string,
+  { element: any; description: string; coords: { x: number; y: number; centerX: number; centerY: number } }
+>();
+
+/** Expose internals the adapters legitimately need (read-only intent). */
+export function getSdk(): any {
+  return sdk;
+}
+export function getLastScreenshotBase64(): string | null {
+  return lastScreenshotBase64;
+}
+export function getElementRef(ref: string) {
+  return elementRefs.get(ref);
+}
+
+/**
+ * Cheap, side-effect-free check for a usable in-process session: a live SDK and
+ * a current session that hasn't expired. Durable adapters call this *before*
+ * doing any expensive reconnect prep (e.g. re-resolving an API key), so the
+ * common warm-process path stays free. Note: unlike `requireActiveSession`, this
+ * does NOT refresh the keepAlive window — it only reports liveness.
+ */
+export function hasLiveSession(): boolean {
+  const session = sessionManager.getCurrentSession();
+  return !!sdk && !!session && sessionManager.isSessionValid(session.sessionId);
+}
+
+// =============================================================================
+// Helpers
+// =============================================================================
+
+/**
+ * Load the TestDriver SDK, working in two very different hosts:
+ *
+ *  - eve's dev-runtime / build: it externalizes the `testdriverai` package, so
+ *    the bare specifier resolves from eve's node_modules and is NOT pulled into
+ *    the bundle/snapshot (which would drag in sdk.js's monorepo-relative
+ *    `require('../shared/load-env')` and fail).
+ *  - the standalone MCP server (stdio/CLI): it is a separate package nested in
+ *    `sdk/`, so `testdriverai` is not self-referenceable there; fall back to the
+ *    relative `../../../sdk.js` it has always used.
+ *
+ * Both specifiers are computed/indirected so a static bundler can't follow them.
+ */
+async function loadTestDriverSdk(): Promise<{ default: any }> {
+  const pkg = "testdriverai";
+  try {
+    return (await import(/* @vite-ignore */ pkg)) as { default: any };
+  } catch {
+    const rel = ["..", "..", "..", "sdk.js"].join("/");
+    return (await import(/* @vite-ignore */ rel)) as { default: any };
+  }
+}
+
+/**
+ * Strip the large/derived fields the SDK attaches to a response so they don't
+ * bloat the agent's context. Returns a shallow clone safe to mutate further.
+ */
+function leanResponse(raw: any): Record<string, unknown> {
+  const r = { ...(raw || {}) } as Record<string, unknown>;
+  delete r.croppedImage;
+  delete r.extractedText;
+  delete r.pixelDiffImage;
+  return r;
+}
+
+/** Normalize a possibly-`data:`-prefixed base64 string to bare base64. */
+function bareBase64(s: string): string {
+  return s.startsWith("data:") ? s.replace(/^data:image\/\w+;base64,/, "") : s;
+}
+
+/**
+ * Whether the live SDK's realtime (Ably) transport is actually usable right now.
+ *
+ * A recycled host gives us a brand-new singleton (`sdk === null`) — caught by the
+ * `!sdk` check below. But a *parked* host is subtler: the process froze between
+ * steps, Ably's idle timer tore down the transport ("No activity seen from
+ * realtime…"), and on thaw the singleton + session still look valid while the
+ * socket underneath is dead. Sending over it doesn't fail fast — it hangs for the
+ * SDK's full command timeout. So we also treat any non-`connected` Ably state as
+ * "needs reconnect". `_ably` is SDK-internal, so every hop is optional-chained
+ * and a throw counts as unhealthy: a false negative costs a cheap reconnect, a
+ * false positive costs a multi-minute hang, so we fail safe toward reconnecting.
+ */
+function realtimeIsHealthy(s: any): boolean {
+  try {
+    return s?.sandbox?._ably?.connection?.state === "connected";
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Attempt to rebuild the singleton from the adapter-registered recovery hook.
+ * Returns true if a usable session now exists, false if there was nothing to
+ * recover from (no resolver, or it returned null). Throws SESSION_EXPIRED when a
+ * recovery was attempted but the sandbox is genuinely gone.
+ *
+ * Concurrent actions in one step share a single in-flight rebuild via
+ * {@link reconnecting} so a parked socket isn't reconnected N times in parallel.
+ */
+async function tryRecoverSession(): Promise<boolean> {
+  if (!reconnectResolver) return false;
+  if (!reconnecting) {
+    const resolver = reconnectResolver;
+    reconnecting = (async () => {
+      const params = await resolver();
+      if (!params?.sandboxId) return; // nothing provisioned this session
+      // Close any half-dead socket on the outgoing SDK before replacing it, so a
+      // parked-then-rebuilt session doesn't leak an orphaned Ably connection.
+      try {
+        sdk?.sandbox?._ably?.close?.();
+      } catch {
+        /* best effort — reconnectSession installs a fresh SDK regardless */
+      }
+      await reconnectSession(params);
+    })().finally(() => {
+      reconnecting = null;
+    });
+  }
+  try {
+    await reconnecting;
+  } catch (err) {
+    sdk = null;
+    if (err instanceof NoActiveSessionError) throw err;
+    throw new NoActiveSessionError(
+      "SESSION_EXPIRED",
+      `Could not reconnect to the sandbox: ${err instanceof Error ? err.message : String(err)}. The sandbox has expired — call session_start again to create a new one.`
+    );
+  }
+  const session = sessionManager.getCurrentSession();
+  return !!sdk && !!session && sessionManager.isSessionValid(session.sessionId);
+}
+
+/**
+ * Validate the active session and auto-extend it (active use keeps it alive),
+ * mirroring the MCP server's `requireActiveSession`. When the in-process
+ * connection was lost (host recycle wiped the singletons, or a park killed the
+ * realtime socket), this first tries to self-heal via the adapter-registered
+ * recovery hook ({@link setReconnectResolver}) — so every action recovers at one
+ * chokepoint instead of each tool wrapping its own reconnect. Throws
+ * {@link NoActiveSessionError} only when there is no usable sandbox AND nothing
+ * to recover from.
+ */
+async function requireActiveSession(): Promise<void> {
+  const session = sessionManager.getCurrentSession();
+
+  // Recoverable conditions:
+  //  - singletons wiped by a host recycle (`!sdk || !session`), or
+  //  - the SDK exists but its realtime socket died during a park
+  //    (`!realtimeIsHealthy`). We only treat a dead socket as recoverable when a
+  //    recovery hook is registered — i.e. a durable host (eve) that actually
+  //    parks. Long-lived hosts (stdio MCP, CLI) register no resolver, so a
+  //    transient in-process Ably blip is left to Ably's own auto-reconnect rather
+  //    than forcing a full singleton rebuild — preserving their prior behavior.
+  const socketDead = !!sdk && !!session && !!reconnectResolver && !realtimeIsHealthy(sdk);
+  const needsRecovery = !sdk || !session || socketDead;
+  if (needsRecovery) {
+    const recovered = await tryRecoverSession();
+    if (!recovered) {
+      // Nothing to recover from (no resolver, or it had no sandbox to rebuild).
+      // If the socket was dead, don't send the action into the void — surface
+      // SESSION_EXPIRED so the agent re-provisions instead of hanging.
+      if (socketDead) {
+        sdk = null;
+        throw new NoActiveSessionError(
+          "SESSION_EXPIRED",
+          "The sandbox connection was lost and could not be restored. Call session_start again to create a new sandbox session.",
+          session?.sessionId
+        );
+      }
+      // Otherwise the singleton was simply absent — the standard NO_SESSION.
+      throw new NoActiveSessionError(
+        "NO_SESSION",
+        "No active session. Call session_start first to create a sandbox before using any other tools."
+      );
+    }
+  }
+
+  const current = sessionManager.getCurrentSession();
+  if (!current || !sessionManager.isSessionValid(current.sessionId)) {
+    sdk = null;
+    throw new NoActiveSessionError(
+      "SESSION_EXPIRED",
+      "Session has expired or timed out. Call session_start again to create a new sandbox session.",
+      current?.sessionId
+    );
+  }
+
+  // Reset the keepAlive timer on each command so active use doesn't expire.
+  sessionManager.refreshSession(current.sessionId);
+}
+
+/** Capture a fresh full-screen screenshot as bare base64 (or null on failure). */
+async function captureScreen(): Promise<string | null> {
+  try {
+    const b64 = await sdk.agent.system.captureScreenBase64(1, false, true);
+    return b64 || null;
+  } catch {
+    return null;
+  }
+}
+
+// =============================================================================
+// Session lifecycle
+// =============================================================================
+
+export interface SessionStartHooks {
+  /** Called before a long await; return a stop fn. Lets adapters heartbeat. */
+  onProgress?: (message: string) => void;
+}
+
+/**
+ * Start a session and provision a sandbox. The adapter is responsible for
+ * progress/heartbeat and abort; this performs the provisioning and returns the
+ * neutral result (including the initial screenshot and provision code).
+ */
+export async function sessionStart(
+  params: SessionStartInput,
+  resolved: { os: "linux" | "windows"; e2bTemplateId?: string; apiKey?: string },
+  hooks: SessionStartHooks = {}
+): Promise<ActionResult> {
+  const progress = hooks.onProgress ?? (() => {});
+
+  // Validate required fields for specific provision types (unless reconnecting).
+  if (!params.sandboxId) {
+    if (params.type === "installer" && !params.installerUrl) {
+      return { ok: false, text: "installer type requires 'installerUrl' parameter", data: { error: "Missing required parameter: installerUrl" } };
+    }
+    if (params.type === "electron" && !params.appPath) {
+      return { ok: false, text: "electron type requires 'appPath' parameter", data: { error: "Missing required parameter: appPath" } };
+    }
+  }
+
+  const newSession = sessionManager.createSession({
+    os: resolved.os,
+    keepAlive: params.keepAlive,
+    testFile: params.testFile,
+  });
+
+  const apiRoot = params.apiRoot || process.env.TD_API_ROOT || "https://api.testdriver.ai";
+  const previewMode = process.env.TD_PREVIEW || "ide";
+  const instanceIp = params.ip || process.env.TD_IP;
+  // An adapter may resolve a per-caller key out-of-band (e.g. the eve agent's
+  // browser OAuth device flow, where each user carries their own team key).
+  // Prefer that; otherwise fall back to the process-wide env keys the MCP/CLI
+  // host has always used.
+  const apiKey = resolved.apiKey || process.env.TD_API_KEY || process.env.COPILOT_MCP_TD_API_KEY || "";
+
+  if (!apiKey) {
+    return {
+      ok: false,
+      text: "No API key found. Please set TD_API_KEY or COPILOT_MCP_TD_API_KEY environment variable.",
+      data: {
+        error: "Missing API key",
+        action: "session_start",
+        hint: "For GitHub Copilot coding agent, create a Copilot environment secret named COPILOT_MCP_TD_API_KEY",
+      },
+    };
+  }
+
+  const TestDriverSDK = (await loadTestDriverSdk()).default;
+  sdk = new TestDriverSDK(apiKey, {
+    os: resolved.os,
+    logging: false,
+    apiRoot,
+    preview: previewMode as "browser" | "ide" | "none",
+    ip: instanceIp,
+    e2bTemplateId: resolved.e2bTemplateId,
+  });
+
+  // Debug mode — attach to an existing sandbox, skip provisioning.
+  if (params.sandboxId) {
+    progress(`Connecting to existing sandbox ${params.sandboxId}...`);
+    await sdk.connect({ sandboxId: params.sandboxId, keepAlive: params.keepAlive });
+    const instance = sdk.getInstance();
+    sessionManager.activateSession(newSession.sessionId, instance?.instanceId || params.sandboxId);
+
+    progress("Capturing screenshot...");
+    const shot = await captureScreen();
+    if (shot) lastScreenshotBase64 = shot;
+
+    return {
+      ok: true,
+      text: `Connected to existing sandbox (debug mode)\nSession: ${newSession.sessionId}\nSandbox: ${params.sandboxId}\n\nUse find, click, type, etc. to interact.`,
+      data: { action: "session_start", sessionId: newSession.sessionId, sandboxId: params.sandboxId, debugMode: true },
+      code: "// Connected to existing sandbox - no provision code needed",
+      images: shot ? [{ kind: "screenshot", base64: shot }] : undefined,
+    };
+  }
+
+  progress(instanceIp ? `Connecting to self-hosted instance ${instanceIp}...` : "Connecting to cloud sandbox...");
+  await sdk.connect({ reconnect: params.reconnect, keepAlive: params.keepAlive, ip: instanceIp });
+
+  const instance = sdk.getInstance();
+  sessionManager.activateSession(newSession.sessionId, instance?.instanceId || "unknown");
+
+  const provisionOptions = getProvisionOptions(params);
+  let provisionCmd = "";
+
+  progress(`Provisioning ${params.type}...`);
+  switch (params.type) {
+    case "chrome":
+      await sdk.provision.chrome(provisionOptions);
+      provisionCmd = "provision.chrome";
+      break;
+    case "chromeExtension":
+      await sdk.provision.chromeExtension(provisionOptions);
+      provisionCmd = "provision.chromeExtension";
+      break;
+    case "vscode":
+      await sdk.provision.vscode(provisionOptions);
+      provisionCmd = "provision.vscode";
+      break;
+    case "installer":
+      await sdk.provision.installer(provisionOptions);
+      provisionCmd = "provision.installer";
+      break;
+    case "electron":
+      await sdk.provision.electron(provisionOptions);
+      provisionCmd = "provision.electron";
+      break;
+  }
+
+  progress("Capturing screenshot...");
+  const shot = await captureScreen();
+  if (shot) lastScreenshotBase64 = shot;
+
+  const debuggerUrl = instance?.debuggerUrl || (instanceIp ? `http://${instanceIp}:9222` : null);
+  const connectionType = instanceIp ? `Self-hosted (${instanceIp})` : "Cloud";
+
+  return {
+    ok: true,
+    text: `Session started: ${newSession.sessionId}\nConnection: ${connectionType}\nType: ${params.type}\nSandbox: ${instance?.instanceId}\nExpires in: ${Math.round(params.keepAlive / 1000)}s`,
+    data: {
+      action: "session_start",
+      sessionId: newSession.sessionId,
+      // The cloud sandbox id (`sb-…`). Surfaced so durable adapters (eve) can
+      // persist it and later reconnect the singleton after a process recycle —
+      // see `reconnectSession`/`ensureActiveSession` below.
+      sandboxId: instance?.instanceId,
+      provisionType: params.type,
+      selfHosted: !!instanceIp,
+      instanceIp: instanceIp || undefined,
+      debuggerUrl,
+    },
+    code: generateActionCode(provisionCmd, provisionOptions),
+    images: shot ? [{ kind: "screenshot", base64: shot }] : undefined,
+  };
+}
+
+// =============================================================================
+// Reconnect (durable adapters)
+// =============================================================================
+
+/**
+ * Parameters needed to rebuild the singleton SDK and reconnect to a sandbox
+ * that is still alive on the server but whose in-process connection was lost
+ * (e.g. the host process was recycled between durable workflow steps).
+ */
+export interface ReconnectParams {
+  /** Cloud sandbox id (`sb-…`) to reconnect to. */
+  sandboxId: string;
+  os: "linux" | "windows";
+  keepAlive: number;
+  apiKey: string;
+  apiRoot?: string;
+  e2bTemplateId?: string;
+  /** Self-hosted instance IP, when the original session used one. */
+  ip?: string;
+  testFile?: string;
+}
+
+/**
+ * Rebuild the process-global SDK and reconnect it to an existing sandbox by id,
+ * skipping provisioning. This is the recovery path for hosts that do NOT keep a
+ * single long-lived process: eve runs each turn as a durable workflow on Vercel
+ * and may park or recycle the process between steps, which drops the SDK's
+ * realtime connection and wipes this module's singletons. The sandbox itself can
+ * still be alive on the server (the API holds it for `keepAlive`/grace), so we
+ * re-`connect({ sandboxId })` and re-register a local session instead of failing
+ * the action with NO_SESSION.
+ *
+ * Returns the (re)activated session id. Throws if the reconnect fails (the
+ * sandbox is gone) — callers translate that into a SESSION_EXPIRED so the agent
+ * knows to call session_start again.
+ */
+export async function reconnectSession(params: ReconnectParams): Promise<string> {
+  const apiRoot = params.apiRoot || process.env.TD_API_ROOT || "https://api.testdriver.ai";
+  const previewMode = process.env.TD_PREVIEW || "ide";
+
+  const session = sessionManager.createSession({
+    os: params.os,
+    keepAlive: params.keepAlive,
+    testFile: params.testFile,
+  });
+
+  const TestDriverSDK = (await loadTestDriverSdk()).default;
+  sdk = new TestDriverSDK(params.apiKey, {
+    os: params.os,
+    logging: false,
+    apiRoot,
+    preview: previewMode as "browser" | "ide" | "none",
+    ip: params.ip,
+    e2bTemplateId: params.e2bTemplateId,
+  });
+
+  await sdk.connect({ sandboxId: params.sandboxId, keepAlive: params.keepAlive });
+  const instance = sdk.getInstance();
+  sessionManager.activateSession(session.sessionId, instance?.instanceId || params.sandboxId);
+  return session.sessionId;
+}
+
+/**
+ * Ensure there is a usable session before an action, reconnecting from durable
+ * state if the in-process singleton was lost. Returns true when a fresh
+ * reconnect happened (the caller may want to recapture context), false when the
+ * existing session was already usable.
+ *
+ * Durable adapters call this at the top of each tool, passing the sandbox id and
+ * config they persisted at session_start. Hosts with a single long-lived process
+ * (the stdio MCP server, the CLI) never persist a sandbox id, so they pass none
+ * and this is a no-op — `requireActiveSession()` inside each action still guards
+ * them exactly as before.
+ */
+export async function ensureActiveSession(params?: ReconnectParams): Promise<boolean> {
+  const session = sessionManager.getCurrentSession();
+  // Already have a live SDK + valid session — nothing to do.
+  if (sdk && session && sessionManager.isSessionValid(session.sessionId)) {
+    return false;
+  }
+  // No durable handle to recover from — leave the (missing/expired) state as-is
+  // so the action's own requireActiveSession() throws the right NO_SESSION /
+  // SESSION_EXPIRED error.
+  if (!params?.sandboxId) {
+    return false;
+  }
+  try {
+    await reconnectSession(params);
+    return true;
+  } catch (err) {
+    // The sandbox is gone (server killed it after grace/disconnect cap, or the
+    // id is stale). Surface a SESSION_EXPIRED so the adapter's mapper tells the
+    // agent to call session_start again, instead of a raw connect error.
+    sdk = null;
+    throw new NoActiveSessionError(
+      "SESSION_EXPIRED",
+      `Could not reconnect to sandbox ${params.sandboxId}: ${err instanceof Error ? err.message : String(err)}. The sandbox has expired — call session_start again to create a new one.`,
+      params.sandboxId
+    );
+  }
+}
+
+/** Disconnect the SDK (best-effort); used by adapters on cancel/teardown. */
+export async function disconnect(): Promise<void> {
+  try {
+    await sdk?.disconnect?.();
+  } catch {
+    /* best effort */
+  }
+}
+
+export function sessionStatus(): ActionResult {
+  const session = sessionManager.getCurrentSession();
+  if (!session) {
+    return { ok: false, text: "No active session", data: { error: "No active session. Call session_start first." } };
+  }
+  const summary = sessionManager.getSessionSummary(session.sessionId);
+  return {
+    ok: true,
+    text: `Session: ${session.sessionId}\nStatus: ${session.status}\nTime remaining: ${Math.round((summary?.timeRemaining || 0) / 1000)}s`,
+    data: { action: "session_status", ...summary, sessionId: session.sessionId, status: session.status },
+  };
+}
+
+export function sessionExtend(additionalMs: number): ActionResult {
+  const session = sessionManager.getCurrentSession();
+  if (!session) return { ok: false, text: "No active session", data: { action: "session_extend" } };
+  sessionManager.extendSession(session.sessionId, additionalMs);
+  const newExpiry = sessionManager.getTimeRemaining(session.sessionId);
+  return {
+    ok: true,
+    text: `Session extended by ${additionalMs / 1000}s. New expiry: ${Math.round(newExpiry / 1000)}s`,
+    data: { action: "session_extend", newExpiry },
+  };
+}
+
+// =============================================================================
+// Element location
+// =============================================================================
+
+export async function find(description: string, timeout?: number): Promise<ActionResult> {
+  await requireActiveSession();
+  const element = await sdk.find(description, timeout ? { timeout } : undefined);
+  const found = element.found();
+  const coords = element.getCoordinates();
+
+  const ref = `el-${Date.now()}`;
+  if (found && coords) {
+    elementRefs.set(ref, {
+      element,
+      description,
+      coords: { x: coords.x, y: coords.y, centerX: coords.centerX, centerY: coords.centerY },
+    });
+  }
+
+  const raw = element._response || {};
+  const images: ActionImage[] = [];
+  if (raw.croppedImage) {
+    images.push({ kind: "cropped", base64: bareBase64(raw.croppedImage) });
+  } else if (!found) {
+    const shot = await captureScreen();
+    if (shot) images.push({ kind: "screenshot", base64: shot });
+  }
+  const data = leanResponse(raw);
+
+  return {
+    ok: found,
+    text: found
+      ? `Found: "${description}" at (${(raw.coordinates as any)?.x}, ${(raw.coordinates as any)?.y})\nRef: ${ref}`
+      : `Element not found: "${description}"`,
+    data: {
+      ...data,
+      action: "find",
+      element: found ? { description, centerX: coords?.centerX, centerY: coords?.centerY, confidence: element.confidence, ref } : undefined,
+      ref,
+    },
+    code: found ? generateActionCode("find", { description }) : undefined,
+    images: images.length ? images : undefined,
+  };
+}
+
+export async function findAll(description: string, timeout?: number): Promise<ActionResult> {
+  await requireActiveSession();
+  const elements = await sdk.findAll(description, timeout ? { timeout } : undefined);
+  const count = elements.length;
+
+  const refs: string[] = [];
+  const elementInfos: Array<{ ref: string; x: number; y: number; centerX: number; centerY: number; confidence: number }> = [];
+  for (let i = 0; i < elements.length; i++) {
+    const el = elements[i];
+    const coords = el.getCoordinates();
+    if (!coords) continue;
+    const ref = `el-${Date.now()}-${i}`;
+    elementRefs.set(ref, { element: el, description: `${description} [${i}]`, coords });
+    refs.push(ref);
+    elementInfos.push({ ref, x: coords.x, y: coords.y, centerX: coords.centerX, centerY: coords.centerY, confidence: el.confidence });
+  }
+
+  const raw = elements[0]?._response || {};
+  const images: ActionImage[] = [];
+  if (raw.croppedImage) {
+    images.push({ kind: "cropped", base64: bareBase64(raw.croppedImage) });
+  } else if (count === 0) {
+    const shot = await captureScreen();
+    if (shot) images.push({ kind: "screenshot", base64: shot });
+  }
+  const data = leanResponse(raw);
+
+  return {
+    ok: count > 0,
+    text:
+      count > 0
+        ? `Found ${count} elements matching "${description}":\n${refs.map((r, i) => `  [${i}] ${r}`).join("\n")}`
+        : `No elements found matching: "${description}"`,
+    data: { ...data, action: "findall", count, refs, elements: elementInfos },
+    code: count > 0 ? generateActionCode("findall", { description }) : undefined,
+    images: images.length ? images : undefined,
+  };
+}
+
+// =============================================================================
+// Element interaction (require a ref from find/findall)
+// =============================================================================
+
+async function actOnRef(
+  ref: string,
+  verb: "click" | "double-click" | "right-click" | "hover",
+  action: "click" | "hover"
+): Promise<ActionResult> {
+  await requireActiveSession();
+  const stored = elementRefs.get(ref);
+  if (!stored) {
+    return { ok: false, text: `Element reference "${ref}" not found. Use 'find' first to locate the element.`, data: { error: "Element reference not found" } };
+  }
+  const { element, description, coords } = stored;
+
+  if (verb === "click") await element.click();
+  else if (verb === "double-click") await element.doubleClick();
+  else if (verb === "right-click") await element.rightClick();
+  else if (verb === "hover") await element.hover();
+
+  const shot = await captureScreen();
+  if (shot) lastScreenshotBase64 = shot;
+  const data = leanResponse(element._response);
+
+  if (action === "hover") {
+    return {
+      ok: true,
+      text: `Hovered over "${description}"`,
+      data: { ...data, action: "hover" },
+      code: generateActionCode("hover", {}),
+      images: shot ? [{ kind: "screenshot", base64: shot }] : undefined,
+    };
+  }
+  return {
+    ok: true,
+    text: `Clicked on "${description}"`,
+    data: { ...data, action: "click", clickAction: verb, clickPosition: coords },
+    code: generateActionCode("click", { action: verb }),
+    images: shot ? [{ kind: "screenshot", base64: shot }] : undefined,
+  };
+}
+
+export function click(ref: string, action: "click" | "double-click" | "right-click" = "click"): Promise<ActionResult> {
+  return actOnRef(ref, action, "click");
+}
+export function hover(ref: string): Promise<ActionResult> {
+  return actOnRef(ref, "hover", "hover");
+}
+
+export async function findAndClick(description: string, action: "click" | "double-click" | "right-click" = "click"): Promise<ActionResult> {
+  await requireActiveSession();
+  const element = await sdk.find(description);
+  const found = element.found();
+
+  if (!found) {
+    const raw = element._response || {};
+    const images: ActionImage[] = [];
+    if (raw.croppedImage) images.push({ kind: "screenshot", base64: bareBase64(raw.croppedImage) });
+    else {
+      const shot = await captureScreen();
+      if (shot) images.push({ kind: "screenshot", base64: shot });
+    }
+    return {
+      ok: false,
+      text: `Element not found: "${description}"`,
+      data: { ...leanResponse(raw), action: "find_and_click", error: "Element not found" },
+      images: images.length ? images : undefined,
+    };
+  }
+
+  const coords = element.getCoordinates();
+  const ref = `el-${Date.now()}`;
+  if (coords) elementRefs.set(ref, { element, description, coords });
+
+  if (action === "click") await element.click();
+  else if (action === "double-click") await element.doubleClick();
+  else if (action === "right-click") await element.rightClick();
+
+  const raw = element._response || {};
+  const images: ActionImage[] = [];
+  if (raw.croppedImage) images.push({ kind: "cropped", base64: bareBase64(raw.croppedImage) });
+  const data = leanResponse(raw);
+
+  return {
+    ok: true,
+    text: `Found and clicked: "${description}" at (${(raw.coordinates as any)?.x}, ${(raw.coordinates as any)?.y})\nRef: ${ref}`,
+    data: {
+      ...data,
+      action: "find_and_click",
+      element: coords ? { description, centerX: coords.centerX, centerY: coords.centerY, confidence: element.confidence, ref } : undefined,
+      ref,
+      clickAction: action,
+      clickPosition: coords ? { x: coords.centerX, y: coords.centerY } : undefined,
+    },
+    code: generateActionCode("find_and_click", { description, action }),
+    images: images.length ? images : undefined,
+  };
+}
+
+// =============================================================================
+// Input
+// =============================================================================
+
+export async function type(text: string, secret = false, delay?: number): Promise<ActionResult> {
+  await requireActiveSession();
+  await sdk.type(text, { secret, delay });
+  return {
+    ok: true,
+    text: `Typed: ${secret ? "[secret text]" : `"${text}"`}`,
+    data: { action: "type", text: secret ? "[SECRET]" : text },
+    code: generateActionCode("type", { text, secret }),
+  };
+}
+
+export async function pressKeys(keys: string[]): Promise<ActionResult> {
+  await requireActiveSession();
+  await sdk.pressKeys(keys);
+  return {
+    ok: true,
+    text: `Pressed keys: ${keys.join(" + ")}`,
+    data: { action: "press_keys", keys },
+    code: generateActionCode("press_keys", { keys }),
+  };
+}
+
+export async function scroll(direction: "up" | "down" | "left" | "right" = "down", amount?: number): Promise<ActionResult> {
+  await requireActiveSession();
+  await sdk.scroll(direction, amount ? { amount } : undefined);
+  return {
+    ok: true,
+    text: `Scrolled ${direction}${amount ? ` by ${amount}px` : ""}`,
+    data: { action: "scroll", scrollDirection: direction, direction, amount },
+    code: generateActionCode("scroll", { direction, amount }),
+  };
+}
+
+export async function focusApplication(name: string): Promise<ActionResult> {
+  await requireActiveSession();
+  await sdk.focusApplication(name);
+  return {
+    ok: true,
+    text: `Focused application: "${name}"`,
+    data: { action: "focus", name },
+    code: generateActionCode("focus_application", { name }),
+  };
+}
+
+export async function wait(timeout: number): Promise<ActionResult> {
+  await requireActiveSession();
+  await sdk.wait(timeout);
+  return {
+    ok: true,
+    text: `Waited for ${timeout}ms`,
+    data: { action: "wait", timeout },
+    code: generateActionCode("wait", { timeout }),
+  };
+}
+
+export async function exec(language: "sh" | "pwsh", code: string, timeout = 30000): Promise<ActionResult> {
+  await requireActiveSession();
+  const output = await sdk.exec(language, code, timeout);
+  return {
+    ok: true,
+    text: `Executed ${language} code:\n${output || "(no output)"}`,
+    data: { action: "exec", language, output },
+    code: generateActionCode("exec", { language, code, timeout }),
+  };
+}
+
+// =============================================================================
+// Verification
+// =============================================================================
+
+export async function assert(assertion: string): Promise<ActionResult> {
+  await requireActiveSession();
+  const result = await sdk.assert(assertion);
+  return {
+    ok: result,
+    text: result ? `✓ Assertion passed: "${assertion}"` : `✗ Assertion failed: "${assertion}"`,
+    data: { action: "assert", assertion, passed: result },
+    code: generateActionCode("assert", { assertion }),
+  };
+}
+
+/**
+ * AI analysis of the current screen vs. a "before" state. Does NOT generate
+ * code. `referenceImage` (bare base64), if provided, is used as the before
+ * image instead of the last captured screenshot.
+ */
+export async function check(task: string, referenceImage?: string): Promise<ActionResult> {
+  await requireActiveSession();
+  const currentScreenshot = await sdk.agent.system.captureScreenBase64(1, false, true);
+  const beforeScreenshot = referenceImage || lastScreenshotBase64 || currentScreenshot;
+  lastScreenshotBase64 = currentScreenshot;
+
+  const mousePosition = await sdk.agent.system.getMousePosition();
+  const activeWindow = await sdk.agent.system.activeWin();
+
+  const response = await sdk.agent.sdk.req("check", {
+    tasks: [task],
+    images: [beforeScreenshot, currentScreenshot],
+    mousePosition,
+    activeWindow,
+  });
+  const aiResponse = response.data;
+
+  const hasCodeBlocks =
+    aiResponse && (aiResponse.includes("```yml") || aiResponse.includes("```yaml") || aiResponse.includes("- command:"));
+  const isComplete = !hasCodeBlocks;
+
+  return {
+    ok: isComplete,
+    text: isComplete
+      ? `✓ Task appears complete: "${task}"\n\nAI Analysis:\n${aiResponse}`
+      : `⚠ Task may not be complete: "${task}"\n\nAI Analysis:\n${aiResponse}`,
+    data: { action: "check", task, complete: isComplete, aiResponse },
+    images: currentScreenshot ? [{ kind: "screenshot", base64: currentScreenshot }] : undefined,
+  };
+}
+
+/** Capture a screenshot for the user (cursor visible). */
+export async function screenshot(): Promise<ActionResult> {
+  await requireActiveSession();
+  const shot = await sdk.agent.system.captureScreenBase64(1, false, true);
+  if (shot) lastScreenshotBase64 = shot;
+  return {
+    ok: !!shot,
+    text: shot ? "Captured screenshot" : "Failed to capture screenshot",
+    data: { action: "screenshot" },
+    images: shot ? [{ kind: "screenshot", base64: shot }] : undefined,
+  };
+}
+
+export type { SessionState };