@testdriverai/mcp 7.10.0-canary.0 → 7.10.1-test

package/lib/resolve-channel.js

@@ -64,10 +64,10 @@ const tdEnv = resolveTdEnv();
 
 /**
  * Resolves the Sentry environment name.
- * Uses the infrastructure tier (dev | staging | production).
+ * Uses the release channel (dev | test | canary | stable).
  */
 function resolveSentryEnvironment() {
-  return tdEnv;
+  return active;
 }
 
 const sentryEnvironment = resolveSentryEnvironment();

package/lib/sentry.js

@@ -48,8 +48,8 @@ if (isEnabled()) {
       "https://452bd5a00dbd83a38ee8813e11c57694@o4510262629236736.ingest.us.sentry.io/4510480443637760",
     environment: sentryEnvironment,
     release: version,
-    sampleRate: 1.0,
-    tracesSampleRate: 1.0, // Sample 20% of transactions for performance
+    sampleRate: 0.01,
+    tracesSampleRate: 0.01,
     enableLogs: true,
     integrations: [Sentry.httpIntegration(), Sentry.nodeContextIntegration()],
     // Set initial context
@@ -414,6 +414,18 @@ function startTransaction(name, op = "cli") {
   return Sentry.startSpan({ name, op });
 }
 
+/**
+ * Execute a callback within a Sentry performance span.
+ * If Sentry is not enabled, the callback is executed directly.
+ * @param {Object} options - Span options ({ name, op, attributes })
+ * @param {Function} callback - Async callback receiving the span
+ * @returns {Promise<*>} Result of the callback
+ */
+async function withSpan(options, callback) {
+  if (!isEnabled()) return callback(null);
+  return Sentry.startSpan(options, callback);
+}
+
 /**
  * Flush pending events before process exit
  * @param {number} timeout - Timeout in milliseconds
@@ -436,5 +448,6 @@ module.exports = {
   getTraceId,
   attachLogListeners,
   startTransaction,
+  withSpan,
   flush,
 };

package/lib/vitest/hooks.mjs

@@ -433,25 +433,20 @@ export function TestDriver(context, options = {}) {
   testdriver._debugOnFailure = mergedOptions.debugOnFailure || false;
   testDriverInstances.set(context.task, testdriver);
 
-  // Set platform metadata early so the reporter can show the correct OS from the start
+  // Set platform metadata early so the reporter can show the correct OS from
+  // the start. Test identity (file / suite / name) comes from Vitest's own
+  // task tree in the reporter — we deliberately do not duplicate it here.
   if (!context.task.meta) {
     context.task.meta = {};
   }
-  const platform = mergedOptions.os || "linux";
-  const absolutePath =
-    context.task.file?.filepath || context.task.file?.name || "unknown";
-  const projectRoot = process.cwd();
-  const testFile =
-    absolutePath !== "unknown"
-      ? path.relative(projectRoot, absolutePath)
-      : absolutePath;
-
-  context.task.meta.platform = platform;
-  context.task.meta.testFile = testFile;
-  context.task.meta.testOrder = 0;
-
-  // Pass test file name to SDK for debugger display
-  testdriver.testFile = testFile;
+  context.task.meta.platform = mergedOptions.os || "linux";
+
+  // Pass test file name to SDK for debugger display. Use Vitest's standard
+  // runner-task `file.filepath` rather than reconstructing identity ourselves.
+  const taskFilepath = context.task.file?.filepath || null;
+  testdriver.testFile = taskFilepath
+    ? path.relative(process.cwd(), taskFilepath)
+    : null;
 
   const debugConsoleSpy = process.env.TD_DEBUG_CONSOLE_SPY === "true";
 
@@ -570,21 +565,10 @@ export function TestDriver(context, options = {}) {
         context.task.meta = {};
       }
 
-      // Always set test metadata, even if dashcam never started or fails to stop
-      // This ensures the reporter can record test results even for early failures
-      const platform = currentInstance.os || "linux";
-      const absolutePath =
-        context.task.file?.filepath || context.task.file?.name || "unknown";
-      const projectRoot = process.cwd();
-      const testFile =
-        absolutePath !== "unknown"
-          ? path.relative(projectRoot, absolutePath)
-          : absolutePath;
-
-      // Set basic metadata that's always available
-      context.task.meta.platform = platform;
-      context.task.meta.testFile = testFile;
-      context.task.meta.testOrder = 0;
+      // Refresh SDK-owned metadata. Test identity (file / suite / name) is
+      // read directly from the Vitest task tree by the reporter — we only
+      // store things Vitest can't know about (platform, session, dashcam).
+      context.task.meta.platform = currentInstance.os || "linux";
       context.task.meta.sessionId = currentInstance.getSessionId?.() || null;
 
       // Initialize dashcamUrls array for tracking per-attempt URLs (persists across retries)
@@ -597,6 +581,11 @@ export function TestDriver(context, options = {}) {
       const isRetry = attemptNumber > 1;
       const attemptLabel = isRetry ? ` (attempt ${attemptNumber})` : "";
 
+      // Capture the current attempt's test error (before retry resets result)
+      const attemptResult = context.task.result?.errors?.[0];
+      const attemptErrorMessage = attemptResult?.message || null;
+      const attemptErrorStack = attemptResult?.stack || null;
+
       // Stop dashcam if it was started - with timeout to prevent hanging
       if (currentInstance._dashcam && currentInstance._dashcam.recording) {
         try {
@@ -607,6 +596,8 @@ export function TestDriver(context, options = {}) {
             attempt: attemptNumber,
             url: dashcamUrl || null,
             sessionId: currentInstance.getSessionId?.() || null,
+            errorMessage: attemptErrorMessage,
+            errorStack: attemptErrorStack,
           });
 
           // Keep backward compatibility - last attempt's URL
@@ -617,7 +608,7 @@ export function TestDriver(context, options = {}) {
             globalThis.__testdriverPlugin.registerDashcamUrl(
               context.task.id,
               dashcamUrl,
-              platform,
+              context.task.meta.platform,
               attemptNumber,
             );
           }
@@ -660,6 +651,8 @@ export function TestDriver(context, options = {}) {
             url: null,
             sessionId: currentInstance.getSessionId?.() || null,
             error: error.message,
+            errorMessage: attemptErrorMessage,
+            errorStack: attemptErrorStack,
           });
           // Ensure dashcamUrl is set to null if stop failed
           context.task.meta.dashcamUrl = null;
@@ -670,6 +663,8 @@ export function TestDriver(context, options = {}) {
           attempt: attemptNumber,
           url: null,
           sessionId: currentInstance.getSessionId?.() || null,
+          errorMessage: attemptErrorMessage,
+          errorStack: attemptErrorStack,
         });
         context.task.meta.dashcamUrl = null;
       }
@@ -684,6 +679,21 @@ export function TestDriver(context, options = {}) {
         const sbx = currentInstance.sandbox || {};
         const apiRoot = currentInstance.config?.TD_API_ROOT || null;
 
+        const taskFilepath = context.task.file?.filepath || null;
+        const testFileRel = taskFilepath
+          ? path.relative(process.cwd(), taskFilepath)
+          : null;
+        const suiteParts = [];
+        {
+          let cursor = context.task.suite;
+          while (cursor && cursor.type === "suite") {
+            suiteParts.unshift(cursor.name);
+            cursor = cursor.suite;
+          }
+        }
+        const fullSuiteName =
+          suiteParts.length > 0 ? suiteParts.join(" > ") : null;
+
         context.task.meta.testResult = {
           // Versions
           sdkVersion: sdkPkg.version || null,
@@ -704,9 +714,9 @@ export function TestDriver(context, options = {}) {
           sessionId: currentInstance.getSessionId?.() || null,
 
           // Test info
-          testFile: context.task.meta.testFile || null,
+          testFile: testFileRel,
           testName: context.task.name || null,
-          suiteName: context.task.suite?.name || null,
+          suiteName: fullSuiteName,
 
           // Test result
           testPassed: context.task.result?.state === "pass",

package/mcp-server/README.md

@@ -89,6 +89,38 @@ To use the TestDriver MCP server with GitHub Copilot coding agent:
 
 For more information, see [Extending GitHub Copilot coding agent with Model Context Protocol](https://docs.github.com/en/copilot/customizing-copilot/extending-copilot-coding-agent-with-mcp).
 
+### With Claude Code (plugin)
+
+TestDriver ships as a [Claude Code plugin](https://docs.claude.com/en/docs/claude-code/plugins) so you get the MCP server, the `testdriver` expert agent, and all TestDriver skills with a single install.
+
+1. Set your TestDriver API key in your shell:
+
+   ```bash
+   export TD_API_KEY=your_api_key_from_https_console_testdriver_ai_team
+   ```
+
+2. Add this repo as a Claude Code plugin marketplace and install the `testdriver` plugin:
+
+   ```bash
+   # Add the marketplace (sources `.claude-plugin/marketplace.json` from this repo)
+   /plugin marketplace add testdriverai/testdriverai
+
+   # Install the plugin
+   /plugin install testdriver@testdriver
+   ```
+
+   This reads `.claude-plugin/plugin.json` (shipped inside the `testdriverai` npm package) and:
+
+   - Registers the `testdriver` MCP server (`npx -p testdriverai testdriverai-mcp`)
+   - Registers the `testdriver` sub-agent (from `ai/agents/testdriver.md`)
+   - Registers all TestDriver skills (from `ai/skills/testdriver-*/SKILL.md`)
+
+3. In a Claude Code session, delegate to the agent with:
+
+   ```text
+   @testdriver Write a test that signs into example.com and adds an item to the cart.
+   ```
+
 ### With Claude Desktop / Cursor
 
 Add to your MCP config (`~/.cursor/mcp.json` or `~/Library/Application Support/Claude/claude_desktop_config.json`):

package/mcp-server/dist/core/actions.d.ts

@@ -0,0 +1,176 @@
+/**
+ * TestDriver action core — framework-agnostic.
+ *
+ * This module owns the actual computer-use logic: it holds the singleton SDK,
+ * the element-reference map, and the "last screenshot" used by `check`, and
+ * exposes one async function per action. Every function returns a neutral
+ * {@link ActionResult} — no MCP types, no eve types — so it can be wrapped by:
+ *
+ *   - the MCP server (`server.ts`), which turns images into resource URIs and
+ *     calls `createToolResult`, and
+ *   - eve `defineTool()` files, which return the text/code/images as eve content.
+ *
+ * Keep MCP- and eve-specific concerns (Sentry, progress heartbeats, abort
+ * racing, image stores, UI resources) in the adapters. This file only knows
+ * about the TestDriver SDK and `generateActionCode`.
+ *
+ * State is process-global and single-session by design: one sandbox at a time,
+ * mirroring how the stdio MCP server has always behaved.
+ */
+import { SessionStartInputSchema, type SessionStartInput } from "../provision-types.js";
+import { type SessionState } from "../session.js";
+export { SessionStartInputSchema, type SessionStartInput };
+export interface ActionImage {
+    kind: "screenshot" | "cropped";
+    /** Raw base64 (no `data:` prefix). */
+    base64: string;
+}
+export interface ActionResult {
+    /** Whether the action succeeded (assertions/finds may legitimately be false). */
+    ok: boolean;
+    /** Human/agent-readable summary line(s). */
+    text: string;
+    /** Structured fields for programmatic consumers and UIs. */
+    data: Record<string, unknown>;
+    /** Code to append to the test file, when the action produces test code. */
+    code?: string;
+    /** Images produced by the action (screenshots / cropped element shots). */
+    images?: ActionImage[];
+}
+/** Thrown when an action runs without an active/valid session. */
+export declare class NoActiveSessionError extends Error {
+    readonly code: "NO_SESSION" | "SESSION_EXPIRED";
+    readonly expiredSessionId?: string;
+    constructor(code: "NO_SESSION" | "SESSION_EXPIRED", message: string, expiredSessionId?: string);
+}
+/**
+ * Optional adapter-supplied recovery hook. When the singleton SDK is missing or
+ * stale at action time, {@link requireActiveSession} calls this to obtain the
+ * params needed to rebuild the connection from the still-alive sandbox, then
+ * reconnects before throwing NO_SESSION.
+ *
+ * Why a hook instead of mcp-core owning the handle: the durable facts (sandbox
+ * id, config) and the API key must survive a host process recycle, but this
+ * module's globals do NOT — they reset on the very recycle we're recovering
+ * from. So the *durable owner* (eve's per-session state) registers a resolver
+ * that reads its own durable store and re-resolves the key per call. Hosts with
+ * one long-lived process (stdio MCP server, CLI) register nothing: the resolver
+ * stays null and behavior is unchanged — `requireActiveSession` throws the same
+ * NO_SESSION/SESSION_EXPIRED as before.
+ *
+ * The resolver returns `null` when it has nothing to recover from (no sandbox
+ * provisioned this session), in which case we fall through to the normal throw.
+ * It is re-registered each step by the adapter (module globals reset on recycle),
+ * so a stale closure can't outlive the process it was bound to.
+ */
+export type ReconnectResolver = () => Promise<ReconnectParams | null>;
+/**
+ * Register (or clear, with `null`) the recovery hook. Durable adapters call this
+ * at the start of each step with a resolver bound to the current tool context.
+ */
+export declare function setReconnectResolver(resolver: ReconnectResolver | null): void;
+/** Expose internals the adapters legitimately need (read-only intent). */
+export declare function getSdk(): any;
+export declare function getLastScreenshotBase64(): string | null;
+export declare function getElementRef(ref: string): {
+    element: any;
+    description: string;
+    coords: {
+        x: number;
+        y: number;
+        centerX: number;
+        centerY: number;
+    };
+} | undefined;
+/**
+ * Cheap, side-effect-free check for a usable in-process session: a live SDK and
+ * a current session that hasn't expired. Durable adapters call this *before*
+ * doing any expensive reconnect prep (e.g. re-resolving an API key), so the
+ * common warm-process path stays free. Note: unlike `requireActiveSession`, this
+ * does NOT refresh the keepAlive window — it only reports liveness.
+ */
+export declare function hasLiveSession(): boolean;
+export interface SessionStartHooks {
+    /** Called before a long await; return a stop fn. Lets adapters heartbeat. */
+    onProgress?: (message: string) => void;
+}
+/**
+ * Start a session and provision a sandbox. The adapter is responsible for
+ * progress/heartbeat and abort; this performs the provisioning and returns the
+ * neutral result (including the initial screenshot and provision code).
+ */
+export declare function sessionStart(params: SessionStartInput, resolved: {
+    os: "linux" | "windows";
+    e2bTemplateId?: string;
+    apiKey?: string;
+}, hooks?: SessionStartHooks): Promise<ActionResult>;
+/**
+ * Parameters needed to rebuild the singleton SDK and reconnect to a sandbox
+ * that is still alive on the server but whose in-process connection was lost
+ * (e.g. the host process was recycled between durable workflow steps).
+ */
+export interface ReconnectParams {
+    /** Cloud sandbox id (`sb-…`) to reconnect to. */
+    sandboxId: string;
+    os: "linux" | "windows";
+    keepAlive: number;
+    apiKey: string;
+    apiRoot?: string;
+    e2bTemplateId?: string;
+    /** Self-hosted instance IP, when the original session used one. */
+    ip?: string;
+    testFile?: string;
+}
+/**
+ * Rebuild the process-global SDK and reconnect it to an existing sandbox by id,
+ * skipping provisioning. This is the recovery path for hosts that do NOT keep a
+ * single long-lived process: eve runs each turn as a durable workflow on Vercel
+ * and may park or recycle the process between steps, which drops the SDK's
+ * realtime connection and wipes this module's singletons. The sandbox itself can
+ * still be alive on the server (the API holds it for `keepAlive`/grace), so we
+ * re-`connect({ sandboxId })` and re-register a local session instead of failing
+ * the action with NO_SESSION.
+ *
+ * Returns the (re)activated session id. Throws if the reconnect fails (the
+ * sandbox is gone) — callers translate that into a SESSION_EXPIRED so the agent
+ * knows to call session_start again.
+ */
+export declare function reconnectSession(params: ReconnectParams): Promise<string>;
+/**
+ * Ensure there is a usable session before an action, reconnecting from durable
+ * state if the in-process singleton was lost. Returns true when a fresh
+ * reconnect happened (the caller may want to recapture context), false when the
+ * existing session was already usable.
+ *
+ * Durable adapters call this at the top of each tool, passing the sandbox id and
+ * config they persisted at session_start. Hosts with a single long-lived process
+ * (the stdio MCP server, the CLI) never persist a sandbox id, so they pass none
+ * and this is a no-op — `requireActiveSession()` inside each action still guards
+ * them exactly as before.
+ */
+export declare function ensureActiveSession(params?: ReconnectParams): Promise<boolean>;
+/** Disconnect the SDK (best-effort); used by adapters on cancel/teardown. */
+export declare function disconnect(): Promise<void>;
+export declare function sessionStatus(): ActionResult;
+export declare function sessionExtend(additionalMs: number): ActionResult;
+export declare function find(description: string, timeout?: number): Promise<ActionResult>;
+export declare function findAll(description: string, timeout?: number): Promise<ActionResult>;
+export declare function click(ref: string, action?: "click" | "double-click" | "right-click"): Promise<ActionResult>;
+export declare function hover(ref: string): Promise<ActionResult>;
+export declare function findAndClick(description: string, action?: "click" | "double-click" | "right-click"): Promise<ActionResult>;
+export declare function type(text: string, secret?: boolean, delay?: number): Promise<ActionResult>;
+export declare function pressKeys(keys: string[]): Promise<ActionResult>;
+export declare function scroll(direction?: "up" | "down" | "left" | "right", amount?: number): Promise<ActionResult>;
+export declare function focusApplication(name: string): Promise<ActionResult>;
+export declare function wait(timeout: number): Promise<ActionResult>;
+export declare function exec(language: "sh" | "pwsh", code: string, timeout?: number): Promise<ActionResult>;
+export declare function assert(assertion: string): Promise<ActionResult>;
+/**
+ * AI analysis of the current screen vs. a "before" state. Does NOT generate
+ * code. `referenceImage` (bare base64), if provided, is used as the before
+ * image instead of the last captured screenshot.
+ */
+export declare function check(task: string, referenceImage?: string): Promise<ActionResult>;
+/** Capture a screenshot for the user (cursor visible). */
+export declare function screenshot(): Promise<ActionResult>;
+export type { SessionState };