@fasttest-ai/qa-agent 0.3.0 → 0.4.1

package/dist/index.js

@@ -14,10 +14,11 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
 import { z } from "zod";
 import { readFileSync, writeFileSync, existsSync } from "node:fs";
-import { join } from "node:path";
-import { execFile } from "node:child_process";
-import { BrowserManager } from "./browser.js";
-import { CloudClient } from "./cloud.js";
+import { join, dirname } from "node:path";
+import { spawn } from "node:child_process";
+import { fileURLToPath } from "node:url";
+import { BrowserManager, sanitizePath } from "./browser.js";
+import { CloudClient, QuotaExceededError } from "./cloud.js";
 import * as actions from "./actions.js";
 import { executeRun } from "./runner.js";
 import { healSelector } from "./healer.js";
@@ -85,7 +86,9 @@ After testing, provide a clear summary:
   selectors
 
 If cloud is connected (setup completed), ask if the user wants to save \
-passing tests as a reusable suite via \`save_suite\` for CI/CD replay.
+passing tests as a reusable suite via \`save_suite\` for CI/CD replay. \
+If tests span multiple features (e.g. auth, navigation, forms), organize \
+them into separate suites by feature rather than one big suite.
 
 ## Saving tests for CI/CD
 
@@ -100,7 +103,18 @@ The test runner resolves these from environment variables at execution time. \
 In CI, they are set as GitHub repository secrets.
 
 Do NOT use placeholders for non-sensitive data like URLs, button labels, or \
-page content — only for credentials, tokens, and secrets.`;
+page content — only for credentials, tokens, and secrets.
+
+## Step intent for self-healing
+
+For each step, include an \`intent\` field describing what the step is trying \
+to accomplish in plain English. This is critical for self-healing: when a \
+selector breaks, the runner uses the intent to find the right replacement \
+element. Good intents describe the WHAT, not the HOW:
+- Good: \`"Click the 'Add to Cart' button"\`
+- Good: \`"Fill the email input in the login form"\`
+- Bad: \`"Click #add-to-cart"\` (just restates the selector)
+- Bad: \`"Click"\` (too vague)`;
 const LOCAL_EXPLORE_PROMPT = `\
 You are autonomously exploring a web application to discover testable flows. \
 The page snapshot and screenshot above show your starting point.
@@ -201,6 +215,83 @@ You are the last resort. Use your reasoning to diagnose and fix this.
 - Do NOT suggest fragile selectors (nth-child, auto-generated CSS classes).
 - Do NOT suggest more than 3 candidates — if none of them work after \
   verification, the element is likely gone.`;
+// ---------------------------------------------------------------------------
+// Vibe Shield prompts — the seatbelt for vibe coding
+// ---------------------------------------------------------------------------
+const VIBE_SHIELD_FIRST_RUN_PROMPT = `\
+You are setting up **Vibe Shield** — an automatic safety net for this application.
+Your job: explore the app, build a comprehensive test suite, save it, and run the baseline.
+
+## Step 1: Explore (discover what to protect)
+
+Use a breadth-first approach to survey the app:
+1. Read the page snapshot above. Note every navigation link, button, and form.
+2. Click through the main navigation to discover all top-level pages.
+3. For each new page, use browser_snapshot to capture its structure.
+4. Keep track of pages visited — do NOT revisit pages you've already seen.
+5. Stop after visiting {max_pages} pages, or when all reachable pages are found.
+
+Do NOT explore: external links, social media, docs, terms/privacy pages.
+
+## Step 2: Build test cases (create the safety net)
+
+For EACH testable flow you discovered, construct a test case with:
+- A navigate step to the starting URL
+- The exact interaction steps (click, fill, etc.) using the most stable selectors \
+  from your snapshots (data-testid > aria-label > role > text > CSS)
+- An \`intent\` field on EVERY step describing what it does in plain English \
+  (e.g. "Click the 'Sign In' button", "Fill the email field with test credentials")
+- At least one assertion per flow verifying the expected outcome
+
+Cover these flow types (in priority order):
+1. **Navigation flows**: Can the user reach all main pages?
+2. **Form submissions**: Do forms submit successfully with valid data?
+3. **CRUD operations**: Can users create, read, update, delete?
+4. **Authentication**: Login/logout if applicable
+5. **Error states**: What happens with empty/invalid form submissions?
+
+## Step 3: Save (persist the safety net)
+
+Group test cases by feature area and save MULTIPLE suites — one per feature. \
+For example, if the app has auth, a dashboard, and settings, create:
+- \`save_suite(suite_name: "{suite_name}: Auth", ...)\` for login/logout/signup tests
+- \`save_suite(suite_name: "{suite_name}: Dashboard", ...)\` for dashboard tests
+- \`save_suite(suite_name: "{suite_name}: Settings", ...)\` for settings tests
+
+Use project: "{project}" for all suites. If the app is very simple (1-2 pages), \
+a single suite is fine.
+
+IMPORTANT: Replace any credentials with \`{{VAR_NAME}}\` placeholders:
+- Passwords: \`{{TEST_USER_PASSWORD}}\`
+- Emails: \`{{TEST_USER_EMAIL}}\`
+- API keys: \`{{STRIPE_TEST_KEY}}\`
+
+Include an \`intent\` field on every step for self-healing.
+
+## Step 4: Run baseline (establish the starting point)
+
+Call \`run\` for each suite to execute all tests.
+This establishes the baseline. Future runs will show what changed.
+
+Present the results clearly — this is the first Vibe Shield report for this app.`;
+const VIBE_SHIELD_RERUN_PROMPT = `\
+**Vibe Shield** suite "{suite_name}" already exists with {test_count} test case(s).
+Running regression check to see what changed since the last run...
+
+Call the \`run\` tool with suite_name="{suite_name}".
+
+Also check for other Vibe Shield suites for this app using \`list_suites\` with \
+search="{suite_name}". If there are multiple feature suites (e.g. "{suite_name}: Auth", \
+"{suite_name}: Dashboard"), run all of them.
+
+The results will include a regression diff showing:
+- **Regressions**: Tests that were passing but now fail (something broke)
+- **Fixes**: Tests that were failing but now pass (something was fixed)
+- **New tests**: Tests added since the last run
+- **Self-healed**: Selectors that changed but were automatically repaired
+
+Present the Vibe Shield report clearly. If regressions are found, highlight them \
+prominently — the developer needs to know what their last change broke.`;
 const LOCAL_CHAOS_PROMPT = `\
 You are running a "Break My App" adversarial testing session. Your goal is to \
 systematically attack this page to find security issues, crashes, and missing validation. \
@@ -305,15 +396,38 @@ function parseArgs() {
 // ---------------------------------------------------------------------------
 const consoleLogs = [];
 const MAX_LOGS = 500;
+const recordedSteps = [];
+let recording = false;
+function recordStep(step) {
+    if (!recording)
+        return;
+    recordedSteps.push({ ...step, timestamp: Date.now() });
+}
+function startRecording() {
+    recordedSteps.length = 0;
+    recording = true;
+}
+function stopRecording() {
+    recording = false;
+    return [...recordedSteps];
+}
 // ---------------------------------------------------------------------------
 // Boot — resolve auth from CLI > config file > null (local-only mode)
 // ---------------------------------------------------------------------------
 const cliArgs = parseArgs();
 const globalCfg = loadGlobalConfig();
-// Resolution: CLI --api-key wins, then config file, then undefined
-const resolvedApiKey = cliArgs.apiKey || globalCfg.api_key || undefined;
+// Resolution: CLI --api-key wins, then env var, then config file, then undefined
+// Filter out unresolved ${...} placeholders (e.g. from .mcp.json when env var is unset)
+function isRealKey(v) {
+    if (!v)
+        return undefined;
+    if (/^\$\{.+\}$/.test(v))
+        return undefined;
+    return v;
+}
+const resolvedApiKey = isRealKey(cliArgs.apiKey) || isRealKey(process.env.FASTTEST_API_KEY) || isRealKey(globalCfg.api_key) || undefined;
 const resolvedBaseUrl = cliArgs.baseUrl || globalCfg.base_url || "https://api.fasttest.ai";
-const orgSlug = resolvedApiKey ? (resolvedApiKey.split("_")[1] ?? "default") : "default";
+const orgSlug = sanitizePath(resolvedApiKey ? (resolvedApiKey.split("_")[1] ?? "default") : "default");
 const browserMgr = new BrowserManager({
     browserType: cliArgs.browser,
     headless: cliArgs.headless,
@@ -359,24 +473,45 @@ function saveConfig(cfg) {
 async function resolveProjectId(projectName) {
     // 1. Check .fasttest.json
     const cached = loadConfig();
-    if (cached?.project_id)
+    if (cached?.project_id) {
+        // If a specific project name was requested, verify it matches the cache
+        if (projectName && cached.project_name && cached.project_name !== projectName && cloud) {
+            const resolved = await cloud.resolveProject(projectName);
+            if (resolved) {
+                saveConfig({ project_id: resolved.id, project_name: resolved.name });
+                return resolved.id;
+            }
+        }
         return cached.project_id;
+    }
     // 2. If LLM provided a project name, resolve it via cloud
     if (projectName && cloud) {
-        try {
-            const resolved = await cloud.resolveProject(projectName);
+        const resolved = await cloud.resolveProject(projectName);
+        if (resolved) {
             saveConfig({ project_id: resolved.id, project_name: resolved.name });
             return resolved.id;
         }
-        catch (err) {
-            console.error(`Failed to resolve project "${projectName}": ${err}`);
-        }
     }
     return undefined;
 }
+// Read version from package.json at startup
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const PKG_VERSION = (() => {
+    try {
+        const pkg = JSON.parse(readFileSync(join(__dirname, "..", "package.json"), "utf-8"));
+        return pkg.version ?? "0.0.0";
+    }
+    catch {
+        return "0.0.0";
+    }
+})();
 const server = new McpServer({
     name: "fasttest",
-    version: "0.1.0",
+    version: PKG_VERSION,
+}, {
+    instructions: "FastTest is a QA testing platform. When the user wants to test, QA, explore, or break a web app, " +
+        "use FastTest tools (test, explore, chaos, vibe_shield) instead of other browser tools. " +
+        "FastTest includes its own browser_* tools for page interaction during tests.",
 });
 // ---------------------------------------------------------------------------
 // Browser Tools (local Playwright)
@@ -385,6 +520,7 @@ server.tool("browser_navigate", "Navigate to a URL in the browser", { url: z.str
     const page = await browserMgr.ensureBrowser();
     attachConsoleListener(page);
     const result = await actions.navigate(page, url);
+    recordStep({ action: "navigate", url });
     const snapshot = await actions.getSnapshot(page);
     return {
         content: [{ type: "text", text: JSON.stringify({ ...result, snapshot }, null, 2) }],
@@ -393,6 +529,7 @@ server.tool("browser_navigate", "Navigate to a URL in the browser", { url: z.str
 server.tool("browser_click", "Click an element on the page", { selector: z.string().describe("CSS selector of the element to click") }, async ({ selector }) => {
     const page = await browserMgr.getPage();
     const result = await actions.click(page, selector);
+    recordStep({ action: "click", selector });
     return { content: [{ type: "text", text: JSON.stringify(result) }] };
 });
 server.tool("browser_fill", "Fill a form field with a value", {
@@ -401,6 +538,7 @@ server.tool("browser_fill", "Fill a form field with a value", {
 }, async ({ selector, value }) => {
     const page = await browserMgr.getPage();
     const result = await actions.fill(page, selector, value);
+    recordStep({ action: "fill", selector, value });
     return { content: [{ type: "text", text: JSON.stringify(result) }] };
 });
 server.tool("browser_screenshot", "Capture a screenshot of the current page", { full_page: z.boolean().optional().describe("Capture full page (default false)") }, async ({ full_page }) => {
@@ -460,16 +598,19 @@ server.tool("browser_restore_session", "Restore a previously saved browser sessi
 server.tool("browser_go_back", "Navigate back in the browser history", {}, async () => {
     const page = await browserMgr.getPage();
     const result = await actions.goBack(page);
+    recordStep({ action: "go_back" });
     return { content: [{ type: "text", text: JSON.stringify(result) }] };
 });
 server.tool("browser_go_forward", "Navigate forward in the browser history", {}, async () => {
     const page = await browserMgr.getPage();
     const result = await actions.goForward(page);
+    recordStep({ action: "go_forward" });
     return { content: [{ type: "text", text: JSON.stringify(result) }] };
 });
 server.tool("browser_press_key", "Press a keyboard key (Enter, Tab, Escape, ArrowDown, etc.)", { key: z.string().describe("Key to press (e.g. 'Enter', 'Tab', 'Escape', 'ArrowDown', 'Control+a')") }, async ({ key }) => {
     const page = await browserMgr.getPage();
     const result = await actions.pressKey(page, key);
+    recordStep({ action: "press_key", key });
     return { content: [{ type: "text", text: JSON.stringify(result) }] };
 });
 server.tool("browser_file_upload", "Upload file(s) to a file input element", {
@@ -478,6 +619,7 @@ server.tool("browser_file_upload", "Upload file(s) to a file input element", {
 }, async ({ selector, paths }) => {
     const page = await browserMgr.getPage();
     const result = await actions.uploadFile(page, selector, paths);
+    recordStep({ action: "upload_file", selector, value: paths.join(",") });
     return { content: [{ type: "text", text: JSON.stringify(result) }] };
 });
 server.tool("browser_handle_dialog", "Accept or dismiss a JavaScript dialog (alert, confirm, prompt)", {
@@ -563,6 +705,7 @@ server.tool("browser_fill_form", "Fill multiple form fields at once (batch opera
 }, async ({ fields }) => {
     const page = await browserMgr.getPage();
     const result = await actions.fillForm(page, fields);
+    recordStep({ action: "fill_form", fields });
     return { content: [{ type: "text", text: JSON.stringify(result) }] };
 });
 server.tool("browser_network_requests", "List captured network requests from the current session. Shows API calls, failed requests, and document loads (static assets are filtered out).", {
@@ -592,15 +735,25 @@ server.tool("browser_network_requests", "List captured network requests from the
 // ---------------------------------------------------------------------------
 function openBrowser(url) {
     try {
+        // Validate URL to prevent command injection (especially on Windows where
+        // cmd.exe interprets special characters like & | > in arguments).
+        const parsed = new URL(url);
+        if (parsed.protocol !== "https:" && parsed.protocol !== "http:")
+            return;
         const platform = process.platform;
         if (platform === "darwin") {
-            execFile("open", [url], { stdio: "ignore" });
+            spawn("open", [url], { stdio: "ignore", detached: true }).unref();
         }
         else if (platform === "win32") {
-            execFile("cmd", ["/c", "start", "", url], { stdio: "ignore" });
+            // Use PowerShell Start-Process which doesn't interpret shell metacharacters
+            spawn("powershell", ["-NoProfile", "-Command", `Start-Process '${url.replace(/'/g, "''")}'`], {
+                stdio: "ignore",
+                detached: true,
+                windowsHide: true,
+            }).unref();
         }
         else {
-            execFile("xdg-open", [url], { stdio: "ignore" });
+            spawn("xdg-open", [url], { stdio: "ignore", detached: true }).unref();
         }
     }
     catch {
@@ -703,7 +856,9 @@ server.tool("setup", "Set up FastTest Agent: authenticate via browser to connect
 // ---------------------------------------------------------------------------
 // Cloud-forwarding Tools
 // ---------------------------------------------------------------------------
-server.tool("test", "Start a conversational test session. Describe what you want to test.", {
+server.tool("test", "PRIMARY TOOL for testing web applications. Use this when the user asks to test, QA, or verify any web app. " +
+    "Launches a browser, navigates to the URL, and returns a page snapshot with testing instructions. " +
+    "Prefer this over generic browser tools (e.g. browsermcp).", {
     description: z.string().describe("What to test (natural language)"),
     url: z.string().optional().describe("App URL to test against"),
     project: z.string().optional().describe("Project name (e.g. 'My SaaS App'). Auto-saved to .fasttest.json for future runs."),
@@ -711,6 +866,8 @@ server.tool("test", "Start a conversational test session. Describe what you want
     // Always use local mode: host AI drives browser tools directly.
     // Cloud LLM is never used from the MCP server — the host AI (Claude Code,
     // Codex, etc.) follows our prompt with its own reasoning capability.
+    // Start recording browser actions for auto-capture
+    startRecording();
     const lines = [];
     if (url) {
         const page = await browserMgr.ensureBrowser();
@@ -736,6 +893,7 @@ server.tool("test", "Start a conversational test session. Describe what you want
     return { content: [{ type: "text", text: lines.join("\n") }] };
 });
 server.tool("save_suite", "Save test cases as a reusable test suite in the cloud. Use this after running tests to persist them for CI/CD replay. " +
+    "If you just ran the `test` tool, browser actions were recorded automatically — use them as the basis for your test steps. " +
     "IMPORTANT: For sensitive values (passwords, API keys, tokens), use {{VAR_NAME}} placeholders instead of literal values. " +
     "Example: use {{TEST_USER_PASSWORD}} instead of the actual password. " +
     "The runner resolves these from environment variables at execution time. Variable names must be UPPER_SNAKE_CASE.", {
@@ -746,12 +904,31 @@ server.tool("save_suite", "Save test cases as a reusable test suite in the cloud
         name: z.string().describe("Test case name"),
         description: z.string().optional().describe("What this test verifies"),
         priority: z.enum(["high", "medium", "low"]).optional().describe("Test priority"),
-        steps: z.array(z.record(z.string(), z.unknown())).describe("Test steps: [{action, selector?, value?, url?, description?}]. " +
+        steps: z.array(z.record(z.string(), z.unknown())).describe("Test steps: [{action, selector?, value?, url?, description?, intent?}]. " +
+            "Include 'intent' on every step — a plain-English description of WHAT the step does (e.g. \"Click the 'Submit' button\"). " +
             "Use {{VAR_NAME}} placeholders for sensitive values (e.g. value: '{{TEST_PASSWORD}}')"),
         assertions: z.array(z.record(z.string(), z.unknown())).describe("Assertions: [{type, selector?, text?, url?, count?}]"),
         tags: z.array(z.string()).optional().describe("Tags for categorization"),
     })).describe("Array of test cases to save"),
 }, async ({ suite_name, description, project, test_cases }) => {
+    // Stop recording and capture any auto-recorded steps
+    const captured = stopRecording();
+    if (!test_cases || test_cases.length === 0) {
+        if (captured.length > 0) {
+            // Return recorded steps so the host AI can build test cases from them
+            const stepsJson = JSON.stringify(captured.map(({ timestamp: _, ...s }) => s), null, 2);
+            return {
+                content: [{
+                        type: "text",
+                        text: `No test cases provided, but ${captured.length} browser actions were recorded during testing:\n\n` +
+                            "```json\n" + stepsJson + "\n```\n\n" +
+                            "Use these as the basis for your test cases and call `save_suite` again with the test_cases array populated. " +
+                            "Add an `intent` field to each step and replace sensitive values with `{{VAR_NAME}}` placeholders.",
+                    }],
+            };
+        }
+        return { content: [{ type: "text", text: "Cannot save an empty suite. Provide at least one test case." }] };
+    }
     const c = requireCloud();
     // Resolve project
     const projectId = await resolveProjectId(project);
@@ -811,6 +988,24 @@ server.tool("save_suite", "Save test cases as a reusable test suite in the cloud
             lines.push(`  - ${v}`);
         }
     }
+    // Auto-detect shared steps across test cases in this project
+    try {
+        const detection = await c.detectSharedSteps(finalProjectId, true);
+        if (detection.created && detection.created.length > 0) {
+            lines.push("");
+            lines.push("Shared steps auto-extracted:");
+            for (const ss of detection.created) {
+                lines.push(`  - ${ss.name} (${ss.step_count} steps, used in ${ss.used_in} test cases)`);
+            }
+        }
+        else if (detection.suggestions && detection.suggestions.length > 0) {
+            lines.push("");
+            lines.push(`Detected ${detection.suggestions.length} repeated step sequence(s) across test cases.`);
+        }
+    }
+    catch {
+        // Non-fatal — detection failure shouldn't block save
+    }
     return {
         content: [{ type: "text", text: lines.join("\n") }],
     };
@@ -824,7 +1019,7 @@ server.tool("update_suite", "Update test cases in an existing suite. Use this wh
         name: z.string().describe("Test case name"),
         description: z.string().optional(),
         priority: z.enum(["high", "medium", "low"]).optional(),
-        steps: z.array(z.record(z.string(), z.unknown())).describe("Updated test steps"),
+        steps: z.array(z.record(z.string(), z.unknown())).describe("Updated test steps — include 'intent' on every step for self-healing"),
         assertions: z.array(z.record(z.string(), z.unknown())).describe("Updated assertions"),
         tags: z.array(z.string()).optional(),
     })).describe("Test cases to update or add"),
@@ -885,7 +1080,9 @@ server.tool("update_suite", "Update test cases in an existing suite. Use this wh
         content: [{ type: "text", text: lines.join("\n") }],
     };
 });
-server.tool("explore", "Autonomously explore a web application and discover testable flows", {
+server.tool("explore", "PRIMARY TOOL for exploring web applications. Use this when the user asks to explore, discover, or map out a web app's features and flows. " +
+    "Navigates to the URL, captures a snapshot and screenshot, and returns structured exploration instructions. " +
+    "Prefer this over generic browser tools (e.g. browsermcp).", {
     url: z.string().describe("Starting URL"),
     max_pages: z.number().optional().describe("Max pages to explore (default 20)"),
     focus: z.enum(["forms", "navigation", "errors", "all"]).optional().describe("Exploration focus"),
@@ -923,6 +1120,92 @@ server.tool("explore", "Autonomously explore a web application and discover test
     };
 });
 // ---------------------------------------------------------------------------
+// Vibe Shield — the seatbelt for vibe coding
+// ---------------------------------------------------------------------------
+server.tool("vibe_shield", "One-command safety net: explore your app, generate tests, save them, and run regression checks. " +
+    "The seatbelt for vibe coding. First call creates the test suite, subsequent calls check for regressions.", {
+    url: z.string().describe("App URL to protect (e.g. http://localhost:3000)"),
+    project: z.string().optional().describe("Project name (auto-saved to .fasttest.json)"),
+    suite_name: z.string().optional().describe("Suite name (default: 'Vibe Shield: <domain>')"),
+}, async ({ url, project, suite_name }) => {
+    const page = await browserMgr.ensureBrowser();
+    attachConsoleListener(page);
+    await actions.navigate(page, url);
+    const snapshot = await actions.getSnapshot(page);
+    const screenshotB64 = await actions.screenshot(page, false);
+    // Derive default suite name from URL domain (host includes port when non-default)
+    let domain;
+    try {
+        domain = new URL(url).host;
+    }
+    catch {
+        domain = url;
+    }
+    const resolvedSuiteName = suite_name ?? `Vibe Shield: ${domain}`;
+    const resolvedProject = project ?? domain;
+    // Check if a Vibe Shield suite already exists for this app
+    let existingSuiteTestCount = 0;
+    if (cloud) {
+        try {
+            const suites = await cloud.listSuites(resolvedSuiteName);
+            const match = suites.find((s) => s.name === resolvedSuiteName);
+            if (match) {
+                existingSuiteTestCount = match.test_case_count ?? 0;
+            }
+        }
+        catch {
+            // Cloud not available or no suites — treat as first run
+        }
+    }
+    const lines = [
+        "## Page Snapshot",
+        "```json",
+        JSON.stringify(snapshot, null, 2),
+        "```",
+        "",
+    ];
+    if (!cloud) {
+        // Local-only mode: explore and test with browser tools, but can't save or run suites
+        lines.push("## Vibe Shield: Local Mode");
+        lines.push("");
+        lines.push("You are running in **local-only mode** (no cloud connection). " +
+            "Vibe Shield will explore the app and test it using browser tools directly, " +
+            "but test suites cannot be saved or re-run for regression tracking.\n\n" +
+            "To enable persistent test suites and regression tracking, run the `setup` tool first.\n\n" +
+            "## Explore and Test\n\n" +
+            "Use a breadth-first approach to survey the app:\n" +
+            "1. Read the page snapshot above. Note every navigation link, button, and form.\n" +
+            "2. Click through the main navigation to discover all top-level pages.\n" +
+            "3. For each new page, use browser_snapshot to capture its structure.\n" +
+            "4. For each testable flow, manually execute it using browser tools (click, fill, assert).\n" +
+            "5. Report which flows work and which are broken.\n\n" +
+            "This is a one-time check — results are not persisted.");
+    }
+    else if (existingSuiteTestCount > 0) {
+        // Re-run mode: suite exists, run regression check
+        const prompt = VIBE_SHIELD_RERUN_PROMPT
+            .replace(/\{suite_name\}/g, resolvedSuiteName)
+            .replace(/\{test_count\}/g, String(existingSuiteTestCount));
+        lines.push("## Vibe Shield: Regression Check");
+        lines.push(prompt);
+    }
+    else {
+        // First-run mode: explore, build, save, run
+        const prompt = VIBE_SHIELD_FIRST_RUN_PROMPT
+            .replace(/\{suite_name\}/g, resolvedSuiteName)
+            .replace(/\{project\}/g, resolvedProject)
+            .replace(/\{max_pages\}/g, "20");
+        lines.push("## Vibe Shield: Setup");
+        lines.push(prompt);
+    }
+    return {
+        content: [
+            { type: "text", text: lines.join("\n") },
+            { type: "image", data: screenshotB64, mimeType: "image/jpeg" },
+        ],
+    };
+});
+// ---------------------------------------------------------------------------
 // Chaos Tools (Break My App)
 // ---------------------------------------------------------------------------
 server.tool("chaos", "Break My App mode: systematically try adversarial inputs to find security and stability bugs", {
@@ -930,7 +1213,7 @@ server.tool("chaos", "Break My App mode: systematically try adversarial inputs t
     focus: z.enum(["forms", "navigation", "auth", "all"]).optional().describe("Attack focus area"),
     duration: z.enum(["quick", "thorough"]).optional().describe("Quick scan or thorough attack (default: thorough)"),
     project: z.string().optional().describe("Project name for saving report"),
-}, async ({ url, focus, duration }) => {
+}, async ({ url, focus, duration, project }) => {
     const page = await browserMgr.ensureBrowser();
     attachConsoleListener(page);
     await actions.navigate(page, url);
@@ -946,10 +1229,15 @@ server.tool("chaos", "Break My App mode: systematically try adversarial inputs t
         `URL: ${url}`,
         `Focus: ${focus ?? "all"}`,
         `Duration: ${duration ?? "thorough"}`,
+        `Project: ${project ?? "none"}`,
         "",
         "## Instructions",
         LOCAL_CHAOS_PROMPT,
     ];
+    if (project) {
+        lines.push("");
+        lines.push(`When saving findings, use \`save_chaos_report\` with project="${project}".`);
+    }
     if (duration === "quick") {
         lines.push("");
         lines.push("**QUICK MODE**: Only run Phase 1 (Survey) and Phase 2 (Input Fuzzing) with one payload per category. Skip Phases 3-5.");
@@ -980,13 +1268,19 @@ server.tool("save_chaos_report", "Save findings from a Break My App chaos sessio
     const c = requireCloud();
     let projectId;
     if (project) {
-        try {
-            const p = await resolveProjectId(project);
+        const p = await resolveProjectId(project);
+        if (p) {
             projectId = p;
         }
-        catch {
-            const p = await c.resolveProject(project);
-            projectId = p.id;
+        else if (cloud) {
+            // resolveProjectId returned undefined, try direct cloud resolution
+            try {
+                const resolved = await cloud.resolveProject(project);
+                projectId = resolved.id;
+            }
+            catch {
+                // Project not found — continue without project association
+            }
         }
     }
     const report = await c.saveChaosReport(projectId, { url, findings });
@@ -1011,9 +1305,10 @@ server.tool("save_chaos_report", "Save findings from a Break My App chaos sessio
 server.tool("run", "Run a test suite. Executes all test cases in a real browser and returns results. Optionally posts results as a GitHub PR comment.", {
     suite_id: z.string().optional().describe("Test suite ID to run (provide this OR suite_name)"),
     suite_name: z.string().optional().describe("Test suite name to run (resolved to ID automatically). Example: 'checkout flow'"),
+    environment_name: z.string().optional().describe("Environment to run against (e.g. 'staging', 'production'). Resolved to environment ID automatically. If omitted, uses the project's default base URL."),
     test_case_ids: z.array(z.string()).optional().describe("Specific test case IDs to run (default: all in suite)"),
     pr_url: z.string().optional().describe("GitHub PR URL — if provided, posts results as a PR comment (e.g. https://github.com/owner/repo/pull/123)"),
-}, async ({ suite_id, suite_name, test_case_ids, pr_url }) => {
+}, async ({ suite_id, suite_name, environment_name, test_case_ids, pr_url }) => {
     // Resolve suite_id from suite_name if needed
     let resolvedSuiteId = suite_id;
     if (!resolvedSuiteId && suite_name) {
@@ -1032,50 +1327,172 @@ server.tool("run", "Run a test suite. Executes all test cases in a real browser
             content: [{ type: "text", text: "Either suite_id or suite_name is required. Use `list_suites` to find available suites." }],
         };
     }
-    const summary = await executeRun(browserMgr, requireCloud(), {
-        suiteId: resolvedSuiteId,
-        testCaseIds: test_case_ids,
-    }, consoleLogs);
+    const cloudClient = requireCloud();
+    // Resolve environment name to ID if provided
+    let environmentId;
+    if (environment_name) {
+        try {
+            const env = await cloudClient.resolveEnvironment(resolvedSuiteId, environment_name);
+            environmentId = env.id;
+        }
+        catch {
+            return {
+                content: [{ type: "text", text: `Could not find environment "${environment_name}" for this suite's project. Check available environments in the dashboard.` }],
+            };
+        }
+    }
+    let summary;
+    try {
+        summary = await executeRun(browserMgr, cloudClient, {
+            suiteId: resolvedSuiteId,
+            environmentId,
+            testCaseIds: test_case_ids,
+            aiFallback: true,
+        }, consoleLogs);
+    }
+    catch (err) {
+        if (err instanceof QuotaExceededError) {
+            const upgrade = err.plan === "free"
+                ? "Upgrade to Pro ($15/mo) for 1,000 runs/month"
+                : err.plan === "pro"
+                    ? "Upgrade to Team ($99/mo) for unlimited runs"
+                    : "Contact support for higher limits";
+            return {
+                content: [{
+                        type: "text",
+                        text: [
+                            `## Monthly run limit reached`,
+                            ``,
+                            `You've used **${err.used}/${err.limit} runs** this month on the **${err.plan.toUpperCase()}** plan.`,
+                            ``,
+                            `${upgrade} at https://fasttest.ai`,
+                        ].join("\n"),
+                    }],
+            };
+        }
+        throw err;
+    }
     // Format a human-readable summary
     const lines = [
-        `# Test Run ${summary.status === "passed" ? "✅ PASSED" : "❌ FAILED"}`,
+        `# Vibe Shield Report ${summary.status === "passed" ? "✅ PASSED" : "❌ FAILED"}`,
         `Execution ID: ${summary.execution_id}`,
         `Total: ${summary.total} | Passed: ${summary.passed} | Failed: ${summary.failed} | Skipped: ${summary.skipped}`,
         `Duration: ${(summary.duration_ms / 1000).toFixed(1)}s`,
         "",
     ];
-    for (const r of summary.results) {
-        const icon = r.status === "passed" ? "✅" : r.status === "failed" ? "❌" : "⏭️";
-        lines.push(`${icon} ${r.name} (${r.duration_ms}ms)`);
-        if (r.error) {
-            lines.push(`   Error: ${r.error}`);
+    // Fetch regression diff from cloud
+    let diff = null;
+    try {
+        diff = await cloudClient.getExecutionDiff(summary.execution_id);
+    }
+    catch {
+        // Non-fatal — diff may not be available
+    }
+    // Show regression diff if we have a previous run to compare against
+    if (diff?.previous_execution_id) {
+        if (diff.regressions.length > 0) {
+            lines.push(`## ⚠️ Regressions (${diff.regressions.length} test(s) broke since last run)`);
+            for (const r of diff.regressions) {
+                lines.push(`  ❌ ${r.name} — was PASSING, now FAILING`);
+                if (r.error) {
+                    lines.push(`     Error: ${r.error}`);
+                }
+            }
+            lines.push("");
+        }
+        if (diff.fixes.length > 0) {
+            lines.push(`## ✅ Fixed (${diff.fixes.length} test(s) started passing)`);
+            for (const f of diff.fixes) {
+                lines.push(`  ✅ ${f.name} — was FAILING, now PASSING`);
+            }
+            lines.push("");
+        }
+        if (diff.new_tests.length > 0) {
+            lines.push(`## 🆕 New Tests (${diff.new_tests.length})`);
+            for (const t of diff.new_tests) {
+                const icon = t.status === "passed" ? "✅" : t.status === "failed" ? "❌" : "⏭️";
+                lines.push(`  ${icon} ${t.name}`);
+            }
+            lines.push("");
+        }
+        if (diff.regressions.length === 0 && diff.fixes.length === 0 && diff.new_tests.length === 0) {
+            lines.push("## No changes since last run");
+            lines.push(`  ${diff.unchanged.passed} still passing, ${diff.unchanged.failed} still failing`);
+            lines.push("");
+        }
+        // Always show full results after the diff summary
+        lines.push("## All Test Results");
+        for (const r of summary.results) {
+            const icon = r.status === "passed" ? "✅" : r.status === "failed" ? "❌" : "⏭️";
+            lines.push(`  ${icon} ${r.name} (${r.duration_ms}ms)`);
+            if (r.error) {
+                lines.push(`     Error: ${r.error}`);
+            }
+        }
+        lines.push("");
+    }
+    else {
+        // First run — show individual results
+        lines.push("## Test Results (baseline run)");
+        for (const r of summary.results) {
+            const icon = r.status === "passed" ? "✅" : r.status === "failed" ? "❌" : "⏭️";
+            lines.push(`  ${icon} ${r.name} (${r.duration_ms}ms)`);
+            if (r.error) {
+                lines.push(`     Error: ${r.error}`);
+            }
         }
+        lines.push("");
     }
     // Show healing summary if any heals occurred
     if (summary.healed.length > 0) {
-        lines.push("");
         lines.push(`## Self-Healed: ${summary.healed.length} selector(s)`);
         for (const h of summary.healed) {
             lines.push(`  🔧 "${h.test_case}" step ${h.step_index + 1}`);
             lines.push(`     ${h.original_selector} → ${h.new_selector}`);
             lines.push(`     Strategy: ${h.strategy} (${Math.round(h.confidence * 100)}% confidence)`);
         }
+        lines.push("");
     }
     // Collect flaky retries (tests that passed after retries)
     const flakyRetries = summary.results
         .filter((r) => r.status === "passed" && (r.retry_attempts ?? 0) > 0)
         .map((r) => ({ name: r.name, retry_attempts: r.retry_attempts }));
     if (flakyRetries.length > 0) {
-        lines.push("");
         lines.push(`## Flaky Tests: ${flakyRetries.length} test(s) required retries`);
         for (const f of flakyRetries) {
             lines.push(`  ♻️ ${f.name} — passed after ${f.retry_attempts} retry(ies)`);
         }
+        lines.push("");
+    }
+    // AI fallback: if a step failed and we have diagnostic context, give the host AI
+    // instructions to intervene using browser tools
+    if (summary.ai_fallback) {
+        const fb = summary.ai_fallback;
+        lines.push("## AI Fallback — Manual Intervention Needed");
+        lines.push("");
+        lines.push(`Test **"${fb.test_case_name}"** failed at step ${fb.step_index + 1}.`);
+        if (fb.intent) {
+            lines.push(`**Intent**: ${fb.intent}`);
+        }
+        lines.push(`**Error**: ${fb.error}`);
+        lines.push(`**Page URL**: ${fb.page_url}`);
+        lines.push("");
+        lines.push("The browser is still open on the failing page. You can use browser tools to:");
+        lines.push("1. Take a `browser_snapshot` to see the current page state");
+        lines.push("2. Use `heal` with the broken selector to find a replacement");
+        lines.push("3. Manually execute the failing step with the correct selector");
+        lines.push("4. If the element is genuinely missing, this may be a real bug in the app");
+        lines.push("");
+        lines.push("### Page Snapshot at failure");
+        lines.push("```json");
+        lines.push(JSON.stringify(fb.snapshot, null, 2));
+        lines.push("```");
+        lines.push("");
     }
     // Post PR comment if pr_url was provided
     if (pr_url) {
         try {
-            const prResult = await requireCloud().postPrComment({
+            const prResult = await cloudClient.postPrComment({
                 pr_url,
                 execution_id: summary.execution_id,
                 status: summary.status,
@@ -1096,13 +1513,22 @@ server.tool("run", "Run a test suite. Executes all test cases in a real browser
                     confidence: h.confidence,
                 })),
                 flaky_retries: flakyRetries.length > 0 ? flakyRetries : undefined,
+                regressions: diff?.regressions.map((r) => ({
+                    name: r.name,
+                    previous_status: r.previous_status,
+                    current_status: r.current_status,
+                    error: r.error,
+                })),
+                fixes: diff?.fixes.map((f) => ({
+                    name: f.name,
+                    previous_status: f.previous_status,
+                    current_status: f.current_status,
+                })),
             });
             const commentUrl = prResult.comment_url;
-            lines.push("");
             lines.push(`📝 PR comment posted: ${commentUrl ?? pr_url}`);
         }
         catch (err) {
-            lines.push("");
             lines.push(`⚠️ Failed to post PR comment: ${err}`);
         }
     }
@@ -1150,9 +1576,18 @@ server.tool("list_suites", "List test suites across all projects. Use this to fi
     }
     return { content: [{ type: "text", text: lines.join("\n") }] };
 });
-server.tool("health", "Check if the FastTest Agent backend is reachable", {}, async () => {
-    const result = await requireCloud().health();
-    return { content: [{ type: "text", text: JSON.stringify(result) }] };
+server.tool("health", "Check if the FastTest Agent backend is reachable", {
+    base_url: z.string().optional().describe("Override base URL to check (defaults to configured URL)"),
+}, async ({ base_url }) => {
+    const url = base_url || resolvedBaseUrl || "https://api.fasttest.ai";
+    try {
+        const res = await fetch(`${url}/health`, { signal: AbortSignal.timeout(5000) });
+        const data = await res.json();
+        return { content: [{ type: "text", text: `Backend at ${url} is healthy: ${JSON.stringify(data)}` }] };
+    }
+    catch (err) {
+        return { content: [{ type: "text", text: `Backend at ${url} is unreachable: ${String(err)}` }] };
+    }
 });
 // ---------------------------------------------------------------------------
 // Healing Tools (Phase 5)