@fasttest-ai/qa-agent 0.1.3 → 0.3.0

package/dist/index.js

@@ -1,29 +1,287 @@
 #!/usr/bin/env node
 /**
- * QA Agent — MCP server (stdio transport).
+ * FastTest Agent — MCP server (stdio transport).
  *
  * This is the ONLY MCP server in the architecture.
  * Flow: Claude Code → MCP → Local Skill → HTTPS → Cloud API
  *
  * Exposes:
- *   - 10 browser tools (Playwright, runs locally)
- *   - Cloud-forwarding tools (test, answer, approve, explore, run, status, cancel, etc.)
+ *   - 21 browser tools (Playwright, runs locally)
+ *   - Local-first tools (test, explore, heal — host AI drives via structured prompts)
+ *   - Cloud tools (save_suite, update_suite, run, status, cancel, etc. — require setup)
  */
 import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
 import { z } from "zod";
+import { readFileSync, writeFileSync, existsSync } from "node:fs";
+import { join } from "node:path";
+import { execFile } from "node:child_process";
 import { BrowserManager } from "./browser.js";
 import { CloudClient } from "./cloud.js";
 import * as actions from "./actions.js";
 import { executeRun } from "./runner.js";
 import { healSelector } from "./healer.js";
+import { loadGlobalConfig, saveGlobalConfig } from "./config.js";
+// ---------------------------------------------------------------------------
+// Local-mode test prompt (used when no cloud is connected)
+// ---------------------------------------------------------------------------
+const LOCAL_TEST_PROMPT = `\
+You are executing QA tests by driving a real browser via tools. The page \
+snapshot above shows the current state of the page. Follow this methodology:
+
+## Execution loop (repeat for each test scenario)
+
+1. **Plan**: Read the page snapshot. Identify the elements you need to \
+   interact with. Pick the most stable selectors (data-testid > aria-label \
+   > role > text > CSS).
+2. **Act**: Execute steps using browser tools:
+   - browser_navigate — load a URL
+   - browser_click — click elements (use CSS selectors from the snapshot)
+   - browser_fill — type into inputs
+   - browser_press_key — keyboard actions (Enter, Tab, Escape)
+   - browser_fill_form — fill multiple form fields at once
+   - browser_drag — drag and drop elements
+   - browser_resize — resize viewport for responsive testing
+   - browser_tabs — manage browser tabs (list, create, switch, close)
+   - browser_wait — wait for elements or a timeout
+3. **Verify**: After each significant action, use browser_assert to check \
+   the expected outcome. Available assertion types: element_visible, \
+   element_hidden, text_contains, text_equals, url_contains, url_equals, \
+   element_count, attribute_value.
+4. **Snapshot**: After actions that change the page (form submit, navigation, \
+   modal open), use browser_snapshot to get the updated page state before \
+   continuing.
+5. **Evidence**: Use browser_screenshot after key assertions to capture proof.
+
+## Error recovery
+
+- If browser_click or browser_fill fails with "element not found", use the \
+  \`heal\` tool with the broken selector. It will try multiple strategies \
+  to find the element.
+- If heal also fails, take a browser_snapshot and analyze the page state — \
+  the element may be behind a loading spinner, inside an iframe, or require \
+  scrolling.
+- If an assertion fails, do NOT retry the same assertion. Report it as a \
+  failure — it may be a real bug.
+
+## What to test
+
+Based on the test request above, cover these scenarios in order:
+1. **Happy path**: The primary flow as described. This is the most important.
+2. **Input validation**: If the flow has form fields, test empty submission \
+   and one invalid input format.
+3. **Error states**: If the flow involves API calls or actions that can fail, \
+   test one failure scenario.
+
+Only test scenarios that are relevant to the request. Don't force edge cases \
+that don't apply.
+
+## Output format
+
+After testing, provide a clear summary:
+- List each scenario tested with PASS or FAIL
+- For failures, include what was expected vs. what happened
+- If any selectors were healed during testing, note the original and new \
+  selectors
+
+If cloud is connected (setup completed), ask if the user wants to save \
+passing tests as a reusable suite via \`save_suite\` for CI/CD replay.
+
+## Saving tests for CI/CD
+
+When saving test suites via \`save_suite\`, replace sensitive values with \
+\`{{VAR_NAME}}\` placeholders (UPPER_SNAKE_CASE):
+- Passwords: \`{{TEST_USER_PASSWORD}}\`
+- Emails/usernames: \`{{TEST_USER_EMAIL}}\`
+- API keys: \`{{STRIPE_TEST_KEY}}\`
+- Any value from .env files: use the matching env var name
+
+The test runner resolves these from environment variables at execution time. \
+In CI, they are set as GitHub repository secrets.
+
+Do NOT use placeholders for non-sensitive data like URLs, button labels, or \
+page content — only for credentials, tokens, and secrets.`;
+const LOCAL_EXPLORE_PROMPT = `\
+You are autonomously exploring a web application to discover testable flows. \
+The page snapshot and screenshot above show your starting point.
+
+## Exploration methodology
+
+Use a breadth-first approach: survey the app's structure before diving deep.
+
+### Phase 1: Survey (explore broadly)
+1. Read the current page snapshot. Note every navigation link, button, and form.
+2. Click through the main navigation to discover all top-level pages.
+3. For each new page, use browser_snapshot to capture its structure.
+4. Keep a mental map of pages visited and their URLs — do NOT revisit pages \
+   you've already seen.
+
+### Phase 2: Catalog (go deeper on high-value pages)
+For pages that have forms, CRUD operations, or multi-step flows:
+1. Identify the form fields and their types.
+2. Note any authentication requirements (login walls, role-based access).
+3. Look for state-changing actions (create, edit, delete).
+
+## Stopping criteria
+- Stop after visiting the number of pages specified in "Max pages" above.
+- Stop if you encounter a login wall and don't have credentials.
+- Stop if you've visited all reachable pages from the main navigation.
+- Do NOT explore: external links, social media, terms/privacy pages, \
+  documentation, or links that would leave the application domain.
+
+## Tools to use
+- browser_click — navigate to pages, open menus, expand sections
+- browser_navigate — go to a specific URL
+- browser_snapshot — capture the accessibility tree of the current page
+- browser_screenshot — capture visual evidence of interesting pages
+- browser_go_back — return to the previous page
+
+## Output format
+
+After exploring, present a structured summary:
+
+**Pages discovered:**
+| URL | Page type | Key elements |
+|-----|-----------|--------------|
+
+**Testable flows discovered:**
+1. Flow name — brief description (pages involved)
+2. Flow name — brief description (pages involved)
+
+**Forms found:**
+- Page URL: field names and types
+
+Then ask: "Which flows would you like me to test? I can run them now with \
+the \`test\` tool, or save them as a reusable suite with \`save_suite\`."`;
+const LOCAL_HEAL_PROMPT = `\
+A test step failed because a CSS selector no longer matches any element. \
+Four automated repair strategies (data-testid matching, ARIA label matching, \
+text content matching, structural matching) have already been tried and failed.
+
+You are the last resort. Use your reasoning to diagnose and fix this.
+
+## Broken selector details
+- Selector: {selector}
+- Error: {error_message}
+- Page URL: {page_url}
+
+## Diagnosis steps
+
+1. **Understand the intent**: What element was the selector trying to target? \
+   Parse the selector to determine: is it a button, input, link, container? \
+   What was its purpose in the test?
+
+2. **Search the snapshot**: The page snapshot is below. Look for elements \
+   that match the INTENT of the original selector, not its syntax. Search by:
+   - Role (button, textbox, link, heading)
+   - Label or visible text
+   - Position in the page structure (e.g., "the submit button in the login form")
+
+3. **Determine root cause**: Why did the selector break?
+   - **Renamed**: Element exists but with a different ID/class/attribute
+   - **Moved**: Element exists but in a different part of the DOM
+   - **Replaced**: Old element removed, new one added with different markup
+   - **Hidden**: Element exists but is not visible (behind a modal, in a \
+     collapsed section, requires scrolling)
+   - **Removed**: Element genuinely doesn't exist — this is a REAL BUG
+
+4. **Construct a new selector**: Build the most stable selector possible.
+   Priority: [data-testid] > [aria-label] > role-based > text-based > \
+   structural.
+
+5. **Verify**: Use browser_assert with type "element_visible" and your new \
+   selector. If it passes, report the fix. If it fails, try your next best \
+   candidate.
+
+## Important
+
+- If the element genuinely doesn't exist on the page (not renamed, not \
+  moved, not hidden), report it as a REAL BUG. Say: "This appears to be a \
+  real bug — the [element description] is missing from the page."
+- Do NOT suggest fragile selectors (nth-child, auto-generated CSS classes).
+- Do NOT suggest more than 3 candidates — if none of them work after \
+  verification, the element is likely gone.`;
+const LOCAL_CHAOS_PROMPT = `\
+You are running a "Break My App" adversarial testing session. Your goal is to \
+systematically attack this page to find security issues, crashes, and missing validation. \
+Use the browser tools (browser_fill, browser_click, browser_evaluate, browser_console_logs, \
+browser_screenshot) to execute each attack.
+
+WARNING: Run against staging/dev environments only. Adversarial payloads may trigger WAF rules.
+
+## Phase 1: Survey
+
+Read the page snapshot below carefully. Catalog every form, input field, button, \
+link, and interactive element. Identify the most interesting targets — forms with \
+auth, payment, CRUD operations, file uploads. Note the current URL and page title.
+
+## Phase 2: Input Fuzzing
+
+For each input field you found, try these payloads one at a time, submitting the \
+form after each:
+
+**XSS payloads:**
+- \`<script>alert(1)</script>\`
+- \`<img onerror=alert(1) src=x>\`
+- \`javascript:alert(1)\`
+
+**SQL injection:**
+- \`' OR 1=1 --\`
+- \`'; DROP TABLE users; --\`
+
+**Boundary testing:**
+- Empty submission (clear all fields, submit)
+- Long string (paste 5000+ chars of "AAAA...")
+- Unicode: RTL mark \\u200F, zero-width space \\u200B, emoji-only "🔥💀🎉"
+- Negative numbers: \`-1\`, \`-999999\`
+
+After each submission: call \`browser_console_logs\` and check for any \`[error]\` \
+entries. Take a screenshot if you find something interesting.
+
+## Phase 3: Interaction Fuzzing
+
+- Double-click submit buttons rapidly (click twice with no delay)
+- Rapid-fire click the same action button 5 times quickly
+- Use \`browser_evaluate\` to click disabled buttons: \
+  \`document.querySelector('button[disabled]')?.removeAttribute('disabled'); \
+  document.querySelector('button[disabled]')?.click();\`
+- Press browser back during form submission (navigate, then immediately go back)
+
+## Phase 4: Auth & Access
+
+- Use \`browser_evaluate\` to read localStorage and cookies: \
+  \`JSON.stringify({localStorage: {...localStorage}, cookies: document.cookie})\`
+- If tokens are found, clear them: \
+  \`localStorage.clear(); document.cookie.split(';').forEach(c => \
+  document.cookie = c.trim().split('=')[0] + '=;expires=Thu, 01 Jan 1970');\`
+- After clearing, try accessing the same page — does it still show protected content?
+
+## Phase 5: Console Monitoring
+
+After every action, check \`browser_console_logs\` for:
+- Unhandled exceptions or promise rejections
+- 404 or 500 network errors
+- Exposed stack traces or sensitive data in error messages
+
+## Output Format
+
+After testing, summarize your findings as a structured list. For each finding:
+- **severity**: critical (XSS executes, app crashes, data leak), high (unhandled \
+  exception, auth bypass), medium (missing validation, accepts garbage), low (cosmetic issue)
+- **category**: xss, injection, crash, validation, error, auth
+- **description**: What you found
+- **reproduction_steps**: Numbered steps to reproduce
+- **console_errors**: Any relevant console errors
+
+If you want to save these findings, call the \`save_chaos_report\` tool with \
+the URL and findings array.`;
 // ---------------------------------------------------------------------------
 // CLI arg parsing
 // ---------------------------------------------------------------------------
 function parseArgs() {
     const args = process.argv.slice(2);
-    let apiKey = "";
-    let baseUrl = "http://localhost:9000";
+    let apiKey;
+    let baseUrl = "";
     let headless = true;
     let browserType = "chromium";
     for (let i = 0; i < args.length; i++) {
@@ -40,10 +298,6 @@ function parseArgs() {
             browserType = args[++i];
         }
     }
-    if (!apiKey) {
-        console.error("Usage: qa-agent --api-key <key> [--base-url <url>] [--headed] [--browser chromium|firefox|webkit]");
-        process.exit(1);
-    }
     return { apiKey, baseUrl, headless, browser: browserType };
 }
 // ---------------------------------------------------------------------------
@@ -52,18 +306,76 @@ function parseArgs() {
 const consoleLogs = [];
 const MAX_LOGS = 500;
 // ---------------------------------------------------------------------------
-// Boot
+// Boot — resolve auth from CLI > config file > null (local-only mode)
 // ---------------------------------------------------------------------------
-const config = parseArgs();
-const orgSlug = config.apiKey.split("_")[1] ?? "default";
+const cliArgs = parseArgs();
+const globalCfg = loadGlobalConfig();
+// Resolution: CLI --api-key wins, then config file, then undefined
+const resolvedApiKey = cliArgs.apiKey || globalCfg.api_key || undefined;
+const resolvedBaseUrl = cliArgs.baseUrl || globalCfg.base_url || "https://api.fasttest.ai";
+const orgSlug = resolvedApiKey ? (resolvedApiKey.split("_")[1] ?? "default") : "default";
 const browserMgr = new BrowserManager({
-    browserType: config.browser,
-    headless: config.headless,
+    browserType: cliArgs.browser,
+    headless: cliArgs.headless,
     orgSlug,
 });
-const cloud = new CloudClient({ apiKey: config.apiKey, baseUrl: config.baseUrl });
+// cloud is null until auth is available (lazy initialization)
+let cloud = resolvedApiKey
+    ? new CloudClient({ apiKey: resolvedApiKey, baseUrl: resolvedBaseUrl })
+    : null;
+// ---------------------------------------------------------------------------
+// Cloud guard — returns CloudClient or throws a user-friendly error
+// ---------------------------------------------------------------------------
+function requireCloud() {
+    if (!cloud) {
+        throw new Error("Not connected to FastTest cloud. Run the `setup` tool first to create an organization.");
+    }
+    return cloud;
+}
+const FASTTEST_CONFIG = ".fasttest.json";
+function configPath() {
+    return join(process.cwd(), FASTTEST_CONFIG);
+}
+function loadConfig() {
+    const p = configPath();
+    if (!existsSync(p))
+        return null;
+    try {
+        return JSON.parse(readFileSync(p, "utf-8"));
+    }
+    catch {
+        return null;
+    }
+}
+function saveConfig(cfg) {
+    writeFileSync(configPath(), JSON.stringify(cfg, null, 2) + "\n");
+}
+/**
+ * Resolve a project_id. Priority:
+ *   1. .fasttest.json in cwd (cached from previous run)
+ *   2. Explicit project name from the LLM → resolve via cloud API
+ *   3. null (no project scoping)
+ */
+async function resolveProjectId(projectName) {
+    // 1. Check .fasttest.json
+    const cached = loadConfig();
+    if (cached?.project_id)
+        return cached.project_id;
+    // 2. If LLM provided a project name, resolve it via cloud
+    if (projectName && cloud) {
+        try {
+            const resolved = await cloud.resolveProject(projectName);
+            saveConfig({ project_id: resolved.id, project_name: resolved.name });
+            return resolved.id;
+        }
+        catch (err) {
+            console.error(`Failed to resolve project "${projectName}": ${err}`);
+        }
+    }
+    return undefined;
+}
 const server = new McpServer({
-    name: "qa-agent",
+    name: "fasttest",
     version: "0.1.0",
 });
 // ---------------------------------------------------------------------------
@@ -185,64 +497,543 @@ server.tool("browser_evaluate", "Execute JavaScript in the page context and retu
     const result = await actions.evaluate(page, expression);
     return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
 });
+server.tool("browser_drag", "Drag an element and drop it onto another element", {
+    source: z.string().describe("CSS selector of the element to drag"),
+    target: z.string().describe("CSS selector of the drop target"),
+}, async ({ source, target }) => {
+    const page = await browserMgr.getPage();
+    const result = await actions.drag(page, source, target);
+    return { content: [{ type: "text", text: JSON.stringify(result) }] };
+});
+server.tool("browser_resize", "Resize the browser viewport (useful for responsive/mobile testing)", {
+    width: z.number().describe("Viewport width in pixels"),
+    height: z.number().describe("Viewport height in pixels"),
+}, async ({ width, height }) => {
+    const page = await browserMgr.getPage();
+    const result = await actions.resize(page, width, height);
+    return { content: [{ type: "text", text: JSON.stringify(result) }] };
+});
+server.tool("browser_tabs", "Manage browser tabs: list, create, switch, or close tabs", {
+    action: z.enum(["list", "create", "switch", "close"]).describe("Tab action to perform"),
+    url: z.string().optional().describe("URL to open in new tab (only for 'create' action)"),
+    index: z.number().optional().describe("Tab index (for 'switch' and 'close' actions)"),
+}, async ({ action, url, index }) => {
+    try {
+        switch (action) {
+            case "list": {
+                const pages = await browserMgr.listPagesAsync();
+                return { content: [{ type: "text", text: JSON.stringify({ success: true, tabs: pages }) }] };
+            }
+            case "create": {
+                const page = await browserMgr.createPage(url);
+                return {
+                    content: [{
+                            type: "text",
+                            text: JSON.stringify({ success: true, url: page.url(), title: await page.title() }),
+                        }],
+                };
+            }
+            case "switch": {
+                if (index === undefined) {
+                    return { content: [{ type: "text", text: JSON.stringify({ success: false, error: "index is required for switch" }) }] };
+                }
+                const page = await browserMgr.switchToPage(index);
+                return {
+                    content: [{
+                            type: "text",
+                            text: JSON.stringify({ success: true, url: page.url(), title: await page.title() }),
+                        }],
+                };
+            }
+            case "close": {
+                if (index === undefined) {
+                    return { content: [{ type: "text", text: JSON.stringify({ success: false, error: "index is required for close" }) }] };
+                }
+                await browserMgr.closePage(index);
+                return { content: [{ type: "text", text: JSON.stringify({ success: true }) }] };
+            }
+        }
+    }
+    catch (err) {
+        return { content: [{ type: "text", text: JSON.stringify({ success: false, error: String(err) }) }] };
+    }
+});
+server.tool("browser_fill_form", "Fill multiple form fields at once (batch operation, fewer round-trips than individual browser_fill calls)", {
+    fields: z.record(z.string(), z.string()).describe("Map of CSS selector → value to fill (e.g. {\"#email\": \"test@example.com\", \"#password\": \"secret\"})"),
+}, async ({ fields }) => {
+    const page = await browserMgr.getPage();
+    const result = await actions.fillForm(page, fields);
+    return { content: [{ type: "text", text: JSON.stringify(result) }] };
+});
+server.tool("browser_network_requests", "List captured network requests from the current session. Shows API calls, failed requests, and document loads (static assets are filtered out).", {
+    filter_status: z.number().optional().describe("Only show requests with this HTTP status code or higher (e.g. 400 for errors only)"),
+}, async ({ filter_status }) => {
+    const entries = browserMgr.getNetworkSummary();
+    // Filter static assets — only show API/document/error requests
+    const filtered = entries.filter((e) => {
+        const mime = e.mimeType.toLowerCase();
+        const isRelevant = mime.includes("json") || mime.includes("text/html") ||
+            mime.includes("text/plain") || e.status >= 400;
+        if (!isRelevant)
+            return false;
+        if (filter_status !== undefined && e.status < filter_status)
+            return false;
+        return true;
+    });
+    return {
+        content: [{
+                type: "text",
+                text: JSON.stringify({ total: filtered.length, requests: filtered.slice(-100) }, null, 2),
+            }],
+    };
+});
+// ---------------------------------------------------------------------------
+// Setup Tool — device auth flow (opens browser for secure authentication)
+// ---------------------------------------------------------------------------
+function openBrowser(url) {
+    try {
+        const platform = process.platform;
+        if (platform === "darwin") {
+            execFile("open", [url], { stdio: "ignore" });
+        }
+        else if (platform === "win32") {
+            execFile("cmd", ["/c", "start", "", url], { stdio: "ignore" });
+        }
+        else {
+            execFile("xdg-open", [url], { stdio: "ignore" });
+        }
+    }
+    catch {
+        // Silently fail — the URL is shown to the user as fallback
+    }
+}
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+server.tool("setup", "Set up FastTest Agent: authenticate via browser to connect your editor to your FastTest account. Opens a browser window for secure login.", {
+    base_url: z.string().optional().describe("Cloud API base URL (default: https://api.fasttest.ai)"),
+}, async ({ base_url }) => {
+    if (cloud) {
+        return {
+            content: [{
+                    type: "text",
+                    text: "Already connected to FastTest cloud. To switch organizations, edit ~/.fasttest/config.json or pass --api-key on the CLI.",
+                }],
+        };
+    }
+    const targetBaseUrl = base_url ?? resolvedBaseUrl;
+    try {
+        // 1. Request a device code from the backend
+        const deviceCode = await CloudClient.requestDeviceCode(targetBaseUrl);
+        // 2. Open the browser to the verification URL
+        openBrowser(deviceCode.verification_url);
+        const lines = [
+            "Opening your browser to authenticate...",
+            "",
+            "If it doesn't open automatically, visit:",
+            `  ${deviceCode.verification_url}`,
+            "",
+            `Device code: **${deviceCode.code}**`,
+            "",
+            "Waiting for confirmation (expires in 5 minutes)...",
+        ];
+        // 3. Poll for completion
+        const pollIntervalMs = 2000;
+        const maxAttempts = Math.ceil((deviceCode.expires_in * 1000) / pollIntervalMs);
+        for (let i = 0; i < maxAttempts; i++) {
+            await sleep(pollIntervalMs);
+            const status = await CloudClient.pollDeviceCode(targetBaseUrl, deviceCode.poll_token);
+            if (status.status === "completed" && status.api_key) {
+                // Save the API key
+                saveGlobalConfig({
+                    api_key: status.api_key,
+                    base_url: targetBaseUrl,
+                });
+                cloud = new CloudClient({ apiKey: status.api_key, baseUrl: targetBaseUrl });
+                return {
+                    content: [{
+                            type: "text",
+                            text: [
+                                ...lines,
+                                "",
+                                `Authenticated as **${status.org_name}** (${status.org_slug}).`,
+                                "",
+                                `  Config saved to: ~/.fasttest/config.json`,
+                                "",
+                                "Cloud features are now active. You can use `test`, `run`, `explore`, and all other tools.",
+                            ].join("\n"),
+                        }],
+                };
+            }
+            if (status.status === "expired") {
+                return {
+                    content: [{
+                            type: "text",
+                            text: [
+                                ...lines,
+                                "",
+                                "Device code expired. Run `setup` again to get a new code.",
+                            ].join("\n"),
+                        }],
+                };
+            }
+            // Still pending — continue polling
+        }
+        // Timed out
+        return {
+            content: [{
+                    type: "text",
+                    text: [
+                        ...lines,
+                        "",
+                        "Timed out waiting for browser confirmation. Run `setup` again to retry.",
+                    ].join("\n"),
+                }],
+        };
+    }
+    catch (err) {
+        return {
+            content: [{
+                    type: "text",
+                    text: `Setup failed: ${String(err)}`,
+                }],
+        };
+    }
+});
 // ---------------------------------------------------------------------------
 // Cloud-forwarding Tools
 // ---------------------------------------------------------------------------
 server.tool("test", "Start a conversational test session. Describe what you want to test.", {
     description: z.string().describe("What to test (natural language)"),
     url: z.string().optional().describe("App URL to test against"),
-}, async ({ description, url }) => {
-    const result = await cloud.startConversation({ description, url });
-    return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
+    project: z.string().optional().describe("Project name (e.g. 'My SaaS App'). Auto-saved to .fasttest.json for future runs."),
+}, async ({ description, url, project }) => {
+    // Always use local mode: host AI drives browser tools directly.
+    // Cloud LLM is never used from the MCP server — the host AI (Claude Code,
+    // Codex, etc.) follows our prompt with its own reasoning capability.
+    const lines = [];
+    if (url) {
+        const page = await browserMgr.ensureBrowser();
+        attachConsoleListener(page);
+        await actions.navigate(page, url);
+        const snapshot = await actions.getSnapshot(page);
+        lines.push("## Page Snapshot");
+        lines.push("```json");
+        lines.push(JSON.stringify(snapshot, null, 2));
+        lines.push("```");
+        lines.push("");
+    }
+    lines.push("## Test Request");
+    lines.push(description);
+    lines.push("");
+    lines.push("## Instructions");
+    lines.push(LOCAL_TEST_PROMPT);
+    if (!cloud) {
+        lines.push("");
+        lines.push("---");
+        lines.push("*Running in local-only mode. Run the `setup` tool to enable persistent test suites, CI/CD, and the dashboard.*");
+    }
+    return { content: [{ type: "text", text: lines.join("\n") }] };
 });
-server.tool("answer", "Respond to the agent's clarifying questions", {
-    session_id: z.string().describe("Session ID from the test call"),
-    answers: z.string().describe("Your responses to the agent's questions"),
-}, async ({ session_id, answers }) => {
-    const result = await cloud.answerConversation(session_id, answers);
-    return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
+server.tool("save_suite", "Save test cases as a reusable test suite in the cloud. Use this after running tests to persist them for CI/CD replay. " +
+    "IMPORTANT: For sensitive values (passwords, API keys, tokens), use {{VAR_NAME}} placeholders instead of literal values. " +
+    "Example: use {{TEST_USER_PASSWORD}} instead of the actual password. " +
+    "The runner resolves these from environment variables at execution time. Variable names must be UPPER_SNAKE_CASE.", {
+    suite_name: z.string().describe("Name for the test suite (e.g. 'Login Flow', 'Checkout Tests')"),
+    description: z.string().optional().describe("What this suite tests"),
+    project: z.string().optional().describe("Project name (auto-resolved or created)"),
+    test_cases: z.array(z.object({
+        name: z.string().describe("Test case name"),
+        description: z.string().optional().describe("What this test verifies"),
+        priority: z.enum(["high", "medium", "low"]).optional().describe("Test priority"),
+        steps: z.array(z.record(z.string(), z.unknown())).describe("Test steps: [{action, selector?, value?, url?, description?}]. " +
+            "Use {{VAR_NAME}} placeholders for sensitive values (e.g. value: '{{TEST_PASSWORD}}')"),
+        assertions: z.array(z.record(z.string(), z.unknown())).describe("Assertions: [{type, selector?, text?, url?, count?}]"),
+        tags: z.array(z.string()).optional().describe("Tags for categorization"),
+    })).describe("Array of test cases to save"),
+}, async ({ suite_name, description, project, test_cases }) => {
+    const c = requireCloud();
+    // Resolve project
+    const projectId = await resolveProjectId(project);
+    let finalProjectId = projectId;
+    if (!finalProjectId) {
+        const resolved = await c.resolveProject(project ?? "Default");
+        finalProjectId = resolved.id;
+        saveConfig({ project_id: resolved.id, project_name: resolved.name });
+    }
+    // Create suite
+    const suite = await c.createSuite(finalProjectId, {
+        name: suite_name,
+        description,
+        auto_generated: true,
+        test_type: "functional",
+    });
+    // Create test cases linked to the suite
+    const savedCases = [];
+    for (const tc of test_cases) {
+        const created = await c.createTestCase({
+            name: tc.name,
+            description: tc.description,
+            priority: tc.priority ?? "medium",
+            steps: tc.steps,
+            assertions: tc.assertions,
+            tags: tc.tags ?? [],
+            test_suite_ids: [suite.id],
+            auto_generated: true,
+            generated_by_agent: true,
+            natural_language_source: suite_name,
+        });
+        savedCases.push(`  - ${created.name} (${created.id})`);
+    }
+    // Scan for {{VAR}} placeholders to show CI/CD guidance
+    const allVars = new Set();
+    for (const tc of test_cases) {
+        const raw = JSON.stringify(tc.steps) + JSON.stringify(tc.assertions);
+        const matches = raw.matchAll(/\{\{([A-Z][A-Z0-9_]*)\}\}/g);
+        for (const m of matches)
+            allVars.add(m[1]);
+    }
+    const lines = [
+        `Suite "${suite.name}" saved successfully.`,
+        `  Suite ID: ${suite.id}`,
+        `  Project: ${finalProjectId}`,
+        `  Test cases (${savedCases.length}):`,
+        ...savedCases,
+        "",
+        `To replay: \`run(suite_id: "${suite.id}")\``,
+        `To replay by name: \`run(suite_name: "${suite_name}")\``,
+    ];
+    if (allVars.size > 0) {
+        lines.push("");
+        lines.push("Environment variables required for CI/CD:");
+        lines.push("Set these as GitHub repository secrets before running in CI:");
+        for (const v of Array.from(allVars).sort()) {
+            lines.push(`  - ${v}`);
+        }
+    }
+    return {
+        content: [{ type: "text", text: lines.join("\n") }],
+    };
 });
-server.tool("approve", "Approve the generated test plan and start execution", {
-    session_id: z.string().describe("Session ID"),
-    exclude: z.array(z.number()).optional().describe("Test case numbers to skip"),
-}, async ({ session_id, exclude }) => {
-    const result = await cloud.approveConversation(session_id, exclude);
-    return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
+server.tool("update_suite", "Update test cases in an existing suite. Use this when the app has changed and tests need updating. " +
+    "Use {{VAR_NAME}} placeholders for sensitive values (passwords, API keys, tokens) — same as save_suite.", {
+    suite_id: z.string().optional().describe("Suite ID to update (provide this OR suite_name)"),
+    suite_name: z.string().optional().describe("Suite name to update (resolved automatically)"),
+    test_cases: z.array(z.object({
+        id: z.string().optional().describe("Existing test case ID to update (omit to add a new case)"),
+        name: z.string().describe("Test case name"),
+        description: z.string().optional(),
+        priority: z.enum(["high", "medium", "low"]).optional(),
+        steps: z.array(z.record(z.string(), z.unknown())).describe("Updated test steps"),
+        assertions: z.array(z.record(z.string(), z.unknown())).describe("Updated assertions"),
+        tags: z.array(z.string()).optional(),
+    })).describe("Test cases to update or add"),
+}, async ({ suite_id, suite_name, test_cases }) => {
+    const c = requireCloud();
+    // Resolve suite ID
+    let resolvedSuiteId = suite_id;
+    if (!resolvedSuiteId && suite_name) {
+        const resolved = await c.resolveSuite(suite_name);
+        resolvedSuiteId = resolved.id;
+    }
+    if (!resolvedSuiteId) {
+        return {
+            content: [{ type: "text", text: "Either suite_id or suite_name is required." }],
+        };
+    }
+    const updated = [];
+    const created = [];
+    for (const tc of test_cases) {
+        if (tc.id) {
+            // Update existing test case
+            const result = await c.updateTestCase(tc.id, {
+                name: tc.name,
+                description: tc.description,
+                priority: tc.priority,
+                steps: tc.steps,
+                assertions: tc.assertions,
+                tags: tc.tags,
+            });
+            updated.push(`  - ${result.name} (${result.id})`);
+        }
+        else {
+            // Create new test case and link to suite
+            const result = await c.createTestCase({
+                name: tc.name,
+                description: tc.description,
+                priority: tc.priority ?? "medium",
+                steps: tc.steps,
+                assertions: tc.assertions,
+                tags: tc.tags ?? [],
+                test_suite_ids: [resolvedSuiteId],
+                auto_generated: true,
+                generated_by_agent: true,
+            });
+            created.push(`  - ${result.name} (${result.id})`);
+        }
+    }
+    const lines = [`Suite "${resolvedSuiteId}" updated.`];
+    if (updated.length > 0) {
+        lines.push(`Updated (${updated.length}):`);
+        lines.push(...updated);
+    }
+    if (created.length > 0) {
+        lines.push(`Added (${created.length}):`);
+        lines.push(...created);
+    }
+    return {
+        content: [{ type: "text", text: lines.join("\n") }],
+    };
 });
 server.tool("explore", "Autonomously explore a web application and discover testable flows", {
     url: z.string().describe("Starting URL"),
     max_pages: z.number().optional().describe("Max pages to explore (default 20)"),
     focus: z.enum(["forms", "navigation", "errors", "all"]).optional().describe("Exploration focus"),
 }, async ({ url, max_pages, focus }) => {
-    // Step 1: Navigate locally to get initial snapshot
+    // Always local-first: navigate, snapshot, return prompt for host AI
     const page = await browserMgr.ensureBrowser();
     attachConsoleListener(page);
     await actions.navigate(page, url);
     const snapshot = await actions.getSnapshot(page);
     const screenshotB64 = await actions.screenshot(page, false);
-    // Step 2: Send to cloud for AI analysis
-    const result = await cloud.startExploration({
-        url,
-        max_pages: max_pages ?? 20,
-        focus: focus ?? "all",
-    });
+    const lines = [
+        "## Page Snapshot",
+        "```json",
+        JSON.stringify(snapshot, null, 2),
+        "```",
+        "",
+        "## Exploration Request",
+        `URL: ${url}`,
+        `Focus: ${focus ?? "all"}`,
+        `Max pages: ${max_pages ?? 20}`,
+        "",
+        "## Instructions",
+        LOCAL_EXPLORE_PROMPT,
+    ];
+    if (!cloud) {
+        lines.push("");
+        lines.push("---");
+        lines.push("*Running in local-only mode. Run the `setup` tool to enable persistent test suites and CI/CD.*");
+    }
+    return {
+        content: [
+            { type: "text", text: lines.join("\n") },
+            { type: "image", data: screenshotB64, mimeType: "image/jpeg" },
+        ],
+    };
+});
+// ---------------------------------------------------------------------------
+// Chaos Tools (Break My App)
+// ---------------------------------------------------------------------------
+server.tool("chaos", "Break My App mode: systematically try adversarial inputs to find security and stability bugs", {
+    url: z.string().describe("URL to attack"),
+    focus: z.enum(["forms", "navigation", "auth", "all"]).optional().describe("Attack focus area"),
+    duration: z.enum(["quick", "thorough"]).optional().describe("Quick scan or thorough attack (default: thorough)"),
+    project: z.string().optional().describe("Project name for saving report"),
+}, async ({ url, focus, duration }) => {
+    const page = await browserMgr.ensureBrowser();
+    attachConsoleListener(page);
+    await actions.navigate(page, url);
+    const snapshot = await actions.getSnapshot(page);
+    const screenshotB64 = await actions.screenshot(page, false);
+    const lines = [
+        "## Page Snapshot",
+        "```json",
+        JSON.stringify(snapshot, null, 2),
+        "```",
+        "",
+        "## Chaos Configuration",
+        `URL: ${url}`,
+        `Focus: ${focus ?? "all"}`,
+        `Duration: ${duration ?? "thorough"}`,
+        "",
+        "## Instructions",
+        LOCAL_CHAOS_PROMPT,
+    ];
+    if (duration === "quick") {
+        lines.push("");
+        lines.push("**QUICK MODE**: Only run Phase 1 (Survey) and Phase 2 (Input Fuzzing) with one payload per category. Skip Phases 3-5.");
+    }
+    if (!cloud) {
+        lines.push("");
+        lines.push("---");
+        lines.push("*Running in local-only mode. Run the `setup` tool to enable saving chaos reports.*");
+    }
     return {
         content: [
-            { type: "text", text: JSON.stringify({ ...result, initial_snapshot: snapshot }, null, 2) },
+            { type: "text", text: lines.join("\n") },
             { type: "image", data: screenshotB64, mimeType: "image/jpeg" },
         ],
     };
 });
+server.tool("save_chaos_report", "Save findings from a Break My App chaos session to the cloud", {
+    url: z.string().describe("URL that was tested"),
+    project: z.string().optional().describe("Project name (auto-resolved or created)"),
+    findings: z.array(z.object({
+        severity: z.enum(["critical", "high", "medium", "low"]),
+        category: z.string().describe("e.g. xss, injection, crash, validation, error, auth"),
+        description: z.string(),
+        reproduction_steps: z.array(z.string()),
+        console_errors: z.array(z.string()).optional(),
+    })).describe("List of findings from the chaos session"),
+}, async ({ url, project, findings }) => {
+    const c = requireCloud();
+    let projectId;
+    if (project) {
+        try {
+            const p = await resolveProjectId(project);
+            projectId = p;
+        }
+        catch {
+            const p = await c.resolveProject(project);
+            projectId = p.id;
+        }
+    }
+    const report = await c.saveChaosReport(projectId, { url, findings });
+    const sevCounts = { critical: 0, high: 0, medium: 0, low: 0 };
+    for (const f of findings) {
+        sevCounts[f.severity]++;
+    }
+    const lines = [
+        `Chaos report saved (${findings.length} findings)`,
+        "",
+        `Critical: ${sevCounts.critical} | High: ${sevCounts.high} | Medium: ${sevCounts.medium} | Low: ${sevCounts.low}`,
+        "",
+        `Report ID: ${report.id ?? "saved"}`,
+    ];
+    return {
+        content: [{ type: "text", text: lines.join("\n") }],
+    };
+});
 // ---------------------------------------------------------------------------
 // Execution Tools (Phase 3)
 // ---------------------------------------------------------------------------
 server.tool("run", "Run a test suite. Executes all test cases in a real browser and returns results. Optionally posts results as a GitHub PR comment.", {
-    suite_id: z.string().describe("Test suite ID to run"),
+    suite_id: z.string().optional().describe("Test suite ID to run (provide this OR suite_name)"),
+    suite_name: z.string().optional().describe("Test suite name to run (resolved to ID automatically). Example: 'checkout flow'"),
     test_case_ids: z.array(z.string()).optional().describe("Specific test case IDs to run (default: all in suite)"),
     pr_url: z.string().optional().describe("GitHub PR URL — if provided, posts results as a PR comment (e.g. https://github.com/owner/repo/pull/123)"),
-}, async ({ suite_id, test_case_ids, pr_url }) => {
-    const summary = await executeRun(browserMgr, cloud, {
-        suiteId: suite_id,
+}, async ({ suite_id, suite_name, test_case_ids, pr_url }) => {
+    // Resolve suite_id from suite_name if needed
+    let resolvedSuiteId = suite_id;
+    if (!resolvedSuiteId && suite_name) {
+        try {
+            const resolved = await requireCloud().resolveSuite(suite_name);
+            resolvedSuiteId = resolved.id;
+        }
+        catch {
+            return {
+                content: [{ type: "text", text: `Could not find a suite matching "${suite_name}". Use \`list_suites\` to see available suites.` }],
+            };
+        }
+    }
+    if (!resolvedSuiteId) {
+        return {
+            content: [{ type: "text", text: "Either suite_id or suite_name is required. Use `list_suites` to find available suites." }],
+        };
+    }
+    const summary = await executeRun(browserMgr, requireCloud(), {
+        suiteId: resolvedSuiteId,
         testCaseIds: test_case_ids,
     }, consoleLogs);
     // Format a human-readable summary
@@ -270,10 +1061,21 @@ server.tool("run", "Run a test suite. Executes all test cases in a real browser
             lines.push(`     Strategy: ${h.strategy} (${Math.round(h.confidence * 100)}% confidence)`);
         }
     }
+    // Collect flaky retries (tests that passed after retries)
+    const flakyRetries = summary.results
+        .filter((r) => r.status === "passed" && (r.retry_attempts ?? 0) > 0)
+        .map((r) => ({ name: r.name, retry_attempts: r.retry_attempts }));
+    if (flakyRetries.length > 0) {
+        lines.push("");
+        lines.push(`## Flaky Tests: ${flakyRetries.length} test(s) required retries`);
+        for (const f of flakyRetries) {
+            lines.push(`  ♻️ ${f.name} — passed after ${f.retry_attempts} retry(ies)`);
+        }
+    }
     // Post PR comment if pr_url was provided
     if (pr_url) {
         try {
-            const prResult = await cloud.postPrComment({
+            const prResult = await requireCloud().postPrComment({
                 pr_url,
                 execution_id: summary.execution_id,
                 status: summary.status,
@@ -293,6 +1095,7 @@ server.tool("run", "Run a test suite. Executes all test cases in a real browser
                     strategy: h.strategy,
                     confidence: h.confidence,
                 })),
+                flaky_retries: flakyRetries.length > 0 ? flakyRetries : undefined,
             });
             const commentUrl = prResult.comment_url;
             lines.push("");
@@ -310,27 +1113,45 @@ server.tool("run", "Run a test suite. Executes all test cases in a real browser
 server.tool("github_token", "Set the GitHub personal access token for PR integration", {
     token: z.string().describe("GitHub personal access token (PAT) with repo scope"),
 }, async ({ token }) => {
-    await cloud.setGithubToken(token);
+    await requireCloud().setGithubToken(token);
     return { content: [{ type: "text", text: "GitHub token stored securely." }] };
 });
 server.tool("status", "Check the status of a test execution", {
     execution_id: z.string().describe("Execution ID to check"),
 }, async ({ execution_id }) => {
-    const result = await cloud.getExecutionStatus(execution_id);
+    const result = await requireCloud().getExecutionStatus(execution_id);
     return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
 });
 server.tool("cancel", "Cancel a running test execution", {
     execution_id: z.string().describe("Execution ID to cancel"),
 }, async ({ execution_id }) => {
-    const result = await cloud.cancelExecution(execution_id);
+    const result = await requireCloud().cancelExecution(execution_id);
     return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
 });
 server.tool("list_projects", "List all QA projects in the organization", {}, async () => {
-    const result = await cloud.listProjects();
+    const result = await requireCloud().listProjects();
     return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
 });
-server.tool("health", "Check if the QA agent backend is reachable", {}, async () => {
-    const result = await cloud.health();
+server.tool("list_suites", "List test suites across all projects. Use this to find suite IDs for the `run` tool.", {
+    search: z.string().optional().describe("Filter suites by name (e.g. 'checkout')"),
+}, async ({ search }) => {
+    const suites = await requireCloud().listSuites(search);
+    if (!Array.isArray(suites) || suites.length === 0) {
+        return { content: [{ type: "text", text: "No test suites found." }] };
+    }
+    const lines = ["# Test Suites", ""];
+    for (const s of suites) {
+        lines.push(`- **${s.name}**`);
+        lines.push(`  ID: \`${s.id}\``);
+        lines.push(`  Project: ${s.project_name} | Type: ${s.test_type}`);
+        if (s.description)
+            lines.push(`  ${s.description}`);
+        lines.push("");
+    }
+    return { content: [{ type: "text", text: lines.join("\n") }] };
+});
+server.tool("health", "Check if the FastTest Agent backend is reachable", {}, async () => {
+    const result = await requireCloud().health();
     return { content: [{ type: "text", text: JSON.stringify(result) }] };
 });
 // ---------------------------------------------------------------------------
@@ -343,13 +1164,14 @@ server.tool("heal", "Attempt to heal a broken selector by trying alternative loc
 }, async ({ selector, page_url, error_message }) => {
     const page = await browserMgr.getPage();
     const url = page_url ?? page.url();
+    // Try local deterministic strategies first (no cloud needed)
     const result = await healSelector(page, cloud, selector, "ELEMENT_NOT_FOUND", error_message ?? "Element not found", url);
     if (result.healed) {
         return {
             content: [{
                     type: "text",
                     text: [
-                        `🔧 Selector healed!`,
+                        `Selector healed!`,
                         `  Original: ${selector}`,
                         `  New:      ${result.newSelector}`,
                         `  Strategy: ${result.strategy} (${Math.round((result.confidence ?? 0) * 100)}% confidence)`,
@@ -357,19 +1179,36 @@ server.tool("heal", "Attempt to heal a broken selector by trying alternative loc
                 }],
         };
     }
+    // Local strategies exhausted — return snapshot + prompt for host AI to reason
+    const snapshot = await actions.getSnapshot(page);
+    const healPrompt = LOCAL_HEAL_PROMPT
+        .replace("{selector}", selector)
+        .replace("{error_message}", error_message ?? "Element not found")
+        .replace("{page_url}", url);
     return {
         content: [{
                 type: "text",
-                text: `❌ Could not heal selector: ${selector}\n  ${result.error ?? "All strategies exhausted"}`,
+                text: [
+                    `Local healing strategies could not fix: ${selector}`,
+                    "",
+                    "## Page Snapshot",
+                    "```json",
+                    JSON.stringify(snapshot, null, 2),
+                    "```",
+                    "",
+                    "## Instructions",
+                    healPrompt,
+                ].join("\n"),
             }],
     };
 });
 server.tool("healing_history", "View healing patterns and statistics for the organization", {
     limit: z.number().optional().describe("Max patterns to return (default 20)"),
 }, async ({ limit }) => {
+    const c = requireCloud();
     const [patterns, stats] = await Promise.all([
-        cloud.get(`/qa/healing/patterns?limit=${limit ?? 20}`),
-        cloud.get("/qa/healing/statistics"),
+        c.get(`/qa/healing/patterns?limit=${limit ?? 20}`),
+        c.get("/qa/healing/statistics"),
     ]);
     const lines = [
         `# Healing Statistics`,