@askqa/mcp 1.1.2 → 1.2.2

package/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "askqa",
-  "version": "1.1.2",
+  "version": "1.2.2",
   "description": "AskQA skills — set up notifications and monitoring for your websites",
   "mcpServers": {
     "askqa": {

package/README.md

@@ -41,6 +41,7 @@ Get your API key from [askqa.ai](https://askqa.ai) after signing in.
 | `delete_test`                 | Delete a test (with confirmation)                       |
 | `run_test`                    | Run a test and wait for results                         |
 | `get_test_results`            | Get recent test run results with step details           |
+| `heal_test`                   | Diagnose a layout-change regression and help fix selectors |
 | `get_test_screenshots`        | Get screenshots from a test run                         |
 | `list_templates`              | List available test templates                           |
 | `screenshot_url`              | Screenshot a URL and extract page structure             |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@askqa/mcp",
-  "version": "1.1.2",
+  "version": "1.2.2",
   "description": "MCP server for AskQA — monitor websites with automated tests by chatting with AI",
   "type": "module",
   "license": "MIT",

package/server.js

@@ -138,6 +138,9 @@ function buildTestRunText(testRun, testName) {
   }
 
   lines.push(`  View details: ${testRun.details_url || `${WEBSITE_URL}/runs/${testRun.id}`}`);
+  if (testRun.trace_viewer_url) {
+    lines.push(`  View trace: ${testRun.trace_viewer_url}`);
+  }
 
   return lines.join("\n");
 }
@@ -381,7 +384,7 @@ server.registerTool(
 server.registerTool(
   "validate_test",
   {
-    description: "Dry-run custom Playwright test code against a URL. Returns execution results, screenshots, and page structure for debugging. Steps continue even on failure to maximize debug signal. Use this to iterate on code before calling create_test.",
+    description: "Start here when writing a new test. Dry-runs Playwright code against a URL without saving it — returns step results, screenshots, and page structure. Steps continue even on failure for maximum debug signal. Iterate here until the test passes, then call create_test to save it.",
     readOnlyHint: true,
     inputSchema: {
       code: z.string().describe("Custom Playwright test code. Must define an async function test({ page, step, log })."),
@@ -390,28 +393,33 @@ server.registerTool(
   },
   async ({ code, url }) => {
     try {
-      const result = await apiPost("/api/tests/validate", { code, url });
+      // POST returns immediately with test_run_id; poll until done (same as run_test)
+      const { test_run_id } = await apiPost("/api/tests/validate", { code, url });
+      const testRun = await pollTestRun(test_run_id);
+      const runResult = testRun.result || {};
       const content = [];
 
       // Build text summary
-      const icon = result.status === "passed" ? "✓" : "✗";
-      const lines = [`${icon} Validation: ${result.status}`];
-      if (result.durationMs) lines.push(`  Duration: ${(result.durationMs / 1000).toFixed(1)}s`);
-      if (result.error) lines.push(`  Error: ${result.error}`);
-      for (const step of (result.steps || [])) {
+      const overallStatus = runResult.status || (testRun.status === "completed" ? "passed" : "failed");
+      const icon = overallStatus === "passed" ? "✓" : "✗";
+      const lines = [`${icon} Validation: ${overallStatus}`];
+      if (runResult.durationMs) lines.push(`  Duration: ${(runResult.durationMs / 1000).toFixed(1)}s`);
+      if (testRun.error) lines.push(`  Error: ${testRun.error}`);
+      if (runResult.error) lines.push(`  Error: ${runResult.error}`);
+      for (const step of (runResult.steps || [])) {
         const stepIcon = step.status === "passed" ? "✓" : step.status === "failed" ? "✗" : "?";
         lines.push(`  ${stepIcon} ${step.name} — ${step.status}`);
         if (step.error) lines.push(`    Error: ${step.error}`);
       }
-      if (result.logs?.length) {
+      if (runResult.logs?.length) {
         lines.push("  Logs:");
-        for (const msg of result.logs) lines.push(`    ${msg}`);
+        for (const msg of runResult.logs) lines.push(`    ${msg}`);
       }
       content.push({ type: "text", text: lines.join("\n") });
 
       // Include page info for debugging selectors
-      if (result.pageInfo) {
-        const info = result.pageInfo;
+      if (runResult.pageInfo) {
+        const info = runResult.pageInfo;
         const infoLines = ["", "Page structure (for fixing selectors):"];
         if (info.buttons?.length) {
           infoLines.push("  Buttons:");
@@ -431,14 +439,10 @@ server.registerTool(
         content.push({ type: "text", text: infoLines.join("\n") });
       }
 
-      // Include screenshots as labeled images
-      if (result.screenshots) {
-        for (const [stepName, base64] of Object.entries(result.screenshots)) {
-          if (base64) {
-            content.push({ type: "text", text: `Screenshot: ${stepName}` });
-            content.push({ type: "image", data: base64, mimeType: "image/png" });
-          }
-        }
+      // Include screenshots from storage (via execution_id)
+      if (testRun.execution_id) {
+        const screenshots = await buildTestRunScreenshots(testRun);
+        content.push(...screenshots);
       }
 
       return { content };
@@ -498,9 +502,11 @@ server.registerTool(
       secrets: z.record(z.string()).nullable().optional().describe("Updated secrets (pass null to clear). Encrypted at rest, never returned in API responses — must be provided again when updating a test that uses secrets."),
       headers: z.record(z.string()).nullable().optional().describe("Updated HTTP headers (pass null to clear)"),
       enable_test_mode: z.boolean().optional().describe("Send X-AskQA-Secret header to the target site, enabling test mode on sites that support it (default: true)"),
+      healing_disabled: z.boolean().optional().describe("Set true to stop AskQA from suggesting fixes for this test (e.g. a failing test that's acceptable as-is, or one you'd rather fix yourself). Auto-clears once the test passes again. Set false to re-enable."),
+      healing_note: z.string().nullable().optional().describe("Optional note explaining why healing was disabled (pass null to clear)."),
     },
   },
-  async ({ test_id, name, url, code, template_id, params, secrets, headers, enable_test_mode }) => {
+  async ({ test_id, name, url, code, template_id, params, secrets, headers, enable_test_mode, healing_disabled, healing_note }) => {
     try {
       const body = {};
       if (name !== undefined) body.name = name;
@@ -511,6 +517,8 @@ server.registerTool(
       if (secrets !== undefined) body.secrets = secrets;
       if (headers !== undefined) body.headers = headers;
       if (enable_test_mode !== undefined) body.enable_test_mode = enable_test_mode;
+      if (healing_disabled !== undefined) body.healing_disabled = healing_disabled;
+      if (healing_note !== undefined) body.healing_note = healing_note;
       const test = await apiPatch(`/api/tests/${test_id}`, body);
       return { content: [{ type: "text", text: JSON.stringify(test, null, 2) }] };
     } catch (err) {
@@ -762,6 +770,150 @@ server.registerTool(
   }
 );
 
+const REASON_EXPLANATIONS = {
+  "healing-disabled": "Healing is turned off for this test. Re-enable with update_test healing_disabled=false, or pass force=true to heal anyway.",
+  "not-custom-code": "This is a template-based test with no custom selectors to heal. Layout-change healing only applies to custom-code tests.",
+  "currently-passing": "The test's most recent run passed — there's nothing to heal.",
+  "not-enough-failures": "The test hasn't failed enough consecutive times yet to confirm a regression (a single/transient failure isn't healed). Wait for the confirmation threshold.",
+  "never-passed": "This test has no prior passing run, so a failure isn't a regression — it likely needs to be authored/fixed from scratch rather than healed.",
+  "transient-recovered": "On a fresh re-run the test passed, so the earlier failure was transient — no layout fix needed.",
+};
+
+function renderPageStructure(pageInfo) {
+  if (!pageInfo) return "";
+  const lines = ["", "Current page structure (selectors that exist NOW — map old selectors to these):"];
+  if (pageInfo.buttons?.length) {
+    lines.push("  Buttons:");
+    for (const b of pageInfo.buttons) lines.push(`    "${b.text}"  →  ${b.selector}`);
+  }
+  if (pageInfo.inputs?.length) {
+    lines.push("  Inputs:");
+    for (const inp of pageInfo.inputs) {
+      const desc = inp.placeholder || inp.name || inp.type || inp.tag;
+      lines.push(`    [${desc}]  →  ${inp.selector}`);
+    }
+  }
+  if (pageInfo.links?.length) {
+    lines.push("  Links:");
+    for (const l of pageInfo.links) lines.push(`    "${l.text}"  →  ${l.selector}`);
+  }
+  return lines.join("\n");
+}
+
+server.registerTool(
+  "heal_test",
+  {
+    description: "Diagnose and help heal a test that recently started failing because the website's layout changed (e.g. an input id or button selector changed). Accepts a test_id or name. Checks run history to confirm a real regression (was passing, now consistently failing) and that the failure is selector/layout-related — NOT a site outage, network error, or genuine feature breakage. When healable, it re-runs the test live, captures the CURRENT page structure + screenshots, and returns a brief for you to write new selectors. IMPORTANT: only fix the selectors — never change the test's logic, step order, assertions, or navigation, and never skip steps. After proposing new code, verify it with validate_test, and only then apply with update_test. If the failing test is actually acceptable, or the user wants to fix it themselves, call update_test healing_disabled=true to stop future suggestions.",
+    readOnlyHint: true,
+    inputSchema: {
+      test_id: z.coerce.number().optional().describe("The test ID to heal (from list_tests). Provide this or name."),
+      name: z.string().optional().describe("Test name (case-insensitive substring match) if you don't have the ID."),
+      reproduce: z.boolean().optional().describe("Re-run the test live to confirm the failure and capture current layout (default: true)."),
+      force: z.boolean().optional().describe("Heal even if healing_disabled is set on the test (default: false)."),
+    },
+  },
+  async ({ test_id, name, reproduce, force }) => {
+    try {
+      // Resolve test by id or name.
+      let resolvedId = test_id;
+      if (!resolvedId) {
+        if (!name) {
+          return { content: [{ type: "text", text: "Provide a test_id or name." }], isError: true };
+        }
+        const data = await apiGet("/api/tests/list");
+        const needle = name.toLowerCase();
+        const matches = data.tests.filter((t) => t.name && t.name.toLowerCase().includes(needle));
+        if (matches.length === 0) {
+          return { content: [{ type: "text", text: `No test found matching "${name}". Use list_tests to see available tests.` }], isError: true };
+        }
+        if (matches.length > 1) {
+          const list = matches.map((t) => `  #${t.id} — ${t.name} (${t.url})`).join("\n");
+          return { content: [{ type: "text", text: `Multiple tests match "${name}":\n${list}\n\nRe-run heal_test with a specific test_id.` }] };
+        }
+        resolvedId = matches[0].id;
+      }
+
+      const result = await apiPost(`/api/tests/${resolvedId}/heal`, { mode: "diagnose", reproduce: reproduce !== false });
+      const { test, diagnosis, reproduction } = result;
+      const content = [];
+      const lines = [`Healing diagnosis for "${test.name}" (#${test.id})`, `  URL: ${test.url}`, ""];
+
+      // Healing disabled — respect unless forced.
+      if (diagnosis.reason === "healing-disabled" && !force) {
+        lines.push("⚠ " + REASON_EXPLANATIONS["healing-disabled"]);
+        if (diagnosis.healing_note) lines.push(`  Note: ${diagnosis.healing_note}`);
+        return { content: [{ type: "text", text: lines.join("\n") }] };
+      }
+
+      // Not healable — explain and stop.
+      if (!diagnosis.healable && diagnosis.reason !== "healing-disabled") {
+        lines.push(`Not healable (${diagnosis.reason}).`);
+        if (REASON_EXPLANATIONS[diagnosis.reason]) lines.push(`  ${REASON_EXPLANATIONS[diagnosis.reason]}`);
+        if (diagnosis.reason === "regression" && diagnosis.failure_class && diagnosis.failure_class !== "selector") {
+          lines.push(`  The failure looks like a "${diagnosis.failure_class}" issue, not a layout/selector change, so healing won't help.`);
+          if (diagnosis.failing_step) lines.push(`  Failing step: ${diagnosis.failing_step.name} — ${diagnosis.failing_step.error || ""}`);
+        }
+        return { content: [{ type: "text", text: lines.join("\n") }] };
+      }
+
+      // Healable (or forced past a disabled flag). Build the brief.
+      const reg = diagnosis.regression;
+      if (reg) {
+        lines.push("✓ Confirmed regression (layout change suspected):");
+        lines.push(`  Last passed: run #${reg.lastPassRunId} at ${reg.lastPassAt}`);
+        lines.push(`  Started failing: run #${reg.firstFailRunId} at ${reg.firstFailAt}`);
+        lines.push(`  Consecutive failures: ${reg.failureStreak}`);
+      }
+      if (diagnosis.failing_step) {
+        lines.push("", `Failing step: ${diagnosis.failing_step.name}`);
+        if (diagnosis.failing_step.error) lines.push(`  Error: ${diagnosis.failing_step.error}`);
+      }
+
+      if (reproduction && !reproduction.still_failing) {
+        lines.push("", "On a fresh re-run the test PASSED — the failure was transient. No fix needed right now.");
+        return { content: [{ type: "text", text: lines.join("\n") }] };
+      }
+
+      // Current test code
+      const fullTest = await apiGet(`/api/tests/${test.id}`);
+      lines.push("", "Current test code:", "```javascript", fullTest.code || "(no code)", "```");
+
+      // Current page structure
+      if (reproduction?.page_info) lines.push(renderPageStructure(reproduction.page_info));
+
+      // Guardrails + instructions for the calling agent
+      lines.push(
+        "",
+        "── How to heal this test ──",
+        "1. Propose updated code that changes ONLY the broken selector(s) to match the current page structure above.",
+        "   • Do NOT change the test's logic, step order, assertions, waits, or navigation.",
+        "   • Do NOT skip/remove steps or add workarounds (no JS .click(), no waitForTimeout).",
+        "2. Verify your new code with validate_test (code + url). Iterate until it passes.",
+        "3. Only after it passes, apply it with update_test (confirm with the user first).",
+        "",
+        "If this failing test is actually acceptable, or the user wants to fix it themselves, call",
+        "update_test healing_disabled=true (optionally with healing_note) to stop future suggestions.",
+      );
+
+      content.push({ type: "text", text: lines.join("\n") });
+
+      // Reproduction screenshots
+      if (reproduction?.screenshots) {
+        for (const [stepName, base64] of Object.entries(reproduction.screenshots)) {
+          if (base64) {
+            content.push({ type: "text", text: `Screenshot: ${stepName}` });
+            content.push({ type: "image", data: base64, mimeType: "image/png" });
+          }
+        }
+      }
+
+      return { content };
+    } catch (err) {
+      return { content: [{ type: "text", text: `Error: ${err.message}` }], isError: true };
+    }
+  }
+);
+
 // --- Notification channel tools ---
 
 server.registerTool(