@askqa/mcp 1.1.1 → 1.2.0

package/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "askqa",
-  "version": "1.1.1",
+  "version": "1.2.0",
   "description": "AskQA skills — set up notifications and monitoring for your websites",
   "mcpServers": {
     "askqa": {

package/README.md

@@ -41,6 +41,7 @@ Get your API key from [askqa.ai](https://askqa.ai) after signing in.
 | `delete_test`                 | Delete a test (with confirmation)                       |
 | `run_test`                    | Run a test and wait for results                         |
 | `get_test_results`            | Get recent test run results with step details           |
+| `heal_test`                   | Diagnose a layout-change regression and help fix selectors |
 | `get_test_screenshots`        | Get screenshots from a test run                         |
 | `list_templates`              | List available test templates                           |
 | `screenshot_url`              | Screenshot a URL and extract page structure             |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@askqa/mcp",
-  "version": "1.1.1",
+  "version": "1.2.0",
   "description": "MCP server for AskQA — monitor websites with automated tests by chatting with AI",
   "type": "module",
   "license": "MIT",

package/server.js

@@ -170,8 +170,19 @@ const server = new McpServer(
       'When the user asks whether something is working (e.g. "is checkout working?", "is the site up?"),',
       "your FIRST step should be to call list_tests to find a matching test by name or URL,",
       "then call get_test_results for that test to check the latest run status and step details.",
-      "If the latest run passed, confirm it\'s working. If it failed, report what failed.",
+      "If the latest run passed, confirm it's working. If it failed, report what failed.",
       "Only call run_test if the user explicitly asks to run a new test — checking status should use existing results.",
+      "",
+      "When the user asks to monitor a site, use list_templates to see available templates.",
+      "There are two kinds of templates:",
+      "",
+      "1. Universal templates (e.g. 'quick-checks') — work on any website with no configuration.",
+      "   Just call create_test with template_id and you're done.",
+      "",
+      "2. Site-specific templates (e.g. 'shopify-cart') — need to discover CSS selectors for the target site.",
+      "   Call detect_template first to probe the site and generate standalone Playwright code,",
+      "   then call create_test with that generated code (not the template_id).",
+      "   Templates that support detection have supportsDetection: true in list_templates output.",
     ].join("\n"),
   }
 );
@@ -179,13 +190,94 @@ const server = new McpServer(
 server.registerTool(
   "list_templates",
   {
-    description: "List available test templates. Returns template IDs, names, descriptions, and steps.",
+    description: "List available test templates with usage hints. Some templates work directly with create_test (universal), others need detect_template first to discover site-specific selectors (site-specific).",
     readOnlyHint: true,
   },
   async () => {
     try {
       const data = await apiGet("/api/tests/templates");
-      return { content: [{ type: "text", text: JSON.stringify(data.templates, null, 2) }] };
+      const lines = [];
+      for (const t of data.templates) {
+        lines.push(`${t.id}`);
+        lines.push(`  Name: ${t.name}`);
+        lines.push(`  Description: ${t.description}`);
+        lines.push(`  Steps: ${t.steps.join(", ")}`);
+        if (t.supportsCodeGeneration) {
+          lines.push(`  supportsDetection: true`);
+          lines.push(`  Usage: call detect_template → get generated code → create_test with code`);
+        } else {
+          lines.push(`  Usage: call create_test with template_id="${t.id}" directly`);
+        }
+        lines.push("");
+      }
+      return { content: [{ type: "text", text: lines.join("\n") }] };
+    } catch (err) {
+      return { content: [{ type: "text", text: `Error: ${err.message}` }], isError: true };
+    }
+  }
+);
+
+server.registerTool(
+  "detect_template",
+  {
+    description: "Run template detection against a URL to discover selectors and generate custom test code. This is the fastest way to create a custom test — detection runs the template flow against the site, discovers which CSS selectors work, and generates standalone Playwright test code with those selectors baked in. Use the generated code with create_test to save it.",
+    readOnlyHint: true,
+    inputSchema: {
+      template_id: z.string().describe("Template ID from list_templates (e.g. 'shopify-cart')"),
+      url: z.string().describe("The target URL to detect against (e.g. 'https://my-store.myshopify.com')"),
+    },
+  },
+  async ({ template_id, url }) => {
+    try {
+      const result = await apiPost(`/api/tests/templates/${template_id}/detect`, { url });
+      const content = [];
+      const lines = [];
+
+      const statusLabel = result.status === "passed" ? "PASSED" : "FAILED";
+      lines.push(`Detection: ${statusLabel}`);
+      lines.push("");
+
+      if (result.steps) {
+        lines.push("Steps:");
+        for (const step of result.steps) {
+          const icon = step.status === "passed" ? "+" : "x";
+          lines.push(`  ${icon} ${step.name} — ${step.status}`);
+          if (step.error) lines.push(`    Error: ${step.error}`);
+        }
+        lines.push("");
+      }
+
+      if (result.selectors) {
+        lines.push("Discovered selectors:");
+        for (const [key, value] of Object.entries(result.selectors)) {
+          lines.push(`  ${key}: ${value}`);
+        }
+        lines.push("");
+      }
+
+      if (result.code) {
+        lines.push("Generated test code (use with create_test):");
+        lines.push("```");
+        lines.push(result.code);
+        lines.push("```");
+      }
+
+      content.push({ type: "text", text: lines.join("\n") });
+
+      // Fetch screenshots for each step
+      if (result.steps) {
+        for (const step of result.steps) {
+          if (!step.screenshot) continue;
+          const screenshotUrl = `${API_URL}/api/screenshots/${result.executionId}/${step.screenshot}`;
+          const base64 = await fetchScreenshot(screenshotUrl);
+          if (base64) {
+            content.push({ type: "text", text: `Screenshot: ${step.name}` });
+            content.push({ type: "image", data: base64, mimeType: "image/png" });
+          }
+        }
+      }
+
+      return { content };
     } catch (err) {
       return { content: [{ type: "text", text: `Error: ${err.message}` }], isError: true };
     }
@@ -195,7 +287,7 @@ server.registerTool(
 server.registerTool(
   "create_test",
   {
-    description: "Create a saved test. Use template_id for built-in templates, or code for custom Playwright tests. Provide one or the other, not both.",
+    description: "Create a saved test. Use template_id for universal templates (e.g. 'quick-checks') that work on any site. Use code for custom tests or for site-specific templates after running detect_template. Provide template_id or code, not both.",
     destructiveHint: true,
     inputSchema: {
       name: z.string().describe("A name for this test (e.g. 'Homepage health check')"),
@@ -403,12 +495,14 @@ server.registerTool(
       code: z.string().optional().describe("Updated custom Playwright test code"),
       template_id: z.string().optional().describe("Updated template ID"),
       params: z.record(z.string()).optional().describe("Updated template parameters"),
-      secrets: z.record(z.string()).nullable().optional().describe("Updated secrets (pass null to clear)"),
+      secrets: z.record(z.string()).nullable().optional().describe("Updated secrets (pass null to clear). Encrypted at rest, never returned in API responses — must be provided again when updating a test that uses secrets."),
       headers: z.record(z.string()).nullable().optional().describe("Updated HTTP headers (pass null to clear)"),
       enable_test_mode: z.boolean().optional().describe("Send X-AskQA-Secret header to the target site, enabling test mode on sites that support it (default: true)"),
+      healing_disabled: z.boolean().optional().describe("Set true to stop AskQA from suggesting fixes for this test (e.g. a failing test that's acceptable as-is, or one you'd rather fix yourself). Auto-clears once the test passes again. Set false to re-enable."),
+      healing_note: z.string().nullable().optional().describe("Optional note explaining why healing was disabled (pass null to clear)."),
     },
   },
-  async ({ test_id, name, url, code, template_id, params, secrets, headers, enable_test_mode }) => {
+  async ({ test_id, name, url, code, template_id, params, secrets, headers, enable_test_mode, healing_disabled, healing_note }) => {
     try {
       const body = {};
       if (name !== undefined) body.name = name;
@@ -419,6 +513,8 @@ server.registerTool(
       if (secrets !== undefined) body.secrets = secrets;
       if (headers !== undefined) body.headers = headers;
       if (enable_test_mode !== undefined) body.enable_test_mode = enable_test_mode;
+      if (healing_disabled !== undefined) body.healing_disabled = healing_disabled;
+      if (healing_note !== undefined) body.healing_note = healing_note;
       const test = await apiPatch(`/api/tests/${test_id}`, body);
       return { content: [{ type: "text", text: JSON.stringify(test, null, 2) }] };
     } catch (err) {
@@ -670,6 +766,150 @@ server.registerTool(
   }
 );
 
+const REASON_EXPLANATIONS = {
+  "healing-disabled": "Healing is turned off for this test. Re-enable with update_test healing_disabled=false, or pass force=true to heal anyway.",
+  "not-custom-code": "This is a template-based test with no custom selectors to heal. Layout-change healing only applies to custom-code tests.",
+  "currently-passing": "The test's most recent run passed — there's nothing to heal.",
+  "not-enough-failures": "The test hasn't failed enough consecutive times yet to confirm a regression (a single/transient failure isn't healed). Wait for the confirmation threshold.",
+  "never-passed": "This test has no prior passing run, so a failure isn't a regression — it likely needs to be authored/fixed from scratch rather than healed.",
+  "transient-recovered": "On a fresh re-run the test passed, so the earlier failure was transient — no layout fix needed.",
+};
+
+function renderPageStructure(pageInfo) {
+  if (!pageInfo) return "";
+  const lines = ["", "Current page structure (selectors that exist NOW — map old selectors to these):"];
+  if (pageInfo.buttons?.length) {
+    lines.push("  Buttons:");
+    for (const b of pageInfo.buttons) lines.push(`    "${b.text}"  →  ${b.selector}`);
+  }
+  if (pageInfo.inputs?.length) {
+    lines.push("  Inputs:");
+    for (const inp of pageInfo.inputs) {
+      const desc = inp.placeholder || inp.name || inp.type || inp.tag;
+      lines.push(`    [${desc}]  →  ${inp.selector}`);
+    }
+  }
+  if (pageInfo.links?.length) {
+    lines.push("  Links:");
+    for (const l of pageInfo.links) lines.push(`    "${l.text}"  →  ${l.selector}`);
+  }
+  return lines.join("\n");
+}
+
+server.registerTool(
+  "heal_test",
+  {
+    description: "Diagnose and help heal a test that recently started failing because the website's layout changed (e.g. an input id or button selector changed). Accepts a test_id or name. Checks run history to confirm a real regression (was passing, now consistently failing) and that the failure is selector/layout-related — NOT a site outage, network error, or genuine feature breakage. When healable, it re-runs the test live, captures the CURRENT page structure + screenshots, and returns a brief for you to write new selectors. IMPORTANT: only fix the selectors — never change the test's logic, step order, assertions, or navigation, and never skip steps. After proposing new code, verify it with validate_test, and only then apply with update_test. If the failing test is actually acceptable, or the user wants to fix it themselves, call update_test healing_disabled=true to stop future suggestions.",
+    readOnlyHint: true,
+    inputSchema: {
+      test_id: z.coerce.number().optional().describe("The test ID to heal (from list_tests). Provide this or name."),
+      name: z.string().optional().describe("Test name (case-insensitive substring match) if you don't have the ID."),
+      reproduce: z.boolean().optional().describe("Re-run the test live to confirm the failure and capture current layout (default: true)."),
+      force: z.boolean().optional().describe("Heal even if healing_disabled is set on the test (default: false)."),
+    },
+  },
+  async ({ test_id, name, reproduce, force }) => {
+    try {
+      // Resolve test by id or name.
+      let resolvedId = test_id;
+      if (!resolvedId) {
+        if (!name) {
+          return { content: [{ type: "text", text: "Provide a test_id or name." }], isError: true };
+        }
+        const data = await apiGet("/api/tests/list");
+        const needle = name.toLowerCase();
+        const matches = data.tests.filter((t) => t.name && t.name.toLowerCase().includes(needle));
+        if (matches.length === 0) {
+          return { content: [{ type: "text", text: `No test found matching "${name}". Use list_tests to see available tests.` }], isError: true };
+        }
+        if (matches.length > 1) {
+          const list = matches.map((t) => `  #${t.id} — ${t.name} (${t.url})`).join("\n");
+          return { content: [{ type: "text", text: `Multiple tests match "${name}":\n${list}\n\nRe-run heal_test with a specific test_id.` }] };
+        }
+        resolvedId = matches[0].id;
+      }
+
+      const result = await apiPost(`/api/tests/${resolvedId}/heal`, { mode: "diagnose", reproduce: reproduce !== false });
+      const { test, diagnosis, reproduction } = result;
+      const content = [];
+      const lines = [`Healing diagnosis for "${test.name}" (#${test.id})`, `  URL: ${test.url}`, ""];
+
+      // Healing disabled — respect unless forced.
+      if (diagnosis.reason === "healing-disabled" && !force) {
+        lines.push("⚠ " + REASON_EXPLANATIONS["healing-disabled"]);
+        if (diagnosis.healing_note) lines.push(`  Note: ${diagnosis.healing_note}`);
+        return { content: [{ type: "text", text: lines.join("\n") }] };
+      }
+
+      // Not healable — explain and stop.
+      if (!diagnosis.healable && diagnosis.reason !== "healing-disabled") {
+        lines.push(`Not healable (${diagnosis.reason}).`);
+        if (REASON_EXPLANATIONS[diagnosis.reason]) lines.push(`  ${REASON_EXPLANATIONS[diagnosis.reason]}`);
+        if (diagnosis.reason === "regression" && diagnosis.failure_class && diagnosis.failure_class !== "selector") {
+          lines.push(`  The failure looks like a "${diagnosis.failure_class}" issue, not a layout/selector change, so healing won't help.`);
+          if (diagnosis.failing_step) lines.push(`  Failing step: ${diagnosis.failing_step.name} — ${diagnosis.failing_step.error || ""}`);
+        }
+        return { content: [{ type: "text", text: lines.join("\n") }] };
+      }
+
+      // Healable (or forced past a disabled flag). Build the brief.
+      const reg = diagnosis.regression;
+      if (reg) {
+        lines.push("✓ Confirmed regression (layout change suspected):");
+        lines.push(`  Last passed: run #${reg.lastPassRunId} at ${reg.lastPassAt}`);
+        lines.push(`  Started failing: run #${reg.firstFailRunId} at ${reg.firstFailAt}`);
+        lines.push(`  Consecutive failures: ${reg.failureStreak}`);
+      }
+      if (diagnosis.failing_step) {
+        lines.push("", `Failing step: ${diagnosis.failing_step.name}`);
+        if (diagnosis.failing_step.error) lines.push(`  Error: ${diagnosis.failing_step.error}`);
+      }
+
+      if (reproduction && !reproduction.still_failing) {
+        lines.push("", "On a fresh re-run the test PASSED — the failure was transient. No fix needed right now.");
+        return { content: [{ type: "text", text: lines.join("\n") }] };
+      }
+
+      // Current test code
+      const fullTest = await apiGet(`/api/tests/${test.id}`);
+      lines.push("", "Current test code:", "```javascript", fullTest.code || "(no code)", "```");
+
+      // Current page structure
+      if (reproduction?.page_info) lines.push(renderPageStructure(reproduction.page_info));
+
+      // Guardrails + instructions for the calling agent
+      lines.push(
+        "",
+        "── How to heal this test ──",
+        "1. Propose updated code that changes ONLY the broken selector(s) to match the current page structure above.",
+        "   • Do NOT change the test's logic, step order, assertions, waits, or navigation.",
+        "   • Do NOT skip/remove steps or add workarounds (no JS .click(), no waitForTimeout).",
+        "2. Verify your new code with validate_test (code + url). Iterate until it passes.",
+        "3. Only after it passes, apply it with update_test (confirm with the user first).",
+        "",
+        "If this failing test is actually acceptable, or the user wants to fix it themselves, call",
+        "update_test healing_disabled=true (optionally with healing_note) to stop future suggestions.",
+      );
+
+      content.push({ type: "text", text: lines.join("\n") });
+
+      // Reproduction screenshots
+      if (reproduction?.screenshots) {
+        for (const [stepName, base64] of Object.entries(reproduction.screenshots)) {
+          if (base64) {
+            content.push({ type: "text", text: `Screenshot: ${stepName}` });
+            content.push({ type: "image", data: base64, mimeType: "image/png" });
+          }
+        }
+      }
+
+      return { content };
+    } catch (err) {
+      return { content: [{ type: "text", text: `Error: ${err.message}` }], isError: true };
+    }
+  }
+);
+
 // --- Notification channel tools ---
 
 server.registerTool(