@validate.qa/mcp 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js CHANGED
@@ -24,7 +24,7 @@ function parseInitArgs(argv) {
24
24
  args[arg.slice(2)] = argv[++i];
25
25
  }
26
26
  }
27
- const apiUrl = args["api-url"] ?? process.env.VALIDATEQA_API_URL ?? "https://shared-server.up.railway.app";
27
+ const apiUrl = args["api-url"] ?? process.env.VALIDATEQA_API_URL ?? "https://shared.validate.qa";
28
28
  const token = args["token"] ?? process.env.VALIDATEQA_TOKEN;
29
29
  const projectId = args["project-id"] ?? process.env.VALIDATEQA_PROJECT_ID;
30
30
  if (!token) {
@@ -80,7 +80,7 @@ function runInit() {
80
80
  const entry = buildMcpEntry(config2);
81
81
  const cwd = process.cwd();
82
82
  console.log("");
83
- console.log(" Validate.QA \u2014 MCP Setup");
83
+ console.log(" Validate.QA - MCP Setup");
84
84
  console.log(" \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500");
85
85
  console.log("");
86
86
  let anyWritten = false;
@@ -91,9 +91,9 @@ function runInit() {
91
91
  }
92
92
  const file = join(target.configDir, target.configFile);
93
93
  if (result === "created") {
94
- console.log(` \u2713 ${target.name} \u2014 created ${file}`);
94
+ console.log(` \u2713 ${target.name} - created ${file}`);
95
95
  } else {
96
- console.log(` \u2713 ${target.name} \u2014 updated ${file}`);
96
+ console.log(` \u2713 ${target.name} - updated ${file}`);
97
97
  }
98
98
  anyWritten = true;
99
99
  }
@@ -174,6 +174,24 @@ function resolveConfig() {
174
174
  }
175
175
 
176
176
  // src/api-client.ts
177
+ var PLAN_FEATURE_LOCKED_CODE = "PLAN_FEATURE_LOCKED";
178
+ var ApiResponseError = class extends Error {
179
+ status;
180
+ body;
181
+ constructor(status, body) {
182
+ const message = typeof body.error === "string" && body.error.length > 0 ? body.error : `API error ${status}`;
183
+ super(message);
184
+ this.name = "ApiResponseError";
185
+ this.status = status;
186
+ this.body = body;
187
+ }
188
+ };
189
+ function toolErrorText(err) {
190
+ if (err instanceof ApiResponseError && err.body.code === PLAN_FEATURE_LOCKED_CODE) {
191
+ return JSON.stringify(err.body);
192
+ }
193
+ return `Error: ${err instanceof Error ? err.message : String(err)}`;
194
+ }
177
195
  var ValidateQAClient = class {
178
196
  constructor(baseUrl, token, projectId) {
179
197
  this.baseUrl = baseUrl;
@@ -199,11 +217,28 @@ var ValidateQAClient = class {
199
217
  throw new Error(`Connection failed: ${msg}${cause ? ` (${cause})` : ""}`);
200
218
  }
201
219
  if (!res.ok) {
202
- const body = await res.json().catch(() => ({ error: res.statusText }));
203
- throw new Error(body.error || `API error ${res.status}`);
220
+ const parsed = await res.json().catch(() => ({ error: res.statusText }));
221
+ const body = parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : { error: res.statusText };
222
+ throw new ApiResponseError(res.status, body);
204
223
  }
205
224
  return res.json();
206
225
  }
226
+ /** project://test-conventions from the server (GET /api/mcp/test-conventions); null when the server predates it. */
227
+ async getTestConventions() {
228
+ try {
229
+ const res = await fetch(`${this.baseUrl}/api/mcp/test-conventions`, {
230
+ headers: {
231
+ "Authorization": `Bearer ${this.token}`,
232
+ "X-Project-Id": this.projectId
233
+ },
234
+ signal: AbortSignal.timeout(3e4)
235
+ });
236
+ if (!res.ok) return null;
237
+ return await res.json();
238
+ } catch {
239
+ return null;
240
+ }
241
+ }
207
242
  async healthCheck() {
208
243
  try {
209
244
  const res = await fetch(`${this.baseUrl}/health`, {
@@ -227,7 +262,49 @@ function tool(server2, client2, name, description, schema, annotations) {
227
262
  };
228
263
  } catch (err) {
229
264
  return {
230
- content: [{ type: "text", text: `Error: ${err instanceof Error ? err.message : String(err)}` }],
265
+ content: [{ type: "text", text: toolErrorText(err) }],
266
+ isError: true
267
+ };
268
+ }
269
+ };
270
+ if (annotations) {
271
+ server2.tool(name, description, schema, annotations, handler);
272
+ } else {
273
+ server2.tool(name, description, schema, handler);
274
+ }
275
+ }
276
+ var SERVER_WAIT_MAX_SECONDS = 55;
277
+ var WAIT_DEFAULT_SECONDS = 50;
278
+ var WAIT_MAX_SECONDS = 120;
279
+ function isRecord(value) {
280
+ return typeof value === "object" && value !== null && !Array.isArray(value);
281
+ }
282
+ function waitSecondsFrom(raw, fallback) {
283
+ const n = typeof raw === "number" ? raw : Number.NaN;
284
+ if (!Number.isFinite(n)) return fallback;
285
+ return Math.min(Math.max(n, 0), WAIT_MAX_SECONDS);
286
+ }
287
+ async function executeWithWait(client2, name, args, waitKey, totalSeconds, kind) {
288
+ const deadline = Date.now() + totalSeconds * 1e3;
289
+ const first = Math.min(totalSeconds, SERVER_WAIT_MAX_SECONDS);
290
+ let result = await client2.executeTool(name, { ...args, [waitKey]: first });
291
+ for (; ; ) {
292
+ if (!isRecord(result) || result.timedOut !== true) return result;
293
+ const id = result[kind.idKey];
294
+ const left = Math.floor((deadline - Date.now()) / 1e3);
295
+ if (typeof id !== "string" || left < 1) return result;
296
+ result = await client2.executeTool(kind.tool, { [kind.idKey]: id, timeoutSeconds: Math.min(left, SERVER_WAIT_MAX_SECONDS) });
297
+ }
298
+ }
299
+ function waitTool(server2, client2, name, description, schema, waitKey, defaultSeconds, kind, annotations) {
300
+ const handler = async (args) => {
301
+ try {
302
+ const seconds = waitSecondsFrom(args[waitKey], defaultSeconds);
303
+ const result = seconds > 0 ? await executeWithWait(client2, name, args, waitKey, seconds, kind) : await client2.executeTool(name, args);
304
+ return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
305
+ } catch (err) {
306
+ return {
307
+ content: [{ type: "text", text: toolErrorText(err) }],
231
308
  isError: true
232
309
  };
233
310
  }
@@ -243,7 +320,7 @@ function registerAllTools(server2, client2) {
243
320
  server2,
244
321
  client2,
245
322
  "list_tests",
246
- "List tests in the current project. Returns test name, status, suite, tags, confidence, and the last run result. Paginated \u2014 use limit/offset to page through all tests.",
323
+ "List tests in the current project. Returns test name, status, suite, tags, confidence, and the last run result. Paginated - use limit/offset to page through all tests.",
247
324
  {
248
325
  status: z.enum(["DRAFT", "ACTIVE", "DISABLED", "QUARANTINED"]).optional().describe("Filter by test lifecycle status"),
249
326
  suite: z.string().optional().describe("Filter by suite name"),
@@ -258,16 +335,24 @@ function registerAllTools(server2, client2) {
258
335
  server2,
259
336
  client2,
260
337
  "get_test",
261
- "Get details of a specific test: name, status, tags, steps, session, and the 5 most recent run results. Code is truncated to 200 lines \u2014 use get_test_code for the full Playwright code.",
338
+ "Get details of a specific test: name, status, tags, steps, session, and the 5 most recent run results. Code is truncated to 200 lines - use get_test_code for the full Playwright code.",
262
339
  { testId: z.string().describe("The test case ID") },
263
340
  { readOnlyHint: true }
264
341
  );
265
- tool(
342
+ waitTool(
266
343
  server2,
267
344
  client2,
268
345
  "run_test",
269
- "Queue a test for execution. Creates a PENDING test run that will be picked up by an online runner. Returns the run ID.",
270
- { testId: z.string().describe("The test case ID to run") }
346
+ "Queue a test for execution. Creates a PENDING test run that will be picked up by an online runner. Returns the run ID; with waitSeconds, the finished run (as wait_for_run). For a mobile (Appium) test, pass platform to run it on the OTHER platform than it was authored on - if some steps lack a native locator for that platform, a one-time locator-discovery pass runs first (supply appId once for the target-platform build), then the test runs automatically.",
347
+ {
348
+ testId: z.string().describe("The test case ID to run"),
349
+ platform: z.enum(["IOS", "ANDROID"]).optional().describe("Mobile only: run this test on the given platform (cross-platform run). Omit to run on the authoring platform."),
350
+ appId: z.string().optional().describe("Mobile cross-platform only: the target-platform app id (bundleId/appPackage). Only needed the first time this project runs on the other platform."),
351
+ waitSeconds: z.number().min(0).max(WAIT_MAX_SECONDS).optional().describe(`Optional. Wait up to this many seconds (max ${WAIT_MAX_SECONDS}) for the run to finish and return its final result, like wait_for_run (timedOut: true if still going). 0 or omitted: return at once with the run ID.`)
352
+ },
353
+ "waitSeconds",
354
+ 0,
355
+ { tool: "wait_for_run", idKey: "runId" }
271
356
  );
272
357
  tool(
273
358
  server2,
@@ -288,10 +373,11 @@ function registerAllTools(server2, client2) {
288
373
  server2,
289
374
  client2,
290
375
  "edit_test_code",
291
- "Update the Playwright code, name, or tags of an existing test case. Read project://test-conventions for patterns and pitfalls. Code is validated \u2014 fix any warnings returned.",
376
+ "Update an existing test case. WEB (Playwright) tests: provide the full replacement playwrightCode (read project://test-conventions for patterns; code is validated - fix any warnings returned). MOBILE (Appium) tests: provide the FULL replacement steps array (playwrightCode is rejected - mobile tests execute structured steps, not code). Read the current form first with get_test_code.",
292
377
  {
293
378
  testId: z.string().describe("The test case ID to edit"),
294
- playwrightCode: z.string().optional().describe("New Playwright test code (full file replacement)"),
379
+ playwrightCode: z.string().optional().describe("Web tests only: new Playwright test code (full file replacement)"),
380
+ steps: z.array(z.record(z.unknown())).optional().describe("Mobile (Appium) tests only: FULL replacement structured step array (same shape as create_test steps). Include every step, not just the changed ones."),
295
381
  name: z.string().optional().describe("New test name"),
296
382
  tags: z.array(z.string()).optional().describe('New tags array (e.g. ["@regression", "@smoke"])')
297
383
  }
@@ -300,7 +386,7 @@ function registerAllTools(server2, client2) {
300
386
  server2,
301
387
  client2,
302
388
  "update_test_status",
303
- "Change the lifecycle status of one or more tests (e.g. promote DRAFT\u2192ACTIVE, disable a flaky test).",
389
+ "Change the lifecycle status of one or more tests (e.g. promote DRAFT\u2192ACTIVE, disable a flaky test). Re-activating exactly ONE bug-found test (QUARANTINED\u2192ACTIVE) also queues a verification run and returns its testRunId; with several ids no run is queued.",
304
390
  {
305
391
  testIds: z.array(z.string()).describe("Test IDs to update"),
306
392
  status: z.enum(["DRAFT", "ACTIVE", "DISABLED", "QUARANTINED"]).describe("New status to set")
@@ -310,13 +396,65 @@ function registerAllTools(server2, client2) {
310
396
  server2,
311
397
  client2,
312
398
  "get_test_run",
313
- "Get the status and result of a specific test run, including error details if it failed. Can look up by runId or get the latest run for a testId.",
399
+ "Get the status and result of a specific test run once (to wait for it to finish, use wait_for_run, not a loop of this). Look up by runId, or the latest run for a testId. A FAILED or ERROR run also carries its failure evidence: error (up to 4 KB), consoleLog (the last 200 lines, at most 20 KB), failedSteps, and for a mobile run domSnapshot (the page source at the failure). Screenshots: get_test_screenshots. No Playwright trace is recorded.",
314
400
  {
315
401
  runId: z.string().optional().describe("The test run ID"),
316
402
  testId: z.string().optional().describe("Get the most recent run for this test case ID")
317
403
  },
318
404
  { readOnlyHint: true }
319
405
  );
406
+ waitTool(
407
+ server2,
408
+ client2,
409
+ "wait_for_run",
410
+ `Wait until a test run (run_test, heal_test, a verification run) is PASSED, FAILED or ERROR, then return it in the get_test_run shape, failure evidence included. Returns after at most timeoutSeconds (default ${WAIT_DEFAULT_SECONDS}, max ${WAIT_MAX_SECONDS}) with timedOut: true and the current status if it is still pending or running; call it again to keep waiting. Use this instead of polling get_test_run.`,
411
+ {
412
+ runId: z.string().describe("The test run ID to wait for"),
413
+ timeoutSeconds: z.number().min(0).max(WAIT_MAX_SECONDS).optional().describe(`How long to wait, in seconds (default ${WAIT_DEFAULT_SECONDS}, max ${WAIT_MAX_SECONDS})`)
414
+ },
415
+ "timeoutSeconds",
416
+ WAIT_DEFAULT_SECONDS,
417
+ { tool: "wait_for_run", idKey: "runId" },
418
+ { readOnlyHint: true }
419
+ );
420
+ waitTool(
421
+ server2,
422
+ client2,
423
+ "wait_for_batch",
424
+ `Wait until a batch from batch_run_tests is COMPLETED, then return its counts (total, completed, passed, failed, errored) and, when tests failed, up to 20 failures with their runId and error. Returns after at most timeoutSeconds (default ${WAIT_DEFAULT_SECONDS}, max ${WAIT_MAX_SECONDS}) with timedOut: true and the current counts if it is still running; call it again to keep waiting. Use get_test_run on a failure's runId for its console log.`,
425
+ {
426
+ batchId: z.string().describe("The batch ID returned by batch_run_tests"),
427
+ timeoutSeconds: z.number().min(0).max(WAIT_MAX_SECONDS).optional().describe(`How long to wait, in seconds (default ${WAIT_DEFAULT_SECONDS}, max ${WAIT_MAX_SECONDS})`)
428
+ },
429
+ "timeoutSeconds",
430
+ WAIT_DEFAULT_SECONDS,
431
+ { tool: "wait_for_batch", idKey: "batchId" },
432
+ { readOnlyHint: true }
433
+ );
434
+ tool(
435
+ server2,
436
+ client2,
437
+ "run_discovery",
438
+ "Start AI discovery for this project. For web, discovery uses the configured base URL. For mobile, pass platform and appId to explore a native app on a connected runner/device. Use survey to map features, deep with featureName to generate tests for one feature, or full for broad generation.",
439
+ {
440
+ mode: z.enum(["survey", "deep", "full"]).optional().describe("Discovery mode. survey maps features/screens and generates no tests; deep generates tests for one feature; full explores and generates broadly."),
441
+ featureName: z.string().optional().describe("Required for deep mode: the feature name to generate tests for."),
442
+ focusAreas: z.array(z.string()).optional().describe("Optional routes, screens, or areas to focus discovery on."),
443
+ platform: z.enum(["IOS", "ANDROID"]).optional().describe("Mobile only: target platform. Requires appId."),
444
+ appId: z.string().optional().describe("Mobile only: bundle id/package id to explore. Requires platform."),
445
+ resetPolicy: z.enum(["AUTO", "CLEAR_APP_DATA", "PRESERVE_APP_DATA"]).optional().describe("Mobile only: app-state policy for discovery/replay. AUTO uses default safe reset behavior.")
446
+ }
447
+ );
448
+ tool(
449
+ server2,
450
+ client2,
451
+ "get_discovery_status",
452
+ "Check progress for a discovery run. Omit discoveryRunId to get the latest discovery run for this project.",
453
+ {
454
+ discoveryRunId: z.string().optional().describe("Specific discovery run ID. Omit for latest.")
455
+ },
456
+ { readOnlyHint: true }
457
+ );
320
458
  server2.tool(
321
459
  "get_test_screenshots",
322
460
  "Get screenshots from a specific test run as images the AI can actually see. Use this to visually inspect what happened during a failed or passing run.",
@@ -339,7 +477,7 @@ function registerAllTools(server2, client2) {
339
477
  const content = [
340
478
  {
341
479
  type: "text",
342
- text: `Run ${result.runId} \u2014 status: ${result.status} \u2014 showing ${result.returned} of ${result.total} screenshot(s)`
480
+ text: `Run ${result.runId} - status: ${result.status} - showing ${result.returned} of ${result.total} screenshot(s)`
343
481
  }
344
482
  ];
345
483
  for (const shot of result.screenshots) {
@@ -348,7 +486,7 @@ function registerAllTools(server2, client2) {
348
486
  return { content };
349
487
  } catch (err) {
350
488
  return {
351
- content: [{ type: "text", text: `Error: ${err instanceof Error ? err.message : String(err)}` }],
489
+ content: [{ type: "text", text: toolErrorText(err) }],
352
490
  isError: true
353
491
  };
354
492
  }
@@ -364,15 +502,19 @@ function registerAllTools(server2, client2) {
364
502
  cancelBatch: z.boolean().optional().describe("If true, cancel the latest batch run instead")
365
503
  }
366
504
  );
367
- tool(
505
+ waitTool(
368
506
  server2,
369
507
  client2,
370
508
  "batch_run_tests",
371
- "Run multiple tests from the database individually. Filter by test IDs or suite name; omit both to run all ACTIVE tests. Each test executes in its own browser.",
509
+ "Run multiple tests from the database individually. Filter by test IDs or suite name; omit both to run all ACTIVE tests. Each test executes in its own browser. Returns the batchId; follow it with wait_for_batch, or pass waitSeconds.",
372
510
  {
373
511
  testIds: z.array(z.string()).optional().describe("Specific test IDs to run"),
374
- suite: z.string().optional().describe("Run all active tests in this suite")
375
- }
512
+ suite: z.string().optional().describe("Run all active tests in this suite"),
513
+ waitSeconds: z.number().min(0).max(WAIT_MAX_SECONDS).optional().describe(`Optional. Wait up to this many seconds (max ${WAIT_MAX_SECONDS}) for the batch to complete and return the wait_for_batch result (timedOut: true if still running). 0 or omitted: return at once with the batchId.`)
514
+ },
515
+ "waitSeconds",
516
+ 0,
517
+ { tool: "wait_for_batch", idKey: "batchId" }
376
518
  );
377
519
  tool(
378
520
  server2,
@@ -382,6 +524,16 @@ function registerAllTools(server2, client2) {
382
524
  {},
383
525
  { readOnlyHint: true }
384
526
  );
527
+ tool(
528
+ server2,
529
+ client2,
530
+ "generate_tests_for_feature",
531
+ "Start test generation for an existing feature by id or name. For mobile, run_discovery with mode=deep, featureName, platform, and appId gives explicit device/platform control.",
532
+ {
533
+ featureId: z.string().optional().describe("Feature ID. Either featureId or featureName is required."),
534
+ featureName: z.string().optional().describe("Feature name. Either featureId or featureName is required.")
535
+ }
536
+ );
385
537
  tool(
386
538
  server2,
387
539
  client2,
@@ -416,6 +568,14 @@ function registerAllTools(server2, client2) {
416
568
  {},
417
569
  { readOnlyHint: true }
418
570
  );
571
+ tool(
572
+ server2,
573
+ client2,
574
+ "get_regression_stats",
575
+ "Get test counts by lifecycle status (active, quarantined, draft, disabled, total; API specs count their inner tests) and the 7-day pass rate, with rate-limit, config-issue and cancelled runs left out. The same numbers as the dashboard's regression stats.",
576
+ {},
577
+ { readOnlyHint: true }
578
+ );
419
579
  tool(
420
580
  server2,
421
581
  client2,
@@ -434,7 +594,7 @@ function registerAllTools(server2, client2) {
434
594
  mode: z.enum(["batch"]).describe("Run mode: batch (runs project tests from the database)"),
435
595
  intervalMinutes: z.union([z.literal(60), z.literal(360), z.literal(720), z.literal(1440), z.literal(2880), z.literal(10080)]).describe("Interval in minutes: 60 (hourly), 360 (6h), 720 (12h), 1440 (daily), 2880 (2 days), 10080 (weekly)"),
436
596
  tags: z.array(z.string()).optional().describe('Tags to filter which tests run (e.g. ["@smoke"]). Omit to run all ACTIVE tests.'),
437
- autoHeal: z.boolean().optional().describe("Automatically heal failing tests after each run"),
597
+ autoHeal: z.boolean().optional().describe("Heal this schedule's failed runs (default false). Takes effect only while the project's auto-heal (toggle_auto_heal) is on: a run is healed only when both are on."),
438
598
  enabled: z.boolean().optional().describe("Whether the schedule is active (default: true)"),
439
599
  runImmediately: z.boolean().optional().describe("Whether to trigger the first run immediately (default: true)")
440
600
  }
@@ -450,7 +610,7 @@ function registerAllTools(server2, client2) {
450
610
  intervalMinutes: z.union([z.literal(60), z.literal(360), z.literal(720), z.literal(1440), z.literal(2880), z.literal(10080)]).optional().describe("New interval in minutes"),
451
611
  suite: z.string().optional().describe("New suite name to filter tests"),
452
612
  tags: z.array(z.string()).optional().describe("New tag filter"),
453
- autoHeal: z.boolean().optional().describe("Toggle auto-heal"),
613
+ autoHeal: z.boolean().optional().describe("Heal this schedule's failed runs. Takes effect only while the project's auto-heal (toggle_auto_heal) is on."),
454
614
  enabled: z.boolean().optional().describe("Enable or disable the schedule")
455
615
  }
456
616
  );
@@ -492,10 +652,11 @@ function registerAllTools(server2, client2) {
492
652
  server2,
493
653
  client2,
494
654
  "create_test",
495
- "Create a new test case with Playwright code. Read project://test-conventions for templates and pitfalls BEFORE writing code. The test starts as DRAFT \u2014 auto-runs and auto-promotes to ACTIVE on pass. IMPORTANT: (1) Set suite to the EXACT feature name. (2) After creating all tests for a feature, you MUST call add_api_endpoints.",
655
+ "Create a new test case. WEB projects: provide playwrightCode (read project://test-conventions for templates BEFORE writing code; on a production BASE_URL use its read-only guard template). The response may carry warnings (weak selectors, hardcoded hosts in URL assertions, vacuous checks, a missing read-only guard) - they never block, fix them with edit_test_code. MOBILE (Appium) projects: provide steps - a structured step array is the ONLY executable form for mobile (code is never executed). The test starts as DRAFT - auto-runs and auto-promotes to ACTIVE on pass. IMPORTANT: (1) Set suite to the EXACT feature name. (2) After creating all tests for a web feature, you MUST call add_api_endpoints.",
496
656
  {
497
- name: z.string().describe('Test name (e.g. "Login \u2014 Happy Path")'),
498
- playwrightCode: z.string().describe("Full Playwright test code"),
657
+ name: z.string().describe('Test name (e.g. "Login - Happy Path")'),
658
+ playwrightCode: z.string().optional().describe("Web projects: full Playwright test code (required). Mobile projects: optional WebdriverIO export for humans - never executed."),
659
+ steps: z.array(z.record(z.unknown())).optional().describe('Mobile (Appium) projects ONLY - required. Ordered steps: { action: tap|type|clear|swipe|scroll|back|home|waitForElement|assertVisible|assertText, description, target?: { primary, fallbacks?, screenName? }, value? }. target.primary: "css=#domId" for WebView-rendered (hybrid) controls, resource-id/accessibility-id/text= only for genuinely native UI. type/assertText require value ({{TEST_VAR}} placeholders supported). Do not author OS permission-dialog taps. End with an assert step. Call get_feature first for mapped screens + known selectors.'),
499
660
  suite: z.string().optional().describe('MUST match an existing feature name exactly to link the test to that feature (e.g. "Authentication", not "Auth Tests")'),
500
661
  tags: z.array(z.string()).optional().describe('Tags (e.g. ["@smoke", "@auth"])')
501
662
  }
@@ -508,11 +669,22 @@ function registerAllTools(server2, client2) {
508
669
  { featureName: z.string().describe("The feature name (case-insensitive match)") },
509
670
  { readOnlyHint: true }
510
671
  );
672
+ tool(
673
+ server2,
674
+ client2,
675
+ "get_spec_coverage",
676
+ `Map spec items to the existing tests of a feature (heuristic: test names, tags and the copy in their code) and list the items no test covers yet. Pass the spec you were given as one item per line item, scoped by page ("Sign up: malformed email shows an error"); without specItems it uses the feature's stored test plans. Call it before you stop writing tests for a feature, then cover every uncovered item.`,
677
+ {
678
+ featureName: z.string().describe("The feature name (case-insensitive) - its suite holds the tests"),
679
+ specItems: z.array(z.string()).optional().describe("Spec items, one per behaviour and page (max 200)")
680
+ },
681
+ { readOnlyHint: true }
682
+ );
511
683
  tool(
512
684
  server2,
513
685
  client2,
514
686
  "create_feature",
515
- "Create a new feature with page routes. Read the developer's route files to provide accurate page URLs. Pages are used for coverage tracking and site map.",
687
+ "Create a new feature with page routes. Read the developer's route files to provide accurate page URLs. Pages are used for coverage tracking and site map. Idempotent, not an upsert: if an active feature with this name already exists it is returned unchanged with existing: true (use update_feature with its id to change it). Fails only when the name belongs to an archived feature (restore it with restore_feature, or pick another name).",
516
688
  {
517
689
  name: z.string().describe('Feature name (e.g. "Checkout Flow")'),
518
690
  description: z.string().optional().describe("What the feature does and what to test"),
@@ -539,7 +711,7 @@ function registerAllTools(server2, client2) {
539
711
  server2,
540
712
  client2,
541
713
  "toggle_auto_heal",
542
- "Enable or disable automatic AI healing when tests fail in batch or suite runs.",
714
+ "Enable or disable automatic AI healing when tests fail: the verification run after create_test, batch and suite runs. When off, no heal is queued automatically; heal_test still works.",
543
715
  { enabled: z.boolean().describe("true to enable auto-heal, false to disable") }
544
716
  );
545
717
  tool(
@@ -586,10 +758,17 @@ function registerAllTools(server2, client2) {
586
758
  server2,
587
759
  client2,
588
760
  "archive_feature",
589
- "Archive a feature (soft delete). Archived features are hidden from the dashboard but can be restored.",
761
+ "Archive a feature (soft delete). Archived features are hidden from the dashboard but can be restored with restore_feature.",
590
762
  { featureId: z.string().describe("The feature ID to archive") },
591
763
  { destructiveHint: true }
592
764
  );
765
+ tool(
766
+ server2,
767
+ client2,
768
+ "restore_feature",
769
+ "Restore an archived feature so it shows in the dashboard again. Does nothing for a feature that is already active. Fails if an active feature now holds the same name.",
770
+ { featureId: z.string().describe("The ID of the archived feature to restore") }
771
+ );
593
772
  tool(
594
773
  server2,
595
774
  client2,
@@ -602,15 +781,28 @@ function registerAllTools(server2, client2) {
602
781
  server2,
603
782
  client2,
604
783
  "list_bugs",
605
- "Get all detected bugs: app bugs (quarantined tests with @bug-found), security findings, and flaky tests.",
784
+ "Get all detected bugs: app bugs (quarantined tests with @bug-found), security findings, flaky tests, and visual issues flagged as bugs.",
606
785
  {},
607
786
  { readOnlyHint: true }
608
787
  );
788
+ tool(
789
+ server2,
790
+ client2,
791
+ "list_visual_issues",
792
+ "List visual issues found on the app's pages by the layout checks that run during surveys and discovery (clipped or overflowing text, covered or cut-off controls, broken images, sideways scrolling on phones, small tap targets, low contrast). Each has the page route, viewport, CSS selector, element box and a description, so you can find and fix the element in the code. Visual issues never fail tests.",
793
+ {
794
+ status: z.enum(["OPEN", "ACCEPTED", "DISMISSED", "GONE"]).optional().describe("Filter by status (default OPEN). ACCEPTED = flagged as a bug; GONE = no longer seen on the page"),
795
+ severity: z.enum(["high", "medium", "low"]).optional().describe("Filter by severity"),
796
+ viewport: z.enum(["desktop", "phone"]).optional().describe("Filter by viewport"),
797
+ route: z.string().optional().describe("Only this page route, e.g. /pricing")
798
+ },
799
+ { readOnlyHint: true }
800
+ );
609
801
  tool(
610
802
  server2,
611
803
  client2,
612
804
  "flag_as_bug",
613
- "Mark a failing test as a confirmed application bug \u2014 quarantines it with @bug-found and records the bug description. Use when the failure is caused by broken app behaviour, NOT a test script issue. Do NOT use for selector changes or timeout failures \u2014 use heal_test or edit_test_code for those.",
805
+ "Mark a failing test as a confirmed application bug - quarantines it with @bug-found and records the bug description. Use when the failure is caused by broken app behaviour, NOT a test script issue. Do NOT use for selector changes or timeout failures - use heal_test or edit_test_code for those.",
614
806
  {
615
807
  testId: z.string().describe("The failing test case ID"),
616
808
  description: z.string().describe('What is broken in the application (e.g. "POST /api/checkout returns 500 when payment method is valid")')
@@ -620,7 +812,7 @@ function registerAllTools(server2, client2) {
620
812
  server2,
621
813
  client2,
622
814
  "run_security_audit",
623
- 'Run an AI security audit on captured API traffic. Requires prior browser-based test execution (discovery or capture) \u2014 analyzes real network traffic, not just registered endpoints. If no traffic exists, the developer can trigger "Security Audit" from the Dashboard which launches a live browser to capture it.',
815
+ 'Run an AI security audit on captured API traffic. Requires prior browser-based test execution (discovery or capture) - analyzes real network traffic, not just registered endpoints. If no traffic exists, the developer can trigger "Security Audit" from the Dashboard which launches a live browser to capture it.',
624
816
  {}
625
817
  );
626
818
  tool(
@@ -638,14 +830,14 @@ function registerAllTools(server2, client2) {
638
830
  server2,
639
831
  client2,
640
832
  "set_app_brief",
641
- "Set or update the project's application brief \u2014 a description of the app used by AI for better test generation.",
833
+ "Set or update the project's application brief - a description of the app used by AI for better test generation.",
642
834
  { appBrief: z.string().describe("Application description. Set empty to clear.") }
643
835
  );
644
836
  tool(
645
837
  server2,
646
838
  client2,
647
839
  "get_coverage_summary",
648
- "Get feature-level test coverage: planned vs existing vs passing tests, plus API endpoint coverage. Shows uncoveredEndpoints per feature \u2014 use this to find which API endpoints still need tests.",
840
+ "Get feature-level test coverage: planned vs existing vs passing tests, plus API endpoint coverage. Shows uncoveredEndpoints per feature - use this to find which API endpoints still need tests.",
649
841
  {},
650
842
  { readOnlyHint: true }
651
843
  );
@@ -662,7 +854,7 @@ function registerAllTools(server2, client2) {
662
854
  server2,
663
855
  client2,
664
856
  "setup_project",
665
- 'One-shot project setup from codebase knowledge. Sets app brief, base URL, test credentials, and creates features with pages \u2014 all in one call. Use this when a developer says "set up test coverage for my app."',
857
+ 'One-shot project setup from codebase knowledge. Sets app brief, base URL, test credentials, and creates features with pages - all in one call. Use this when a developer says "set up test coverage for my app."',
666
858
  {
667
859
  appBrief: z.string().optional().describe("Description of the application (what it does, key features, tech stack)"),
668
860
  baseUrl: z.string().describe('Application base URL (e.g. "https://myapp.com" or "http://localhost:3000")'),
@@ -719,7 +911,7 @@ var config = resolveConfig();
719
911
  var client = new ValidateQAClient(config.apiUrl, config.token, config.projectId);
720
912
  var server = new McpServer(
721
913
  { name: "validate.qa", version: "1.0.0" },
722
- { instructions: `Validate.QA \u2014 AI-powered QA platform: generates, runs, and heals Playwright E2E tests.
914
+ { instructions: `Validate.QA - AI-powered QA platform: generates, runs, and heals Playwright E2E tests.
723
915
 
724
916
  ## Capabilities
725
917
  - Setup projects via setup_project (read source code first)
@@ -733,12 +925,12 @@ var server = new McpServer(
733
925
  2. Check health: list_features (failingTests > 0), get_test_failures per feature, edit_test_code or heal_test.
734
926
  3. Add coverage: READ code, create_feature, create_test x N (suite = feature name), add_api_endpoints.
735
927
  4. Nightly automation: create_schedule (daily, autoHeal: true).
736
- 5. After code change: batch_run_tests, then list_features + get_test_failures.
928
+ 5. After code change: batch_run_tests (waitSeconds: 120, or then wait_for_batch), then list_features + get_test_failures.
737
929
 
738
930
  ## Automatic lifecycle
739
- Do NOT call run_test after create_test (auto-queued) or heal_test after failures (auto-heals).
740
- Do NOT check run status in a loop \u2014 call get_test_failures after a reasonable wait to see what failed.
741
- Lifecycle: create > auto-run > pass=ACTIVE, fail=auto-heal > heal fail=@needs-review.
931
+ Do NOT call run_test after create_test (auto-queued) or heal_test after failures (auto-heals, unless toggle_auto_heal turned it off).
932
+ Do NOT poll get_test_run in a loop. To wait for a run, call wait_for_run(runId) (or wait_for_batch(batchId)); it blocks until the run finishes or timeoutSeconds passes (max 120) and returns timedOut: true if it is still going - then call it again. run_test and batch_run_tests also take waitSeconds to queue and wait in one call. A FAILED run comes back with its error, consoleLog and failedSteps.
933
+ Lifecycle: create > auto-run > pass=ACTIVE, fail=auto-heal > heal fail=@needs-review. With toggle_auto_heal(false), no failure is auto-healed; call heal_test yourself when you want one.
742
934
  DO call add_api_endpoints per feature (not automatic).
743
935
  DO call run_security_audit after UI tests to analyze captured API traffic. If it errors about missing data, tell the developer to click "Security Audit" in the Dashboard to run a live browser capture first. Use list_security_findings to review results.
744
936
 
@@ -747,34 +939,38 @@ DO call run_security_audit after UI tests to analyze captured API traffic. If it
747
939
 
748
940
  ## Triaging failures
749
941
  - App broken: flag_as_bug(testId, description). Selector/timing: heal_test. Bad test logic: edit_test_code.
750
- - Unsure: heal_test first \u2014 it detects app bugs automatically.
942
+ - Unsure: heal_test first - it detects app bugs automatically.
751
943
 
752
944
  ## Using the codebase
753
945
  You have full source code access. Always READ before writing: routes for URLs, components for selectors, API handlers for endpoints.
754
946
 
755
- ## Before writing any test \u2014 preflight check
947
+ ## Before writing any test - preflight check
756
948
 
757
949
  Read project://test-conventions first. Check the variables section:
758
950
  - BASE_URL must be set (the runner needs it). If missing, call setup_project or set_variable.
759
951
  - TEST_USER_EMAIL and TEST_USER_PASSWORD must be set if tests require login. If missing, ask the developer for credentials and call set_variable.
760
952
  - If starting fresh (no features, no variables), use workflow #1 (setup_project) before anything else.
761
953
 
762
- ## Critical rules \u2014 see project://test-conventions for full details and code templates
763
- - URLs: relative only, never hardcoded domains \u2014 runner sets baseURL
764
- - Credentials: process.env.TEST_USER_EMAIL! (non-null assertion) \u2014 never ?? or || fallbacks
954
+ ## Critical rules - see project://test-conventions for full details and code templates
955
+ - URLs: relative only, never hardcoded domains - runner sets baseURL
956
+ - Credentials: process.env.TEST_USER_EMAIL! (non-null assertion) - never ?? or || fallbacks
765
957
  - Registration: testuser-\${Date.now()}@example.com
766
958
  - Suite name must match feature name exactly (case-sensitive)
767
959
  - @auth must NOT combine with @smoke/@regression/@needs-auth/@critical
768
960
  - Each test = own browser process; @needs-auth tests MUST login inline
769
961
  - API tests: test.describe.serial() with __QA_HTTP__ log after every request
770
962
  - UI tests: console.log() milestones after key actions
771
- - create_test/edit_test_code validate code \u2014 fix warnings before moving on
963
+ - Selectors: getByRole(role, { name, exact: true }), getByLabel(label, { exact: true }), getByText(text, { exact: true }); CSS ids only for nodes with no accessible name
964
+ - URL assertions: path only - await expect(page).toHaveURL(/\\/dashboard$/) - never a host
965
+ - Production BASE_URL: read-only tests get the read-only guard (page.route aborts auth/mutation requests and fails the test) - template in project://test-conventions
966
+ - Every assertion must be able to fail: no toHaveCount(0) on transient text, no toBeHidden on an element never shown, full message text instead of a toContainText fragment
967
+ - create_test/edit_test_code validate code - fix warnings before moving on
772
968
 
773
969
  ## Setting up coverage
774
- 1. SCAN: Read router, nav, pages, API handlers \u2014 map ALL routes and features.
970
+ 1. SCAN: Read router, nav, pages, API handlers - map ALL routes and features.
775
971
  2. SETUP: setup_project with features, pages, baseUrl, testCredentials, appBrief.
776
- 3. WRITE: One feature at a time. For EACH test, read the SPECIFIC source code first (see codeReadingRequirements in test-conventions). For login blocks: read the login component. For UI tests: read the page component. For API tests: read the route handler. Templates are structure guides \u2014 selectors MUST come from the real code. Use create_test for BOTH UI and API tests. Then call add_api_endpoints.
777
- 4. VERIFY: get_coverage_summary + list_features for failures + get_test_failures.
972
+ 3. WRITE: One feature at a time. For EACH test, read the SPECIFIC source code first (see codeReadingRequirements in test-conventions). For login blocks: read the login component. For UI tests: read the page component. For API tests: read the route handler. Templates are structure guides - selectors MUST come from the real code. Use create_test for BOTH UI and API tests. Then call add_api_endpoints.
973
+ 4. VERIFY: get_spec_coverage(featureName, specItems) - write tests for every uncovered item - then get_coverage_summary + list_features for failures + get_test_failures.
778
974
 
779
975
  Test real user flows (forms, validation, errors). "Page loads" tests are worthless.` }
780
976
  );
@@ -807,22 +1003,26 @@ server.resource(
807
1003
  "project://test-conventions",
808
1004
  { description: "Project-specific test conventions: environment variables, credential patterns, tag meanings, and registration email rules. Read this before writing or editing any test." },
809
1005
  async () => {
1006
+ const served = await client.getTestConventions();
1007
+ if (served && typeof served === "object") {
1008
+ return { contents: [{ uri: "project://test-conventions", mimeType: "application/json", text: JSON.stringify(served, null, 2) }] };
1009
+ }
810
1010
  try {
811
1011
  const vars = await client.executeTool("get_variables");
812
1012
  const varNames = Object.keys(vars.variables || {});
813
1013
  const hasAuth = varNames.includes("TEST_USER_EMAIL") && varNames.includes("TEST_USER_PASSWORD");
814
1014
  const conventions = {
815
1015
  codeReadingRequirements: {
816
- CRITICAL: "Templates below are STRUCTURE GUIDES \u2014 every selector, route, and endpoint MUST come from reading the actual source code. Never copy a template verbatim.",
817
- forLogin: "BEFORE writing any @needs-auth login block: READ the login/auth page component. Find: (1) the login route (/login, /auth/login, /signin?), (2) the flow type (single-step or multi-step with Continue/Next button?), (3) exact input selectors (labels, placeholders, input types \u2014 is it getByLabel, getByPlaceholder, or input[type=password]?), (4) exact submit button text, (5) post-login redirect URL. Every app is different \u2014 single-step, multi-step, OAuth buttons, magic links.",
818
- forUITests: "BEFORE writing a UI test for a page: READ the page component. Find: headings (text + level), form fields (labels, placeholders, types), buttons (text, role), navigation elements, validation error messages, success/error states. Use the exact text from the component \u2014 do not guess.",
1016
+ CRITICAL: "Templates below are STRUCTURE GUIDES - every selector, route, and endpoint MUST come from reading the actual source code. Never copy a template verbatim.",
1017
+ forLogin: "BEFORE writing any @needs-auth login block: READ the login/auth page component. Find: (1) the login route (/login, /auth/login, /signin?), (2) the flow type (single-step or multi-step with Continue/Next button?), (3) each field's label text for getByLabel(label, { exact: true }) - fall back to the placeholder or input[type=password] only when the field has no label, (4) the submit button's accessible name for getByRole('button', { name, exact: true }), (5) post-login redirect URL. Every app is different - single-step, multi-step, OAuth buttons, magic links.",
1018
+ forUITests: "BEFORE writing a UI test for a page: READ the page component. Find: headings (text + level), form fields (labels, placeholders, types), buttons (text, role), navigation elements, validation error messages, success/error states. Use the exact text from the component - do not guess.",
819
1019
  forAPITests: "BEFORE writing an API test: READ the route handler. Find: the endpoint path, HTTP method, request body schema (what fields, what types), response body shape (is it flat or wrapped in { data: {...} }?), auth requirements (Bearer token? Cookie? API key?), error response shapes.",
820
1020
  forAPIMapping: "BEFORE calling add_api_endpoints: READ ALL route handlers for the feature. Register every endpoint with accurate method, path, requestShape, responseShapes, requiresAuth."
821
1021
  },
822
1022
  credentials: {
823
1023
  rule: "Never hardcode credentials in test code",
824
- loginFields: hasAuth ? "Use process.env.TEST_USER_EMAIL! and process.env.TEST_USER_PASSWORD!" : "No credentials configured \u2014 set TEST_USER_EMAIL and TEST_USER_PASSWORD via set_variable",
825
- registrationFields: 'MUST use unique email per run: `testuser-${Date.now()}@example.com` \u2014 reusing emails causes "already registered" failures',
1024
+ loginFields: hasAuth ? "Use process.env.TEST_USER_EMAIL! and process.env.TEST_USER_PASSWORD!" : "No credentials configured - set TEST_USER_EMAIL and TEST_USER_PASSWORD via set_variable",
1025
+ registrationFields: 'MUST use unique email per run: `testuser-${Date.now()}@example.com` - reusing emails causes "already registered" failures',
826
1026
  registrationPassword: "Use 'TestPass123!' for new account passwords (meets typical strength rules)"
827
1027
  },
828
1028
  variables: {
@@ -831,10 +1031,10 @@ server.resource(
831
1031
  },
832
1032
  tags: {
833
1033
  "@auth": "Tests the login/auth flow itself. MUST NOT combine with @smoke, @regression, @needs-auth, @critical. CAN include @discovered.",
834
- "@needs-auth": "Tests for authenticated pages \u2014 MUST include inline login block. CAN combine with @smoke, @regression, @critical.",
835
- "@smoke": "Quick sanity checks \u2014 run on every deploy",
1034
+ "@needs-auth": "Tests for authenticated pages - MUST include inline login block. CAN combine with @smoke, @regression, @critical.",
1035
+ "@smoke": "Quick sanity checks - run on every deploy",
836
1036
  "@regression": "Full regression suite tests",
837
- "@critical": "Business-critical paths (informational label \u2014 no special suite filtering)",
1037
+ "@critical": "Business-critical paths (informational label - no special suite filtering)",
838
1038
  "@api": "API-level tests. Runs in its own project, excluded from suite/smoke/regression runs",
839
1039
  "@discovered": "Auto-added to tests generated by AI discovery",
840
1040
  "@generated": "Auto-added by AI discovery and API test generation",
@@ -842,75 +1042,80 @@ server.resource(
842
1042
  "@flaky": "Auto-added after repeated failures. Excluded from suite/smoke/regression runs",
843
1043
  "@bug-found": "Added when healer detects an app bug. Test is auto-quarantined",
844
1044
  "@needs-review": "Added when DRAFT test fails healing. Requires human attention",
845
- "@heal-skip": "Added by healer when test is permanently unfixable. Prevents future auto-heal"
1045
+ "@heal-skip": "Added by healer when test is permanently unfixable. Prevents future auto-heal",
1046
+ "@rate-limit-blocked": "Auto-added by pre-heal classifier when test failure was a rate-limit (HTTP 429). Test skips heal until next pass clears the tag. NOT for use by test authors.",
1047
+ "@quota-exceeded": "Auto-added when failure is a quota/daily-limit response. Skips heal; clears on next pass.",
1048
+ "@auth-lost": "Auto-added when a signed-in test (@needs-auth, authed @api, or a stored-login body) fails on a 401/403 or falls back to the login page. Never on @auth / login-form tests. Skips heal; clears after re-auth + pass.",
1049
+ "@rate-limit-expected": 'OPT-OUT tag for tests that intentionally exercise rate-limit behavior (e.g., "POST /api/login 6 times to trigger lockout"). Add this tag to ANY @api test that asserts rate-limit responses, otherwise the rate-limit detector will silently downgrade your failure to ERROR. Distinct from auto-pushed @rate-limit-blocked.'
846
1050
  },
847
1051
  executionModel: {
848
1052
  isolation: "Each test runs as its own separate Playwright process in a fresh browser. No shared state, no storageState, no global setup.",
849
- authRequired: "Every @needs-auth test MUST include its own login block. Do NOT suggest storageState or performLogin() \u2014 those only work in the regression suite.",
850
- concurrency: "Tests run in parallel \u2014 runner claims up to 10 per poll, launched 3s apart (~30s per batch). Multiple batches can overlap. Use test.setTimeout(120_000) if the target app is slow under concurrent login load."
1053
+ authRequired: "Every @needs-auth test MUST include its own login block. Do NOT suggest storageState or performLogin() - those only work in the regression suite.",
1054
+ concurrency: "Tests run in parallel - runner claims up to 10 per poll, launched 3s apart (~30s per batch). Multiple batches can overlap. Use test.setTimeout(120_000) if the target app is slow under concurrent login load."
851
1055
  },
852
1056
  lifecycle: {
853
- DRAFT: "New or unverified test \u2014 not included in scheduled runs",
854
- ACTIVE: "Verified and running \u2014 included in batch/suite executions",
855
- DISABLED: "Manually paused \u2014 excluded from all runs",
1057
+ DRAFT: "New or unverified test - not included in scheduled runs",
1058
+ ACTIVE: "Verified and running - included in batch/suite executions",
1059
+ DISABLED: "Manually paused - excluded from all runs",
856
1060
  QUARANTINED: "Auto-quarantined after repeated failures"
857
1061
  },
858
1062
  logging: {
859
1063
  CRITICAL: "EVERY request.get/post/put/patch/delete call MUST be followed by a __QA_HTTP__ log line. This is how the platform surfaces request/response data in the Network panel. Missing this = invisible API calls.",
860
1064
  apiPattern: "const body = await res.json();\nconsole.log('__QA_HTTP__:' + JSON.stringify({ method: 'POST', url: '/api/path', status: res.status(), requestBody: { field: value, password: '[REDACTED]' }, responseBody: body }));",
861
1065
  rules: [
862
- "Call res.json() ONCE, store in body variable \u2014 reuse for log and assertions (calling twice throws)",
1066
+ "Call res.json() ONCE, store in body variable - reuse for log and assertions (calling twice throws)",
863
1067
  "Log AFTER the request, BEFORE assertions",
864
1068
  'Redact sensitive fields: password: "[REDACTED]", token: "[REDACTED]"',
865
1069
  "Use the literal URL string (same as the request call), not a variable",
866
- "UI tests using page fixture do NOT need __QA_HTTP__ \u2014 only request fixture tests"
1070
+ "UI tests using page fixture do NOT need __QA_HTTP__ - only request fixture tests"
867
1071
  ],
868
- uiMilestones: "UI tests (page fixture): add plain console.log() at key milestones \u2014 e.g. console.log('Navigated to login page') after page.goto(). These appear in the Console Output panel.",
1072
+ uiMilestones: "UI tests (page fixture): add plain console.log() at key milestones - e.g. console.log('Navigated to login page') after page.goto(). These appear in the Console Output panel.",
869
1073
  uiLoggingRules: [
870
- "Log AFTER the action, not before \u2014 confirms the action succeeded",
1074
+ "Log AFTER the action, not before - confirms the action succeeded",
871
1075
  'Include dynamic values where useful: console.log("Current URL:", page.url())',
872
- "Keep messages short and human-readable \u2014 they show verbatim in the UI",
1076
+ "Keep messages short and human-readable - they show verbatim in the UI",
873
1077
  "Do NOT log sensitive values (passwords, tokens)"
874
1078
  ],
875
- apiTestStructure: "Use test.describe.serial() for API tests sharing auth. Independent validation tests (401, 404 checks) go in a SEPARATE non-serial describe block \u2014 do not mix them with the auth-dependent serial flow."
1079
+ apiTestStructure: "Use test.describe.serial() for API tests sharing auth. Independent validation tests (401, 404 checks) go in a SEPARATE non-serial describe block - do not mix them with the auth-dependent serial flow."
876
1080
  },
877
1081
  bestPractices: [
878
1082
  "Every @needs-auth test must start with a login block (goto login, fill creds, click submit, waitForURL)",
879
- "Use { timeout: 30000 } on waitForURL after login \u2014 target apps can be slow under concurrent test load",
880
- "Keep login blocks minimal \u2014 no assertions on the login page (that is what @auth tests are for)",
881
- "After login, page.goto() directly to the page being tested \u2014 do not navigate via UI clicks",
882
- "Use role-based selectors (getByRole, getByLabel) over CSS selectors",
1083
+ "Use { timeout: 30000 } on waitForURL after login - target apps can be slow under concurrent test load",
1084
+ "Keep login blocks minimal - no assertions on the login page (that is what @auth tests are for)",
1085
+ "After login, page.goto() directly to the page being tested - do not navigate via UI clicks",
1086
+ "Selectors: getByRole(role, { name, exact: true }) for buttons/links/headings, getByLabel(label, { exact: true }) for form fields, getByText(text, { exact: true }) for static copy. CSS ids/classes only for nodes with no accessible name",
1087
+ "URL assertions: path only - await expect(page).toHaveURL(/\\/dashboard$/) - never a host",
883
1088
  "Registration tests must generate unique email with Date.now()",
884
1089
  "Group tests into suites matching feature names exactly (case-sensitive)",
885
- "Do NOT suggest storageState or performLogin() for create_test \u2014 those only work in the regression suite",
886
- "EVERY API test (request fixture) MUST include __QA_HTTP__ log after each request call \u2014 see logging section above",
887
- 'Avoid waitForLoadState("networkidle") \u2014 apps with SSE, WebSocket, or analytics connections may never reach idle, causing 60s timeouts. Wait for specific elements instead.',
888
- 'For headings, use exact: true or level: 1 \u2014 getByRole("heading", { name: "X" }) matches ALL h1-h6 containing "X", causing strict mode violations'
1090
+ "Do NOT suggest storageState or performLogin() for create_test - those only work in the regression suite",
1091
+ "EVERY API test (request fixture) MUST include __QA_HTTP__ log after each request call - see logging section above",
1092
+ 'Avoid waitForLoadState("networkidle") - apps with SSE, WebSocket, or analytics connections may never reach idle, causing 60s timeouts. Wait for specific elements instead.',
1093
+ 'For headings, use exact: true or level: 1 - getByRole("heading", { name: "X" }) matches ALL h1-h6 containing "X", causing strict mode violations'
889
1094
  ],
890
1095
  pitfalls: {
891
1096
  CRITICAL: "These are the top causes of test failures. Read before writing ANY test.",
892
- strictModeViolations: `Playwright throws when a locator resolves to 2+ elements. Common causes: getByLabel('Password') matches input AND 'Show password' toggle; getByRole('button', { name: 'Continue' }) matches 'Continue' AND 'Continue with GitHub'. Prevention: use input[type="password"] for password fields, use { exact: true } for ambiguous button text, scope to a parent container (page.locator('form#login').getByLabel('Email')).`,
893
- networkIdle: "NEVER use waitForLoadState('networkidle') \u2014 waits for 500ms of zero network activity, which is fragile and causes 30s+ timeouts in apps with polling, analytics, or WebSockets. Use waitForURL() or wait for a specific element instead.",
894
- headingStrictMode: "getByRole('heading', { name: 'Dashboard' }) can match multiple headings (h1 + h2 + breadcrumb). Add { level: 1 } or { exact: true } to disambiguate. Same for getByLabel('Password') matching input AND 'Show password' toggle.",
895
- postLoginRedirect: "NEVER assert exact URL after login (e.g. expect(page).toHaveURL('/dashboard')) \u2014 apps redirect to different routes per user/role. Use: await page.waitForURL(url => !url.pathname.startsWith('/login'), { timeout: 30000 })",
896
- resJsonOnce: "res.json() consumes the response body stream \u2014 calling it twice throws. Store result: const body = await res.json(); then reuse body for logging and assertions.",
1097
+ strictModeViolations: `Playwright throws when a locator resolves to 2+ elements. Common causes: getByLabel('Password') matches input AND 'Show password' toggle; getByRole('button', { name: 'Continue' }) matches 'Continue' AND 'Continue with GitHub'. Prevention: getByLabel('Password', { exact: true }) (exact keeps it from matching 'Show password'), { exact: true } for ambiguous button text, scope to a parent container (page.locator('form#login').getByLabel('Email', { exact: true })). Use input[type="password"] only when the field has no label.`,
1098
+ networkIdle: "NEVER use waitForLoadState('networkidle') - waits for 500ms of zero network activity, which is fragile and causes 30s+ timeouts in apps with polling, analytics, or WebSockets. Use waitForURL() or wait for a specific element instead.",
1099
+ headingStrictMode: "getByRole('heading', { name: 'Dashboard' }) can match multiple headings (h1 + h2 + breadcrumb). Add { level: 1 } or { exact: true } to disambiguate. Same fix for getByLabel('Password') matching the input AND a 'Show password' toggle: getByLabel('Password', { exact: true }).",
1100
+ postLoginRedirect: "NEVER assert exact URL after login (e.g. expect(page).toHaveURL('/dashboard')) - apps redirect to different routes per user/role. Use: await page.waitForURL(url => !url.pathname.startsWith('/login'), { timeout: 30000 })",
1101
+ resJsonOnce: "res.json() consumes the response body stream - calling it twice throws. Store result: const body = await res.json(); then reuse body for logging and assertions.",
897
1102
  hiddenInputs: "Many UI frameworks use visually hidden <input> with custom overlays. page.check() fails because the overlay intercepts clicks. Click the <label> or visible wrapper: page.getByText('Remember me').click()",
898
- apiResponseShapes: "API endpoints often wrap data in a property ({ user: {...} }, { data: [...] }). Read the route handler's res.json() call to see the actual shape. If the handler does res.json({ ticket }), then body is { ticket: { id, ... } } \u2014 NOT { id, ... }.",
899
- invalidIds: "Not all routes validate ID format before querying the database. If a route passes the raw ID to an ORM query and gets null back, it returns 404 'not found' \u2014 not 400 'bad request'. If unsure, expect either: expect([400, 404]).toContain(res.status())",
900
- hardcodedUrls: "NEVER hardcode full URLs in test code. For UI tests: always use relative paths \u2014 page.goto('/login'). The runner sets baseURL from the project's BASE_URL variable. For API tests: use relative paths if the API is on the same domain as the frontend (request.post('/api/auth/login')). If the API is on a SEPARATE domain, use process.env.BASE_API_URL! as a test variable (set via set_variable) and construct URLs: request.post(process.env.BASE_API_URL! + '/auth/login'). NEVER hardcode 'https://example.com/...' in test code \u2014 use env vars or relative paths.",
1103
+ apiResponseShapes: "API endpoints often wrap data in a property ({ user: {...} }, { data: [...] }). Read the route handler's res.json() call to see the actual shape. If the handler does res.json({ ticket }), then body is { ticket: { id, ... } } - NOT { id, ... }.",
1104
+ invalidIds: "Not all routes validate ID format before querying the database. If a route passes the raw ID to an ORM query and gets null back, it returns 404 'not found' - not 400 'bad request'. If unsure, expect either: expect([400, 404]).toContain(res.status())",
1105
+ hardcodedUrls: "NEVER hardcode full URLs in test code. For UI tests: always use relative paths - page.goto('/login'). The runner sets baseURL from the project's BASE_URL variable. For API tests: use relative paths if the API is on the same domain as the frontend (request.post('/api/auth/login')). If the API is on a SEPARATE domain, use process.env.BASE_API_URL! as a test variable (set via set_variable) and construct URLs: request.post(process.env.BASE_API_URL! + '/auth/login'). NEVER hardcode 'https://example.com/...' in test code - use env vars or relative paths.",
901
1106
  credentialFallbacks: "NEVER use process.env.TEST_USER_EMAIL ?? 'fallback' or || 'fallback'. Use ! (non-null assertion). Missing variable = test should fail immediately, not silently use hardcoded value."
902
1107
  },
903
1108
  templates: {
904
- needsAuth: `test('Feature Name \u2014 Test Name', {
1109
+ needsAuth: `test('Feature Name - Test Name', {
905
1110
  tag: ['@needs-auth', '@smoke'],
906
1111
  }, async ({ page }) => {
907
- // Login block \u2014 ADAPTED from reading the app's login page component
1112
+ // Login block - ADAPTED from reading the app's login page component
908
1113
  // YOU MUST READ the login page component BEFORE writing this block
909
1114
  await page.goto('/login'); // read the router for actual login route
910
- await page.getByRole('textbox', { name: /email/i }).fill(process.env.TEST_USER_EMAIL!);
911
- await page.locator('input[type="password"]').waitFor({ state: 'visible', timeout: 10000 });
912
- await page.locator('input[type="password"]').fill(process.env.TEST_USER_PASSWORD!);
913
- await page.getByRole('button', { name: /sign in|log in|submit/i }).click();
1115
+ await page.getByLabel('Email', { exact: true }).fill(process.env.TEST_USER_EMAIL!); // label text from the component
1116
+ await page.getByLabel('Password', { exact: true }).waitFor({ state: 'visible', timeout: 10000 });
1117
+ await page.getByLabel('Password', { exact: true }).fill(process.env.TEST_USER_PASSWORD!); // input[type="password"] only if the field has no label
1118
+ await page.getByRole('button', { name: 'Sign in', exact: true }).click(); // the button's accessible name
914
1119
  await page.waitForURL(url => !url.pathname.startsWith('/login'), { timeout: 30000 });
915
1120
  console.log('Logged in, current URL:', page.url());
916
1121
 
@@ -919,7 +1124,7 @@ server.resource(
919
1124
  console.log('On feature page');
920
1125
  await expect(page.getByRole('heading', { name: /feature name/i })).toBeVisible();
921
1126
  });`,
922
- authTest: "test('Login \u2014 Happy Path', {\n tag: ['@auth'],\n}, async ({ page }) => {\n await page.goto('/auth/login');\n // test the login form itself \u2014 selectors from reading the actual component\n});",
1127
+ authTest: "test('Login - Happy Path', {\n tag: ['@auth'],\n}, async ({ page }) => {\n await page.goto('/auth/login');\n // test the login form itself - selectors from reading the actual component\n});",
923
1128
  apiSerial: "test.describe.serial('Feature Name', () => {\n let token: string;\n test('authenticate', async ({ request }) => {\n const res = await request.post('/api/auth/login', {\n data: { email: process.env.TEST_USER_EMAIL!, password: process.env.TEST_USER_PASSWORD! }\n });\n const body = await res.json();\n console.log('__QA_HTTP__:' + JSON.stringify({\n method: 'POST', url: '/api/auth/login', status: res.status(),\n requestBody: { email: process.env.TEST_USER_EMAIL!, password: '[REDACTED]' },\n responseBody: body,\n }));\n expect(res.status()).toBe(200);\n token = body.access_token; // \u2190 adapt to your API (e.g. body.token, body.jwt)\n });\n test('get profile', async ({ request }) => {\n const res = await request.get('/api/users/profile', {\n headers: { Authorization: `Bearer ${token}` }\n });\n const body = await res.json();\n console.log('__QA_HTTP__:' + JSON.stringify({\n method: 'GET', url: '/api/users/profile', status: res.status(),\n responseBody: body,\n }));\n expect(res.status()).toBe(200);\n });\n});",
924
1129
  apiEndpointMapping: `// After reading an Express route handler like:
925
1130
  // router.post('/api/auth/login', async (req, res) => { ... })
@@ -934,24 +1139,10 @@ server.resource(
934
1139
  responseShapes: { "200": { "token": "string", "user": "object" }, "401": { "error": "string" } }
935
1140
  }]
936
1141
  }`,
937
- uiWithMilestones: `test('Feature \u2014 Happy Path', async ({ page }) => {
938
- await page.goto('/login');
939
- console.log('Navigated to login page');
940
-
941
- await page.getByRole('textbox', { name: /email/i }).fill(process.env.TEST_USER_EMAIL!);
942
- await page.locator('input[type="password"]').fill(process.env.TEST_USER_PASSWORD!);
943
- await page.getByRole('button', { name: /sign in/i }).click();
944
- console.log('Credentials submitted');
945
-
946
- await page.waitForURL(url => !url.pathname.startsWith('/login'), { timeout: 15000 });
947
- console.log('Logged in, current URL:', page.url());
948
-
949
- await page.goto('/feature-path');
950
- console.log('On feature page');
951
- });`
1142
+ uiWithMilestones: "test('Feature - Happy Path', async ({ page }) => {\n await page.goto('/login');\n console.log('Navigated to login page');\n\n await page.getByLabel('Email', { exact: true }).fill(process.env.TEST_USER_EMAIL!);\n await page.getByLabel('Password', { exact: true }).fill(process.env.TEST_USER_PASSWORD!);\n await page.getByRole('button', { name: 'Sign in', exact: true }).click();\n console.log('Credentials submitted');\n\n await page.waitForURL(url => !url.pathname.startsWith('/login'), { timeout: 15000 });\n console.log('Logged in, current URL:', page.url());\n\n await page.goto('/feature-path');\n console.log('On feature page');\n});"
952
1143
  },
953
1144
  testQuality: [
954
- 'Test actual user flows, not just "page loads" \u2014 a test that only checks goto + heading visible is nearly worthless',
1145
+ 'Test actual user flows, not just "page loads" - a test that only checks goto + heading visible is nearly worthless',
955
1146
  "Test form submission: fill fields, submit, verify success/redirect",
956
1147
  "Test validation: leave required fields empty, enter invalid data, verify error messages",
957
1148
  "Test error states: wrong password, expired session, invalid IDs",
@@ -960,14 +1151,14 @@ server.resource(
960
1151
  ],
961
1152
  securityAudit: {
962
1153
  workflow: "After creating UI tests for a feature, call run_security_audit to analyze captured API traffic for auth issues, data exposure, and vulnerabilities.",
963
- prerequisite: 'Requires prior browser-based test execution (the auto-verification run from create_test counts). If run_security_audit errors about missing data, tell the developer to click "Security Audit" in the Tests dropdown on the Dashboard \u2014 this launches a live browser to capture real network traffic (HAR) and runs analysis automatically.',
1154
+ prerequisite: 'Requires prior browser-based test execution (the auto-verification run from create_test counts). If run_security_audit errors about missing data, tell the developer to click "Security Audit" in the Tests dropdown on the Dashboard - this launches a live browser to capture real network traffic (HAR) and runs analysis automatically.',
964
1155
  reviewing: "Use list_security_findings to review results. Filter by severity (critical/high/medium/low/info) or status (open/acknowledged/resolved/false-positive). Critical and high findings should be flagged to the developer immediately."
965
1156
  },
966
1157
  keyConcepts: {
967
1158
  features: 'Group related pages and tests (e.g., "Authentication" covers /auth/login, /auth/register). MUST include page routes.',
968
- pages: 'Route URLs on a feature \u2014 used for coverage tracking and site map (e.g., ["/login", "/register"])',
969
- suites: "Group tests for batch execution \u2014 suite name MUST match feature name exactly (case-sensitive)",
970
- runner: "Executes tests \u2014 can be local (developer machine) or cloud",
1159
+ pages: 'Route URLs on a feature - used for coverage tracking and site map (e.g., ["/login", "/register"])',
1160
+ suites: "Group tests for batch execution - suite name MUST match feature name exactly (case-sensitive)",
1161
+ runner: "Executes tests - can be local (developer machine) or cloud",
971
1162
  testData: "Key-value variables injected into tests (e.g., credentials, URLs)"
972
1163
  },
973
1164
  retrievedAt: (/* @__PURE__ */ new Date()).toISOString()