@validate.qa/mcp 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -3
- package/bin.cjs +1 -1
- package/dist/index.js +304 -113
- package/dist/index.js.map +1 -1
- package/package.json +5 -3
package/dist/index.js
CHANGED
|
@@ -24,7 +24,7 @@ function parseInitArgs(argv) {
|
|
|
24
24
|
args[arg.slice(2)] = argv[++i];
|
|
25
25
|
}
|
|
26
26
|
}
|
|
27
|
-
const apiUrl = args["api-url"] ?? process.env.VALIDATEQA_API_URL ?? "https://shared
|
|
27
|
+
const apiUrl = args["api-url"] ?? process.env.VALIDATEQA_API_URL ?? "https://shared.validate.qa";
|
|
28
28
|
const token = args["token"] ?? process.env.VALIDATEQA_TOKEN;
|
|
29
29
|
const projectId = args["project-id"] ?? process.env.VALIDATEQA_PROJECT_ID;
|
|
30
30
|
if (!token) {
|
|
@@ -80,7 +80,7 @@ function runInit() {
|
|
|
80
80
|
const entry = buildMcpEntry(config2);
|
|
81
81
|
const cwd = process.cwd();
|
|
82
82
|
console.log("");
|
|
83
|
-
console.log(" Validate.QA
|
|
83
|
+
console.log(" Validate.QA - MCP Setup");
|
|
84
84
|
console.log(" \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500");
|
|
85
85
|
console.log("");
|
|
86
86
|
let anyWritten = false;
|
|
@@ -91,9 +91,9 @@ function runInit() {
|
|
|
91
91
|
}
|
|
92
92
|
const file = join(target.configDir, target.configFile);
|
|
93
93
|
if (result === "created") {
|
|
94
|
-
console.log(` \u2713 ${target.name}
|
|
94
|
+
console.log(` \u2713 ${target.name} - created ${file}`);
|
|
95
95
|
} else {
|
|
96
|
-
console.log(` \u2713 ${target.name}
|
|
96
|
+
console.log(` \u2713 ${target.name} - updated ${file}`);
|
|
97
97
|
}
|
|
98
98
|
anyWritten = true;
|
|
99
99
|
}
|
|
@@ -174,6 +174,24 @@ function resolveConfig() {
|
|
|
174
174
|
}
|
|
175
175
|
|
|
176
176
|
// src/api-client.ts
|
|
177
|
+
var PLAN_FEATURE_LOCKED_CODE = "PLAN_FEATURE_LOCKED";
|
|
178
|
+
var ApiResponseError = class extends Error {
|
|
179
|
+
status;
|
|
180
|
+
body;
|
|
181
|
+
constructor(status, body) {
|
|
182
|
+
const message = typeof body.error === "string" && body.error.length > 0 ? body.error : `API error ${status}`;
|
|
183
|
+
super(message);
|
|
184
|
+
this.name = "ApiResponseError";
|
|
185
|
+
this.status = status;
|
|
186
|
+
this.body = body;
|
|
187
|
+
}
|
|
188
|
+
};
|
|
189
|
+
function toolErrorText(err) {
|
|
190
|
+
if (err instanceof ApiResponseError && err.body.code === PLAN_FEATURE_LOCKED_CODE) {
|
|
191
|
+
return JSON.stringify(err.body);
|
|
192
|
+
}
|
|
193
|
+
return `Error: ${err instanceof Error ? err.message : String(err)}`;
|
|
194
|
+
}
|
|
177
195
|
var ValidateQAClient = class {
|
|
178
196
|
constructor(baseUrl, token, projectId) {
|
|
179
197
|
this.baseUrl = baseUrl;
|
|
@@ -199,11 +217,28 @@ var ValidateQAClient = class {
|
|
|
199
217
|
throw new Error(`Connection failed: ${msg}${cause ? ` (${cause})` : ""}`);
|
|
200
218
|
}
|
|
201
219
|
if (!res.ok) {
|
|
202
|
-
const
|
|
203
|
-
|
|
220
|
+
const parsed = await res.json().catch(() => ({ error: res.statusText }));
|
|
221
|
+
const body = parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : { error: res.statusText };
|
|
222
|
+
throw new ApiResponseError(res.status, body);
|
|
204
223
|
}
|
|
205
224
|
return res.json();
|
|
206
225
|
}
|
|
226
|
+
/** project://test-conventions from the server (GET /api/mcp/test-conventions); null when the server predates it. */
|
|
227
|
+
async getTestConventions() {
|
|
228
|
+
try {
|
|
229
|
+
const res = await fetch(`${this.baseUrl}/api/mcp/test-conventions`, {
|
|
230
|
+
headers: {
|
|
231
|
+
"Authorization": `Bearer ${this.token}`,
|
|
232
|
+
"X-Project-Id": this.projectId
|
|
233
|
+
},
|
|
234
|
+
signal: AbortSignal.timeout(3e4)
|
|
235
|
+
});
|
|
236
|
+
if (!res.ok) return null;
|
|
237
|
+
return await res.json();
|
|
238
|
+
} catch {
|
|
239
|
+
return null;
|
|
240
|
+
}
|
|
241
|
+
}
|
|
207
242
|
async healthCheck() {
|
|
208
243
|
try {
|
|
209
244
|
const res = await fetch(`${this.baseUrl}/health`, {
|
|
@@ -227,7 +262,49 @@ function tool(server2, client2, name, description, schema, annotations) {
|
|
|
227
262
|
};
|
|
228
263
|
} catch (err) {
|
|
229
264
|
return {
|
|
230
|
-
content: [{ type: "text", text:
|
|
265
|
+
content: [{ type: "text", text: toolErrorText(err) }],
|
|
266
|
+
isError: true
|
|
267
|
+
};
|
|
268
|
+
}
|
|
269
|
+
};
|
|
270
|
+
if (annotations) {
|
|
271
|
+
server2.tool(name, description, schema, annotations, handler);
|
|
272
|
+
} else {
|
|
273
|
+
server2.tool(name, description, schema, handler);
|
|
274
|
+
}
|
|
275
|
+
}
|
|
276
|
+
var SERVER_WAIT_MAX_SECONDS = 55;
|
|
277
|
+
var WAIT_DEFAULT_SECONDS = 50;
|
|
278
|
+
var WAIT_MAX_SECONDS = 120;
|
|
279
|
+
function isRecord(value) {
|
|
280
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
281
|
+
}
|
|
282
|
+
function waitSecondsFrom(raw, fallback) {
|
|
283
|
+
const n = typeof raw === "number" ? raw : Number.NaN;
|
|
284
|
+
if (!Number.isFinite(n)) return fallback;
|
|
285
|
+
return Math.min(Math.max(n, 0), WAIT_MAX_SECONDS);
|
|
286
|
+
}
|
|
287
|
+
async function executeWithWait(client2, name, args, waitKey, totalSeconds, kind) {
|
|
288
|
+
const deadline = Date.now() + totalSeconds * 1e3;
|
|
289
|
+
const first = Math.min(totalSeconds, SERVER_WAIT_MAX_SECONDS);
|
|
290
|
+
let result = await client2.executeTool(name, { ...args, [waitKey]: first });
|
|
291
|
+
for (; ; ) {
|
|
292
|
+
if (!isRecord(result) || result.timedOut !== true) return result;
|
|
293
|
+
const id = result[kind.idKey];
|
|
294
|
+
const left = Math.floor((deadline - Date.now()) / 1e3);
|
|
295
|
+
if (typeof id !== "string" || left < 1) return result;
|
|
296
|
+
result = await client2.executeTool(kind.tool, { [kind.idKey]: id, timeoutSeconds: Math.min(left, SERVER_WAIT_MAX_SECONDS) });
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
function waitTool(server2, client2, name, description, schema, waitKey, defaultSeconds, kind, annotations) {
|
|
300
|
+
const handler = async (args) => {
|
|
301
|
+
try {
|
|
302
|
+
const seconds = waitSecondsFrom(args[waitKey], defaultSeconds);
|
|
303
|
+
const result = seconds > 0 ? await executeWithWait(client2, name, args, waitKey, seconds, kind) : await client2.executeTool(name, args);
|
|
304
|
+
return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
|
|
305
|
+
} catch (err) {
|
|
306
|
+
return {
|
|
307
|
+
content: [{ type: "text", text: toolErrorText(err) }],
|
|
231
308
|
isError: true
|
|
232
309
|
};
|
|
233
310
|
}
|
|
@@ -243,7 +320,7 @@ function registerAllTools(server2, client2) {
|
|
|
243
320
|
server2,
|
|
244
321
|
client2,
|
|
245
322
|
"list_tests",
|
|
246
|
-
"List tests in the current project. Returns test name, status, suite, tags, confidence, and the last run result. Paginated
|
|
323
|
+
"List tests in the current project. Returns test name, status, suite, tags, confidence, and the last run result. Paginated - use limit/offset to page through all tests.",
|
|
247
324
|
{
|
|
248
325
|
status: z.enum(["DRAFT", "ACTIVE", "DISABLED", "QUARANTINED"]).optional().describe("Filter by test lifecycle status"),
|
|
249
326
|
suite: z.string().optional().describe("Filter by suite name"),
|
|
@@ -258,16 +335,24 @@ function registerAllTools(server2, client2) {
|
|
|
258
335
|
server2,
|
|
259
336
|
client2,
|
|
260
337
|
"get_test",
|
|
261
|
-
"Get details of a specific test: name, status, tags, steps, session, and the 5 most recent run results. Code is truncated to 200 lines
|
|
338
|
+
"Get details of a specific test: name, status, tags, steps, session, and the 5 most recent run results. Code is truncated to 200 lines - use get_test_code for the full Playwright code.",
|
|
262
339
|
{ testId: z.string().describe("The test case ID") },
|
|
263
340
|
{ readOnlyHint: true }
|
|
264
341
|
);
|
|
265
|
-
|
|
342
|
+
waitTool(
|
|
266
343
|
server2,
|
|
267
344
|
client2,
|
|
268
345
|
"run_test",
|
|
269
|
-
"Queue a test for execution. Creates a PENDING test run that will be picked up by an online runner. Returns the run ID.",
|
|
270
|
-
{
|
|
346
|
+
"Queue a test for execution. Creates a PENDING test run that will be picked up by an online runner. Returns the run ID; with waitSeconds, the finished run (as wait_for_run). For a mobile (Appium) test, pass platform to run it on the OTHER platform than it was authored on - if some steps lack a native locator for that platform, a one-time locator-discovery pass runs first (supply appId once for the target-platform build), then the test runs automatically.",
|
|
347
|
+
{
|
|
348
|
+
testId: z.string().describe("The test case ID to run"),
|
|
349
|
+
platform: z.enum(["IOS", "ANDROID"]).optional().describe("Mobile only: run this test on the given platform (cross-platform run). Omit to run on the authoring platform."),
|
|
350
|
+
appId: z.string().optional().describe("Mobile cross-platform only: the target-platform app id (bundleId/appPackage). Only needed the first time this project runs on the other platform."),
|
|
351
|
+
waitSeconds: z.number().min(0).max(WAIT_MAX_SECONDS).optional().describe(`Optional. Wait up to this many seconds (max ${WAIT_MAX_SECONDS}) for the run to finish and return its final result, like wait_for_run (timedOut: true if still going). 0 or omitted: return at once with the run ID.`)
|
|
352
|
+
},
|
|
353
|
+
"waitSeconds",
|
|
354
|
+
0,
|
|
355
|
+
{ tool: "wait_for_run", idKey: "runId" }
|
|
271
356
|
);
|
|
272
357
|
tool(
|
|
273
358
|
server2,
|
|
@@ -288,10 +373,11 @@ function registerAllTools(server2, client2) {
|
|
|
288
373
|
server2,
|
|
289
374
|
client2,
|
|
290
375
|
"edit_test_code",
|
|
291
|
-
"Update
|
|
376
|
+
"Update an existing test case. WEB (Playwright) tests: provide the full replacement playwrightCode (read project://test-conventions for patterns; code is validated - fix any warnings returned). MOBILE (Appium) tests: provide the FULL replacement steps array (playwrightCode is rejected - mobile tests execute structured steps, not code). Read the current form first with get_test_code.",
|
|
292
377
|
{
|
|
293
378
|
testId: z.string().describe("The test case ID to edit"),
|
|
294
|
-
playwrightCode: z.string().optional().describe("
|
|
379
|
+
playwrightCode: z.string().optional().describe("Web tests only: new Playwright test code (full file replacement)"),
|
|
380
|
+
steps: z.array(z.record(z.unknown())).optional().describe("Mobile (Appium) tests only: FULL replacement structured step array (same shape as create_test steps). Include every step, not just the changed ones."),
|
|
295
381
|
name: z.string().optional().describe("New test name"),
|
|
296
382
|
tags: z.array(z.string()).optional().describe('New tags array (e.g. ["@regression", "@smoke"])')
|
|
297
383
|
}
|
|
@@ -300,7 +386,7 @@ function registerAllTools(server2, client2) {
|
|
|
300
386
|
server2,
|
|
301
387
|
client2,
|
|
302
388
|
"update_test_status",
|
|
303
|
-
"Change the lifecycle status of one or more tests (e.g. promote DRAFT\u2192ACTIVE, disable a flaky test).",
|
|
389
|
+
"Change the lifecycle status of one or more tests (e.g. promote DRAFT\u2192ACTIVE, disable a flaky test). Re-activating exactly ONE bug-found test (QUARANTINED\u2192ACTIVE) also queues a verification run and returns its testRunId; with several ids no run is queued.",
|
|
304
390
|
{
|
|
305
391
|
testIds: z.array(z.string()).describe("Test IDs to update"),
|
|
306
392
|
status: z.enum(["DRAFT", "ACTIVE", "DISABLED", "QUARANTINED"]).describe("New status to set")
|
|
@@ -310,13 +396,65 @@ function registerAllTools(server2, client2) {
|
|
|
310
396
|
server2,
|
|
311
397
|
client2,
|
|
312
398
|
"get_test_run",
|
|
313
|
-
"Get the status and result of a specific test run
|
|
399
|
+
"Get the status and result of a specific test run once (to wait for it to finish, use wait_for_run, not a loop of this). Look up by runId, or the latest run for a testId. A FAILED or ERROR run also carries its failure evidence: error (up to 4 KB), consoleLog (the last 200 lines, at most 20 KB), failedSteps, and for a mobile run domSnapshot (the page source at the failure). Screenshots: get_test_screenshots. No Playwright trace is recorded.",
|
|
314
400
|
{
|
|
315
401
|
runId: z.string().optional().describe("The test run ID"),
|
|
316
402
|
testId: z.string().optional().describe("Get the most recent run for this test case ID")
|
|
317
403
|
},
|
|
318
404
|
{ readOnlyHint: true }
|
|
319
405
|
);
|
|
406
|
+
waitTool(
|
|
407
|
+
server2,
|
|
408
|
+
client2,
|
|
409
|
+
"wait_for_run",
|
|
410
|
+
`Wait until a test run (run_test, heal_test, a verification run) is PASSED, FAILED or ERROR, then return it in the get_test_run shape, failure evidence included. Returns after at most timeoutSeconds (default ${WAIT_DEFAULT_SECONDS}, max ${WAIT_MAX_SECONDS}) with timedOut: true and the current status if it is still pending or running; call it again to keep waiting. Use this instead of polling get_test_run.`,
|
|
411
|
+
{
|
|
412
|
+
runId: z.string().describe("The test run ID to wait for"),
|
|
413
|
+
timeoutSeconds: z.number().min(0).max(WAIT_MAX_SECONDS).optional().describe(`How long to wait, in seconds (default ${WAIT_DEFAULT_SECONDS}, max ${WAIT_MAX_SECONDS})`)
|
|
414
|
+
},
|
|
415
|
+
"timeoutSeconds",
|
|
416
|
+
WAIT_DEFAULT_SECONDS,
|
|
417
|
+
{ tool: "wait_for_run", idKey: "runId" },
|
|
418
|
+
{ readOnlyHint: true }
|
|
419
|
+
);
|
|
420
|
+
waitTool(
|
|
421
|
+
server2,
|
|
422
|
+
client2,
|
|
423
|
+
"wait_for_batch",
|
|
424
|
+
`Wait until a batch from batch_run_tests is COMPLETED, then return its counts (total, completed, passed, failed, errored) and, when tests failed, up to 20 failures with their runId and error. Returns after at most timeoutSeconds (default ${WAIT_DEFAULT_SECONDS}, max ${WAIT_MAX_SECONDS}) with timedOut: true and the current counts if it is still running; call it again to keep waiting. Use get_test_run on a failure's runId for its console log.`,
|
|
425
|
+
{
|
|
426
|
+
batchId: z.string().describe("The batch ID returned by batch_run_tests"),
|
|
427
|
+
timeoutSeconds: z.number().min(0).max(WAIT_MAX_SECONDS).optional().describe(`How long to wait, in seconds (default ${WAIT_DEFAULT_SECONDS}, max ${WAIT_MAX_SECONDS})`)
|
|
428
|
+
},
|
|
429
|
+
"timeoutSeconds",
|
|
430
|
+
WAIT_DEFAULT_SECONDS,
|
|
431
|
+
{ tool: "wait_for_batch", idKey: "batchId" },
|
|
432
|
+
{ readOnlyHint: true }
|
|
433
|
+
);
|
|
434
|
+
tool(
|
|
435
|
+
server2,
|
|
436
|
+
client2,
|
|
437
|
+
"run_discovery",
|
|
438
|
+
"Start AI discovery for this project. For web, discovery uses the configured base URL. For mobile, pass platform and appId to explore a native app on a connected runner/device. Use survey to map features, deep with featureName to generate tests for one feature, or full for broad generation.",
|
|
439
|
+
{
|
|
440
|
+
mode: z.enum(["survey", "deep", "full"]).optional().describe("Discovery mode. survey maps features/screens and generates no tests; deep generates tests for one feature; full explores and generates broadly."),
|
|
441
|
+
featureName: z.string().optional().describe("Required for deep mode: the feature name to generate tests for."),
|
|
442
|
+
focusAreas: z.array(z.string()).optional().describe("Optional routes, screens, or areas to focus discovery on."),
|
|
443
|
+
platform: z.enum(["IOS", "ANDROID"]).optional().describe("Mobile only: target platform. Requires appId."),
|
|
444
|
+
appId: z.string().optional().describe("Mobile only: bundle id/package id to explore. Requires platform."),
|
|
445
|
+
resetPolicy: z.enum(["AUTO", "CLEAR_APP_DATA", "PRESERVE_APP_DATA"]).optional().describe("Mobile only: app-state policy for discovery/replay. AUTO uses default safe reset behavior.")
|
|
446
|
+
}
|
|
447
|
+
);
|
|
448
|
+
tool(
|
|
449
|
+
server2,
|
|
450
|
+
client2,
|
|
451
|
+
"get_discovery_status",
|
|
452
|
+
"Check progress for a discovery run. Omit discoveryRunId to get the latest discovery run for this project.",
|
|
453
|
+
{
|
|
454
|
+
discoveryRunId: z.string().optional().describe("Specific discovery run ID. Omit for latest.")
|
|
455
|
+
},
|
|
456
|
+
{ readOnlyHint: true }
|
|
457
|
+
);
|
|
320
458
|
server2.tool(
|
|
321
459
|
"get_test_screenshots",
|
|
322
460
|
"Get screenshots from a specific test run as images the AI can actually see. Use this to visually inspect what happened during a failed or passing run.",
|
|
@@ -339,7 +477,7 @@ function registerAllTools(server2, client2) {
|
|
|
339
477
|
const content = [
|
|
340
478
|
{
|
|
341
479
|
type: "text",
|
|
342
|
-
text: `Run ${result.runId}
|
|
480
|
+
text: `Run ${result.runId} - status: ${result.status} - showing ${result.returned} of ${result.total} screenshot(s)`
|
|
343
481
|
}
|
|
344
482
|
];
|
|
345
483
|
for (const shot of result.screenshots) {
|
|
@@ -348,7 +486,7 @@ function registerAllTools(server2, client2) {
|
|
|
348
486
|
return { content };
|
|
349
487
|
} catch (err) {
|
|
350
488
|
return {
|
|
351
|
-
content: [{ type: "text", text:
|
|
489
|
+
content: [{ type: "text", text: toolErrorText(err) }],
|
|
352
490
|
isError: true
|
|
353
491
|
};
|
|
354
492
|
}
|
|
@@ -364,15 +502,19 @@ function registerAllTools(server2, client2) {
|
|
|
364
502
|
cancelBatch: z.boolean().optional().describe("If true, cancel the latest batch run instead")
|
|
365
503
|
}
|
|
366
504
|
);
|
|
367
|
-
|
|
505
|
+
waitTool(
|
|
368
506
|
server2,
|
|
369
507
|
client2,
|
|
370
508
|
"batch_run_tests",
|
|
371
|
-
"Run multiple tests from the database individually. Filter by test IDs or suite name; omit both to run all ACTIVE tests. Each test executes in its own browser.",
|
|
509
|
+
"Run multiple tests from the database individually. Filter by test IDs or suite name; omit both to run all ACTIVE tests. Each test executes in its own browser. Returns the batchId; follow it with wait_for_batch, or pass waitSeconds.",
|
|
372
510
|
{
|
|
373
511
|
testIds: z.array(z.string()).optional().describe("Specific test IDs to run"),
|
|
374
|
-
suite: z.string().optional().describe("Run all active tests in this suite")
|
|
375
|
-
|
|
512
|
+
suite: z.string().optional().describe("Run all active tests in this suite"),
|
|
513
|
+
waitSeconds: z.number().min(0).max(WAIT_MAX_SECONDS).optional().describe(`Optional. Wait up to this many seconds (max ${WAIT_MAX_SECONDS}) for the batch to complete and return the wait_for_batch result (timedOut: true if still running). 0 or omitted: return at once with the batchId.`)
|
|
514
|
+
},
|
|
515
|
+
"waitSeconds",
|
|
516
|
+
0,
|
|
517
|
+
{ tool: "wait_for_batch", idKey: "batchId" }
|
|
376
518
|
);
|
|
377
519
|
tool(
|
|
378
520
|
server2,
|
|
@@ -382,6 +524,16 @@ function registerAllTools(server2, client2) {
|
|
|
382
524
|
{},
|
|
383
525
|
{ readOnlyHint: true }
|
|
384
526
|
);
|
|
527
|
+
tool(
|
|
528
|
+
server2,
|
|
529
|
+
client2,
|
|
530
|
+
"generate_tests_for_feature",
|
|
531
|
+
"Start test generation for an existing feature by id or name. For mobile, run_discovery with mode=deep, featureName, platform, and appId gives explicit device/platform control.",
|
|
532
|
+
{
|
|
533
|
+
featureId: z.string().optional().describe("Feature ID. Either featureId or featureName is required."),
|
|
534
|
+
featureName: z.string().optional().describe("Feature name. Either featureId or featureName is required.")
|
|
535
|
+
}
|
|
536
|
+
);
|
|
385
537
|
tool(
|
|
386
538
|
server2,
|
|
387
539
|
client2,
|
|
@@ -416,6 +568,14 @@ function registerAllTools(server2, client2) {
|
|
|
416
568
|
{},
|
|
417
569
|
{ readOnlyHint: true }
|
|
418
570
|
);
|
|
571
|
+
tool(
|
|
572
|
+
server2,
|
|
573
|
+
client2,
|
|
574
|
+
"get_regression_stats",
|
|
575
|
+
"Get test counts by lifecycle status (active, quarantined, draft, disabled, total; API specs count their inner tests) and the 7-day pass rate, with rate-limit, config-issue and cancelled runs left out. The same numbers as the dashboard's regression stats.",
|
|
576
|
+
{},
|
|
577
|
+
{ readOnlyHint: true }
|
|
578
|
+
);
|
|
419
579
|
tool(
|
|
420
580
|
server2,
|
|
421
581
|
client2,
|
|
@@ -434,7 +594,7 @@ function registerAllTools(server2, client2) {
|
|
|
434
594
|
mode: z.enum(["batch"]).describe("Run mode: batch (runs project tests from the database)"),
|
|
435
595
|
intervalMinutes: z.union([z.literal(60), z.literal(360), z.literal(720), z.literal(1440), z.literal(2880), z.literal(10080)]).describe("Interval in minutes: 60 (hourly), 360 (6h), 720 (12h), 1440 (daily), 2880 (2 days), 10080 (weekly)"),
|
|
436
596
|
tags: z.array(z.string()).optional().describe('Tags to filter which tests run (e.g. ["@smoke"]). Omit to run all ACTIVE tests.'),
|
|
437
|
-
autoHeal: z.boolean().optional().describe("
|
|
597
|
+
autoHeal: z.boolean().optional().describe("Heal this schedule's failed runs (default false). Takes effect only while the project's auto-heal (toggle_auto_heal) is on: a run is healed only when both are on."),
|
|
438
598
|
enabled: z.boolean().optional().describe("Whether the schedule is active (default: true)"),
|
|
439
599
|
runImmediately: z.boolean().optional().describe("Whether to trigger the first run immediately (default: true)")
|
|
440
600
|
}
|
|
@@ -450,7 +610,7 @@ function registerAllTools(server2, client2) {
|
|
|
450
610
|
intervalMinutes: z.union([z.literal(60), z.literal(360), z.literal(720), z.literal(1440), z.literal(2880), z.literal(10080)]).optional().describe("New interval in minutes"),
|
|
451
611
|
suite: z.string().optional().describe("New suite name to filter tests"),
|
|
452
612
|
tags: z.array(z.string()).optional().describe("New tag filter"),
|
|
453
|
-
autoHeal: z.boolean().optional().describe("
|
|
613
|
+
autoHeal: z.boolean().optional().describe("Heal this schedule's failed runs. Takes effect only while the project's auto-heal (toggle_auto_heal) is on."),
|
|
454
614
|
enabled: z.boolean().optional().describe("Enable or disable the schedule")
|
|
455
615
|
}
|
|
456
616
|
);
|
|
@@ -492,10 +652,11 @@ function registerAllTools(server2, client2) {
|
|
|
492
652
|
server2,
|
|
493
653
|
client2,
|
|
494
654
|
"create_test",
|
|
495
|
-
"Create a new test case
|
|
655
|
+
"Create a new test case. WEB projects: provide playwrightCode (read project://test-conventions for templates BEFORE writing code; on a production BASE_URL use its read-only guard template). The response may carry warnings (weak selectors, hardcoded hosts in URL assertions, vacuous checks, a missing read-only guard) - they never block, fix them with edit_test_code. MOBILE (Appium) projects: provide steps - a structured step array is the ONLY executable form for mobile (code is never executed). The test starts as DRAFT - auto-runs and auto-promotes to ACTIVE on pass. IMPORTANT: (1) Set suite to the EXACT feature name. (2) After creating all tests for a web feature, you MUST call add_api_endpoints.",
|
|
496
656
|
{
|
|
497
|
-
name: z.string().describe('Test name (e.g. "Login
|
|
498
|
-
playwrightCode: z.string().describe("
|
|
657
|
+
name: z.string().describe('Test name (e.g. "Login - Happy Path")'),
|
|
658
|
+
playwrightCode: z.string().optional().describe("Web projects: full Playwright test code (required). Mobile projects: optional WebdriverIO export for humans - never executed."),
|
|
659
|
+
steps: z.array(z.record(z.unknown())).optional().describe('Mobile (Appium) projects ONLY - required. Ordered steps: { action: tap|type|clear|swipe|scroll|back|home|waitForElement|assertVisible|assertText, description, target?: { primary, fallbacks?, screenName? }, value? }. target.primary: "css=#domId" for WebView-rendered (hybrid) controls, resource-id/accessibility-id/text= only for genuinely native UI. type/assertText require value ({{TEST_VAR}} placeholders supported). Do not author OS permission-dialog taps. End with an assert step. Call get_feature first for mapped screens + known selectors.'),
|
|
499
660
|
suite: z.string().optional().describe('MUST match an existing feature name exactly to link the test to that feature (e.g. "Authentication", not "Auth Tests")'),
|
|
500
661
|
tags: z.array(z.string()).optional().describe('Tags (e.g. ["@smoke", "@auth"])')
|
|
501
662
|
}
|
|
@@ -508,11 +669,22 @@ function registerAllTools(server2, client2) {
|
|
|
508
669
|
{ featureName: z.string().describe("The feature name (case-insensitive match)") },
|
|
509
670
|
{ readOnlyHint: true }
|
|
510
671
|
);
|
|
672
|
+
tool(
|
|
673
|
+
server2,
|
|
674
|
+
client2,
|
|
675
|
+
"get_spec_coverage",
|
|
676
|
+
`Map spec items to the existing tests of a feature (heuristic: test names, tags and the copy in their code) and list the items no test covers yet. Pass the spec you were given as one item per line item, scoped by page ("Sign up: malformed email shows an error"); without specItems it uses the feature's stored test plans. Call it before you stop writing tests for a feature, then cover every uncovered item.`,
|
|
677
|
+
{
|
|
678
|
+
featureName: z.string().describe("The feature name (case-insensitive) - its suite holds the tests"),
|
|
679
|
+
specItems: z.array(z.string()).optional().describe("Spec items, one per behaviour and page (max 200)")
|
|
680
|
+
},
|
|
681
|
+
{ readOnlyHint: true }
|
|
682
|
+
);
|
|
511
683
|
tool(
|
|
512
684
|
server2,
|
|
513
685
|
client2,
|
|
514
686
|
"create_feature",
|
|
515
|
-
"Create a new feature with page routes. Read the developer's route files to provide accurate page URLs. Pages are used for coverage tracking and site map.",
|
|
687
|
+
"Create a new feature with page routes. Read the developer's route files to provide accurate page URLs. Pages are used for coverage tracking and site map. Idempotent, not an upsert: if an active feature with this name already exists it is returned unchanged with existing: true (use update_feature with its id to change it). Fails only when the name belongs to an archived feature (restore it with restore_feature, or pick another name).",
|
|
516
688
|
{
|
|
517
689
|
name: z.string().describe('Feature name (e.g. "Checkout Flow")'),
|
|
518
690
|
description: z.string().optional().describe("What the feature does and what to test"),
|
|
@@ -539,7 +711,7 @@ function registerAllTools(server2, client2) {
|
|
|
539
711
|
server2,
|
|
540
712
|
client2,
|
|
541
713
|
"toggle_auto_heal",
|
|
542
|
-
"Enable or disable automatic AI healing when tests fail
|
|
714
|
+
"Enable or disable automatic AI healing when tests fail: the verification run after create_test, batch and suite runs. When off, no heal is queued automatically; heal_test still works.",
|
|
543
715
|
{ enabled: z.boolean().describe("true to enable auto-heal, false to disable") }
|
|
544
716
|
);
|
|
545
717
|
tool(
|
|
@@ -586,10 +758,17 @@ function registerAllTools(server2, client2) {
|
|
|
586
758
|
server2,
|
|
587
759
|
client2,
|
|
588
760
|
"archive_feature",
|
|
589
|
-
"Archive a feature (soft delete). Archived features are hidden from the dashboard but can be restored.",
|
|
761
|
+
"Archive a feature (soft delete). Archived features are hidden from the dashboard but can be restored with restore_feature.",
|
|
590
762
|
{ featureId: z.string().describe("The feature ID to archive") },
|
|
591
763
|
{ destructiveHint: true }
|
|
592
764
|
);
|
|
765
|
+
tool(
|
|
766
|
+
server2,
|
|
767
|
+
client2,
|
|
768
|
+
"restore_feature",
|
|
769
|
+
"Restore an archived feature so it shows in the dashboard again. Does nothing for a feature that is already active. Fails if an active feature now holds the same name.",
|
|
770
|
+
{ featureId: z.string().describe("The ID of the archived feature to restore") }
|
|
771
|
+
);
|
|
593
772
|
tool(
|
|
594
773
|
server2,
|
|
595
774
|
client2,
|
|
@@ -602,15 +781,28 @@ function registerAllTools(server2, client2) {
|
|
|
602
781
|
server2,
|
|
603
782
|
client2,
|
|
604
783
|
"list_bugs",
|
|
605
|
-
"Get all detected bugs: app bugs (quarantined tests with @bug-found), security findings, and
|
|
784
|
+
"Get all detected bugs: app bugs (quarantined tests with @bug-found), security findings, flaky tests, and visual issues flagged as bugs.",
|
|
606
785
|
{},
|
|
607
786
|
{ readOnlyHint: true }
|
|
608
787
|
);
|
|
788
|
+
tool(
|
|
789
|
+
server2,
|
|
790
|
+
client2,
|
|
791
|
+
"list_visual_issues",
|
|
792
|
+
"List visual issues found on the app's pages by the layout checks that run during surveys and discovery (clipped or overflowing text, covered or cut-off controls, broken images, sideways scrolling on phones, small tap targets, low contrast). Each has the page route, viewport, CSS selector, element box and a description, so you can find and fix the element in the code. Visual issues never fail tests.",
|
|
793
|
+
{
|
|
794
|
+
status: z.enum(["OPEN", "ACCEPTED", "DISMISSED", "GONE"]).optional().describe("Filter by status (default OPEN). ACCEPTED = flagged as a bug; GONE = no longer seen on the page"),
|
|
795
|
+
severity: z.enum(["high", "medium", "low"]).optional().describe("Filter by severity"),
|
|
796
|
+
viewport: z.enum(["desktop", "phone"]).optional().describe("Filter by viewport"),
|
|
797
|
+
route: z.string().optional().describe("Only this page route, e.g. /pricing")
|
|
798
|
+
},
|
|
799
|
+
{ readOnlyHint: true }
|
|
800
|
+
);
|
|
609
801
|
tool(
|
|
610
802
|
server2,
|
|
611
803
|
client2,
|
|
612
804
|
"flag_as_bug",
|
|
613
|
-
"Mark a failing test as a confirmed application bug
|
|
805
|
+
"Mark a failing test as a confirmed application bug - quarantines it with @bug-found and records the bug description. Use when the failure is caused by broken app behaviour, NOT a test script issue. Do NOT use for selector changes or timeout failures - use heal_test or edit_test_code for those.",
|
|
614
806
|
{
|
|
615
807
|
testId: z.string().describe("The failing test case ID"),
|
|
616
808
|
description: z.string().describe('What is broken in the application (e.g. "POST /api/checkout returns 500 when payment method is valid")')
|
|
@@ -620,7 +812,7 @@ function registerAllTools(server2, client2) {
|
|
|
620
812
|
server2,
|
|
621
813
|
client2,
|
|
622
814
|
"run_security_audit",
|
|
623
|
-
'Run an AI security audit on captured API traffic. Requires prior browser-based test execution (discovery or capture)
|
|
815
|
+
'Run an AI security audit on captured API traffic. Requires prior browser-based test execution (discovery or capture) - analyzes real network traffic, not just registered endpoints. If no traffic exists, the developer can trigger "Security Audit" from the Dashboard which launches a live browser to capture it.',
|
|
624
816
|
{}
|
|
625
817
|
);
|
|
626
818
|
tool(
|
|
@@ -638,14 +830,14 @@ function registerAllTools(server2, client2) {
|
|
|
638
830
|
server2,
|
|
639
831
|
client2,
|
|
640
832
|
"set_app_brief",
|
|
641
|
-
"Set or update the project's application brief
|
|
833
|
+
"Set or update the project's application brief - a description of the app used by AI for better test generation.",
|
|
642
834
|
{ appBrief: z.string().describe("Application description. Set empty to clear.") }
|
|
643
835
|
);
|
|
644
836
|
tool(
|
|
645
837
|
server2,
|
|
646
838
|
client2,
|
|
647
839
|
"get_coverage_summary",
|
|
648
|
-
"Get feature-level test coverage: planned vs existing vs passing tests, plus API endpoint coverage. Shows uncoveredEndpoints per feature
|
|
840
|
+
"Get feature-level test coverage: planned vs existing vs passing tests, plus API endpoint coverage. Shows uncoveredEndpoints per feature - use this to find which API endpoints still need tests.",
|
|
649
841
|
{},
|
|
650
842
|
{ readOnlyHint: true }
|
|
651
843
|
);
|
|
@@ -662,7 +854,7 @@ function registerAllTools(server2, client2) {
|
|
|
662
854
|
server2,
|
|
663
855
|
client2,
|
|
664
856
|
"setup_project",
|
|
665
|
-
'One-shot project setup from codebase knowledge. Sets app brief, base URL, test credentials, and creates features with pages
|
|
857
|
+
'One-shot project setup from codebase knowledge. Sets app brief, base URL, test credentials, and creates features with pages - all in one call. Use this when a developer says "set up test coverage for my app."',
|
|
666
858
|
{
|
|
667
859
|
appBrief: z.string().optional().describe("Description of the application (what it does, key features, tech stack)"),
|
|
668
860
|
baseUrl: z.string().describe('Application base URL (e.g. "https://myapp.com" or "http://localhost:3000")'),
|
|
@@ -719,7 +911,7 @@ var config = resolveConfig();
|
|
|
719
911
|
var client = new ValidateQAClient(config.apiUrl, config.token, config.projectId);
|
|
720
912
|
var server = new McpServer(
|
|
721
913
|
{ name: "validate.qa", version: "1.0.0" },
|
|
722
|
-
{ instructions: `Validate.QA
|
|
914
|
+
{ instructions: `Validate.QA - AI-powered QA platform: generates, runs, and heals Playwright E2E tests.
|
|
723
915
|
|
|
724
916
|
## Capabilities
|
|
725
917
|
- Setup projects via setup_project (read source code first)
|
|
@@ -733,12 +925,12 @@ var server = new McpServer(
|
|
|
733
925
|
2. Check health: list_features (failingTests > 0), get_test_failures per feature, edit_test_code or heal_test.
|
|
734
926
|
3. Add coverage: READ code, create_feature, create_test x N (suite = feature name), add_api_endpoints.
|
|
735
927
|
4. Nightly automation: create_schedule (daily, autoHeal: true).
|
|
736
|
-
5. After code change: batch_run_tests, then list_features + get_test_failures.
|
|
928
|
+
5. After code change: batch_run_tests (waitSeconds: 120, or then wait_for_batch), then list_features + get_test_failures.
|
|
737
929
|
|
|
738
930
|
## Automatic lifecycle
|
|
739
|
-
Do NOT call run_test after create_test (auto-queued) or heal_test after failures (auto-heals).
|
|
740
|
-
Do NOT
|
|
741
|
-
Lifecycle: create > auto-run > pass=ACTIVE, fail=auto-heal > heal fail=@needs-review.
|
|
931
|
+
Do NOT call run_test after create_test (auto-queued) or heal_test after failures (auto-heals, unless toggle_auto_heal turned it off).
|
|
932
|
+
Do NOT poll get_test_run in a loop. To wait for a run, call wait_for_run(runId) (or wait_for_batch(batchId)); it blocks until the run finishes or timeoutSeconds passes (max 120) and returns timedOut: true if it is still going - then call it again. run_test and batch_run_tests also take waitSeconds to queue and wait in one call. A FAILED run comes back with its error, consoleLog and failedSteps.
|
|
933
|
+
Lifecycle: create > auto-run > pass=ACTIVE, fail=auto-heal > heal fail=@needs-review. With toggle_auto_heal(false), no failure is auto-healed; call heal_test yourself when you want one.
|
|
742
934
|
DO call add_api_endpoints per feature (not automatic).
|
|
743
935
|
DO call run_security_audit after UI tests to analyze captured API traffic. If it errors about missing data, tell the developer to click "Security Audit" in the Dashboard to run a live browser capture first. Use list_security_findings to review results.
|
|
744
936
|
|
|
@@ -747,34 +939,38 @@ DO call run_security_audit after UI tests to analyze captured API traffic. If it
|
|
|
747
939
|
|
|
748
940
|
## Triaging failures
|
|
749
941
|
- App broken: flag_as_bug(testId, description). Selector/timing: heal_test. Bad test logic: edit_test_code.
|
|
750
|
-
- Unsure: heal_test first
|
|
942
|
+
- Unsure: heal_test first - it detects app bugs automatically.
|
|
751
943
|
|
|
752
944
|
## Using the codebase
|
|
753
945
|
You have full source code access. Always READ before writing: routes for URLs, components for selectors, API handlers for endpoints.
|
|
754
946
|
|
|
755
|
-
## Before writing any test
|
|
947
|
+
## Before writing any test - preflight check
|
|
756
948
|
|
|
757
949
|
Read project://test-conventions first. Check the variables section:
|
|
758
950
|
- BASE_URL must be set (the runner needs it). If missing, call setup_project or set_variable.
|
|
759
951
|
- TEST_USER_EMAIL and TEST_USER_PASSWORD must be set if tests require login. If missing, ask the developer for credentials and call set_variable.
|
|
760
952
|
- If starting fresh (no features, no variables), use workflow #1 (setup_project) before anything else.
|
|
761
953
|
|
|
762
|
-
## Critical rules
|
|
763
|
-
- URLs: relative only, never hardcoded domains
|
|
764
|
-
- Credentials: process.env.TEST_USER_EMAIL! (non-null assertion)
|
|
954
|
+
## Critical rules - see project://test-conventions for full details and code templates
|
|
955
|
+
- URLs: relative only, never hardcoded domains - runner sets baseURL
|
|
956
|
+
- Credentials: process.env.TEST_USER_EMAIL! (non-null assertion) - never ?? or || fallbacks
|
|
765
957
|
- Registration: testuser-\${Date.now()}@example.com
|
|
766
958
|
- Suite name must match feature name exactly (case-sensitive)
|
|
767
959
|
- @auth must NOT combine with @smoke/@regression/@needs-auth/@critical
|
|
768
960
|
- Each test = own browser process; @needs-auth tests MUST login inline
|
|
769
961
|
- API tests: test.describe.serial() with __QA_HTTP__ log after every request
|
|
770
962
|
- UI tests: console.log() milestones after key actions
|
|
771
|
-
-
|
|
963
|
+
- Selectors: getByRole(role, { name, exact: true }), getByLabel(label, { exact: true }), getByText(text, { exact: true }); CSS ids only for nodes with no accessible name
|
|
964
|
+
- URL assertions: path only - await expect(page).toHaveURL(/\\/dashboard$/) - never a host
|
|
965
|
+
- Production BASE_URL: read-only tests get the read-only guard (page.route aborts auth/mutation requests and fails the test) - template in project://test-conventions
|
|
966
|
+
- Every assertion must be able to fail: no toHaveCount(0) on transient text, no toBeHidden on an element never shown, full message text instead of a toContainText fragment
|
|
967
|
+
- create_test/edit_test_code validate code - fix warnings before moving on
|
|
772
968
|
|
|
773
969
|
## Setting up coverage
|
|
774
|
-
1. SCAN: Read router, nav, pages, API handlers
|
|
970
|
+
1. SCAN: Read router, nav, pages, API handlers - map ALL routes and features.
|
|
775
971
|
2. SETUP: setup_project with features, pages, baseUrl, testCredentials, appBrief.
|
|
776
|
-
3. WRITE: One feature at a time. For EACH test, read the SPECIFIC source code first (see codeReadingRequirements in test-conventions). For login blocks: read the login component. For UI tests: read the page component. For API tests: read the route handler. Templates are structure guides
|
|
777
|
-
4. VERIFY: get_coverage_summary + list_features for failures + get_test_failures.
|
|
972
|
+
3. WRITE: One feature at a time. For EACH test, read the SPECIFIC source code first (see codeReadingRequirements in test-conventions). For login blocks: read the login component. For UI tests: read the page component. For API tests: read the route handler. Templates are structure guides - selectors MUST come from the real code. Use create_test for BOTH UI and API tests. Then call add_api_endpoints.
|
|
973
|
+
4. VERIFY: get_spec_coverage(featureName, specItems) - write tests for every uncovered item - then get_coverage_summary + list_features for failures + get_test_failures.
|
|
778
974
|
|
|
779
975
|
Test real user flows (forms, validation, errors). "Page loads" tests are worthless.` }
|
|
780
976
|
);
|
|
@@ -807,22 +1003,26 @@ server.resource(
|
|
|
807
1003
|
"project://test-conventions",
|
|
808
1004
|
{ description: "Project-specific test conventions: environment variables, credential patterns, tag meanings, and registration email rules. Read this before writing or editing any test." },
|
|
809
1005
|
async () => {
|
|
1006
|
+
const served = await client.getTestConventions();
|
|
1007
|
+
if (served && typeof served === "object") {
|
|
1008
|
+
return { contents: [{ uri: "project://test-conventions", mimeType: "application/json", text: JSON.stringify(served, null, 2) }] };
|
|
1009
|
+
}
|
|
810
1010
|
try {
|
|
811
1011
|
const vars = await client.executeTool("get_variables");
|
|
812
1012
|
const varNames = Object.keys(vars.variables || {});
|
|
813
1013
|
const hasAuth = varNames.includes("TEST_USER_EMAIL") && varNames.includes("TEST_USER_PASSWORD");
|
|
814
1014
|
const conventions = {
|
|
815
1015
|
codeReadingRequirements: {
|
|
816
|
-
CRITICAL: "Templates below are STRUCTURE GUIDES
|
|
817
|
-
forLogin: "BEFORE writing any @needs-auth login block: READ the login/auth page component. Find: (1) the login route (/login, /auth/login, /signin?), (2) the flow type (single-step or multi-step with Continue/Next button?), (3)
|
|
818
|
-
forUITests: "BEFORE writing a UI test for a page: READ the page component. Find: headings (text + level), form fields (labels, placeholders, types), buttons (text, role), navigation elements, validation error messages, success/error states. Use the exact text from the component
|
|
1016
|
+
CRITICAL: "Templates below are STRUCTURE GUIDES - every selector, route, and endpoint MUST come from reading the actual source code. Never copy a template verbatim.",
|
|
1017
|
+
forLogin: "BEFORE writing any @needs-auth login block: READ the login/auth page component. Find: (1) the login route (/login, /auth/login, /signin?), (2) the flow type (single-step or multi-step with Continue/Next button?), (3) each field's label text for getByLabel(label, { exact: true }) - fall back to the placeholder or input[type=password] only when the field has no label, (4) the submit button's accessible name for getByRole('button', { name, exact: true }), (5) post-login redirect URL. Every app is different - single-step, multi-step, OAuth buttons, magic links.",
|
|
1018
|
+
forUITests: "BEFORE writing a UI test for a page: READ the page component. Find: headings (text + level), form fields (labels, placeholders, types), buttons (text, role), navigation elements, validation error messages, success/error states. Use the exact text from the component - do not guess.",
|
|
819
1019
|
forAPITests: "BEFORE writing an API test: READ the route handler. Find: the endpoint path, HTTP method, request body schema (what fields, what types), response body shape (is it flat or wrapped in { data: {...} }?), auth requirements (Bearer token? Cookie? API key?), error response shapes.",
|
|
820
1020
|
forAPIMapping: "BEFORE calling add_api_endpoints: READ ALL route handlers for the feature. Register every endpoint with accurate method, path, requestShape, responseShapes, requiresAuth."
|
|
821
1021
|
},
|
|
822
1022
|
credentials: {
|
|
823
1023
|
rule: "Never hardcode credentials in test code",
|
|
824
|
-
loginFields: hasAuth ? "Use process.env.TEST_USER_EMAIL! and process.env.TEST_USER_PASSWORD!" : "No credentials configured
|
|
825
|
-
registrationFields: 'MUST use unique email per run: `testuser-${Date.now()}@example.com`
|
|
1024
|
+
loginFields: hasAuth ? "Use process.env.TEST_USER_EMAIL! and process.env.TEST_USER_PASSWORD!" : "No credentials configured - set TEST_USER_EMAIL and TEST_USER_PASSWORD via set_variable",
|
|
1025
|
+
registrationFields: 'MUST use unique email per run: `testuser-${Date.now()}@example.com` - reusing emails causes "already registered" failures',
|
|
826
1026
|
registrationPassword: "Use 'TestPass123!' for new account passwords (meets typical strength rules)"
|
|
827
1027
|
},
|
|
828
1028
|
variables: {
|
|
@@ -831,10 +1031,10 @@ server.resource(
|
|
|
831
1031
|
},
|
|
832
1032
|
tags: {
|
|
833
1033
|
"@auth": "Tests the login/auth flow itself. MUST NOT combine with @smoke, @regression, @needs-auth, @critical. CAN include @discovered.",
|
|
834
|
-
"@needs-auth": "Tests for authenticated pages
|
|
835
|
-
"@smoke": "Quick sanity checks
|
|
1034
|
+
"@needs-auth": "Tests for authenticated pages - MUST include inline login block. CAN combine with @smoke, @regression, @critical.",
|
|
1035
|
+
"@smoke": "Quick sanity checks - run on every deploy",
|
|
836
1036
|
"@regression": "Full regression suite tests",
|
|
837
|
-
"@critical": "Business-critical paths (informational label
|
|
1037
|
+
"@critical": "Business-critical paths (informational label - no special suite filtering)",
|
|
838
1038
|
"@api": "API-level tests. Runs in its own project, excluded from suite/smoke/regression runs",
|
|
839
1039
|
"@discovered": "Auto-added to tests generated by AI discovery",
|
|
840
1040
|
"@generated": "Auto-added by AI discovery and API test generation",
|
|
@@ -842,75 +1042,80 @@ server.resource(
|
|
|
842
1042
|
"@flaky": "Auto-added after repeated failures. Excluded from suite/smoke/regression runs",
|
|
843
1043
|
"@bug-found": "Added when healer detects an app bug. Test is auto-quarantined",
|
|
844
1044
|
"@needs-review": "Added when DRAFT test fails healing. Requires human attention",
|
|
845
|
-
"@heal-skip": "Added by healer when test is permanently unfixable. Prevents future auto-heal"
|
|
1045
|
+
"@heal-skip": "Added by healer when test is permanently unfixable. Prevents future auto-heal",
|
|
1046
|
+
"@rate-limit-blocked": "Auto-added by pre-heal classifier when test failure was a rate-limit (HTTP 429). Test skips heal until next pass clears the tag. NOT for use by test authors.",
|
|
1047
|
+
"@quota-exceeded": "Auto-added when failure is a quota/daily-limit response. Skips heal; clears on next pass.",
|
|
1048
|
+
"@auth-lost": "Auto-added when a signed-in test (@needs-auth, authed @api, or a stored-login body) fails on a 401/403 or falls back to the login page. Never on @auth / login-form tests. Skips heal; clears after re-auth + pass.",
|
|
1049
|
+
"@rate-limit-expected": 'OPT-OUT tag for tests that intentionally exercise rate-limit behavior (e.g., "POST /api/login 6 times to trigger lockout"). Add this tag to ANY @api test that asserts rate-limit responses, otherwise the rate-limit detector will silently downgrade your failure to ERROR. Distinct from auto-pushed @rate-limit-blocked.'
|
|
846
1050
|
},
|
|
847
1051
|
executionModel: {
|
|
848
1052
|
isolation: "Each test runs as its own separate Playwright process in a fresh browser. No shared state, no storageState, no global setup.",
|
|
849
|
-
authRequired: "Every @needs-auth test MUST include its own login block. Do NOT suggest storageState or performLogin()
|
|
850
|
-
concurrency: "Tests run in parallel
|
|
1053
|
+
authRequired: "Every @needs-auth test MUST include its own login block. Do NOT suggest storageState or performLogin() - those only work in the regression suite.",
|
|
1054
|
+
concurrency: "Tests run in parallel - runner claims up to 10 per poll, launched 3s apart (~30s per batch). Multiple batches can overlap. Use test.setTimeout(120_000) if the target app is slow under concurrent login load."
|
|
851
1055
|
},
|
|
852
1056
|
lifecycle: {
|
|
853
|
-
DRAFT: "New or unverified test
|
|
854
|
-
ACTIVE: "Verified and running
|
|
855
|
-
DISABLED: "Manually paused
|
|
1057
|
+
DRAFT: "New or unverified test - not included in scheduled runs",
|
|
1058
|
+
ACTIVE: "Verified and running - included in batch/suite executions",
|
|
1059
|
+
DISABLED: "Manually paused - excluded from all runs",
|
|
856
1060
|
QUARANTINED: "Auto-quarantined after repeated failures"
|
|
857
1061
|
},
|
|
858
1062
|
logging: {
|
|
859
1063
|
CRITICAL: "EVERY request.get/post/put/patch/delete call MUST be followed by a __QA_HTTP__ log line. This is how the platform surfaces request/response data in the Network panel. Missing this = invisible API calls.",
|
|
860
1064
|
apiPattern: "const body = await res.json();\nconsole.log('__QA_HTTP__:' + JSON.stringify({ method: 'POST', url: '/api/path', status: res.status(), requestBody: { field: value, password: '[REDACTED]' }, responseBody: body }));",
|
|
861
1065
|
rules: [
|
|
862
|
-
"Call res.json() ONCE, store in body variable
|
|
1066
|
+
"Call res.json() ONCE, store in body variable - reuse for log and assertions (calling twice throws)",
|
|
863
1067
|
"Log AFTER the request, BEFORE assertions",
|
|
864
1068
|
'Redact sensitive fields: password: "[REDACTED]", token: "[REDACTED]"',
|
|
865
1069
|
"Use the literal URL string (same as the request call), not a variable",
|
|
866
|
-
"UI tests using page fixture do NOT need __QA_HTTP__
|
|
1070
|
+
"UI tests using page fixture do NOT need __QA_HTTP__ - only request fixture tests"
|
|
867
1071
|
],
|
|
868
|
-
uiMilestones: "UI tests (page fixture): add plain console.log() at key milestones
|
|
1072
|
+
uiMilestones: "UI tests (page fixture): add plain console.log() at key milestones - e.g. console.log('Navigated to login page') after page.goto(). These appear in the Console Output panel.",
|
|
869
1073
|
uiLoggingRules: [
|
|
870
|
-
"Log AFTER the action, not before
|
|
1074
|
+
"Log AFTER the action, not before - confirms the action succeeded",
|
|
871
1075
|
'Include dynamic values where useful: console.log("Current URL:", page.url())',
|
|
872
|
-
"Keep messages short and human-readable
|
|
1076
|
+
"Keep messages short and human-readable - they show verbatim in the UI",
|
|
873
1077
|
"Do NOT log sensitive values (passwords, tokens)"
|
|
874
1078
|
],
|
|
875
|
-
apiTestStructure: "Use test.describe.serial() for API tests sharing auth. Independent validation tests (401, 404 checks) go in a SEPARATE non-serial describe block
|
|
1079
|
+
apiTestStructure: "Use test.describe.serial() for API tests sharing auth. Independent validation tests (401, 404 checks) go in a SEPARATE non-serial describe block - do not mix them with the auth-dependent serial flow."
|
|
876
1080
|
},
|
|
877
1081
|
bestPractices: [
|
|
878
1082
|
"Every @needs-auth test must start with a login block (goto login, fill creds, click submit, waitForURL)",
|
|
879
|
-
"Use { timeout: 30000 } on waitForURL after login
|
|
880
|
-
"Keep login blocks minimal
|
|
881
|
-
"After login, page.goto() directly to the page being tested
|
|
882
|
-
"
|
|
1083
|
+
"Use { timeout: 30000 } on waitForURL after login - target apps can be slow under concurrent test load",
|
|
1084
|
+
"Keep login blocks minimal - no assertions on the login page (that is what @auth tests are for)",
|
|
1085
|
+
"After login, page.goto() directly to the page being tested - do not navigate via UI clicks",
|
|
1086
|
+
"Selectors: getByRole(role, { name, exact: true }) for buttons/links/headings, getByLabel(label, { exact: true }) for form fields, getByText(text, { exact: true }) for static copy. CSS ids/classes only for nodes with no accessible name",
|
|
1087
|
+
"URL assertions: path only - await expect(page).toHaveURL(/\\/dashboard$/) - never a host",
|
|
883
1088
|
"Registration tests must generate unique email with Date.now()",
|
|
884
1089
|
"Group tests into suites matching feature names exactly (case-sensitive)",
|
|
885
|
-
"Do NOT suggest storageState or performLogin() for create_test
|
|
886
|
-
"EVERY API test (request fixture) MUST include __QA_HTTP__ log after each request call
|
|
887
|
-
'Avoid waitForLoadState("networkidle")
|
|
888
|
-
'For headings, use exact: true or level: 1
|
|
1090
|
+
"Do NOT suggest storageState or performLogin() for create_test - those only work in the regression suite",
|
|
1091
|
+
"EVERY API test (request fixture) MUST include __QA_HTTP__ log after each request call - see logging section above",
|
|
1092
|
+
'Avoid waitForLoadState("networkidle") - apps with SSE, WebSocket, or analytics connections may never reach idle, causing 60s timeouts. Wait for specific elements instead.',
|
|
1093
|
+
'For headings, use exact: true or level: 1 - getByRole("heading", { name: "X" }) matches ALL h1-h6 containing "X", causing strict mode violations'
|
|
889
1094
|
],
|
|
890
1095
|
pitfalls: {
|
|
891
1096
|
CRITICAL: "These are the top causes of test failures. Read before writing ANY test.",
|
|
892
|
-
strictModeViolations: `Playwright throws when a locator resolves to 2+ elements. Common causes: getByLabel('Password') matches input AND 'Show password' toggle; getByRole('button', { name: 'Continue' }) matches 'Continue' AND 'Continue with GitHub'. Prevention:
|
|
893
|
-
networkIdle: "NEVER use waitForLoadState('networkidle')
|
|
894
|
-
headingStrictMode: "getByRole('heading', { name: 'Dashboard' }) can match multiple headings (h1 + h2 + breadcrumb). Add { level: 1 } or { exact: true } to disambiguate. Same for getByLabel('Password') matching input AND 'Show password' toggle.",
|
|
895
|
-
postLoginRedirect: "NEVER assert exact URL after login (e.g. expect(page).toHaveURL('/dashboard'))
|
|
896
|
-
resJsonOnce: "res.json() consumes the response body stream
|
|
1097
|
+
strictModeViolations: `Playwright throws when a locator resolves to 2+ elements. Common causes: getByLabel('Password') matches input AND 'Show password' toggle; getByRole('button', { name: 'Continue' }) matches 'Continue' AND 'Continue with GitHub'. Prevention: getByLabel('Password', { exact: true }) (exact keeps it from matching 'Show password'), { exact: true } for ambiguous button text, scope to a parent container (page.locator('form#login').getByLabel('Email', { exact: true })). Use input[type="password"] only when the field has no label.`,
|
|
1098
|
+
networkIdle: "NEVER use waitForLoadState('networkidle') - waits for 500ms of zero network activity, which is fragile and causes 30s+ timeouts in apps with polling, analytics, or WebSockets. Use waitForURL() or wait for a specific element instead.",
|
|
1099
|
+
headingStrictMode: "getByRole('heading', { name: 'Dashboard' }) can match multiple headings (h1 + h2 + breadcrumb). Add { level: 1 } or { exact: true } to disambiguate. Same fix for getByLabel('Password') matching the input AND a 'Show password' toggle: getByLabel('Password', { exact: true }).",
|
|
1100
|
+
postLoginRedirect: "NEVER assert exact URL after login (e.g. expect(page).toHaveURL('/dashboard')) - apps redirect to different routes per user/role. Use: await page.waitForURL(url => !url.pathname.startsWith('/login'), { timeout: 30000 })",
|
|
1101
|
+
resJsonOnce: "res.json() consumes the response body stream - calling it twice throws. Store result: const body = await res.json(); then reuse body for logging and assertions.",
|
|
897
1102
|
hiddenInputs: "Many UI frameworks use visually hidden <input> with custom overlays. page.check() fails because the overlay intercepts clicks. Click the <label> or visible wrapper: page.getByText('Remember me').click()",
|
|
898
|
-
apiResponseShapes: "API endpoints often wrap data in a property ({ user: {...} }, { data: [...] }). Read the route handler's res.json() call to see the actual shape. If the handler does res.json({ ticket }), then body is { ticket: { id, ... } }
|
|
899
|
-
invalidIds: "Not all routes validate ID format before querying the database. If a route passes the raw ID to an ORM query and gets null back, it returns 404 'not found'
|
|
900
|
-
hardcodedUrls: "NEVER hardcode full URLs in test code. For UI tests: always use relative paths
|
|
1103
|
+
apiResponseShapes: "API endpoints often wrap data in a property ({ user: {...} }, { data: [...] }). Read the route handler's res.json() call to see the actual shape. If the handler does res.json({ ticket }), then body is { ticket: { id, ... } } - NOT { id, ... }.",
|
|
1104
|
+
invalidIds: "Not all routes validate ID format before querying the database. If a route passes the raw ID to an ORM query and gets null back, it returns 404 'not found' - not 400 'bad request'. If unsure, expect either: expect([400, 404]).toContain(res.status())",
|
|
1105
|
+
hardcodedUrls: "NEVER hardcode full URLs in test code. For UI tests: always use relative paths - page.goto('/login'). The runner sets baseURL from the project's BASE_URL variable. For API tests: use relative paths if the API is on the same domain as the frontend (request.post('/api/auth/login')). If the API is on a SEPARATE domain, use process.env.BASE_API_URL! as a test variable (set via set_variable) and construct URLs: request.post(process.env.BASE_API_URL! + '/auth/login'). NEVER hardcode 'https://example.com/...' in test code - use env vars or relative paths.",
|
|
901
1106
|
credentialFallbacks: "NEVER use process.env.TEST_USER_EMAIL ?? 'fallback' or || 'fallback'. Use ! (non-null assertion). Missing variable = test should fail immediately, not silently use hardcoded value."
|
|
902
1107
|
},
|
|
903
1108
|
templates: {
|
|
904
|
-
needsAuth: `test('Feature Name
|
|
1109
|
+
needsAuth: `test('Feature Name - Test Name', {
|
|
905
1110
|
tag: ['@needs-auth', '@smoke'],
|
|
906
1111
|
}, async ({ page }) => {
|
|
907
|
-
// Login block
|
|
1112
|
+
// Login block - ADAPTED from reading the app's login page component
|
|
908
1113
|
// YOU MUST READ the login page component BEFORE writing this block
|
|
909
1114
|
await page.goto('/login'); // read the router for actual login route
|
|
910
|
-
await page.
|
|
911
|
-
await page.
|
|
912
|
-
await page.
|
|
913
|
-
await page.getByRole('button', { name:
|
|
1115
|
+
await page.getByLabel('Email', { exact: true }).fill(process.env.TEST_USER_EMAIL!); // label text from the component
|
|
1116
|
+
await page.getByLabel('Password', { exact: true }).waitFor({ state: 'visible', timeout: 10000 });
|
|
1117
|
+
await page.getByLabel('Password', { exact: true }).fill(process.env.TEST_USER_PASSWORD!); // input[type="password"] only if the field has no label
|
|
1118
|
+
await page.getByRole('button', { name: 'Sign in', exact: true }).click(); // the button's accessible name
|
|
914
1119
|
await page.waitForURL(url => !url.pathname.startsWith('/login'), { timeout: 30000 });
|
|
915
1120
|
console.log('Logged in, current URL:', page.url());
|
|
916
1121
|
|
|
@@ -919,7 +1124,7 @@ server.resource(
|
|
|
919
1124
|
console.log('On feature page');
|
|
920
1125
|
await expect(page.getByRole('heading', { name: /feature name/i })).toBeVisible();
|
|
921
1126
|
});`,
|
|
922
|
-
authTest: "test('Login
|
|
1127
|
+
authTest: "test('Login - Happy Path', {\n tag: ['@auth'],\n}, async ({ page }) => {\n await page.goto('/auth/login');\n // test the login form itself - selectors from reading the actual component\n});",
|
|
923
1128
|
apiSerial: "test.describe.serial('Feature Name', () => {\n let token: string;\n test('authenticate', async ({ request }) => {\n const res = await request.post('/api/auth/login', {\n data: { email: process.env.TEST_USER_EMAIL!, password: process.env.TEST_USER_PASSWORD! }\n });\n const body = await res.json();\n console.log('__QA_HTTP__:' + JSON.stringify({\n method: 'POST', url: '/api/auth/login', status: res.status(),\n requestBody: { email: process.env.TEST_USER_EMAIL!, password: '[REDACTED]' },\n responseBody: body,\n }));\n expect(res.status()).toBe(200);\n token = body.access_token; // \u2190 adapt to your API (e.g. body.token, body.jwt)\n });\n test('get profile', async ({ request }) => {\n const res = await request.get('/api/users/profile', {\n headers: { Authorization: `Bearer ${token}` }\n });\n const body = await res.json();\n console.log('__QA_HTTP__:' + JSON.stringify({\n method: 'GET', url: '/api/users/profile', status: res.status(),\n responseBody: body,\n }));\n expect(res.status()).toBe(200);\n });\n});",
|
|
924
1129
|
apiEndpointMapping: `// After reading an Express route handler like:
|
|
925
1130
|
// router.post('/api/auth/login', async (req, res) => { ... })
|
|
@@ -934,24 +1139,10 @@ server.resource(
|
|
|
934
1139
|
responseShapes: { "200": { "token": "string", "user": "object" }, "401": { "error": "string" } }
|
|
935
1140
|
}]
|
|
936
1141
|
}`,
|
|
937
|
-
uiWithMilestones:
|
|
938
|
-
await page.goto('/login');
|
|
939
|
-
console.log('Navigated to login page');
|
|
940
|
-
|
|
941
|
-
await page.getByRole('textbox', { name: /email/i }).fill(process.env.TEST_USER_EMAIL!);
|
|
942
|
-
await page.locator('input[type="password"]').fill(process.env.TEST_USER_PASSWORD!);
|
|
943
|
-
await page.getByRole('button', { name: /sign in/i }).click();
|
|
944
|
-
console.log('Credentials submitted');
|
|
945
|
-
|
|
946
|
-
await page.waitForURL(url => !url.pathname.startsWith('/login'), { timeout: 15000 });
|
|
947
|
-
console.log('Logged in, current URL:', page.url());
|
|
948
|
-
|
|
949
|
-
await page.goto('/feature-path');
|
|
950
|
-
console.log('On feature page');
|
|
951
|
-
});`
|
|
1142
|
+
uiWithMilestones: "test('Feature - Happy Path', async ({ page }) => {\n await page.goto('/login');\n console.log('Navigated to login page');\n\n await page.getByLabel('Email', { exact: true }).fill(process.env.TEST_USER_EMAIL!);\n await page.getByLabel('Password', { exact: true }).fill(process.env.TEST_USER_PASSWORD!);\n await page.getByRole('button', { name: 'Sign in', exact: true }).click();\n console.log('Credentials submitted');\n\n await page.waitForURL(url => !url.pathname.startsWith('/login'), { timeout: 15000 });\n console.log('Logged in, current URL:', page.url());\n\n await page.goto('/feature-path');\n console.log('On feature page');\n});"
|
|
952
1143
|
},
|
|
953
1144
|
testQuality: [
|
|
954
|
-
'Test actual user flows, not just "page loads"
|
|
1145
|
+
'Test actual user flows, not just "page loads" - a test that only checks goto + heading visible is nearly worthless',
|
|
955
1146
|
"Test form submission: fill fields, submit, verify success/redirect",
|
|
956
1147
|
"Test validation: leave required fields empty, enter invalid data, verify error messages",
|
|
957
1148
|
"Test error states: wrong password, expired session, invalid IDs",
|
|
@@ -960,14 +1151,14 @@ server.resource(
|
|
|
960
1151
|
],
|
|
961
1152
|
securityAudit: {
|
|
962
1153
|
workflow: "After creating UI tests for a feature, call run_security_audit to analyze captured API traffic for auth issues, data exposure, and vulnerabilities.",
|
|
963
|
-
prerequisite: 'Requires prior browser-based test execution (the auto-verification run from create_test counts). If run_security_audit errors about missing data, tell the developer to click "Security Audit" in the Tests dropdown on the Dashboard
|
|
1154
|
+
prerequisite: 'Requires prior browser-based test execution (the auto-verification run from create_test counts). If run_security_audit errors about missing data, tell the developer to click "Security Audit" in the Tests dropdown on the Dashboard - this launches a live browser to capture real network traffic (HAR) and runs analysis automatically.',
|
|
964
1155
|
reviewing: "Use list_security_findings to review results. Filter by severity (critical/high/medium/low/info) or status (open/acknowledged/resolved/false-positive). Critical and high findings should be flagged to the developer immediately."
|
|
965
1156
|
},
|
|
966
1157
|
keyConcepts: {
|
|
967
1158
|
features: 'Group related pages and tests (e.g., "Authentication" covers /auth/login, /auth/register). MUST include page routes.',
|
|
968
|
-
pages: 'Route URLs on a feature
|
|
969
|
-
suites: "Group tests for batch execution
|
|
970
|
-
runner: "Executes tests
|
|
1159
|
+
pages: 'Route URLs on a feature - used for coverage tracking and site map (e.g., ["/login", "/register"])',
|
|
1160
|
+
suites: "Group tests for batch execution - suite name MUST match feature name exactly (case-sensitive)",
|
|
1161
|
+
runner: "Executes tests - can be local (developer machine) or cloud",
|
|
971
1162
|
testData: "Key-value variables injected into tests (e.g., credentials, URLs)"
|
|
972
1163
|
},
|
|
973
1164
|
retrievedAt: (/* @__PURE__ */ new Date()).toISOString()
|