@validate.qa/mcp 1.0.1 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +64 -11
- package/dist/index.js +1158 -185
- package/dist/index.js.map +1 -1
- package/package.json +6 -5
package/dist/index.js
CHANGED
|
@@ -1,21 +1,81 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
var __defProp = Object.defineProperty;
|
|
3
3
|
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
4
|
-
var __esm = (fn, res) => function __init() {
|
|
5
|
-
|
|
4
|
+
var __esm = (fn, res, err) => function __init() {
|
|
5
|
+
if (err) throw err[0];
|
|
6
|
+
try {
|
|
7
|
+
return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
|
|
8
|
+
} catch (e) {
|
|
9
|
+
throw err = [e], e;
|
|
10
|
+
}
|
|
6
11
|
};
|
|
7
12
|
var __export = (target, all) => {
|
|
8
13
|
for (var name in all)
|
|
9
14
|
__defProp(target, name, { get: all[name], enumerable: true });
|
|
10
15
|
};
|
|
11
16
|
|
|
17
|
+
// src/token-store.ts
|
|
18
|
+
import { chmodSync, mkdirSync, readFileSync, renameSync, writeFileSync } from "fs";
|
|
19
|
+
import { homedir } from "os";
|
|
20
|
+
import { join } from "path";
|
|
21
|
+
function defaultTokenStoreDir() {
|
|
22
|
+
return join(homedir(), ".validateqa");
|
|
23
|
+
}
|
|
24
|
+
function tokenStorePath(dir = defaultTokenStoreDir()) {
|
|
25
|
+
return join(dir, "credentials.json");
|
|
26
|
+
}
|
|
27
|
+
function storeKey(apiUrl, projectId) {
|
|
28
|
+
try {
|
|
29
|
+
return `${new URL(apiUrl).origin} ${projectId}`;
|
|
30
|
+
} catch {
|
|
31
|
+
return null;
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
function readTokenFile(dir) {
|
|
35
|
+
try {
|
|
36
|
+
const parsed = JSON.parse(readFileSync(tokenStorePath(dir), "utf-8"));
|
|
37
|
+
if (parsed && typeof parsed.tokens === "object" && parsed.tokens !== null) {
|
|
38
|
+
return { version: 1, tokens: parsed.tokens };
|
|
39
|
+
}
|
|
40
|
+
} catch {
|
|
41
|
+
}
|
|
42
|
+
return { version: 1, tokens: {} };
|
|
43
|
+
}
|
|
44
|
+
function readStoredToken(apiUrl, projectId, dir = defaultTokenStoreDir()) {
|
|
45
|
+
const key = storeKey(apiUrl, projectId);
|
|
46
|
+
if (!key) return void 0;
|
|
47
|
+
const token = readTokenFile(dir).tokens[key];
|
|
48
|
+
return typeof token === "string" && token ? token : void 0;
|
|
49
|
+
}
|
|
50
|
+
function saveToken(apiUrl, projectId, token, dir = defaultTokenStoreDir()) {
|
|
51
|
+
const key = storeKey(apiUrl, projectId);
|
|
52
|
+
if (!key) throw new Error(`Invalid API URL: ${apiUrl}`);
|
|
53
|
+
mkdirSync(dir, { recursive: true, mode: 448 });
|
|
54
|
+
const file = readTokenFile(dir);
|
|
55
|
+
file.tokens[key] = token;
|
|
56
|
+
const path = tokenStorePath(dir);
|
|
57
|
+
const tmp = `${path}.${process.pid}.tmp`;
|
|
58
|
+
writeFileSync(tmp, JSON.stringify(file, null, 2) + "\n", { encoding: "utf-8", mode: 384 });
|
|
59
|
+
renameSync(tmp, path);
|
|
60
|
+
chmodSync(path, 384);
|
|
61
|
+
return path;
|
|
62
|
+
}
|
|
63
|
+
var init_token_store = __esm({
|
|
64
|
+
"src/token-store.ts"() {
|
|
65
|
+
"use strict";
|
|
66
|
+
}
|
|
67
|
+
});
|
|
68
|
+
|
|
12
69
|
// src/init.ts
|
|
13
70
|
var init_exports = {};
|
|
14
71
|
__export(init_exports, {
|
|
72
|
+
buildMcpEntry: () => buildMcpEntry,
|
|
73
|
+
initProject: () => initProject,
|
|
74
|
+
parseInitArgs: () => parseInitArgs,
|
|
15
75
|
runInit: () => runInit
|
|
16
76
|
});
|
|
17
|
-
import { existsSync, mkdirSync, readFileSync as
|
|
18
|
-
import { resolve as resolve2, join } from "path";
|
|
77
|
+
import { appendFileSync, existsSync, mkdirSync as mkdirSync2, readFileSync as readFileSync3, writeFileSync as writeFileSync2 } from "fs";
|
|
78
|
+
import { resolve as resolve2, join as join2 } from "path";
|
|
19
79
|
function parseInitArgs(argv) {
|
|
20
80
|
const args = {};
|
|
21
81
|
for (let i = 0; i < argv.length; i++) {
|
|
@@ -24,7 +84,7 @@ function parseInitArgs(argv) {
|
|
|
24
84
|
args[arg.slice(2)] = argv[++i];
|
|
25
85
|
}
|
|
26
86
|
}
|
|
27
|
-
const apiUrl = args["api-url"] ?? process.env.VALIDATEQA_API_URL ?? "https://shared
|
|
87
|
+
const apiUrl = args["api-url"] ?? process.env.VALIDATEQA_API_URL ?? "https://shared.validate.qa";
|
|
28
88
|
const token = args["token"] ?? process.env.VALIDATEQA_TOKEN;
|
|
29
89
|
const projectId = args["project-id"] ?? process.env.VALIDATEQA_PROJECT_ID;
|
|
30
90
|
if (!token) {
|
|
@@ -37,7 +97,7 @@ function parseInitArgs(argv) {
|
|
|
37
97
|
}
|
|
38
98
|
return { apiUrl: apiUrl.replace(/\/$/, ""), token, projectId };
|
|
39
99
|
}
|
|
40
|
-
function buildMcpEntry(config2) {
|
|
100
|
+
function buildMcpEntry(config2, opts) {
|
|
41
101
|
return {
|
|
42
102
|
command: "npx",
|
|
43
103
|
args: [
|
|
@@ -45,8 +105,7 @@ function buildMcpEntry(config2) {
|
|
|
45
105
|
"@validate.qa/mcp@latest",
|
|
46
106
|
"--api-url",
|
|
47
107
|
config2.apiUrl,
|
|
48
|
-
"--token",
|
|
49
|
-
config2.token,
|
|
108
|
+
...opts.withToken ? ["--token", config2.token] : [],
|
|
50
109
|
"--project-id",
|
|
51
110
|
config2.projectId
|
|
52
111
|
]
|
|
@@ -61,7 +120,7 @@ function writeConfigForIDE(cwd, target, entry) {
|
|
|
61
120
|
let existing = {};
|
|
62
121
|
if (existsSync(filePath)) {
|
|
63
122
|
try {
|
|
64
|
-
existing = JSON.parse(
|
|
123
|
+
existing = JSON.parse(readFileSync3(filePath, "utf-8"));
|
|
65
124
|
} catch {
|
|
66
125
|
}
|
|
67
126
|
}
|
|
@@ -70,39 +129,61 @@ function writeConfigForIDE(cwd, target, entry) {
|
|
|
70
129
|
servers["validate.qa"] = entry;
|
|
71
130
|
existing[target.serversKey] = servers;
|
|
72
131
|
if (!existsSync(dir)) {
|
|
73
|
-
|
|
132
|
+
mkdirSync2(dir, { recursive: true });
|
|
74
133
|
}
|
|
75
|
-
|
|
134
|
+
writeFileSync2(filePath, JSON.stringify(existing, null, 2) + "\n", "utf-8");
|
|
76
135
|
return hadEntry ? "updated" : "created";
|
|
77
136
|
}
|
|
137
|
+
function ignoreInGit(cwd, file) {
|
|
138
|
+
const gitignore = resolve2(cwd, ".gitignore");
|
|
139
|
+
const line = "/" + file.replace(/\\/g, "/").replace(/^\.\//, "");
|
|
140
|
+
const current = existsSync(gitignore) ? readFileSync3(gitignore, "utf-8") : "";
|
|
141
|
+
if (current.split(/\r?\n/).includes(line)) return;
|
|
142
|
+
appendFileSync(gitignore, `${current && !current.endsWith("\n") ? "\n" : ""}${line}
|
|
143
|
+
`, "utf-8");
|
|
144
|
+
}
|
|
145
|
+
function initProject(cwd, config2, storeDir = defaultTokenStoreDir()) {
|
|
146
|
+
let tokenFile = null;
|
|
147
|
+
let tokenStoreError;
|
|
148
|
+
try {
|
|
149
|
+
tokenFile = saveToken(config2.apiUrl, config2.projectId, config2.token, storeDir);
|
|
150
|
+
} catch (err) {
|
|
151
|
+
tokenStoreError = err instanceof Error ? err.message : String(err);
|
|
152
|
+
}
|
|
153
|
+
const entry = buildMcpEntry(config2, { withToken: tokenFile === null });
|
|
154
|
+
const written = [];
|
|
155
|
+
for (const target of IDE_TARGETS) {
|
|
156
|
+
const result = writeConfigForIDE(cwd, target, entry);
|
|
157
|
+
if (result === "skipped") continue;
|
|
158
|
+
written.push({ name: target.name, file: join2(target.configDir, target.configFile), result });
|
|
159
|
+
}
|
|
160
|
+
if (written.length === 0) {
|
|
161
|
+
const target = IDE_TARGETS[0];
|
|
162
|
+
const result = writeConfigForIDE(cwd, target, entry);
|
|
163
|
+
if (result !== "skipped") written.push({ name: target.name, file: target.configFile, result });
|
|
164
|
+
}
|
|
165
|
+
if (tokenFile === null) {
|
|
166
|
+
for (const { file } of written) ignoreInGit(cwd, file);
|
|
167
|
+
}
|
|
168
|
+
return { written, tokenFile, ...tokenStoreError ? { tokenStoreError } : {} };
|
|
169
|
+
}
|
|
78
170
|
function runInit() {
|
|
79
171
|
const config2 = parseInitArgs(process.argv.slice(3));
|
|
80
|
-
const entry = buildMcpEntry(config2);
|
|
81
|
-
const cwd = process.cwd();
|
|
82
172
|
console.log("");
|
|
83
173
|
console.log(" Validate.QA - MCP Setup");
|
|
84
174
|
console.log(" \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500");
|
|
85
175
|
console.log("");
|
|
86
|
-
|
|
87
|
-
for (const
|
|
88
|
-
|
|
89
|
-
if (result === "skipped") {
|
|
90
|
-
continue;
|
|
91
|
-
}
|
|
92
|
-
const file = join(target.configDir, target.configFile);
|
|
93
|
-
if (result === "created") {
|
|
94
|
-
console.log(` \u2713 ${target.name} - created ${file}`);
|
|
95
|
-
} else {
|
|
96
|
-
console.log(` \u2713 ${target.name} - updated ${file}`);
|
|
97
|
-
}
|
|
98
|
-
anyWritten = true;
|
|
176
|
+
const { written, tokenFile, tokenStoreError } = initProject(process.cwd(), config2);
|
|
177
|
+
for (const { name, file, result } of written) {
|
|
178
|
+
console.log(` \u2713 ${name} - ${result} ${file}`);
|
|
99
179
|
}
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
console.log(
|
|
104
|
-
|
|
105
|
-
console.log(
|
|
180
|
+
console.log("");
|
|
181
|
+
if (tokenFile) {
|
|
182
|
+
console.log(` Your token is saved in ${tokenFile} (only you can read it), not in these files,`);
|
|
183
|
+
console.log(" so they are safe to commit. Teammates run their own init command.");
|
|
184
|
+
} else {
|
|
185
|
+
console.log(` \u26A0 Could not save your token outside the repo (${tokenStoreError}).`);
|
|
186
|
+
console.log(" The config files above hold it, so they were added to .gitignore. Do not commit them.");
|
|
106
187
|
}
|
|
107
188
|
console.log("");
|
|
108
189
|
console.log(" Next: restart your IDE or reload MCP servers.");
|
|
@@ -113,6 +194,7 @@ var IDE_TARGETS;
|
|
|
113
194
|
var init_init = __esm({
|
|
114
195
|
"src/init.ts"() {
|
|
115
196
|
"use strict";
|
|
197
|
+
init_token_store();
|
|
116
198
|
IDE_TARGETS = [
|
|
117
199
|
{ name: "Claude Code", configDir: ".", configFile: ".mcp.json", serversKey: "mcpServers" },
|
|
118
200
|
{ name: "Cursor", configDir: ".cursor", configFile: "mcp.json", serversKey: "mcpServers" },
|
|
@@ -122,12 +204,292 @@ var init_init = __esm({
|
|
|
122
204
|
}
|
|
123
205
|
});
|
|
124
206
|
|
|
207
|
+
// src/agent-rules.ts
|
|
208
|
+
var AGENT_RULES;
|
|
209
|
+
var init_agent_rules = __esm({
|
|
210
|
+
"src/agent-rules.ts"() {
|
|
211
|
+
"use strict";
|
|
212
|
+
AGENT_RULES = `## Validate.QA Rules
|
|
213
|
+
|
|
214
|
+
Use the \`validate.qa\` MCP server alongside Playwright so explored flows and saved regression tests run and stay verified in Validate.QA.
|
|
215
|
+
|
|
216
|
+
### Loop Workflow
|
|
217
|
+
1. Call \`start_here\` first (\`start_here({ feature })\`, or \`start_here({})\` to list features) to read the feature's pages, existing tests, \`knownSelectors\`, credential readiness, and \`openBugs\`. Skip bugs already listed in \`openBugs\`.
|
|
218
|
+
2. Save each bug or regression test with \`save_regression_test\` using Playwright code that asserts the **correct** behaviour (fails while the bug exists, passes once fixed). \`save_regression_test\` returns immediately while the run executes in the cloud \u2014 keep exploring without waiting.
|
|
219
|
+
3. Call \`collect_verdicts\` at the end of the session. \`reproduces\` (the test fails on its check) means the bug is locked in: keep that test as it is. \`passes\` means the test misses the bug: check again in your browser and fix its assertion. \`test_problem\` means it broke before its check: fix the steps and keep the expected value. Fix with \`edit_test_code\`, then call \`collect_verdicts\` again.
|
|
220
|
+
|
|
221
|
+
### Test Code Rules
|
|
222
|
+
- Never weaken an assertion to make a test pass; always assert the exact correct behaviour.
|
|
223
|
+
- One deterministic \`expect()\` per outcome: cover one bug or outcome per test and end with a deterministic \`expect()\` on the exact value or state the app should produce.
|
|
224
|
+
- Relative URLs only (\`await page.goto('/orders')\`, path-only URL checks) \u2014 never hardcode a host or scheme; the runner sets \`baseURL\`.
|
|
225
|
+
- Read credentials from \`process.env\` (\`process.env.TEST_USER_EMAIL!\` and \`process.env.TEST_USER_PASSWORD!\` with non-null assertions) \u2014 never hardcode literals or add \`??\` / \`||\` fallbacks.
|
|
226
|
+
- Prefer verified locators from \`knownSelectors\`, \`getByRole(role, { name, exact: true })\`, \`getByLabel\`, and \`getByTestId\`; never use \`waitForLoadState('networkidle')\`.
|
|
227
|
+
|
|
228
|
+
### Healer
|
|
229
|
+
- A heal may change locators and waits, but never removes or loosens an \`expect()\`.
|
|
230
|
+
- If the app is wrong, do not alter the expected value or silence the test with \`test.fixme()\` \u2014 flag it as a bug with \`flag_as_bug\` instead.`;
|
|
231
|
+
}
|
|
232
|
+
});
|
|
233
|
+
|
|
234
|
+
// src/init-agents.ts
|
|
235
|
+
var init_agents_exports = {};
|
|
236
|
+
__export(init_agents_exports, {
|
|
237
|
+
RULES_END_MARKER: () => RULES_END_MARKER,
|
|
238
|
+
RULES_START_MARKER: () => RULES_START_MARKER,
|
|
239
|
+
VALID_LOOPS: () => VALID_LOOPS,
|
|
240
|
+
applyRulesSection: () => applyRulesSection,
|
|
241
|
+
initAgents: () => initAgents,
|
|
242
|
+
parseInitAgentsArgs: () => parseInitAgentsArgs,
|
|
243
|
+
runInitAgents: () => runInitAgents,
|
|
244
|
+
spawnPlaywrightInitAgents: () => spawnPlaywrightInitAgents,
|
|
245
|
+
validateLoop: () => validateLoop
|
|
246
|
+
});
|
|
247
|
+
import { spawn } from "child_process";
|
|
248
|
+
import { existsSync as existsSync2, mkdirSync as mkdirSync3, readdirSync, readFileSync as readFileSync4, statSync, writeFileSync as writeFileSync3 } from "fs";
|
|
249
|
+
import { dirname, join as join3, resolve as resolve3 } from "path";
|
|
250
|
+
function validateLoop(loop) {
|
|
251
|
+
if (VALID_LOOPS.includes(loop)) {
|
|
252
|
+
return loop;
|
|
253
|
+
}
|
|
254
|
+
throw new Error(`Unknown --loop "${loop}". Expected one of: ${VALID_LOOPS.join(", ")}.`);
|
|
255
|
+
}
|
|
256
|
+
function parseInitAgentsArgs(argv, cwd = process.cwd()) {
|
|
257
|
+
let rawLoop;
|
|
258
|
+
let rawDir;
|
|
259
|
+
for (let i = 0; i < argv.length; i++) {
|
|
260
|
+
const arg = argv[i];
|
|
261
|
+
if (arg === "--loop") {
|
|
262
|
+
const next = argv[i + 1];
|
|
263
|
+
if (next === void 0 || next.startsWith("--")) {
|
|
264
|
+
throw new Error(`Missing value for --loop. Expected one of: ${VALID_LOOPS.join(", ")}.`);
|
|
265
|
+
}
|
|
266
|
+
rawLoop = next;
|
|
267
|
+
i++;
|
|
268
|
+
} else if (arg.startsWith("--loop=")) {
|
|
269
|
+
rawLoop = arg.slice("--loop=".length);
|
|
270
|
+
} else if (arg === "--dir") {
|
|
271
|
+
const next = argv[i + 1];
|
|
272
|
+
if (next === void 0 || next.startsWith("--")) {
|
|
273
|
+
throw new Error("Missing value for --dir.");
|
|
274
|
+
}
|
|
275
|
+
rawDir = next;
|
|
276
|
+
i++;
|
|
277
|
+
} else if (arg.startsWith("--dir=")) {
|
|
278
|
+
rawDir = arg.slice("--dir=".length);
|
|
279
|
+
if (!rawDir) throw new Error("Missing value for --dir.");
|
|
280
|
+
} else {
|
|
281
|
+
throw new Error(`Unknown argument "${arg}". Usage: init-agents [--loop claude|vscode|opencode] [--dir <repo>]`);
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
return {
|
|
285
|
+
loop: validateLoop(rawLoop ?? "claude"),
|
|
286
|
+
dir: rawDir ? resolve3(cwd, rawDir) : resolve3(cwd)
|
|
287
|
+
};
|
|
288
|
+
}
|
|
289
|
+
function spawnPlaywrightInitAgents({ dir, loop }) {
|
|
290
|
+
const command = process.platform === "win32" ? "npx.cmd" : "npx";
|
|
291
|
+
const args = ["playwright", "init-agents", `--loop=${loop}`];
|
|
292
|
+
return new Promise((resolvePromise, rejectPromise) => {
|
|
293
|
+
const child = spawn(command, args, { cwd: dir, stdio: "inherit", shell: process.platform === "win32" });
|
|
294
|
+
child.on("error", (err) => {
|
|
295
|
+
rejectPromise(new Error(`Failed to run npx playwright init-agents --loop=${loop}: ${err.message}`));
|
|
296
|
+
});
|
|
297
|
+
child.on("close", (code, signal) => {
|
|
298
|
+
if (code === 0) {
|
|
299
|
+
resolvePromise();
|
|
300
|
+
} else {
|
|
301
|
+
const detail = code !== null ? `exit code ${code}` : `signal ${signal}`;
|
|
302
|
+
rejectPromise(new Error(`npx playwright init-agents --loop=${loop} failed (${detail}).`));
|
|
303
|
+
}
|
|
304
|
+
});
|
|
305
|
+
});
|
|
306
|
+
}
|
|
307
|
+
function escapeRegExp(value) {
|
|
308
|
+
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
309
|
+
}
|
|
310
|
+
function applyRulesSection(content, rules = AGENT_RULES) {
|
|
311
|
+
const block = `${RULES_START_MARKER}
|
|
312
|
+
${rules.trim()}
|
|
313
|
+
${RULES_END_MARKER}`;
|
|
314
|
+
const delimitedPattern = new RegExp(
|
|
315
|
+
`${escapeRegExp(RULES_START_MARKER)}[\\s\\S]*?${escapeRegExp(RULES_END_MARKER)}`,
|
|
316
|
+
"g"
|
|
317
|
+
);
|
|
318
|
+
if (delimitedPattern.test(content)) {
|
|
319
|
+
let replacedFirst = false;
|
|
320
|
+
const updated = content.replace(delimitedPattern, () => {
|
|
321
|
+
if (replacedFirst) return "";
|
|
322
|
+
replacedFirst = true;
|
|
323
|
+
return block;
|
|
324
|
+
});
|
|
325
|
+
return `${updated.replace(/\s+$/, "")}
|
|
326
|
+
`;
|
|
327
|
+
}
|
|
328
|
+
if (content.includes(RULES_START_MARKER)) {
|
|
329
|
+
const start = content.indexOf(RULES_START_MARKER);
|
|
330
|
+
const before = content.slice(0, start).replace(/\s+$/, "");
|
|
331
|
+
return before ? `${before}
|
|
332
|
+
|
|
333
|
+
${block}
|
|
334
|
+
` : `${block}
|
|
335
|
+
`;
|
|
336
|
+
}
|
|
337
|
+
const trimmed = content.replace(/\s+$/, "");
|
|
338
|
+
return trimmed ? `${trimmed}
|
|
339
|
+
|
|
340
|
+
${block}
|
|
341
|
+
` : `${block}
|
|
342
|
+
`;
|
|
343
|
+
}
|
|
344
|
+
function candidateAgentDirs(loop) {
|
|
345
|
+
switch (loop) {
|
|
346
|
+
case "claude":
|
|
347
|
+
return [".claude/agents"];
|
|
348
|
+
case "vscode":
|
|
349
|
+
return [".github/agents", ".github/chatmodes"];
|
|
350
|
+
case "opencode":
|
|
351
|
+
return [".opencode/prompts", ".opencode/agents"];
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
function findAgentFiles(dir, loop) {
|
|
355
|
+
for (const relDir of candidateAgentDirs(loop)) {
|
|
356
|
+
const fullDir = resolve3(dir, relDir);
|
|
357
|
+
if (!existsSync2(fullDir) || !statSync(fullDir).isDirectory()) continue;
|
|
358
|
+
const matches = readdirSync(fullDir).filter((entry) => entry.endsWith(".md") && AGENT_FILE_RE.test(entry)).sort().map((entry) => join3(fullDir, entry));
|
|
359
|
+
if (matches.length > 0) return matches;
|
|
360
|
+
}
|
|
361
|
+
return [];
|
|
362
|
+
}
|
|
363
|
+
function mcpTargetForLoop(loop) {
|
|
364
|
+
switch (loop) {
|
|
365
|
+
case "claude":
|
|
366
|
+
return {
|
|
367
|
+
relativePath: ".mcp.json",
|
|
368
|
+
serversKey: "mcpServers",
|
|
369
|
+
entry: {
|
|
370
|
+
command: "npx",
|
|
371
|
+
args: ["-y", "@validate.qa/mcp"],
|
|
372
|
+
env: { VALIDATEQA_TOOLSET: "agent" }
|
|
373
|
+
}
|
|
374
|
+
};
|
|
375
|
+
case "vscode":
|
|
376
|
+
return {
|
|
377
|
+
relativePath: join3(".vscode", "mcp.json"),
|
|
378
|
+
serversKey: "servers",
|
|
379
|
+
entry: {
|
|
380
|
+
type: "stdio",
|
|
381
|
+
command: "npx",
|
|
382
|
+
args: ["-y", "@validate.qa/mcp"],
|
|
383
|
+
env: { VALIDATEQA_TOOLSET: "agent" }
|
|
384
|
+
}
|
|
385
|
+
};
|
|
386
|
+
case "opencode":
|
|
387
|
+
return {
|
|
388
|
+
relativePath: "opencode.json",
|
|
389
|
+
serversKey: "mcp",
|
|
390
|
+
entry: {
|
|
391
|
+
type: "local",
|
|
392
|
+
command: ["npx", "-y", "@validate.qa/mcp"],
|
|
393
|
+
environment: { VALIDATEQA_TOOLSET: "agent" },
|
|
394
|
+
enabled: true
|
|
395
|
+
}
|
|
396
|
+
};
|
|
397
|
+
}
|
|
398
|
+
}
|
|
399
|
+
function isPlainObject(value) {
|
|
400
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
401
|
+
}
|
|
402
|
+
function readJsonObject(filePath) {
|
|
403
|
+
if (!existsSync2(filePath)) return {};
|
|
404
|
+
let parsed;
|
|
405
|
+
try {
|
|
406
|
+
parsed = JSON.parse(readFileSync4(filePath, "utf-8"));
|
|
407
|
+
} catch {
|
|
408
|
+
parsed = void 0;
|
|
409
|
+
}
|
|
410
|
+
if (!isPlainObject(parsed)) {
|
|
411
|
+
throw new Error(
|
|
412
|
+
`${filePath} is not a plain JSON object (comments and trailing commas are not supported), so init-agents left it untouched. Make it plain JSON, or move it aside, and run init-agents again.`
|
|
413
|
+
);
|
|
414
|
+
}
|
|
415
|
+
return parsed;
|
|
416
|
+
}
|
|
417
|
+
async function initAgents(options = {}) {
|
|
418
|
+
const loop = validateLoop(options.loop ?? "claude");
|
|
419
|
+
const dir = options.dir ? resolve3(options.dir) : resolve3(process.cwd());
|
|
420
|
+
const runPlaywright = options.runPlaywright ?? spawnPlaywrightInitAgents;
|
|
421
|
+
const log = options.log ?? console.log;
|
|
422
|
+
if (!existsSync2(dir) || !statSync(dir).isDirectory()) {
|
|
423
|
+
throw new Error(`Target directory does not exist: ${dir}`);
|
|
424
|
+
}
|
|
425
|
+
const mcpTarget = mcpTargetForLoop(loop);
|
|
426
|
+
const mcpConfigFile = join3(dir, mcpTarget.relativePath);
|
|
427
|
+
const beforeDoc = readJsonObject(mcpConfigFile);
|
|
428
|
+
const beforeServers = isPlainObject(beforeDoc[mcpTarget.serversKey]) ? { ...beforeDoc[mcpTarget.serversKey] } : {};
|
|
429
|
+
await runPlaywright({ dir, loop });
|
|
430
|
+
const agentFiles = findAgentFiles(dir, loop);
|
|
431
|
+
if (agentFiles.length === 0) {
|
|
432
|
+
throw new Error(`No Playwright agent files (planner, generator, healer) found in ${dir} for loop "${loop}".`);
|
|
433
|
+
}
|
|
434
|
+
for (const filePath of agentFiles) {
|
|
435
|
+
const current = readFileSync4(filePath, "utf-8");
|
|
436
|
+
writeFileSync3(filePath, applyRulesSection(current, AGENT_RULES), "utf-8");
|
|
437
|
+
}
|
|
438
|
+
const afterDoc = readJsonObject(mcpConfigFile);
|
|
439
|
+
const afterServers = isPlainObject(afterDoc[mcpTarget.serversKey]) ? afterDoc[mcpTarget.serversKey] : {};
|
|
440
|
+
const mergedServers = { ...beforeServers, ...afterServers };
|
|
441
|
+
if (Object.prototype.hasOwnProperty.call(beforeServers, "validate.qa")) {
|
|
442
|
+
mergedServers["validate.qa"] = beforeServers["validate.qa"];
|
|
443
|
+
}
|
|
444
|
+
const hadValidateQa = Object.prototype.hasOwnProperty.call(mergedServers, "validate.qa");
|
|
445
|
+
if (!hadValidateQa) {
|
|
446
|
+
mergedServers["validate.qa"] = mcpTarget.entry;
|
|
447
|
+
}
|
|
448
|
+
const mergedDoc = {
|
|
449
|
+
...beforeDoc,
|
|
450
|
+
...afterDoc,
|
|
451
|
+
[mcpTarget.serversKey]: mergedServers
|
|
452
|
+
};
|
|
453
|
+
mkdirSync3(dirname(mcpConfigFile), { recursive: true });
|
|
454
|
+
writeFileSync3(mcpConfigFile, `${JSON.stringify(mergedDoc, null, 2)}
|
|
455
|
+
`, "utf-8");
|
|
456
|
+
log("No token was written. The server reads its project from .validateqa.json and its token from your token store: run the init command from your Validate.QA dashboard once, or set VALIDATEQA_TOKEN, VALIDATEQA_PROJECT_ID and VALIDATEQA_API_URL.");
|
|
457
|
+
return {
|
|
458
|
+
loop,
|
|
459
|
+
dir,
|
|
460
|
+
agentFiles,
|
|
461
|
+
mcpConfigFile,
|
|
462
|
+
mcpEntryStatus: hadValidateQa ? "preserved" : "added"
|
|
463
|
+
};
|
|
464
|
+
}
|
|
465
|
+
async function runInitAgents(argv = process.argv.slice(3)) {
|
|
466
|
+
try {
|
|
467
|
+
const parsed = parseInitAgentsArgs(argv);
|
|
468
|
+
await initAgents(parsed);
|
|
469
|
+
} catch (err) {
|
|
470
|
+
console.error(err instanceof Error ? err.message : String(err));
|
|
471
|
+
process.exit(1);
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
var VALID_LOOPS, RULES_START_MARKER, RULES_END_MARKER, AGENT_FILE_RE;
|
|
475
|
+
var init_init_agents = __esm({
|
|
476
|
+
"src/init-agents.ts"() {
|
|
477
|
+
"use strict";
|
|
478
|
+
init_agent_rules();
|
|
479
|
+
VALID_LOOPS = ["claude", "vscode", "opencode"];
|
|
480
|
+
RULES_START_MARKER = "<!-- validate.qa:start -->";
|
|
481
|
+
RULES_END_MARKER = "<!-- validate.qa:end -->";
|
|
482
|
+
AGENT_FILE_RE = /(?:^|[^a-z])(planner|generator|healer)(?:[^a-z]|$)/i;
|
|
483
|
+
}
|
|
484
|
+
});
|
|
485
|
+
|
|
125
486
|
// src/index.ts
|
|
126
487
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
127
488
|
import { StdioServerTransport } from "@modelcontextprotocol/sdk/server/stdio.js";
|
|
128
489
|
|
|
129
490
|
// src/config.ts
|
|
130
|
-
|
|
491
|
+
init_token_store();
|
|
492
|
+
import { readFileSync as readFileSync2 } from "fs";
|
|
131
493
|
import { resolve } from "path";
|
|
132
494
|
function parseArgs(argv) {
|
|
133
495
|
const args = {};
|
|
@@ -142,7 +504,7 @@ function parseArgs(argv) {
|
|
|
142
504
|
}
|
|
143
505
|
function loadDotFile() {
|
|
144
506
|
try {
|
|
145
|
-
const content =
|
|
507
|
+
const content = readFileSync2(resolve(process.cwd(), ".validateqa.json"), "utf-8");
|
|
146
508
|
const json = JSON.parse(content);
|
|
147
509
|
return {
|
|
148
510
|
apiUrl: typeof json.apiUrl === "string" ? json.apiUrl : void 0,
|
|
@@ -152,34 +514,62 @@ function loadDotFile() {
|
|
|
152
514
|
return {};
|
|
153
515
|
}
|
|
154
516
|
}
|
|
517
|
+
function toolsetFrom(value) {
|
|
518
|
+
if (value === void 0 || value === "" || value === "full") return "full";
|
|
519
|
+
if (value === "agent") return "agent";
|
|
520
|
+
console.error(`Unknown toolset "${value}" (--toolset, VALIDATEQA_TOOLSET): expected "agent" or "full". Serving every tool.`);
|
|
521
|
+
return "full";
|
|
522
|
+
}
|
|
155
523
|
function resolveConfig() {
|
|
156
524
|
const cliArgs = parseArgs(process.argv.slice(2));
|
|
157
525
|
const dotFile = loadDotFile();
|
|
158
526
|
const apiUrl = cliArgs["api-url"] ?? process.env.VALIDATEQA_API_URL ?? dotFile.apiUrl;
|
|
159
|
-
const token = cliArgs["token"] ?? process.env.VALIDATEQA_TOKEN;
|
|
160
527
|
const projectId = cliArgs["project-id"] ?? process.env.VALIDATEQA_PROJECT_ID ?? dotFile.projectId;
|
|
528
|
+
const token = cliArgs["token"] ?? process.env.VALIDATEQA_TOKEN ?? (apiUrl && projectId ? readStoredToken(apiUrl, projectId) : void 0);
|
|
161
529
|
if (!apiUrl) {
|
|
162
530
|
console.error("Missing required config: api-url (--api-url, VALIDATEQA_API_URL, or .validateqa.json)");
|
|
163
531
|
process.exit(1);
|
|
164
532
|
}
|
|
165
533
|
if (!token) {
|
|
166
|
-
console.error("Missing required config: token (--token or
|
|
534
|
+
console.error("Missing required config: token (--token, VALIDATEQA_TOKEN, or run the init command from your Validate.QA dashboard again)");
|
|
167
535
|
process.exit(1);
|
|
168
536
|
}
|
|
169
537
|
if (!projectId) {
|
|
170
538
|
console.error("Missing required config: project-id (--project-id, VALIDATEQA_PROJECT_ID, or .validateqa.json)");
|
|
171
539
|
process.exit(1);
|
|
172
540
|
}
|
|
173
|
-
|
|
541
|
+
const toolset = toolsetFrom(cliArgs["toolset"] ?? process.env.VALIDATEQA_TOOLSET);
|
|
542
|
+
return { apiUrl: apiUrl.replace(/\/$/, ""), token, projectId, toolset };
|
|
174
543
|
}
|
|
175
544
|
|
|
176
545
|
// src/api-client.ts
|
|
546
|
+
var PLAN_FEATURE_LOCKED_CODE = "PLAN_FEATURE_LOCKED";
|
|
547
|
+
var ApiResponseError = class extends Error {
|
|
548
|
+
status;
|
|
549
|
+
body;
|
|
550
|
+
constructor(status, body) {
|
|
551
|
+
const message = typeof body.error === "string" && body.error.length > 0 ? body.error : `API error ${status}`;
|
|
552
|
+
super(message);
|
|
553
|
+
this.name = "ApiResponseError";
|
|
554
|
+
this.status = status;
|
|
555
|
+
this.body = body;
|
|
556
|
+
}
|
|
557
|
+
};
|
|
558
|
+
function toolErrorText(err) {
|
|
559
|
+
if (err instanceof ApiResponseError && err.body.code === PLAN_FEATURE_LOCKED_CODE) {
|
|
560
|
+
return JSON.stringify(err.body);
|
|
561
|
+
}
|
|
562
|
+
return `Error: ${err instanceof Error ? err.message : String(err)}`;
|
|
563
|
+
}
|
|
177
564
|
var ValidateQAClient = class {
|
|
178
565
|
constructor(baseUrl, token, projectId) {
|
|
179
566
|
this.baseUrl = baseUrl;
|
|
180
567
|
this.token = token;
|
|
181
568
|
this.projectId = projectId;
|
|
182
569
|
}
|
|
570
|
+
baseUrl;
|
|
571
|
+
token;
|
|
572
|
+
projectId;
|
|
183
573
|
async executeTool(tool2, args = {}) {
|
|
184
574
|
let res;
|
|
185
575
|
try {
|
|
@@ -199,11 +589,28 @@ var ValidateQAClient = class {
|
|
|
199
589
|
throw new Error(`Connection failed: ${msg}${cause ? ` (${cause})` : ""}`);
|
|
200
590
|
}
|
|
201
591
|
if (!res.ok) {
|
|
202
|
-
const
|
|
203
|
-
|
|
592
|
+
const parsed = await res.json().catch(() => ({ error: res.statusText }));
|
|
593
|
+
const body = parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : { error: res.statusText };
|
|
594
|
+
throw new ApiResponseError(res.status, body);
|
|
204
595
|
}
|
|
205
596
|
return res.json();
|
|
206
597
|
}
|
|
598
|
+
/** project://test-conventions from the server (GET /api/mcp/test-conventions); null when the server predates it. */
|
|
599
|
+
async getTestConventions() {
|
|
600
|
+
try {
|
|
601
|
+
const res = await fetch(`${this.baseUrl}/api/mcp/test-conventions`, {
|
|
602
|
+
headers: {
|
|
603
|
+
"Authorization": `Bearer ${this.token}`,
|
|
604
|
+
"X-Project-Id": this.projectId
|
|
605
|
+
},
|
|
606
|
+
signal: AbortSignal.timeout(3e4)
|
|
607
|
+
});
|
|
608
|
+
if (!res.ok) return null;
|
|
609
|
+
return await res.json();
|
|
610
|
+
} catch {
|
|
611
|
+
return null;
|
|
612
|
+
}
|
|
613
|
+
}
|
|
207
614
|
async healthCheck() {
|
|
208
615
|
try {
|
|
209
616
|
const res = await fetch(`${this.baseUrl}/health`, {
|
|
@@ -217,7 +624,397 @@ var ValidateQAClient = class {
|
|
|
217
624
|
};
|
|
218
625
|
|
|
219
626
|
// src/tools/index.ts
|
|
627
|
+
import { z as z2 } from "zod";
|
|
628
|
+
|
|
629
|
+
// src/tools/agent-tools.ts
|
|
220
630
|
import { z } from "zod";
|
|
631
|
+
var AGENT_TOOL_NAMES = /* @__PURE__ */ new Set([
|
|
632
|
+
"start_here",
|
|
633
|
+
"get_feature",
|
|
634
|
+
"save_regression_test",
|
|
635
|
+
"collect_verdicts",
|
|
636
|
+
"edit_test_code",
|
|
637
|
+
"get_test_code",
|
|
638
|
+
"list_tests",
|
|
639
|
+
"get_test_run",
|
|
640
|
+
"get_test_screenshots",
|
|
641
|
+
"list_bugs",
|
|
642
|
+
"flag_as_bug",
|
|
643
|
+
"delete_test"
|
|
644
|
+
]);
|
|
645
|
+
function limitTools(server2, names) {
|
|
646
|
+
return new Proxy(server2, {
|
|
647
|
+
get(target, prop, receiver) {
|
|
648
|
+
if (prop !== "tool") return Reflect.get(target, prop, receiver);
|
|
649
|
+
return (name, ...rest) => names.has(name) ? target.tool.call(target, name, ...rest) : void 0;
|
|
650
|
+
}
|
|
651
|
+
});
|
|
652
|
+
}
|
|
653
|
+
var AgentSession = class {
|
|
654
|
+
saved = /* @__PURE__ */ new Map();
|
|
655
|
+
remember(test) {
|
|
656
|
+
this.saved.set(test.testId, test);
|
|
657
|
+
}
|
|
658
|
+
/** An edited test's last run checked the old code: collect_verdicts runs it again. */
|
|
659
|
+
markEdited(testId) {
|
|
660
|
+
const test = this.saved.get(testId);
|
|
661
|
+
if (test) test.edited = true;
|
|
662
|
+
}
|
|
663
|
+
/**
|
|
664
|
+
* `client`, noting each edit_test_code call so collect_verdicts knows which runs are out of date. An edit of a
|
|
665
|
+
* test saved with save_regression_test is linted as a bug test, like its save (assertsBug).
|
|
666
|
+
*/
|
|
667
|
+
track(client2) {
|
|
668
|
+
return new Proxy(client2, {
|
|
669
|
+
get: (target, prop, receiver) => {
|
|
670
|
+
if (prop !== "executeTool") return Reflect.get(target, prop, receiver);
|
|
671
|
+
return async (tool2, args) => {
|
|
672
|
+
const edited = tool2 === "edit_test_code" && typeof args?.testId === "string" ? args.testId : void 0;
|
|
673
|
+
const result = await target.executeTool(tool2, edited && this.saved.has(edited) ? { ...args, assertsBug: true } : args);
|
|
674
|
+
if (edited) this.markEdited(edited);
|
|
675
|
+
return result;
|
|
676
|
+
};
|
|
677
|
+
}
|
|
678
|
+
});
|
|
679
|
+
}
|
|
680
|
+
};
|
|
681
|
+
var ANSI = /\u001b\[[0-9;]*m/g;
|
|
682
|
+
var NOT_FOUND = /element\(s\) not found|resolved to 0 elements/i;
|
|
683
|
+
var ACTION_TIMEOUT = /Test timeout of \d+ms exceeded|\b(locator|page)\.\w+: Timeout \d+ms exceeded/;
|
|
684
|
+
var INFRA = /net::ERR_|ECONNREFUSED|ECONNRESET|ENOTFOUND|browserType\.launch|Target (page, context or browser|closed)|runner (lost|disconnected)/i;
|
|
685
|
+
var CHECK_MAX = 300;
|
|
686
|
+
function clip(text, max = CHECK_MAX) {
|
|
687
|
+
return text.length > max ? `${text.slice(0, max - 1)}\u2026` : text;
|
|
688
|
+
}
|
|
689
|
+
function checkLine(error) {
|
|
690
|
+
const lines = error.replace(ANSI, "").split("\n").map((line) => line.trim()).filter(Boolean);
|
|
691
|
+
const check = lines.find((line) => /\bexpect\(/.test(line)) ?? lines[0] ?? "";
|
|
692
|
+
const expected = lines.find((line) => /^Expected\b/.test(line) && line !== check);
|
|
693
|
+
const received = lines.find((line) => /^Received\b/.test(line));
|
|
694
|
+
return clip([check, expected, received].filter(Boolean).join(" | "));
|
|
695
|
+
}
|
|
696
|
+
function failedMatcher(error) {
|
|
697
|
+
const m = /\bexpect\([^)]*\)\.(not\.)?(to\w+)\(/.exec(error.replace(ANSI, ""));
|
|
698
|
+
return m ? `${m[1] ?? ""}${m[2]}` : null;
|
|
699
|
+
}
|
|
700
|
+
var MATCHER = /\.\s*(not\s*\.\s*)?(to(?:Be|Have|Equal|StrictEqual|Contain|Match|Throw|Pass)\w*)\s*\(/g;
|
|
701
|
+
function stripComments(code) {
|
|
702
|
+
let out = "";
|
|
703
|
+
let from = 0;
|
|
704
|
+
for (let at = 0; at < code.length; ) {
|
|
705
|
+
const c = code[at];
|
|
706
|
+
if (c === '"' || c === "'" || c === "`") {
|
|
707
|
+
at++;
|
|
708
|
+
while (at < code.length && code[at] !== c && (c === "`" || code[at] !== "\n")) at += code[at] === "\\" ? 2 : 1;
|
|
709
|
+
at++;
|
|
710
|
+
} else if (c === "/" && (code[at + 1] === "/" || code[at + 1] === "*") && code[at - 1] !== "\\") {
|
|
711
|
+
out += code.slice(from, at);
|
|
712
|
+
const end = code[at + 1] === "/" ? code.indexOf("\n", at) : code.indexOf("*/", at + 2);
|
|
713
|
+
if (end < 0) return out;
|
|
714
|
+
at = from = code[at + 1] === "/" ? end : end + 2;
|
|
715
|
+
} else {
|
|
716
|
+
at++;
|
|
717
|
+
}
|
|
718
|
+
}
|
|
719
|
+
return out + code.slice(from);
|
|
720
|
+
}
|
|
721
|
+
function lastMatcher(code) {
|
|
722
|
+
let last = null;
|
|
723
|
+
for (const m of stripComments(code).matchAll(MATCHER)) last = `${m[1] ? "not." : ""}${m[2]}`;
|
|
724
|
+
return last;
|
|
725
|
+
}
|
|
726
|
+
var PRESENCE = /* @__PURE__ */ new Set(["toBeGreaterThan", "toBeGreaterThanOrEqual", "toBeTruthy", "toBeDefined", "not.toBeNull", "not.toBeUndefined", "not.toBeFalsy"]);
|
|
727
|
+
function classifyRun(run, code) {
|
|
728
|
+
const status = String(run.status ?? "");
|
|
729
|
+
const error = typeof run.error === "string" ? run.error.replace(ANSI, "") : "";
|
|
730
|
+
if (status === "PASSED") return { verdict: "passes", check: "" };
|
|
731
|
+
if (status !== "FAILED" && status !== "ERROR") return { verdict: "pending", check: "" };
|
|
732
|
+
const check = error ? checkLine(error) : "";
|
|
733
|
+
if (INFRA.test(error)) return { verdict: "infra_error", check };
|
|
734
|
+
const assertion = /\bexpect\(/.test(error) && /Expected/.test(error) && /Received/.test(error);
|
|
735
|
+
if (status === "FAILED" && assertion && !NOT_FOUND.test(error) && !ACTION_TIMEOUT.test(error)) {
|
|
736
|
+
const failed = failedMatcher(error);
|
|
737
|
+
const last = code ? lastMatcher(code) : null;
|
|
738
|
+
if (failed && last && failed !== last && PRESENCE.has(failed)) {
|
|
739
|
+
return { verdict: "test_problem", check: clip(`Failed on an earlier check (${failed}), not the last one (${last}): ${check}`) };
|
|
740
|
+
}
|
|
741
|
+
return { verdict: "reproduces", check };
|
|
742
|
+
}
|
|
743
|
+
if (!error && status === "ERROR") return { verdict: "infra_error", check };
|
|
744
|
+
return { verdict: "test_problem", check };
|
|
745
|
+
}
|
|
746
|
+
var VERDICT_NEXT = {
|
|
747
|
+
reproduces: "Locked in: the test fails on the bug and will pass once it is fixed.",
|
|
748
|
+
passes: "The test passed, so it does not show the bug yet; that does not mean the bug is gone. Check it again in your browser: if you still see it, the test checks the wrong thing (assert the rule you saw broken over all the rows, not one record), fix it with edit_test_code and collect again. delete_test only when you can no longer see the bug.",
|
|
749
|
+
test_problem: "The test broke before its last check (timeout, locator not found, a failed precondition, script error), so it never compared the value. Fix how it reaches and reads the value: look at the page in your browser, use the exact label or text you see there, and keep the compared value inside expect(), not in an await that can throw first. Keep the expected value as it is: it states what a correct app must show. Fix with edit_test_code, then collect_verdicts again.",
|
|
750
|
+
pending: "Still queued or running: call collect_verdicts again in a minute.",
|
|
751
|
+
infra_error: "The run could not reach the app or the browser died: call collect_verdicts with rerun for this test."
|
|
752
|
+
};
|
|
753
|
+
function ok(value) {
|
|
754
|
+
return { content: [{ type: "text", text: JSON.stringify(value, null, 2) }] };
|
|
755
|
+
}
|
|
756
|
+
function fail(err) {
|
|
757
|
+
return { content: [{ type: "text", text: toolErrorText(err) }], isError: true };
|
|
758
|
+
}
|
|
759
|
+
function isRecord(value) {
|
|
760
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
761
|
+
}
|
|
762
|
+
function list(value) {
|
|
763
|
+
return Array.isArray(value) ? value.filter(isRecord) : [];
|
|
764
|
+
}
|
|
765
|
+
var SERVER_WAIT_MAX_SECONDS = 55;
|
|
766
|
+
var COLLECT_DEFAULT_SECONDS = 90;
|
|
767
|
+
var COLLECT_MAX_SECONDS = 120;
|
|
768
|
+
var TERMINAL = /* @__PURE__ */ new Set(["PASSED", "FAILED", "ERROR"]);
|
|
769
|
+
var CONVENTION_WARNING = /console\.log\(\) milestones|__QA_HTTP__|but has no request guard/;
|
|
770
|
+
var CODE_RULES = [
|
|
771
|
+
"Assert the CORRECT behaviour, so the test fails now and passes once the bug is fixed; one bug per test",
|
|
772
|
+
"Relative URLs only (page.goto('/orders')): the runner sets the base URL",
|
|
773
|
+
"Sign in inside the test with process.env.TEST_USER_EMAIL! and process.env.TEST_USER_PASSWORD!, never literal values",
|
|
774
|
+
"Locators you saw in your browser: getByRole(role, { name, exact: true }), getByLabel, getByTestId; prefer knownSelectors",
|
|
775
|
+
"End with an expect() on the value the app gets wrong, with the exact expected value",
|
|
776
|
+
"The test runs later on its own, on an account that has none of what your browser did: from sign-in, redo every step that made the state you saw (a report you generated, a check you started, a record you created or booked first) and then assert on the value the bug gets wrong, not on something that only exists after your earlier steps"
|
|
777
|
+
];
|
|
778
|
+
function featurePages(feature) {
|
|
779
|
+
const plans = isRecord(feature.testPlans) ? list(feature.testPlans.pages) : [];
|
|
780
|
+
const routes = Array.isArray(feature.pages) ? feature.pages.filter((p) => typeof p === "string") : [];
|
|
781
|
+
for (const plan of plans) {
|
|
782
|
+
const route = String(plan.route ?? "");
|
|
783
|
+
if (route && !routes.includes(route)) routes.push(route);
|
|
784
|
+
}
|
|
785
|
+
return routes;
|
|
786
|
+
}
|
|
787
|
+
async function startHere(client2, featureName) {
|
|
788
|
+
const warnings = [];
|
|
789
|
+
const [features, variables, bugs] = await Promise.all([
|
|
790
|
+
client2.executeTool("list_features"),
|
|
791
|
+
client2.executeTool("get_variables").catch((err) => {
|
|
792
|
+
warnings.push(`get_variables failed (${toolErrorText(err)}): credentials and baseUrl below are unknown, not missing. Call start_here again.`);
|
|
793
|
+
return null;
|
|
794
|
+
}),
|
|
795
|
+
client2.executeTool("list_bugs", { limit: 200 }).catch((err) => {
|
|
796
|
+
warnings.push(`list_bugs failed (${toolErrorText(err)}): openBugs below is unknown, not empty. Call start_here again before filing bugs.`);
|
|
797
|
+
return null;
|
|
798
|
+
})
|
|
799
|
+
]);
|
|
800
|
+
if (isRecord(bugs) && bugs.appBugsTruncated === true) {
|
|
801
|
+
warnings.push("list_bugs returned only part of the open bugs: openBugs below may be missing some.");
|
|
802
|
+
}
|
|
803
|
+
const featureList = isRecord(features) ? list(features.features) : [];
|
|
804
|
+
const chosen = featureName?.trim() || (featureList.length === 1 ? String(featureList[0].name) : void 0);
|
|
805
|
+
const vars = isRecord(variables) && isRecord(variables.variables) ? variables.variables : {};
|
|
806
|
+
const names = Object.keys(vars);
|
|
807
|
+
const baseUrl = isRecord(vars.BASE_URL) && vars.BASE_URL.secret !== true && typeof vars.BASE_URL.value === "string" ? vars.BASE_URL.value : null;
|
|
808
|
+
const credentials = {
|
|
809
|
+
baseUrl,
|
|
810
|
+
loginReady: names.includes("TEST_USER_EMAIL") && names.includes("TEST_USER_PASSWORD"),
|
|
811
|
+
variables: names.map((name) => ({ name, secret: isRecord(vars[name]) && vars[name].secret === true }))
|
|
812
|
+
};
|
|
813
|
+
let feature = null;
|
|
814
|
+
if (chosen) {
|
|
815
|
+
try {
|
|
816
|
+
const detail = await client2.executeTool("get_feature", { featureName: chosen });
|
|
817
|
+
if (isRecord(detail)) feature = detail;
|
|
818
|
+
} catch (err) {
|
|
819
|
+
warnings.push(`get_feature("${chosen}") failed (${toolErrorText(err)}): the feature's pages, tests and selectors are missing below. Check the name against project.features and call start_here again.`);
|
|
820
|
+
}
|
|
821
|
+
}
|
|
822
|
+
const suite = feature ? String(feature.name) : void 0;
|
|
823
|
+
const openBugs = (isRecord(bugs) ? list(bugs.appBugs) : []).filter((bug) => !suite || String(bug.suite ?? "").toLowerCase() === suite.toLowerCase()).map((bug) => ({
|
|
824
|
+
testId: bug.id,
|
|
825
|
+
name: bug.name,
|
|
826
|
+
reviewStatus: bug.reviewStatus,
|
|
827
|
+
check: isRecord(bug.evidence) ? bug.evidence.failingCheck ?? null : null
|
|
828
|
+
}));
|
|
829
|
+
return {
|
|
830
|
+
project: {
|
|
831
|
+
features: featureList.map((f) => ({
|
|
832
|
+
name: f.name,
|
|
833
|
+
pages: f.pages,
|
|
834
|
+
criticality: f.criticality,
|
|
835
|
+
tests: f.existingTests,
|
|
836
|
+
failing: f.failingTests
|
|
837
|
+
}))
|
|
838
|
+
},
|
|
839
|
+
credentials,
|
|
840
|
+
...warnings.length > 0 ? { warnings } : {},
|
|
841
|
+
...feature ? {
|
|
842
|
+
feature: {
|
|
843
|
+
name: feature.name,
|
|
844
|
+
description: feature.description,
|
|
845
|
+
pages: featurePages(feature),
|
|
846
|
+
tests: list(feature.tests).map((t) => ({ id: t.id, name: t.name, status: t.status, lastRunStatus: t.lastRunStatus ?? null })),
|
|
847
|
+
knownSelectors: Array.isArray(feature.knownSelectors) ? feature.knownSelectors : [],
|
|
848
|
+
...typeof feature.surveyNotes === "string" ? { surveyNotes: feature.surveyNotes } : {}
|
|
849
|
+
}
|
|
850
|
+
} : {},
|
|
851
|
+
openBugs,
|
|
852
|
+
codeRules: CODE_RULES,
|
|
853
|
+
next: [
|
|
854
|
+
credentials.loginReady || !variables ? null : "TEST_USER_EMAIL / TEST_USER_PASSWORD are not set: saved tests cannot sign in. Ask the developer for a test account.",
|
|
855
|
+
"Explore each page with your browser. Skip bugs already in openBugs.",
|
|
856
|
+
"For each bug: save_regression_test right away, then keep exploring. Do not wait for its run.",
|
|
857
|
+
"When you are done exploring: collect_verdicts. Fix test_problem tests with edit_test_code and collect again."
|
|
858
|
+
].filter(Boolean)
|
|
859
|
+
};
|
|
860
|
+
}
|
|
861
|
+
async function waitForRun(client2, runId, deadline) {
|
|
862
|
+
let view = null;
|
|
863
|
+
for (; ; ) {
|
|
864
|
+
const left = Math.floor((deadline - Date.now()) / 1e3);
|
|
865
|
+
view = left >= 1 ? await client2.executeTool("wait_for_run", { runId, timeoutSeconds: Math.min(left, SERVER_WAIT_MAX_SECONDS) }) : await client2.executeTool("get_test_run", { runId });
|
|
866
|
+
if (!isRecord(view)) return {};
|
|
867
|
+
if (TERMINAL.has(String(view.status)) || view.timedOut !== true || left < 1) return view;
|
|
868
|
+
}
|
|
869
|
+
}
|
|
870
|
+
async function collectVerdicts(client2, session, args) {
|
|
871
|
+
const seconds = Math.min(Math.max(args.timeoutSeconds ?? COLLECT_DEFAULT_SECONDS, 0), COLLECT_MAX_SECONDS);
|
|
872
|
+
const deadline = Date.now() + seconds * 1e3;
|
|
873
|
+
const ids = args.testIds?.length ? args.testIds : [...session.saved.keys()];
|
|
874
|
+
if (ids.length === 0) {
|
|
875
|
+
return { summary: {}, verdicts: [], next: "Nothing saved in this session yet: save_regression_test first, or pass testIds." };
|
|
876
|
+
}
|
|
877
|
+
const rerun = new Set(args.rerun ?? []);
|
|
878
|
+
const targets = [];
|
|
879
|
+
for (const testId of ids) {
|
|
880
|
+
const saved = session.saved.get(testId);
|
|
881
|
+
try {
|
|
882
|
+
if (rerun.has(testId) || saved?.edited) {
|
|
883
|
+
const queued = await client2.executeTool("run_test", { testId });
|
|
884
|
+
const runId = isRecord(queued) && typeof queued.runId === "string" ? queued.runId : void 0;
|
|
885
|
+
if (saved) {
|
|
886
|
+
saved.edited = false;
|
|
887
|
+
saved.runId = runId;
|
|
888
|
+
}
|
|
889
|
+
targets.push({ testId, runId });
|
|
890
|
+
} else if (saved?.runId) {
|
|
891
|
+
targets.push({ testId, runId: saved.runId });
|
|
892
|
+
} else {
|
|
893
|
+
const latest = await client2.executeTool("get_test_run", { testId });
|
|
894
|
+
targets.push({ testId, runId: isRecord(latest) && typeof latest.id === "string" ? latest.id : void 0 });
|
|
895
|
+
}
|
|
896
|
+
} catch (err) {
|
|
897
|
+
targets.push({ testId, error: toolErrorText(err) });
|
|
898
|
+
}
|
|
899
|
+
}
|
|
900
|
+
const verdicts = [];
|
|
901
|
+
const summary = { reproduces: 0, passes: 0, test_problem: 0, pending: 0, infra_error: 0 };
|
|
902
|
+
for (const { testId, runId, error } of targets) {
|
|
903
|
+
const saved = session.saved.get(testId);
|
|
904
|
+
if (!runId) {
|
|
905
|
+
summary.pending++;
|
|
906
|
+
verdicts.push({ testId, name: saved?.name, verdict: "pending", check: error ?? "no run yet", next: VERDICT_NEXT.pending });
|
|
907
|
+
continue;
|
|
908
|
+
}
|
|
909
|
+
let view;
|
|
910
|
+
try {
|
|
911
|
+
view = await waitForRun(client2, runId, deadline);
|
|
912
|
+
} catch (err) {
|
|
913
|
+
summary.infra_error++;
|
|
914
|
+
verdicts.push({ testId, name: saved?.name, runId, verdict: "infra_error", check: toolErrorText(err), next: VERDICT_NEXT.infra_error });
|
|
915
|
+
continue;
|
|
916
|
+
}
|
|
917
|
+
let test = {};
|
|
918
|
+
if (TERMINAL.has(String(view.status))) {
|
|
919
|
+
test = await client2.executeTool("get_test_code", { testId }).then((t) => isRecord(t) ? t : {}, () => ({}));
|
|
920
|
+
}
|
|
921
|
+
const { verdict, check } = classifyRun(view, typeof test.playwrightCode === "string" ? test.playwrightCode : void 0);
|
|
922
|
+
summary[verdict]++;
|
|
923
|
+
const tags = Array.isArray(test.tags) ? test.tags : [];
|
|
924
|
+
verdicts.push({
|
|
925
|
+
testId,
|
|
926
|
+
name: saved?.name ?? view.testName ?? test.name,
|
|
927
|
+
...saved?.bug ? { bug: saved.bug } : {},
|
|
928
|
+
runId,
|
|
929
|
+
verdict,
|
|
930
|
+
...check ? { check } : {},
|
|
931
|
+
...typeof test.status === "string" ? { testStatus: test.status } : {},
|
|
932
|
+
...tags.includes("@bug-found") ? { triagedAsAppBug: true } : {},
|
|
933
|
+
next: VERDICT_NEXT[verdict]
|
|
934
|
+
});
|
|
935
|
+
}
|
|
936
|
+
const next = summary.pending > 0 ? "Some runs are still going: call collect_verdicts again." : summary.test_problem + summary.passes + summary.infra_error > 0 ? "Keep the reproduces tests as they are. Fix the others (see each next), then collect_verdicts again." : "Every saved test reproduces its bug.";
|
|
937
|
+
return { summary, verdicts, next };
|
|
938
|
+
}
|
|
939
|
+
function registerAgentTools(server2, client2, session) {
|
|
940
|
+
server2.tool(
|
|
941
|
+
"start_here",
|
|
942
|
+
"START HERE when asked to test, find bugs in or lock in a feature. One read with everything to begin: the project's features, the feature's pages, its existing tests and knownSelectors (verified locators), whether test credentials are set, open bugs, and the rules for saved test code.",
|
|
943
|
+
{ feature: z.string().optional().describe("The feature to work on (case-insensitive). Leave out to list the features.") },
|
|
944
|
+
{ readOnlyHint: true },
|
|
945
|
+
async (args) => {
|
|
946
|
+
try {
|
|
947
|
+
return ok(await startHere(client2, args.feature));
|
|
948
|
+
} catch (err) {
|
|
949
|
+
return fail(err);
|
|
950
|
+
}
|
|
951
|
+
}
|
|
952
|
+
);
|
|
953
|
+
server2.tool(
|
|
954
|
+
"save_regression_test",
|
|
955
|
+
"Save a bug you found as a Playwright test that asserts the CORRECT behaviour (it fails now and passes once fixed). Returns at once: the test runs in the cloud while you keep exploring. Call collect_verdicts at the end to learn which tests reproduce their bug. It runs later on an account with none of your browser's changes, so it redoes every step from sign-in. Same code rules as create_test (see start_here codeRules).",
|
|
956
|
+
{
|
|
957
|
+
name: z.string().describe('What should happen, e.g. "Orders - status filter shows only Paid orders"'),
|
|
958
|
+
feature: z.string().describe("The feature name exactly as start_here lists it (becomes the suite)"),
|
|
959
|
+
playwrightCode: z.string().describe("Full Playwright test file"),
|
|
960
|
+
bug: z.string().optional().describe("One line: what the app does wrong"),
|
|
961
|
+
tags: z.array(z.string()).optional().describe("Extra tags; @regression is added")
|
|
962
|
+
},
|
|
963
|
+
async (args) => {
|
|
964
|
+
try {
|
|
965
|
+
const extra = args.tags ?? [];
|
|
966
|
+
const tags = extra.includes("@auth") ? extra : [.../* @__PURE__ */ new Set([...extra, "@regression"])];
|
|
967
|
+
const created = await client2.executeTool("create_test", {
|
|
968
|
+
name: args.name,
|
|
969
|
+
playwrightCode: args.playwrightCode,
|
|
970
|
+
suite: args.feature,
|
|
971
|
+
tags,
|
|
972
|
+
assertsBug: true
|
|
973
|
+
});
|
|
974
|
+
if (!isRecord(created) || typeof created.id !== "string") return ok(created);
|
|
975
|
+
const runId = isRecord(created.verification) && typeof created.verification.runId === "string" ? created.verification.runId : void 0;
|
|
976
|
+
session.remember({ testId: created.id, name: args.name, bug: args.bug, runId, edited: false });
|
|
977
|
+
const warnings = (Array.isArray(created.warnings) ? created.warnings : []).filter((w) => typeof w === "string" && !CONVENTION_WARNING.test(w));
|
|
978
|
+
return ok({
|
|
979
|
+
testId: created.id,
|
|
980
|
+
name: created.name ?? args.name,
|
|
981
|
+
status: created.status,
|
|
982
|
+
...runId ? { runId } : {},
|
|
983
|
+
...warnings.length ? { warnings } : {},
|
|
984
|
+
next: warnings.length ? "Saved; its run is queued. Fix the warnings with edit_test_code, then keep exploring. collect_verdicts at the end." : "Saved; its run is queued. Keep exploring. collect_verdicts at the end."
|
|
985
|
+
});
|
|
986
|
+
} catch (err) {
|
|
987
|
+
return fail(err);
|
|
988
|
+
}
|
|
989
|
+
}
|
|
990
|
+
);
|
|
991
|
+
server2.tool(
|
|
992
|
+
"collect_verdicts",
|
|
993
|
+
"After exploring: wait for the runs of the tests you saved this session (or testIds) and say for each whether it reproduces its bug (fails on its check), passes (does not show the bug), has a test problem (broke before its check), is still pending, or hit an infrastructure error. Tests changed with edit_test_code since their run are run again first. Waits up to timeoutSeconds in total. A test that fails on its last check is the goal: it reproduces the bug, so keep it as it is. A test that passes is the one to question.",
|
|
994
|
+
{
|
|
995
|
+
testIds: z.array(z.string()).optional().describe("Tests to check (default: every test saved this session)"),
|
|
996
|
+
rerun: z.array(z.string()).optional().describe("Tests to run again before checking"),
|
|
997
|
+
timeoutSeconds: z.number().optional().describe(`Total wait (default ${COLLECT_DEFAULT_SECONDS}, max ${COLLECT_MAX_SECONDS})`)
|
|
998
|
+
},
|
|
999
|
+
async (args) => {
|
|
1000
|
+
try {
|
|
1001
|
+
return ok(await collectVerdicts(client2, session, args));
|
|
1002
|
+
} catch (err) {
|
|
1003
|
+
return fail(err);
|
|
1004
|
+
}
|
|
1005
|
+
}
|
|
1006
|
+
);
|
|
1007
|
+
}
|
|
1008
|
+
var AGENT_INSTRUCTIONS = `Validate.QA saves the bugs you find as Playwright tests that run in its cloud and keep running.
|
|
1009
|
+
|
|
1010
|
+
Finding and locking in bugs in a feature:
|
|
1011
|
+
1. start_here({ feature }): its pages, existing tests, knownSelectors, credentials, and open bugs.
|
|
1012
|
+
2. Explore the pages with your browser tools.
|
|
1013
|
+
3. Each bug: save_regression_test with code that asserts the CORRECT behaviour. It runs later on an account with none of your browser's changes, so from sign-in it redoes every click that made the state you saw. It returns at once; keep exploring.
|
|
1014
|
+
4. At the end: collect_verdicts. reproduces (the test fails on its check) = the bug is locked in: keep that test as it is. passes = the test misses the bug: check again in your browser and fix its assertion. test_problem = it broke before its check: fix the steps. Fix with edit_test_code, then collect_verdicts again.
|
|
1015
|
+
Rate limit: 50 calls a minute.`;
|
|
1016
|
+
|
|
1017
|
+
// src/tools/index.ts
|
|
221
1018
|
function tool(server2, client2, name, description, schema, annotations) {
|
|
222
1019
|
const handler = async (args) => {
|
|
223
1020
|
try {
|
|
@@ -227,7 +1024,7 @@ function tool(server2, client2, name, description, schema, annotations) {
|
|
|
227
1024
|
};
|
|
228
1025
|
} catch (err) {
|
|
229
1026
|
return {
|
|
230
|
-
content: [{ type: "text", text:
|
|
1027
|
+
content: [{ type: "text", text: toolErrorText(err) }],
|
|
231
1028
|
isError: true
|
|
232
1029
|
};
|
|
233
1030
|
}
|
|
@@ -238,19 +1035,66 @@ function tool(server2, client2, name, description, schema, annotations) {
|
|
|
238
1035
|
server2.tool(name, description, schema, handler);
|
|
239
1036
|
}
|
|
240
1037
|
}
|
|
241
|
-
|
|
1038
|
+
var SERVER_WAIT_MAX_SECONDS2 = 55;
|
|
1039
|
+
var WAIT_DEFAULT_SECONDS = 50;
|
|
1040
|
+
var WAIT_MAX_SECONDS = 120;
|
|
1041
|
+
function isRecord2(value) {
|
|
1042
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
1043
|
+
}
|
|
1044
|
+
function waitSecondsFrom(raw, fallback) {
|
|
1045
|
+
const n = typeof raw === "number" ? raw : Number.NaN;
|
|
1046
|
+
if (!Number.isFinite(n)) return fallback;
|
|
1047
|
+
return Math.min(Math.max(n, 0), WAIT_MAX_SECONDS);
|
|
1048
|
+
}
|
|
1049
|
+
async function executeWithWait(client2, name, args, waitKey, totalSeconds, kind) {
|
|
1050
|
+
const deadline = Date.now() + totalSeconds * 1e3;
|
|
1051
|
+
const first = Math.min(totalSeconds, SERVER_WAIT_MAX_SECONDS2);
|
|
1052
|
+
let result = await client2.executeTool(name, { ...args, [waitKey]: first });
|
|
1053
|
+
let id = args[kind.idKey];
|
|
1054
|
+
for (; ; ) {
|
|
1055
|
+
if (!isRecord2(result) || result.timedOut !== true) return result;
|
|
1056
|
+
id = result[kind.idKey] ?? id;
|
|
1057
|
+
const left = Math.floor((deadline - Date.now()) / 1e3);
|
|
1058
|
+
if (typeof id !== "string" || left < 1) return result;
|
|
1059
|
+
result = await client2.executeTool(kind.tool, { [kind.idKey]: id, timeoutSeconds: Math.min(left, SERVER_WAIT_MAX_SECONDS2) });
|
|
1060
|
+
}
|
|
1061
|
+
}
|
|
1062
|
+
function waitTool(server2, client2, name, description, schema, waitKey, defaultSeconds, kind, annotations) {
|
|
1063
|
+
const handler = async (args) => {
|
|
1064
|
+
try {
|
|
1065
|
+
const seconds = waitSecondsFrom(args[waitKey], defaultSeconds);
|
|
1066
|
+
const result = seconds > 0 ? await executeWithWait(client2, name, args, waitKey, seconds, kind) : await client2.executeTool(name, args);
|
|
1067
|
+
return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }] };
|
|
1068
|
+
} catch (err) {
|
|
1069
|
+
return {
|
|
1070
|
+
content: [{ type: "text", text: toolErrorText(err) }],
|
|
1071
|
+
isError: true
|
|
1072
|
+
};
|
|
1073
|
+
}
|
|
1074
|
+
};
|
|
1075
|
+
if (annotations) {
|
|
1076
|
+
server2.tool(name, description, schema, annotations, handler);
|
|
1077
|
+
} else {
|
|
1078
|
+
server2.tool(name, description, schema, handler);
|
|
1079
|
+
}
|
|
1080
|
+
}
|
|
1081
|
+
function registerAllTools(server2, client2, toolset = "full") {
|
|
1082
|
+
const session = new AgentSession();
|
|
1083
|
+
if (toolset === "agent") server2 = limitTools(server2, AGENT_TOOL_NAMES);
|
|
1084
|
+
client2 = session.track(client2);
|
|
1085
|
+
registerAgentTools(server2, client2, session);
|
|
242
1086
|
tool(
|
|
243
1087
|
server2,
|
|
244
1088
|
client2,
|
|
245
1089
|
"list_tests",
|
|
246
1090
|
"List tests in the current project. Returns test name, status, suite, tags, confidence, and the last run result. Paginated - use limit/offset to page through all tests.",
|
|
247
1091
|
{
|
|
248
|
-
status:
|
|
249
|
-
suite:
|
|
250
|
-
search:
|
|
251
|
-
tag:
|
|
252
|
-
limit:
|
|
253
|
-
offset:
|
|
1092
|
+
status: z2.enum(["DRAFT", "ACTIVE", "DISABLED", "QUARANTINED"]).optional().describe("Filter by test lifecycle status"),
|
|
1093
|
+
suite: z2.string().optional().describe("Filter by suite name"),
|
|
1094
|
+
search: z2.string().optional().describe("Search test names (case-insensitive substring match)"),
|
|
1095
|
+
tag: z2.string().optional().describe("Filter by tag (e.g. @regression, @smoke, @auth)"),
|
|
1096
|
+
limit: z2.number().optional().describe("Max results to return (default: 50, max: 200)"),
|
|
1097
|
+
offset: z2.number().optional().describe("Number of results to skip for pagination (default: 0)")
|
|
254
1098
|
},
|
|
255
1099
|
{ readOnlyHint: true }
|
|
256
1100
|
);
|
|
@@ -259,22 +1103,31 @@ function registerAllTools(server2, client2) {
|
|
|
259
1103
|
client2,
|
|
260
1104
|
"get_test",
|
|
261
1105
|
"Get details of a specific test: name, status, tags, steps, session, and the 5 most recent run results. Code is truncated to 200 lines - use get_test_code for the full Playwright code.",
|
|
262
|
-
{ testId:
|
|
1106
|
+
{ testId: z2.string().describe("The test case ID") },
|
|
263
1107
|
{ readOnlyHint: true }
|
|
264
1108
|
);
|
|
265
|
-
|
|
1109
|
+
waitTool(
|
|
266
1110
|
server2,
|
|
267
1111
|
client2,
|
|
268
1112
|
"run_test",
|
|
269
|
-
"Queue a test for execution. Creates a PENDING test run that will be picked up by an online runner. Returns the run ID.",
|
|
270
|
-
{
|
|
1113
|
+
"Queue a test for execution. Creates a PENDING test run that will be picked up by an online runner. Returns the run ID; with waitSeconds, the finished run (as wait_for_run). For a mobile (Appium) test, pass platform to run it on the OTHER platform than it was authored on - if some steps lack a native locator for that platform, a one-time locator-discovery pass runs first (supply appId once for the target-platform build), then the test runs automatically.",
|
|
1114
|
+
{
|
|
1115
|
+
testId: z2.string().describe("The test case ID to run"),
|
|
1116
|
+
platform: z2.enum(["IOS", "ANDROID"]).optional().describe("Mobile only: run this test on the given platform (cross-platform run). Omit to run on the authoring platform."),
|
|
1117
|
+
appId: z2.string().optional().describe("Mobile cross-platform only: the target-platform app id (bundleId/appPackage). Only needed the first time this project runs on the other platform."),
|
|
1118
|
+
deeplink: z2.string().optional().describe("Mobile cross-platform only: target-platform deep link URL. Required when the authored test uses a custom app scheme that may differ between iOS and Android (the server answers need_deeplink); http/https App Links are reused automatically."),
|
|
1119
|
+
waitSeconds: z2.number().min(0).max(WAIT_MAX_SECONDS).optional().describe(`Optional. Wait up to this many seconds (max ${WAIT_MAX_SECONDS}) for the run to finish and return its final result, like wait_for_run (timedOut: true if still going). 0 or omitted: return at once with the run ID.`)
|
|
1120
|
+
},
|
|
1121
|
+
"waitSeconds",
|
|
1122
|
+
0,
|
|
1123
|
+
{ tool: "wait_for_run", idKey: "runId" }
|
|
271
1124
|
);
|
|
272
1125
|
tool(
|
|
273
1126
|
server2,
|
|
274
1127
|
client2,
|
|
275
1128
|
"delete_test",
|
|
276
1129
|
"Permanently delete a test case and all its run history. Cannot be undone.",
|
|
277
|
-
{ testId:
|
|
1130
|
+
{ testId: z2.string().describe("The test case ID to delete") },
|
|
278
1131
|
{ destructiveHint: true }
|
|
279
1132
|
);
|
|
280
1133
|
tool(
|
|
@@ -282,38 +1135,93 @@ function registerAllTools(server2, client2) {
|
|
|
282
1135
|
client2,
|
|
283
1136
|
"heal_test",
|
|
284
1137
|
"Queue an AI healing run for a failing test. The AI agent replays the test in a real browser, analyzes the failure, and either fixes the Playwright code OR detects an application bug (quarantines the test with @bug-found if the app itself is broken, not the test).",
|
|
285
|
-
{ testId:
|
|
1138
|
+
{ testId: z2.string().describe("The failing test case ID") }
|
|
286
1139
|
);
|
|
287
1140
|
tool(
|
|
288
1141
|
server2,
|
|
289
1142
|
client2,
|
|
290
1143
|
"edit_test_code",
|
|
291
|
-
"Update
|
|
1144
|
+
"Update an existing test case. WEB (Playwright) tests: provide the full replacement playwrightCode (read project://test-conventions for patterns; code is validated - fix any warnings returned). MOBILE (Appium) tests: provide the FULL replacement steps array (playwrightCode is rejected - mobile tests execute structured steps, not code). Read the current form first with get_test_code.",
|
|
292
1145
|
{
|
|
293
|
-
testId:
|
|
294
|
-
playwrightCode:
|
|
295
|
-
|
|
296
|
-
|
|
1146
|
+
testId: z2.string().describe("The test case ID to edit"),
|
|
1147
|
+
playwrightCode: z2.string().optional().describe("Web tests only: new Playwright test code (full file replacement)"),
|
|
1148
|
+
steps: z2.array(z2.record(z2.string(), z2.unknown())).optional().describe("Mobile (Appium) tests only: FULL replacement structured step array (same shape as create_test steps). Include every step, not just the changed ones."),
|
|
1149
|
+
name: z2.string().optional().describe("New test name"),
|
|
1150
|
+
description: z2.string().optional().describe("New short description of what the test verifies"),
|
|
1151
|
+
tags: z2.array(z2.string()).optional().describe('New tags array (e.g. ["@regression", "@smoke"])'),
|
|
1152
|
+
status: z2.enum(["DRAFT", "ACTIVE", "DISABLED", "QUARANTINED"]).optional().describe("New test status. Re-activating a bug-found test (QUARANTINED\u2192ACTIVE) also queues a verification run and returns its testRunId.")
|
|
297
1153
|
}
|
|
298
1154
|
);
|
|
299
1155
|
tool(
|
|
300
1156
|
server2,
|
|
301
1157
|
client2,
|
|
302
1158
|
"update_test_status",
|
|
303
|
-
"Change the lifecycle status of one or more tests (e.g. promote DRAFT\u2192ACTIVE, disable a flaky test).",
|
|
1159
|
+
"Change the lifecycle status of one or more tests (e.g. promote DRAFT\u2192ACTIVE, disable a flaky test). Re-activating exactly ONE bug-found test (QUARANTINED\u2192ACTIVE) also queues a verification run and returns its testRunId; with several ids no run is queued: any bug-found tests among them come back as unverifiedTestIds, to check with batch_run_tests.",
|
|
304
1160
|
{
|
|
305
|
-
testIds:
|
|
306
|
-
status:
|
|
1161
|
+
testIds: z2.array(z2.string()).describe("Test IDs to update"),
|
|
1162
|
+
status: z2.enum(["DRAFT", "ACTIVE", "DISABLED", "QUARANTINED"]).describe("New status to set")
|
|
307
1163
|
}
|
|
308
1164
|
);
|
|
309
1165
|
tool(
|
|
310
1166
|
server2,
|
|
311
1167
|
client2,
|
|
312
1168
|
"get_test_run",
|
|
313
|
-
"Get the status and result of a specific test run
|
|
1169
|
+
"Get the status and result of a specific test run once (to wait for it to finish, use wait_for_run, not a loop of this). Look up by runId, or the latest run for a testId. A FAILED or ERROR run also carries its failure evidence: error (up to 4 KB), consoleLog (the last 200 lines, at most 20 KB), failedSteps, and for a mobile run domSnapshot (the page source at the failure). Screenshots: get_test_screenshots. No Playwright trace is recorded.",
|
|
1170
|
+
{
|
|
1171
|
+
runId: z2.string().optional().describe("The test run ID"),
|
|
1172
|
+
testId: z2.string().optional().describe("Get the most recent run for this test case ID")
|
|
1173
|
+
},
|
|
1174
|
+
{ readOnlyHint: true }
|
|
1175
|
+
);
|
|
1176
|
+
waitTool(
|
|
1177
|
+
server2,
|
|
1178
|
+
client2,
|
|
1179
|
+
"wait_for_run",
|
|
1180
|
+
`Wait until a test run (run_test, heal_test, a verification run) is PASSED, FAILED or ERROR, then return it in the get_test_run shape, failure evidence included. Returns after at most timeoutSeconds (default ${WAIT_DEFAULT_SECONDS}, max ${WAIT_MAX_SECONDS}) with timedOut: true and the current status if it is still pending or running; call it again to keep waiting. Use this instead of polling get_test_run.`,
|
|
1181
|
+
{
|
|
1182
|
+
runId: z2.string().describe("The test run ID to wait for"),
|
|
1183
|
+
timeoutSeconds: z2.number().min(0).max(WAIT_MAX_SECONDS).optional().describe(`How long to wait, in seconds (default ${WAIT_DEFAULT_SECONDS}, max ${WAIT_MAX_SECONDS})`)
|
|
1184
|
+
},
|
|
1185
|
+
"timeoutSeconds",
|
|
1186
|
+
WAIT_DEFAULT_SECONDS,
|
|
1187
|
+
{ tool: "wait_for_run", idKey: "runId" },
|
|
1188
|
+
{ readOnlyHint: true }
|
|
1189
|
+
);
|
|
1190
|
+
waitTool(
|
|
1191
|
+
server2,
|
|
1192
|
+
client2,
|
|
1193
|
+
"wait_for_batch",
|
|
1194
|
+
`Wait until a batch from batch_run_tests is COMPLETED, then return its counts (total, completed, passed, failed, errored) and, when tests failed, up to 20 failures with their runId and error. Returns after at most timeoutSeconds (default ${WAIT_DEFAULT_SECONDS}, max ${WAIT_MAX_SECONDS}) with timedOut: true and the current counts if it is still running; call it again to keep waiting. Use get_test_run on a failure's runId for its console log.`,
|
|
1195
|
+
{
|
|
1196
|
+
batchId: z2.string().describe("The batch ID returned by batch_run_tests"),
|
|
1197
|
+
timeoutSeconds: z2.number().min(0).max(WAIT_MAX_SECONDS).optional().describe(`How long to wait, in seconds (default ${WAIT_DEFAULT_SECONDS}, max ${WAIT_MAX_SECONDS})`)
|
|
1198
|
+
},
|
|
1199
|
+
"timeoutSeconds",
|
|
1200
|
+
WAIT_DEFAULT_SECONDS,
|
|
1201
|
+
{ tool: "wait_for_batch", idKey: "batchId" },
|
|
1202
|
+
{ readOnlyHint: true }
|
|
1203
|
+
);
|
|
1204
|
+
tool(
|
|
1205
|
+
server2,
|
|
1206
|
+
client2,
|
|
1207
|
+
"run_discovery",
|
|
1208
|
+
"Start AI discovery for this project. For web, discovery uses the configured base URL. For mobile, pass platform and appId to explore a native app on a connected runner/device. Use survey to map features, deep with featureName to generate tests for one feature, or full for broad generation. On a project with no platform yet, a workspace owner must choose Mobile first; tokens can't make that first choice.",
|
|
1209
|
+
{
|
|
1210
|
+
mode: z2.enum(["survey", "deep", "full"]).optional().describe("Discovery mode. survey maps features/screens and generates no tests; deep generates tests for one feature; full explores and generates broadly."),
|
|
1211
|
+
featureName: z2.string().optional().describe("Required for deep mode: the feature name to generate tests for."),
|
|
1212
|
+
focusAreas: z2.array(z2.string()).optional().describe("Optional routes, screens, or areas to focus discovery on."),
|
|
1213
|
+
platform: z2.enum(["IOS", "ANDROID"]).optional().describe("Mobile only: target platform. Requires appId."),
|
|
1214
|
+
appId: z2.string().optional().describe("Mobile only: bundle id/package id to explore. Requires platform."),
|
|
1215
|
+
resetPolicy: z2.enum(["AUTO", "CLEAR_APP_DATA", "PRESERVE_APP_DATA"]).optional().describe("Mobile only: app-state policy for discovery/replay. AUTO uses default safe reset behavior.")
|
|
1216
|
+
}
|
|
1217
|
+
);
|
|
1218
|
+
tool(
|
|
1219
|
+
server2,
|
|
1220
|
+
client2,
|
|
1221
|
+
"get_discovery_status",
|
|
1222
|
+
"Check progress for a discovery run. Omit discoveryRunId to get the latest discovery run for this project.",
|
|
314
1223
|
{
|
|
315
|
-
|
|
316
|
-
testId: z.string().optional().describe("Get the most recent run for this test case ID")
|
|
1224
|
+
discoveryRunId: z2.string().optional().describe("Specific discovery run ID. Omit for latest.")
|
|
317
1225
|
},
|
|
318
1226
|
{ readOnlyHint: true }
|
|
319
1227
|
);
|
|
@@ -321,8 +1229,8 @@ function registerAllTools(server2, client2) {
|
|
|
321
1229
|
"get_test_screenshots",
|
|
322
1230
|
"Get screenshots from a specific test run as images the AI can actually see. Use this to visually inspect what happened during a failed or passing run.",
|
|
323
1231
|
{
|
|
324
|
-
runId:
|
|
325
|
-
limit:
|
|
1232
|
+
runId: z2.string().describe("The test run ID to fetch screenshots from"),
|
|
1233
|
+
limit: z2.number().optional().describe("Max screenshots to return (default: 5, max: 10)")
|
|
326
1234
|
},
|
|
327
1235
|
{ readOnlyHint: true },
|
|
328
1236
|
async (args) => {
|
|
@@ -348,7 +1256,7 @@ function registerAllTools(server2, client2) {
|
|
|
348
1256
|
return { content };
|
|
349
1257
|
} catch (err) {
|
|
350
1258
|
return {
|
|
351
|
-
content: [{ type: "text", text:
|
|
1259
|
+
content: [{ type: "text", text: toolErrorText(err) }],
|
|
352
1260
|
isError: true
|
|
353
1261
|
};
|
|
354
1262
|
}
|
|
@@ -360,19 +1268,23 @@ function registerAllTools(server2, client2) {
|
|
|
360
1268
|
"cancel_test_run",
|
|
361
1269
|
"Cancel a pending or running test run. Can cancel by run ID or cancel the latest running batch.",
|
|
362
1270
|
{
|
|
363
|
-
runId:
|
|
364
|
-
cancelBatch:
|
|
1271
|
+
runId: z2.string().optional().describe("The specific test run ID to cancel"),
|
|
1272
|
+
cancelBatch: z2.boolean().optional().describe("If true, cancel the latest batch run instead")
|
|
365
1273
|
}
|
|
366
1274
|
);
|
|
367
|
-
|
|
1275
|
+
waitTool(
|
|
368
1276
|
server2,
|
|
369
1277
|
client2,
|
|
370
1278
|
"batch_run_tests",
|
|
371
|
-
"Run multiple tests from the database individually. Filter by test IDs or suite name; omit both to run all ACTIVE tests. Each test executes in its own browser.",
|
|
1279
|
+
"Run multiple tests from the database individually. Filter by test IDs or suite name; omit both to run all ACTIVE tests. Each test executes in its own browser. Returns the batchId; follow it with wait_for_batch, or pass waitSeconds.",
|
|
372
1280
|
{
|
|
373
|
-
testIds:
|
|
374
|
-
suite:
|
|
375
|
-
|
|
1281
|
+
testIds: z2.array(z2.string()).optional().describe("Specific test IDs to run"),
|
|
1282
|
+
suite: z2.string().optional().describe("Run all active tests in this suite"),
|
|
1283
|
+
waitSeconds: z2.number().min(0).max(WAIT_MAX_SECONDS).optional().describe(`Optional. Wait up to this many seconds (max ${WAIT_MAX_SECONDS}) for the batch to complete and return the wait_for_batch result (timedOut: true if still running). 0 or omitted: return at once with the batchId.`)
|
|
1284
|
+
},
|
|
1285
|
+
"waitSeconds",
|
|
1286
|
+
0,
|
|
1287
|
+
{ tool: "wait_for_batch", idKey: "batchId" }
|
|
376
1288
|
);
|
|
377
1289
|
tool(
|
|
378
1290
|
server2,
|
|
@@ -382,13 +1294,23 @@ function registerAllTools(server2, client2) {
|
|
|
382
1294
|
{},
|
|
383
1295
|
{ readOnlyHint: true }
|
|
384
1296
|
);
|
|
1297
|
+
tool(
|
|
1298
|
+
server2,
|
|
1299
|
+
client2,
|
|
1300
|
+
"generate_tests_for_feature",
|
|
1301
|
+
"Start test generation for an existing feature by id or name. For mobile, run_discovery with mode=deep, featureName, platform, and appId gives explicit device/platform control.",
|
|
1302
|
+
{
|
|
1303
|
+
featureId: z2.string().optional().describe("Feature ID. Either featureId or featureName is required."),
|
|
1304
|
+
featureName: z2.string().optional().describe("Feature name. Either featureId or featureName is required.")
|
|
1305
|
+
}
|
|
1306
|
+
);
|
|
385
1307
|
tool(
|
|
386
1308
|
server2,
|
|
387
1309
|
client2,
|
|
388
1310
|
"list_api_endpoints",
|
|
389
1311
|
"List captured API endpoints for the current project. Optionally scope to a specific feature name.",
|
|
390
1312
|
{
|
|
391
|
-
featureName:
|
|
1313
|
+
featureName: z2.string().optional().describe("Optional feature name to scope the endpoint list")
|
|
392
1314
|
},
|
|
393
1315
|
{ readOnlyHint: true }
|
|
394
1316
|
);
|
|
@@ -416,6 +1338,14 @@ function registerAllTools(server2, client2) {
|
|
|
416
1338
|
{},
|
|
417
1339
|
{ readOnlyHint: true }
|
|
418
1340
|
);
|
|
1341
|
+
tool(
|
|
1342
|
+
server2,
|
|
1343
|
+
client2,
|
|
1344
|
+
"get_regression_stats",
|
|
1345
|
+
"Get test counts by lifecycle status (active, quarantined, draft, disabled, total; API specs count their inner tests) and the 7-day pass rate, with rate-limit, config-issue and cancelled runs left out. The same numbers as the dashboard's regression stats.",
|
|
1346
|
+
{},
|
|
1347
|
+
{ readOnlyHint: true }
|
|
1348
|
+
);
|
|
419
1349
|
tool(
|
|
420
1350
|
server2,
|
|
421
1351
|
client2,
|
|
@@ -430,13 +1360,13 @@ function registerAllTools(server2, client2) {
|
|
|
430
1360
|
"create_schedule",
|
|
431
1361
|
"Create a new recurring test schedule. Runs all matching tests from the database on the configured interval.",
|
|
432
1362
|
{
|
|
433
|
-
name:
|
|
434
|
-
mode:
|
|
435
|
-
intervalMinutes:
|
|
436
|
-
tags:
|
|
437
|
-
autoHeal:
|
|
438
|
-
enabled:
|
|
439
|
-
runImmediately:
|
|
1363
|
+
name: z2.string().describe('Name for the schedule (e.g. "Nightly smoke tests")'),
|
|
1364
|
+
mode: z2.enum(["batch"]).describe("Run mode: batch (runs project tests from the database)"),
|
|
1365
|
+
intervalMinutes: z2.union([z2.literal(60), z2.literal(360), z2.literal(720), z2.literal(1440), z2.literal(2880), z2.literal(10080)]).describe("Interval in minutes: 60 (hourly), 360 (6h), 720 (12h), 1440 (daily), 2880 (2 days), 10080 (weekly)"),
|
|
1366
|
+
tags: z2.array(z2.string()).optional().describe('Tags to filter which tests run (e.g. ["@smoke"]). Omit to run all ACTIVE tests.'),
|
|
1367
|
+
autoHeal: z2.boolean().optional().describe("Heal this schedule's failed runs (default false). Takes effect only while the project's auto-heal (toggle_auto_heal) is on: a run is healed only when both are on."),
|
|
1368
|
+
enabled: z2.boolean().optional().describe("Whether the schedule is active (default: true)"),
|
|
1369
|
+
runImmediately: z2.boolean().optional().describe("Whether to trigger the first run immediately (default: true)")
|
|
440
1370
|
}
|
|
441
1371
|
);
|
|
442
1372
|
tool(
|
|
@@ -445,13 +1375,13 @@ function registerAllTools(server2, client2) {
|
|
|
445
1375
|
"update_schedule",
|
|
446
1376
|
"Update an existing test schedule.",
|
|
447
1377
|
{
|
|
448
|
-
scheduleId:
|
|
449
|
-
name:
|
|
450
|
-
intervalMinutes:
|
|
451
|
-
suite:
|
|
452
|
-
tags:
|
|
453
|
-
autoHeal:
|
|
454
|
-
enabled:
|
|
1378
|
+
scheduleId: z2.string().describe("The schedule ID to update"),
|
|
1379
|
+
name: z2.string().optional().describe("New name for the schedule"),
|
|
1380
|
+
intervalMinutes: z2.union([z2.literal(60), z2.literal(360), z2.literal(720), z2.literal(1440), z2.literal(2880), z2.literal(10080)]).optional().describe("New interval in minutes"),
|
|
1381
|
+
suite: z2.string().optional().describe("New suite name to filter tests"),
|
|
1382
|
+
tags: z2.array(z2.string()).optional().describe("New tag filter"),
|
|
1383
|
+
autoHeal: z2.boolean().optional().describe("Heal this schedule's failed runs. Takes effect only while the project's auto-heal (toggle_auto_heal) is on."),
|
|
1384
|
+
enabled: z2.boolean().optional().describe("Enable or disable the schedule")
|
|
455
1385
|
}
|
|
456
1386
|
);
|
|
457
1387
|
tool(
|
|
@@ -459,7 +1389,7 @@ function registerAllTools(server2, client2) {
|
|
|
459
1389
|
client2,
|
|
460
1390
|
"delete_schedule",
|
|
461
1391
|
"Delete (remove) a test schedule permanently.",
|
|
462
|
-
{ scheduleId:
|
|
1392
|
+
{ scheduleId: z2.string().describe("The schedule ID to delete") },
|
|
463
1393
|
{ destructiveHint: true }
|
|
464
1394
|
);
|
|
465
1395
|
tool(
|
|
@@ -467,7 +1397,7 @@ function registerAllTools(server2, client2) {
|
|
|
467
1397
|
client2,
|
|
468
1398
|
"run_schedule_now",
|
|
469
1399
|
"Trigger a scheduled run immediately instead of waiting for the next scheduled time.",
|
|
470
|
-
{ scheduleId:
|
|
1400
|
+
{ scheduleId: z2.string().describe("The schedule ID to trigger") }
|
|
471
1401
|
);
|
|
472
1402
|
tool(
|
|
473
1403
|
server2,
|
|
@@ -483,21 +1413,22 @@ function registerAllTools(server2, client2) {
|
|
|
483
1413
|
"set_variable",
|
|
484
1414
|
"Set or update a test variable (environment variable) for the project. Used to configure base URLs, credentials, etc.",
|
|
485
1415
|
{
|
|
486
|
-
key:
|
|
487
|
-
value:
|
|
488
|
-
secret:
|
|
1416
|
+
key: z2.string().describe("Variable name (e.g. BASE_URL, TEST_USER_EMAIL, API_KEY)"),
|
|
1417
|
+
value: z2.string().describe("Variable value. Set value to empty string to remove a variable."),
|
|
1418
|
+
secret: z2.boolean().optional().describe("Whether this variable is a secret (will be redacted in UI)")
|
|
489
1419
|
}
|
|
490
1420
|
);
|
|
491
1421
|
tool(
|
|
492
1422
|
server2,
|
|
493
1423
|
client2,
|
|
494
1424
|
"create_test",
|
|
495
|
-
"Create a new test case
|
|
1425
|
+
"Create a new test case. WEB projects: provide playwrightCode (read project://test-conventions for templates BEFORE writing code; on a production BASE_URL use its read-only guard template). The response may carry warnings (weak selectors, hardcoded hosts in URL assertions, vacuous checks, a missing read-only guard) - they never block, fix them with edit_test_code. MOBILE (Appium) projects: provide steps - a structured step array is the ONLY executable form for mobile (code is never executed). The test starts as DRAFT - auto-runs and auto-promotes to ACTIVE on pass. IMPORTANT: (1) Set suite to the EXACT feature name. (2) After creating all tests for a web feature, you MUST call add_api_endpoints.",
|
|
496
1426
|
{
|
|
497
|
-
name:
|
|
498
|
-
playwrightCode:
|
|
499
|
-
|
|
500
|
-
|
|
1427
|
+
name: z2.string().describe('Test name (e.g. "Login - Happy Path")'),
|
|
1428
|
+
playwrightCode: z2.string().optional().describe("Web projects: full Playwright test code (required). Mobile projects: optional WebdriverIO export for humans - never executed."),
|
|
1429
|
+
steps: z2.array(z2.record(z2.string(), z2.unknown())).optional().describe('Mobile (Appium) projects ONLY - required. Ordered steps: { action: tap|type|clear|swipe|scroll|back|home|waitForElement|assertVisible|assertText, description, target?: { primary, fallbacks?, screenName? }, value? }. target.primary: "css=#domId" for WebView-rendered (hybrid) controls, resource-id/accessibility-id/text= only for genuinely native UI. type/assertText require value ({{TEST_VAR}} placeholders supported). Do not author OS permission-dialog taps. End with an assert step. Call get_feature first for mapped screens + known selectors.'),
|
|
1430
|
+
suite: z2.string().optional().describe('MUST match an existing feature name exactly to link the test to that feature (e.g. "Authentication", not "Auth Tests")'),
|
|
1431
|
+
tags: z2.array(z2.string()).optional().describe('Tags (e.g. ["@smoke", "@auth"])')
|
|
501
1432
|
}
|
|
502
1433
|
);
|
|
503
1434
|
tool(
|
|
@@ -505,19 +1436,30 @@ function registerAllTools(server2, client2) {
|
|
|
505
1436
|
client2,
|
|
506
1437
|
"get_feature",
|
|
507
1438
|
"Get detailed info about a feature including its pages, test plans, existing tests with lastRunStatus, and AI insights.",
|
|
508
|
-
{ featureName:
|
|
1439
|
+
{ featureName: z2.string().describe("The feature name (case-insensitive match)") },
|
|
1440
|
+
{ readOnlyHint: true }
|
|
1441
|
+
);
|
|
1442
|
+
tool(
|
|
1443
|
+
server2,
|
|
1444
|
+
client2,
|
|
1445
|
+
"get_spec_coverage",
|
|
1446
|
+
`Map spec items to the existing tests of a feature (heuristic: test names, tags and the copy in their code) and list the items no test covers yet. Pass the spec you were given as one item per line item, scoped by page ("Sign up: malformed email shows an error"); without specItems it uses the feature's stored test plans. Call it before you stop writing tests for a feature, then cover every uncovered item.`,
|
|
1447
|
+
{
|
|
1448
|
+
featureName: z2.string().describe("The feature name (case-insensitive) - its suite holds the tests"),
|
|
1449
|
+
specItems: z2.array(z2.string()).optional().describe("Spec items, one per behaviour and page (max 200)")
|
|
1450
|
+
},
|
|
509
1451
|
{ readOnlyHint: true }
|
|
510
1452
|
);
|
|
511
1453
|
tool(
|
|
512
1454
|
server2,
|
|
513
1455
|
client2,
|
|
514
1456
|
"create_feature",
|
|
515
|
-
"Create a new feature with page routes. Read the developer's route files to provide accurate page URLs. Pages are used for coverage tracking and site map.",
|
|
1457
|
+
"Create a new feature with page routes. Read the developer's route files to provide accurate page URLs. Pages are used for coverage tracking and site map. Idempotent, not an upsert: if an active feature with this name already exists it is returned unchanged with existing: true (use update_feature with its id to change it). Fails only when the name belongs to an archived feature (restore it with restore_feature, or pick another name).",
|
|
516
1458
|
{
|
|
517
|
-
name:
|
|
518
|
-
description:
|
|
519
|
-
pages:
|
|
520
|
-
criticality:
|
|
1459
|
+
name: z2.string().describe('Feature name (e.g. "Checkout Flow")'),
|
|
1460
|
+
description: z2.string().optional().describe("What the feature does and what to test"),
|
|
1461
|
+
pages: z2.array(z2.string()).describe('Page routes this feature covers (e.g. ["/checkout", "/checkout/payment"]). Required for test generation.'),
|
|
1462
|
+
criticality: z2.enum(["critical", "high", "medium", "low"]).optional().describe("Feature criticality level (default: medium)")
|
|
521
1463
|
}
|
|
522
1464
|
);
|
|
523
1465
|
tool(
|
|
@@ -525,22 +1467,22 @@ function registerAllTools(server2, client2) {
|
|
|
525
1467
|
client2,
|
|
526
1468
|
"get_test_code",
|
|
527
1469
|
"Get the full (untruncated) Playwright test code for a test case. Use this when you need to read, analyze, or prepare edits for the complete test code.",
|
|
528
|
-
{ testId:
|
|
1470
|
+
{ testId: z2.string().describe("The test case ID") },
|
|
529
1471
|
{ readOnlyHint: true }
|
|
530
1472
|
);
|
|
531
1473
|
tool(
|
|
532
1474
|
server2,
|
|
533
1475
|
client2,
|
|
534
1476
|
"set_runner_preference",
|
|
535
|
-
"Switch between cloud and local runner for test execution.",
|
|
536
|
-
{ preference:
|
|
1477
|
+
"Switch between cloud and local runner for test execution. Copilot (browser session) only; access tokens and MCP connectors get 403 \u2014 change the runner in Project Settings.",
|
|
1478
|
+
{ preference: z2.enum(["cloud", "local"]).describe("Runner type: cloud (shared infrastructure) or local (runs on your machine)") }
|
|
537
1479
|
);
|
|
538
1480
|
tool(
|
|
539
1481
|
server2,
|
|
540
1482
|
client2,
|
|
541
1483
|
"toggle_auto_heal",
|
|
542
|
-
"Enable or disable automatic AI healing when tests fail
|
|
543
|
-
{ enabled:
|
|
1484
|
+
"Enable or disable automatic AI healing when tests fail: the verification run after create_test, batch and suite runs. When off, no heal is queued automatically; heal_test still works.",
|
|
1485
|
+
{ enabled: z2.boolean().describe("true to enable auto-heal, false to disable") }
|
|
544
1486
|
);
|
|
545
1487
|
tool(
|
|
546
1488
|
server2,
|
|
@@ -556,9 +1498,9 @@ function registerAllTools(server2, client2) {
|
|
|
556
1498
|
"create_test_data",
|
|
557
1499
|
"Create a new test data set with key-value pairs. Max 200 entries per set.",
|
|
558
1500
|
{
|
|
559
|
-
name:
|
|
560
|
-
description:
|
|
561
|
-
data:
|
|
1501
|
+
name: z2.string().describe("Name of the test data set (max 100 chars)"),
|
|
1502
|
+
description: z2.string().optional().describe("Optional description (max 1000 chars)"),
|
|
1503
|
+
data: z2.record(z2.string(), z2.string()).describe('Key-value pairs (e.g. {"email": "test@example.com", "password": "secret"})')
|
|
562
1504
|
}
|
|
563
1505
|
);
|
|
564
1506
|
tool(
|
|
@@ -566,7 +1508,7 @@ function registerAllTools(server2, client2) {
|
|
|
566
1508
|
client2,
|
|
567
1509
|
"delete_test_data",
|
|
568
1510
|
"Delete a test data set by ID.",
|
|
569
|
-
{ dataSetId:
|
|
1511
|
+
{ dataSetId: z2.string().describe("The test data set ID to delete") },
|
|
570
1512
|
{ destructiveHint: true }
|
|
571
1513
|
);
|
|
572
1514
|
tool(
|
|
@@ -575,21 +1517,28 @@ function registerAllTools(server2, client2) {
|
|
|
575
1517
|
"update_feature",
|
|
576
1518
|
"Update a feature's name, description, pages, or criticality.",
|
|
577
1519
|
{
|
|
578
|
-
featureId:
|
|
579
|
-
name:
|
|
580
|
-
description:
|
|
581
|
-
pages:
|
|
582
|
-
criticality:
|
|
1520
|
+
featureId: z2.string().describe("The feature ID to update"),
|
|
1521
|
+
name: z2.string().optional().describe("New feature name"),
|
|
1522
|
+
description: z2.string().optional().describe("New description"),
|
|
1523
|
+
pages: z2.array(z2.string()).optional().describe("Updated page routes"),
|
|
1524
|
+
criticality: z2.enum(["critical", "high", "medium", "low"]).optional().describe("Feature criticality level")
|
|
583
1525
|
}
|
|
584
1526
|
);
|
|
585
1527
|
tool(
|
|
586
1528
|
server2,
|
|
587
1529
|
client2,
|
|
588
1530
|
"archive_feature",
|
|
589
|
-
"Archive a feature (soft delete). Archived features are hidden from the dashboard but can be restored.",
|
|
590
|
-
{ featureId:
|
|
1531
|
+
"Archive a feature (soft delete). Archived features are hidden from the dashboard but can be restored with restore_feature.",
|
|
1532
|
+
{ featureId: z2.string().describe("The feature ID to archive") },
|
|
591
1533
|
{ destructiveHint: true }
|
|
592
1534
|
);
|
|
1535
|
+
tool(
|
|
1536
|
+
server2,
|
|
1537
|
+
client2,
|
|
1538
|
+
"restore_feature",
|
|
1539
|
+
"Restore an archived feature so it shows in the dashboard again. Does nothing for a feature that is already active. Fails if an active feature now holds the same name.",
|
|
1540
|
+
{ featureId: z2.string().describe("The ID of the archived feature to restore") }
|
|
1541
|
+
);
|
|
593
1542
|
tool(
|
|
594
1543
|
server2,
|
|
595
1544
|
client2,
|
|
@@ -598,12 +1547,33 @@ function registerAllTools(server2, client2) {
|
|
|
598
1547
|
{},
|
|
599
1548
|
{ readOnlyHint: true }
|
|
600
1549
|
);
|
|
1550
|
+
tool(
|
|
1551
|
+
server2,
|
|
1552
|
+
client2,
|
|
1553
|
+
"get_app_map",
|
|
1554
|
+
'Get the app map: the features, sub-features, user journeys and page states found for this project, each with its routes, sign-in need, what it writes (orders, bookings, sign-ups, payments, messages), where it came from (survey, code, bundle, network, the customer) and whether the customer ticked it. Returns { map: null } before the first build. Use format "eval" for the compact features/journeys/states export.',
|
|
1555
|
+
{ format: z2.enum(["eval"]).optional().describe('"eval" returns the compact features/journeys/states export instead of the full map') },
|
|
1556
|
+
{ readOnlyHint: true }
|
|
1557
|
+
);
|
|
601
1558
|
tool(
|
|
602
1559
|
server2,
|
|
603
1560
|
client2,
|
|
604
1561
|
"list_bugs",
|
|
605
|
-
"Get all detected bugs: app bugs (
|
|
606
|
-
{},
|
|
1562
|
+
"Get all detected bugs: app bugs (confirmed, and waiting for a person), security findings, flaky tests, and visual issues flagged as bugs. appBugs holds at most `limit` bugs (default 50, max 200); appBugsTruncated says when the project has more, summary.totalBugs is the full count.",
|
|
1563
|
+
{ limit: z2.number().int().min(1).max(200).optional().describe("Max app bugs to return (default 50, max 200). When appBugsTruncated is true, ask again with a higher limit.") },
|
|
1564
|
+
{ readOnlyHint: true }
|
|
1565
|
+
);
|
|
1566
|
+
tool(
|
|
1567
|
+
server2,
|
|
1568
|
+
client2,
|
|
1569
|
+
"list_visual_issues",
|
|
1570
|
+
"List visual issues found on the app's pages by the layout checks that run during surveys and discovery (clipped or overflowing text, covered or cut-off controls, broken images, sideways scrolling on phones, small tap targets, low contrast). Each has the page route, viewport, CSS selector, element box and a description, so you can find and fix the element in the code. Visual issues never fail tests.",
|
|
1571
|
+
{
|
|
1572
|
+
status: z2.enum(["OPEN", "ACCEPTED", "DISMISSED", "GONE"]).optional().describe("Filter by status (default OPEN). ACCEPTED = flagged as a bug; GONE = no longer seen on the page"),
|
|
1573
|
+
severity: z2.enum(["high", "medium", "low"]).optional().describe("Filter by severity"),
|
|
1574
|
+
viewport: z2.enum(["desktop", "phone"]).optional().describe("Filter by viewport"),
|
|
1575
|
+
route: z2.string().optional().describe("Only this page route, e.g. /pricing")
|
|
1576
|
+
},
|
|
607
1577
|
{ readOnlyHint: true }
|
|
608
1578
|
);
|
|
609
1579
|
tool(
|
|
@@ -612,8 +1582,8 @@ function registerAllTools(server2, client2) {
|
|
|
612
1582
|
"flag_as_bug",
|
|
613
1583
|
"Mark a failing test as a confirmed application bug - quarantines it with @bug-found and records the bug description. Use when the failure is caused by broken app behaviour, NOT a test script issue. Do NOT use for selector changes or timeout failures - use heal_test or edit_test_code for those.",
|
|
614
1584
|
{
|
|
615
|
-
testId:
|
|
616
|
-
description:
|
|
1585
|
+
testId: z2.string().describe("The failing test case ID"),
|
|
1586
|
+
description: z2.string().describe('What is broken in the application (e.g. "POST /api/checkout returns 500 when payment method is valid")')
|
|
617
1587
|
}
|
|
618
1588
|
);
|
|
619
1589
|
tool(
|
|
@@ -629,8 +1599,9 @@ function registerAllTools(server2, client2) {
|
|
|
629
1599
|
"list_security_findings",
|
|
630
1600
|
"List security audit findings with optional filters. Returns severity, category, status, and affected endpoints.",
|
|
631
1601
|
{
|
|
632
|
-
severity:
|
|
633
|
-
status:
|
|
1602
|
+
severity: z2.enum(["critical", "high", "medium", "low", "info"]).optional().describe("Filter by severity"),
|
|
1603
|
+
status: z2.enum(["open", "acknowledged", "resolved", "false-positive", "false-positive-auto"]).optional().describe("Filter by status"),
|
|
1604
|
+
verificationStatus: z2.enum(["confirmed", "false-positive-auto", "unverifiable", "error"]).optional().describe("Filter by verification probe result")
|
|
634
1605
|
},
|
|
635
1606
|
{ readOnlyHint: true }
|
|
636
1607
|
);
|
|
@@ -639,7 +1610,7 @@ function registerAllTools(server2, client2) {
|
|
|
639
1610
|
client2,
|
|
640
1611
|
"set_app_brief",
|
|
641
1612
|
"Set or update the project's application brief - a description of the app used by AI for better test generation.",
|
|
642
|
-
{ appBrief:
|
|
1613
|
+
{ appBrief: z2.string().describe("Application description. Set empty to clear.") }
|
|
643
1614
|
);
|
|
644
1615
|
tool(
|
|
645
1616
|
server2,
|
|
@@ -653,9 +1624,9 @@ function registerAllTools(server2, client2) {
|
|
|
653
1624
|
server2,
|
|
654
1625
|
client2,
|
|
655
1626
|
"run_draft_tests",
|
|
656
|
-
"Run
|
|
1627
|
+
"Run DRAFT tests to verify them, 50 at most per call (oldest first; hasMore says when more remain). Tests that pass auto-promote to ACTIVE. Optionally scope to a feature. Returns a batchId: wait with wait_for_batch.",
|
|
657
1628
|
{
|
|
658
|
-
featureName:
|
|
1629
|
+
featureName: z2.string().optional().describe("Only run drafts for this feature (optional)")
|
|
659
1630
|
}
|
|
660
1631
|
);
|
|
661
1632
|
tool(
|
|
@@ -664,17 +1635,17 @@ function registerAllTools(server2, client2) {
|
|
|
664
1635
|
"setup_project",
|
|
665
1636
|
'One-shot project setup from codebase knowledge. Sets app brief, base URL, test credentials, and creates features with pages - all in one call. Use this when a developer says "set up test coverage for my app."',
|
|
666
1637
|
{
|
|
667
|
-
appBrief:
|
|
668
|
-
baseUrl:
|
|
669
|
-
testCredentials:
|
|
670
|
-
email:
|
|
671
|
-
password:
|
|
1638
|
+
appBrief: z2.string().optional().describe("Description of the application (what it does, key features, tech stack)"),
|
|
1639
|
+
baseUrl: z2.string().describe('Application base URL (e.g. "https://myapp.com" or "http://localhost:3000")'),
|
|
1640
|
+
testCredentials: z2.object({
|
|
1641
|
+
email: z2.string().optional().describe("Test user email"),
|
|
1642
|
+
password: z2.string().optional().describe("Test user password")
|
|
672
1643
|
}).optional().describe("Login credentials for test execution"),
|
|
673
|
-
features:
|
|
674
|
-
name:
|
|
675
|
-
description:
|
|
676
|
-
pages:
|
|
677
|
-
criticality:
|
|
1644
|
+
features: z2.array(z2.object({
|
|
1645
|
+
name: z2.string().describe("Feature name"),
|
|
1646
|
+
description: z2.string().optional().describe("What the feature does"),
|
|
1647
|
+
pages: z2.array(z2.string()).describe('Page routes (e.g. ["/login", "/register"])'),
|
|
1648
|
+
criticality: z2.enum(["critical", "high", "medium", "low"]).optional().describe("Feature criticality")
|
|
678
1649
|
})).describe("Features to create with their page routes")
|
|
679
1650
|
}
|
|
680
1651
|
);
|
|
@@ -684,13 +1655,13 @@ function registerAllTools(server2, client2) {
|
|
|
684
1655
|
"add_api_endpoints",
|
|
685
1656
|
"Register API endpoints for a feature from codebase knowledge. Read the developer's route handlers to extract methods, paths, request/response shapes. Populates the API intelligence dashboard.",
|
|
686
1657
|
{
|
|
687
|
-
featureName:
|
|
688
|
-
endpoints:
|
|
689
|
-
method:
|
|
690
|
-
path:
|
|
691
|
-
requestShape:
|
|
692
|
-
responseShapes:
|
|
693
|
-
requiresAuth:
|
|
1658
|
+
featureName: z2.string().describe("Feature name to link these endpoints to"),
|
|
1659
|
+
endpoints: z2.array(z2.object({
|
|
1660
|
+
method: z2.enum(["GET", "POST", "PUT", "PATCH", "DELETE"]).describe("HTTP method"),
|
|
1661
|
+
path: z2.string().describe('API path (e.g. "/api/auth/login")'),
|
|
1662
|
+
requestShape: z2.record(z2.string(), z2.string()).optional().describe('Request body fields with types (e.g. { "email": "string" })'),
|
|
1663
|
+
responseShapes: z2.record(z2.string(), z2.record(z2.string(), z2.string())).optional().describe('Response shapes by status code (e.g. { "200": { "token": "string" } })'),
|
|
1664
|
+
requiresAuth: z2.boolean().optional().describe("Whether endpoint requires authentication")
|
|
694
1665
|
})).describe("API endpoints to register")
|
|
695
1666
|
}
|
|
696
1667
|
);
|
|
@@ -700,10 +1671,10 @@ function registerAllTools(server2, client2) {
|
|
|
700
1671
|
"get_test_failures",
|
|
701
1672
|
"Get recent test failures with full error messages, Playwright code, step-level failures, and heal context. Use this to understand why tests are failing.",
|
|
702
1673
|
{
|
|
703
|
-
limit:
|
|
704
|
-
suite:
|
|
705
|
-
mode:
|
|
706
|
-
sinceHours:
|
|
1674
|
+
limit: z2.number().optional().describe("Max failures to return (default: 50, max: 100)"),
|
|
1675
|
+
suite: z2.string().optional().describe("Filter by feature/suite name"),
|
|
1676
|
+
mode: z2.enum(["run", "heal", "all"]).optional().describe("run = regular failures, heal = heal attempt failures, all = both (default)"),
|
|
1677
|
+
sinceHours: z2.number().optional().describe("Only failures from the last N hours (default: 24)")
|
|
707
1678
|
},
|
|
708
1679
|
{ readOnlyHint: true }
|
|
709
1680
|
);
|
|
@@ -715,11 +1686,16 @@ if (process.argv[2] === "init") {
|
|
|
715
1686
|
runInit2();
|
|
716
1687
|
process.exit(0);
|
|
717
1688
|
}
|
|
1689
|
+
if (process.argv[2] === "init-agents") {
|
|
1690
|
+
const { runInitAgents: runInitAgents2 } = await Promise.resolve().then(() => (init_init_agents(), init_agents_exports));
|
|
1691
|
+
await runInitAgents2();
|
|
1692
|
+
process.exit(0);
|
|
1693
|
+
}
|
|
718
1694
|
var config = resolveConfig();
|
|
719
1695
|
var client = new ValidateQAClient(config.apiUrl, config.token, config.projectId);
|
|
720
1696
|
var server = new McpServer(
|
|
721
|
-
{ name: "validate.qa", version: "1.
|
|
722
|
-
{ instructions: `Validate.QA - AI-powered QA platform: generates, runs, and heals Playwright E2E tests.
|
|
1697
|
+
{ name: "validate.qa", version: "1.2.0" },
|
|
1698
|
+
{ instructions: config.toolset === "agent" ? AGENT_INSTRUCTIONS : `Validate.QA - AI-powered QA platform: generates, runs, and heals Playwright E2E tests.
|
|
723
1699
|
|
|
724
1700
|
## Capabilities
|
|
725
1701
|
- Setup projects via setup_project (read source code first)
|
|
@@ -733,12 +1709,13 @@ var server = new McpServer(
|
|
|
733
1709
|
2. Check health: list_features (failingTests > 0), get_test_failures per feature, edit_test_code or heal_test.
|
|
734
1710
|
3. Add coverage: READ code, create_feature, create_test x N (suite = feature name), add_api_endpoints.
|
|
735
1711
|
4. Nightly automation: create_schedule (daily, autoHeal: true).
|
|
736
|
-
5. After code change: batch_run_tests, then list_features + get_test_failures.
|
|
1712
|
+
5. After code change: batch_run_tests (waitSeconds: 120, or then wait_for_batch), then list_features + get_test_failures.
|
|
1713
|
+
6. Find and lock in bugs in a feature: start_here({ feature }), explore with your browser, save_regression_test for each bug (returns at once, keep exploring), then collect_verdicts.
|
|
737
1714
|
|
|
738
1715
|
## Automatic lifecycle
|
|
739
|
-
Do NOT call run_test after create_test (auto-queued) or heal_test after failures (auto-heals).
|
|
740
|
-
Do NOT
|
|
741
|
-
Lifecycle: create > auto-run > pass=ACTIVE, fail=auto-heal > heal fail=@needs-review.
|
|
1716
|
+
Do NOT call run_test after create_test (auto-queued) or heal_test after failures (auto-heals, unless toggle_auto_heal turned it off).
|
|
1717
|
+
Do NOT poll get_test_run in a loop. To wait for a run, call wait_for_run(runId) (or wait_for_batch(batchId)); it blocks until the run finishes or timeoutSeconds passes (max 120) and returns timedOut: true if it is still going - then call it again. run_test and batch_run_tests also take waitSeconds to queue and wait in one call. A FAILED run comes back with its error, consoleLog and failedSteps.
|
|
1718
|
+
Lifecycle: create > auto-run > pass=ACTIVE, fail=auto-heal > heal fail=@needs-review. With toggle_auto_heal(false), no failure is auto-healed; call heal_test yourself when you want one.
|
|
742
1719
|
DO call add_api_endpoints per feature (not automatic).
|
|
743
1720
|
DO call run_security_audit after UI tests to analyze captured API traffic. If it errors about missing data, tell the developer to click "Security Audit" in the Dashboard to run a live browser capture first. Use list_security_findings to review results.
|
|
744
1721
|
|
|
@@ -762,23 +1739,27 @@ Read project://test-conventions first. Check the variables section:
|
|
|
762
1739
|
## Critical rules - see project://test-conventions for full details and code templates
|
|
763
1740
|
- URLs: relative only, never hardcoded domains - runner sets baseURL
|
|
764
1741
|
- Credentials: process.env.TEST_USER_EMAIL! (non-null assertion) - never ?? or || fallbacks
|
|
765
|
-
- Registration: testuser-\${Date.now()}@example.
|
|
1742
|
+
- Registration: testuser-\${Date.now()}@example.test when the sign-up sends no email; if it emails a code or link, register with process.env.VQA_TEST_INBOX_EMAIL! / process.env.VQA_TEST_ACCOUNT_PASSWORD! and read it with await __vqaWaitForEmail('code' | 'link'), first asserting the app's 'email sent' state (code field visible / 'check your email') before each call (runner-injected; see test-conventions)
|
|
766
1743
|
- Suite name must match feature name exactly (case-sensitive)
|
|
767
1744
|
- @auth must NOT combine with @smoke/@regression/@needs-auth/@critical
|
|
768
1745
|
- Each test = own browser process; @needs-auth tests MUST login inline
|
|
769
1746
|
- API tests: test.describe.serial() with __QA_HTTP__ log after every request
|
|
770
1747
|
- UI tests: console.log() milestones after key actions
|
|
1748
|
+
- Selectors: getByRole(role, { name, exact: true }), getByLabel(label, { exact: true }), getByText(text, { exact: true }); CSS ids only for nodes with no accessible name
|
|
1749
|
+
- URL assertions: path only - await expect(page).toHaveURL(/\\/dashboard$/) - never a host
|
|
1750
|
+
- Production BASE_URL: read-only tests get the read-only guard (page.route aborts auth/mutation requests and fails the test) - template in project://test-conventions
|
|
1751
|
+
- Every assertion must be able to fail: no toHaveCount(0) on transient text, no toBeHidden on an element never shown, full message text instead of a toContainText fragment
|
|
771
1752
|
- create_test/edit_test_code validate code - fix warnings before moving on
|
|
772
1753
|
|
|
773
1754
|
## Setting up coverage
|
|
774
1755
|
1. SCAN: Read router, nav, pages, API handlers - map ALL routes and features.
|
|
775
1756
|
2. SETUP: setup_project with features, pages, baseUrl, testCredentials, appBrief.
|
|
776
1757
|
3. WRITE: One feature at a time. For EACH test, read the SPECIFIC source code first (see codeReadingRequirements in test-conventions). For login blocks: read the login component. For UI tests: read the page component. For API tests: read the route handler. Templates are structure guides - selectors MUST come from the real code. Use create_test for BOTH UI and API tests. Then call add_api_endpoints.
|
|
777
|
-
4. VERIFY: get_coverage_summary + list_features for failures + get_test_failures.
|
|
1758
|
+
4. VERIFY: get_spec_coverage(featureName, specItems) - write tests for every uncovered item - then get_coverage_summary + list_features for failures + get_test_failures.
|
|
778
1759
|
|
|
779
1760
|
Test real user flows (forms, validation, errors). "Page loads" tests are worthless.` }
|
|
780
1761
|
);
|
|
781
|
-
registerAllTools(server, client);
|
|
1762
|
+
registerAllTools(server, client, config.toolset);
|
|
782
1763
|
server.resource(
|
|
783
1764
|
"Project Overview",
|
|
784
1765
|
"project://overview",
|
|
@@ -807,6 +1788,10 @@ server.resource(
|
|
|
807
1788
|
"project://test-conventions",
|
|
808
1789
|
{ description: "Project-specific test conventions: environment variables, credential patterns, tag meanings, and registration email rules. Read this before writing or editing any test." },
|
|
809
1790
|
async () => {
|
|
1791
|
+
const served = await client.getTestConventions();
|
|
1792
|
+
if (served && typeof served === "object") {
|
|
1793
|
+
return { contents: [{ uri: "project://test-conventions", mimeType: "application/json", text: JSON.stringify(served, null, 2) }] };
|
|
1794
|
+
}
|
|
810
1795
|
try {
|
|
811
1796
|
const vars = await client.executeTool("get_variables");
|
|
812
1797
|
const varNames = Object.keys(vars.variables || {});
|
|
@@ -814,7 +1799,7 @@ server.resource(
|
|
|
814
1799
|
const conventions = {
|
|
815
1800
|
codeReadingRequirements: {
|
|
816
1801
|
CRITICAL: "Templates below are STRUCTURE GUIDES - every selector, route, and endpoint MUST come from reading the actual source code. Never copy a template verbatim.",
|
|
817
|
-
forLogin: "BEFORE writing any @needs-auth login block: READ the login/auth page component. Find: (1) the login route (/login, /auth/login, /signin?), (2) the flow type (single-step or multi-step with Continue/Next button?), (3)
|
|
1802
|
+
forLogin: "BEFORE writing any @needs-auth login block: READ the login/auth page component. Find: (1) the login route (/login, /auth/login, /signin?), (2) the flow type (single-step or multi-step with Continue/Next button?), (3) each field's label text for getByLabel(label, { exact: true }) - fall back to the placeholder or input[type=password] only when the field has no label, (4) the submit button's accessible name for getByRole('button', { name, exact: true }), (5) post-login redirect URL. Every app is different - single-step, multi-step, OAuth buttons, magic links.",
|
|
818
1803
|
forUITests: "BEFORE writing a UI test for a page: READ the page component. Find: headings (text + level), form fields (labels, placeholders, types), buttons (text, role), navigation elements, validation error messages, success/error states. Use the exact text from the component - do not guess.",
|
|
819
1804
|
forAPITests: "BEFORE writing an API test: READ the route handler. Find: the endpoint path, HTTP method, request body schema (what fields, what types), response body shape (is it flat or wrapped in { data: {...} }?), auth requirements (Bearer token? Cookie? API key?), error response shapes.",
|
|
820
1805
|
forAPIMapping: "BEFORE calling add_api_endpoints: READ ALL route handlers for the feature. Register every endpoint with accurate method, path, requestShape, responseShapes, requiresAuth."
|
|
@@ -822,7 +1807,8 @@ server.resource(
|
|
|
822
1807
|
credentials: {
|
|
823
1808
|
rule: "Never hardcode credentials in test code",
|
|
824
1809
|
loginFields: hasAuth ? "Use process.env.TEST_USER_EMAIL! and process.env.TEST_USER_PASSWORD!" : "No credentials configured - set TEST_USER_EMAIL and TEST_USER_PASSWORD via set_variable",
|
|
825
|
-
registrationFields: 'MUST use unique email per run: `testuser-${Date.now()}@example.
|
|
1810
|
+
registrationFields: 'MUST use unique email per run: `testuser-${Date.now()}@example.test` - reusing emails causes "already registered" failures. When the sign-up emails a verification code or link, register with `process.env.VQA_TEST_INBOX_EMAIL!` / `process.env.VQA_TEST_ACCOUNT_PASSWORD!` (a fresh real inbox address the runner injects per run) instead',
|
|
1811
|
+
emailedCodesAndLinks: "Read emails at runtime with the runner-injected helpers (never define them): `const code = await __vqaWaitForEmail('code'); await __vqaEnterVerificationCode(page, code);` or `await page.goto(await __vqaWaitForEmail('link'));` for the run's sign-up identity; `await __vqaWaitForEmail('code', { to: process.env.TEST_USER_EMAIL })` for the saved account's sign-in code or magic link (works only when TEST_USER_EMAIL is the project's Validate.QA inbox address). Each call returns an email no earlier call returned. Assert the app's 'email sent' state (code field visible / 'check your email') before each __vqaWaitForEmail call, so a send error fails at once instead of after the inbox wait. Never hardcode or guess a code.",
|
|
826
1812
|
registrationPassword: "Use 'TestPass123!' for new account passwords (meets typical strength rules)"
|
|
827
1813
|
},
|
|
828
1814
|
variables: {
|
|
@@ -845,7 +1831,7 @@ server.resource(
|
|
|
845
1831
|
"@heal-skip": "Added by healer when test is permanently unfixable. Prevents future auto-heal",
|
|
846
1832
|
"@rate-limit-blocked": "Auto-added by pre-heal classifier when test failure was a rate-limit (HTTP 429). Test skips heal until next pass clears the tag. NOT for use by test authors.",
|
|
847
1833
|
"@quota-exceeded": "Auto-added when failure is a quota/daily-limit response. Skips heal; clears on next pass.",
|
|
848
|
-
"@auth-lost": "Auto-added when
|
|
1834
|
+
"@auth-lost": "Auto-added when a signed-in test (@needs-auth, authed @api, or a stored-login body) fails on a 401/403 or falls back to the login page. Never on @auth / login-form tests. Skips heal; clears after re-auth + pass.",
|
|
849
1835
|
"@rate-limit-expected": 'OPT-OUT tag for tests that intentionally exercise rate-limit behavior (e.g., "POST /api/login 6 times to trigger lockout"). Add this tag to ANY @api test that asserts rate-limit responses, otherwise the rate-limit detector will silently downgrade your failure to ERROR. Distinct from auto-pushed @rate-limit-blocked.'
|
|
850
1836
|
},
|
|
851
1837
|
executionModel: {
|
|
@@ -883,7 +1869,8 @@ server.resource(
|
|
|
883
1869
|
"Use { timeout: 30000 } on waitForURL after login - target apps can be slow under concurrent test load",
|
|
884
1870
|
"Keep login blocks minimal - no assertions on the login page (that is what @auth tests are for)",
|
|
885
1871
|
"After login, page.goto() directly to the page being tested - do not navigate via UI clicks",
|
|
886
|
-
"
|
|
1872
|
+
"Selectors: getByRole(role, { name, exact: true }) for buttons/links/headings, getByLabel(label, { exact: true }) for form fields, getByText(text, { exact: true }) for static copy. CSS ids/classes only for nodes with no accessible name",
|
|
1873
|
+
"URL assertions: path only - await expect(page).toHaveURL(/\\/dashboard$/) - never a host",
|
|
887
1874
|
"Registration tests must generate unique email with Date.now()",
|
|
888
1875
|
"Group tests into suites matching feature names exactly (case-sensitive)",
|
|
889
1876
|
"Do NOT suggest storageState or performLogin() for create_test - those only work in the regression suite",
|
|
@@ -893,9 +1880,9 @@ server.resource(
|
|
|
893
1880
|
],
|
|
894
1881
|
pitfalls: {
|
|
895
1882
|
CRITICAL: "These are the top causes of test failures. Read before writing ANY test.",
|
|
896
|
-
strictModeViolations: `Playwright throws when a locator resolves to 2+ elements. Common causes: getByLabel('Password') matches input AND 'Show password' toggle; getByRole('button', { name: 'Continue' }) matches 'Continue' AND 'Continue with GitHub'. Prevention:
|
|
1883
|
+
strictModeViolations: `Playwright throws when a locator resolves to 2+ elements. Common causes: getByLabel('Password') matches input AND 'Show password' toggle; getByRole('button', { name: 'Continue' }) matches 'Continue' AND 'Continue with GitHub'. Prevention: getByLabel('Password', { exact: true }) (exact keeps it from matching 'Show password'), { exact: true } for ambiguous button text, scope to a parent container (page.locator('form#login').getByLabel('Email', { exact: true })). Use input[type="password"] only when the field has no label.`,
|
|
897
1884
|
networkIdle: "NEVER use waitForLoadState('networkidle') - waits for 500ms of zero network activity, which is fragile and causes 30s+ timeouts in apps with polling, analytics, or WebSockets. Use waitForURL() or wait for a specific element instead.",
|
|
898
|
-
headingStrictMode: "getByRole('heading', { name: 'Dashboard' }) can match multiple headings (h1 + h2 + breadcrumb). Add { level: 1 } or { exact: true } to disambiguate. Same for getByLabel('Password') matching input AND 'Show password' toggle.",
|
|
1885
|
+
headingStrictMode: "getByRole('heading', { name: 'Dashboard' }) can match multiple headings (h1 + h2 + breadcrumb). Add { level: 1 } or { exact: true } to disambiguate. Same fix for getByLabel('Password') matching the input AND a 'Show password' toggle: getByLabel('Password', { exact: true }).",
|
|
899
1886
|
postLoginRedirect: "NEVER assert exact URL after login (e.g. expect(page).toHaveURL('/dashboard')) - apps redirect to different routes per user/role. Use: await page.waitForURL(url => !url.pathname.startsWith('/login'), { timeout: 30000 })",
|
|
900
1887
|
resJsonOnce: "res.json() consumes the response body stream - calling it twice throws. Store result: const body = await res.json(); then reuse body for logging and assertions.",
|
|
901
1888
|
hiddenInputs: "Many UI frameworks use visually hidden <input> with custom overlays. page.check() fails because the overlay intercepts clicks. Click the <label> or visible wrapper: page.getByText('Remember me').click()",
|
|
@@ -911,10 +1898,10 @@ server.resource(
|
|
|
911
1898
|
// Login block - ADAPTED from reading the app's login page component
|
|
912
1899
|
// YOU MUST READ the login page component BEFORE writing this block
|
|
913
1900
|
await page.goto('/login'); // read the router for actual login route
|
|
914
|
-
await page.
|
|
915
|
-
await page.
|
|
916
|
-
await page.
|
|
917
|
-
await page.getByRole('button', { name:
|
|
1901
|
+
await page.getByLabel('Email', { exact: true }).fill(process.env.TEST_USER_EMAIL!); // label text from the component
|
|
1902
|
+
await page.getByLabel('Password', { exact: true }).waitFor({ state: 'visible', timeout: 10000 });
|
|
1903
|
+
await page.getByLabel('Password', { exact: true }).fill(process.env.TEST_USER_PASSWORD!); // input[type="password"] only if the field has no label
|
|
1904
|
+
await page.getByRole('button', { name: 'Sign in', exact: true }).click(); // the button's accessible name
|
|
918
1905
|
await page.waitForURL(url => !url.pathname.startsWith('/login'), { timeout: 30000 });
|
|
919
1906
|
console.log('Logged in, current URL:', page.url());
|
|
920
1907
|
|
|
@@ -938,21 +1925,7 @@ server.resource(
|
|
|
938
1925
|
responseShapes: { "200": { "token": "string", "user": "object" }, "401": { "error": "string" } }
|
|
939
1926
|
}]
|
|
940
1927
|
}`,
|
|
941
|
-
uiWithMilestones:
|
|
942
|
-
await page.goto('/login');
|
|
943
|
-
console.log('Navigated to login page');
|
|
944
|
-
|
|
945
|
-
await page.getByRole('textbox', { name: /email/i }).fill(process.env.TEST_USER_EMAIL!);
|
|
946
|
-
await page.locator('input[type="password"]').fill(process.env.TEST_USER_PASSWORD!);
|
|
947
|
-
await page.getByRole('button', { name: /sign in/i }).click();
|
|
948
|
-
console.log('Credentials submitted');
|
|
949
|
-
|
|
950
|
-
await page.waitForURL(url => !url.pathname.startsWith('/login'), { timeout: 15000 });
|
|
951
|
-
console.log('Logged in, current URL:', page.url());
|
|
952
|
-
|
|
953
|
-
await page.goto('/feature-path');
|
|
954
|
-
console.log('On feature page');
|
|
955
|
-
});`
|
|
1928
|
+
uiWithMilestones: "test('Feature - Happy Path', async ({ page }) => {\n await page.goto('/login');\n console.log('Navigated to login page');\n\n await page.getByLabel('Email', { exact: true }).fill(process.env.TEST_USER_EMAIL!);\n await page.getByLabel('Password', { exact: true }).fill(process.env.TEST_USER_PASSWORD!);\n await page.getByRole('button', { name: 'Sign in', exact: true }).click();\n console.log('Credentials submitted');\n\n await page.waitForURL(url => !url.pathname.startsWith('/login'), { timeout: 15000 });\n console.log('Logged in, current URL:', page.url());\n\n await page.goto('/feature-path');\n console.log('On feature page');\n});"
|
|
956
1929
|
},
|
|
957
1930
|
testQuality: [
|
|
958
1931
|
'Test actual user flows, not just "page loads" - a test that only checks goto + heading visible is nearly worthless',
|