@skyramp/mcp 0.2.3 → 0.2.5-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/playwright/registerPlaywrightTools.js +21 -25
- package/build/playwright/traceRecordingPrompt.js +2 -2
- package/build/prompts/test-maintenance/actionsInstructions.js +60 -0
- package/build/prompts/test-maintenance/drift-analysis-prompt.js +18 -101
- package/build/prompts/test-maintenance/driftAnalysisSections.js +210 -171
- package/build/prompts/test-recommendation/analysisOutputPrompt.js +1 -1
- package/build/prompts/test-recommendation/diffExecutionPlan.js +4 -3
- package/build/prompts/test-recommendation/recommendationSections.js +6 -6
- package/build/prompts/test-recommendation/scopeAssessment.js +3 -1
- package/build/prompts/test-recommendation/scopeAssessment.test.js +13 -0
- package/build/prompts/test-recommendation/test-recommendation-prompt.js +2 -2
- package/build/prompts/test-recommendation/test-recommendation-prompt.test.js +3 -3
- package/build/prompts/testbot/testbot-prompts.js +21 -17
- package/build/prompts/testbot/testbot-prompts.test.js +21 -17
- package/build/services/TestDiscoveryService.js +11 -43
- package/build/tools/submitReportTool.js +9 -12
- package/build/tools/submitReportTool.test.js +4 -5
- package/build/tools/test-management/actionsTool.js +160 -240
- package/build/tools/test-management/analyzeChangesTool.js +43 -18
- package/build/tools/test-management/analyzeTestHealthTool.js +17 -29
- package/build/utils/docker.test.js +1 -1
- package/build/utils/versions.js +1 -1
- package/node_modules/playwright/lib/mcp/skyramp/common/visualSnapshot.js +95 -0
- package/node_modules/playwright/lib/mcp/skyramp/loadTraceTool.js +2 -0
- package/node_modules/playwright/lib/mcp/skyramp/traceRecordingBackend.js +150 -2
- package/node_modules/playwright/lib/mcp/skyramp/visualSnapshotTool.js +63 -0
- package/node_modules/playwright/lib/mcp/test/skyRampExport.js +36 -0
- package/package.json +2 -2
- package/build/prompts/test-maintenance/drift-analysis-prompt.test.js +0 -116
|
@@ -358,15 +358,9 @@ export function registerAnalyzeChangesTool(server) {
|
|
|
358
358
|
idempotentHint: false,
|
|
359
359
|
openWorldHint: true, // may fetch PR comments from GitHub
|
|
360
360
|
},
|
|
361
|
-
description: `Scan repository API endpoints and discover existing tests — first step of the unified Test Health Analysis Flow.
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
**Output:** stateFile path + ranked test recommendations + enrichment instructions for calling \`skyramp_recommend_tests\`.`,
|
|
366
|
-
// TODO: Define outputSchema here instead of embedding structured output format in the
|
|
367
|
-
// description string — per Archit's review comment. outputSchema reduces token usage
|
|
368
|
-
// by letting the MCP client understand the response shape structurally rather than
|
|
369
|
-
// through natural language in the description.
|
|
361
|
+
description: `Scan repository API endpoints and discover existing tests — first step of the unified Test Health Analysis Flow. Returns a stateFile path and ranked test recommendations. Pass stateFile to skyramp_analyze_test_health and skyramp_actions.`,
|
|
362
|
+
// TODO: Replace description-embedded output format with outputSchema — structural
|
|
363
|
+
// output schema reduces token usage vs natural language in description.
|
|
370
364
|
inputSchema: analyzeChangesInputSchema,
|
|
371
365
|
}, async (params, extra) => {
|
|
372
366
|
let errorResult;
|
|
@@ -538,6 +532,14 @@ Combines API endpoint scanning, branch diff computation, and test discovery into
|
|
|
538
532
|
}
|
|
539
533
|
await sendProgress(50, 100, "Discovering existing tests...");
|
|
540
534
|
// ── Step 3: Discover existing tests ──
|
|
535
|
+
// Always scan from repositoryPath so tests in any subdirectory (e.g.
|
|
536
|
+
// apps/api/v2/src/modules/) are found regardless of workspace.yml
|
|
537
|
+
// testDirectory. In PR mode, partitionByRelevance already filters the
|
|
538
|
+
// results to files relevant to the changed endpoints — no flooding.
|
|
539
|
+
// testDirectory only controls where generation tools write new files.
|
|
540
|
+
// testsRepoDir is a cross-repo path override — honour it when set.
|
|
541
|
+
// Otherwise always scan the full repo root.
|
|
542
|
+
const testDir = params.testsRepoDir ?? undefined;
|
|
541
543
|
// Compute changedResources from classified endpoints for test discovery filtering.
|
|
542
544
|
// undefined → full-repo mode (no diff context)
|
|
543
545
|
// [] → PR mode, no endpoints found → skip external tests
|
|
@@ -552,21 +554,44 @@ Combines API endpoint scanning, branch diff computation, and test discovery into
|
|
|
552
554
|
...classifiedEndpoints.newEndpoints,
|
|
553
555
|
...classifiedEndpoints.removedEndpoints,
|
|
554
556
|
];
|
|
557
|
+
const FRAMEWORK_SUFFIX_RE = /\.(service|controller|transformer|handler|middleware|resolver|repository|module|guard|interceptor|pipe|filter|decorator|input|output|dto|schema)$/i;
|
|
555
558
|
if (allClassified.length > 0) {
|
|
556
559
|
// Scanned endpoints always have full paths — extractResourceFromPath
|
|
557
560
|
// never returns "unknown" for properly resolved paths.
|
|
561
|
+
// Exception: NestJS versioned controllers register relative paths (e.g. "GET /")
|
|
562
|
+
// which resolve to "unknown". Fall through to file-path extraction in that case.
|
|
558
563
|
const resolved = allClassified
|
|
559
564
|
.map((ep) => extractResourceFromPath(ep.path))
|
|
560
565
|
.filter((r, i, arr) => r !== "unknown" && arr.indexOf(r) === i);
|
|
561
|
-
|
|
566
|
+
if (resolved.length > 0) {
|
|
567
|
+
changedResources = resolved;
|
|
568
|
+
}
|
|
569
|
+
else {
|
|
570
|
+
// All endpoints resolved to "unknown" (e.g. NestJS relative paths) —
|
|
571
|
+
// extract resource names from changed file paths instead.
|
|
572
|
+
const candidateFiles = classifiedEndpoints.unmatchedFiles.length > 0
|
|
573
|
+
? classifiedEndpoints.unmatchedFiles
|
|
574
|
+
: classifiedEndpoints.changedFiles ?? [];
|
|
575
|
+
const fromFiles = candidateFiles
|
|
576
|
+
.map((f) => extractResourceFromPath(f.replace(/\.[^./]+$/, "").replace(FRAMEWORK_SUFFIX_RE, "")))
|
|
577
|
+
.filter((r, i, arr) => r !== "unknown" && arr.indexOf(r) === i);
|
|
578
|
+
changedResources = fromFiles.length > 0 ? fromFiles : ["unknown"];
|
|
579
|
+
}
|
|
562
580
|
}
|
|
563
581
|
else if (classifiedEndpoints.unmatchedFiles.length > 0) {
|
|
564
|
-
// Changed files don't map to any endpoint (e.g.
|
|
565
|
-
// migration
|
|
566
|
-
// tests
|
|
567
|
-
//
|
|
568
|
-
//
|
|
569
|
-
|
|
582
|
+
// Changed files don't map to any endpoint (e.g. DTO, service, output
|
|
583
|
+
// formatter, migration). Extract resource names from the file paths so
|
|
584
|
+
// external tests whose names overlap with these resources are surfaced
|
|
585
|
+
// for drift assessment.
|
|
586
|
+
// Strip the file extension and common framework suffixes (.service,
|
|
587
|
+
// .controller, .input, .output, etc.) before extracting so that
|
|
588
|
+
// "event-types.service.ts" yields "event-types" rather than
|
|
589
|
+
// "event-types.service", which would fail relevance scoring against
|
|
590
|
+
// test files that contain "event" and "types" but not "service".
|
|
591
|
+
const fromFiles = classifiedEndpoints.unmatchedFiles
|
|
592
|
+
.map((f) => extractResourceFromPath(f.replace(/\.[^./]+$/, "").replace(FRAMEWORK_SUFFIX_RE, "")))
|
|
593
|
+
.filter((r, i, arr) => r !== "unknown" && arr.indexOf(r) === i);
|
|
594
|
+
changedResources = fromFiles.length > 0 ? fromFiles : ["unknown"];
|
|
570
595
|
}
|
|
571
596
|
else {
|
|
572
597
|
changedResources = [];
|
|
@@ -577,8 +602,7 @@ Combines API endpoint scanning, branch diff computation, and test discovery into
|
|
|
577
602
|
try {
|
|
578
603
|
const testDiscoveryService = new TestDiscoveryService();
|
|
579
604
|
setTestsRepoDir(params.testsRepoDir);
|
|
580
|
-
const
|
|
581
|
-
const discoveryResult = await testDiscoveryService.discoverTests(testScanPath, { changedResources });
|
|
605
|
+
const discoveryResult = await testDiscoveryService.discoverTests(testDir ?? params.repositoryPath, { changedResources });
|
|
582
606
|
existingTests = discoveryResult.tests.map((test) => ({
|
|
583
607
|
testFile: test.testFile,
|
|
584
608
|
testType: test.testType,
|
|
@@ -1188,6 +1212,7 @@ Combines API endpoint scanning, branch diff computation, and test discovery into
|
|
|
1188
1212
|
sessionId,
|
|
1189
1213
|
routerMountContext,
|
|
1190
1214
|
candidateRouteFiles,
|
|
1215
|
+
relevantExternalTestPaths,
|
|
1191
1216
|
},
|
|
1192
1217
|
};
|
|
1193
1218
|
// Clean up old state files (>24 hours) before creating new one
|
|
@@ -15,16 +15,7 @@ export function registerAnalyzeTestHealthTool(server) {
|
|
|
15
15
|
idempotentHint: true,
|
|
16
16
|
openWorldHint: false,
|
|
17
17
|
},
|
|
18
|
-
description: `Generate drift
|
|
19
|
-
|
|
20
|
-
**PREREQUISITE:** Call \`skyramp_analyze_changes\` first to get a stateFile.
|
|
21
|
-
|
|
22
|
-
This tool reads existing tests, the branch diff, and scanned endpoints from the stateFile,
|
|
23
|
-
then returns a structured prompt for the LLM to assess each test for drift and health.
|
|
24
|
-
|
|
25
|
-
The LLM follows the returned prompt to assign drift details and actions (UPDATE / REGENERATE / VERIFY / DELETE / IGNORE) for each test, then calls \`skyramp_actions\`.
|
|
26
|
-
|
|
27
|
-
(Optional) Execute tests using \`skyramp_execute_test\` with \`stateFile\` parameter before \`skyramp_actions\` to validate tests live.`,
|
|
18
|
+
description: `Generate drift assessment instructions for existing tests — second step of the unified Test Health Analysis Flow. Optionally execute tests with skyramp_execute_test before calling skyramp_actions to capture beforeStatus.`,
|
|
28
19
|
inputSchema: {
|
|
29
20
|
stateFile: z
|
|
30
21
|
.string()
|
|
@@ -45,19 +36,24 @@ The LLM follows the returned prompt to assign drift details and actions (UPDATE
|
|
|
45
36
|
if (!stateData) {
|
|
46
37
|
return toolError(`State file is empty or invalid: ${args.stateFile}. Call skyramp_analyze_changes first to generate a valid state file.`);
|
|
47
38
|
}
|
|
48
|
-
// Only Skyramp tests are candidates for drift analysis and maintenance actions.
|
|
49
|
-
// External (user-written) tests are used only for recommendation deduplication.
|
|
50
|
-
// Default source to Skyramp for backwards compat with state files created before the source field existed.
|
|
51
|
-
const existingTests = (stateData.existingTests || []).filter((t) => (t.source ?? TestSource.Skyramp) !== TestSource.External);
|
|
52
|
-
logger.info(`Loaded ${existingTests.length} existing Skyramp tests from state file (excluded external)`);
|
|
53
39
|
if (!repositoryPath || typeof repositoryPath !== "string") {
|
|
54
40
|
return toolError(`repositoryPath not found in state file metadata. The state file was likely created by an older version — re-run skyramp_analyze_changes to regenerate it.`);
|
|
55
41
|
}
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
42
|
+
// Skyramp tests: full drift analysis + all actions permitted.
|
|
43
|
+
// Relevant external tests (user-written, relevant to this PR's endpoints): drift analysis
|
|
44
|
+
// + UPDATE only — REGENERATE and DELETE are report-only (enforced in skyramp_actions).
|
|
45
|
+
// Other external tests: excluded entirely (deduplication only, not analysed).
|
|
46
|
+
// relevantExternalTestPaths are stored relative to repositoryPath in the state file.
|
|
47
|
+
// Re-absolutize here so has() comparisons against t.testFile (absolute) work correctly.
|
|
48
|
+
const relevantExternalPaths = new Set((stateData.repositoryAnalysis?.relevantExternalTestPaths ?? []).map((p) => path.isAbsolute(p) ? p : path.resolve(repositoryPath, p)));
|
|
49
|
+
const existingTests = (stateData.existingTests || []).filter((t) => {
|
|
50
|
+
if ((t.source ?? TestSource.Skyramp) !== TestSource.External)
|
|
51
|
+
return true;
|
|
52
|
+
return relevantExternalPaths.has(t.testFile);
|
|
53
|
+
});
|
|
54
|
+
const skyrampCount = existingTests.filter((t) => (t.source ?? TestSource.Skyramp) !== TestSource.External).length;
|
|
55
|
+
const externalCount = existingTests.length - skyrampCount;
|
|
56
|
+
logger.info(`Loaded ${skyrampCount} Skyramp + ${externalCount} relevant external tests from state file`);
|
|
61
57
|
// Sweep stale diff files on this natural follow-up call so they don't accumulate.
|
|
62
58
|
// Pass [] for stateTypes so only skyramp-diff-*.diff files are deleted — state files
|
|
63
59
|
// (skyramp-analysis-*, skyramp-recommendation-*) must not be removed here because the
|
|
@@ -68,15 +64,7 @@ The LLM follows the returned prompt to assign drift details and actions (UPDATE
|
|
|
68
64
|
catch (error) {
|
|
69
65
|
logger.warning(`Failed to cleanup old diff files: ${error.message}`);
|
|
70
66
|
}
|
|
71
|
-
const promptText = buildDriftAnalysisPrompt({
|
|
72
|
-
existingTests,
|
|
73
|
-
scannedEndpoints,
|
|
74
|
-
repositoryPath: absoluteRepoPath,
|
|
75
|
-
stateFile: stateManager.getStatePath(),
|
|
76
|
-
routerMountContext,
|
|
77
|
-
candidateRouteFiles,
|
|
78
|
-
diffFilePath,
|
|
79
|
-
});
|
|
67
|
+
const promptText = buildDriftAnalysisPrompt(stateManager.getStatePath(), existingTests.map((t) => ({ testFile: t.testFile, source: t.source })));
|
|
80
68
|
return {
|
|
81
69
|
structuredContent: { prompt: promptText },
|
|
82
70
|
content: [{ type: "text", text: "Drift analysis prompt generated. Follow the prompt field to assess each test." }],
|
|
@@ -54,7 +54,7 @@ describe("dockerImageExistsLocally", () => {
|
|
|
54
54
|
});
|
|
55
55
|
});
|
|
56
56
|
describe("pullDockerImage", () => {
|
|
57
|
-
const IMAGE = "skyramp/executor:v1.3.
|
|
57
|
+
const IMAGE = "skyramp/executor:v1.3.27";
|
|
58
58
|
beforeEach(() => jest.clearAllMocks());
|
|
59
59
|
describe("on amd64 host", () => {
|
|
60
60
|
const originalArch = process.arch;
|
package/build/utils/versions.js
CHANGED
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __defProp = Object.defineProperty;
|
|
3
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
+
var __export = (target, all) => {
|
|
7
|
+
for (var name in all)
|
|
8
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
+
};
|
|
10
|
+
var __copyProps = (to, from, except, desc) => {
|
|
11
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
12
|
+
for (let key of __getOwnPropNames(from))
|
|
13
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
14
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
15
|
+
}
|
|
16
|
+
return to;
|
|
17
|
+
};
|
|
18
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
|
+
var visualSnapshot_exports = {};
|
|
20
|
+
__export(visualSnapshot_exports, {
|
|
21
|
+
buildVisualSnapshotJsonl: () => buildVisualSnapshotJsonl,
|
|
22
|
+
nextSnapshotFilename: () => nextSnapshotFilename,
|
|
23
|
+
parseSnapshotCounter: () => parseSnapshotCounter,
|
|
24
|
+
visualSnapshotToCode: () => visualSnapshotToCode
|
|
25
|
+
});
|
|
26
|
+
module.exports = __toCommonJS(visualSnapshot_exports);
|
|
27
|
+
const FILENAME_PREFIX = {
|
|
28
|
+
page: "page",
|
|
29
|
+
element: "el",
|
|
30
|
+
region: "region"
|
|
31
|
+
};
|
|
32
|
+
function nextSnapshotFilename(type, counter) {
|
|
33
|
+
return `${FILENAME_PREFIX[type]}-${String(counter).padStart(3, "0")}.png`;
|
|
34
|
+
}
|
|
35
|
+
function parseSnapshotCounter(type, filename) {
|
|
36
|
+
const m = new RegExp(`^${FILENAME_PREFIX[type]}-(\\d+)\\.png$`, "i").exec(filename);
|
|
37
|
+
return m ? parseInt(m[1], 10) : null;
|
|
38
|
+
}
|
|
39
|
+
function buildVisualSnapshotJsonl(input) {
|
|
40
|
+
if (!input.filename.toLowerCase().endsWith(".png"))
|
|
41
|
+
return { error: `visual snapshot filename must end in .png (got "${input.filename}").` };
|
|
42
|
+
switch (input.snapshotType) {
|
|
43
|
+
case "page": {
|
|
44
|
+
const action = { name: "visualSnapshot", snapshotType: "page", filename: input.filename };
|
|
45
|
+
if (input.fullPage !== void 0)
|
|
46
|
+
action.fullPage = input.fullPage;
|
|
47
|
+
if (input.screenshotStyle !== void 0)
|
|
48
|
+
action.screenshotStyle = input.screenshotStyle;
|
|
49
|
+
return { action };
|
|
50
|
+
}
|
|
51
|
+
case "element": {
|
|
52
|
+
if (!input.selector)
|
|
53
|
+
return { error: 'visual snapshot "element" requires a resolved selector (pass a ref to the tool).' };
|
|
54
|
+
return { action: { name: "visualSnapshot", snapshotType: "element", filename: input.filename, selector: input.selector } };
|
|
55
|
+
}
|
|
56
|
+
case "region": {
|
|
57
|
+
const c = input.clip;
|
|
58
|
+
if (!c || c.width === void 0 || c.height === void 0 || c.x === void 0 || c.y === void 0)
|
|
59
|
+
return { error: 'visual snapshot "region" requires a clip { x, y, width, height }.' };
|
|
60
|
+
return { action: { name: "visualSnapshot", snapshotType: "region", filename: input.filename, clip: { x: c.x, y: c.y, width: c.width, height: c.height } } };
|
|
61
|
+
}
|
|
62
|
+
default:
|
|
63
|
+
return { error: `Unknown visual snapshot type: ${input.snapshotType}` };
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
function visualSnapshotToCode(a, locatorExpr) {
|
|
67
|
+
const q = (s) => s.replace(/\\/g, "\\\\").replace(/'/g, "\\'");
|
|
68
|
+
const file = q(a.filename);
|
|
69
|
+
switch (a.snapshotType) {
|
|
70
|
+
case "page": {
|
|
71
|
+
const opts = [];
|
|
72
|
+
if (a.fullPage)
|
|
73
|
+
opts.push("fullPage: true");
|
|
74
|
+
if (a.screenshotStyle)
|
|
75
|
+
opts.push(`style: '${q(a.screenshotStyle)}'`);
|
|
76
|
+
const optsStr = opts.length ? `, { ${opts.join(", ")} }` : "";
|
|
77
|
+
return `await expect(page).toHaveScreenshot('${file}'${optsStr});`;
|
|
78
|
+
}
|
|
79
|
+
case "element": {
|
|
80
|
+
const target = locatorExpr ? `page.${locatorExpr}` : `page.locator('${q(a.selector)}')`;
|
|
81
|
+
return `await expect(${target}).toHaveScreenshot('${file}');`;
|
|
82
|
+
}
|
|
83
|
+
case "region": {
|
|
84
|
+
const { x, y, width, height } = a.clip;
|
|
85
|
+
return `await expect(page).toHaveScreenshot('${file}', { clip: { x: ${x}, y: ${y}, width: ${width}, height: ${height} } });`;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
90
|
+
0 && (module.exports = {
|
|
91
|
+
buildVisualSnapshotJsonl,
|
|
92
|
+
nextSnapshotFilename,
|
|
93
|
+
parseSnapshotCounter,
|
|
94
|
+
visualSnapshotToCode
|
|
95
|
+
});
|
|
@@ -104,6 +104,8 @@ function describeStep(action, index) {
|
|
|
104
104
|
detail += ` = ${JSON.stringify(a.value)}`;
|
|
105
105
|
else if (a.name === "press" && a.key !== void 0)
|
|
106
106
|
detail += ` ${a.key}`;
|
|
107
|
+
else if (a.name === "visualSnapshot")
|
|
108
|
+
detail = `${a.snapshotType ?? ""}${a.filename ? ` ${a.filename}` : ""}`.trim();
|
|
107
109
|
return `#${index + 1} ${a.name}${onPage}${detail ? ` ${detail}` : ""}`;
|
|
108
110
|
}
|
|
109
111
|
function listStepsFrom(allActions, fromIndex) {
|
|
@@ -47,11 +47,13 @@ var import_assertApiRequestTool = require("./assertApiRequestTool");
|
|
|
47
47
|
var import_loadTraceTool = require("./loadTraceTool");
|
|
48
48
|
var import_skyRampImport = require("./skyRampImport");
|
|
49
49
|
var import_mouseActionTool = require("./mouseActionTool");
|
|
50
|
+
var import_visualSnapshotTool = require("./visualSnapshotTool");
|
|
51
|
+
var import_visualSnapshot = require("./common/visualSnapshot");
|
|
50
52
|
var import_utils = require("playwright-core/lib/utils");
|
|
51
53
|
var import_types = require("./types");
|
|
52
54
|
const traceDebug = (0, import_utilsBundle.debug)("pw:mcp:trace");
|
|
53
55
|
class TraceRecordingBackend {
|
|
54
|
-
//
|
|
56
|
+
// per-type baseline filename counter
|
|
55
57
|
constructor(options) {
|
|
56
58
|
this._trackedActions = [];
|
|
57
59
|
this._initialized = false;
|
|
@@ -62,6 +64,8 @@ class TraceRecordingBackend {
|
|
|
62
64
|
this._pendingPopupAlias = null;
|
|
63
65
|
// popup alias to stamp on the NEXT tracked click
|
|
64
66
|
this._reloading = false;
|
|
67
|
+
// true while page.reload() is in progress — suppresses spurious popup tracking
|
|
68
|
+
this._visualSnapshotCounters = { page: 0, element: 0, region: 0 };
|
|
65
69
|
this._options = options || {};
|
|
66
70
|
this._outputDir = options?.outputDir || process.cwd();
|
|
67
71
|
this._tempDir = import_fs.default.mkdtempSync(import_path.default.join(import_os.default.tmpdir(), "skyramp-trace-"));
|
|
@@ -120,7 +124,7 @@ class TraceRecordingBackend {
|
|
|
120
124
|
}
|
|
121
125
|
async listTools() {
|
|
122
126
|
const browserTools = await this._browserBackend.listTools();
|
|
123
|
-
return [...browserTools, (0, import_exportTool.exportZipMcpTool)(), (0, import_assertTool.assertMcpTool)(), (0, import_assertApiRequestTool.assertApiRequestMcpTool)(), (0, import_loadTraceTool.loadTraceMcpTool)(), (0, import_mouseActionTool.mouseActionMcpTool)()];
|
|
127
|
+
return [...browserTools, (0, import_exportTool.exportZipMcpTool)(), (0, import_assertTool.assertMcpTool)(), (0, import_assertApiRequestTool.assertApiRequestMcpTool)(), (0, import_loadTraceTool.loadTraceMcpTool)(), (0, import_mouseActionTool.mouseActionMcpTool)(), (0, import_visualSnapshotTool.visualSnapshotMcpTool)()];
|
|
124
128
|
}
|
|
125
129
|
async callTool(name, args, progress) {
|
|
126
130
|
if (!this._initialized)
|
|
@@ -151,6 +155,10 @@ class TraceRecordingBackend {
|
|
|
151
155
|
const parsed = import_mouseActionTool.mouseActionSchema.inputSchema.parse(args || {});
|
|
152
156
|
return this._handleMouseAction(parsed);
|
|
153
157
|
}
|
|
158
|
+
if (name === import_visualSnapshotTool.visualSnapshotSchema.name) {
|
|
159
|
+
const parsed = import_visualSnapshotTool.visualSnapshotSchema.inputSchema.parse(args || {});
|
|
160
|
+
return this._handleVisualSnapshot(parsed);
|
|
161
|
+
}
|
|
154
162
|
if (name === import_assertTool.assertToolSchema.name) {
|
|
155
163
|
const parsed = import_assertTool.assertToolSchema.inputSchema.parse(args || {});
|
|
156
164
|
return this._handleAssert(parsed);
|
|
@@ -352,6 +360,93 @@ Reloaded current page: ${currentUrl}
|
|
|
352
360
|
traceDebug(`Tracked ${actions.length} mouse sub-action(s) for "${params.action}" on ${pageAlias}`);
|
|
353
361
|
return result;
|
|
354
362
|
}
|
|
363
|
+
/**
|
|
364
|
+
* Handle browser_visual_snapshot: record a `visualSnapshot` marker that
|
|
365
|
+
* exports to expect(...).toHaveScreenshot(filename), so the generated test
|
|
366
|
+
* pixel-compares against a baseline.
|
|
367
|
+
*
|
|
368
|
+
* This is marker-only by design: the baseline image is created/updated by
|
|
369
|
+
* Playwright on the first test run (into its snapshot dir), NOT captured here.
|
|
370
|
+
* Taking a live screenshot at record time would be throwaway, and browser_
|
|
371
|
+
* take_screenshot has no clip parameter, so a region screenshot could not even
|
|
372
|
+
* be honored — it would mislead by returning a full-viewport image. So we only
|
|
373
|
+
* emit the marker, mirroring browser_assert_api_request.
|
|
374
|
+
*
|
|
375
|
+
* For an element snapshot, the ref is still resolved to a durable selector via
|
|
376
|
+
* the same hover->selector path browser_assert uses (testid > role > text, with
|
|
377
|
+
* the snapshot-accessible-name fallback for brittle/Flutter ids); that also
|
|
378
|
+
* validates the ref exists. Iframe and GoJS-diagram snapshots from the recorder
|
|
379
|
+
* are out of scope here (see common/visualSnapshot.ts).
|
|
380
|
+
*/
|
|
381
|
+
async _handleVisualSnapshot(params) {
|
|
382
|
+
const timestamp = Date.now();
|
|
383
|
+
const pageAlias = this._currentPageAlias;
|
|
384
|
+
const input = {
|
|
385
|
+
snapshotType: params.snapshotType,
|
|
386
|
+
filename: params.filename ?? this._nextSnapshotFilename(params.snapshotType),
|
|
387
|
+
fullPage: params.fullPage,
|
|
388
|
+
clip: params.clip
|
|
389
|
+
};
|
|
390
|
+
if (params.snapshotType === "element") {
|
|
391
|
+
if (!params.ref)
|
|
392
|
+
return { content: [{ type: "text", text: '### Error\nsnapshotType "element" requires a ref (from the latest browser_snapshot).' }], isError: true };
|
|
393
|
+
const resolved = await this._resolveRefToLocator(params.ref, params.element ?? "");
|
|
394
|
+
if (!resolved)
|
|
395
|
+
return { content: [{ type: "text", text: `### Error
|
|
396
|
+
Could not resolve a durable selector for ref=${params.ref}. Take a fresh browser_snapshot and retry, or use snapshotType "region".` }], isError: true };
|
|
397
|
+
input.selector = resolved.selector;
|
|
398
|
+
}
|
|
399
|
+
const built = (0, import_visualSnapshot.buildVisualSnapshotJsonl)(input);
|
|
400
|
+
if ("error" in built)
|
|
401
|
+
return { content: [{ type: "text", text: `### Error
|
|
402
|
+
${built.error}` }], isError: true };
|
|
403
|
+
this._visualSnapshotCounters[params.snapshotType]++;
|
|
404
|
+
this._advanceSnapshotCounterFor(params.snapshotType, input.filename);
|
|
405
|
+
this._trackedActions.push({
|
|
406
|
+
toolName: "browser_visual_snapshot",
|
|
407
|
+
args: built.action,
|
|
408
|
+
code: "",
|
|
409
|
+
timestamp,
|
|
410
|
+
pageAlias
|
|
411
|
+
});
|
|
412
|
+
traceDebug(`Tracked visualSnapshot (${params.snapshotType}) "${input.filename}" on ${pageAlias}`);
|
|
413
|
+
const targetDesc = params.snapshotType === "element" ? ` (${input.selector})` : params.snapshotType === "region" && params.clip ? ` (clip ${params.clip.width}x${params.clip.height} at ${params.clip.x},${params.clip.y})` : params.fullPage ? " (full page)" : "";
|
|
414
|
+
return { content: [{ type: "text", text: `### Visual snapshot recorded
|
|
415
|
+
Baseline "${input.filename}" (${params.snapshotType})${targetDesc} recorded; the generated test will assert toHaveScreenshot against it. The baseline image is created on the first test run.` }] };
|
|
416
|
+
}
|
|
417
|
+
/** Next auto-generated baseline filename for a snapshot type (page-NNN.png, etc.). */
|
|
418
|
+
_nextSnapshotFilename(type) {
|
|
419
|
+
return (0, import_visualSnapshot.nextSnapshotFilename)(type, this._visualSnapshotCounters[type] + 1);
|
|
420
|
+
}
|
|
421
|
+
/**
|
|
422
|
+
* Resolve a snapshot ref to a durable Playwright selector, mirroring the
|
|
423
|
+
* element-resolution path of _handleAssert: hover to get the resolved code,
|
|
424
|
+
* parse it to a selector, and prefer a snapshot-accessible-name selector over
|
|
425
|
+
* a brittle raw-CSS id (the Flutter-durable fallback). Returns null if the ref
|
|
426
|
+
* can't be resolved.
|
|
427
|
+
*/
|
|
428
|
+
async _resolveRefToLocator(ref, element) {
|
|
429
|
+
const hoverResult = await this._browserBackend.callTool("browser_hover", { element, ref });
|
|
430
|
+
if (hoverResult.isError)
|
|
431
|
+
return null;
|
|
432
|
+
const hoverCode = (0, import_response.parseResponse)(hoverResult)?.code ?? "";
|
|
433
|
+
const locatorMatch = hoverCode.match(/await\s+page\.(.*?)\.hover\(\)/s);
|
|
434
|
+
if (!locatorMatch)
|
|
435
|
+
return null;
|
|
436
|
+
const locatorExpr = locatorMatch[1].trim();
|
|
437
|
+
let parsed = this._codeToLocator(locatorExpr);
|
|
438
|
+
if (!parsed || parsed.locator.kind === "css") {
|
|
439
|
+
const snapResult = await this._browserBackend.callTool("browser_snapshot", {});
|
|
440
|
+
if (!snapResult.isError) {
|
|
441
|
+
const snapText = snapResult.content?.map((c) => c.type === "text" ? c.text : "").join("") || "";
|
|
442
|
+
const refLine = snapText.split("\n").find((l) => l.includes(`[ref=${ref}]`)) || "";
|
|
443
|
+
const fromSnapshot = this._extractLocatorForRef(refLine);
|
|
444
|
+
if (fromSnapshot)
|
|
445
|
+
parsed = fromSnapshot;
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
return parsed;
|
|
449
|
+
}
|
|
355
450
|
/**
|
|
356
451
|
* Load a prior Skyramp trace and replay it against the live browser, honoring
|
|
357
452
|
* an optional stop point, then seed _trackedActions with the replayed actions
|
|
@@ -465,6 +560,8 @@ Continue recording with browser_* tools, then call skyramp_export_zip to write t
|
|
|
465
560
|
const seeded = this._seedTrackedActionFields(a, locatorExpr);
|
|
466
561
|
if (!seeded)
|
|
467
562
|
return;
|
|
563
|
+
if (seeded.toolName === "browser_visual_snapshot")
|
|
564
|
+
this._advanceSnapshotCounterFor(seeded.args.snapshotType, seeded.args.filename);
|
|
468
565
|
this._trackedActions.push({
|
|
469
566
|
...seeded,
|
|
470
567
|
timestamp: action.startTime,
|
|
@@ -472,6 +569,19 @@ Continue recording with browser_* tools, then call skyramp_export_zip to write t
|
|
|
472
569
|
framePath: action.frame.framePath?.length ? action.frame.framePath : void 0
|
|
473
570
|
});
|
|
474
571
|
}
|
|
572
|
+
/**
|
|
573
|
+
* Bump the per-type snapshot counter to at least the number embedded in a
|
|
574
|
+
* seeded baseline filename (`<prefix>-NNN.png`), so subsequently-recorded
|
|
575
|
+
* snapshots of that type don't reuse a loaded trace's filename. No-op if the
|
|
576
|
+
* filename doesn't carry a parseable counter.
|
|
577
|
+
*/
|
|
578
|
+
_advanceSnapshotCounterFor(snapshotType, filename) {
|
|
579
|
+
if (!snapshotType || !filename || this._visualSnapshotCounters[snapshotType] === void 0)
|
|
580
|
+
return;
|
|
581
|
+
const n = (0, import_visualSnapshot.parseSnapshotCounter)(snapshotType, filename);
|
|
582
|
+
if (n !== null && n > this._visualSnapshotCounters[snapshotType])
|
|
583
|
+
this._visualSnapshotCounters[snapshotType] = n;
|
|
584
|
+
}
|
|
475
585
|
/**
|
|
476
586
|
* Build the { toolName, code, args } triple a seeded (replayed) action must
|
|
477
587
|
* carry so it round-trips through skyRampExport.buildJsonlContent exactly as
|
|
@@ -566,6 +676,44 @@ Continue recording with browser_* tools, then call skyramp_export_zip to write t
|
|
|
566
676
|
return { toolName: "browser_assert", code: `assertChecked:${a.selector}:${!!a.checked}`, args: { type: "checked", selector: a.selector, checked: !!a.checked } };
|
|
567
677
|
case "assertVisible":
|
|
568
678
|
return { toolName: "browser_assert", code: `assertVisible:${a.selector}`, args: { type: "visible", selector: a.selector } };
|
|
679
|
+
case "visualSnapshot": {
|
|
680
|
+
const args = this._seedVisualSnapshotArgs(a);
|
|
681
|
+
return args ? { toolName: "browser_visual_snapshot", code: "", args } : null;
|
|
682
|
+
}
|
|
683
|
+
default:
|
|
684
|
+
return null;
|
|
685
|
+
}
|
|
686
|
+
}
|
|
687
|
+
/**
|
|
688
|
+
* Build the normalized JSONL args for a re-seeded visualSnapshot action,
|
|
689
|
+
* following the VisualSnapshotJsonl contract in common/visualSnapshot.ts. Only
|
|
690
|
+
* known fields per snapshotType are emitted; a snapshot missing its required
|
|
691
|
+
* field (element->selector, region->clip) or with an unsupported type
|
|
692
|
+
* (gojsDiagram is recorder-only) is rejected (returns null) so the caller
|
|
693
|
+
* skips it rather than exporting an invalid shape.
|
|
694
|
+
*/
|
|
695
|
+
_seedVisualSnapshotArgs(a) {
|
|
696
|
+
if (!a.filename || !String(a.filename).toLowerCase().endsWith(".png"))
|
|
697
|
+
return null;
|
|
698
|
+
switch (a.snapshotType) {
|
|
699
|
+
case "page": {
|
|
700
|
+
const args = { name: "visualSnapshot", snapshotType: "page", filename: a.filename };
|
|
701
|
+
if (a.fullPage !== void 0)
|
|
702
|
+
args.fullPage = a.fullPage;
|
|
703
|
+
if (a.screenshotStyle !== void 0)
|
|
704
|
+
args.screenshotStyle = a.screenshotStyle;
|
|
705
|
+
return args;
|
|
706
|
+
}
|
|
707
|
+
case "element":
|
|
708
|
+
if (!a.selector)
|
|
709
|
+
return null;
|
|
710
|
+
return { name: "visualSnapshot", snapshotType: "element", filename: a.filename, selector: a.selector };
|
|
711
|
+
case "region": {
|
|
712
|
+
const c = a.clip;
|
|
713
|
+
if (!c || c.x === void 0 || c.y === void 0 || c.width === void 0 || c.height === void 0)
|
|
714
|
+
return null;
|
|
715
|
+
return { name: "visualSnapshot", snapshotType: "region", filename: a.filename, clip: { x: c.x, y: c.y, width: c.width, height: c.height } };
|
|
716
|
+
}
|
|
569
717
|
default:
|
|
570
718
|
return null;
|
|
571
719
|
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __defProp = Object.defineProperty;
|
|
3
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
4
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
5
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
6
|
+
var __export = (target, all) => {
|
|
7
|
+
for (var name in all)
|
|
8
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
9
|
+
};
|
|
10
|
+
var __copyProps = (to, from, except, desc) => {
|
|
11
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
12
|
+
for (let key of __getOwnPropNames(from))
|
|
13
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
14
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
15
|
+
}
|
|
16
|
+
return to;
|
|
17
|
+
};
|
|
18
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
19
|
+
var visualSnapshotTool_exports = {};
|
|
20
|
+
__export(visualSnapshotTool_exports, {
|
|
21
|
+
visualSnapshotMcpTool: () => visualSnapshotMcpTool,
|
|
22
|
+
visualSnapshotSchema: () => visualSnapshotSchema
|
|
23
|
+
});
|
|
24
|
+
module.exports = __toCommonJS(visualSnapshotTool_exports);
|
|
25
|
+
var import_mcpBundle = require("playwright-core/lib/mcpBundle");
|
|
26
|
+
var import_tool = require("../sdk/tool");
|
|
27
|
+
const visualSnapshotSchema = {
|
|
28
|
+
name: "browser_visual_snapshot",
|
|
29
|
+
title: "Visual snapshot (screenshot baseline)",
|
|
30
|
+
description: [
|
|
31
|
+
"Record a visual-regression baseline: stores a toHaveScreenshot() assertion in the trace so the generated test pixel-compares the page/element/region against a baseline image on every run (the baseline is created on the test's first run).",
|
|
32
|
+
"Use this to lock the visual appearance of a page, an element, or a screen region after a key action (e.g. a rendered chart, a styled component, a confirmation screen).",
|
|
33
|
+
"The `snapshotType` parameter selects the target:",
|
|
34
|
+
"- page: the whole page (set fullPage to capture the full scrollable page rather than just the viewport).",
|
|
35
|
+
"- element: a single element, identified by its snapshot ref. The ref is resolved to a durable selector for the generated test.",
|
|
36
|
+
"- region: a rectangular area given by clip (x, y, width, height) in viewport pixels, read from a normal (non-fullPage) screenshot.",
|
|
37
|
+
"This complements browser_assert (which checks text/value/state): use browser_visual_snapshot when the thing to verify is how it LOOKS, not its text."
|
|
38
|
+
].join(" "),
|
|
39
|
+
inputSchema: import_mcpBundle.z.object({
|
|
40
|
+
snapshotType: import_mcpBundle.z.enum(["page", "element", "region"]).describe("What to capture: whole page, a single element (by ref), or a pixel region (by clip)."),
|
|
41
|
+
ref: import_mcpBundle.z.string().optional().describe('Element snapshot ref to capture. Required for snapshotType "element".'),
|
|
42
|
+
element: import_mcpBundle.z.string().optional().describe("Human-readable description of the element (paired with ref) for permission and logging."),
|
|
43
|
+
fullPage: import_mcpBundle.z.boolean().optional().describe('For snapshotType "page": capture the full scrollable page instead of just the viewport.'),
|
|
44
|
+
clip: import_mcpBundle.z.object({
|
|
45
|
+
x: import_mcpBundle.z.number().describe("Left edge, in viewport pixels (distance from the visible top-left, not the document top)."),
|
|
46
|
+
y: import_mcpBundle.z.number().describe("Top edge, in viewport pixels (distance from the visible top-left, not the document top)."),
|
|
47
|
+
width: import_mcpBundle.z.number().positive().describe("Region width in pixels."),
|
|
48
|
+
height: import_mcpBundle.z.number().positive().describe("Region height in pixels.")
|
|
49
|
+
}).optional().describe('For snapshotType "region": the rectangle to capture, in VIEWPORT pixels. Read these coordinates from a normal (viewport) browser_take_screenshot, NOT a fullPage one \u2014 the region is clipped to the visible viewport, so document/scrolled coordinates will be off.'),
|
|
50
|
+
filename: import_mcpBundle.z.string().optional().describe("Baseline filename. Auto-generated (page-NNN.png / el-NNN.png / region-NNN.png) when omitted.")
|
|
51
|
+
}),
|
|
52
|
+
// Marker-only: records a trace marker, does not mutate the page. Mirrors the
|
|
53
|
+
// other marker tools (browser_assert, browser_assert_api_request).
|
|
54
|
+
type: "readOnly"
|
|
55
|
+
};
|
|
56
|
+
function visualSnapshotMcpTool() {
|
|
57
|
+
return (0, import_tool.toMcpTool)(visualSnapshotSchema);
|
|
58
|
+
}
|
|
59
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
60
|
+
0 && (module.exports = {
|
|
61
|
+
visualSnapshotMcpTool,
|
|
62
|
+
visualSnapshotSchema
|
|
63
|
+
});
|
|
@@ -352,6 +352,32 @@ function assertActionToJsonl(action, pageGuid, timestamp) {
|
|
|
352
352
|
return null;
|
|
353
353
|
}
|
|
354
354
|
}
|
|
355
|
+
function visualSnapshotActionToJsonl(action, pageGuid, timestamp) {
|
|
356
|
+
const args = action.args;
|
|
357
|
+
if (!args || !args.filename || !String(args.filename).toLowerCase().endsWith(".png"))
|
|
358
|
+
return null;
|
|
359
|
+
const base = {
|
|
360
|
+
signals: [],
|
|
361
|
+
timestamp: String(timestamp),
|
|
362
|
+
pageGuid,
|
|
363
|
+
pageAlias: action.pageAlias ?? DEFAULT_PAGE_ALIAS,
|
|
364
|
+
framePath: action.framePath ?? DEFAULT_FRAME_PATH
|
|
365
|
+
};
|
|
366
|
+
if (args.snapshotType === "page")
|
|
367
|
+
return JSON.stringify({ name: "visualSnapshot", snapshotType: "page", filename: args.filename, ...args.fullPage ? { fullPage: true } : {}, ...args.screenshotStyle ? { screenshotStyle: args.screenshotStyle } : {}, ...base });
|
|
368
|
+
if (args.snapshotType === "element") {
|
|
369
|
+
if (!args.selector)
|
|
370
|
+
return null;
|
|
371
|
+
return JSON.stringify({ name: "visualSnapshot", snapshotType: "element", filename: args.filename, selector: args.selector, ...base });
|
|
372
|
+
}
|
|
373
|
+
if (args.snapshotType === "region") {
|
|
374
|
+
const c = args.clip;
|
|
375
|
+
if (!c || c.x === void 0 || c.y === void 0 || c.width === void 0 || c.height === void 0)
|
|
376
|
+
return null;
|
|
377
|
+
return JSON.stringify({ name: "visualSnapshot", snapshotType: "region", filename: args.filename, clip: { x: c.x, y: c.y, width: c.width, height: c.height }, ...base });
|
|
378
|
+
}
|
|
379
|
+
return null;
|
|
380
|
+
}
|
|
355
381
|
function selectorToLocator(selector) {
|
|
356
382
|
const testidMatch = selector.match(/internal:testid=\[data-testid="([^"]+)"/);
|
|
357
383
|
if (testidMatch)
|
|
@@ -518,6 +544,16 @@ function buildJsonlContent(actions, browserName, harPath) {
|
|
|
518
544
|
actionCount++;
|
|
519
545
|
continue;
|
|
520
546
|
}
|
|
547
|
+
if (action.toolName === "browser_visual_snapshot") {
|
|
548
|
+
const vsLine = visualSnapshotActionToJsonl(action, pageGuid, action.timestamp);
|
|
549
|
+
if (vsLine) {
|
|
550
|
+
lines.push(vsLine);
|
|
551
|
+
actionCount++;
|
|
552
|
+
} else {
|
|
553
|
+
skipped.push(action.toolName);
|
|
554
|
+
}
|
|
555
|
+
continue;
|
|
556
|
+
}
|
|
521
557
|
if ((action.toolName === "browser_type" || action.toolName === "browser_press_sequentially") && action.args.submit) {
|
|
522
558
|
const fillLine = trackedActionToJsonl(action, pageGuid, action.timestamp);
|
|
523
559
|
if (fillLine) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@skyramp/mcp",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.5-rc.1",
|
|
4
4
|
"main": "build/index.js",
|
|
5
5
|
"exports": {
|
|
6
6
|
".": "./build/index.js",
|
|
@@ -55,7 +55,7 @@
|
|
|
55
55
|
"dependencies": {
|
|
56
56
|
"@modelcontextprotocol/sdk": "^1.24.3",
|
|
57
57
|
"@playwright/test": "^1.55.0",
|
|
58
|
-
"@skyramp/skyramp": "1.3.
|
|
58
|
+
"@skyramp/skyramp": "1.3.27",
|
|
59
59
|
"dockerode": "^5.0.0",
|
|
60
60
|
"fast-glob": "^3.3.3",
|
|
61
61
|
"js-yaml": "^4.1.1",
|