@skyramp/mcp 0.2.3 → 0.2.5-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/playwright/registerPlaywrightTools.js +21 -25
- package/build/playwright/traceRecordingPrompt.js +2 -2
- package/build/prompts/test-maintenance/actionsInstructions.js +60 -0
- package/build/prompts/test-maintenance/drift-analysis-prompt.js +18 -101
- package/build/prompts/test-maintenance/driftAnalysisSections.js +210 -171
- package/build/prompts/test-recommendation/analysisOutputPrompt.js +1 -1
- package/build/prompts/test-recommendation/diffExecutionPlan.js +4 -3
- package/build/prompts/test-recommendation/recommendationSections.js +6 -6
- package/build/prompts/test-recommendation/scopeAssessment.js +3 -1
- package/build/prompts/test-recommendation/scopeAssessment.test.js +13 -0
- package/build/prompts/test-recommendation/test-recommendation-prompt.js +2 -2
- package/build/prompts/test-recommendation/test-recommendation-prompt.test.js +3 -3
- package/build/prompts/testbot/testbot-prompts.js +21 -17
- package/build/prompts/testbot/testbot-prompts.test.js +21 -17
- package/build/services/TestDiscoveryService.js +11 -43
- package/build/tools/submitReportTool.js +9 -12
- package/build/tools/submitReportTool.test.js +4 -5
- package/build/tools/test-management/actionsTool.js +160 -240
- package/build/tools/test-management/analyzeChangesTool.js +43 -18
- package/build/tools/test-management/analyzeTestHealthTool.js +17 -29
- package/build/utils/docker.test.js +1 -1
- package/build/utils/versions.js +1 -1
- package/node_modules/playwright/lib/mcp/skyramp/common/visualSnapshot.js +95 -0
- package/node_modules/playwright/lib/mcp/skyramp/loadTraceTool.js +2 -0
- package/node_modules/playwright/lib/mcp/skyramp/traceRecordingBackend.js +150 -2
- package/node_modules/playwright/lib/mcp/skyramp/visualSnapshotTool.js +63 -0
- package/node_modules/playwright/lib/mcp/test/skyRampExport.js +36 -0
- package/package.json +2 -2
- package/build/prompts/test-maintenance/drift-analysis-prompt.test.js +0 -116
|
@@ -40,11 +40,6 @@ export async function registerPlaywrightTools(server, options) {
|
|
|
40
40
|
'browser_select_option',
|
|
41
41
|
'browser_hover',
|
|
42
42
|
'browser_drag',
|
|
43
|
-
'browser_mouse_move_xy',
|
|
44
|
-
'browser_mouse_click_xy',
|
|
45
|
-
'browser_mouse_drag_xy',
|
|
46
|
-
'browser_mouse_down',
|
|
47
|
-
'browser_mouse_up',
|
|
48
43
|
'browser_file_upload',
|
|
49
44
|
'browser_evaluate',
|
|
50
45
|
'browser_tabs',
|
|
@@ -54,6 +49,8 @@ export async function registerPlaywrightTools(server, options) {
|
|
|
54
49
|
'browser_assert',
|
|
55
50
|
'browser_assert_api_request',
|
|
56
51
|
'browser_assert_table_cell',
|
|
52
|
+
'browser_mouse_action',
|
|
53
|
+
'browser_visual_snapshot',
|
|
57
54
|
'skyramp_export_zip',
|
|
58
55
|
'skyramp_load_trace',
|
|
59
56
|
'browser_mouse_action',
|
|
@@ -93,26 +90,7 @@ function jsonSchemaToZod(schema) {
|
|
|
93
90
|
const required = new Set(schema.required || []);
|
|
94
91
|
const shape = {};
|
|
95
92
|
for (const [key, prop] of Object.entries(properties)) {
|
|
96
|
-
let field;
|
|
97
|
-
switch (prop.type) {
|
|
98
|
-
case "string":
|
|
99
|
-
field = prop.enum
|
|
100
|
-
? z.enum(prop.enum)
|
|
101
|
-
: z.string();
|
|
102
|
-
break;
|
|
103
|
-
case "number":
|
|
104
|
-
case "integer":
|
|
105
|
-
field = z.number();
|
|
106
|
-
break;
|
|
107
|
-
case "boolean":
|
|
108
|
-
field = z.boolean();
|
|
109
|
-
break;
|
|
110
|
-
case "array":
|
|
111
|
-
field = z.array(prop.items ? jsonSchemaPropertyToZod(prop.items) : z.unknown());
|
|
112
|
-
break;
|
|
113
|
-
default:
|
|
114
|
-
field = z.unknown();
|
|
115
|
-
}
|
|
93
|
+
let field = jsonSchemaPropertyToZod(prop);
|
|
116
94
|
if (prop.description)
|
|
117
95
|
field = field.describe(prop.description);
|
|
118
96
|
if (prop.default !== undefined)
|
|
@@ -136,6 +114,24 @@ function jsonSchemaPropertyToZod(prop) {
|
|
|
136
114
|
return z.boolean();
|
|
137
115
|
case "array":
|
|
138
116
|
return z.array(prop.items ? jsonSchemaPropertyToZod(prop.items) : z.unknown());
|
|
117
|
+
case "object": {
|
|
118
|
+
// Nested object (e.g. a clip rect). Recurse into its properties so the
|
|
119
|
+
// SDK registers a real object schema; without this it falls through to
|
|
120
|
+
// z.unknown(), which serializes the nested object as a string and the
|
|
121
|
+
// inner backend's z.object(...) then rejects it.
|
|
122
|
+
const nestedProps = prop.properties || {};
|
|
123
|
+
const nestedRequired = new Set(prop.required || []);
|
|
124
|
+
const nestedShape = {};
|
|
125
|
+
for (const [k, p] of Object.entries(nestedProps)) {
|
|
126
|
+
let f = jsonSchemaPropertyToZod(p);
|
|
127
|
+
if (p.description)
|
|
128
|
+
f = f.describe(p.description);
|
|
129
|
+
if (!nestedRequired.has(k))
|
|
130
|
+
f = f.optional();
|
|
131
|
+
nestedShape[k] = f;
|
|
132
|
+
}
|
|
133
|
+
return z.object(nestedShape);
|
|
134
|
+
}
|
|
139
135
|
default:
|
|
140
136
|
return z.unknown();
|
|
141
137
|
}
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
*/
|
|
5
5
|
import { z } from "zod";
|
|
6
6
|
import { logger } from "../utils/logger.js";
|
|
7
|
-
import {
|
|
7
|
+
import { getPersonaPrefix } from "../prompts/personas.js";
|
|
8
8
|
export function getTraceRecordingPromptText(opts) {
|
|
9
9
|
const outputDir = opts?.outputDir;
|
|
10
10
|
const modularize = opts?.modularize ?? true;
|
|
@@ -19,7 +19,7 @@ export function getTraceRecordingPromptText(opts) {
|
|
|
19
19
|
: `- Do NOT run \`skyramp_modularization\` — skip modularization in CI.`;
|
|
20
20
|
return `## Skyramp UI Test Recording
|
|
21
21
|
|
|
22
|
-
${
|
|
22
|
+
${getPersonaPrefix()} For UI recording, every action must be grounded in what \`browser_snapshot\` returns. If an element is not visible in the snapshot, do not interact with it.
|
|
23
23
|
|
|
24
24
|
### Required workflow
|
|
25
25
|
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Prompt language for skyramp_actions output — the LLM_INSTRUCTIONS block
|
|
3
|
+
* emitted to the agent that applies UPDATE and REGENERATE edits.
|
|
4
|
+
*
|
|
5
|
+
* Kept separate from actionsTool.ts so prompt text is co-located with other
|
|
6
|
+
* maintenance prompts and can be reviewed/edited without touching tool logic.
|
|
7
|
+
*/
|
|
8
|
+
/** Strategy string for URL-path find-and-replace updates. */
|
|
9
|
+
export function buildRenameStrategy() {
|
|
10
|
+
return `For each file, find-and-replace all occurrences of oldPath with newPath. Do NOT regenerate or restructure the test — only update the URL paths.`;
|
|
11
|
+
}
|
|
12
|
+
/** Strategy string for file renames after path substitution. */
|
|
13
|
+
export function buildFileRenameStrategy() {
|
|
14
|
+
return `After updating path content in each file, rename the file using 'mv' or equivalent. Use git mv if the repo tracks the file.`;
|
|
15
|
+
}
|
|
16
|
+
/** Strategy string for in-place UPDATE edits. */
|
|
17
|
+
export function buildUpdateStrategy() {
|
|
18
|
+
return `For each file in update_context, apply the changes described in context. Preserve all existing test logic — only add or adjust what is described in context. After applying all edits, call skyramp_enhance_assertions with each updated file path.`;
|
|
19
|
+
}
|
|
20
|
+
/** Strategy string for REGENERATE — call generation tool to overwrite the file. */
|
|
21
|
+
export function buildRegenerateStrategy() {
|
|
22
|
+
return `For each file in regenerate_context, call the appropriate generation tool (skyramp_integration_test_generation or skyramp_contract_test_generation) with outputDir set to the file's directory and output set to the filename. Use existing_content to determine the test type, endpoint, auth pattern, and language. The generation tool will overwrite the file. Do NOT use skyramp_ui_test_generation here — UI test regeneration requires a recorded trace (playwrightInput) and must be handled separately.`;
|
|
23
|
+
}
|
|
24
|
+
/** Per-file instruction block for a single UPDATE recommendation. */
|
|
25
|
+
export function buildUpdateFileInstruction(params) {
|
|
26
|
+
const { testFile, renames, suggestedNewFile, updateInstructions, rationale } = params;
|
|
27
|
+
const renameTable = renames.length > 0 ? [
|
|
28
|
+
`**Endpoint Rename Detected — Path Substitution Required:**\n`,
|
|
29
|
+
`| Old Path | New Path | Method |`,
|
|
30
|
+
`|----------|----------|--------|`,
|
|
31
|
+
...renames.map((r) => `| \`${r.oldPath}\` | \`${r.newPath}\` | ${r.method} |`),
|
|
32
|
+
``,
|
|
33
|
+
`**Action:** Find-and-replace all occurrences of the old path with the new path in this test file. Do NOT change any test logic, assertions, or structure — only update the URL paths.\n`,
|
|
34
|
+
...(suggestedNewFile ? [
|
|
35
|
+
`**File Rename:** After updating the paths, rename this file:`,
|
|
36
|
+
`- From: \`${testFile.split("/").pop()}\``,
|
|
37
|
+
`- To: \`${suggestedNewFile.split("/").pop()}\`\n`,
|
|
38
|
+
] : []),
|
|
39
|
+
].join("\n") : "";
|
|
40
|
+
const changeBlock = updateInstructions
|
|
41
|
+
? `**What to change:**\n\n${updateInstructions}\n`
|
|
42
|
+
: renames.length === 0
|
|
43
|
+
? [
|
|
44
|
+
rationale ? `**Why:** ${rationale}\n` : "",
|
|
45
|
+
`**Action:** Update this test file based on the rationale above.\n`,
|
|
46
|
+
].filter(Boolean).join("\n")
|
|
47
|
+
: "";
|
|
48
|
+
return `\n### ${testFile}\n\n${renameTable}${changeBlock}`;
|
|
49
|
+
}
|
|
50
|
+
/** Per-file instruction block for a single REGENERATE recommendation. */
|
|
51
|
+
export function buildRegenerateFileInstruction(params) {
|
|
52
|
+
const { testFile, updateInstructions, outputDir, outputFile } = params;
|
|
53
|
+
const whatChanged = updateInstructions ? `**What changed:**\n\n${updateInstructions}\n\n` : "";
|
|
54
|
+
return [
|
|
55
|
+
`\n### ${testFile}\n`,
|
|
56
|
+
`**Action: REGENERATE** — the response shape changed too drastically for targeted edits.\n`,
|
|
57
|
+
whatChanged,
|
|
58
|
+
`Call the appropriate generation tool (e.g. \`skyramp_integration_test_generation\`, \`skyramp_contract_test_generation\`) with \`outputDir: "${outputDir}"\` and \`output: "${outputFile}"\` to overwrite this file from scratch. Use the provided existing file content for context on the endpoint, auth pattern, and test structure — replicate the test type and language.\n`,
|
|
59
|
+
].join("\n");
|
|
60
|
+
}
|
|
@@ -1,102 +1,19 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
`;
|
|
20
|
-
}
|
|
21
|
-
const testListSection = existingTests.length > 0
|
|
22
|
-
? `## Existing Test Files (${existingTests.length})
|
|
23
|
-
${existingTests.map((t) => `- ${t.testFile} (${t.testType})`).join("\n")}
|
|
24
|
-
`
|
|
25
|
-
: `## Existing Test Files
|
|
26
|
-
No existing Skyramp tests found in repository.
|
|
27
|
-
`;
|
|
28
|
-
const scannedSection = scannedEndpoints.length > 0
|
|
29
|
-
? `## Scanned Endpoints (${scannedEndpoints.length})
|
|
30
|
-
Note: paths below come from static analysis and may be incomplete for nested resources or unsupported frameworks. Use the Routing entry-point files section below to verify and reconstruct full paths.
|
|
31
|
-
${scannedEndpoints.map((ep) => {
|
|
32
|
-
let methods;
|
|
33
|
-
if (Array.isArray(ep.methods)) {
|
|
34
|
-
methods = ep.methods.map((m) => (typeof m === "string" ? m : m.method)).join("|");
|
|
35
|
-
}
|
|
36
|
-
else {
|
|
37
|
-
methods = ep.method;
|
|
38
|
-
}
|
|
39
|
-
return `- ${methods} ${ep.path}`;
|
|
40
|
-
}).join("\n")}
|
|
41
|
-
`
|
|
42
|
-
: "";
|
|
43
|
-
const mountSection = routerMountContext?.length
|
|
44
|
-
? `## Routing entry-point files
|
|
45
|
-
Read these to trace the full router/module hierarchy when verifying endpoint paths:
|
|
46
|
-
${routerMountContext.map(f => `- \`${f}\``).join("\n")}
|
|
47
|
-
`
|
|
48
|
-
: "";
|
|
49
|
-
const hasJavaFiles = candidateRouteFiles?.some(f => /\.(java|kt)$/.test(f)) ?? false;
|
|
50
|
-
const candidateFilesSection = candidateRouteFiles && candidateRouteFiles.length > 0
|
|
51
|
-
? `## Route Files (read these to find endpoints from any framework)
|
|
52
|
-
${candidateRouteFiles.map(f => `- ${f}`).join("\n")}
|
|
53
|
-
${hasJavaFiles ? "Note — Java Spring: full URL = class-level `@RequestMapping` prefix + method-level path. If the prefix is a constant reference (e.g. `@RequestMapping(Url.PAGE_URL)`), find the constant — same file, inner class, or a separate `Url.java` — and resolve it (including `+` concatenation)." : ""}
|
|
54
|
-
`
|
|
55
|
-
: "";
|
|
56
|
-
const diffFileSection = diffFilePath
|
|
57
|
-
? `## Raw Diff File
|
|
58
|
-
Read \`${diffFilePath}\` to get the full line-by-line diff. Use it to detect:
|
|
59
|
-
- Additive response fields: lines starting with \`+\` inside a view/serializer/controller (e.g. \`+ "newField":\`, \`+ newField =\`)
|
|
60
|
-
- Renamed routes: \`- @app.route("/old")\` / \`+ @app.route("/new")\` or similar framework patterns
|
|
61
|
-
- Status code changes: \`- return 200\` / \`+ return 201\`, \`- res.status(200)\` / \`+ res.status(204)\`
|
|
62
|
-
- Auth additions/removals: \`+ @require_auth\`, \`- @login_required\`, middleware changes
|
|
63
|
-
Read the file once and cache its contents — it is the primary source for per-line breaking-change detection. Use it as evidence for Checks A–D below.
|
|
64
|
-
`
|
|
65
|
-
: "";
|
|
66
|
-
// In inline mode (testbot), skip the context header — existing tests and diff
|
|
67
|
-
// are provided by skyramp_analyze_changes at runtime, not at prompt-build time.
|
|
68
|
-
const contextSection = isTestbotEnabled()
|
|
69
|
-
? ""
|
|
70
|
-
: `# Test Health Analysis
|
|
71
|
-
|
|
72
|
-
**Repository**: \`${repositoryPath}\`
|
|
73
|
-
**Existing tests**: ${existingTests.length}
|
|
74
|
-
**New endpoints in diff**: ${newEndpointCount}
|
|
75
|
-
|
|
76
|
-
${diffSection}
|
|
77
|
-
${diffFileSection}
|
|
78
|
-
${testListSection}
|
|
79
|
-
${scannedSection}
|
|
80
|
-
${mountSection}
|
|
81
|
-
${candidateFilesSection}`;
|
|
82
|
-
if (isTestbotEnabled()) {
|
|
83
|
-
// Testbot inline mode: all maintenance logic lives here so the testbot
|
|
84
|
-
// prompt only orchestrates steps without duplicating rules.
|
|
85
|
-
// No persona statement here — the outer testbot prompt already establishes
|
|
86
|
-
// the agent's context; a nested identity statement causes role confusion.
|
|
87
|
-
return `<drift_analysis_rules>
|
|
88
|
-
${buildActionDecisionMatrix()}
|
|
89
|
-
${buildUpdateExecutionRules()}
|
|
90
|
-
${buildDriftOutputChecklist(existingTests.length, newEndpointCount, isTestbotEnabled())}
|
|
91
|
-
</drift_analysis_rules>`;
|
|
92
|
-
}
|
|
93
|
-
return `You are acting as a Skyramp Integration Architect. Your responsibility is to assess each existing test against the branch diff and determine the correct maintenance action.
|
|
94
|
-
|
|
95
|
-
${contextSection}
|
|
96
|
-
${buildActionDecisionMatrix()}
|
|
97
|
-
${buildBreakingChangePatterns()}
|
|
98
|
-
${buildTestAssessmentGuidelines()}
|
|
99
|
-
${buildUpdateExecutionRules()}
|
|
100
|
-
${buildAddRecommendationGuidelines()}
|
|
101
|
-
${buildDriftOutputChecklist(existingTests.length, newEndpointCount, isTestbotEnabled(), stateFile)}`;
|
|
1
|
+
import { buildActionDecisionTree, buildCheckAdditiveFields, buildCheckEndpointExistence, buildCheckResponseShape, buildCheckAuthAndAuthorization, buildCheckBehavioralContract, buildCheckAssignAction, buildDriftOutputChecklist, } from "./driftAnalysisSections.js";
|
|
2
|
+
import { PromptPlan } from "../test-recommendation/promptPlan.js";
|
|
3
|
+
const _plan = new PromptPlan()
|
|
4
|
+
.addPhase("maintenance", "Test Maintenance Assessment", {
|
|
5
|
+
headerLevel: "##",
|
|
6
|
+
stepFormat: "hash",
|
|
7
|
+
})
|
|
8
|
+
.step("ASSESS", "Action Decision Tree — assess each existing test against the diff", () => buildActionDecisionTree())
|
|
9
|
+
.subStep("ENDPOINT_EXISTENCE", "Endpoint existence", () => buildCheckEndpointExistence())
|
|
10
|
+
.subStep("RESPONSE_SHAPE", "Request/response shape (breaking changes)", () => buildCheckResponseShape())
|
|
11
|
+
.subStep("ADDITIVE_FIELDS", "Additive response fields (coverage gaps)", () => buildCheckAdditiveFields())
|
|
12
|
+
.subStep("AUTH_AUTHZ", "Auth and authorization changes", () => buildCheckAuthAndAuthorization())
|
|
13
|
+
.subStep("BEHAVIORAL_CONTRACT", "Behavioral and semantic contract changes", () => buildCheckBehavioralContract())
|
|
14
|
+
.subStep("ASSIGN_ACTION", "Assign action", () => buildCheckAssignAction())
|
|
15
|
+
.step("CALL_TOOL", "Submit recommendations", (p) => buildDriftOutputChecklist(p.stateFile, p.existingTests))
|
|
16
|
+
.done();
|
|
17
|
+
export function buildDriftAnalysisPrompt(stateFile, existingTests) {
|
|
18
|
+
return `<drift_analysis_rules>\n${_plan.render({ stateFile, existingTests })}\n</drift_analysis_rules>`;
|
|
102
19
|
}
|