@skyramp/mcp 0.2.3 → 0.2.5-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. package/build/playwright/registerPlaywrightTools.js +21 -25
  2. package/build/playwright/traceRecordingPrompt.js +2 -2
  3. package/build/prompts/test-maintenance/actionsInstructions.js +60 -0
  4. package/build/prompts/test-maintenance/drift-analysis-prompt.js +18 -101
  5. package/build/prompts/test-maintenance/driftAnalysisSections.js +210 -171
  6. package/build/prompts/test-recommendation/analysisOutputPrompt.js +1 -1
  7. package/build/prompts/test-recommendation/diffExecutionPlan.js +4 -3
  8. package/build/prompts/test-recommendation/recommendationSections.js +6 -6
  9. package/build/prompts/test-recommendation/scopeAssessment.js +3 -1
  10. package/build/prompts/test-recommendation/scopeAssessment.test.js +13 -0
  11. package/build/prompts/test-recommendation/test-recommendation-prompt.js +2 -2
  12. package/build/prompts/test-recommendation/test-recommendation-prompt.test.js +3 -3
  13. package/build/prompts/testbot/testbot-prompts.js +21 -17
  14. package/build/prompts/testbot/testbot-prompts.test.js +21 -17
  15. package/build/services/TestDiscoveryService.js +11 -43
  16. package/build/tools/submitReportTool.js +9 -12
  17. package/build/tools/submitReportTool.test.js +4 -5
  18. package/build/tools/test-management/actionsTool.js +160 -240
  19. package/build/tools/test-management/analyzeChangesTool.js +43 -18
  20. package/build/tools/test-management/analyzeTestHealthTool.js +17 -29
  21. package/build/utils/docker.test.js +1 -1
  22. package/build/utils/versions.js +1 -1
  23. package/node_modules/playwright/lib/mcp/skyramp/common/visualSnapshot.js +95 -0
  24. package/node_modules/playwright/lib/mcp/skyramp/loadTraceTool.js +2 -0
  25. package/node_modules/playwright/lib/mcp/skyramp/traceRecordingBackend.js +150 -2
  26. package/node_modules/playwright/lib/mcp/skyramp/visualSnapshotTool.js +63 -0
  27. package/node_modules/playwright/lib/mcp/test/skyRampExport.js +36 -0
  28. package/package.json +2 -2
  29. package/build/prompts/test-maintenance/drift-analysis-prompt.test.js +0 -116
@@ -40,11 +40,6 @@ export async function registerPlaywrightTools(server, options) {
40
40
  'browser_select_option',
41
41
  'browser_hover',
42
42
  'browser_drag',
43
- 'browser_mouse_move_xy',
44
- 'browser_mouse_click_xy',
45
- 'browser_mouse_drag_xy',
46
- 'browser_mouse_down',
47
- 'browser_mouse_up',
48
43
  'browser_file_upload',
49
44
  'browser_evaluate',
50
45
  'browser_tabs',
@@ -54,6 +49,8 @@ export async function registerPlaywrightTools(server, options) {
54
49
  'browser_assert',
55
50
  'browser_assert_api_request',
56
51
  'browser_assert_table_cell',
52
+ 'browser_mouse_action',
53
+ 'browser_visual_snapshot',
57
54
  'skyramp_export_zip',
58
55
  'skyramp_load_trace',
59
56
  'browser_mouse_action',
@@ -93,26 +90,7 @@ function jsonSchemaToZod(schema) {
93
90
  const required = new Set(schema.required || []);
94
91
  const shape = {};
95
92
  for (const [key, prop] of Object.entries(properties)) {
96
- let field;
97
- switch (prop.type) {
98
- case "string":
99
- field = prop.enum
100
- ? z.enum(prop.enum)
101
- : z.string();
102
- break;
103
- case "number":
104
- case "integer":
105
- field = z.number();
106
- break;
107
- case "boolean":
108
- field = z.boolean();
109
- break;
110
- case "array":
111
- field = z.array(prop.items ? jsonSchemaPropertyToZod(prop.items) : z.unknown());
112
- break;
113
- default:
114
- field = z.unknown();
115
- }
93
+ let field = jsonSchemaPropertyToZod(prop);
116
94
  if (prop.description)
117
95
  field = field.describe(prop.description);
118
96
  if (prop.default !== undefined)
@@ -136,6 +114,24 @@ function jsonSchemaPropertyToZod(prop) {
136
114
  return z.boolean();
137
115
  case "array":
138
116
  return z.array(prop.items ? jsonSchemaPropertyToZod(prop.items) : z.unknown());
117
+ case "object": {
118
+ // Nested object (e.g. a clip rect). Recurse into its properties so the
119
+ // SDK registers a real object schema; without this it falls through to
120
+ // z.unknown(), which serializes the nested object as a string and the
121
+ // inner backend's z.object(...) then rejects it.
122
+ const nestedProps = prop.properties || {};
123
+ const nestedRequired = new Set(prop.required || []);
124
+ const nestedShape = {};
125
+ for (const [k, p] of Object.entries(nestedProps)) {
126
+ let f = jsonSchemaPropertyToZod(p);
127
+ if (p.description)
128
+ f = f.describe(p.description);
129
+ if (!nestedRequired.has(k))
130
+ f = f.optional();
131
+ nestedShape[k] = f;
132
+ }
133
+ return z.object(nestedShape);
134
+ }
139
135
  default:
140
136
  return z.unknown();
141
137
  }
@@ -4,7 +4,7 @@
4
4
  */
5
5
  import { z } from "zod";
6
6
  import { logger } from "../utils/logger.js";
7
- import { SKYRAMP_QA_PERSONA } from "../prompts/personas.js";
7
+ import { getPersonaPrefix } from "../prompts/personas.js";
8
8
  export function getTraceRecordingPromptText(opts) {
9
9
  const outputDir = opts?.outputDir;
10
10
  const modularize = opts?.modularize ?? true;
@@ -19,7 +19,7 @@ export function getTraceRecordingPromptText(opts) {
19
19
  : `- Do NOT run \`skyramp_modularization\` — skip modularization in CI.`;
20
20
  return `## Skyramp UI Test Recording
21
21
 
22
- ${SKYRAMP_QA_PERSONA} For UI recording, every action must be grounded in what \`browser_snapshot\` returns. If an element is not visible in the snapshot, do not interact with it.
22
+ ${getPersonaPrefix()} For UI recording, every action must be grounded in what \`browser_snapshot\` returns. If an element is not visible in the snapshot, do not interact with it.
23
23
 
24
24
  ### Required workflow
25
25
 
@@ -0,0 +1,60 @@
1
+ /**
2
+ * Prompt language for skyramp_actions output — the LLM_INSTRUCTIONS block
3
+ * emitted to the agent that applies UPDATE and REGENERATE edits.
4
+ *
5
+ * Kept separate from actionsTool.ts so prompt text is co-located with other
6
+ * maintenance prompts and can be reviewed/edited without touching tool logic.
7
+ */
8
+ /** Strategy string for URL-path find-and-replace updates. */
9
+ export function buildRenameStrategy() {
10
+ return `For each file, find-and-replace all occurrences of oldPath with newPath. Do NOT regenerate or restructure the test — only update the URL paths.`;
11
+ }
12
+ /** Strategy string for file renames after path substitution. */
13
+ export function buildFileRenameStrategy() {
14
+ return `After updating path content in each file, rename the file using 'mv' or equivalent. Use git mv if the repo tracks the file.`;
15
+ }
16
+ /** Strategy string for in-place UPDATE edits. */
17
+ export function buildUpdateStrategy() {
18
+ return `For each file in update_context, apply the changes described in context. Preserve all existing test logic — only add or adjust what is described in context. After applying all edits, call skyramp_enhance_assertions with each updated file path.`;
19
+ }
20
+ /** Strategy string for REGENERATE — call generation tool to overwrite the file. */
21
+ export function buildRegenerateStrategy() {
22
+ return `For each file in regenerate_context, call the appropriate generation tool (skyramp_integration_test_generation or skyramp_contract_test_generation) with outputDir set to the file's directory and output set to the filename. Use existing_content to determine the test type, endpoint, auth pattern, and language. The generation tool will overwrite the file. Do NOT use skyramp_ui_test_generation here — UI test regeneration requires a recorded trace (playwrightInput) and must be handled separately.`;
23
+ }
24
+ /** Per-file instruction block for a single UPDATE recommendation. */
25
+ export function buildUpdateFileInstruction(params) {
26
+ const { testFile, renames, suggestedNewFile, updateInstructions, rationale } = params;
27
+ const renameTable = renames.length > 0 ? [
28
+ `**Endpoint Rename Detected — Path Substitution Required:**\n`,
29
+ `| Old Path | New Path | Method |`,
30
+ `|----------|----------|--------|`,
31
+ ...renames.map((r) => `| \`${r.oldPath}\` | \`${r.newPath}\` | ${r.method} |`),
32
+ ``,
33
+ `**Action:** Find-and-replace all occurrences of the old path with the new path in this test file. Do NOT change any test logic, assertions, or structure — only update the URL paths.\n`,
34
+ ...(suggestedNewFile ? [
35
+ `**File Rename:** After updating the paths, rename this file:`,
36
+ `- From: \`${testFile.split("/").pop()}\``,
37
+ `- To: \`${suggestedNewFile.split("/").pop()}\`\n`,
38
+ ] : []),
39
+ ].join("\n") : "";
40
+ const changeBlock = updateInstructions
41
+ ? `**What to change:**\n\n${updateInstructions}\n`
42
+ : renames.length === 0
43
+ ? [
44
+ rationale ? `**Why:** ${rationale}\n` : "",
45
+ `**Action:** Update this test file based on the rationale above.\n`,
46
+ ].filter(Boolean).join("\n")
47
+ : "";
48
+ return `\n### ${testFile}\n\n${renameTable}${changeBlock}`;
49
+ }
50
+ /** Per-file instruction block for a single REGENERATE recommendation. */
51
+ export function buildRegenerateFileInstruction(params) {
52
+ const { testFile, updateInstructions, outputDir, outputFile } = params;
53
+ const whatChanged = updateInstructions ? `**What changed:**\n\n${updateInstructions}\n\n` : "";
54
+ return [
55
+ `\n### ${testFile}\n`,
56
+ `**Action: REGENERATE** — the response shape changed too drastically for targeted edits.\n`,
57
+ whatChanged,
58
+ `Call the appropriate generation tool (e.g. \`skyramp_integration_test_generation\`, \`skyramp_contract_test_generation\`) with \`outputDir: "${outputDir}"\` and \`output: "${outputFile}"\` to overwrite this file from scratch. Use the provided existing file content for context on the endpoint, auth pattern, and test structure — replicate the test type and language.\n`,
59
+ ].join("\n");
60
+ }
@@ -1,102 +1,19 @@
1
- import { buildActionDecisionMatrix, buildBreakingChangePatterns, buildTestAssessmentGuidelines, buildAddRecommendationGuidelines, buildDriftOutputChecklist, buildUpdateExecutionRules, } from "./driftAnalysisSections.js";
2
- import { isTestbotEnabled } from "../../utils/featureFlags.js";
3
- import { readDiffFile } from "../../utils/utils.js";
4
- export function buildDriftAnalysisPrompt(params) {
5
- const { existingTests, scannedEndpoints, repositoryPath, stateFile, routerMountContext, candidateRouteFiles, diffFilePath } = params;
6
- // Read raw diff once — used for both the inline summary block and the per-line file reference.
7
- const rawDiff = readDiffFile(diffFilePath);
8
- let newEndpointCount = 0;
9
- let diffSection = "";
10
- if (rawDiff) {
11
- const lines = rawDiff.split("\n");
12
- const newEndpointMatch = rawDiff.match(/\*\*New Endpoints\*\*\s+\((\d+)\)/);
13
- if (newEndpointMatch)
14
- newEndpointCount = parseInt(newEndpointMatch[1], 10);
15
- diffSection = `## Branch Diff
16
- \`\`\`
17
- ${lines.slice(0, 200).join("\n")}
18
- \`\`\`
19
- `;
20
- }
21
- const testListSection = existingTests.length > 0
22
- ? `## Existing Test Files (${existingTests.length})
23
- ${existingTests.map((t) => `- ${t.testFile} (${t.testType})`).join("\n")}
24
- `
25
- : `## Existing Test Files
26
- No existing Skyramp tests found in repository.
27
- `;
28
- const scannedSection = scannedEndpoints.length > 0
29
- ? `## Scanned Endpoints (${scannedEndpoints.length})
30
- Note: paths below come from static analysis and may be incomplete for nested resources or unsupported frameworks. Use the Routing entry-point files section below to verify and reconstruct full paths.
31
- ${scannedEndpoints.map((ep) => {
32
- let methods;
33
- if (Array.isArray(ep.methods)) {
34
- methods = ep.methods.map((m) => (typeof m === "string" ? m : m.method)).join("|");
35
- }
36
- else {
37
- methods = ep.method;
38
- }
39
- return `- ${methods} ${ep.path}`;
40
- }).join("\n")}
41
- `
42
- : "";
43
- const mountSection = routerMountContext?.length
44
- ? `## Routing entry-point files
45
- Read these to trace the full router/module hierarchy when verifying endpoint paths:
46
- ${routerMountContext.map(f => `- \`${f}\``).join("\n")}
47
- `
48
- : "";
49
- const hasJavaFiles = candidateRouteFiles?.some(f => /\.(java|kt)$/.test(f)) ?? false;
50
- const candidateFilesSection = candidateRouteFiles && candidateRouteFiles.length > 0
51
- ? `## Route Files (read these to find endpoints from any framework)
52
- ${candidateRouteFiles.map(f => `- ${f}`).join("\n")}
53
- ${hasJavaFiles ? "Note — Java Spring: full URL = class-level `@RequestMapping` prefix + method-level path. If the prefix is a constant reference (e.g. `@RequestMapping(Url.PAGE_URL)`), find the constant — same file, inner class, or a separate `Url.java` — and resolve it (including `+` concatenation)." : ""}
54
- `
55
- : "";
56
- const diffFileSection = diffFilePath
57
- ? `## Raw Diff File
58
- Read \`${diffFilePath}\` to get the full line-by-line diff. Use it to detect:
59
- - Additive response fields: lines starting with \`+\` inside a view/serializer/controller (e.g. \`+ "newField":\`, \`+ newField =\`)
60
- - Renamed routes: \`- @app.route("/old")\` / \`+ @app.route("/new")\` or similar framework patterns
61
- - Status code changes: \`- return 200\` / \`+ return 201\`, \`- res.status(200)\` / \`+ res.status(204)\`
62
- - Auth additions/removals: \`+ @require_auth\`, \`- @login_required\`, middleware changes
63
- Read the file once and cache its contents — it is the primary source for per-line breaking-change detection. Use it as evidence for Checks A–D below.
64
- `
65
- : "";
66
- // In inline mode (testbot), skip the context header — existing tests and diff
67
- // are provided by skyramp_analyze_changes at runtime, not at prompt-build time.
68
- const contextSection = isTestbotEnabled()
69
- ? ""
70
- : `# Test Health Analysis
71
-
72
- **Repository**: \`${repositoryPath}\`
73
- **Existing tests**: ${existingTests.length}
74
- **New endpoints in diff**: ${newEndpointCount}
75
-
76
- ${diffSection}
77
- ${diffFileSection}
78
- ${testListSection}
79
- ${scannedSection}
80
- ${mountSection}
81
- ${candidateFilesSection}`;
82
- if (isTestbotEnabled()) {
83
- // Testbot inline mode: all maintenance logic lives here so the testbot
84
- // prompt only orchestrates steps without duplicating rules.
85
- // No persona statement here — the outer testbot prompt already establishes
86
- // the agent's context; a nested identity statement causes role confusion.
87
- return `<drift_analysis_rules>
88
- ${buildActionDecisionMatrix()}
89
- ${buildUpdateExecutionRules()}
90
- ${buildDriftOutputChecklist(existingTests.length, newEndpointCount, isTestbotEnabled())}
91
- </drift_analysis_rules>`;
92
- }
93
- return `You are acting as a Skyramp Integration Architect. Your responsibility is to assess each existing test against the branch diff and determine the correct maintenance action.
94
-
95
- ${contextSection}
96
- ${buildActionDecisionMatrix()}
97
- ${buildBreakingChangePatterns()}
98
- ${buildTestAssessmentGuidelines()}
99
- ${buildUpdateExecutionRules()}
100
- ${buildAddRecommendationGuidelines()}
101
- ${buildDriftOutputChecklist(existingTests.length, newEndpointCount, isTestbotEnabled(), stateFile)}`;
1
+ import { buildActionDecisionTree, buildCheckAdditiveFields, buildCheckEndpointExistence, buildCheckResponseShape, buildCheckAuthAndAuthorization, buildCheckBehavioralContract, buildCheckAssignAction, buildDriftOutputChecklist, } from "./driftAnalysisSections.js";
2
+ import { PromptPlan } from "../test-recommendation/promptPlan.js";
3
+ const _plan = new PromptPlan()
4
+ .addPhase("maintenance", "Test Maintenance Assessment", {
5
+ headerLevel: "##",
6
+ stepFormat: "hash",
7
+ })
8
+ .step("ASSESS", "Action Decision Tree — assess each existing test against the diff", () => buildActionDecisionTree())
9
+ .subStep("ENDPOINT_EXISTENCE", "Endpoint existence", () => buildCheckEndpointExistence())
10
+ .subStep("RESPONSE_SHAPE", "Request/response shape (breaking changes)", () => buildCheckResponseShape())
11
+ .subStep("ADDITIVE_FIELDS", "Additive response fields (coverage gaps)", () => buildCheckAdditiveFields())
12
+ .subStep("AUTH_AUTHZ", "Auth and authorization changes", () => buildCheckAuthAndAuthorization())
13
+ .subStep("BEHAVIORAL_CONTRACT", "Behavioral and semantic contract changes", () => buildCheckBehavioralContract())
14
+ .subStep("ASSIGN_ACTION", "Assign action", () => buildCheckAssignAction())
15
+ .step("CALL_TOOL", "Submit recommendations", (p) => buildDriftOutputChecklist(p.stateFile, p.existingTests))
16
+ .done();
17
+ export function buildDriftAnalysisPrompt(stateFile, existingTests) {
18
+ return `<drift_analysis_rules>\n${_plan.render({ stateFile, existingTests })}\n</drift_analysis_rules>`;
102
19
  }