@skyramp/mcp 0.1.8 → 0.2.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/index.js +4 -2
- package/build/playwright/registerPlaywrightTools.js +12 -0
- package/build/playwright/traceRecordingPrompt.js +15 -0
- package/build/prompts/code-reuse.js +106 -7
- package/build/prompts/pom-aware-code-reuse.js +106 -7
- package/build/prompts/startTraceCollectionPrompts.js +37 -15
- package/build/prompts/test-maintenance/drift-analysis-prompt.js +26 -31
- package/build/prompts/test-maintenance/drift-analysis-prompt.test.js +40 -1
- package/build/prompts/test-maintenance/driftAnalysisSections.js +90 -86
- package/build/prompts/test-recommendation/analysisOutputPrompt.js +286 -163
- package/build/prompts/test-recommendation/analysisOutputPrompt.test.js +154 -45
- package/build/prompts/test-recommendation/diffExecutionPlan.js +246 -117
- package/build/prompts/test-recommendation/promptPlan.js +290 -0
- package/build/prompts/test-recommendation/promptPlan.test.js +336 -0
- package/build/prompts/test-recommendation/recommendationSections.js +4 -3
- package/build/prompts/test-recommendation/recommendationShared.js +23 -1
- package/build/prompts/test-recommendation/scopeAssessment.js +65 -14
- package/build/prompts/test-recommendation/scopeAssessment.test.js +93 -2
- package/build/prompts/test-recommendation/test-recommendation-prompt.js +36 -12
- package/build/prompts/test-recommendation/test-recommendation-prompt.test.js +316 -1
- package/build/prompts/testbot/testbot-prompts.js +73 -13
- package/build/prompts/testbot/testbot-prompts.test.js +114 -1
- package/build/resources/testbotResource.js +1 -1
- package/build/services/ScenarioGenerationService.integration.test.js +158 -0
- package/build/services/ScenarioGenerationService.js +47 -4
- package/build/services/ScenarioGenerationService.test.js +158 -22
- package/build/services/TestExecutionService.js +73 -15
- package/build/services/TestExecutionService.test.js +105 -0
- package/build/services/TestGenerationService.js +11 -1
- package/build/tools/executeSkyrampTestTool.js +1 -10
- package/build/tools/generate-tests/generateBatchScenarioRestTool.js +16 -4
- package/build/tools/generate-tests/generateIntegrationRestTool.js +2 -0
- package/build/tools/generate-tests/generateUIRestTool.js +2 -0
- package/build/tools/test-management/actionsTool.js +152 -63
- package/build/tools/test-management/analyzeChangesTool.js +178 -64
- package/build/tools/test-management/analyzeChangesTool.test.js +103 -16
- package/build/tools/test-management/analyzeTestHealthTool.js +30 -81
- package/build/tools/test-management/index.js +1 -0
- package/build/tools/test-management/uiAnalyzeChangesTool.js +149 -0
- package/build/tools/test-management/uiAnalyzeChangesTool.test.js +100 -0
- package/build/tools/trace/resolveSaveStoragePath.js +16 -0
- package/build/tools/trace/resolveSaveStoragePath.test.js +17 -0
- package/build/tools/trace/resolveSessionPaths.js +39 -0
- package/build/tools/trace/resolveSessionPaths.test.js +103 -0
- package/build/tools/trace/sessionState.js +14 -0
- package/build/tools/trace/sessionState.test.js +17 -0
- package/build/tools/trace/startTraceCollectionTool.js +84 -14
- package/build/tools/trace/stopTraceCollectionTool.js +9 -2
- package/build/types/TestAnalysis.js +50 -0
- package/build/types/TestRecommendation.js +6 -58
- package/build/types/TestTypes.js +1 -1
- package/build/utils/AnalysisStateManager.js +22 -11
- package/build/utils/branchDiff.js +11 -2
- package/build/utils/docker.test.js +1 -1
- package/build/utils/gitStaging.js +52 -3
- package/build/utils/gitStaging.test.js +19 -1
- package/build/utils/repoScanner.js +18 -10
- package/build/utils/repoScanner.test.js +92 -0
- package/build/utils/routeParsers.js +180 -25
- package/build/utils/routeParsers.test.js +180 -1
- package/build/utils/scenarioDrafting.js +220 -17
- package/build/utils/scenarioDrafting.test.js +182 -9
- package/build/utils/sourceRouteExtractor.js +806 -0
- package/build/utils/sourceRouteExtractor.test.js +565 -0
- package/build/utils/uiPageEnumerator.js +319 -0
- package/build/utils/uiPageEnumerator.test.js +422 -0
- package/build/utils/utils.js +27 -0
- package/build/utils/versions.js +1 -1
- package/build/utils/workspaceAuth.js +33 -4
- package/node_modules/playwright/ThirdPartyNotices.txt +6 -6
- package/node_modules/playwright/lib/dom-analyzer/analyze.js +111 -0
- package/node_modules/playwright/lib/dom-analyzer/blueprint.js +1210 -0
- package/node_modules/playwright/lib/dom-analyzer/blueprint.test.js +396 -0
- package/node_modules/playwright/lib/dom-analyzer/blueprintCache.js +57 -0
- package/node_modules/playwright/lib/dom-analyzer/blueprintCache.test.js +57 -0
- package/node_modules/playwright/lib/dom-analyzer/blueprintDiff.js +254 -0
- package/node_modules/playwright/lib/dom-analyzer/blueprintDiff.test.js +304 -0
- package/node_modules/playwright/lib/dom-analyzer/crawler.js +384 -0
- package/node_modules/playwright/lib/dom-analyzer/curatedWidgets.js +73 -0
- package/node_modules/playwright/lib/dom-analyzer/dynamicId.js +43 -0
- package/node_modules/playwright/lib/dom-analyzer/dynamicId.test.js +85 -0
- package/node_modules/playwright/lib/dom-analyzer/fingerprint.js +90 -0
- package/node_modules/playwright/lib/dom-analyzer/fingerprint.test.js +231 -0
- package/node_modules/playwright/lib/dom-analyzer/fingerprintAblation.fixtures.js +145 -0
- package/node_modules/playwright/lib/dom-analyzer/fingerprintAblation.test.js +41 -0
- package/node_modules/playwright/lib/dom-analyzer/graph.js +36 -0
- package/node_modules/playwright/lib/dom-analyzer/liveFingerprints.js +43 -0
- package/node_modules/playwright/lib/dom-analyzer/logicalNameResolver.js +72 -0
- package/node_modules/playwright/lib/dom-analyzer/logicalNameResolver.test.js +182 -0
- package/node_modules/playwright/lib/dom-analyzer/possibleAssertions.js +150 -0
- package/node_modules/playwright/lib/dom-analyzer/possibleAssertions.test.js +470 -0
- package/node_modules/playwright/lib/dom-analyzer/sectionGrouper.js +169 -0
- package/node_modules/playwright/lib/dom-analyzer/sectionGrouper.test.js +269 -0
- package/node_modules/playwright/lib/dom-analyzer/serialization.js +75 -0
- package/node_modules/playwright/lib/dom-analyzer/slug.js +30 -0
- package/node_modules/playwright/lib/dom-analyzer/slug.test.js +84 -0
- package/node_modules/playwright/lib/dom-analyzer/widgetContract.js +127 -0
- package/node_modules/playwright/lib/dom-analyzer/widgetContract.test.js +212 -0
- package/node_modules/playwright/lib/mcp/browser/browserContextFactory.js +3 -1
- package/node_modules/playwright/lib/mcp/browser/config.js +1 -1
- package/node_modules/playwright/lib/mcp/browser/context.js +17 -1
- package/node_modules/playwright/lib/mcp/browser/tab.js +38 -0
- package/node_modules/playwright/lib/mcp/browser/tools/domAnalyzer.js +261 -0
- package/node_modules/playwright/lib/mcp/browser/tools/keyboard.js +3 -3
- package/node_modules/playwright/lib/mcp/browser/tools/pageBlueprint.js +146 -0
- package/node_modules/playwright/lib/mcp/browser/tools/pageBlueprint.test.js +140 -0
- package/node_modules/playwright/lib/mcp/browser/tools/sitemap.js +226 -0
- package/node_modules/playwright/lib/mcp/browser/tools/snapshot.js +2 -2
- package/node_modules/playwright/lib/mcp/browser/tools/widgetContract.js +168 -0
- package/node_modules/playwright/lib/mcp/browser/tools.js +6 -0
- package/node_modules/playwright/lib/mcp/skyramp/traceRecordingBackend.js +52 -12
- package/node_modules/playwright/lib/mcp/test/skyRampExport.js +64 -13
- package/node_modules/playwright/package.json +1 -1
- package/node_modules/playwright/skyramp-playwright-1.58.2-skyramp.8.9.3.tgz +0 -0
- package/node_modules/playwright/skyramp-playwright-1.58.2-skyramp.8.9.4.tgz +0 -0
- package/node_modules/playwright/skyramp-playwright-1.58.2-skyramp.8.9.5.tgz +0 -0
- package/node_modules/playwright/skyramp-playwright-1.58.2-skyramp.8.9.6.tgz +0 -0
- package/package.json +3 -3
- package/build/services/TestHealthService.js +0 -694
- package/build/services/TestHealthService.test.js +0 -241
- package/build/types/TestDriftAnalysis.js +0 -1
- package/build/types/TestHealth.js +0 -4
|
@@ -1,241 +0,0 @@
|
|
|
1
|
-
import { TestHealthService } from "./TestHealthService.js";
|
|
2
|
-
describe("TestHealthService", () => {
|
|
3
|
-
let service;
|
|
4
|
-
beforeEach(() => {
|
|
5
|
-
service = new TestHealthService();
|
|
6
|
-
});
|
|
7
|
-
describe("identifyIssues - endpoint rename detection", () => {
|
|
8
|
-
function identifyIssues(execution, drift) {
|
|
9
|
-
return service["identifyIssues"](execution, drift);
|
|
10
|
-
}
|
|
11
|
-
it("should create an endpoints_renamed issue when drift has endpoint_renamed changes", () => {
|
|
12
|
-
const drift = {
|
|
13
|
-
testFile: "products_smoke_test.py",
|
|
14
|
-
lastCommit: "abc123",
|
|
15
|
-
currentCommit: "def456",
|
|
16
|
-
driftScore: 30,
|
|
17
|
-
changes: [
|
|
18
|
-
{
|
|
19
|
-
type: "endpoint_renamed",
|
|
20
|
-
file: "API Schema",
|
|
21
|
-
description: "Endpoint renamed: get /api/v1/products -> /api/v1/items",
|
|
22
|
-
severity: "high",
|
|
23
|
-
details: "Path changed from /api/v1/products to /api/v1/items",
|
|
24
|
-
},
|
|
25
|
-
],
|
|
26
|
-
affectedFiles: { files: ["src/routers/product.py"] },
|
|
27
|
-
analysisTimestamp: new Date().toISOString(),
|
|
28
|
-
};
|
|
29
|
-
const issues = identifyIssues(undefined, drift);
|
|
30
|
-
const renameIssue = issues.find((i) => i.type === "endpoints_renamed");
|
|
31
|
-
expect(renameIssue).toBeDefined();
|
|
32
|
-
expect(renameIssue?.severity).toBe("high");
|
|
33
|
-
expect(renameIssue?.description).toContain("1 API endpoint(s) renamed");
|
|
34
|
-
});
|
|
35
|
-
it("should not create endpoints_renamed issue when no renames in drift", () => {
|
|
36
|
-
const drift = {
|
|
37
|
-
testFile: "products_smoke_test.py",
|
|
38
|
-
lastCommit: "abc123",
|
|
39
|
-
currentCommit: "def456",
|
|
40
|
-
driftScore: 15,
|
|
41
|
-
changes: [
|
|
42
|
-
{
|
|
43
|
-
type: "endpoint_removed",
|
|
44
|
-
file: "API Schema",
|
|
45
|
-
description: "1 endpoint(s) removed",
|
|
46
|
-
severity: "high",
|
|
47
|
-
},
|
|
48
|
-
],
|
|
49
|
-
affectedFiles: { files: [] },
|
|
50
|
-
analysisTimestamp: new Date().toISOString(),
|
|
51
|
-
};
|
|
52
|
-
const issues = identifyIssues(undefined, drift);
|
|
53
|
-
const renameIssue = issues.find((i) => i.type === "endpoints_renamed");
|
|
54
|
-
expect(renameIssue).toBeUndefined();
|
|
55
|
-
const removeIssue = issues.find((i) => i.type === "endpoints_removed");
|
|
56
|
-
expect(removeIssue).toBeDefined();
|
|
57
|
-
});
|
|
58
|
-
it("should handle multiple rename changes", () => {
|
|
59
|
-
const drift = {
|
|
60
|
-
testFile: "products_smoke_test.py",
|
|
61
|
-
lastCommit: "abc123",
|
|
62
|
-
currentCommit: "def456",
|
|
63
|
-
driftScore: 40,
|
|
64
|
-
changes: [
|
|
65
|
-
{
|
|
66
|
-
type: "endpoint_renamed",
|
|
67
|
-
file: "API Schema",
|
|
68
|
-
description: "Endpoint renamed: get /api/v1/products -> /api/v1/items",
|
|
69
|
-
severity: "high",
|
|
70
|
-
},
|
|
71
|
-
{
|
|
72
|
-
type: "endpoint_renamed",
|
|
73
|
-
file: "API Schema",
|
|
74
|
-
description: "Endpoint renamed: post /api/v1/products -> /api/v1/items",
|
|
75
|
-
severity: "high",
|
|
76
|
-
},
|
|
77
|
-
],
|
|
78
|
-
affectedFiles: { files: [] },
|
|
79
|
-
analysisTimestamp: new Date().toISOString(),
|
|
80
|
-
};
|
|
81
|
-
const issues = identifyIssues(undefined, drift);
|
|
82
|
-
const renameIssue = issues.find((i) => i.type === "endpoints_renamed");
|
|
83
|
-
expect(renameIssue).toBeDefined();
|
|
84
|
-
expect(renameIssue?.description).toContain("2 API endpoint(s) renamed");
|
|
85
|
-
});
|
|
86
|
-
});
|
|
87
|
-
describe("generateRecommendation - endpoint rename handling", () => {
|
|
88
|
-
function generateRecommendation(testFile, driftScore, execution, issues, apiEndpoint, apiSchemaChanges) {
|
|
89
|
-
const healthScore = service["calculateHealthScore"](execution
|
|
90
|
-
? service["calculateExecutionScore"](execution).score
|
|
91
|
-
: undefined, driftScore);
|
|
92
|
-
return service["generateRecommendation"](testFile, healthScore, driftScore, execution, issues, apiEndpoint, apiSchemaChanges);
|
|
93
|
-
}
|
|
94
|
-
it("should return UPDATE action for endpoint renames regardless of drift score", () => {
|
|
95
|
-
const issues = [
|
|
96
|
-
{
|
|
97
|
-
type: "endpoints_renamed",
|
|
98
|
-
severity: "high",
|
|
99
|
-
description: "1 API endpoint(s) renamed",
|
|
100
|
-
details: "Endpoint renamed: get /api/v1/products -> /api/v1/items",
|
|
101
|
-
},
|
|
102
|
-
];
|
|
103
|
-
const apiSchemaChanges = {
|
|
104
|
-
endpointsRemoved: [],
|
|
105
|
-
endpointsRenamed: [
|
|
106
|
-
{ oldPath: "/api/v1/products", newPath: "/api/v1/items", method: "get" },
|
|
107
|
-
],
|
|
108
|
-
endpointsModified: [],
|
|
109
|
-
authenticationChanged: false,
|
|
110
|
-
};
|
|
111
|
-
// Even with low drift score, renames should trigger UPDATE
|
|
112
|
-
const rec = generateRecommendation("products_smoke_test.py", 12, // low drift
|
|
113
|
-
undefined, issues, undefined, apiSchemaChanges);
|
|
114
|
-
expect(rec.action).toBe("UPDATE");
|
|
115
|
-
expect(rec.priority).toBe("HIGH");
|
|
116
|
-
expect(rec.rationale).toContain("renamed");
|
|
117
|
-
expect(rec.estimatedWork).toBe("SMALL");
|
|
118
|
-
});
|
|
119
|
-
it("should include renamedEndpoints in recommendation details", () => {
|
|
120
|
-
const issues = [
|
|
121
|
-
{
|
|
122
|
-
type: "endpoints_renamed",
|
|
123
|
-
severity: "high",
|
|
124
|
-
description: "1 API endpoint(s) renamed",
|
|
125
|
-
details: "Endpoint renamed: get /api/v1/products -> /api/v1/items",
|
|
126
|
-
},
|
|
127
|
-
];
|
|
128
|
-
const apiSchemaChanges = {
|
|
129
|
-
endpointsRemoved: [],
|
|
130
|
-
endpointsRenamed: [
|
|
131
|
-
{ oldPath: "/api/v1/products", newPath: "/api/v1/items", method: "get" },
|
|
132
|
-
],
|
|
133
|
-
endpointsModified: [],
|
|
134
|
-
authenticationChanged: false,
|
|
135
|
-
};
|
|
136
|
-
const rec = generateRecommendation("products_smoke_test.py", 30, undefined, issues, undefined, apiSchemaChanges);
|
|
137
|
-
expect(rec.details?.endpointStatus).toBe("renamed");
|
|
138
|
-
expect(rec.details?.renamedEndpoints).toBeDefined();
|
|
139
|
-
expect(rec.details?.renamedEndpoints).toHaveLength(1);
|
|
140
|
-
expect(rec.details?.renamedEndpoints?.[0]).toEqual({
|
|
141
|
-
oldPath: "/api/v1/products",
|
|
142
|
-
newPath: "/api/v1/items",
|
|
143
|
-
method: "get",
|
|
144
|
-
});
|
|
145
|
-
});
|
|
146
|
-
it("should mention test failure in rationale when test is failing due to rename", () => {
|
|
147
|
-
const issues = [
|
|
148
|
-
{
|
|
149
|
-
type: "endpoints_renamed",
|
|
150
|
-
severity: "high",
|
|
151
|
-
description: "1 API endpoint(s) renamed",
|
|
152
|
-
details: "Endpoint renamed: get /api/v1/products -> /api/v1/items",
|
|
153
|
-
},
|
|
154
|
-
];
|
|
155
|
-
const execution = {
|
|
156
|
-
testFile: "products_smoke_test.py",
|
|
157
|
-
executedAt: new Date().toISOString(),
|
|
158
|
-
passed: false,
|
|
159
|
-
duration: 10000,
|
|
160
|
-
errors: ["404 Not Found"],
|
|
161
|
-
warnings: [],
|
|
162
|
-
crashed: false,
|
|
163
|
-
};
|
|
164
|
-
const apiSchemaChanges = {
|
|
165
|
-
endpointsRemoved: [],
|
|
166
|
-
endpointsRenamed: [
|
|
167
|
-
{ oldPath: "/api/v1/products", newPath: "/api/v1/items", method: "get" },
|
|
168
|
-
],
|
|
169
|
-
endpointsModified: [],
|
|
170
|
-
authenticationChanged: false,
|
|
171
|
-
};
|
|
172
|
-
const rec = generateRecommendation("products_smoke_test.py", 30, execution, issues, undefined, apiSchemaChanges);
|
|
173
|
-
expect(rec.action).toBe("UPDATE");
|
|
174
|
-
expect(rec.rationale).toContain("failing");
|
|
175
|
-
});
|
|
176
|
-
it("should escalate to UPDATE when drift is 10-30 and test is failing", () => {
|
|
177
|
-
const execution = {
|
|
178
|
-
testFile: "products_integration_test.py",
|
|
179
|
-
executedAt: new Date().toISOString(),
|
|
180
|
-
passed: false,
|
|
181
|
-
duration: 2770,
|
|
182
|
-
errors: ["POST returns 400: Input should be a valid integer"],
|
|
183
|
-
warnings: [],
|
|
184
|
-
crashed: false,
|
|
185
|
-
};
|
|
186
|
-
const rec = generateRecommendation("products_integration_test.py", 15, // low drift (10-30 range)
|
|
187
|
-
execution, []);
|
|
188
|
-
expect(rec.action).toBe("UPDATE");
|
|
189
|
-
expect(rec.priority).toBe("HIGH");
|
|
190
|
-
expect(rec.estimatedWork).toBe("MEDIUM");
|
|
191
|
-
});
|
|
192
|
-
it("should keep VERIFY when drift is 10-30 and test is passing", () => {
|
|
193
|
-
const execution = {
|
|
194
|
-
testFile: "products_fuzz_test.py",
|
|
195
|
-
executedAt: new Date().toISOString(),
|
|
196
|
-
passed: true,
|
|
197
|
-
duration: 75000,
|
|
198
|
-
errors: [],
|
|
199
|
-
warnings: [],
|
|
200
|
-
crashed: false,
|
|
201
|
-
};
|
|
202
|
-
const rec = generateRecommendation("products_fuzz_test.py", 15, execution, []);
|
|
203
|
-
expect(rec.action).toBe("VERIFY");
|
|
204
|
-
expect(rec.priority).toBe("LOW");
|
|
205
|
-
});
|
|
206
|
-
it("should not set renamedEndpoints when there are no renames", () => {
|
|
207
|
-
const rec = generateRecommendation("orders_smoke_test.py", 5, undefined, [], { exists: true }, undefined);
|
|
208
|
-
expect(rec.action).toBe("VERIFY");
|
|
209
|
-
expect(rec.details?.renamedEndpoints).toBeUndefined();
|
|
210
|
-
expect(rec.details?.endpointStatus).toBe("exists");
|
|
211
|
-
});
|
|
212
|
-
it("should prefer rename handling over high-drift REGENERATE", () => {
|
|
213
|
-
// If drift is > 70 but it's caused by a rename, we should UPDATE not REGENERATE
|
|
214
|
-
const issues = [
|
|
215
|
-
{
|
|
216
|
-
type: "endpoints_renamed",
|
|
217
|
-
severity: "high",
|
|
218
|
-
description: "5 API endpoint(s) renamed",
|
|
219
|
-
details: "Multiple renames",
|
|
220
|
-
},
|
|
221
|
-
];
|
|
222
|
-
const apiSchemaChanges = {
|
|
223
|
-
endpointsRemoved: [],
|
|
224
|
-
endpointsRenamed: [
|
|
225
|
-
{ oldPath: "/api/v1/products", newPath: "/api/v1/items", method: "get" },
|
|
226
|
-
{ oldPath: "/api/v1/products", newPath: "/api/v1/items", method: "post" },
|
|
227
|
-
{ oldPath: "/api/v1/products/{id}", newPath: "/api/v1/items/{id}", method: "get" },
|
|
228
|
-
{ oldPath: "/api/v1/products/{id}", newPath: "/api/v1/items/{id}", method: "put" },
|
|
229
|
-
{ oldPath: "/api/v1/products/{id}", newPath: "/api/v1/items/{id}", method: "delete" },
|
|
230
|
-
],
|
|
231
|
-
endpointsModified: [],
|
|
232
|
-
authenticationChanged: false,
|
|
233
|
-
};
|
|
234
|
-
const rec = generateRecommendation("products_smoke_test.py", 75, // would normally trigger REGENERATE
|
|
235
|
-
undefined, issues, undefined, apiSchemaChanges);
|
|
236
|
-
// Rename detection should take priority over drift threshold
|
|
237
|
-
expect(rec.action).toBe("UPDATE");
|
|
238
|
-
expect(rec.estimatedWork).toBe("SMALL"); // renames are simple substitutions
|
|
239
|
-
});
|
|
240
|
-
});
|
|
241
|
-
});
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
export {};
|