@skyramp/mcp 0.2.8 → 0.2.10-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/index.js +1 -8
- package/build/prompts/enhance-assertions/uiAssertionsPrompt.js +28 -1
- package/build/prompts/initialize-workspace/initializeWorkspacePrompt.js +1 -0
- package/build/prompts/personas.d.ts +2 -2
- package/build/prompts/personas.js +2 -2
- package/build/prompts/sut-setup/modes/adaptWorkflowPrompt.js +3 -1
- package/build/prompts/sut-setup/modes/dockerComposePrompt.js +6 -1
- package/build/prompts/sut-setup/shared.d.ts +20 -0
- package/build/prompts/sut-setup/shared.js +67 -7
- package/build/prompts/test-maintenance/actionsInstructions.js +5 -1
- package/build/prompts/test-maintenance/drift-analysis-prompt.d.ts +15 -2
- package/build/prompts/test-maintenance/drift-analysis-prompt.js +63 -3
- package/build/prompts/test-maintenance/driftAnalysisSections.js +7 -25
- package/build/prompts/test-maintenance/driftAnalysisShared.d.ts +20 -0
- package/build/prompts/test-maintenance/driftAnalysisShared.js +80 -0
- package/build/prompts/test-maintenance/uiDriftAnalysisSections.d.ts +32 -0
- package/build/prompts/test-maintenance/uiDriftAnalysisSections.js +212 -0
- package/build/prompts/test-recommendation/recommendationShared.js +1 -1
- package/build/prompts/test-recommendation/scopeAssessment.js +12 -2
- package/build/prompts/testbot/testbot-prompts.js +10 -9
- package/build/resources/testbotResource.js +1 -1
- package/build/services/TestDiscoveryService.d.ts +12 -10
- package/build/services/TestDiscoveryService.js +137 -83
- package/build/services/TestDiscoveryService.test.js +370 -15
- package/build/services/TestExecutionService.d.ts +1 -1
- package/build/services/TestExecutionService.js +7 -7
- package/build/services/TestExecutionService.test.js +4 -1
- package/build/tools/executeSkyrampTestTool.js +40 -71
- package/build/tools/submitReportTool.d.ts +25 -25
- package/build/tools/submitReportTool.js +123 -34
- package/build/tools/submitReportTool.test.js +413 -94
- package/build/tools/test-management/actionsTool.js +177 -77
- package/build/tools/test-management/actionsTool.test.d.ts +1 -0
- package/build/tools/test-management/actionsTool.test.js +297 -0
- package/build/tools/test-management/analyzeChangesTool.d.ts +9 -0
- package/build/tools/test-management/analyzeChangesTool.js +74 -169
- package/build/tools/test-management/analyzeChangesTool.test.js +34 -3
- package/build/tools/test-management/analyzeTestHealthTool.js +63 -22
- package/build/tools/test-management/analyzeTestHealthTool.test.d.ts +1 -0
- package/build/tools/test-management/analyzeTestHealthTool.test.js +268 -0
- package/build/tools/workspace/initializeWorkspaceTool.js +37 -24
- package/build/tools/workspace/initializeWorkspaceTool.test.js +9 -4
- package/build/types/FrontendIntegration.d.ts +25 -0
- package/build/types/FrontendIntegration.js +19 -0
- package/build/types/RepositoryAnalysis.d.ts +8 -8
- package/build/types/TestAnalysis.d.ts +18 -49
- package/build/types/TestAnalysis.js +0 -28
- package/build/types/TestTypes.d.ts +3 -1
- package/build/types/TestTypes.js +3 -1
- package/build/types/TestbotReport.d.ts +1 -1
- package/build/utils/AnalysisStateManager.d.ts +16 -22
- package/build/utils/docker.test.js +1 -1
- package/build/utils/featureFlags.d.ts +1 -1
- package/build/utils/featureFlags.js +1 -1
- package/build/utils/frontendIntegration.d.ts +9 -0
- package/build/utils/frontendIntegration.js +237 -0
- package/build/utils/frontendIntegration.test.d.ts +1 -0
- package/build/utils/frontendIntegration.test.js +229 -0
- package/build/utils/pr-comment-parser.d.ts +3 -3
- package/build/utils/pr-comment-parser.js +6 -6
- package/build/utils/pr-comment-parser.test.js +3 -3
- package/build/utils/repoScanner.d.ts +7 -0
- package/build/utils/repoScanner.js +14 -6
- package/build/utils/repoScanner.test.js +9 -9
- package/build/utils/routeParsers.d.ts +32 -0
- package/build/utils/routeParsers.js +205 -2
- package/build/utils/routeParsers.test.js +279 -29
- package/build/utils/sourceRouteExtractor.js +174 -5
- package/build/utils/sourceRouteExtractor.test.js +173 -0
- package/build/utils/utils.d.ts +11 -0
- package/build/utils/utils.js +19 -0
- package/build/utils/utils.test.js +23 -1
- package/build/utils/versions.d.ts +3 -3
- package/build/utils/versions.js +1 -1
- package/build/workspace/workspace.d.ts +21 -21
- package/build/workspace/workspace.js +7 -4
- package/build/workspace/workspace.test.js +65 -6
- package/node_modules/playwright/lib/mcp/browser/tools/files.js +8 -3
- package/node_modules/playwright/lib/mcp/browser/tools/wait.js +1 -1
- package/node_modules/playwright/lib/mcp/skyramp/exportTool.js +10 -1
- package/node_modules/playwright/lib/mcp/skyramp/traceRecordingBackend.js +33 -9
- package/node_modules/playwright/lib/mcp/test/skyRampExport.js +7 -1
- package/node_modules/playwright/lib/mcp/test/skyRampExport.test.js +44 -0
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/{index.B_7ywgmr.js → index.d4gkVSou.js} +1 -1
- package/node_modules/playwright/node_modules/playwright-core/lib/vite/traceViewer/index.html +1 -1
- package/node_modules/playwright/node_modules/playwright-core/package.json +1 -1
- package/node_modules/playwright/package.json +1 -1
- package/package.json +3 -3
|
@@ -12,11 +12,12 @@ import { StateManager, registerSession, storeSessionData, setTestsRepoDir, getAc
|
|
|
12
12
|
import { buildRecommendationPrompt } from "../../prompts/test-recommendation/test-recommendation-prompt.js";
|
|
13
13
|
import { hasFlutterSdkDep, isFrontendFile, isTestFile } from "../../prompts/test-recommendation/scopeAssessment.js";
|
|
14
14
|
import { enumerateCandidateUiPages } from "../../utils/uiPageEnumerator.js";
|
|
15
|
+
import { checkFrontendFileIntegration } from "../../utils/frontendIntegration.js";
|
|
15
16
|
import { MAX_RECOMMENDATIONS, MAX_TESTS_TO_GENERATE } from "../../prompts/test-recommendation/recommendationSections.js";
|
|
16
17
|
import { TestDiscoveryService } from "../../services/TestDiscoveryService.js";
|
|
17
18
|
import { ScenarioSource, AnalysisScope } from "../../types/RepositoryAnalysis.js";
|
|
18
19
|
import { computeBranchDiff } from "../../utils/branchDiff.js";
|
|
19
|
-
import {
|
|
20
|
+
import { classifyEndpointsByChangedFiles, selectRemovalCandidateFiles, recoverRemovedEndpointsFromBase, } from "../../utils/routeParsers.js";
|
|
20
21
|
import { scanAllRepoEndpoints, scanRelatedEndpoints, grepRouterMountingContext, findCandidateRouteFiles, } from "../../utils/repoScanner.js";
|
|
21
22
|
import { detectProjectMetadata } from "../../utils/projectMetadata.js";
|
|
22
23
|
import { draftScenariosFromEndpoints } from "../../utils/scenarioDrafting.js";
|
|
@@ -32,24 +33,6 @@ const GENERIC_MODULE_NAMES = new Set([
|
|
|
32
33
|
"config", "settings", "exceptions", "errors", "base", "common",
|
|
33
34
|
"app", "main", "index", "server", "init", "deps", "dependencies",
|
|
34
35
|
]);
|
|
35
|
-
function resolveMountPrefixTokens(mountFileContents, changedFiles) {
|
|
36
|
-
const tokens = changedFiles.flatMap((file) => {
|
|
37
|
-
const mod = path.basename(file).replace(/\.[^.]+$/, "").toLowerCase();
|
|
38
|
-
if (GENERIC_MODULE_NAMES.has(mod))
|
|
39
|
-
return [];
|
|
40
|
-
const modRe = new RegExp(`\\b${mod.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")}\\b`);
|
|
41
|
-
return mountFileContents.flatMap((content) => {
|
|
42
|
-
if (!modRe.test(content.toLowerCase()))
|
|
43
|
-
return [];
|
|
44
|
-
const matches = [...content.matchAll(/prefix\s*=\s*["'](\/?[^"']+)/g)];
|
|
45
|
-
return matches.flatMap((m) => {
|
|
46
|
-
const token = extractResourceFromPath(m[1]);
|
|
47
|
-
return token !== "unknown" ? [token] : [];
|
|
48
|
-
});
|
|
49
|
-
});
|
|
50
|
-
});
|
|
51
|
-
return [...new Set(tokens)];
|
|
52
|
-
}
|
|
53
36
|
/** Exported for testing: maps a parsed trace result to a TraceFile. */
|
|
54
37
|
export function buildTraceFileEntry(tracePath, result) {
|
|
55
38
|
return {
|
|
@@ -285,54 +268,6 @@ function isNonApplicationFile(filePath) {
|
|
|
285
268
|
const ROUTE_FILE_PATTERN = /route|controller|endpoint|handler|view|urls|api|router|service|gateway|resolver|\bserver\b/i;
|
|
286
269
|
const ROUTE_FILE_BASENAME_PATTERN = /\bapp\b|\bmain\b/i;
|
|
287
270
|
const SOURCE_EXTS = /\.(ts|tsx|js|jsx|py|java|kt|go|rb|php|rs|cs|ex|exs)$/;
|
|
288
|
-
/**
|
|
289
|
-
* Recover endpoints from files deleted in this branch by reading their
|
|
290
|
-
* content from the base branch via `git show`. Only checks files whose
|
|
291
|
-
* name matches route-file heuristics to keep I/O bounded.
|
|
292
|
-
*/
|
|
293
|
-
async function recoverDeletedFileEndpoints(repositoryPath, baseBranch, deletedFiles) {
|
|
294
|
-
const candidates = deletedFiles.filter((f) => SOURCE_EXTS.test(f) && (ROUTE_FILE_PATTERN.test(f) || ROUTE_FILE_BASENAME_PATTERN.test(path.basename(f))));
|
|
295
|
-
if (candidates.length === 0)
|
|
296
|
-
return [];
|
|
297
|
-
const git = simpleGit(repositoryPath);
|
|
298
|
-
const results = [];
|
|
299
|
-
const endpointMap = new Map();
|
|
300
|
-
for (const file of candidates) {
|
|
301
|
-
try {
|
|
302
|
-
const content = await git.show([`${baseBranch}:${file}`]);
|
|
303
|
-
for (const ep of parseFileEndpoints(content, file)) {
|
|
304
|
-
const normalizedPath = ep.path.startsWith("/") ? ep.path : `/${ep.path}`;
|
|
305
|
-
const key = `${file}::${normalizedPath}`;
|
|
306
|
-
const existing = endpointMap.get(key);
|
|
307
|
-
if (existing) {
|
|
308
|
-
existing.methods.add(ep.method);
|
|
309
|
-
}
|
|
310
|
-
else {
|
|
311
|
-
endpointMap.set(key, {
|
|
312
|
-
path: normalizedPath,
|
|
313
|
-
methods: new Set([ep.method]),
|
|
314
|
-
sourceFile: file,
|
|
315
|
-
});
|
|
316
|
-
}
|
|
317
|
-
}
|
|
318
|
-
}
|
|
319
|
-
catch (err) {
|
|
320
|
-
logger.debug("Failed to recover endpoints from deleted file", {
|
|
321
|
-
file,
|
|
322
|
-
baseBranch,
|
|
323
|
-
error: err instanceof Error ? err.message : String(err),
|
|
324
|
-
});
|
|
325
|
-
}
|
|
326
|
-
}
|
|
327
|
-
for (const data of endpointMap.values()) {
|
|
328
|
-
results.push({
|
|
329
|
-
path: data.path,
|
|
330
|
-
methods: Array.from(data.methods),
|
|
331
|
-
sourceFile: data.sourceFile,
|
|
332
|
-
});
|
|
333
|
-
}
|
|
334
|
-
return results;
|
|
335
|
-
}
|
|
336
271
|
export const analyzeChangesInputSchema = {
|
|
337
272
|
repositoryPath: z
|
|
338
273
|
.string()
|
|
@@ -364,7 +299,7 @@ export const analyzeChangesInputSchema = {
|
|
|
364
299
|
prNumber: z
|
|
365
300
|
.number()
|
|
366
301
|
.optional()
|
|
367
|
-
.describe("GitHub PR number. When provided, fetches previous
|
|
302
|
+
.describe("GitHub PR number. When provided, fetches previous Testbot comments on this PR and skips re-recommending tests already suggested in earlier commits — reduces duplicate recommendations across multiple pushes to the same PR."),
|
|
368
303
|
repository: z
|
|
369
304
|
.string()
|
|
370
305
|
.optional()
|
|
@@ -403,6 +338,30 @@ If a candidate URL 404s or redirects unexpectedly, navigate from the workspace b
|
|
|
403
338
|
|
|
404
339
|
If \`browser_blueprint\` fails on every candidate URL (app unreachable, all 404s), proceed and log an \`issuesFound\` info entry. Recommendations will be source-grounded; non-UI work is unaffected.`;
|
|
405
340
|
}
|
|
341
|
+
/**
|
|
342
|
+
* Instruction block for changed frontend files the server determined have no
|
|
343
|
+
* production importer (SKYR-3855) — dead/orphaned components. Used both to
|
|
344
|
+
* replace the capture instructions entirely (every changed frontend file is
|
|
345
|
+
* unintegrated) and to append a scoped note when only some files are
|
|
346
|
+
* (`buildCaptureInstructions` still runs for the rest in that case).
|
|
347
|
+
*/
|
|
348
|
+
export function buildUnintegratedInstructions(results) {
|
|
349
|
+
const listing = results
|
|
350
|
+
.filter((r) => !r.integrated)
|
|
351
|
+
.map((r) => {
|
|
352
|
+
const token = path.basename(r.file).replace(/\.[^.]+$/, "");
|
|
353
|
+
return ` - \`${r.file}\` (checked for a production import of \`${token}\` — none found)`;
|
|
354
|
+
})
|
|
355
|
+
.join("\n");
|
|
356
|
+
return `**Server-side integration check found no production importer for the following changed component(s) — do not treat this as unproven:**
|
|
357
|
+
${listing}
|
|
358
|
+
|
|
359
|
+
These components have no DOM presence in the running app: no route, page, or other production file imports or renders them. Handle them as follows:
|
|
360
|
+
- **Never record traces or generate UI tests against a substitute surface** (a different page or component that happens to be visible). A test on an unrelated surface validates nothing about this PR.
|
|
361
|
+
- **DO write the change-targeting test directly from code analysis** — you know the component's structure, selectors, and expected behavior from the diff. Target the component's own elements/attributes. The test is EXPECTED TO FAIL because the component never mounts; that failing test documents the integration gap and is more valuable than prose. Mark its report entry accordingly (e.g. "expected failure: component has no production importer").
|
|
362
|
+
- Log an \`issuesFound\` entry (high severity) naming the unintegrated file(s) above and stating that the changed code is unreachable in the running app.
|
|
363
|
+
- Backend/other work in this PR is unaffected — proceed with it normally.`;
|
|
364
|
+
}
|
|
406
365
|
export function registerAnalyzeChangesTool(server) {
|
|
407
366
|
server.registerTool(TOOL_NAME, {
|
|
408
367
|
annotations: {
|
|
@@ -526,11 +485,16 @@ export function registerAnalyzeChangesTool(server) {
|
|
|
526
485
|
// HTTP methods, eliminating the need for path resolution and MULTI sentinels.
|
|
527
486
|
let classifiedEndpoints;
|
|
528
487
|
if (diffData) {
|
|
529
|
-
// Recover endpoints
|
|
530
|
-
|
|
531
|
-
|
|
488
|
+
// Recover endpoints that existed in the base branch from every candidate
|
|
489
|
+
// file (deleted files AND modified files that may have dropped a route via
|
|
490
|
+
// line-edit). classifyEndpointsByChangedFiles filters out any that still
|
|
491
|
+
// exist in the current catalog, leaving only genuine removals.
|
|
492
|
+
const removalCandidateFiles = selectRemovalCandidateFiles(diffData);
|
|
493
|
+
const git = simpleGit(params.repositoryPath);
|
|
494
|
+
const recoveredBaseEndpoints = removalCandidateFiles.length > 0
|
|
495
|
+
? await recoverRemovedEndpointsFromBase(removalCandidateFiles, (file) => git.show([`${diffData.baseBranch}:${file}`]))
|
|
532
496
|
: [];
|
|
533
|
-
classifiedEndpoints = classifyEndpointsByChangedFiles(diffData, scannedEndpoints,
|
|
497
|
+
classifiedEndpoints = classifyEndpointsByChangedFiles(diffData, scannedEndpoints, recoveredBaseEndpoints);
|
|
534
498
|
classifiedEndpoints = {
|
|
535
499
|
...classifiedEndpoints,
|
|
536
500
|
// changed/new endpoints come from scannedEndpoints, which is already
|
|
@@ -617,91 +581,26 @@ export function registerAnalyzeChangesTool(server) {
|
|
|
617
581
|
return [];
|
|
618
582
|
return [{ file: f, content: content.trimEnd() }];
|
|
619
583
|
});
|
|
620
|
-
//
|
|
621
|
-
//
|
|
622
|
-
|
|
623
|
-
//
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
const allClassified = [
|
|
630
|
-
...classifiedEndpoints.changedEndpoints,
|
|
631
|
-
...classifiedEndpoints.newEndpoints,
|
|
632
|
-
...classifiedEndpoints.removedEndpoints,
|
|
633
|
-
];
|
|
634
|
-
const FRAMEWORK_SUFFIX_RE = /\.(service|controller|transformer|handler|middleware|resolver|repository|module|guard|interceptor|pipe|filter|decorator|input|output|dto|schema)$/i;
|
|
635
|
-
if (allClassified.length > 0) {
|
|
636
|
-
// Scanned endpoints always have full paths — extractResourceFromPath
|
|
637
|
-
// never returns "unknown" for properly resolved paths.
|
|
638
|
-
// Exception: NestJS versioned controllers register relative paths (e.g. "GET /")
|
|
639
|
-
// which resolve to "unknown". Fall through to file-path extraction in that case.
|
|
640
|
-
const resolved = allClassified
|
|
641
|
-
.map((ep) => extractResourceFromPath(ep.path))
|
|
642
|
-
.filter((r, i, arr) => r !== "unknown" && arr.indexOf(r) === i);
|
|
643
|
-
if (resolved.length > 0) {
|
|
644
|
-
changedResources = resolved;
|
|
645
|
-
}
|
|
646
|
-
else {
|
|
647
|
-
// All endpoints resolved to "unknown" (e.g. NestJS relative paths) —
|
|
648
|
-
// first try to extract resource names from router mount prefixes.
|
|
649
|
-
const changedSourceFiles = classifiedEndpoints.changedFiles ?? [];
|
|
650
|
-
const mountedResources = resolveMountPrefixTokens(routerMountContextContents, changedSourceFiles);
|
|
651
|
-
if (mountedResources.length > 0) {
|
|
652
|
-
changedResources = mountedResources;
|
|
653
|
-
}
|
|
654
|
-
else {
|
|
655
|
-
// Fall back to extracting resource names from changed file paths.
|
|
656
|
-
const candidateFiles = classifiedEndpoints.unmatchedFiles.length > 0
|
|
657
|
-
? classifiedEndpoints.unmatchedFiles
|
|
658
|
-
: classifiedEndpoints.changedFiles ?? [];
|
|
659
|
-
const fromFiles = candidateFiles
|
|
660
|
-
.map((f) => extractResourceFromPath(f.replace(/\.[^./]+$/, "").replace(FRAMEWORK_SUFFIX_RE, "")))
|
|
661
|
-
.filter((r, i, arr) => r !== "unknown" && arr.indexOf(r) === i);
|
|
662
|
-
changedResources = fromFiles.length > 0 ? fromFiles : ["unknown"];
|
|
663
|
-
}
|
|
664
|
-
}
|
|
665
|
-
}
|
|
666
|
-
else if (classifiedEndpoints.unmatchedFiles.length > 0) {
|
|
667
|
-
// Changed files don't map to any endpoint (e.g. DTO, service, output
|
|
668
|
-
// formatter, migration). Extract resource names from the file paths so
|
|
669
|
-
// external tests whose names overlap with these resources are surfaced
|
|
670
|
-
// for drift assessment.
|
|
671
|
-
// Strip the file extension and common framework suffixes (.service,
|
|
672
|
-
// .controller, .input, .output, etc.) before extracting so that
|
|
673
|
-
// "event-types.service.ts" yields "event-types" rather than
|
|
674
|
-
// "event-types.service", which would fail relevance scoring against
|
|
675
|
-
// test files that contain "event" and "types" but not "service".
|
|
676
|
-
const fromFiles = classifiedEndpoints.unmatchedFiles
|
|
677
|
-
.map((f) => extractResourceFromPath(f.replace(/\.[^./]+$/, "").replace(FRAMEWORK_SUFFIX_RE, "")))
|
|
678
|
-
.filter((r, i, arr) => r !== "unknown" && arr.indexOf(r) === i);
|
|
679
|
-
changedResources = fromFiles.length > 0 ? fromFiles : ["unknown"];
|
|
680
|
-
}
|
|
681
|
-
else {
|
|
682
|
-
// No endpoints and no unmatched files — the diff touches only non-route
|
|
683
|
-
// files (e.g. schema, model, DTO). Extract resource tokens from the
|
|
684
|
-
// changed file paths so tests that import the changed symbol are still
|
|
685
|
-
// surfaced for drift assessment. Fall back to [] only if extraction
|
|
686
|
-
// yields nothing (e.g. a file with a non-resource name like utils.py).
|
|
687
|
-
const fromChangedFiles = (classifiedEndpoints.changedFiles ?? [])
|
|
688
|
-
.map((f) => extractResourceFromPath(f.replace(/\.[^./]+$/, "").replace(FRAMEWORK_SUFFIX_RE, "")))
|
|
689
|
-
.filter((r, i, arr) => r !== "unknown" && arr.indexOf(r) === i);
|
|
690
|
-
changedResources = fromChangedFiles.length > 0 ? fromChangedFiles : [];
|
|
691
|
-
}
|
|
692
|
-
}
|
|
584
|
+
// PR mode → [] excludes external tests from pre-loading (agent discovers via grep).
|
|
585
|
+
// No diff context → undefined triggers full-repo mode in TestDiscoveryService.
|
|
586
|
+
const changedResources = classifiedEndpoints ? [] : undefined;
|
|
587
|
+
// Compute changed frontend files once — reused for both discoverTests (component
|
|
588
|
+
// test promotion) and uiContext below (page enumeration). Uses classifiedEndpoints
|
|
589
|
+
// changedFiles so it reflects the same diff scope as the rest of the analysis.
|
|
590
|
+
const flutterSdkDep = hasFlutterSdkDep(params.repositoryPath);
|
|
591
|
+
const changedFrontendFiles = (classifiedEndpoints?.changedFiles ?? diffData?.changedFiles ?? [])
|
|
592
|
+
.filter(f => isFrontendFile(f, { hasFlutterSdkDep: flutterSdkDep }) && !isTestFile(f));
|
|
693
593
|
let existingTests = [];
|
|
694
594
|
let discoveredRelevantExternalPaths = [];
|
|
695
595
|
try {
|
|
696
596
|
const testDiscoveryService = new TestDiscoveryService();
|
|
697
597
|
setTestsRepoDir(params.testsRepoDir);
|
|
698
|
-
const discoveryResult = await testDiscoveryService.discoverTests(testDir ?? params.repositoryPath, { changedResources });
|
|
598
|
+
const discoveryResult = await testDiscoveryService.discoverTests(testDir ?? params.repositoryPath, { changedResources, hasFrontendChanges: changedFrontendFiles.length > 0 });
|
|
699
599
|
existingTests = discoveryResult.tests.map((test) => ({
|
|
700
600
|
testFile: test.testFile,
|
|
701
601
|
testType: test.testType,
|
|
702
602
|
language: test.language,
|
|
703
603
|
framework: test.framework,
|
|
704
|
-
apiSchema: test.apiSchema,
|
|
705
604
|
apiEndpoint: test.apiEndpoint,
|
|
706
605
|
source: test.source,
|
|
707
606
|
generatedAt: test.generatedAt,
|
|
@@ -1032,7 +931,7 @@ export function registerAnalyzeChangesTool(server) {
|
|
|
1032
931
|
const entry = t.apiEndpoint
|
|
1033
932
|
? `${dp} [external] (covers: ${t.apiEndpoint})`
|
|
1034
933
|
: `${dp} [external]`;
|
|
1035
|
-
addEntry(t.testType
|
|
934
|
+
addEntry(t.testType, entry);
|
|
1036
935
|
}
|
|
1037
936
|
// Pass 2: Skyramp tests — tagged [skyramp] so the LLM can positively identify the source
|
|
1038
937
|
for (const t of existingTests) {
|
|
@@ -1042,7 +941,7 @@ export function registerAnalyzeChangesTool(server) {
|
|
|
1042
941
|
const entry = t.apiEndpoint
|
|
1043
942
|
? `${dp} [skyramp] (covers: ${t.apiEndpoint})`
|
|
1044
943
|
: `${dp} [skyramp]`;
|
|
1045
|
-
addEntry(t.testType
|
|
944
|
+
addEntry(t.testType, entry);
|
|
1046
945
|
}
|
|
1047
946
|
// Pass 3: low-relevance external tests (name-only) — capped to avoid context overflow
|
|
1048
947
|
const LOW_RELEVANCE_CAP = 20;
|
|
@@ -1054,7 +953,7 @@ export function registerAnalyzeChangesTool(server) {
|
|
|
1054
953
|
if (relevantExternalSet.has(t.testFile))
|
|
1055
954
|
continue;
|
|
1056
955
|
if (lowRelevanceCount < LOW_RELEVANCE_CAP) {
|
|
1057
|
-
addEntry(t.testType
|
|
956
|
+
addEntry(t.testType, `${displayPath(t.testFile)} [external]`);
|
|
1058
957
|
lowRelevanceCount++;
|
|
1059
958
|
}
|
|
1060
959
|
else {
|
|
@@ -1218,25 +1117,18 @@ export function registerAnalyzeChangesTool(server) {
|
|
|
1218
1117
|
// browser_blueprints — see uiInstructions below, which this tool returns
|
|
1219
1118
|
// so the agent captures element vocabulary for UI rec reasoning.
|
|
1220
1119
|
const uiContext = await (async () => {
|
|
1221
|
-
|
|
1222
|
-
if (
|
|
1120
|
+
// changedFrontendFiles computed above (before discoverTests) — reuse here.
|
|
1121
|
+
if (changedFrontendFiles.length === 0)
|
|
1223
1122
|
return undefined;
|
|
1224
|
-
|
|
1225
|
-
//
|
|
1226
|
-
//
|
|
1227
|
-
//
|
|
1228
|
-
|
|
1229
|
-
// in a Flutter project are recognised as frontend. See Confluence
|
|
1230
|
-
// "Flutter support in Testbot" — this is the budget-driving call site
|
|
1231
|
-
// that must thread the flag so a Flutter PR enters the UI pipeline.
|
|
1232
|
-
const flutterSdk = hasFlutterSdkDep(params.repositoryPath);
|
|
1233
|
-
const frontendFiles = changedFiles.filter((f) => isFrontendFile(f, { hasFlutterSdkDep: flutterSdk }) && !isTestFile(f));
|
|
1234
|
-
if (frontendFiles.length === 0)
|
|
1235
|
-
return undefined;
|
|
1236
|
-
const candidateUiPages = await enumerateCandidateUiPages(params.repositoryPath, frontendFiles);
|
|
1123
|
+
const candidateUiPages = await enumerateCandidateUiPages(params.repositoryPath, changedFrontendFiles);
|
|
1124
|
+
// SKYR-3855: deterministic production-importer check, computed here so
|
|
1125
|
+
// downstream consumers (testbot prompt) can skip UI generation for
|
|
1126
|
+
// unintegrated components on a server fact instead of a mid-run grep.
|
|
1127
|
+
const frontendFileIntegration = checkFrontendFileIntegration(params.repositoryPath, changedFrontendFiles);
|
|
1237
1128
|
return {
|
|
1238
|
-
changedFrontendFiles
|
|
1129
|
+
changedFrontendFiles,
|
|
1239
1130
|
candidateUiPages,
|
|
1131
|
+
frontendFileIntegration,
|
|
1240
1132
|
};
|
|
1241
1133
|
})();
|
|
1242
1134
|
// Derive the UI blueprint-capture instructions the agent acts on after
|
|
@@ -1244,11 +1136,24 @@ export function registerAnalyzeChangesTool(server) {
|
|
|
1244
1136
|
// pre-flight tool, now folded in here so the agent makes one analysis call
|
|
1245
1137
|
// instead of two. The captures stay in the agent's tool-result history and
|
|
1246
1138
|
// serve as element vocabulary when it writes UI recommendation reasoning.
|
|
1139
|
+
//
|
|
1140
|
+
// SKYR-3855: when the integration check finds no production importer for
|
|
1141
|
+
// some/all changed frontend files, that overrides/augments the normal
|
|
1142
|
+
// capture flow — see buildUnintegratedInstructions.
|
|
1143
|
+
const unintegratedFrontendFiles = (uiContext?.frontendFileIntegration ?? []).filter((r) => !r.integrated);
|
|
1144
|
+
const allFrontendFilesUnintegrated = !!uiContext &&
|
|
1145
|
+
uiContext.changedFrontendFiles.length > 0 &&
|
|
1146
|
+
unintegratedFrontendFiles.length === uiContext.changedFrontendFiles.length;
|
|
1247
1147
|
const uiInstructions = !uiContext || uiContext.changedFrontendFiles.length === 0
|
|
1248
1148
|
? NO_UI_INSTRUCTIONS
|
|
1249
|
-
:
|
|
1250
|
-
?
|
|
1251
|
-
:
|
|
1149
|
+
: allFrontendFilesUnintegrated
|
|
1150
|
+
? buildUnintegratedInstructions(unintegratedFrontendFiles)
|
|
1151
|
+
: (uiContext.candidateUiPages.length === 0
|
|
1152
|
+
? NO_RESOLVABLE_URLS_INSTRUCTIONS
|
|
1153
|
+
: buildCaptureInstructions(uiContext.candidateUiPages)) +
|
|
1154
|
+
(unintegratedFrontendFiles.length > 0
|
|
1155
|
+
? `\n\n${buildUnintegratedInstructions(unintegratedFrontendFiles)}`
|
|
1156
|
+
: "");
|
|
1252
1157
|
const unifiedState = {
|
|
1253
1158
|
existingTests,
|
|
1254
1159
|
newEndpoints: newEndpointsForDrafting,
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
// @ts-nocheck - Jest ESM type inference issues
|
|
2
2
|
import { jest } from "@jest/globals";
|
|
3
|
-
import { z } from "zod";
|
|
4
3
|
// Mock all heavy dependencies so the module can be loaded in isolation.
|
|
5
4
|
// Under native ESM, jest.mock is not hoisted — every mock the subject's import
|
|
6
5
|
// graph touches must use unstable_mockModule before the dynamic import below.
|
|
@@ -27,7 +26,10 @@ jest.unstable_mockModule("../../utils/branchDiff.js", () => ({
|
|
|
27
26
|
jest.unstable_mockModule("../../utils/routeParsers.js", () => ({
|
|
28
27
|
classifyEndpointsByChangedFiles: jest.fn(),
|
|
29
28
|
extractResourceFromPath: jest.fn(),
|
|
29
|
+
extractTopLevelResourceFromPath: jest.fn(),
|
|
30
30
|
parseFileEndpoints: jest.fn(),
|
|
31
|
+
selectRemovalCandidateFiles: jest.fn(() => []),
|
|
32
|
+
recoverRemovedEndpointsFromBase: jest.fn(async () => []),
|
|
31
33
|
}));
|
|
32
34
|
jest.unstable_mockModule("../../utils/repoScanner.js", () => ({
|
|
33
35
|
scanAllRepoEndpoints: jest.fn(),
|
|
@@ -87,7 +89,7 @@ jest.unstable_mockModule("@modelcontextprotocol/sdk/shared/protocol.js", () => (
|
|
|
87
89
|
// All subject + mocked-module imports must be dynamic and live AFTER the
|
|
88
90
|
// unstable_mockModule calls above — static imports hoist above the mocks under
|
|
89
91
|
// native ESM, loading the real (heavy) modules unmocked.
|
|
90
|
-
const { analyzeChangesInputSchema, isSecurityRelevantDiff, filterEndpointsBySpec, filterUnsupportedGraphQLEndpoints, isGraphQLFile, registerAnalyzeChangesTool, buildTraceFileEntry, buildCaptureInstructions, NO_UI_INSTRUCTIONS, NO_RESOLVABLE_URLS_INSTRUCTIONS, } = await import("./analyzeChangesTool.js");
|
|
92
|
+
const { analyzeChangesInputSchema, isSecurityRelevantDiff, filterEndpointsBySpec, filterUnsupportedGraphQLEndpoints, isGraphQLFile, registerAnalyzeChangesTool, buildTraceFileEntry, buildCaptureInstructions, buildUnintegratedInstructions, NO_UI_INSTRUCTIONS, NO_RESOLVABLE_URLS_INSTRUCTIONS, } = await import("./analyzeChangesTool.js");
|
|
91
93
|
const { computeBranchDiff } = await import("../../utils/branchDiff.js");
|
|
92
94
|
const { classifyEndpointsByChangedFiles } = await import("../../utils/routeParsers.js");
|
|
93
95
|
const { scanRelatedEndpoints, scanAllRepoEndpoints } = await import("../../utils/repoScanner.js");
|
|
@@ -96,7 +98,6 @@ const os = await import("os");
|
|
|
96
98
|
const path = await import("path");
|
|
97
99
|
const fsSync = await import("fs");
|
|
98
100
|
const fsModule = fsSync;
|
|
99
|
-
const schema = z.object(analyzeChangesInputSchema);
|
|
100
101
|
describe("isSecurityRelevantDiff", () => {
|
|
101
102
|
it("matches auth-specific middleware and admin-key signals", () => {
|
|
102
103
|
expect(isSecurityRelevantDiff("const authMiddleware = requireAuth();")).toBe(true);
|
|
@@ -454,3 +455,33 @@ describe("UI blueprint-capture instructions (merged from ui_analyze_changes)", (
|
|
|
454
455
|
expect(text).not.toMatch(/ui-credentials/i);
|
|
455
456
|
});
|
|
456
457
|
});
|
|
458
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
459
|
+
// buildUnintegratedInstructions — SKYR-3855 server-side integration check.
|
|
460
|
+
// Per .claude/CLAUDE.md, only assert structural branching (which files are
|
|
461
|
+
// listed), not prose wording.
|
|
462
|
+
// ─────────────────────────────────────────────────────────────────────────────
|
|
463
|
+
describe("buildUnintegratedInstructions", () => {
|
|
464
|
+
it("lists a single unintegrated file when only one is passed (all-unintegrated case)", () => {
|
|
465
|
+
const text = buildUnintegratedInstructions([
|
|
466
|
+
{ file: "src/components/EntityExplorerSidebar.tsx", integrated: false, importers: [], reason: "no-importers" },
|
|
467
|
+
]);
|
|
468
|
+
expect(text).toContain("src/components/EntityExplorerSidebar.tsx");
|
|
469
|
+
expect(text).not.toContain("src/components/OtherPanel.tsx");
|
|
470
|
+
});
|
|
471
|
+
it("lists every file passed in — mixed-PR call sites scope this to just the unintegrated subset", () => {
|
|
472
|
+
const text = buildUnintegratedInstructions([
|
|
473
|
+
{ file: "src/components/A.tsx", integrated: false, importers: [], reason: "no-importers" },
|
|
474
|
+
{ file: "src/components/B.tsx", integrated: false, importers: [], reason: "no-importers" },
|
|
475
|
+
]);
|
|
476
|
+
expect(text).toContain("src/components/A.tsx");
|
|
477
|
+
expect(text).toContain("src/components/B.tsx");
|
|
478
|
+
});
|
|
479
|
+
it("filters out integrated files even if accidentally included by a caller", () => {
|
|
480
|
+
const text = buildUnintegratedInstructions([
|
|
481
|
+
{ file: "src/components/Unintegrated.tsx", integrated: false, importers: [], reason: "no-importers" },
|
|
482
|
+
{ file: "src/pages/Dashboard.tsx", integrated: true, importers: ["src/App.tsx"], reason: "imported" },
|
|
483
|
+
]);
|
|
484
|
+
expect(text).toContain("src/components/Unintegrated.tsx");
|
|
485
|
+
expect(text).not.toContain("src/pages/Dashboard.tsx");
|
|
486
|
+
});
|
|
487
|
+
});
|
|
@@ -1,25 +1,48 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
import { logger } from "../../utils/logger.js";
|
|
3
3
|
import { StateManager, } from "../../utils/AnalysisStateManager.js";
|
|
4
|
-
import path from "path";
|
|
5
4
|
import { AnalyticsService } from "../../services/AnalyticsService.js";
|
|
6
5
|
import { buildDriftAnalysisPrompt } from "../../prompts/test-maintenance/drift-analysis-prompt.js";
|
|
7
6
|
import { TestSource } from "../../types/TestAnalysis.js";
|
|
8
7
|
import { toolError } from "../../utils/utils.js";
|
|
9
8
|
const TOOL_NAME = "skyramp_analyze_test_health";
|
|
9
|
+
// UI test frameworks. "playwright-ui" and "cypress-ui" are the browser-interaction
|
|
10
|
+
// variants detected when the file uses the page fixture (not just the request fixture).
|
|
11
|
+
// Plain "playwright"/"cypress" without the -ui suffix = REST API usage — excluded.
|
|
12
|
+
const UI_COMPONENT_FRAMEWORKS = new Set([
|
|
13
|
+
"rtl", "react-testing-library",
|
|
14
|
+
"vue-test-utils", "angular-testbed", "svelte-testing-library",
|
|
15
|
+
"enzyme",
|
|
16
|
+
"flutter-widget", "flutter-test",
|
|
17
|
+
"xcuitest", "espresso", "appium",
|
|
18
|
+
"selenium",
|
|
19
|
+
"playwright-ui", "cypress-ui",
|
|
20
|
+
]);
|
|
21
|
+
// No REST API test is ever written in TSX/JSX — extension alone is sufficient.
|
|
22
|
+
const UI_TEST_EXTENSIONS = /\.(test|spec)\.(tsx|jsx)$/;
|
|
23
|
+
const UI_SNAPSHOT_EXT = /\.snap$/;
|
|
24
|
+
function isUiTest(t) {
|
|
25
|
+
const fw = t.framework?.toLowerCase() ?? "";
|
|
26
|
+
const isUiSnap = UI_SNAPSHOT_EXT.test(t.testFile);
|
|
27
|
+
return UI_COMPONENT_FRAMEWORKS.has(fw) || UI_TEST_EXTENSIONS.test(t.testFile) || isUiSnap;
|
|
28
|
+
}
|
|
10
29
|
export function registerAnalyzeTestHealthTool(server) {
|
|
11
30
|
server.registerTool(TOOL_NAME, {
|
|
12
31
|
annotations: {
|
|
13
|
-
readOnlyHint: false,
|
|
14
|
-
destructiveHint:
|
|
32
|
+
readOnlyHint: false,
|
|
33
|
+
destructiveHint: true, // deletes stale diff files via StateManager.cleanupOldFiles
|
|
15
34
|
idempotentHint: true,
|
|
16
35
|
openWorldHint: false,
|
|
17
36
|
},
|
|
18
|
-
description: `
|
|
37
|
+
description: `Assess existing tests for drift against the current branch diff and generate maintenance instructions (UPDATE, REGENERATE, DELETE, VERIFY, or IGNORE) for each test. Covers both API tests and UI tests (Playwright, Cypress, RTL, component tests). Requires a stateFile from skyramp_analyze_changes.`,
|
|
19
38
|
inputSchema: {
|
|
20
39
|
stateFile: z
|
|
21
40
|
.string()
|
|
22
41
|
.describe("Path to the analyze-changes-state.json file returned by skyramp_analyze_changes (the stateFile field in its output). Do NOT pass summaryOutputFile or testbot-result.txt here."),
|
|
42
|
+
blueprintCaptured: z
|
|
43
|
+
.boolean()
|
|
44
|
+
.optional()
|
|
45
|
+
.describe("If hasFrontendChanges is true in the skyramp_analyze_changes output: call browser_navigate to the primary route affected by the diff, then call browser_blueprint to capture the current DOM, then call this tool with blueprintCaptured: true. If hasFrontendChanges is false: call this tool directly and omit this parameter. Passing true activates blueprint-diff analysis for browser UI tests (Playwright, Cypress, Selenium). Component/widget tests always use source-only analysis regardless of this flag."),
|
|
23
46
|
repository: z
|
|
24
47
|
.string()
|
|
25
48
|
.optional()
|
|
@@ -46,32 +69,50 @@ export function registerAnalyzeTestHealthTool(server) {
|
|
|
46
69
|
if (!Array.isArray(stateData.existingTests)) {
|
|
47
70
|
return toolError(`stateFile does not contain test discovery data (existingTests missing). The stateFile parameter must be the analyze-changes-state.json path returned by skyramp_analyze_changes — not summaryOutputFile or testbot-result.txt.`);
|
|
48
71
|
}
|
|
49
|
-
//
|
|
50
|
-
//
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
// relevantExternalTestPaths are stored relative to repositoryPath in the state file.
|
|
54
|
-
// Re-absolutize here so has() comparisons against t.testFile (absolute) work correctly.
|
|
55
|
-
const relevantExternalPaths = new Set((stateData.repositoryAnalysis?.relevantExternalTestPaths ?? []).map((p) => path.isAbsolute(p) ? p : path.resolve(repositoryPath, p)));
|
|
56
|
-
const existingTests = (stateData.existingTests || []).filter((t) => {
|
|
57
|
-
if ((t.source ?? TestSource.Skyramp) !== TestSource.External)
|
|
58
|
-
return true;
|
|
59
|
-
return relevantExternalPaths.has(t.testFile);
|
|
60
|
-
});
|
|
61
|
-
const skyrampCount = existingTests.filter((t) => (t.source ?? TestSource.Skyramp) !== TestSource.External).length;
|
|
72
|
+
// External tests: UPDATE only — REGENERATE/DELETE are report-only (enforced in
|
|
73
|
+
// skyramp_actions). Already scoped to relevant ones by discovery.
|
|
74
|
+
const existingTests = stateData.existingTests || [];
|
|
75
|
+
const skyrampCount = existingTests.filter((t) => t.source !== TestSource.External).length;
|
|
62
76
|
const externalCount = existingTests.length - skyrampCount;
|
|
63
77
|
logger.info(`Loaded ${skyrampCount} Skyramp + ${externalCount} relevant external tests from state file`);
|
|
64
|
-
//
|
|
65
|
-
// Pass [] for stateTypes so only skyramp-diff-*.diff files are deleted — state files
|
|
66
|
-
// (skyramp-analysis-*, skyramp-recommendation-*) must not be removed here because the
|
|
67
|
-
// caller still needs args.stateFile to pass to skyramp_actions.
|
|
78
|
+
// Delete stale diff files only — state files must remain for the caller's skyramp_actions call.
|
|
68
79
|
try {
|
|
69
80
|
await StateManager.cleanupOldFiles(24, undefined, []);
|
|
70
81
|
}
|
|
71
82
|
catch (error) {
|
|
72
83
|
logger.warning(`Failed to cleanup old diff files: ${error.message}`);
|
|
73
84
|
}
|
|
74
|
-
const
|
|
85
|
+
const uiContext = stateData.uiContext;
|
|
86
|
+
const hasUiFrontend = !!(uiContext?.changedFrontendFiles?.length);
|
|
87
|
+
const uiTests = [];
|
|
88
|
+
const apiTests = [];
|
|
89
|
+
for (const t of existingTests) {
|
|
90
|
+
// Only route to UI drift when frontend files changed — otherwise UI tests
|
|
91
|
+
// (playwright-ui, RTL, etc.) fall through to API drift so they are not
|
|
92
|
+
// silently dropped on API-only PRs.
|
|
93
|
+
(hasUiFrontend && isUiTest(t) ? uiTests : apiTests).push(t);
|
|
94
|
+
}
|
|
95
|
+
let uiDriftParams;
|
|
96
|
+
if (hasUiFrontend) {
|
|
97
|
+
logger.info(`Including UI drift analysis for ${uiTests.length} UI test(s), ${apiTests.length} API test(s)`);
|
|
98
|
+
uiDriftParams = {
|
|
99
|
+
tests: uiTests.map(t => ({ testFile: t.testFile, source: t.source })),
|
|
100
|
+
changedFrontendFiles: uiContext.changedFrontendFiles,
|
|
101
|
+
blueprintCaptured: args.blueprintCaptured,
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
// Collect all checked-out repo paths so the grep instruction covers
|
|
105
|
+
// every repo in a multi-repo run (endpoints in Repo A, tests in Repo B).
|
|
106
|
+
// TODO(multi-repo): existingTests is scoped to the current repo only — related
|
|
107
|
+
// repo tests are not pre-loaded, discoverable only via the grep instruction.
|
|
108
|
+
// Skip the async lookups when there are no tests — symbol discovery is a no-op.
|
|
109
|
+
let allRepoPaths = [repositoryPath];
|
|
110
|
+
if (existingTests.length > 0) {
|
|
111
|
+
const relatedRepoKeys = await stateManager.listRelatedRepos();
|
|
112
|
+
const relatedRepoPaths = (await Promise.all(relatedRepoKeys.map(r => stateManager.getRepoRepositoryPath(r)))).filter((p) => typeof p === "string");
|
|
113
|
+
allRepoPaths = [repositoryPath, ...relatedRepoPaths];
|
|
114
|
+
}
|
|
115
|
+
const promptText = buildDriftAnalysisPrompt(stateManager.getStatePath(), apiTests.map((t) => ({ testFile: t.testFile, source: t.source })), uiDriftParams, allRepoPaths);
|
|
75
116
|
return {
|
|
76
117
|
structuredContent: { prompt: promptText },
|
|
77
118
|
content: [{ type: "text", text: "Drift analysis prompt generated. Follow the prompt field to assess each test." }],
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|