@skyramp/mcp 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/prompts/test-recommendation/analysisOutputPrompt.js +24 -19
- package/build/prompts/test-recommendation/recommendationSections.js +1 -1
- package/build/prompts/test-recommendation/test-recommendation-prompt.js +48 -6
- package/build/prompts/test-recommendation/test-recommendation-prompt.test.js +139 -0
- package/build/services/TestDiscoveryService.js +22 -7
- package/build/services/TestDiscoveryService.test.js +44 -0
- package/build/tools/test-management/analyzeChangesTool.js +259 -140
- package/build/tools/test-management/analyzeChangesTool.test.js +3 -1
- package/build/tools/test-management/analyzeTestHealthTool.js +5 -0
- package/build/types/RepositoryAnalysis.js +8 -0
- package/build/utils/branchDiff.js +24 -8
- package/build/utils/repoScanner.js +16 -2
- package/build/utils/routeParsers.js +79 -79
- package/build/utils/routeParsers.test.js +192 -66
- package/build/utils/scenarioDrafting.js +10 -2
- package/package.json +1 -1
|
@@ -79,19 +79,23 @@ For GET list endpoints: identify query params (\`limit\`, \`offset\`, \`order\`,
|
|
|
79
79
|
${nextStep}`;
|
|
80
80
|
}
|
|
81
81
|
const changedFiles = p.parsedDiff?.changedFiles.join(", ") ?? "";
|
|
82
|
-
// Whether the
|
|
83
|
-
|
|
84
|
-
// frameworks (e.g. Spring class-level @RequestMapping, Django, Rails) are
|
|
85
|
-
// covered even when the static regex returns nothing.
|
|
86
|
-
const regexFoundEndpoints = p.parsedDiff && (p.parsedDiff.newEndpoints.length > 0 || p.parsedDiff.modifiedEndpoints.length > 0);
|
|
82
|
+
// Whether the scanner found API endpoints in any changed file.
|
|
83
|
+
const preDetectedEndpoints = p.parsedDiff && (p.parsedDiff.newEndpoints.length > 0 || p.parsedDiff.modifiedEndpoints.length > 0 || (p.parsedDiff.removedEndpoints?.length ?? 0) > 0);
|
|
87
84
|
const diffFiles = p.parsedDiff?.changedFiles ?? [];
|
|
88
85
|
const isUIOnly = diffFiles.length > 0 &&
|
|
89
|
-
!
|
|
86
|
+
!preDetectedEndpoints &&
|
|
90
87
|
diffFiles.every(f => FRONTEND_EXT.test(f));
|
|
91
88
|
const diffHasJavaFiles = diffFiles.some(f => /\.(java|kt)$/.test(f));
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
89
|
+
// Inline small diffs so the LLM sees them without a tool call. Large diffs
|
|
90
|
+
// stay as a temp file reference to avoid bloating the prompt.
|
|
91
|
+
const INLINE_DIFF_LIMIT = 12_000; // chars — roughly 300 lines
|
|
92
|
+
const canInline = p.diffContent && p.diffContent.length <= INLINE_DIFF_LIMIT;
|
|
93
|
+
const diffFileRef = canInline
|
|
94
|
+
? `\n<diff>\n${p.diffContent}\n</diff>\n`
|
|
95
|
+
+ (p.diffFilePath ? `Full diff also available at \`${p.diffFilePath}\`.\n` : "")
|
|
96
|
+
: p.diffFilePath
|
|
97
|
+
? `\n**Full diff file**: \`${p.diffFilePath}\` — **you MUST read this file before proceeding to Step 2.** It contains the complete unified diff for this PR.\n`
|
|
98
|
+
: "";
|
|
95
99
|
const step2 = isUIOnly
|
|
96
100
|
? `### Step 2: Identify consumed API endpoints and integration status
|
|
97
101
|
UI-only PR — perform two checks:
|
|
@@ -105,26 +109,28 @@ If no production file imports, re-exports, or renders a changed component, mark
|
|
|
105
109
|
Exception: if the same PR also adds a route/page file (e.g. under Next.js \`pages/\` or \`app/\`) that imports the component, the route IS the integration point — do NOT mark it as unintegrated.
|
|
106
110
|
Do NOT apply the unintegrated heuristic to route/entrypoint files themselves — those are always reachable by convention.
|
|
107
111
|
An unintegrated non-route component has no DOM node in the running app and cannot be browser-tested — it qualifies as a dead-code / unintegrated-component no-surface PR regardless of how complex the component logic is.`
|
|
108
|
-
: p.
|
|
109
|
-
? `### Step 2: Extract new and
|
|
110
|
-
Read the
|
|
112
|
+
: (canInline || p.diffFilePath)
|
|
113
|
+
? `### Step 2: Extract new, modified, and removed API endpoints from the diff
|
|
114
|
+
${canInline ? "Read the `<diff>` above" : `Read the diff file at \`${p.diffFilePath}\``} and identify every new or modified API endpoint — route registrations, handler methods, controller annotations. Then use the **Router Mounting / Nesting** section above to reconstruct the full URL path for each endpoint by chaining all parent router prefixes down to the handler (e.g. a handler in a file with \`prefix="/reviews"\` that is mounted at \`/{product_id}\` under a router mounted at \`/api/v1/products\` → full path \`/api/v1/products/{product_id}/reviews\`).
|
|
111
115
|
${diffHasJavaFiles ? JAVA_SPRING_NOTE : ""}
|
|
112
116
|
For each endpoint found: note the HTTP method, full path, and source file.
|
|
113
|
-
${
|
|
117
|
+
${preDetectedEndpoints ? "The endpoint catalog above already lists some changed endpoints — verify and augment with anything it missed." : "No endpoints were pre-detected in the changed files — extract them from the diff."}
|
|
118
|
+
**Also identify removed endpoints**: Look for deleted route annotations (lines starting with \`-\` in the diff) in modified files (files that still exist but had routes deleted). A removed endpoint is a route definition present in the base branch but absent in the current branch. Cross-reference against the scanned endpoint listing below — if a deleted route annotation's endpoint still appears there (e.g. moved to another file), it is NOT removed. Only flag endpoints that are truly gone from the codebase.
|
|
114
119
|
**CRITICAL — Query params vs body:** For GET endpoints (especially search/filter/list),
|
|
115
120
|
identify which parameters are URL query params vs request body. Look at framework-specific
|
|
116
121
|
annotations (FastAPI \`Query()\`, Express \`req.query\`, Spring \`@RequestParam\`, etc.).
|
|
117
122
|
Pass these as \`queryParams\` (not \`requestBody\`) when generating scenarios.`
|
|
118
|
-
: `### Step 2: Extract new and
|
|
123
|
+
: `### Step 2: Extract new, modified, and removed API endpoints from source files
|
|
119
124
|
No diff was available — read the changed source files listed above directly to identify new or modified API endpoints. Use the **Router Mounting / Nesting** section to reconstruct full paths.
|
|
120
125
|
${diffHasJavaFiles ? JAVA_SPRING_NOTE : ""}
|
|
121
|
-
For each endpoint found: note the HTTP method, full path, and source file
|
|
126
|
+
For each endpoint found: note the HTTP method, full path, and source file.
|
|
127
|
+
Also compare against the endpoint catalog to identify any endpoints that appear in the catalog but are no longer present in the source files — these are removed endpoints.`;
|
|
122
128
|
const criticalPatternStep = `### Step 2.5: Identify critical patterns for test categorization
|
|
123
129
|
Look for these patterns in model/schema/handler files to inform test recommendations:
|
|
124
130
|
- **Unique constraints**: \`@unique\`, \`unique: true\`, unique indexes, \`.refine()\` uniqueness checks, \`UNIQUE\` in SQL migrations
|
|
125
131
|
- **Cascade deletes**: \`ON DELETE CASCADE\`, \`.onDelete("cascade")\`, manual cascade logic in delete handlers
|
|
126
132
|
- **Permission checks**: auth middleware, ownership guards (\`req.user.id === resource.ownerId\`), role-based access control, \`isOwner\` assertions
|
|
127
|
-
- **Breaking changes in diff**: route renames, auth header changes, removed required fields, changed status codes
|
|
133
|
+
- **Breaking changes in diff**: route renames, deleted route definitions (endpoints removed from modified files), auth header changes, removed required fields, changed status codes
|
|
128
134
|
Tag each finding with its category (security_boundary, business_rule, data_integrity, breaking_change) for the recommendation step.`;
|
|
129
135
|
const step3Content = useHealthFlow
|
|
130
136
|
? `### Step 3: Identify tests at risk of drift
|
|
@@ -160,8 +166,7 @@ Call \`skyramp_recommend_tests\` with:
|
|
|
160
166
|
return `## Your Task — Enrich & Recommend (PR-scoped)
|
|
161
167
|
|
|
162
168
|
### Step 1: Read the changed files and diff
|
|
163
|
-
${changedFiles}${
|
|
164
|
-
|
|
169
|
+
${changedFiles}${diffFileRef}
|
|
165
170
|
${buildPathResolutionTableStep(p)}${step2}
|
|
166
171
|
|
|
167
172
|
${criticalPatternStep}
|
|
@@ -186,7 +191,7 @@ ${p.routerMountContext.map(f => `- \`${f}\``).join("\n")}`
|
|
|
186
191
|
**Session ID**: \`${p.sessionId}\`
|
|
187
192
|
**Repository**: \`${p.repositoryPath}\`
|
|
188
193
|
**Analysis Scope**: \`${p.analysisScope}\`
|
|
189
|
-
${isDiffScope ? `**Diff endpoints**: ${(p.parsedDiff?.newEndpoints.length ?? 0) + (p.parsedDiff?.modifiedEndpoints.length ?? 0)}` : `**Pre-scanned endpoints**: ${p.scannedEndpoints.length}`}
|
|
194
|
+
${isDiffScope ? `**Diff endpoints**: ${(p.parsedDiff?.newEndpoints.length ?? 0) + (p.parsedDiff?.modifiedEndpoints.length ?? 0) + (p.parsedDiff?.removedEndpoints?.length ?? 0)}` : `**Pre-scanned endpoints**: ${p.scannedEndpoints.length}`}
|
|
190
195
|
${routerSection}
|
|
191
196
|
${enrichment}
|
|
192
197
|
|
|
@@ -48,7 +48,7 @@ Before each GENERATE tool call, confirm WHERE each key value comes from:
|
|
|
48
48
|
- **requestBody / responseBody fields** → source code schema (Zod, Pydantic, DTO), enriched scenario, or OpenAPI spec. **The generation tool rejects empty \`{}\` request bodies for POST/PUT/PATCH** — read the source schema first if the fields are unknown.
|
|
49
49
|
- **endpointURL** → workspace \`baseUrl\` + endpoint path (both required — never path alone)
|
|
50
50
|
- **authHeader / authScheme** → workspace config or OpenAPI \`securitySchemes\`
|
|
51
|
-
- **FK path params** → chained from a prior step's response \`id\` field — not hardcoded
|
|
51
|
+
- **FK path params** → chained from a prior step's response (check the actual field name — it may be \`id\`, \`uuid\`, \`_id\`, or a resource-specific \`*_id\` field). The chaining source can be a response body (POST or GET), a response header (e.g. \`Location\`), or a cookie — not hardcoded
|
|
52
52
|
- **Names / string values** → realistic; append timestamp suffix to avoid re-run conflicts
|
|
53
53
|
|
|
54
54
|
## Ranking Rule
|
|
@@ -35,9 +35,10 @@ function classifyNovelty(scenario, diffContext) {
|
|
|
35
35
|
const paths = scenario.steps.map(s => s.path);
|
|
36
36
|
const newPaths = new Set((diffContext.newEndpoints || []).map(ep => ep.path));
|
|
37
37
|
const modPaths = new Set((diffContext.modifiedEndpoints || []).map(ep => ep.path));
|
|
38
|
+
const removedPaths = new Set((diffContext.removedEndpoints || []).map(ep => ep.path));
|
|
38
39
|
if (paths.some(p => newPaths.has(p)))
|
|
39
40
|
return "new";
|
|
40
|
-
if (paths.some(p => modPaths.has(p)))
|
|
41
|
+
if (paths.some(p => modPaths.has(p) || removedPaths.has(p)))
|
|
41
42
|
return "modified";
|
|
42
43
|
return "existing";
|
|
43
44
|
}
|
|
@@ -530,7 +531,7 @@ function buildExecutionPlan(scored, maxGen, topN, baseUrl, authHeaderValue, auth
|
|
|
530
531
|
? `authHeader: "${authHeaderValue}"${authSchemeSnippet}`
|
|
531
532
|
: "authHeader: <resolve from workspace or OpenAPI securitySchemes>; authScheme: <if Authorization>";
|
|
532
533
|
const prereqNote = s.category === "new_endpoint"
|
|
533
|
-
? `\n**Prerequisite discovery**: Check for FK fields (product_id, user_id, order_id) in the endpoint's request body. If found, prepend a step to create that prerequisite resource first, then chain its
|
|
534
|
+
? `\n**Prerequisite discovery**: Check for FK fields (product_id, user_id, order_id) in the endpoint's request body. If found, prepend a step to create that prerequisite resource first, then chain its primary key field into the dependent step using template variable syntax. Check the actual field name from the response body (\`id\`, \`uuid\`, \`_id\`, etc.), response header (\`Location\`), or cookie — do not assume \`id\`.`
|
|
534
535
|
: "";
|
|
535
536
|
const bugLine = s.bugCatchingTarget
|
|
536
537
|
? `**Bug to catch**: ${s.bugCatchingTarget}\n`
|
|
@@ -703,7 +704,7 @@ export function buildRecommendationPrompt(analysis, analysisScope = AnalysisScop
|
|
|
703
704
|
? filteredChangedFiles.some(f => isFrontendFile(f))
|
|
704
705
|
: false;
|
|
705
706
|
const hasApiChanges = isDiffScope && diffContext
|
|
706
|
-
? (diffContext.newEndpoints.length > 0 || diffContext.modifiedEndpoints.length > 0)
|
|
707
|
+
? (diffContext.newEndpoints.length > 0 || diffContext.modifiedEndpoints.length > 0 || (diffContext.removedEndpoints?.length ?? 0) > 0)
|
|
707
708
|
: false;
|
|
708
709
|
const isUIOnlyPR = hasFrontendChanges && !hasApiChanges;
|
|
709
710
|
const hasTraces = (analysis.artifacts?.traceFiles?.length ?? 0) > 0 ||
|
|
@@ -719,9 +720,46 @@ Output should be concise and immediately actionable.`
|
|
|
719
720
|
: `You are in **Repo mode**. Comprehensive test strategy across all endpoints.`;
|
|
720
721
|
// ── Endpoint listing ──
|
|
721
722
|
const allEndpoints = analysis.apiEndpoints.endpoints;
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
723
|
+
// In PR mode, identify which endpoints were changed so we can partition the listing.
|
|
724
|
+
const changedEndpointKeys = new Set();
|
|
725
|
+
if (isDiffScope && diffContext) {
|
|
726
|
+
for (const ep of [...(diffContext.newEndpoints || []), ...(diffContext.modifiedEndpoints || []), ...(diffContext.removedEndpoints || [])]) {
|
|
727
|
+
for (const m of (ep.methods ?? [])) {
|
|
728
|
+
changedEndpointKeys.add(`${m.method} ${ep.path}`);
|
|
729
|
+
}
|
|
730
|
+
}
|
|
731
|
+
}
|
|
732
|
+
const fmtEndpoint = (m, ep) => ` ${m.method} ${ep.path}${m.authRequired ? " [auth]" : ""} (${(m.interactions ?? []).length} interactions)`;
|
|
733
|
+
let endpointLines;
|
|
734
|
+
if (isDiffScope && changedEndpointKeys.size > 0) {
|
|
735
|
+
const changedLines = [];
|
|
736
|
+
const otherLines = [];
|
|
737
|
+
for (const ep of allEndpoints) {
|
|
738
|
+
for (const m of (ep.methods ?? [])) {
|
|
739
|
+
const line = fmtEndpoint(m, ep);
|
|
740
|
+
if (changedEndpointKeys.has(`${m.method} ${ep.path}`)) {
|
|
741
|
+
changedLines.push(line);
|
|
742
|
+
}
|
|
743
|
+
else {
|
|
744
|
+
otherLines.push(line);
|
|
745
|
+
}
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
// Removed endpoints no longer exist in allEndpoints (current catalog), so they
|
|
749
|
+
// would be silently absent from changedLines. Append them explicitly with a
|
|
750
|
+
// [removed] marker so the LLM knows to generate verify-404/deprecation tests.
|
|
751
|
+
for (const ep of (diffContext?.removedEndpoints || [])) {
|
|
752
|
+
for (const m of (ep.methods ?? [])) {
|
|
753
|
+
changedLines.push(` ${m.method} ${ep.path} [removed]`);
|
|
754
|
+
}
|
|
755
|
+
}
|
|
756
|
+
endpointLines = `**Changed in this PR:**\n${changedLines.join("\n") || " none"}\n\n**Other endpoints (reference only — do not prioritize for testing):**\n${otherLines.join("\n") || " none"}`;
|
|
757
|
+
}
|
|
758
|
+
else {
|
|
759
|
+
endpointLines = allEndpoints
|
|
760
|
+
.flatMap((ep) => (ep.methods ?? []).map((m) => fmtEndpoint(m, ep)))
|
|
761
|
+
.join("\n");
|
|
762
|
+
}
|
|
725
763
|
const authMethod = analysis.authentication.method || "unknown";
|
|
726
764
|
const authTypeValue = workspaceAuthType ?? "";
|
|
727
765
|
let authHeaderValue;
|
|
@@ -803,9 +841,13 @@ New endpoints:
|
|
|
803
841
|
${fmtEps(diffContext.newEndpoints, (m) => `${m.sourceFile}, ${m.interactionCount} interactions`)}
|
|
804
842
|
Modified endpoints:
|
|
805
843
|
${fmtEps(diffContext.modifiedEndpoints, (m) => `${m.sourceFile}, ${m.changeType}`)}
|
|
844
|
+
Removed endpoints:
|
|
845
|
+
${fmtEps(diffContext.removedEndpoints ?? [], (m) => `${m.sourceFile}, removed`)}
|
|
806
846
|
Affected services: ${diffContext.affectedServices.join(", ") || "N/A"}
|
|
807
847
|
|
|
808
848
|
Focus on tests that validate these changes and how they interact with existing resources.
|
|
849
|
+
For removed endpoints: verify they now return 404 or the appropriate deprecation status code.
|
|
850
|
+
Allocate your test budget to endpoints listed under "Changed in this PR". Use other endpoints only as setup steps (e.g. creating a resource before testing its deletion).
|
|
809
851
|
`;
|
|
810
852
|
}
|
|
811
853
|
// ── Interactions ──
|
|
@@ -886,6 +886,145 @@ describe("buildRecommendationPrompt — Tool Contract Framing", () => {
|
|
|
886
886
|
});
|
|
887
887
|
});
|
|
888
888
|
// ---------------------------------------------------------------------------
|
|
889
|
+
// Tests — Multi-method endpoint partitioning
|
|
890
|
+
// ---------------------------------------------------------------------------
|
|
891
|
+
describe("buildRecommendationPrompt — multi-method endpoint partitioning", () => {
|
|
892
|
+
it("classifies all methods of a changed endpoint as changed", () => {
|
|
893
|
+
// When classifyEndpointsByChangedFiles identifies a file as changed,
|
|
894
|
+
// all methods from that endpoint's scanned catalog entry are included
|
|
895
|
+
// with concrete methods (no MULTI sentinels).
|
|
896
|
+
const analysis = minimalAnalysis({
|
|
897
|
+
apiEndpoints: {
|
|
898
|
+
totalCount: 2,
|
|
899
|
+
baseUrl: "http://localhost:3000",
|
|
900
|
+
endpoints: [
|
|
901
|
+
{
|
|
902
|
+
path: "/api/products",
|
|
903
|
+
resourceGroup: "products",
|
|
904
|
+
pathParams: [],
|
|
905
|
+
methods: [
|
|
906
|
+
{ method: "GET", description: "List products", queryParams: [], authRequired: false, sourceFile: "app/api/products/route.ts", interactions: [] },
|
|
907
|
+
{ method: "POST", description: "Create product", queryParams: [], authRequired: false, sourceFile: "app/api/products/route.ts", interactions: [] },
|
|
908
|
+
],
|
|
909
|
+
},
|
|
910
|
+
{
|
|
911
|
+
path: "/api/items",
|
|
912
|
+
resourceGroup: "items",
|
|
913
|
+
pathParams: [],
|
|
914
|
+
methods: [
|
|
915
|
+
{ method: "GET", description: "List items", queryParams: [], authRequired: false, sourceFile: "routes/items.ts", interactions: [] },
|
|
916
|
+
],
|
|
917
|
+
},
|
|
918
|
+
],
|
|
919
|
+
},
|
|
920
|
+
branchDiffContext: {
|
|
921
|
+
baseBranch: "main",
|
|
922
|
+
currentBranch: "feature/products",
|
|
923
|
+
changedFiles: ["app/api/products/route.ts"],
|
|
924
|
+
newEndpoints: [{
|
|
925
|
+
path: "/api/products",
|
|
926
|
+
methods: [
|
|
927
|
+
{ method: "GET", sourceFile: "app/api/products/route.ts", interactionCount: 0 },
|
|
928
|
+
{ method: "POST", sourceFile: "app/api/products/route.ts", interactionCount: 0 },
|
|
929
|
+
],
|
|
930
|
+
}],
|
|
931
|
+
modifiedEndpoints: [],
|
|
932
|
+
affectedServices: [],
|
|
933
|
+
},
|
|
934
|
+
});
|
|
935
|
+
const prompt = buildRecommendationPrompt(analysis, AnalysisScope.CurrentBranchDiff, 10);
|
|
936
|
+
// Both GET and POST for /api/products should be in "Changed in this PR"
|
|
937
|
+
expect(prompt).toContain("Changed in this PR");
|
|
938
|
+
expect(prompt).toMatch(/Changed in this PR:[\s\S]*GET \/api\/products/);
|
|
939
|
+
expect(prompt).toMatch(/Changed in this PR:[\s\S]*POST \/api\/products/);
|
|
940
|
+
// /api/items should NOT be in changed section
|
|
941
|
+
expect(prompt).toMatch(/Other endpoints[\s\S]*GET \/api\/items/);
|
|
942
|
+
});
|
|
943
|
+
it("handles mix of new and modified endpoints with concrete methods", () => {
|
|
944
|
+
const analysis = minimalAnalysis({
|
|
945
|
+
apiEndpoints: {
|
|
946
|
+
totalCount: 2,
|
|
947
|
+
baseUrl: "http://localhost:3000",
|
|
948
|
+
endpoints: [
|
|
949
|
+
{
|
|
950
|
+
path: "/api/products",
|
|
951
|
+
resourceGroup: "products",
|
|
952
|
+
pathParams: [],
|
|
953
|
+
methods: [
|
|
954
|
+
{ method: "GET", description: "List", queryParams: [], authRequired: false, sourceFile: "routes.ts", interactions: [] },
|
|
955
|
+
{ method: "POST", description: "Create", queryParams: [], authRequired: false, sourceFile: "routes.ts", interactions: [] },
|
|
956
|
+
],
|
|
957
|
+
},
|
|
958
|
+
{
|
|
959
|
+
path: "/api/orders",
|
|
960
|
+
resourceGroup: "orders",
|
|
961
|
+
pathParams: [],
|
|
962
|
+
methods: [
|
|
963
|
+
{ method: "POST", description: "Create order", queryParams: [], authRequired: false, sourceFile: "routes.ts", interactions: [] },
|
|
964
|
+
],
|
|
965
|
+
},
|
|
966
|
+
],
|
|
967
|
+
},
|
|
968
|
+
branchDiffContext: {
|
|
969
|
+
baseBranch: "main",
|
|
970
|
+
currentBranch: "feature/mix",
|
|
971
|
+
changedFiles: ["routes.ts"],
|
|
972
|
+
newEndpoints: [
|
|
973
|
+
{ path: "/api/products", methods: [
|
|
974
|
+
{ method: "GET", sourceFile: "routes.ts", interactionCount: 0 },
|
|
975
|
+
{ method: "POST", sourceFile: "routes.ts", interactionCount: 0 },
|
|
976
|
+
] },
|
|
977
|
+
],
|
|
978
|
+
modifiedEndpoints: [
|
|
979
|
+
{ path: "/api/orders", methods: [{ method: "POST", sourceFile: "routes.ts", changeType: "modified" }] },
|
|
980
|
+
],
|
|
981
|
+
affectedServices: [],
|
|
982
|
+
},
|
|
983
|
+
});
|
|
984
|
+
const prompt = buildRecommendationPrompt(analysis, AnalysisScope.CurrentBranchDiff, 10);
|
|
985
|
+
// Both products and orders should be in changed section
|
|
986
|
+
expect(prompt).toMatch(/Changed in this PR:[\s\S]*GET \/api\/products/);
|
|
987
|
+
expect(prompt).toMatch(/Changed in this PR:[\s\S]*POST \/api\/orders/);
|
|
988
|
+
});
|
|
989
|
+
});
|
|
990
|
+
// ---------------------------------------------------------------------------
|
|
991
|
+
// Tests — Removed endpoint [removed] marker in prompt (Fix 3 verification)
|
|
992
|
+
// ---------------------------------------------------------------------------
|
|
993
|
+
describe("buildRecommendationPrompt — removed endpoint listing", () => {
|
|
994
|
+
it("appends [removed] marker for removed endpoints not in current catalog", () => {
|
|
995
|
+
const analysis = minimalAnalysis({
|
|
996
|
+
apiEndpoints: {
|
|
997
|
+
totalCount: 1,
|
|
998
|
+
baseUrl: "http://localhost:3000",
|
|
999
|
+
endpoints: [{
|
|
1000
|
+
path: "/api/items",
|
|
1001
|
+
resourceGroup: "items",
|
|
1002
|
+
pathParams: [],
|
|
1003
|
+
methods: [{
|
|
1004
|
+
method: "GET", description: "List items", queryParams: [],
|
|
1005
|
+
authRequired: false, sourceFile: "routes.ts", interactions: [],
|
|
1006
|
+
}],
|
|
1007
|
+
}],
|
|
1008
|
+
},
|
|
1009
|
+
branchDiffContext: {
|
|
1010
|
+
baseBranch: "main",
|
|
1011
|
+
currentBranch: "feature/remove",
|
|
1012
|
+
changedFiles: ["routes.ts"],
|
|
1013
|
+
newEndpoints: [],
|
|
1014
|
+
modifiedEndpoints: [],
|
|
1015
|
+
removedEndpoints: [{
|
|
1016
|
+
path: "/api/legacy",
|
|
1017
|
+
methods: [{ method: "DELETE", sourceFile: "routes.ts", changeType: "removed" }],
|
|
1018
|
+
}],
|
|
1019
|
+
affectedServices: [],
|
|
1020
|
+
},
|
|
1021
|
+
});
|
|
1022
|
+
const prompt = buildRecommendationPrompt(analysis, AnalysisScope.CurrentBranchDiff, 10);
|
|
1023
|
+
expect(prompt).toContain("DELETE /api/legacy [removed]");
|
|
1024
|
+
expect(prompt).toContain("Changed in this PR");
|
|
1025
|
+
});
|
|
1026
|
+
});
|
|
1027
|
+
// ---------------------------------------------------------------------------
|
|
889
1028
|
// Tests — Long-context best practices: XML tags structure
|
|
890
1029
|
// ---------------------------------------------------------------------------
|
|
891
1030
|
describe("buildRecommendationPrompt — XML tag structure (long-context best practice)", () => {
|
|
@@ -62,10 +62,11 @@ export class TestDiscoveryService {
|
|
|
62
62
|
* Uses fast-glob for cross-platform file scanning, then classifies discovered files
|
|
63
63
|
* as Skyramp-generated tests, external tests, or not-a-test during processing.
|
|
64
64
|
*
|
|
65
|
-
*
|
|
66
|
-
*
|
|
67
|
-
*
|
|
68
|
-
*
|
|
65
|
+
* External test handling depends on `options.changedResources`:
|
|
66
|
+
* - `string[]` with entries (PR mode, endpoints detected): partition by relevance.
|
|
67
|
+
* - `[]` empty array (PR mode, scanner found no endpoints): skip external tests entirely
|
|
68
|
+
* rather than flooding context with irrelevant files.
|
|
69
|
+
* - `undefined` (full-repo mode, no diff): cap at MAX_EXTERNAL_FULL_REPO.
|
|
69
70
|
*/
|
|
70
71
|
async discoverTests(repositoryPath, options = {}) {
|
|
71
72
|
logger.info(`Starting test discovery in: ${repositoryPath}`);
|
|
@@ -86,23 +87,37 @@ export class TestDiscoveryService {
|
|
|
86
87
|
skyrampTests.forEach(t => { t.source = TestSource.Skyramp; });
|
|
87
88
|
// Partition external tests into relevant (full extraction) and low-relevance (name-only).
|
|
88
89
|
//
|
|
89
|
-
// PR mode (changedResources
|
|
90
|
+
// PR mode + endpoints detected (changedResources is non-empty array):
|
|
90
91
|
// Files whose path/name token-overlaps with the changed resource names are "relevant".
|
|
91
92
|
// Only they get full endpoint extraction. Low-relevance files get name-only entries.
|
|
92
93
|
// No hard cap — the relevance filter naturally bounds the read set to PR scope.
|
|
94
|
+
// The sentinel ["unknown"] falls into this branch — most files score 0 (low-relevance)
|
|
95
|
+
// and get name-only entries, so external coverage is preserved without context flood.
|
|
93
96
|
//
|
|
94
|
-
//
|
|
97
|
+
// PR mode + truly no endpoints (changedResources is empty array []):
|
|
98
|
+
// Diff contained no endpoints at all (new, modified, or removed) — skip external
|
|
99
|
+
// tests entirely rather than flooding the prompt with hundreds of irrelevant files.
|
|
100
|
+
//
|
|
101
|
+
// Full-repo mode (changedResources is undefined):
|
|
95
102
|
// No diff context — all external files treated as potentially relevant.
|
|
96
103
|
// Cap at MAX_EXTERNAL_FULL_REPO to avoid reading hundreds of files.
|
|
97
104
|
const { changedResources } = options;
|
|
98
105
|
let relevantExternal;
|
|
99
106
|
let otherExternal;
|
|
100
107
|
if (changedResources?.length) {
|
|
108
|
+
// PR mode with detected endpoints — partition by relevance
|
|
101
109
|
({ relevant: relevantExternal, other: otherExternal } =
|
|
102
110
|
this.partitionByRelevance(classified.external, changedResources));
|
|
103
111
|
}
|
|
112
|
+
else if (changedResources !== undefined) {
|
|
113
|
+
// PR mode with an explicit empty endpoint list from diff parsing — don't flood
|
|
114
|
+
// context with irrelevant external tests. The LLM will work from Skyramp tests
|
|
115
|
+
// and scanned endpoints only.
|
|
116
|
+
relevantExternal = [];
|
|
117
|
+
otherExternal = [];
|
|
118
|
+
}
|
|
104
119
|
else {
|
|
105
|
-
// Full-repo mode: cap full-extraction set, remaining become name-only
|
|
120
|
+
// Full-repo mode (no diff context): cap full-extraction set, remaining become name-only
|
|
106
121
|
relevantExternal = classified.external.slice(0, this.MAX_EXTERNAL_FULL_REPO);
|
|
107
122
|
otherExternal = classified.external.slice(this.MAX_EXTERNAL_FULL_REPO);
|
|
108
123
|
}
|
|
@@ -348,6 +348,50 @@ describe("TestDiscoveryService", () => {
|
|
|
348
348
|
const withEndpoints = externalTests.filter(t => t.apiEndpoint !== "");
|
|
349
349
|
expect(withEndpoints.length).toBe(12);
|
|
350
350
|
});
|
|
351
|
+
it("returns zero external tests when changedResources is empty array (PR mode, no endpoints)", async () => {
|
|
352
|
+
// Simulate PR mode where a parsed diff produced no detected endpoints:
|
|
353
|
+
// newEndpoints=[], modifiedEndpoints=[], and removedEndpoints=[] → changedResources = []
|
|
354
|
+
writeFile("test_orders_api.py", 'import requests\nrequests.get("/api/orders")');
|
|
355
|
+
writeFile("test_products_api.py", 'import requests\nrequests.get("/api/products")');
|
|
356
|
+
const result = await service.discoverTests(tmpDir, { changedResources: [] });
|
|
357
|
+
const externalTests = result.tests.filter(t => t.source === TestSource.External);
|
|
358
|
+
// Empty changedResources = PR mode with no detected endpoints → zero external tests
|
|
359
|
+
expect(externalTests.length).toBe(0);
|
|
360
|
+
expect(result.relevantExternalTestPaths.length).toBe(0);
|
|
361
|
+
});
|
|
362
|
+
it("still returns external tests in full-repo mode (changedResources undefined)", async () => {
|
|
363
|
+
// Full-repo mode: changedResources not provided → should use capped full-repo behavior
|
|
364
|
+
writeFile("test_orders_api.py", 'import requests\nrequests.get("/api/orders")');
|
|
365
|
+
writeFile("test_products_api.py", 'import requests\nrequests.get("/api/products")');
|
|
366
|
+
const result = await service.discoverTests(tmpDir); // no options → undefined
|
|
367
|
+
const externalTests = result.tests.filter(t => t.source === TestSource.External);
|
|
368
|
+
expect(externalTests.length).toBe(2);
|
|
369
|
+
expect(result.relevantExternalTestPaths.length).toBe(2);
|
|
370
|
+
});
|
|
371
|
+
it("Skyramp tests are unaffected by empty changedResources", async () => {
|
|
372
|
+
writeFile("tests/test_orders_smoke.py", '# Generated by Skyramp\nskyramp generate smoke rest');
|
|
373
|
+
writeFile("test_external.py", 'import pytest\ndef test(): pass');
|
|
374
|
+
const result = await service.discoverTests(tmpDir, { changedResources: [] });
|
|
375
|
+
const skyrampTests = result.tests.filter(t => t.source === TestSource.Skyramp);
|
|
376
|
+
const externalTests = result.tests.filter(t => t.source === TestSource.External);
|
|
377
|
+
// Skyramp tests always discovered regardless of changedResources
|
|
378
|
+
expect(skyrampTests.length).toBe(1);
|
|
379
|
+
// External tests suppressed in PR-mode-no-endpoints
|
|
380
|
+
expect(externalTests.length).toBe(0);
|
|
381
|
+
});
|
|
382
|
+
it("returns external tests as name-only with ['unknown'] sentinel (unresolvable resources)", async () => {
|
|
383
|
+
// When diff endpoints exist but all paths resolve to "unknown" (e.g. decorator-relative
|
|
384
|
+
// paths like "/{order_id}"), changedResources = ["unknown"]. External tests should be
|
|
385
|
+
// discovered (not skipped) but scored as low-relevance since "unknown" won't match filenames.
|
|
386
|
+
writeFile("test_orders_api.py", 'import requests\nrequests.get("/api/orders")');
|
|
387
|
+
writeFile("test_products_api.py", 'import requests\nrequests.get("/api/products")');
|
|
388
|
+
const result = await service.discoverTests(tmpDir, { changedResources: ["unknown"] });
|
|
389
|
+
const externalTests = result.tests.filter(t => t.source === TestSource.External);
|
|
390
|
+
// External tests discovered (not skipped like empty array)
|
|
391
|
+
expect(externalTests.length).toBe(2);
|
|
392
|
+
// But all are low-relevance (name-only) since "unknown" doesn't match any filename tokens
|
|
393
|
+
expect(result.relevantExternalTestPaths.length).toBe(0);
|
|
394
|
+
});
|
|
351
395
|
it("low-relevance files have empty apiEndpoint and empty framework in PR mode", async () => {
|
|
352
396
|
writeFile("test_orders_api.py", 'import requests\nrequests.get("/api/orders")');
|
|
353
397
|
writeFile("test_products_api.py", 'import requests\nrequests.get("/api/products")');
|