@skyramp/mcp 0.0.65 → 0.1.0-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/playwright/traceRecordingPrompt.js +30 -36
- package/build/prompts/architectPersona.js +19 -0
- package/build/prompts/test-maintenance/drift-analysis-prompt.js +11 -6
- package/build/prompts/test-maintenance/drift-analysis-prompt.test.js +49 -0
- package/build/prompts/test-maintenance/driftAnalysisSections.js +4 -2
- package/build/prompts/test-recommendation/analysisOutputPrompt.js +42 -50
- package/build/prompts/test-recommendation/mergeEnrichedScenarios.test.js +125 -0
- package/build/prompts/test-recommendation/recommendationSections.js +121 -4
- package/build/prompts/test-recommendation/registerRecommendTestsPrompt.js +151 -9
- package/build/prompts/test-recommendation/test-recommendation-prompt.js +416 -61
- package/build/prompts/test-recommendation/test-recommendation-prompt.test.js +455 -63
- package/build/prompts/testbot/testbot-prompts.js +111 -100
- package/build/prompts/testbot/testbot-prompts.test.js +142 -0
- package/build/resources/analysisResources.js +13 -5
- package/build/services/ScenarioGenerationService.js +2 -2
- package/build/services/ScenarioGenerationService.test.js +35 -0
- package/build/services/TestExecutionService.js +1 -1
- package/build/tools/code-refactor/modularizationTool.js +2 -2
- package/build/tools/executeSkyrampTestTool.js +4 -3
- package/build/tools/generate-tests/generateBatchScenarioRestTool.js +51 -21
- package/build/tools/generate-tests/generateContractRestTool.js +26 -4
- package/build/tools/generate-tests/generateIntegrationRestTool.js +44 -13
- package/build/tools/generate-tests/generateScenarioRestTool.js +17 -39
- package/build/tools/generate-tests/generateUIRestTool.js +69 -4
- package/build/tools/submitReportTool.js +27 -13
- package/build/tools/test-management/analyzeChangesTool.js +32 -10
- package/build/tools/test-management/analyzeChangesTool.test.js +85 -0
- package/build/types/RepositoryAnalysis.js +25 -3
- package/build/types/TestRecommendation.js +5 -4
- package/build/types/TestTypes.js +44 -9
- package/build/utils/AnalysisStateManager.js +43 -9
- package/build/utils/AnalysisStateManager.test.js +35 -0
- package/build/utils/routeParsers.js +35 -0
- package/build/utils/routeParsers.test.js +66 -1
- package/build/utils/scenarioDrafting.js +207 -360
- package/build/utils/scenarioDrafting.test.js +191 -256
- package/build/utils/trace-parser.js +24 -6
- package/build/utils/trace-parser.test.js +140 -0
- package/node_modules/playwright/lib/mcp/browser/browserServerBackend.js +3 -0
- package/node_modules/playwright/lib/mcp/browser/tab.js +8 -1
- package/node_modules/playwright/lib/mcp/browser/tools/keyboard.js +3 -2
- package/node_modules/playwright/lib/mcp/browser/tools/navigate.js +1 -1
- package/node_modules/playwright/lib/mcp/browser/tools/snapshot.js +4 -4
- package/node_modules/playwright/lib/mcp/browser/tools/tabs.js +5 -4
- package/node_modules/playwright/lib/mcp/browser/tools/wait.js +1 -1
- package/node_modules/playwright/lib/mcp/skyramp/exportTool.js +10 -9
- package/node_modules/playwright/lib/mcp/skyramp/traceRecordingBackend.js +304 -7
- package/node_modules/playwright/lib/mcp/test/skyRampExport.js +128 -20
- package/package.json +2 -2
- package/node_modules/playwright/lib/mcp/terminal/help.json +0 -32
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
jest.mock("@skyramp/skyramp", () => ({
|
|
2
2
|
WorkspaceConfigManager: { create: jest.fn() },
|
|
3
3
|
}));
|
|
4
|
+
import { TestType } from "../../types/TestTypes.js";
|
|
4
5
|
import { buildRecommendationPrompt } from "./test-recommendation-prompt.js";
|
|
5
|
-
import { PATH_PARAM_UUID_GUIDANCE, MAX_TESTS_TO_GENERATE, buildTestQualityCriteria } from "./recommendationSections.js";
|
|
6
|
+
import { PATH_PARAM_UUID_GUIDANCE, MAX_TESTS_TO_GENERATE, buildTestQualityCriteria, buildArchitectPreamble, buildContextFetchingGuidance, buildReasoningProtocol, buildFewShotExamples, buildVerificationChecklist, } from "./recommendationSections.js";
|
|
7
|
+
import { AnalysisScope } from "../../types/RepositoryAnalysis.js";
|
|
6
8
|
// ---------------------------------------------------------------------------
|
|
7
9
|
// Minimal fixtures
|
|
8
10
|
// ---------------------------------------------------------------------------
|
|
@@ -60,7 +62,7 @@ describe("buildRecommendationPrompt — PR History section", () => {
|
|
|
60
62
|
expect(prompt).not.toContain("Stability rule");
|
|
61
63
|
});
|
|
62
64
|
it("omits PR History section when prContext has no recommendations", () => {
|
|
63
|
-
const prompt = buildRecommendationPrompt(minimalAnalysis(),
|
|
65
|
+
const prompt = buildRecommendationPrompt(minimalAnalysis(), AnalysisScope.CurrentBranchDiff, 10, makePRContext());
|
|
64
66
|
expect(prompt).not.toContain("PR History");
|
|
65
67
|
});
|
|
66
68
|
it("includes PR History section with prNumber when recommendations exist", () => {
|
|
@@ -70,7 +72,7 @@ describe("buildRecommendationPrompt — PR History section", () => {
|
|
|
70
72
|
{ testType: "integration", endpoint: "POST /api/items", status: "implemented", commentId: "1" },
|
|
71
73
|
],
|
|
72
74
|
});
|
|
73
|
-
const prompt = buildRecommendationPrompt(minimalAnalysis(),
|
|
75
|
+
const prompt = buildRecommendationPrompt(minimalAnalysis(), AnalysisScope.CurrentBranchDiff, 10, ctx);
|
|
74
76
|
expect(prompt).toContain("PR History (PR #99)");
|
|
75
77
|
});
|
|
76
78
|
it("renders Previously Generated Tests for implemented recommendations", () => {
|
|
@@ -80,7 +82,7 @@ describe("buildRecommendationPrompt — PR History section", () => {
|
|
|
80
82
|
],
|
|
81
83
|
implementedTestFiles: ["test_items_contract.py"],
|
|
82
84
|
});
|
|
83
|
-
const prompt = buildRecommendationPrompt(minimalAnalysis(),
|
|
85
|
+
const prompt = buildRecommendationPrompt(minimalAnalysis(), AnalysisScope.CurrentBranchDiff, 10, ctx);
|
|
84
86
|
expect(prompt).toContain("Previously Generated Tests");
|
|
85
87
|
expect(prompt).toContain("contract — GET /api/items");
|
|
86
88
|
expect(prompt).toContain("test_items_contract.py");
|
|
@@ -91,7 +93,7 @@ describe("buildRecommendationPrompt — PR History section", () => {
|
|
|
91
93
|
{ testType: "contract", endpoint: "GET /api/items", status: "implemented", commentId: "1" },
|
|
92
94
|
],
|
|
93
95
|
});
|
|
94
|
-
const prompt = buildRecommendationPrompt(minimalAnalysis(),
|
|
96
|
+
const prompt = buildRecommendationPrompt(minimalAnalysis(), AnalysisScope.CurrentBranchDiff, 10, ctx);
|
|
95
97
|
expect(prompt).not.toContain("Stability rule");
|
|
96
98
|
expect(prompt).not.toContain("Previously Recommended");
|
|
97
99
|
});
|
|
@@ -101,7 +103,7 @@ describe("buildRecommendationPrompt — PR History section", () => {
|
|
|
101
103
|
{ testType: "integration", endpoint: "POST /api/orders", scenarioName: "order-lifecycle", status: "recommended", commentId: "1" },
|
|
102
104
|
],
|
|
103
105
|
});
|
|
104
|
-
const prompt = buildRecommendationPrompt(minimalAnalysis(),
|
|
106
|
+
const prompt = buildRecommendationPrompt(minimalAnalysis(), AnalysisScope.CurrentBranchDiff, 10, ctx);
|
|
105
107
|
expect(prompt).toContain("Previously Recommended (not generated)");
|
|
106
108
|
expect(prompt).toContain("Stability rule");
|
|
107
109
|
expect(prompt).toContain("integration — POST /api/orders (scenario: order-lifecycle)");
|
|
@@ -112,7 +114,7 @@ describe("buildRecommendationPrompt — PR History section", () => {
|
|
|
112
114
|
{ testType: "integration", endpoint: "POST /api/orders", status: "recommended", commentId: "1" },
|
|
113
115
|
],
|
|
114
116
|
});
|
|
115
|
-
const prompt = buildRecommendationPrompt(minimalAnalysis(),
|
|
117
|
+
const prompt = buildRecommendationPrompt(minimalAnalysis(), AnalysisScope.CurrentBranchDiff, 10, ctx);
|
|
116
118
|
const stabilityBlock = prompt.slice(prompt.indexOf("**Stability rule**"), prompt.indexOf("Only add NEW recommendations"));
|
|
117
119
|
expect(stabilityBlock).toContain("scenarioName");
|
|
118
120
|
expect(stabilityBlock).toContain("endpoint");
|
|
@@ -126,7 +128,7 @@ describe("buildRecommendationPrompt — PR History section", () => {
|
|
|
126
128
|
{ testType: "integration", endpoint: "POST /api/orders", status: "recommended", commentId: "1" },
|
|
127
129
|
],
|
|
128
130
|
});
|
|
129
|
-
const prompt = buildRecommendationPrompt(minimalAnalysis(),
|
|
131
|
+
const prompt = buildRecommendationPrompt(minimalAnalysis(), AnalysisScope.CurrentBranchDiff, 10, ctx);
|
|
130
132
|
expect(prompt).toContain("Do not churn recommendations without cause");
|
|
131
133
|
expect(prompt).toContain("Carry forward");
|
|
132
134
|
});
|
|
@@ -138,7 +140,7 @@ describe("buildRecommendationPrompt — PR History section", () => {
|
|
|
138
140
|
],
|
|
139
141
|
implementedTestFiles: ["test_items_contract.py"],
|
|
140
142
|
});
|
|
141
|
-
const prompt = buildRecommendationPrompt(minimalAnalysis(),
|
|
143
|
+
const prompt = buildRecommendationPrompt(minimalAnalysis(), AnalysisScope.CurrentBranchDiff, 10, ctx);
|
|
142
144
|
expect(prompt).toContain("Previously Generated Tests");
|
|
143
145
|
expect(prompt).toContain("Previously Recommended (not generated)");
|
|
144
146
|
expect(prompt).toContain("Stability rule");
|
|
@@ -152,7 +154,7 @@ describe("buildRecommendationPrompt — PR History section", () => {
|
|
|
152
154
|
{ testFile: "test_items_contract.py", status: "fail", timestamp: "2026-01-01T00:00:00Z" },
|
|
153
155
|
],
|
|
154
156
|
});
|
|
155
|
-
const prompt = buildRecommendationPrompt(minimalAnalysis(),
|
|
157
|
+
const prompt = buildRecommendationPrompt(minimalAnalysis(), AnalysisScope.CurrentBranchDiff, 10, ctx);
|
|
156
158
|
expect(prompt).toContain("Execution Results from Prior Run");
|
|
157
159
|
expect(prompt).toContain("test_items_contract.py");
|
|
158
160
|
expect(prompt).toContain("fail");
|
|
@@ -163,7 +165,7 @@ describe("buildRecommendationPrompt — PR History section", () => {
|
|
|
163
165
|
{ testType: "contract", endpoint: "GET /api/items", status: "recommended", commentId: "1" },
|
|
164
166
|
],
|
|
165
167
|
});
|
|
166
|
-
const prompt = buildRecommendationPrompt(minimalAnalysis(),
|
|
168
|
+
const prompt = buildRecommendationPrompt(minimalAnalysis(), AnalysisScope.CurrentBranchDiff, 10, ctx);
|
|
167
169
|
expect(prompt).toContain("contract — GET /api/items");
|
|
168
170
|
expect(prompt).not.toContain("(scenario:");
|
|
169
171
|
});
|
|
@@ -175,7 +177,7 @@ describe("buildRecommendationPrompt — PR History section", () => {
|
|
|
175
177
|
{ testType: "integration", endpoint: "DELETE /api/users/{id}", scenarioName: "cascade-delete-user", status: "recommended", commentId: "1" },
|
|
176
178
|
],
|
|
177
179
|
});
|
|
178
|
-
const prompt = buildRecommendationPrompt(minimalAnalysis(),
|
|
180
|
+
const prompt = buildRecommendationPrompt(minimalAnalysis(), AnalysisScope.CurrentBranchDiff, 10, ctx);
|
|
179
181
|
expect(prompt).toContain("order-lifecycle");
|
|
180
182
|
expect(prompt).toContain("item-schema-check");
|
|
181
183
|
expect(prompt).toContain("cascade-delete-user");
|
|
@@ -188,7 +190,7 @@ describe("buildRecommendationPrompt — PR History section", () => {
|
|
|
188
190
|
{ testType: "contract", endpoint: "GET /api/items", status: "recommended", commentId: "1" },
|
|
189
191
|
],
|
|
190
192
|
});
|
|
191
|
-
const prompt = buildRecommendationPrompt(minimalAnalysis(),
|
|
193
|
+
const prompt = buildRecommendationPrompt(minimalAnalysis(), AnalysisScope.CurrentBranchDiff, 10, ctx);
|
|
192
194
|
expect(prompt).not.toContain("order-lifecycle");
|
|
193
195
|
expect(prompt).toContain("contract — GET /api/items");
|
|
194
196
|
});
|
|
@@ -198,7 +200,7 @@ describe("buildRecommendationPrompt — PR History section", () => {
|
|
|
198
200
|
{ testType: "integration", endpoint: "POST /api/orders", status: "recommended", commentId: "1" },
|
|
199
201
|
],
|
|
200
202
|
});
|
|
201
|
-
const prompt = buildRecommendationPrompt(minimalAnalysis(),
|
|
203
|
+
const prompt = buildRecommendationPrompt(minimalAnalysis(), AnalysisScope.CurrentBranchDiff, 10, ctx);
|
|
202
204
|
expect(prompt).toContain("Promote the highest-priority ones");
|
|
203
205
|
expect(prompt).toContain("into generation slots if capacity allows");
|
|
204
206
|
});
|
|
@@ -208,7 +210,7 @@ describe("buildRecommendationPrompt — PR History section", () => {
|
|
|
208
210
|
{ testType: "contract", endpoint: "GET /api/items", status: "implemented", commentId: "1" },
|
|
209
211
|
],
|
|
210
212
|
});
|
|
211
|
-
const prompt = buildRecommendationPrompt(minimalAnalysis(),
|
|
213
|
+
const prompt = buildRecommendationPrompt(minimalAnalysis(), AnalysisScope.CurrentBranchDiff, 10, ctx);
|
|
212
214
|
expect(prompt).toContain("Do NOT re-recommend");
|
|
213
215
|
expect(prompt).toContain("Previously Generated Tests");
|
|
214
216
|
});
|
|
@@ -220,7 +222,7 @@ describe("buildRecommendationPrompt — PR History section", () => {
|
|
|
220
222
|
{ testType: "integration", endpoint: "GET /api/orders/{id}", scenarioName: "order-lifecycle", status: "recommended", commentId: "1" },
|
|
221
223
|
],
|
|
222
224
|
});
|
|
223
|
-
const prompt = buildRecommendationPrompt(minimalAnalysis(),
|
|
225
|
+
const prompt = buildRecommendationPrompt(minimalAnalysis(), AnalysisScope.CurrentBranchDiff, 10, ctx);
|
|
224
226
|
const recSection = prompt.slice(prompt.indexOf("Previously Recommended"), prompt.indexOf("**Stability rule**"));
|
|
225
227
|
const scenarioOccurrences = (recSection.match(/order-lifecycle/g) || []).length;
|
|
226
228
|
expect(scenarioOccurrences).toBe(1);
|
|
@@ -236,7 +238,7 @@ describe("buildRecommendationPrompt — PR History section", () => {
|
|
|
236
238
|
{ testType: "contract", endpoint: "POST /api/items", status: "recommended", commentId: "1" },
|
|
237
239
|
],
|
|
238
240
|
});
|
|
239
|
-
const prompt = buildRecommendationPrompt(minimalAnalysis(),
|
|
241
|
+
const prompt = buildRecommendationPrompt(minimalAnalysis(), AnalysisScope.CurrentBranchDiff, 10, ctx);
|
|
240
242
|
const recSection = prompt.slice(prompt.indexOf("Previously Recommended"), prompt.indexOf("**Stability rule**"));
|
|
241
243
|
const getOccurrences = (recSection.match(/contract — GET \/api\/items/g) || []).length;
|
|
242
244
|
expect(getOccurrences).toBe(1);
|
|
@@ -252,7 +254,7 @@ describe("buildRecommendationPrompt — PR History section", () => {
|
|
|
252
254
|
{ testType: "integration", endpoint: "DELETE /api/items/{id}", scenarioName: "cascade-delete", status: "recommended", commentId: "1" },
|
|
253
255
|
],
|
|
254
256
|
});
|
|
255
|
-
const prompt = buildRecommendationPrompt(minimalAnalysis(),
|
|
257
|
+
const prompt = buildRecommendationPrompt(minimalAnalysis(), AnalysisScope.CurrentBranchDiff, 10, ctx);
|
|
256
258
|
const recSection = prompt.slice(prompt.indexOf("Previously Recommended"), prompt.indexOf("**Stability rule**"));
|
|
257
259
|
const orderOccurrences = (recSection.match(/order-lifecycle/g) || []).length;
|
|
258
260
|
const cascadeOccurrences = (recSection.match(/cascade-delete/g) || []).length;
|
|
@@ -276,34 +278,34 @@ function minimalScenario(overrides = {}) {
|
|
|
276
278
|
chainingKeys: ["id"],
|
|
277
279
|
requiresAuth: false,
|
|
278
280
|
estimatedComplexity: "simple",
|
|
279
|
-
testType:
|
|
281
|
+
testType: TestType.INTEGRATION,
|
|
280
282
|
...overrides,
|
|
281
283
|
};
|
|
282
284
|
}
|
|
283
285
|
describe("buildRecommendationPrompt — Stability and supplement section", () => {
|
|
284
|
-
it("includes Recommendation Stability section in output when scenarios exist", () => {
|
|
286
|
+
it("includes Recommendation Stability section in output when scenarios exist (PR mode)", () => {
|
|
285
287
|
const analysis = minimalAnalysis({
|
|
286
288
|
businessContext: { mainPurpose: "Test API", userFlows: [], dataFlows: [], integrationPatterns: [], draftedScenarios: [minimalScenario()] },
|
|
287
289
|
});
|
|
288
|
-
const prompt = buildRecommendationPrompt(analysis,
|
|
290
|
+
const prompt = buildRecommendationPrompt(analysis, AnalysisScope.CurrentBranchDiff, 10);
|
|
289
291
|
expect(prompt).toContain("## Recommendation Stability");
|
|
290
292
|
});
|
|
291
|
-
it("stability section uses scenarioName/endpoint matching strategy", () => {
|
|
293
|
+
it("stability section uses scenarioName/endpoint matching strategy (PR mode)", () => {
|
|
292
294
|
const analysis = minimalAnalysis({
|
|
293
295
|
businessContext: { mainPurpose: "Test API", userFlows: [], dataFlows: [], integrationPatterns: [], draftedScenarios: [minimalScenario()] },
|
|
294
296
|
});
|
|
295
|
-
const prompt = buildRecommendationPrompt(analysis,
|
|
297
|
+
const prompt = buildRecommendationPrompt(analysis, AnalysisScope.CurrentBranchDiff, 10);
|
|
296
298
|
const stabilityStart = prompt.indexOf("## Recommendation Stability");
|
|
297
299
|
const stabilityBlock = prompt.slice(stabilityStart, stabilityStart + 500);
|
|
298
300
|
expect(stabilityBlock).toContain("scenarioName");
|
|
299
301
|
expect(stabilityBlock).toContain("endpoint");
|
|
300
302
|
expect(stabilityBlock).toContain("Re-derive category and priority");
|
|
301
303
|
});
|
|
302
|
-
it("stability section specifies when to drop a recommendation", () => {
|
|
304
|
+
it("stability section specifies when to drop a recommendation (PR mode)", () => {
|
|
303
305
|
const analysis = minimalAnalysis({
|
|
304
306
|
businessContext: { mainPurpose: "Test API", userFlows: [], dataFlows: [], integrationPatterns: [], draftedScenarios: [minimalScenario()] },
|
|
305
307
|
});
|
|
306
|
-
const prompt = buildRecommendationPrompt(analysis,
|
|
308
|
+
const prompt = buildRecommendationPrompt(analysis, AnalysisScope.CurrentBranchDiff, 10);
|
|
307
309
|
expect(prompt).toContain("target endpoint was removed");
|
|
308
310
|
expect(prompt).toContain("business logic changed");
|
|
309
311
|
expect(prompt).toContain("covered by a generated test");
|
|
@@ -313,21 +315,21 @@ describe("buildRecommendationPrompt — Stability and supplement section", () =>
|
|
|
313
315
|
const analysis = minimalAnalysis({
|
|
314
316
|
businessContext: { mainPurpose: "Test API", userFlows: [], dataFlows: [], integrationPatterns: [], draftedScenarios: [minimalScenario()] },
|
|
315
317
|
});
|
|
316
|
-
const prompt = buildRecommendationPrompt(analysis,
|
|
317
|
-
expect(prompt).toContain("
|
|
318
|
-
expect(prompt).toContain("
|
|
318
|
+
const prompt = buildRecommendationPrompt(analysis, AnalysisScope.FullRepo, 10);
|
|
319
|
+
expect(prompt).toContain("<supplement_guidance>");
|
|
320
|
+
expect(prompt).toContain("5-dimension rubric");
|
|
319
321
|
});
|
|
320
322
|
// Verify MAX_TESTS_TO_GENERATE is still exported and equals 3
|
|
321
323
|
it("MAX_TESTS_TO_GENERATE is 3", () => {
|
|
322
324
|
expect(MAX_TESTS_TO_GENERATE).toBe(3);
|
|
323
325
|
});
|
|
324
|
-
it("uses MAX_CRITICAL_TESTS in category-aware selection rules", () => {
|
|
326
|
+
it("uses MAX_CRITICAL_TESTS in category-aware selection rules (PR mode)", () => {
|
|
325
327
|
const analysis = minimalAnalysis({
|
|
326
328
|
businessContext: { mainPurpose: "Test API", userFlows: [], dataFlows: [], integrationPatterns: [], draftedScenarios: [minimalScenario()] },
|
|
327
329
|
});
|
|
328
|
-
const prompt = buildRecommendationPrompt(analysis,
|
|
329
|
-
//
|
|
330
|
-
expect(prompt).toContain("GENERATE items
|
|
330
|
+
const prompt = buildRecommendationPrompt(analysis, AnalysisScope.CurrentBranchDiff, 10);
|
|
331
|
+
// MAX_CRITICAL_TESTS applies to PR mode (GENERATE items) — full_repo mode only presents, does not execute
|
|
332
|
+
expect(prompt).toContain("GENERATE items should be from HIGH-priority categories");
|
|
331
333
|
});
|
|
332
334
|
});
|
|
333
335
|
// ---------------------------------------------------------------------------
|
|
@@ -364,7 +366,7 @@ describe("PATH_PARAM_UUID_GUIDANCE — no hardcoded UUID anchor", () => {
|
|
|
364
366
|
}],
|
|
365
367
|
},
|
|
366
368
|
});
|
|
367
|
-
const prompt = buildRecommendationPrompt(analysis,
|
|
369
|
+
const prompt = buildRecommendationPrompt(analysis, AnalysisScope.CurrentBranchDiff, 10);
|
|
368
370
|
expect(prompt).toContain("<random-uuid-v4>");
|
|
369
371
|
expect(prompt).not.toMatch(UUID_V4_REGEX);
|
|
370
372
|
});
|
|
@@ -373,12 +375,24 @@ describe("PATH_PARAM_UUID_GUIDANCE — no hardcoded UUID anchor", () => {
|
|
|
373
375
|
// Tests — maxGenerateOverride parameter in buildRecommendationPrompt
|
|
374
376
|
// ---------------------------------------------------------------------------
|
|
375
377
|
describe("buildRecommendationPrompt — maxGenerateOverride", () => {
|
|
376
|
-
const scenariosForOverride =
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
378
|
+
const scenariosForOverride = [
|
|
379
|
+
// One single-step scenario (auto-classified as contract in full_repo mode) to verify contract tool calls
|
|
380
|
+
minimalScenario({
|
|
381
|
+
scenarioName: "scenario-0",
|
|
382
|
+
description: "Test scenario 0",
|
|
383
|
+
category: "security_boundary",
|
|
384
|
+
priority: "high",
|
|
385
|
+
testType: TestType.CONTRACT,
|
|
386
|
+
steps: [{ order: 1, method: "GET", path: "/api/items", description: "Get items", interactionType: "success", expectedStatusCode: 200 }],
|
|
387
|
+
}),
|
|
388
|
+
// Five multi-step integration scenarios
|
|
389
|
+
...Array.from({ length: 5 }, (_, i) => minimalScenario({
|
|
390
|
+
scenarioName: `scenario-${i + 1}`,
|
|
391
|
+
description: `Test scenario ${i + 1}`,
|
|
392
|
+
category: i < 1 ? "security_boundary" : "crud",
|
|
393
|
+
priority: i < 1 ? "high" : "low",
|
|
394
|
+
})),
|
|
395
|
+
];
|
|
382
396
|
const analysisWithScenarios = minimalAnalysis({
|
|
383
397
|
businessContext: {
|
|
384
398
|
mainPurpose: "Test API",
|
|
@@ -389,35 +403,85 @@ describe("buildRecommendationPrompt — maxGenerateOverride", () => {
|
|
|
389
403
|
},
|
|
390
404
|
});
|
|
391
405
|
it("uses MAX_TESTS_TO_GENERATE as default when maxGenerateOverride is undefined", () => {
|
|
392
|
-
const prompt = buildRecommendationPrompt(analysisWithScenarios,
|
|
406
|
+
const prompt = buildRecommendationPrompt(analysisWithScenarios, AnalysisScope.CurrentBranchDiff, 10);
|
|
393
407
|
expect(prompt).toContain(`Budget: ${MAX_TESTS_TO_GENERATE} generate`);
|
|
394
408
|
});
|
|
395
409
|
it("respects maxGenerateOverride when provided", () => {
|
|
396
|
-
const prompt = buildRecommendationPrompt(analysisWithScenarios,
|
|
410
|
+
const prompt = buildRecommendationPrompt(analysisWithScenarios, AnalysisScope.CurrentBranchDiff, 10, undefined, undefined, undefined, 5);
|
|
397
411
|
expect(prompt).toContain("Budget: 5 generate");
|
|
398
412
|
expect(prompt).toContain("additional = 10 total");
|
|
399
413
|
});
|
|
400
414
|
it("clamps maxGenerateOverride to topN when override exceeds topN", () => {
|
|
401
|
-
const prompt = buildRecommendationPrompt(analysisWithScenarios,
|
|
415
|
+
const prompt = buildRecommendationPrompt(analysisWithScenarios, AnalysisScope.CurrentBranchDiff, 4, undefined, undefined, undefined, 10);
|
|
402
416
|
expect(prompt).toContain("Budget: 4 generate");
|
|
403
417
|
});
|
|
404
418
|
it("clamps maxGenerateOverride to 0 when negative", () => {
|
|
405
|
-
const prompt = buildRecommendationPrompt(analysisWithScenarios,
|
|
419
|
+
const prompt = buildRecommendationPrompt(analysisWithScenarios, AnalysisScope.CurrentBranchDiff, 10, undefined, undefined, undefined, -5);
|
|
406
420
|
expect(prompt).toContain("Budget: 0 generate");
|
|
407
421
|
});
|
|
408
422
|
it("allows maxGenerateOverride of 0 to produce no generate items", () => {
|
|
409
|
-
const prompt = buildRecommendationPrompt(analysisWithScenarios,
|
|
423
|
+
const prompt = buildRecommendationPrompt(analysisWithScenarios, AnalysisScope.CurrentBranchDiff, 10, undefined, undefined, undefined, 0);
|
|
410
424
|
expect(prompt).toContain("Budget: 0 generate");
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
expect(prompt).toContain("Budget:
|
|
420
|
-
|
|
425
|
+
// The few-shot examples section contains "#1 — GENERATE" as an illustration,
|
|
426
|
+
// so check the execution plan's GENERATE section is empty instead.
|
|
427
|
+
expect(prompt).toContain("(no pre-ranked generate items");
|
|
428
|
+
});
|
|
429
|
+
it("uses topN as default maxGen in full_repo scope — Repo mode output format", () => {
|
|
430
|
+
const prompt = buildRecommendationPrompt(analysisWithScenarios, AnalysisScope.FullRepo, 6);
|
|
431
|
+
expect(prompt).toContain("Test Recommendations — 6 total (grouped by test type)");
|
|
432
|
+
expect(prompt).toContain("Repo mode");
|
|
433
|
+
expect(prompt).not.toContain("Budget:");
|
|
434
|
+
// Full-repo mode: no execution, just presentation
|
|
435
|
+
expect(prompt).toContain("no tests are executed");
|
|
436
|
+
expect(prompt).toContain("Present up to 6 recommendations.");
|
|
437
|
+
// Full-repo items must include tool calls for on-demand generation
|
|
438
|
+
expect(prompt).toContain("skyramp_contract_test_generation");
|
|
439
|
+
});
|
|
440
|
+
it("full_repo mode ignores maxGenerateOverride — always presents all", () => {
|
|
441
|
+
const prompt = buildRecommendationPrompt(analysisWithScenarios, AnalysisScope.FullRepo, 6, undefined, undefined, undefined, 2);
|
|
442
|
+
expect(prompt).toContain("Test Recommendations — 6 total (grouped by test type)");
|
|
443
|
+
expect(prompt).toContain("Repo mode");
|
|
444
|
+
expect(prompt).not.toContain("Budget:");
|
|
445
|
+
expect(prompt).toContain("no tests are executed");
|
|
446
|
+
});
|
|
447
|
+
it("full_repo mode includes tool calls in each recommendation item", () => {
|
|
448
|
+
const prompt = buildRecommendationPrompt(analysisWithScenarios, AnalysisScope.FullRepo, 6);
|
|
449
|
+
// Contract items should have tool calls
|
|
450
|
+
expect(prompt).toContain("skyramp_contract_test_generation");
|
|
451
|
+
// Integration items should use the batch tool
|
|
452
|
+
expect(prompt).toContain("skyramp_batch_scenario_test_generation");
|
|
453
|
+
// MANDATORY language for test type mix and item counts
|
|
454
|
+
expect(prompt).toContain("Test type mix");
|
|
455
|
+
});
|
|
456
|
+
it("full_repo mode uses MANDATORY language for item count and test type mix", () => {
|
|
457
|
+
const prompt = buildRecommendationPrompt(analysisWithScenarios, AnalysisScope.FullRepo, 6);
|
|
458
|
+
expect(prompt).toContain("Test type mix — MANDATORY");
|
|
459
|
+
expect(prompt).toContain("Present up to 6 recommendations.");
|
|
460
|
+
});
|
|
461
|
+
it("full_repo mode pre-allocates E2E and UI sections for full-stack repos", () => {
|
|
462
|
+
const fullStackAnalysis = minimalAnalysis({
|
|
463
|
+
projectClassification: {
|
|
464
|
+
projectType: "full-stack",
|
|
465
|
+
primaryLanguage: "TypeScript",
|
|
466
|
+
primaryFramework: "Express",
|
|
467
|
+
deploymentPattern: "traditional",
|
|
468
|
+
},
|
|
469
|
+
businessContext: {
|
|
470
|
+
mainPurpose: "Test API",
|
|
471
|
+
userFlows: [],
|
|
472
|
+
dataFlows: [],
|
|
473
|
+
integrationPatterns: [],
|
|
474
|
+
draftedScenarios: scenariosForOverride,
|
|
475
|
+
},
|
|
476
|
+
});
|
|
477
|
+
const prompt = buildRecommendationPrompt(fullStackAnalysis, AnalysisScope.FullRepo, 10);
|
|
478
|
+
// E2E and UI sections must be present even though scenarioDrafting only produces backend types
|
|
479
|
+
expect(prompt).toContain("### E2E");
|
|
480
|
+
expect(prompt).toContain("### UI");
|
|
481
|
+
expect(prompt).toContain("skyramp_e2e_test_generation");
|
|
482
|
+
expect(prompt).toContain("skyramp_ui_test_generation");
|
|
483
|
+
// Backend sections should still be present
|
|
484
|
+
expect(prompt).toContain("### Integration");
|
|
421
485
|
});
|
|
422
486
|
});
|
|
423
487
|
// ---------------------------------------------------------------------------
|
|
@@ -438,7 +502,7 @@ describe("buildRecommendationPrompt — additional recommendation dedup", () =>
|
|
|
438
502
|
chainingKeys: ["id"],
|
|
439
503
|
requiresAuth: false,
|
|
440
504
|
estimatedComplexity: "complex",
|
|
441
|
-
testType:
|
|
505
|
+
testType: TestType.INTEGRATION,
|
|
442
506
|
...overrides,
|
|
443
507
|
};
|
|
444
508
|
}
|
|
@@ -479,7 +543,7 @@ describe("buildRecommendationPrompt — additional recommendation dedup", () =>
|
|
|
479
543
|
patchOrdersScenario("orders-patch-another-variant"),
|
|
480
544
|
];
|
|
481
545
|
const analysis = analysisWithPatchScenarios(scenarios);
|
|
482
|
-
const prompt = buildRecommendationPrompt(analysis,
|
|
546
|
+
const prompt = buildRecommendationPrompt(analysis, AnalysisScope.CurrentBranchDiff, 10, undefined, undefined, undefined, 2);
|
|
483
547
|
// First 2 become GENERATE, the remaining share orders::integration → should be filtered
|
|
484
548
|
const additionalMatches = prompt.match(/#\d+ \[ADDITIONAL\]/g) || [];
|
|
485
549
|
const ordersPatchAdditional = (prompt.match(/\[ADDITIONAL\].*orders-patch/g) || []);
|
|
@@ -493,12 +557,12 @@ describe("buildRecommendationPrompt — additional recommendation dedup", () =>
|
|
|
493
557
|
// Contract test for same endpoint — different test type, should survive dedup
|
|
494
558
|
{
|
|
495
559
|
...patchOrdersScenario("orders-patch-contract"),
|
|
496
|
-
testType:
|
|
560
|
+
testType: TestType.CONTRACT,
|
|
497
561
|
steps: [{ order: 1, method: "PATCH", path: "/api/v1/orders/{order_id}", description: "Contract test", interactionType: "success", expectedStatusCode: 200 }],
|
|
498
562
|
},
|
|
499
563
|
];
|
|
500
564
|
const analysis = analysisWithPatchScenarios(scenarios);
|
|
501
|
-
const prompt = buildRecommendationPrompt(analysis,
|
|
565
|
+
const prompt = buildRecommendationPrompt(analysis, AnalysisScope.CurrentBranchDiff, 10, undefined, undefined, undefined, 2);
|
|
502
566
|
// Contract test targets orders but is a different type → should be in ADDITIONAL
|
|
503
567
|
expect(prompt).toContain("orders-patch-contract");
|
|
504
568
|
});
|
|
@@ -516,7 +580,7 @@ describe("buildRecommendationPrompt — additional recommendation dedup", () =>
|
|
|
516
580
|
},
|
|
517
581
|
];
|
|
518
582
|
const analysis = analysisWithPatchScenarios(scenarios);
|
|
519
|
-
const prompt = buildRecommendationPrompt(analysis,
|
|
583
|
+
const prompt = buildRecommendationPrompt(analysis, AnalysisScope.CurrentBranchDiff, 10, undefined, undefined, undefined, 2);
|
|
520
584
|
expect(prompt).toContain("products-unique-constraint");
|
|
521
585
|
});
|
|
522
586
|
});
|
|
@@ -533,7 +597,7 @@ describe("buildRecommendationPrompt — E2E slot guard (Fix 2)", () => {
|
|
|
533
597
|
chainingKeys: [],
|
|
534
598
|
requiresAuth: false,
|
|
535
599
|
estimatedComplexity: "simple",
|
|
536
|
-
testType:
|
|
600
|
+
testType: TestType.UI,
|
|
537
601
|
};
|
|
538
602
|
}
|
|
539
603
|
it("suppresses E2E additional slot when UI test is in GENERATE list", () => {
|
|
@@ -552,7 +616,7 @@ describe("buildRecommendationPrompt — E2E slot guard (Fix 2)", () => {
|
|
|
552
616
|
affectedServices: [],
|
|
553
617
|
},
|
|
554
618
|
});
|
|
555
|
-
const prompt = buildRecommendationPrompt(analysis,
|
|
619
|
+
const prompt = buildRecommendationPrompt(analysis, AnalysisScope.CurrentBranchDiff, 10, undefined, undefined, undefined, 3);
|
|
556
620
|
expect(prompt).not.toContain("e2e-flow-for-changed-feature");
|
|
557
621
|
});
|
|
558
622
|
it("includes E2E additional slot when no UI test is generated and no UI slot reserved", () => {
|
|
@@ -573,7 +637,7 @@ describe("buildRecommendationPrompt — E2E slot guard (Fix 2)", () => {
|
|
|
573
637
|
});
|
|
574
638
|
// maxGen=1 so reserveUIGenSlot is false (requires maxGen > 1),
|
|
575
639
|
// and no UI scenario in GENERATE → E2E slot should appear
|
|
576
|
-
const prompt = buildRecommendationPrompt(analysis,
|
|
640
|
+
const prompt = buildRecommendationPrompt(analysis, AnalysisScope.CurrentBranchDiff, 10, undefined, undefined, undefined, 1);
|
|
577
641
|
expect(prompt).toContain("e2e-flow-for-changed-feature");
|
|
578
642
|
});
|
|
579
643
|
});
|
|
@@ -602,7 +666,335 @@ describe("buildTestQualityCriteria — contract test guidance for error-handling
|
|
|
602
666
|
draftedScenarios: [minimalScenario()],
|
|
603
667
|
},
|
|
604
668
|
});
|
|
605
|
-
const prompt = buildRecommendationPrompt(analysis,
|
|
669
|
+
const prompt = buildRecommendationPrompt(analysis, AnalysisScope.FullRepo, 10);
|
|
606
670
|
expect(prompt).toContain("Do NOT add setup steps just to avoid hardcoding an ID");
|
|
607
671
|
});
|
|
608
672
|
});
|
|
673
|
+
// ---------------------------------------------------------------------------
|
|
674
|
+
// Tests — Architect Persona Preamble
|
|
675
|
+
// ---------------------------------------------------------------------------
|
|
676
|
+
describe("buildRecommendationPrompt — Architect Persona Preamble", () => {
|
|
677
|
+
it("includes Skyramp Integration Architect persona at the beginning of the prompt", () => {
|
|
678
|
+
const prompt = buildRecommendationPrompt(minimalAnalysis());
|
|
679
|
+
expect(prompt).toContain("Skyramp Integration Architect");
|
|
680
|
+
});
|
|
681
|
+
it("persona appears before the mode preamble", () => {
|
|
682
|
+
const prompt = buildRecommendationPrompt(minimalAnalysis());
|
|
683
|
+
const architectIdx = prompt.indexOf("Skyramp Integration Architect");
|
|
684
|
+
const modeIdx = prompt.indexOf("Repo mode");
|
|
685
|
+
expect(architectIdx).toBeLessThan(modeIdx);
|
|
686
|
+
expect(architectIdx).toBeGreaterThan(-1);
|
|
687
|
+
});
|
|
688
|
+
it("persona instructs not to guess or fabricate values", () => {
|
|
689
|
+
const prompt = buildRecommendationPrompt(minimalAnalysis());
|
|
690
|
+
expect(prompt).toContain("No guessing");
|
|
691
|
+
expect(prompt).toContain("derive all parameters");
|
|
692
|
+
});
|
|
693
|
+
it("buildArchitectPreamble(diff) returns the diff persona", () => {
|
|
694
|
+
const preamble = buildArchitectPreamble(true);
|
|
695
|
+
expect(preamble).toContain("Skyramp Integration Architect");
|
|
696
|
+
expect(preamble).toContain("generate tests for this PR");
|
|
697
|
+
expect(preamble).toContain("duplicate coverage");
|
|
698
|
+
});
|
|
699
|
+
it("buildArchitectPreamble(full_repo) returns the full-repo persona", () => {
|
|
700
|
+
const preamble = buildArchitectPreamble(false);
|
|
701
|
+
expect(preamble).toContain("Skyramp Integration Architect");
|
|
702
|
+
expect(preamble).toContain("comprehensive test recommendation catalog");
|
|
703
|
+
expect(preamble).toContain("Do not call any generation tools");
|
|
704
|
+
});
|
|
705
|
+
});
|
|
706
|
+
// ---------------------------------------------------------------------------
|
|
707
|
+
// Tests — Mandatory Reasoning Protocol
|
|
708
|
+
// ---------------------------------------------------------------------------
|
|
709
|
+
describe("buildRecommendationPrompt — Mandatory Reasoning Protocol", () => {
|
|
710
|
+
it("includes reasoning protocol section in the prompt", () => {
|
|
711
|
+
const analysis = minimalAnalysis({
|
|
712
|
+
businessContext: {
|
|
713
|
+
mainPurpose: "Test API",
|
|
714
|
+
userFlows: [],
|
|
715
|
+
dataFlows: [],
|
|
716
|
+
integrationPatterns: [],
|
|
717
|
+
draftedScenarios: [minimalScenario()],
|
|
718
|
+
},
|
|
719
|
+
});
|
|
720
|
+
const prompt = buildRecommendationPrompt(analysis, AnalysisScope.FullRepo, 10);
|
|
721
|
+
expect(prompt).toContain("Parameter Grounding Rule");
|
|
722
|
+
});
|
|
723
|
+
it("reasoning protocol covers the key parameter sources", () => {
|
|
724
|
+
const protocol = buildReasoningProtocol();
|
|
725
|
+
expect(protocol).toContain("requestBody");
|
|
726
|
+
expect(protocol).toContain("endpointURL");
|
|
727
|
+
expect(protocol).toContain("authHeader");
|
|
728
|
+
expect(protocol).toContain("FK path params");
|
|
729
|
+
});
|
|
730
|
+
it("reasoning protocol instructs to read source file when value cannot be sourced", () => {
|
|
731
|
+
const protocol = buildReasoningProtocol();
|
|
732
|
+
expect(protocol).toContain("read the relevant source file");
|
|
733
|
+
});
|
|
734
|
+
});
|
|
735
|
+
// ---------------------------------------------------------------------------
|
|
736
|
+
// Tests — Context Fetching Guidance
|
|
737
|
+
// ---------------------------------------------------------------------------
|
|
738
|
+
describe("buildRecommendationPrompt — Context Fetching Guidance", () => {
|
|
739
|
+
it("includes execution plan context when sessionId is provided", () => {
|
|
740
|
+
const prompt = buildRecommendationPrompt(minimalAnalysis(), AnalysisScope.FullRepo, 10, undefined, undefined, undefined, undefined, "test-session-123");
|
|
741
|
+
expect(prompt).toContain("Execution Plan Context");
|
|
742
|
+
expect(prompt).toContain("relevant source file");
|
|
743
|
+
});
|
|
744
|
+
it("omits execution plan context when sessionId is not provided", () => {
|
|
745
|
+
const prompt = buildRecommendationPrompt(minimalAnalysis());
|
|
746
|
+
expect(prompt).not.toContain("Execution Plan Context");
|
|
747
|
+
});
|
|
748
|
+
it("buildContextFetchingGuidance returns empty string when no sessionId", () => {
|
|
749
|
+
expect(buildContextFetchingGuidance()).toBe("");
|
|
750
|
+
expect(buildContextFetchingGuidance(undefined)).toBe("");
|
|
751
|
+
});
|
|
752
|
+
it("buildContextFetchingGuidance references enriched scenarios and source files", () => {
|
|
753
|
+
const guidance = buildContextFetchingGuidance("session-1");
|
|
754
|
+
expect(guidance).toContain("placeholder");
|
|
755
|
+
expect(guidance).toContain("relevant source file");
|
|
756
|
+
});
|
|
757
|
+
});
|
|
758
|
+
// ---------------------------------------------------------------------------
|
|
759
|
+
// Tests — Tool Contract Framing
|
|
760
|
+
// ---------------------------------------------------------------------------
|
|
761
|
+
describe("buildRecommendationPrompt — Tool Contract Framing", () => {
|
|
762
|
+
it("includes contract language in tool workflows section (PR mode only)", () => {
|
|
763
|
+
const prompt = buildRecommendationPrompt(minimalAnalysis(), AnalysisScope.CurrentBranchDiff);
|
|
764
|
+
expect(prompt).toContain("strict technical contracts");
|
|
765
|
+
});
|
|
766
|
+
it("includes thinking block reminder in tool workflows (PR mode only)", () => {
|
|
767
|
+
const prompt = buildRecommendationPrompt(minimalAnalysis(), AnalysisScope.CurrentBranchDiff);
|
|
768
|
+
expect(prompt).toContain("Output a <thinking> block");
|
|
769
|
+
});
|
|
770
|
+
});
|
|
771
|
+
// ---------------------------------------------------------------------------
|
|
772
|
+
// Tests — Long-context best practices: XML tags structure
|
|
773
|
+
// ---------------------------------------------------------------------------
|
|
774
|
+
describe("buildRecommendationPrompt — XML tag structure (long-context best practice)", () => {
|
|
775
|
+
it("wraps repository context in <repository_context> tags", () => {
|
|
776
|
+
const prompt = buildRecommendationPrompt(minimalAnalysis());
|
|
777
|
+
expect(prompt).toContain("<repository_context>");
|
|
778
|
+
expect(prompt).toContain("</repository_context>");
|
|
779
|
+
});
|
|
780
|
+
it("wraps endpoint interactions in <endpoint_interactions> tags", () => {
|
|
781
|
+
const prompt = buildRecommendationPrompt(minimalAnalysis());
|
|
782
|
+
expect(prompt).toContain("<endpoint_interactions>");
|
|
783
|
+
expect(prompt).toContain("</endpoint_interactions>");
|
|
784
|
+
});
|
|
785
|
+
it("wraps existing tests in <existing_tests> tags", () => {
|
|
786
|
+
const prompt = buildRecommendationPrompt(minimalAnalysis());
|
|
787
|
+
expect(prompt).toContain("<existing_tests>");
|
|
788
|
+
expect(prompt).toContain("</existing_tests>");
|
|
789
|
+
});
|
|
790
|
+
it("wraps instructions in <instructions> tags", () => {
|
|
791
|
+
const prompt = buildRecommendationPrompt(minimalAnalysis());
|
|
792
|
+
expect(prompt).toContain("<instructions>");
|
|
793
|
+
expect(prompt).toContain("</instructions>");
|
|
794
|
+
});
|
|
795
|
+
it("wraps branch diff in <branch_diff> tags when diff context is present", () => {
|
|
796
|
+
const analysis = minimalAnalysis({
|
|
797
|
+
branchDiffContext: {
|
|
798
|
+
baseBranch: "main",
|
|
799
|
+
currentBranch: "feature/test",
|
|
800
|
+
changedFiles: ["src/routes.ts"],
|
|
801
|
+
newEndpoints: [{ path: "/api/new", methods: [{ method: "POST", sourceFile: "routes.ts", interactionCount: 0 }] }],
|
|
802
|
+
modifiedEndpoints: [],
|
|
803
|
+
affectedServices: [],
|
|
804
|
+
},
|
|
805
|
+
});
|
|
806
|
+
const prompt = buildRecommendationPrompt(analysis, AnalysisScope.CurrentBranchDiff, 10);
|
|
807
|
+
expect(prompt).toContain("<branch_diff>");
|
|
808
|
+
expect(prompt).toContain("</branch_diff>");
|
|
809
|
+
});
|
|
810
|
+
it("wraps PR history in <pr_history> tags when present", () => {
|
|
811
|
+
const ctx = makePRContext({
|
|
812
|
+
prNumber: 42,
|
|
813
|
+
previousRecommendations: [
|
|
814
|
+
{ testType: "contract", endpoint: "GET /api/items", status: "implemented", commentId: "1" },
|
|
815
|
+
],
|
|
816
|
+
});
|
|
817
|
+
const prompt = buildRecommendationPrompt(minimalAnalysis(), AnalysisScope.CurrentBranchDiff, 10, ctx);
|
|
818
|
+
expect(prompt).toContain("<pr_history>");
|
|
819
|
+
expect(prompt).toContain("</pr_history>");
|
|
820
|
+
});
|
|
821
|
+
});
|
|
822
|
+
// ---------------------------------------------------------------------------
|
|
823
|
+
// Tests — Long-context best practice: data before instructions
|
|
824
|
+
// ---------------------------------------------------------------------------
|
|
825
|
+
describe("buildRecommendationPrompt — data-before-instructions ordering", () => {
|
|
826
|
+
it("places repository_context before instructions", () => {
|
|
827
|
+
const prompt = buildRecommendationPrompt(minimalAnalysis());
|
|
828
|
+
const dataIdx = prompt.indexOf("<repository_context>");
|
|
829
|
+
const instructionsIdx = prompt.indexOf("<instructions>");
|
|
830
|
+
expect(dataIdx).toBeLessThan(instructionsIdx);
|
|
831
|
+
});
|
|
832
|
+
it("places existing_tests before instructions", () => {
|
|
833
|
+
const prompt = buildRecommendationPrompt(minimalAnalysis());
|
|
834
|
+
const testsIdx = prompt.indexOf("<existing_tests>");
|
|
835
|
+
const instructionsIdx = prompt.indexOf("<instructions>");
|
|
836
|
+
expect(testsIdx).toBeLessThan(instructionsIdx);
|
|
837
|
+
});
|
|
838
|
+
it("places endpoint_interactions before instructions", () => {
|
|
839
|
+
const prompt = buildRecommendationPrompt(minimalAnalysis());
|
|
840
|
+
const interactionsIdx = prompt.indexOf("<endpoint_interactions>");
|
|
841
|
+
const instructionsIdx = prompt.indexOf("<instructions>");
|
|
842
|
+
expect(interactionsIdx).toBeLessThan(instructionsIdx);
|
|
843
|
+
});
|
|
844
|
+
});
|
|
845
|
+
// ---------------------------------------------------------------------------
|
|
846
|
+
// Tests — Few-shot examples
|
|
847
|
+
// ---------------------------------------------------------------------------
|
|
848
|
+
describe("buildFewShotExamples — concrete few-shot examples with thinking", () => {
|
|
849
|
+
it("returns examples wrapped in <examples> tags", () => {
|
|
850
|
+
const examples = buildFewShotExamples();
|
|
851
|
+
expect(examples).toContain("<examples>");
|
|
852
|
+
expect(examples).toContain("</examples>");
|
|
853
|
+
});
|
|
854
|
+
it("contains at least 2 example blocks", () => {
|
|
855
|
+
const examples = buildFewShotExamples();
|
|
856
|
+
const count = (examples.match(/<example /g) || []).length;
|
|
857
|
+
expect(count).toBeGreaterThanOrEqual(2);
|
|
858
|
+
});
|
|
859
|
+
it("includes <thinking> blocks inside examples to demonstrate reasoning", () => {
|
|
860
|
+
const examples = buildFewShotExamples();
|
|
861
|
+
expect(examples).toContain("<thinking>");
|
|
862
|
+
expect(examples).toContain("</thinking>");
|
|
863
|
+
});
|
|
864
|
+
it("includes an integration test example with tool calls", () => {
|
|
865
|
+
const examples = buildFewShotExamples();
|
|
866
|
+
expect(examples).toContain("skyramp_batch_scenario_test_generation");
|
|
867
|
+
expect(examples).toContain("skyramp_integration_test_generation");
|
|
868
|
+
});
|
|
869
|
+
it("includes a contract test example", () => {
|
|
870
|
+
const examples = buildFewShotExamples();
|
|
871
|
+
expect(examples).toContain("skyramp_contract_test_generation");
|
|
872
|
+
});
|
|
873
|
+
it("includes an ADDITIONAL example", () => {
|
|
874
|
+
const examples = buildFewShotExamples();
|
|
875
|
+
expect(examples).toContain("[ADDITIONAL]");
|
|
876
|
+
});
|
|
877
|
+
it("thinking blocks demonstrate parameter grounding", () => {
|
|
878
|
+
const examples = buildFewShotExamples();
|
|
879
|
+
expect(examples).toContain("**Parameter grounding**:");
|
|
880
|
+
expect(examples).toContain("workspace");
|
|
881
|
+
expect(examples).toContain("chained from");
|
|
882
|
+
});
|
|
883
|
+
it("integration example uses filePath from batch response, not a guessed filename", () => {
|
|
884
|
+
const examples = buildFewShotExamples();
|
|
885
|
+
expect(examples).toContain("<filePath returned by skyramp_batch_scenario_test_generation");
|
|
886
|
+
expect(examples).not.toContain('scenarioFile: "scenario_');
|
|
887
|
+
});
|
|
888
|
+
it("integration example includes bugCatchingTarget", () => {
|
|
889
|
+
const examples = buildFewShotExamples();
|
|
890
|
+
expect(examples).toContain("bugCatchingTarget:");
|
|
891
|
+
});
|
|
892
|
+
it("is included in the recommendation prompt (PR mode only)", () => {
|
|
893
|
+
const prompt = buildRecommendationPrompt(minimalAnalysis(), AnalysisScope.CurrentBranchDiff);
|
|
894
|
+
expect(prompt).toContain("<examples>");
|
|
895
|
+
});
|
|
896
|
+
});
|
|
897
|
+
// ---------------------------------------------------------------------------
|
|
898
|
+
// Tests — Verification checklist
|
|
899
|
+
// ---------------------------------------------------------------------------
|
|
900
|
+
describe("buildVerificationChecklist — self-check at end of prompt", () => {
|
|
901
|
+
it("returns checklist wrapped in <verification> tags", () => {
|
|
902
|
+
const checklist = buildVerificationChecklist(10, 3);
|
|
903
|
+
expect(checklist).toContain("<verification>");
|
|
904
|
+
expect(checklist).toContain("</verification>");
|
|
905
|
+
});
|
|
906
|
+
it("includes total count check with correct numbers", () => {
|
|
907
|
+
const checklist = buildVerificationChecklist(10, 3);
|
|
908
|
+
expect(checklist).toContain("equals exactly 10");
|
|
909
|
+
expect(checklist).toContain("3 GENERATE + 7 ADDITIONAL");
|
|
910
|
+
});
|
|
911
|
+
it("includes scenarioFile source check", () => {
|
|
912
|
+
const checklist = buildVerificationChecklist(10, 3);
|
|
913
|
+
expect(checklist).toContain("filePath");
|
|
914
|
+
expect(checklist).toContain("skyramp_batch_scenario_test_generation");
|
|
915
|
+
});
|
|
916
|
+
it("includes bugCatchingTarget check", () => {
|
|
917
|
+
const checklist = buildVerificationChecklist(10, 3);
|
|
918
|
+
expect(checklist).toContain("bugCatchingTarget");
|
|
919
|
+
});
|
|
920
|
+
it("includes distinct code path check", () => {
|
|
921
|
+
const checklist = buildVerificationChecklist(10, 3);
|
|
922
|
+
expect(checklist).toContain("distinct code path");
|
|
923
|
+
});
|
|
924
|
+
it("includes auth consistency check", () => {
|
|
925
|
+
const checklist = buildVerificationChecklist(10, 3);
|
|
926
|
+
expect(checklist).toContain("Auth parameters are consistent");
|
|
927
|
+
});
|
|
928
|
+
it("includes endpointURL format check", () => {
|
|
929
|
+
const checklist = buildVerificationChecklist(10, 3);
|
|
930
|
+
expect(checklist).toContain("base URL and the path");
|
|
931
|
+
});
|
|
932
|
+
it("includes placeholder removal check", () => {
|
|
933
|
+
const checklist = buildVerificationChecklist(10, 3);
|
|
934
|
+
expect(checklist).toContain("from source");
|
|
935
|
+
});
|
|
936
|
+
it("is included in the recommendation prompt (PR mode only)", () => {
|
|
937
|
+
const prompt = buildRecommendationPrompt(minimalAnalysis(), AnalysisScope.CurrentBranchDiff);
|
|
938
|
+
expect(prompt).toContain("<verification>");
|
|
939
|
+
});
|
|
940
|
+
});
|
|
941
|
+
// ---------------------------------------------------------------------------
|
|
942
|
+
// Tests — Source evidence grounding
|
|
943
|
+
// ---------------------------------------------------------------------------
|
|
944
|
+
describe("buildRecommendationPrompt — source evidence grounding", () => {
|
|
945
|
+
it("includes source_evidence instruction in repo-mode enrichment", () => {
|
|
946
|
+
const analysis = minimalAnalysis({
|
|
947
|
+
businessContext: {
|
|
948
|
+
mainPurpose: "Test API",
|
|
949
|
+
userFlows: [],
|
|
950
|
+
dataFlows: [],
|
|
951
|
+
integrationPatterns: [],
|
|
952
|
+
draftedScenarios: [minimalScenario()],
|
|
953
|
+
},
|
|
954
|
+
});
|
|
955
|
+
const prompt = buildRecommendationPrompt(analysis, AnalysisScope.FullRepo, 10);
|
|
956
|
+
expect(prompt).toContain("Test Recommendations");
|
|
957
|
+
});
|
|
958
|
+
it("includes source_evidence instruction in PR-mode enrichment", () => {
|
|
959
|
+
const analysis = minimalAnalysis({
|
|
960
|
+
businessContext: {
|
|
961
|
+
mainPurpose: "Test API",
|
|
962
|
+
userFlows: [],
|
|
963
|
+
dataFlows: [],
|
|
964
|
+
integrationPatterns: [],
|
|
965
|
+
draftedScenarios: [minimalScenario()],
|
|
966
|
+
},
|
|
967
|
+
});
|
|
968
|
+
const prompt = buildRecommendationPrompt(analysis, AnalysisScope.CurrentBranchDiff, 10);
|
|
969
|
+
expect(prompt).toContain("Execution Plan");
|
|
970
|
+
});
|
|
971
|
+
});
|
|
972
|
+
// ---------------------------------------------------------------------------
|
|
973
|
+
// Tests — Over-prompting reduction
|
|
974
|
+
// ---------------------------------------------------------------------------
|
|
975
|
+
describe("buildRecommendationPrompt — reduced over-prompting", () => {
|
|
976
|
+
it("uses softer language for step labels (no MANDATORY)", () => {
|
|
977
|
+
const analysis = minimalAnalysis({
|
|
978
|
+
businessContext: {
|
|
979
|
+
mainPurpose: "Test API",
|
|
980
|
+
userFlows: [],
|
|
981
|
+
dataFlows: [],
|
|
982
|
+
integrationPatterns: [],
|
|
983
|
+
draftedScenarios: [minimalScenario()],
|
|
984
|
+
},
|
|
985
|
+
});
|
|
986
|
+
const prompt = buildRecommendationPrompt(analysis, AnalysisScope.CurrentBranchDiff, 10);
|
|
987
|
+
expect(prompt).not.toContain("(MANDATORY before executing anything)");
|
|
988
|
+
expect(prompt).toContain("(before executing anything)");
|
|
989
|
+
});
|
|
990
|
+
it("uses XML tags in context fetching guidance", () => {
|
|
991
|
+
const guidance = buildContextFetchingGuidance("session-1");
|
|
992
|
+
expect(guidance).toContain("<context_fetching_protocol>");
|
|
993
|
+
expect(guidance).toContain("</context_fetching_protocol>");
|
|
994
|
+
});
|
|
995
|
+
it("uses XML tags in reasoning protocol", () => {
|
|
996
|
+
const protocol = buildReasoningProtocol();
|
|
997
|
+
expect(protocol).toContain("<reasoning_protocol>");
|
|
998
|
+
expect(protocol).toContain("</reasoning_protocol>");
|
|
999
|
+
});
|
|
1000
|
+
});
|