@skyramp/mcp 0.1.5 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/index.js +6 -5
- package/build/prompts/initialize-workspace/initializeWorkspacePrompt.js +150 -149
- package/build/prompts/personas.js +2 -1
- package/build/prompts/test-maintenance/drift-analysis-prompt.js +2 -1
- package/build/prompts/test-maintenance/drift-analysis-prompt.test.js +28 -0
- package/build/prompts/test-recommendation/analysisOutputPrompt.js +72 -14
- package/build/prompts/test-recommendation/analysisOutputPrompt.test.js +154 -0
- package/build/prompts/test-recommendation/diffExecutionPlan.js +290 -0
- package/build/prompts/test-recommendation/fullRepoCatalog.js +271 -0
- package/build/prompts/test-recommendation/recommendationSections.js +4 -2
- package/build/prompts/test-recommendation/recommendationShared.js +68 -0
- package/build/prompts/test-recommendation/registerRecommendTestsPrompt.js +20 -4
- package/build/prompts/test-recommendation/test-recommendation-prompt.js +11 -640
- package/build/prompts/test-recommendation/test-recommendation-prompt.test.js +6 -6
- package/build/prompts/testbot/testbot-prompts.js +19 -7
- package/build/prompts/testbot/testbot-prompts.test.js +22 -5
- package/build/resources/analysisResources.js +1 -0
- package/build/services/ScenarioGenerationService.js +5 -1
- package/build/services/TestGenerationService.js +3 -0
- package/build/tools/code-refactor/codeReuseTool.js +3 -0
- package/build/tools/code-refactor/enhanceAssertionsTool.js +5 -1
- package/build/tools/code-refactor/modularizationTool.js +3 -0
- package/build/tools/generate-tests/generateBatchScenarioRestTool.js +123 -1
- package/build/tools/generate-tests/generateBatchScenarioRestTool.test.js +205 -9
- package/build/tools/generate-tests/generateContractRestTool.js +19 -19
- package/build/tools/generate-tests/generateIntegrationRestTool.js +9 -2
- package/build/tools/generate-tests/generateUIRestTool.js +23 -8
- package/build/tools/test-management/analyzeChangesTool.js +218 -2
- package/build/tools/test-management/analyzeChangesTool.test.js +233 -1
- package/build/tools/workspace/initializeWorkspaceTool.js +1 -1
- package/build/utils/docker.test.js +1 -1
- package/build/utils/featureFlags.js +7 -0
- package/build/utils/featureFlags.test.js +81 -0
- package/build/utils/gitStaging.js +18 -0
- package/build/utils/gitStaging.test.js +87 -0
- package/build/utils/httpDefaults.js +17 -0
- package/build/utils/httpDefaults.test.js +21 -0
- package/build/utils/scenarioDrafting.js +37 -15
- package/build/utils/scenarioDrafting.test.js +66 -0
- package/build/utils/telemetry.js +2 -1
- package/build/utils/utils.js +23 -0
- package/build/utils/versions.js +1 -1
- package/node_modules/playwright/lib/mcp/browser/context.js +2 -0
- package/node_modules/playwright/lib/mcp/browser/tools/keyboard.js +2 -2
- package/node_modules/playwright/lib/mcp/skyramp/traceRecordingBackend.js +17 -26
- package/package.json +2 -2
|
@@ -934,9 +934,9 @@ describe("buildRecommendationPrompt — multi-method endpoint partitioning", ()
|
|
|
934
934
|
});
|
|
935
935
|
const prompt = buildRecommendationPrompt(analysis, AnalysisScope.CurrentBranchDiff, 10);
|
|
936
936
|
// Both GET and POST for /api/products should be in "Changed in this PR"
|
|
937
|
-
expect(prompt).toContain("
|
|
938
|
-
expect(prompt).toMatch(/
|
|
939
|
-
expect(prompt).toMatch(/
|
|
937
|
+
expect(prompt).toContain("Likely changed in this PR");
|
|
938
|
+
expect(prompt).toMatch(/Likely changed in this PR[\s\S]*GET \/api\/products/);
|
|
939
|
+
expect(prompt).toMatch(/Likely changed in this PR[\s\S]*POST \/api\/products/);
|
|
940
940
|
// /api/items should NOT be in changed section
|
|
941
941
|
expect(prompt).toMatch(/Other endpoints[\s\S]*GET \/api\/items/);
|
|
942
942
|
});
|
|
@@ -983,8 +983,8 @@ describe("buildRecommendationPrompt — multi-method endpoint partitioning", ()
|
|
|
983
983
|
});
|
|
984
984
|
const prompt = buildRecommendationPrompt(analysis, AnalysisScope.CurrentBranchDiff, 10);
|
|
985
985
|
// Both products and orders should be in changed section
|
|
986
|
-
expect(prompt).toMatch(/
|
|
987
|
-
expect(prompt).toMatch(/
|
|
986
|
+
expect(prompt).toMatch(/Likely changed in this PR[\s\S]*GET \/api\/products/);
|
|
987
|
+
expect(prompt).toMatch(/Likely changed in this PR[\s\S]*POST \/api\/orders/);
|
|
988
988
|
});
|
|
989
989
|
});
|
|
990
990
|
// ---------------------------------------------------------------------------
|
|
@@ -1021,7 +1021,7 @@ describe("buildRecommendationPrompt — removed endpoint listing", () => {
|
|
|
1021
1021
|
});
|
|
1022
1022
|
const prompt = buildRecommendationPrompt(analysis, AnalysisScope.CurrentBranchDiff, 10);
|
|
1023
1023
|
expect(prompt).toContain("DELETE /api/legacy [removed]");
|
|
1024
|
-
expect(prompt).toContain("
|
|
1024
|
+
expect(prompt).toContain("Likely changed in this PR");
|
|
1025
1025
|
});
|
|
1026
1026
|
});
|
|
1027
1027
|
// ---------------------------------------------------------------------------
|
|
@@ -5,9 +5,11 @@ import { MAX_TESTS_TO_GENERATE, MAX_RECOMMENDATIONS, MAX_CRITICAL_TESTS, PATH_PA
|
|
|
5
5
|
import { buildDriftAnalysisPrompt } from "../test-maintenance/drift-analysis-prompt.js";
|
|
6
6
|
import { getTraceRecordingPromptText } from "../../playwright/traceRecordingPrompt.js";
|
|
7
7
|
import { isContractConsumerModeEnabled } from "../../utils/featureFlags.js";
|
|
8
|
+
import { resolveServiceDetailsRef } from "../../utils/utils.js";
|
|
8
9
|
import { readWorkspaceConfigRaw } from "../../utils/workspaceAuth.js";
|
|
9
|
-
// Cached at module-load —
|
|
10
|
+
// Cached at module-load — flags are process-wide and cannot change per call.
|
|
10
11
|
const CONSUMER_MODE_ENABLED = isContractConsumerModeEnabled();
|
|
12
|
+
const SERVICE_REFS = resolveServiceDetailsRef();
|
|
11
13
|
// Mode-aware bullet block that appears inside the "How to generate each type"
|
|
12
14
|
// section. When consumer mode is disabled, only provider-mode guidance is
|
|
13
15
|
// surfaced so the agent never recommends or invokes consumer contract tests.
|
|
@@ -121,8 +123,8 @@ ${userPrompt ? "Generate only the tests that the user requested from the Additio
|
|
|
121
123
|
- Critical-category tests are already ranked first by the pre-computed scores — follow the plan order.
|
|
122
124
|
|
|
123
125
|
**Auth — determine ONCE, apply to EVERY tool call:**
|
|
124
|
-
1. Read auth params from the Execution Plan returned by \`skyramp_analyze_changes\` — they are resolved
|
|
125
|
-
2. If workspace shows \`authType: none\` or \`authHeader: ""\` → proceed with no auth (\`authHeader: ""\`). If tests fail due to 401/403, add to \`issuesFound\`: "Auth may be required — update \`api.authType\` in
|
|
126
|
+
1. Read auth params from the Execution Plan returned by \`skyramp_analyze_changes\` — they are pre-resolved from ${SERVICE_REFS.authSourceRef}. **Use these as-is; do not infer or override.**
|
|
127
|
+
2. If workspace shows \`authType: none\` or \`authHeader: ""\` → proceed with no auth (\`authHeader: ""\`). If tests fail due to 401/403, add to \`issuesFound\`: "Auth may be required — update \`api.authType\` in ${SERVICE_REFS.authSourceRef}."
|
|
126
128
|
3. **Auth params by header type — quick reference:**
|
|
127
129
|
|
|
128
130
|
| \`authHeader\` | \`authType\` examples | \`skyramp_batch_scenario_*\` / \`skyramp_contract_*\` | \`skyramp_integration_test_generation\` (scenarioFile) |
|
|
@@ -133,7 +135,7 @@ ${userPrompt ? "Generate only the tests that the user requested from the Additio
|
|
|
133
135
|
| none / \`""\` | \`none\` | \`authHeader: ""\` only when endpoint confirmed unauthenticated | \`authHeader: ""\` |
|
|
134
136
|
|
|
135
137
|
**Omit \`authToken\` entirely** — \`SKYRAMP_PLACEHOLDER_TOKEN\` is auto-inserted at execution time.
|
|
136
|
-
The \`authScheme\` for \`Authorization\` headers is pre-resolved in the Execution Plan — use it exactly (e.g. \`"Bearer"\`, \`"Token"\`, or a custom scheme from
|
|
138
|
+
The \`authScheme\` for \`Authorization\` headers is pre-resolved in the Execution Plan — use it exactly (e.g. \`"Bearer"\`, \`"Token"\`, or a custom scheme from ${SERVICE_REFS.authSourceRef}).
|
|
137
139
|
|
|
138
140
|
Passing auth alongside workspace \`authType\` on \`skyramp_integration_test_generation\` causes "${AUTH_CONFLICT_ERROR_MSG}" — follow the table.
|
|
139
141
|
4. Only pass \`authHeader: ""\` if you can confirm the endpoint is truly unauthenticated.
|
|
@@ -141,7 +143,7 @@ ${userPrompt ? "Generate only the tests that the user requested from the Additio
|
|
|
141
143
|
**How to generate each type (for ADD):**
|
|
142
144
|
- **Integration**: call \`skyramp_batch_scenario_test_generation\` with ALL steps in a single call (pass the \`steps\` array with method, path, requestBody, statusCode for each step). Then call \`skyramp_integration_test_generation\` with the returned scenario file.
|
|
143
145
|
**Use the pre-built scenario JSON from the Execution Plan** — pass the steps array directly. Do NOT read source code models to construct request bodies if the plan already provides them.
|
|
144
|
-
Scenario JSON and test files go in
|
|
146
|
+
Scenario JSON and test files go in ${SERVICE_REFS.testDirRef}. Do NOT create a new \`tests/\` directory at the repo root — use that path. If no \`testDirectory\` is configured, default to the language-conventional location (e.g. \`src/test/java/...\` for Java, \`tests/\` for Python).
|
|
145
147
|
**Pipeline for speed**: Call ALL \`skyramp_batch_scenario_test_generation\` calls in one batch. When they return, call ALL \`skyramp_integration_test_generation\` calls in the next batch. Do NOT serialize per-scenario (batch→integration→batch→integration) — batch ALL scenarios first, then generate ALL integration tests.
|
|
146
148
|
- **Contract**: call \`skyramp_contract_test_generation\` with \`endpointURL\`, \`method\`, and \`requestData\` for POST/PUT/PATCH.
|
|
147
149
|
Pass \`apiSchema\` if an OpenAPI spec exists.
|
|
@@ -289,7 +291,7 @@ function buildServiceContext(services) {
|
|
|
289
291
|
if (svc.api?.baseUrl)
|
|
290
292
|
parts.push(` <base_url>${escapeXml(svc.api.baseUrl)}</base_url>`);
|
|
291
293
|
if (svc.testDirectory)
|
|
292
|
-
parts.push(` <
|
|
294
|
+
parts.push(` <test_directory>${escapeXml(svc.testDirectory)}</test_directory>`);
|
|
293
295
|
parts.push('</service>');
|
|
294
296
|
return parts.join('\n');
|
|
295
297
|
});
|
|
@@ -303,6 +305,9 @@ export async function readWorkspaceServices(repositoryPath) {
|
|
|
303
305
|
const rawConfig = await readWorkspaceConfigRaw(repositoryPath);
|
|
304
306
|
return (rawConfig?.services ?? []);
|
|
305
307
|
}
|
|
308
|
+
export function buildWorkspaceRecoveryPrefix(repositoryPath) {
|
|
309
|
+
return `IMPORTANT: The existing .skyramp/workspace.yml failed to parse or validate. Before proceeding with any tasks below, you MUST call skyramp_init_scan with workspacePath "${repositoryPath}" and force: true, then call skyramp_init_workspace with workspacePath "${repositoryPath}", the discovered services, scanToken, and force: true to regenerate the workspace file.\n\n`;
|
|
310
|
+
}
|
|
306
311
|
export function registerTestbotPrompt(server) {
|
|
307
312
|
logger.info("Registering testbot prompt");
|
|
308
313
|
server.registerPrompt("skyramp_testbot", {
|
|
@@ -349,10 +354,17 @@ export function registerTestbotPrompt(server) {
|
|
|
349
354
|
.string()
|
|
350
355
|
.optional()
|
|
351
356
|
.describe("Browser login credentials for UI test recording (format: 'username:password', one per line). Injected into the prompt as a <ui-credentials> block so the agent logs in before recording traces."),
|
|
357
|
+
workspaceValidationFailed: z
|
|
358
|
+
.boolean()
|
|
359
|
+
.default(false)
|
|
360
|
+
.describe("Set to true when the testbot detected that .skyramp/workspace.yml exists but failed schema validation. Instructs the agent to regenerate the workspace file before proceeding."),
|
|
352
361
|
},
|
|
353
362
|
}, async (args) => {
|
|
354
363
|
const services = await readWorkspaceServices(args.repositoryPath);
|
|
355
|
-
|
|
364
|
+
let prompt = getTestbotPrompt(args.prTitle, args.prDescription, args.summaryOutputFile, args.repositoryPath, args.baseBranch, args.maxRecommendations, args.maxGenerate, args.maxCritical, args.prNumber, args.userPrompt, services.length ? services : undefined, args.stateOutputFile, args.uiCredentials);
|
|
365
|
+
if (args.workspaceValidationFailed) {
|
|
366
|
+
prompt = buildWorkspaceRecoveryPrefix(args.repositoryPath) + prompt;
|
|
367
|
+
}
|
|
356
368
|
AnalyticsService.pushMCPToolEvent("skyramp_testbot_prompt", undefined, {}).catch(() => { });
|
|
357
369
|
return {
|
|
358
370
|
messages: [
|
|
@@ -59,7 +59,7 @@ describe("buildServiceContext (via getTestbotPrompt)", () => {
|
|
|
59
59
|
expect(prompt).toContain("<language>python</language>");
|
|
60
60
|
expect(prompt).toContain("<framework>pytest</framework>");
|
|
61
61
|
expect(prompt).toContain("<base_url>http://localhost:8000</base_url>");
|
|
62
|
-
expect(prompt).toContain("<
|
|
62
|
+
expect(prompt).toContain("<test_directory>tests/python</test_directory>");
|
|
63
63
|
expect(prompt).toContain("</service>");
|
|
64
64
|
expect(prompt).toContain("<services>");
|
|
65
65
|
expect(prompt).toContain("</services>");
|
|
@@ -70,7 +70,7 @@ describe("buildServiceContext (via getTestbotPrompt)", () => {
|
|
|
70
70
|
expect(prompt).not.toContain("<language>");
|
|
71
71
|
expect(prompt).not.toContain("<framework>");
|
|
72
72
|
expect(prompt).not.toContain("<base_url>");
|
|
73
|
-
expect(prompt).not.toContain("<
|
|
73
|
+
expect(prompt).not.toContain("<test_directory>");
|
|
74
74
|
});
|
|
75
75
|
it("renders multiple services", () => {
|
|
76
76
|
const prompt = callWithServices([
|
|
@@ -104,7 +104,7 @@ describe("buildServiceContext (via getTestbotPrompt)", () => {
|
|
|
104
104
|
api: { baseUrl: "http://host?a=1&b=2" },
|
|
105
105
|
},
|
|
106
106
|
]);
|
|
107
|
-
expect(prompt).toContain("<
|
|
107
|
+
expect(prompt).toContain("<test_directory>tests/a&b</test_directory>");
|
|
108
108
|
expect(prompt).toContain("<base_url>http://host?a=1&b=2</base_url>");
|
|
109
109
|
});
|
|
110
110
|
it("places services block between REPOSITORY PATH and instruction line", () => {
|
|
@@ -211,12 +211,12 @@ describe("drift analysis inline embedding", () => {
|
|
|
211
211
|
expect(prompt).toContain("<drift_analysis_rules>");
|
|
212
212
|
expect(prompt).toContain("</drift_analysis_rules>");
|
|
213
213
|
});
|
|
214
|
-
it("
|
|
214
|
+
it("does not include a persona statement inside the inline XML block", () => {
|
|
215
215
|
const prompt = basePrompt();
|
|
216
216
|
const start = prompt.indexOf("<drift_analysis_rules>");
|
|
217
217
|
const end = prompt.indexOf("</drift_analysis_rules>");
|
|
218
218
|
const block = prompt.slice(start, end);
|
|
219
|
-
expect(block).toContain("You are acting as a Skyramp Integration Architect");
|
|
219
|
+
expect(block).not.toContain("You are acting as a Skyramp Integration Architect");
|
|
220
220
|
});
|
|
221
221
|
it("drift_analysis_rules block appears inside Task 1, before Task 2", () => {
|
|
222
222
|
const prompt = basePrompt();
|
|
@@ -231,3 +231,20 @@ describe("drift analysis inline embedding", () => {
|
|
|
231
231
|
expect(prompt).toContain("rules in `<drift_analysis_rules>`");
|
|
232
232
|
});
|
|
233
233
|
});
|
|
234
|
+
describe("buildWorkspaceRecoveryPrefix", () => {
|
|
235
|
+
const { buildWorkspaceRecoveryPrefix } = require("./testbot-prompts.js");
|
|
236
|
+
it("includes repositoryPath in both init_scan and init_workspace instructions", () => {
|
|
237
|
+
const prefix = buildWorkspaceRecoveryPrefix("/home/user/repo");
|
|
238
|
+
expect(prefix).toContain('skyramp_init_scan with workspacePath "/home/user/repo"');
|
|
239
|
+
expect(prefix).toContain('skyramp_init_workspace with workspacePath "/home/user/repo"');
|
|
240
|
+
});
|
|
241
|
+
it("includes force: true for both tool calls", () => {
|
|
242
|
+
const prefix = buildWorkspaceRecoveryPrefix("/repo");
|
|
243
|
+
expect(prefix).toContain("force: true, then call skyramp_init_workspace");
|
|
244
|
+
expect(prefix).toContain("force: true to regenerate");
|
|
245
|
+
});
|
|
246
|
+
it("starts with IMPORTANT", () => {
|
|
247
|
+
const prefix = buildWorkspaceRecoveryPrefix("/repo");
|
|
248
|
+
expect(prefix).toMatch(/^IMPORTANT:/);
|
|
249
|
+
});
|
|
250
|
+
});
|
|
@@ -29,6 +29,7 @@ export function registerAnalysisResources(server) {
|
|
|
29
29
|
return memData;
|
|
30
30
|
}
|
|
31
31
|
}
|
|
32
|
+
logger.warning(`Session not found in memory (sessionId=${sessionId}) — server may have restarted; falling back to state file`);
|
|
32
33
|
// Fall back to state file for backward compatibility.
|
|
33
34
|
// Try both "analysis" and "recommendation" prefixes since the default changed.
|
|
34
35
|
const registeredPath = getSessionFilePath(sessionId);
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { AUTH_PLACEHOLDER_TOKEN } from "../types/TestTypes.js";
|
|
2
2
|
import { isAuthorizationHeaderName } from "../utils/workspaceAuth.js";
|
|
3
|
+
import { inferExpectedStatus } from "../utils/httpDefaults.js";
|
|
3
4
|
import { logger } from "../utils/logger.js";
|
|
5
|
+
import { stageGeneratedPaths } from "../utils/gitStaging.js";
|
|
4
6
|
import fs from "fs";
|
|
5
7
|
import path from "path";
|
|
6
8
|
export class ScenarioGenerationService {
|
|
@@ -40,6 +42,8 @@ export class ScenarioGenerationService {
|
|
|
40
42
|
}
|
|
41
43
|
existingRequests.push(traceRequest);
|
|
42
44
|
fs.writeFileSync(filePath, JSON.stringify(existingRequests, null, 2), "utf8");
|
|
45
|
+
// Stage so testbot includes the generated files in its output commit.
|
|
46
|
+
await stageGeneratedPaths(filePath);
|
|
43
47
|
logger.info("Trace request added to file", {
|
|
44
48
|
filePath,
|
|
45
49
|
totalRequests: existingRequests.length,
|
|
@@ -124,7 +128,7 @@ ${JSON.stringify(traceRequest, null, 2)}
|
|
|
124
128
|
}
|
|
125
129
|
const timestamp = new Date().toISOString();
|
|
126
130
|
const method = params.method;
|
|
127
|
-
const statusCode = params.statusCode ?? (method
|
|
131
|
+
const statusCode = params.statusCode ?? inferExpectedStatus(method);
|
|
128
132
|
const requestBody = params.requestBody ||
|
|
129
133
|
(method === "GET" || method === "DELETE" ? "" : "{}");
|
|
130
134
|
const responseHeaders = params.responseHeaders
|
|
@@ -8,6 +8,7 @@ import { getEntryPoint } from "../utils/telemetry.js";
|
|
|
8
8
|
import { getLanguageSteps } from "../utils/language-helper.js";
|
|
9
9
|
import { logger } from "../utils/logger.js";
|
|
10
10
|
import { normalizeLanguageParams } from "../utils/normalizeParams.js";
|
|
11
|
+
import { stageGeneratedPaths } from "../utils/gitStaging.js";
|
|
11
12
|
export class TestGenerationService {
|
|
12
13
|
client;
|
|
13
14
|
constructor() {
|
|
@@ -324,6 +325,8 @@ The generated test file remains unchanged and ready to use as-is.
|
|
|
324
325
|
throw new Error(`Test generation failed: ${result}`);
|
|
325
326
|
}
|
|
326
327
|
}
|
|
328
|
+
// Stage so testbot includes the generated files in its output commit.
|
|
329
|
+
await stageGeneratedPaths(generateOptions.outputDir);
|
|
327
330
|
return `
|
|
328
331
|
**Generated Test Details:**
|
|
329
332
|
- Test Type: ${this.getTestType()}
|
|
@@ -4,6 +4,7 @@ import { getCodeReusePrompt } from "../../prompts/code-reuse.js";
|
|
|
4
4
|
import { codeRefactoringSchema, languageSchema, } from "../../types/TestTypes.js";
|
|
5
5
|
import { SKYRAMP_UTILS_HEADER } from "../../utils/utils.js";
|
|
6
6
|
import { AnalyticsService } from "../../services/AnalyticsService.js";
|
|
7
|
+
import { stageGeneratedPaths } from "../../utils/gitStaging.js";
|
|
7
8
|
const codeReuseSchema = z.object({
|
|
8
9
|
testFile: z
|
|
9
10
|
.string()
|
|
@@ -70,6 +71,8 @@ export function registerCodeReuseTool(server) {
|
|
|
70
71
|
language: params.language,
|
|
71
72
|
framework: params.framework,
|
|
72
73
|
});
|
|
74
|
+
// Stage so testbot includes the generated files in its output commit.
|
|
75
|
+
await stageGeneratedPaths(params.testFile);
|
|
73
76
|
const codeReusePrompt = getCodeReusePrompt(params.testFile, params.language, params.framework);
|
|
74
77
|
return {
|
|
75
78
|
content: [
|
|
@@ -4,6 +4,8 @@ import { AnalyticsService } from "../../services/AnalyticsService.js";
|
|
|
4
4
|
import { getContractProviderAssertionsPrompt } from "../../prompts/enhance-assertions/contractProviderAssertionsPrompt.js";
|
|
5
5
|
import { getIntegrationAssertionsPrompt } from "../../prompts/enhance-assertions/integrationAssertionsPrompt.js";
|
|
6
6
|
import { getUIAssertionsPrompt } from "../../prompts/enhance-assertions/uiAssertionsPrompt.js";
|
|
7
|
+
import { isTestbotEnabled } from "../../utils/featureFlags.js";
|
|
8
|
+
import { stageGeneratedPaths } from "../../utils/gitStaging.js";
|
|
7
9
|
const TOOL_NAME = "skyramp_enhance_assertions";
|
|
8
10
|
const TESTBOT_UI_CHECKS = `
|
|
9
11
|
### Additional Testbot-Specific Checks
|
|
@@ -33,11 +35,13 @@ export function registerEnhanceAssertionsTool(server) {
|
|
|
33
35
|
inputSchema: enhanceAssertionsSchema,
|
|
34
36
|
}, async (params) => {
|
|
35
37
|
const { testFile, testType, enhanceType } = params;
|
|
38
|
+
// Stage so testbot includes the generated files in its output commit.
|
|
39
|
+
await stageGeneratedPaths(testFile);
|
|
36
40
|
const enhanceCtx = enhanceType;
|
|
37
41
|
let instructions;
|
|
38
42
|
if (testType === TestType.UI) {
|
|
39
43
|
instructions = getUIAssertionsPrompt(testFile, enhanceCtx);
|
|
40
|
-
if (
|
|
44
|
+
if (isTestbotEnabled()) {
|
|
41
45
|
instructions += TESTBOT_UI_CHECKS;
|
|
42
46
|
}
|
|
43
47
|
}
|
|
@@ -6,6 +6,7 @@ import { ModularizationService, } from "../../services/ModularizationService.js"
|
|
|
6
6
|
import { AnalyticsService } from "../../services/AnalyticsService.js";
|
|
7
7
|
import { normalizeLanguageParams, resolveParamAliases, } from "../../utils/normalizeParams.js";
|
|
8
8
|
import { normalizeSkyrampImportsInFile } from "../../utils/normalizeSkyrampImports.js";
|
|
9
|
+
import { stageGeneratedPaths } from "../../utils/gitStaging.js";
|
|
9
10
|
const modularizationSchema = {
|
|
10
11
|
testFile: z
|
|
11
12
|
.string()
|
|
@@ -79,6 +80,8 @@ After modularization, if errors remain, call skyramp_fix_errors.
|
|
|
79
80
|
if (!params.isTraceBased && [TestType.UI, TestType.E2E, TestType.INTEGRATION].includes(params.testType))
|
|
80
81
|
params.isTraceBased = true;
|
|
81
82
|
normalizeSkyrampImportsInFile(params.testFile);
|
|
83
|
+
// Stage so testbot includes the generated files in its output commit.
|
|
84
|
+
await stageGeneratedPaths(params.testFile);
|
|
82
85
|
// Default prompt to test file content
|
|
83
86
|
if (!params.prompt && params.testFile) {
|
|
84
87
|
try {
|
|
@@ -5,6 +5,7 @@ import fs from "fs";
|
|
|
5
5
|
import { baseSchema, AUTH_PLACEHOLDER_TOKEN, HttpMethod } from "../../types/TestTypes.js";
|
|
6
6
|
import { AnalyticsService } from "../../services/AnalyticsService.js";
|
|
7
7
|
import { getWorkspaceAuthConfig, WorkspaceAuthType, getDefaultAuthHeader, isAuthorizationHeaderName, getAuthScheme } from "../../utils/workspaceAuth.js";
|
|
8
|
+
import yaml from "js-yaml";
|
|
8
9
|
import { logger } from "../../utils/logger.js";
|
|
9
10
|
function isJsonValue(v) {
|
|
10
11
|
if (v === undefined || v === null)
|
|
@@ -184,6 +185,126 @@ Call \`skyramp_integration_test_generation\` with the returned \`scenarioFile\`
|
|
|
184
185
|
logger.warning("Could not resolve auth from workspace config");
|
|
185
186
|
}
|
|
186
187
|
}
|
|
188
|
+
// Separate try/catch so auth errors don't silently swallow schema population.
|
|
189
|
+
// Walk up from outputDir with a sync existsSync check — one stat per level,
|
|
190
|
+
// one read+parse only when found — instead of an async readWorkspaceConfigRaw
|
|
191
|
+
// call (WorkspaceConfigManager instantiation + 2 async I/O ops) per level.
|
|
192
|
+
if (!params.apiSchema) {
|
|
193
|
+
try {
|
|
194
|
+
const WS_SUBPATH = path.join(".skyramp", "workspace.yml");
|
|
195
|
+
// Assumption: outputDir lives somewhere inside the repo tree so the walk-up
|
|
196
|
+
// eventually reaches .skyramp/workspace.yml. If outputDir is outside the repo
|
|
197
|
+
// (e.g. /tmp/skyramp-test), the loop exits at the filesystem root without finding
|
|
198
|
+
// the config — the surrounding try/catch handles this gracefully (non-critical).
|
|
199
|
+
let searchDir = path.resolve(params.outputDir);
|
|
200
|
+
let wsConfigPath = null;
|
|
201
|
+
while (searchDir !== path.dirname(searchDir)) {
|
|
202
|
+
const candidate = path.join(searchDir, WS_SUBPATH);
|
|
203
|
+
if (fs.existsSync(candidate)) {
|
|
204
|
+
wsConfigPath = candidate;
|
|
205
|
+
break;
|
|
206
|
+
}
|
|
207
|
+
searchDir = path.dirname(searchDir);
|
|
208
|
+
}
|
|
209
|
+
if (wsConfigPath) {
|
|
210
|
+
const wsRaw = yaml.load(fs.readFileSync(wsConfigPath, "utf-8"));
|
|
211
|
+
// Best-effort: picks the first service. Multi-service workspaces may have
|
|
212
|
+
// the wrong schema if outputDir belongs to a later service — acceptable
|
|
213
|
+
// limitation for now; the user can always pass apiSchema explicitly.
|
|
214
|
+
const rawSchemaPath = wsRaw?.services?.[0]?.api?.schemaPath;
|
|
215
|
+
if (rawSchemaPath && typeof rawSchemaPath === "string") {
|
|
216
|
+
const isUrl = rawSchemaPath.startsWith("http://") || rawSchemaPath.startsWith("https://");
|
|
217
|
+
// searchDir is the directory where workspace.yml was found — use it as
|
|
218
|
+
// the resolution base so relative paths like "../openapi.yml" resolve
|
|
219
|
+
// correctly regardless of outputDir depth.
|
|
220
|
+
const schemaPath = isUrl
|
|
221
|
+
? rawSchemaPath
|
|
222
|
+
: path.resolve(searchDir, rawSchemaPath);
|
|
223
|
+
params = { ...params, apiSchema: schemaPath };
|
|
224
|
+
logger.info("Auto-populated apiSchema from workspace config", { schemaPath });
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
catch {
|
|
229
|
+
// non-critical
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
// ── Change 10b: Reject GraphQL endpoint steps ──
|
|
233
|
+
// Skyramp supports REST testing only; /graphql* requires introspection not implemented.
|
|
234
|
+
{
|
|
235
|
+
const graphqlSteps = params.steps.filter((s) => {
|
|
236
|
+
if (typeof s.path !== "string")
|
|
237
|
+
return false;
|
|
238
|
+
const normalized = s.path.replace(/\/+$/, "").toLowerCase();
|
|
239
|
+
return normalized.split("/").some((seg) => seg === "graphql");
|
|
240
|
+
});
|
|
241
|
+
if (graphqlSteps.length > 0) {
|
|
242
|
+
return {
|
|
243
|
+
isError: true,
|
|
244
|
+
content: [{ type: "text", text: `GraphQL endpoints are not supported by Skyramp's test generation.\n` +
|
|
245
|
+
`Affected steps: ${graphqlSteps.map((s) => `${s.method} ${s.path}`).join(", ")}\n\n` +
|
|
246
|
+
`Skyramp supports REST API testing only. Remove GraphQL steps and use ` +
|
|
247
|
+
`the REST endpoints for the same resource instead.`,
|
|
248
|
+
}],
|
|
249
|
+
};
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
// ── Change 3: Soft spec path validation ──
|
|
253
|
+
// Warn when step paths are missing from the spec — proceed with generation regardless.
|
|
254
|
+
// Hard rejection was removed: specs frequently lag code (new endpoints, undocumented
|
|
255
|
+
// internal routes, stale auto-gen) so a missing path != a phantom path.
|
|
256
|
+
let specValidationWarning = "";
|
|
257
|
+
if (params.apiSchema) {
|
|
258
|
+
try {
|
|
259
|
+
const isUrl = params.apiSchema.startsWith("http://") || params.apiSchema.startsWith("https://");
|
|
260
|
+
let specText;
|
|
261
|
+
if (isUrl) {
|
|
262
|
+
const specRes = await fetch(params.apiSchema, { signal: AbortSignal.timeout(10_000) });
|
|
263
|
+
if (!specRes.ok) {
|
|
264
|
+
throw new Error(`HTTP ${specRes.status} ${specRes.statusText} fetching spec at ${params.apiSchema}`);
|
|
265
|
+
}
|
|
266
|
+
specText = await specRes.text();
|
|
267
|
+
}
|
|
268
|
+
else {
|
|
269
|
+
// Note: relative apiSchema paths resolve against outputDir, not the workspace root.
|
|
270
|
+
// In practice apiSchema is always absolute (Gap Fix 2 / schema guidance), so this is safe.
|
|
271
|
+
specText = fs.readFileSync(path.resolve(params.outputDir, params.apiSchema), "utf-8");
|
|
272
|
+
}
|
|
273
|
+
// js-yaml handles both JSON and YAML specs
|
|
274
|
+
const specLoaded = yaml.load(specText);
|
|
275
|
+
const specPaths = new Set(Object.keys((specLoaded && typeof specLoaded === "object" ? specLoaded.paths : null) ?? {}));
|
|
276
|
+
if (specPaths.size === 0) {
|
|
277
|
+
logger.warning("Spec loaded but contains no paths — skipping path check", {
|
|
278
|
+
apiSchema: params.apiSchema,
|
|
279
|
+
});
|
|
280
|
+
}
|
|
281
|
+
else {
|
|
282
|
+
const unverifiedSteps = params.steps.filter((s) => {
|
|
283
|
+
if (typeof s.path !== "string")
|
|
284
|
+
return false;
|
|
285
|
+
const norm = s.path.replace(/:[a-zA-Z_][a-zA-Z0-9_]*/g, (m) => `{${m.slice(1)}}`);
|
|
286
|
+
return !specPaths.has(s.path) && !specPaths.has(norm);
|
|
287
|
+
});
|
|
288
|
+
if (unverifiedSteps.length > 0) {
|
|
289
|
+
specValidationWarning =
|
|
290
|
+
`\n\n⚠️ **Spec warning** — the following paths were not found in \`${params.apiSchema}\` ` +
|
|
291
|
+
`(spec may be stale or incomplete — verify paths against source before running tests):\n` +
|
|
292
|
+
unverifiedSteps.map((s) => ` ${s.method} ${s.path}`).join("\n") +
|
|
293
|
+
`\n\nKnown spec paths (first 20): ${[...specPaths].slice(0, 20).join(", ")}`;
|
|
294
|
+
logger.warning("Step paths not found in spec — proceeding (spec may lag code)", {
|
|
295
|
+
unverifiedPaths: unverifiedSteps.map((s) => s.path),
|
|
296
|
+
apiSchema: params.apiSchema,
|
|
297
|
+
});
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
}
|
|
301
|
+
catch (err) {
|
|
302
|
+
logger.warning("Spec check skipped — could not load apiSchema", {
|
|
303
|
+
apiSchema: params.apiSchema,
|
|
304
|
+
error: err instanceof Error ? err.message : String(err),
|
|
305
|
+
});
|
|
306
|
+
}
|
|
307
|
+
}
|
|
187
308
|
const service = new ScenarioGenerationService();
|
|
188
309
|
const steps = params.steps;
|
|
189
310
|
const scenarioSlug = params.scenarioName.toLowerCase().replace(/[^a-z0-9_-]+/g, "-").replace(/^-+|-+$/g, "") || "scenario";
|
|
@@ -261,7 +382,8 @@ Call \`skyramp_integration_test_generation\` with the returned \`scenarioFile\`
|
|
|
261
382
|
+ `**Scenario:** ${params.scenarioName}\n`
|
|
262
383
|
+ `**Steps:**\n${steps.map((s, i) => ` ${i + 1}. ${s.method} ${s.path} → ${s.statusCode ?? "default"}`).join("\n")}\n\n`
|
|
263
384
|
+ `**File:** ${filePath}\n\n`
|
|
264
|
-
+ `**Next:** Call \`skyramp_integration_test_generation\` with \`scenarioFile: "${filePath}"
|
|
385
|
+
+ `**Next:** Call \`skyramp_integration_test_generation\` with \`scenarioFile: "${filePath}"\``
|
|
386
|
+
+ specValidationWarning,
|
|
265
387
|
},
|
|
266
388
|
],
|
|
267
389
|
isError: false,
|