@skyramp/mcp 0.1.4 → 0.1.6

package/build/tools/generate-tests/generateBatchScenarioRestTool.test.js

@@ -1,15 +1,40 @@
-jest.mock("../../services/ScenarioGenerationService.js", () => ({ ScenarioGenerationService: jest.fn() }));
-jest.mock("../../services/AnalyticsService.js", () => ({ AnalyticsService: { pushMCPToolEvent: jest.fn() } }));
-jest.mock("../../utils/workspaceAuth.js", () => ({ getWorkspaceAuthConfig: jest.fn(), WorkspaceAuthType: { None: "none" } }));
+jest.mock("../../services/ScenarioGenerationService.js", () => ({
+    ScenarioGenerationService: jest.fn().mockImplementation(() => ({
+        generateTraceRequestFromInput: jest.fn().mockReturnValue(null),
+    })),
+}));
+jest.mock("../../services/AnalyticsService.js", () => ({ AnalyticsService: { pushMCPToolEvent: jest.fn().mockResolvedValue(undefined) } }));
+jest.mock("../../utils/workspaceAuth.js", () => ({
+    getWorkspaceAuthConfig: jest.fn().mockResolvedValue({ authHeader: undefined, authType: "none" }),
+    WorkspaceAuthType: { None: "none" },
+    getDefaultAuthHeader: jest.fn().mockReturnValue(""),
+    isAuthorizationHeaderName: jest.fn().mockReturnValue(false),
+    getAuthScheme: jest.fn().mockReturnValue(""),
+    readWorkspaceConfigRaw: jest.fn().mockResolvedValue(null),
+}));
 jest.mock("../../utils/logger.js", () => ({ logger: { info: jest.fn(), warning: jest.fn(), error: jest.fn() } }));
 jest.mock("../../prompts/personas.js", () => ({ getPersonaPrefix: () => "" }));
-import { stepSchema } from "./generateBatchScenarioRestTool.js";
+import fs from "fs";
+import { stepSchema, registerBatchScenarioTestTool } from "./generateBatchScenarioRestTool.js";
+/** Build a minimal mock McpServer, register the tool, and return the captured handler. */
+function captureHandler() {
+    let capturedHandler;
+    const mockServer = {
+        registerTool: jest.fn((_name, _meta, handler) => {
+            capturedHandler = handler;
+        }),
+    };
+    registerBatchScenarioTestTool(mockServer);
+    return capturedHandler;
+}
+// ─────────────────────────────────────────────────────────────────────────────
+// stepSchema — requestBody validation
+// ─────────────────────────────────────────────────────────────────────────────
 describe("stepSchema — requestBody validation", () => {
     it("accepts a POST step with a non-empty requestBody", () => {
         const result = stepSchema.safeParse({
             method: "POST",
             path: "/api/v1/products",
-            statusCode: 201,
             requestBody: JSON.stringify({ name: "Widget-123", price: 9.99 }),
         });
         expect(result.success).toBe(true);
@@ -18,7 +43,6 @@ describe("stepSchema — requestBody validation", () => {
         const result = stepSchema.safeParse({
             method: "POST",
             path: "/api/v1/products",
-            statusCode: 201,
             requestBody: "{}",
         });
         expect(result.success).toBe(false);
@@ -30,7 +54,6 @@ describe("stepSchema — requestBody validation", () => {
         const result = stepSchema.safeParse({
             method: "PATCH",
             path: "/api/v1/orders/1",
-            statusCode: 200,
             requestBody: "{}",
         });
         expect(result.success).toBe(false);
@@ -40,7 +63,6 @@ describe("stepSchema — requestBody validation", () => {
         const result = stepSchema.safeParse({
             method: "PUT",
             path: "/api/v1/products/1",
-            statusCode: 200,
             requestBody: "{}",
         });
         expect(result.success).toBe(false);
@@ -49,29 +71,24 @@ describe("stepSchema — requestBody validation", () => {
         expect(issue?.message).toContain("PUT");
     });
     it("accepts a POST step with requestBody omitted entirely", () => {
-        // The validation only rejects {} when present — omitting requestBody is still allowed
         const result = stepSchema.safeParse({
             method: "POST",
             path: "/api/v1/products",
-            statusCode: 201,
         });
         expect(result.success).toBe(true);
     });
     it("does not throw on requestBody: 'null' (valid JSON null)", () => {
-        // parsed === null must not cause Object.keys to throw
         const result = stepSchema.safeParse({
             method: "POST",
             path: "/api/v1/products",
-            statusCode: 201,
             requestBody: "null",
         });
-        expect(result.success).toBe(true); // null is not an empty object — not rejected
+        expect(result.success).toBe(true);
     });
     it("accepts a GET step with no requestBody", () => {
         const result = stepSchema.safeParse({
             method: "GET",
             path: "/api/v1/products/1",
-            statusCode: 200,
         });
         expect(result.success).toBe(true);
     });
@@ -79,19 +96,189 @@ describe("stepSchema — requestBody validation", () => {
         const result = stepSchema.safeParse({
             method: "DELETE",
             path: "/api/v1/products/1",
-            statusCode: 204,
         });
         expect(result.success).toBe(true);
     });
     it("does not reject a GET step that happens to have an empty requestBody", () => {
-        // GET with empty body is unusual but not blocked — the validation only
-        // applies to body methods (POST/PUT/PATCH)
         const result = stepSchema.safeParse({
             method: "GET",
             path: "/api/v1/products",
-            statusCode: 200,
             requestBody: "{}",
         });
         expect(result.success).toBe(true);
     });
 });
+// ─────────────────────────────────────────────────────────────────────────────
+// GraphQL step rejection (Change 10b)
+// ─────────────────────────────────────────────────────────────────────────────
+describe("generateBatchScenarioRestTool — GraphQL step rejection (Change 10b)", () => {
+    let handler;
+    const baseParams = {
+        scenarioName: "test_scenario",
+        destination: "localhost",
+        outputDir: "/tmp/skyramp-test",
+    };
+    beforeEach(() => {
+        jest.clearAllMocks();
+        handler = captureHandler();
+    });
+    it("rejects a step list containing /graphql", async () => {
+        const result = await handler({
+            ...baseParams,
+            steps: [{ method: "POST", path: "/graphql", requestBody: '{"query":"{ users { id } }"}' }],
+        });
+        expect(result.isError).toBe(true);
+        expect(result.content[0].text).toContain("GraphQL endpoints are not supported");
+        expect(result.content[0].text).toContain("POST /graphql");
+    });
+    it("rejects a step list containing /api/graphql (nested segment)", async () => {
+        const result = await handler({
+            ...baseParams,
+            steps: [
+                { method: "GET", path: "/api/v1/users" },
+                { method: "POST", path: "/api/graphql", requestBody: '{"query":"{ id }"}' },
+            ],
+        });
+        expect(result.isError).toBe(true);
+        expect(result.content[0].text).toContain("GraphQL endpoints are not supported");
+        expect(result.content[0].text).toContain("POST /api/graphql");
+    });
+    it("does not reject a path whose segment is 'graphql-config' (not an exact segment match)", async () => {
+        // 'graphql-config' !== 'graphql' — the guard uses exact segment comparison.
+        // The GraphQL rejection message must never appear, regardless of whether the
+        // handler succeeds or fails for some other reason.
+        const result = await handler({
+            ...baseParams,
+            steps: [{ method: "GET", path: "/api/v1/graphql-config" }],
+        });
+        expect(result.content[0].text).not.toContain("GraphQL endpoints are not supported");
+    });
+    it("does not throw on a non-string path (typeof guard)", async () => {
+        // typeof guard returns false for non-string — no crash from .replace()
+        const result = await handler({
+            ...baseParams,
+            steps: [{ method: "GET", path: undefined }],
+        });
+        expect(result).toBeDefined();
+        if (result.isError) {
+            expect(result.content[0].text).not.toContain("Cannot read");
+        }
+    });
+});
+// ─────────────────────────────────────────────────────────────────────────────
+// Spec path validation (Change 3)
+// ─────────────────────────────────────────────────────────────────────────────
+describe("generateBatchScenarioRestTool — spec path validation (Change 3)", () => {
+    let handler;
+    let readFileSyncSpy;
+    const fakeSpec = JSON.stringify({
+        openapi: "3.0.0",
+        paths: {
+            "/api/v1/users": {},
+            "/api/v1/users/{id}": {},
+        },
+    });
+    const baseParams = {
+        scenarioName: "test_scenario",
+        destination: "localhost",
+        outputDir: "/tmp/skyramp-test",
+        apiSchema: "/tmp/openapi.json",
+    };
+    let writeFileSyncSpy;
+    beforeEach(() => {
+        jest.clearAllMocks();
+        handler = captureHandler();
+        readFileSyncSpy = jest.spyOn(fs, "readFileSync").mockReturnValue(fakeSpec);
+        writeFileSyncSpy = jest.spyOn(fs, "writeFileSync").mockImplementation(() => { });
+        // Return a valid trace so tests reach the success path (where the warning is appended)
+        const ScenarioSvc = require("../../services/ScenarioGenerationService.js").ScenarioGenerationService;
+        ScenarioSvc.mockImplementation(() => ({
+            generateTraceRequestFromInput: jest.fn().mockReturnValue({ method: "GET", path: "/mock" }),
+        }));
+    });
+    afterEach(() => {
+        readFileSyncSpy.mockRestore();
+        writeFileSyncSpy.mockRestore();
+    });
+    it("warns (does not hard-reject) when step paths are not in the OpenAPI spec", async () => {
+        // Spec may lag code — missing path != phantom path, so we warn and proceed.
+        const result = await handler({
+            ...baseParams,
+            steps: [{ method: "GET", path: "/api/v1/phantom-endpoint" }],
+        });
+        expect(result.isError).toBeFalsy();
+        expect(result.content[0].text).toContain("Spec warning");
+        expect(result.content[0].text).toContain("/api/v1/phantom-endpoint");
+        expect(result.content[0].text).toContain("Known spec paths");
+    });
+    it("includes no spec warning when step path is present in the OpenAPI spec", async () => {
+        const result = await handler({
+            ...baseParams,
+            steps: [{ method: "GET", path: "/api/v1/users" }],
+        });
+        expect(result.content[0].text).not.toContain("Spec warning");
+    });
+    it("does not warn for Express-style :param paths that normalise to a spec path", async () => {
+        const result = await handler({
+            ...baseParams,
+            steps: [{ method: "GET", path: "/api/v1/users/:id" }],
+        });
+        expect(result.content[0].text).not.toContain("Spec warning");
+    });
+    it("does not throw on a non-string path (typeof guard)", async () => {
+        const result = await handler({
+            ...baseParams,
+            steps: [{ method: "GET", path: undefined }],
+        });
+        expect(result).toBeDefined();
+        expect(result.content[0].text).not.toContain("TypeError");
+        expect(result.content[0].text).not.toContain("Cannot read");
+    });
+    it("includes known spec paths hint in the warning message", async () => {
+        const result = await handler({
+            ...baseParams,
+            steps: [{ method: "POST", path: "/does/not/exist" }],
+        });
+        expect(result.isError).toBeFalsy();
+        expect(result.content[0].text).toContain("/api/v1/users");
+    });
+});
+describe("generateBatchScenarioRestTool — spec URL fetch safety (comment 3203006665)", () => {
+    let handler;
+    const baseParams = {
+        scenarioName: "test_scenario",
+        destination: "localhost",
+        outputDir: "/tmp/skyramp-test",
+        apiSchema: "https://example.com/openapi.json",
+        steps: [{ method: "GET", path: "/api/v1/users" }],
+    };
+    beforeEach(() => {
+        jest.clearAllMocks();
+        handler = captureHandler();
+    });
+    it("skips spec check (no warning emitted) when URL returns non-2xx", async () => {
+        global.fetch = jest.fn().mockResolvedValue({
+            ok: false,
+            status: 404,
+            statusText: "Not Found",
+            text: async () => "<html>Not Found</html>",
+        });
+        const result = await handler({ ...baseParams });
+        // Non-2xx throws → caught → spec check skipped entirely, no warning in output
+        expect(result.content[0].text).not.toContain("Spec warning");
+    });
+    it("skips spec check (no warning emitted) when spec has no paths", async () => {
+        global.fetch = jest.fn().mockResolvedValue({
+            ok: true,
+            status: 200,
+            statusText: "OK",
+            text: async () => JSON.stringify({ openapi: "3.0.0", info: { title: "Test", version: "1.0" } }),
+        });
+        const result = await handler({ ...baseParams });
+        expect(result.content[0].text).not.toContain("Spec warning");
+    });
+    afterEach(() => {
+        // Restore fetch
+        delete global.fetch;
+    });
+});

package/build/tools/generate-tests/generateContractRestTool.js

@@ -4,7 +4,7 @@ import { baseTestSchema, TestType } from "../../types/TestTypes.js";
 import { TestGenerationService, } from "../../services/TestGenerationService.js";
 import { AnalyticsService } from "../../services/AnalyticsService.js";
 import { getPersonaPrefix } from "../../prompts/personas.js";
-import { isContractConsumerModeEnabled } from "../../utils/featureFlags.js";
+import { isContractConsumerModeEnabled, isTestbotEnabled } from "../../utils/featureFlags.js";
 // SKYRAMP_FEATURE_CONTRACT_CONSUMER_MODE gates BOTH:
 //   1. Consumer-side contract test generation (`consumerMode` /
 //      `consumerOutput` schema fields, validation, post-gen instructions,
@@ -16,6 +16,7 @@ import { isContractConsumerModeEnabled } from "../../utils/featureFlags.js";
 //
 // `providerMode` itself is exposed regardless of the flag.
 const CONSUMER_MODE_ENABLED = isContractConsumerModeEnabled();
+const ADD_ASSERTIONS_DEFAULT = isTestbotEnabled();
 // "Requires apiSchema." plus a mode-aware "Not allowed with ..." clause for
 // the parent-provisioning fields. Generated once so wording stays in lockstep
 // with the feature flag.
@@ -23,6 +24,10 @@ const PARENT_FIELD_NOT_ALLOWED_NOTE = CONSUMER_MODE_ENABLED
     ? "Requires apiSchema. Not allowed with consumerMode or skipProvisionParents."
     : "Requires apiSchema. Not allowed with skipProvisionParents.";
 const baseContractTestSchema = {
+    enhanceAssertions: z
+        .boolean()
+        .default(ADD_ASSERTIONS_DEFAULT)
+        .describe("When true, calls skyramp_enhance_assertions after test generation to add richer response-body assertions. Disabled by default. Automatically enabled when running as testbot-feature. Do not override the default value of this parameter, unless the user explicitly asks to enable it."),
     ...baseTestSchema,
     pathParams: z
         .string()
@@ -149,7 +154,7 @@ export class ContractTestService extends TestGenerationService {
         if (params.providerMode) {
             content.push({
                 type: "text",
-                text: this.buildProviderPostGenInstructions(params),
+                text: this.buildProviderPostGenInstructions(params, params.enhanceAssertions),
             });
         }
         else {
@@ -219,27 +224,22 @@ ${step5}
     }
     buildSampleDataInstructions(params) {
         return `
-⏭️ **CRITICAL NEXT STEP — Replace placeholder sample data:**
-
-**Part 1 — Replace placeholder values in the generated file:**
+### CRITICAL NEXT STEP — Replace placeholder sample data
 
 ${this.buildPlaceholderReplacementBlock(params)}
 `;
     }
-    buildProviderPostGenInstructions(params) {
-        return `
-⏭️ **CRITICAL NEXT STEP — Replace placeholder data and enhance assertions:**
-
-⏭️ **CRITICAL — Part 1 — Replace placeholder values in the generated file:**
-
-${this.buildPlaceholderReplacementBlock(params, "continue to Part 2")}
-
----
-
-⏭️ **CRITICAL — Part 2 — Enhance response body assertions:**
-
-Call \`skyramp_enhance_assertions\` with \`testFile\` set to the absolute path of the generated provider contract test file, \`testType: "contract"\`, and \`enhanceType: "generation"\`. Apply every instruction returned to that file.
-`;
+    buildProviderPostGenInstructions(params, enhanceAssertions = false) {
+        const steps = [];
+        const nextStepNote = enhanceAssertions ? "continue to Step 2" : undefined;
+        steps.push(`### Step ${steps.length + 1} — Replace placeholder values [REQUIRED]\n\n${this.buildPlaceholderReplacementBlock(params, nextStepNote)}`);
+        if (enhanceAssertions) {
+            steps.push(`### Step ${steps.length + 1} — Enhance response body assertions [REQUIRED]\nCall \`skyramp_enhance_assertions\` with \`testFile\` set to the absolute path of the generated provider contract test file, \`testType: "contract"\`, and \`enhanceType: "generation"\`. Apply every instruction returned to that file.`);
+        }
+        const heading = enhanceAssertions
+            ? "Replace placeholder data and enhance assertions"
+            : "Replace placeholder data";
+        return `\n### CRITICAL NEXT STEP — ${heading}\n\n${steps.join("\n\n")}`;
     }
     buildConsumerStubReplacementInstructions() {
         return `

package/build/tools/generate-tests/generateIntegrationRestTool.js

@@ -4,8 +4,14 @@ import { TestGenerationService, } from "../../services/TestGenerationService.js"
 import { AnalyticsService } from "../../services/AnalyticsService.js";
 import { AUTH_CONFLICT_ERROR_MSG } from "../../prompts/test-recommendation/recommendationSections.js";
 import { getPersonaPrefix } from "../../prompts/personas.js";
+import { isTestbotEnabled } from "../../utils/featureFlags.js";
+const ADD_ASSERTIONS_DEFAULT = isTestbotEnabled();
 const integrationTestSchema = z
     .object({
+    enhanceAssertions: z
+        .boolean()
+        .default(ADD_ASSERTIONS_DEFAULT)
+        .describe("When true, calls skyramp_enhance_assertions after test generation to add richer response-body assertions. Disabled by default. Automatically enabled when running as testbot-feature. Do not override the default value of this parameter, unless the user explicitly asks to enable it."),
     ...baseTestSchema,
     chainingKey: z
         .string()
@@ -41,6 +47,8 @@ export class IntegrationTestService extends TestGenerationService {
         const result = await super.generateTest(params);
         if (result.isError)
             return result;
+        if (!params.enhanceAssertions)
+            return result;
         const content = [...result.content];
         content.push({
             type: "text",
@@ -50,8 +58,7 @@ export class IntegrationTestService extends TestGenerationService {
     }
     buildAssertionEnhancementInstructions() {
         return `
-**CRITICAL NEXT STEP — Enhance response body assertions after each request:**
-
+### CRITICAL NEXT STEP — Enhance response body assertions after each request
 Call \`skyramp_enhance_assertions\` with \`testFile\` set to the absolute path of the generated test file, \`testType: "integration"\`, and \`enhanceType: "generation"\`. Apply every instruction returned to that file.
 `;
     }

package/build/tools/generate-tests/generateUIRestTool.js

@@ -4,6 +4,7 @@ import { AnalyticsService } from "../../services/AnalyticsService.js";
 import { TestGenerationService, } from "../../services/TestGenerationService.js";
 import { normalizeLanguageParams, resolveParamAliases, } from "../../utils/normalizeParams.js";
 import { getPersonaPrefix } from "../../prompts/personas.js";
+import { isTestbotEnabled } from "../../utils/featureFlags.js";
 const TOOL_NAME = "skyramp_ui_test_generation";
 export class UITestService extends TestGenerationService {
     getTestType() {
@@ -25,25 +26,39 @@ export class UITestService extends TestGenerationService {
         const result = await super.generateTest({ ...params, modularizeCode: false });
         if (result.isError)
             return result;
+        if (!params.enhanceAssertions && !params.modularizeCode)
+            return result;
         const content = [...result.content];
         content.push({
             type: "text",
-            text: this.buildUIAssertionInstructions(params.modularizeCode !== false),
+            text: this.buildUIPostGenInstructions(!!params.enhanceAssertions, !!params.modularizeCode),
         });
         return { ...result, content };
     }
-    buildUIAssertionInstructions(modularize = true) {
-        const modularizeStep = modularize
-            ? `\n⏭️ **Step 2 — Modularize [REQUIRED]**: Call \`skyramp_modularization\` with \`testFile\` set to the absolute path of the generated test file, \`language\`, \`testType: "ui"\`, and \`isTraceBased: true\`.`
-            : "";
-        return `**CRITICAL NEXT STEP — Enhance UI assertions in the generated test:**
-
-Call \`skyramp_enhance_assertions\` with \`testFile\` set to the absolute path of the generated test file, \`testType: "ui"\`, and \`enhanceType: "generation"\`. Apply every instruction returned to that file.${modularizeStep}`;
+    buildUIPostGenInstructions(enhanceAssertions, modularize) {
+        const steps = [];
+        if (enhanceAssertions) {
+            steps.push(`### Step ${steps.length + 1} — Enhance UI assertions [REQUIRED]\nCall \`skyramp_enhance_assertions\` with \`testFile\` set to the absolute path of the generated test file, \`testType: "ui"\`, and \`enhanceType: "generation"\`. Apply every instruction returned to that file.`);
+        }
+        if (modularize) {
+            steps.push(`### Step ${steps.length + 1} — Modularize [REQUIRED]\nCall \`skyramp_modularization\` with \`testFile\` set to the absolute path of the generated test file, \`language\`, \`testType: "ui"\`, and \`isTraceBased: true\`.`);
+        }
+        const heading = enhanceAssertions && modularize
+            ? "Enhance UI assertions and modularize the generated test"
+            : enhanceAssertions
+                ? "Enhance UI assertions in the generated test"
+                : "Modularize the generated test";
+        return `### CRITICAL NEXT STEP — ${heading}\n\n${steps.join("\n\n")}`;
     }
 }
+const ADD_ASSERTIONS_DEFAULT = isTestbotEnabled();
 // Only include the original params in the schema
 const uiTestSchema = {
     ...languageSchema.shape,
+    enhanceAssertions: z
+        .boolean()
+        .default(ADD_ASSERTIONS_DEFAULT)
+        .describe("When true, calls skyramp_enhance_assertions after test generation to add richer response-body assertions. Disabled by default. Automatically enabled when running as testbot-feature. Do not override the default value of this parameter, unless the user explicitly asks to enable it."),
     playwrightInput: z
         .string()
         .describe("MUST be absolute path to the playwright trace zip file like /path/to/trace.zip. Capture using browser_* tools + skyramp_export_zip, or via manual trace collection with skyramp_start_trace_collection/skyramp_stop_trace_collection."),