@skyramp/mcp 0.0.64-rc.2 → 0.0.64-rc.4

package/build/prompts/testbot/testbot-prompts.js

@@ -151,10 +151,19 @@ If a test generation tool call fails:
 4. You MUST generate **at least 1 test** for any PR that touches application code. Zero generated tests is NOT acceptable.
 5. Log skipped candidates in \`issuesFound\` with the error message.
 
-**After generation, fix chaining only:**
+### UI Test Execution Fix-up
+If a generated UI test fails with a timeout waiting for an element after navigation (e.g. \`TimeoutError\` on \`getByTestId\` or \`locator\`), add a hydration wait after each \`page.goto()\` call:
+\`\`\`
+// Wait for React/framework hydration to complete
+await page.waitForTimeout(1500);
+\`\`\`
+Then re-run the test. This is a common issue with SSR/SPA frameworks where the DOM is rendered but not yet interactive.
+
+**After generation, fix chaining and enhance assertions only:**
 - Path params like \`id = 'id'\` → \`skyramp.get_response_value(prev_response, "id")\`
 - Hardcoded IDs in request bodies → dynamic values from prior response
-- Change ONLY chaining values. Preserve everything else exactly as generated.
+- **Integration tests and contract provider tests**: after the test generation, you MUST enhance response body assertions as instructed in the tool output.
+- Change ONLY chaining values and enhance assertions. Preserve everything else exactly as generated.
 
 After all actions, execute ONLY the test files you created (ADD), regenerated (REGENERATE),
 or edited (UPDATE). Do NOT execute VERIFY'd tests — they are unaffected by the diff and do not

package/build/services/TestExecutionService.js

@@ -335,6 +335,7 @@ export class TestExecutionService {
         // Prepare host config with mounts
         const hostConfig = {
             ExtraHosts: ["host.docker.internal:host-gateway"],
+            ...(options.useHostNetwork && process.platform === "linux" ? { NetworkMode: "host" } : {}),
             Mounts: [
                 {
                     Type: "bind",

package/build/services/TestGenerationService.js

@@ -226,6 +226,12 @@ The generated test file remains unchanged and ready to use as-is.
             // Map authScheme → authType for the npm client which uses authType internally
             if (generateOptions.authScheme !== undefined) {
                 generateOptions.authType = generateOptions.authScheme;
+                // authType handles the Authorization header implicitly — the Go CLI treats
+                // --auth-type and --auth-header as mutually exclusive, so clear authHeader
+                // when it's the standard Authorization header to avoid the conflict.
+                if (/^authorization$/i.test(generateOptions.authHeader || '')) {
+                    delete generateOptions.authHeader;
+                }
             }
             delete generateOptions.authScheme;
             const result = await this.client.generateRestTest(generateOptions);

package/build/services/TestGenerationService.test.js

@@ -107,7 +107,8 @@ describe("TestGenerationService — authType/authScheme not passed to library",
         const callArgs = mockGenerateRestTest.mock.calls[0][0];
         expect(callArgs.authType).toBe("Token");
         expect(callArgs.authScheme).toBeUndefined();
-        expect(callArgs.authHeader).toBe("Authorization");
+        // authHeader is cleared for Authorization when authType is set (CLI conflict fix)
+        expect(callArgs.authHeader).toBeUndefined();
     });
     it("passes authHeader through to library without modification", async () => {
         const svc = new StubService();
@@ -237,7 +238,7 @@ describe("TestGenerationService — trace-based auth in executeGeneration", () =
         fs.writeFileSync(filePath, JSON.stringify(requests), "utf8");
         return filePath;
     }
-    it("populates authHeader and authScheme from trace when not provided", async () => {
+    it("populates authScheme from trace when not provided (clears Authorization authHeader)", async () => {
         const traceFile = writeTrace([
             {
                 RequestHeaders: {
@@ -258,7 +259,8 @@ describe("TestGenerationService — trace-based auth in executeGeneration", () =
         svc.handleApiAnalysis = async () => null;
         await svc.generateTest({ ...BASE });
         const callArgs = mockGenerateRestTest.mock.calls[0][0];
-        expect(callArgs.authHeader).toBe("Authorization");
+        // authHeader is cleared for Authorization when authType is set (CLI conflict fix)
+        expect(callArgs.authHeader).toBeUndefined();
         expect(callArgs.authType).toBe("Token");
         expect(callArgs.authScheme).toBeUndefined();
     });
@@ -287,7 +289,7 @@ describe("TestGenerationService — trace-based auth in executeGeneration", () =
         expect(callArgs.authType).toBe("");
         expect(callArgs.authScheme).toBeUndefined();
     });
-    it("overrides conflicting authScheme with trace value", async () => {
+    it("overrides conflicting authScheme with trace value and clears Authorization authHeader", async () => {
         const traceFile = writeTrace([
             {
                 RequestHeaders: {
@@ -308,11 +310,12 @@ describe("TestGenerationService — trace-based auth in executeGeneration", () =
         svc.handleApiAnalysis = async () => null;
         await svc.generateTest({ ...BASE });
         const callArgs = mockGenerateRestTest.mock.calls[0][0];
-        expect(callArgs.authHeader).toBe("Authorization");
+        // authHeader is cleared for Authorization when authType is set (CLI conflict fix)
+        expect(callArgs.authHeader).toBeUndefined();
         expect(callArgs.authType).toBe("Token");
         expect(callArgs.authScheme).toBeUndefined();
     });
-    it("does not override auth when trace has no auth header", async () => {
+    it("does not override auth when trace has no auth header (clears Authorization authHeader)", async () => {
         const traceFile = writeTrace([
             {
                 RequestHeaders: {
@@ -332,7 +335,8 @@ describe("TestGenerationService — trace-based auth in executeGeneration", () =
         svc.handleApiAnalysis = async () => null;
         await svc.generateTest({ ...BASE });
         const callArgs = mockGenerateRestTest.mock.calls[0][0];
-        expect(callArgs.authHeader).toBe("Authorization");
+        // authHeader is cleared for Authorization when authType is set (CLI conflict fix)
+        expect(callArgs.authHeader).toBeUndefined();
         expect(callArgs.authType).toBe("Bearer");
         expect(callArgs.authScheme).toBeUndefined();
     });

package/build/tools/executeSkyrampTestTool.js

@@ -79,31 +79,34 @@ For detailed documentation visit: https://www.skyramp.dev/docs/quickstart`,
         };
         const previousBaseUrl = process.env.SKYRAMP_TEST_BASE_URL;
         let didSetSkyrampBaseUrl = false;
+        let useHostNetwork = false;
         try {
             // Send initial progress
             await sendProgress(0, 100, "Starting test execution...");
-            // Inject SKYRAMP_TEST_BASE_URL from workspace if not already set in env.
-            // Match by testFile path so the correct service URL is used when the
-            // workspace has multiple services with different baseUrls.
-            if (!process.env.SKYRAMP_TEST_BASE_URL && params.workspacePath) {
-                const { baseUrl, candidates } = await getWorkspaceBaseUrl(params.workspacePath, params.testFile, params.language);
-                if (baseUrl) {
-                    process.env.SKYRAMP_TEST_BASE_URL = baseUrl;
-                    didSetSkyrampBaseUrl = true;
-                }
-                else if (candidates.length > 0) {
-                    return {
-                        content: [{
-                                type: "text",
-                                text: [
-                                    `Cannot determine SKYRAMP_TEST_BASE_URL — test file matches multiple services:`,
-                                    ...candidates.map((c) => `  • ${c.serviceName}: ${c.baseUrl}`),
-                                    ``,
-                                    `Re-invoke with SKYRAMP_TEST_BASE_URL set to the correct service URL, or make each service's testDirectory unique in .skyramp/workspace.yml.`,
-                                ].join("\n"),
-                            }],
-                        isError: true,
-                    };
+            // Always resolve workspace config for dockerNetwork (host networking)
+            // and optionally inject SKYRAMP_TEST_BASE_URL if not already set.
+            if (params.workspacePath) {
+                const { baseUrl, dockerNetwork, candidates } = await getWorkspaceBaseUrl(params.workspacePath, params.testFile, params.language);
+                useHostNetwork = !!dockerNetwork;
+                if (!process.env.SKYRAMP_TEST_BASE_URL) {
+                    if (baseUrl) {
+                        process.env.SKYRAMP_TEST_BASE_URL = baseUrl;
+                        didSetSkyrampBaseUrl = true;
+                    }
+                    else if (candidates.length > 0) {
+                        return {
+                            content: [{
+                                    type: "text",
+                                    text: [
+                                        `Cannot determine SKYRAMP_TEST_BASE_URL — test file matches multiple services:`,
+                                        ...candidates.map((c) => `  • ${c.serviceName}: ${c.baseUrl}`),
+                                        ``,
+                                        `Re-invoke with SKYRAMP_TEST_BASE_URL set to the correct service URL, or make each service's testDirectory unique in .skyramp/workspace.yml.`,
+                                    ].join("\n"),
+                                }],
+                            isError: true,
+                        };
+                    }
                 }
             }
             const executionService = new TestExecutionService();
@@ -115,6 +118,7 @@ For detailed documentation visit: https://www.skyramp.dev/docs/quickstart`,
                 testType: params.testType,
                 token: params.token,
                 playwrightSaveStoragePath: params.playwrightSaveStoragePath,
+                useHostNetwork,
             }, onExecutionProgress);
             // Progress is already reported by TestExecutionService
             // Only report final status if not already at 100%

package/build/tools/test-management/executeTestsTool.js

@@ -128,38 +128,47 @@ back into the state file for use by \`skyramp_actions\`.
             // candidates so the LLM can resolve and re-invoke.
             const previousTestBaseUrl = process.env.SKYRAMP_TEST_BASE_URL;
             let didSetTestBaseUrl = false;
-            if (!previousTestBaseUrl) {
+            let useHostNetwork = false;
+            // Always resolve workspace config for dockerNetwork (host networking)
+            // and optionally inject SKYRAMP_TEST_BASE_URL if not already set.
+            {
                 const results = await Promise.all(testOptions.map((t) => getWorkspaceBaseUrl(absoluteWorkspacePath, t.testFile, t.language)));
-                const ambiguous = results.filter((r) => r.candidates.length > 0);
-                if (ambiguous.length > 0) {
-                    const lines = ambiguous.flatMap((r) => r.candidates.map((c) => `  • ${c.serviceName}: ${c.baseUrl}`));
-                    return {
-                        content: [{
-                                type: "text",
-                                text: [
-                                    `Cannot determine SKYRAMP_TEST_BASE_URL — one or more test files match multiple services:`,
-                                    ...lines,
-                                    ``,
-                                    `Re-invoke with SKYRAMP_TEST_BASE_URL set to the correct service URL, or make each service's testDirectory unique in .skyramp/workspace.yml.`,
-                                ].join("\n"),
-                            }],
-                        isError: true,
-                    };
-                }
-                const allResolved = results.every((r) => r.baseUrl);
-                if (allResolved) {
-                    const uniqueUrls = [...new Set(results.map((r) => r.baseUrl))];
-                    if (uniqueUrls.length === 1) {
-                        process.env.SKYRAMP_TEST_BASE_URL = uniqueUrls[0];
-                        didSetTestBaseUrl = true;
+                // Enable host networking if any matched service has dockerNetwork
+                useHostNetwork = results.some((r) => !!r.dockerNetwork);
+                if (!previousTestBaseUrl) {
+                    const ambiguous = results.filter((r) => r.candidates.length > 0);
+                    if (ambiguous.length > 0) {
+                        const lines = ambiguous.flatMap((r) => r.candidates.map((c) => `  • ${c.serviceName}: ${c.baseUrl}`));
+                        return {
+                            content: [{
+                                    type: "text",
+                                    text: [
+                                        `Cannot determine SKYRAMP_TEST_BASE_URL — one or more test files match multiple services:`,
+                                        ...lines,
+                                        ``,
+                                        `Re-invoke with SKYRAMP_TEST_BASE_URL set to the correct service URL, or make each service's testDirectory unique in .skyramp/workspace.yml.`,
+                                    ].join("\n"),
+                                }],
+                            isError: true,
+                        };
+                    }
+                    const allResolved = results.every((r) => r.baseUrl);
+                    if (allResolved) {
+                        const uniqueUrls = [...new Set(results.map((r) => r.baseUrl))];
+                        if (uniqueUrls.length === 1) {
+                            process.env.SKYRAMP_TEST_BASE_URL = uniqueUrls[0];
+                            didSetTestBaseUrl = true;
+                        }
                     }
                 }
             }
+            // Pass useHostNetwork to each test option
+            const enrichedTestOptions = testOptions.map((t) => ({ ...t, useHostNetwork }));
             let executionResult;
             try {
                 logger.info(`Executing ${testOptions.length} tests in parallel batches (max 5 concurrent)`);
                 const executionService = new TestExecutionService();
-                executionResult = await executionService.executeBatch(testOptions);
+                executionResult = await executionService.executeBatch(enrichedTestOptions);
                 logger.info(`Batch execution complete: ${executionResult.passed} passed, ` +
                     `${executionResult.failed} failed, ${executionResult.crashed} crashed`);
             }

package/build/utils/workspaceAuth.js

@@ -29,7 +29,7 @@ export async function getWorkspaceAuthHeader(repositoryPath) {
  *   → { baseUrl: "http://localhost:8000", candidates: [] }
  */
 export async function getWorkspaceBaseUrl(repositoryPath, testFile, language) {
-    const noMatch = { baseUrl: undefined, candidates: [] };
+    const noMatch = { baseUrl: undefined, dockerNetwork: undefined, candidates: [] };
     if (!testFile)
         return noMatch;
     try {
@@ -57,12 +57,17 @@ export async function getWorkspaceBaseUrl(repositoryPath, testFile, language) {
         }
         if (matches.length === 1) {
             const parsed = new URL(matches[0].api.baseUrl);
-            return { baseUrl: `${parsed.protocol}//${parsed.host}`, candidates: [] };
+            return {
+                baseUrl: `${parsed.protocol}//${parsed.host}`,
+                dockerNetwork: matches[0].runtimeDetails?.dockerNetwork,
+                candidates: [],
+            };
         }
         // Still ambiguous — return candidates for the LLM to resolve
         if (matches.length > 1) {
             return {
                 baseUrl: undefined,
+                dockerNetwork: undefined,
                 candidates: matches.map((s) => ({
                     serviceName: s.serviceName ?? s.testDirectory,
                     baseUrl: s.api.baseUrl,

package/build/utils/workspaceAuth.test.js

@@ -146,6 +146,58 @@ describe("getWorkspaceBaseUrl", () => {
         expect(result.baseUrl).toBe("http://localhost:9000");
         expect(result.candidates).toHaveLength(0);
     });
+    it("should return dockerNetwork when matched service has runtimeDetails.dockerNetwork", async () => {
+        const { WorkspaceConfigManager } = await import("@skyramp/skyramp");
+        WorkspaceConfigManager.mockImplementationOnce(() => ({
+            exists: jest.fn().mockResolvedValue(true),
+            read: jest.fn().mockResolvedValue({
+                services: [
+                    {
+                        serviceName: "backend",
+                        testDirectory: "tests",
+                        api: { baseUrl: "http://localhost:8000" },
+                        runtimeDetails: { dockerNetwork: "myapp_default" },
+                    },
+                ],
+            }),
+        }));
+        const result = await getWorkspaceBaseUrl(REPO, `${REPO}/tests/test_api.py`);
+        expect(result.baseUrl).toBe("http://localhost:8000");
+        expect(result.dockerNetwork).toBe("myapp_default");
+    });
+    it("should return undefined dockerNetwork when service has no runtimeDetails", async () => {
+        const result = await getWorkspaceBaseUrl(REPO, `${REPO}/backend/tests/test_api.py`);
+        expect(result.baseUrl).toBe("http://localhost:8000");
+        expect(result.dockerNetwork).toBeUndefined();
+    });
+    it("should return undefined dockerNetwork when ambiguous", async () => {
+        const { WorkspaceConfigManager } = await import("@skyramp/skyramp");
+        WorkspaceConfigManager.mockImplementationOnce(() => ({
+            exists: jest.fn().mockResolvedValue(true),
+            read: jest.fn().mockResolvedValue({
+                services: [
+                    {
+                        serviceName: "svc-a",
+                        language: "python",
+                        testDirectory: "shared/tests",
+                        api: { baseUrl: "http://localhost:8000" },
+                        runtimeDetails: { dockerNetwork: "net-a" },
+                    },
+                    {
+                        serviceName: "svc-b",
+                        language: "python",
+                        testDirectory: "shared/tests",
+                        api: { baseUrl: "http://localhost:9000" },
+                        runtimeDetails: { dockerNetwork: "net-b" },
+                    },
+                ],
+            }),
+        }));
+        const result = await getWorkspaceBaseUrl(REPO, `${REPO}/shared/tests/test_api.py`, "python");
+        expect(result.baseUrl).toBeUndefined();
+        expect(result.dockerNetwork).toBeUndefined();
+        expect(result.candidates).toHaveLength(2);
+    });
     it("should return no match when matched service baseUrl is not a valid URL", async () => {
         const { WorkspaceConfigManager } = await import("@skyramp/skyramp");
         WorkspaceConfigManager.mockImplementationOnce(() => ({

package/node_modules/playwright/lib/mcp/skyramp/traceRecordingBackend.js

@@ -146,7 +146,7 @@ class TraceRecordingBackend {
       this._maybeTrackAction("browser_select_option", args, result);
       return result;
     }
-    const shouldFallback = resultText.includes("not a <select> element") || resultText.includes("selectOption") && resultText.includes("Timeout");
+    const shouldFallback = resultText.includes("not a <select> element") || resultText.includes("selectOption") && resultText.includes("Timeout") || resultText.includes("not found in the current page snapshot");
     if (!shouldFallback)
       return result;
     traceDebug("selectOption failed on custom dropdown, trying combobox fallback");

package/node_modules/playwright/package.json

@@ -1,12 +1,15 @@
 {
-  "name": "playwright",
+  "name": "@skyramp/playwright",
   "version": "1.58.2-skyramp.8.9.0",
-  "description": "A high-level API to automate web browsers",
+  "description": "Skyramp's fork of Playwright with trace recording for UI test generation",
+  "publishConfig": {
+    "access": "public"
+  },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/microsoft/playwright.git"
+    "url": "git+https://github.com/letsramp/playwright.git"
   },
-  "homepage": "https://playwright.dev",
+  "homepage": "https://www.skyramp.dev",
   "engines": {
     "node": ">=18"
   },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@skyramp/mcp",
-  "version": "0.0.64-rc.2",
+  "version": "0.0.64-rc.4",
   "main": "build/index.js",
   "type": "module",
   "bin": {
@@ -52,7 +52,7 @@
     "@skyramp/skyramp": "1.3.16",
     "dockerode": "^4.0.6",
     "fast-glob": "^3.3.3",
-    "playwright": "file:vendor/playwright-1.58.2-skyramp.8.9.0.tgz",
+    "playwright": "file:vendor/skyramp-playwright-1.58.2-skyramp.8.9.0.tgz",
     "simple-git": "^3.30.0",
     "zod": "^3.25.3"
   },