@skyramp/mcp 0.0.62 → 0.0.63-rc.2

package/build/prompts/testbot/testbot-prompts.js

@@ -3,115 +3,123 @@ import { z } from "zod";
 import { logger } from "../../utils/logger.js";
 import { AnalyticsService } from "../../services/AnalyticsService.js";
 import { MAX_TESTS_TO_GENERATE, MAX_RECOMMENDATIONS } from "../test-recommendation/recommendationSections.js";
-function getTestbotPrompt(prTitle, prDescription, diffFile, testDirectory, summaryOutputFile, repositoryPath, baseBranch, maxRecommendations = MAX_RECOMMENDATIONS, maxGenerate = MAX_TESTS_TO_GENERATE) {
+function getTestbotPrompt(prTitle, prDescription, diffFile, testDirectory, summaryOutputFile, repositoryPath, baseBranch, maxRecommendations = MAX_RECOMMENDATIONS, maxGenerate = MAX_TESTS_TO_GENERATE, prNumber) {
     return `<TITLE>${prTitle}</TITLE>
 <DESCRIPTION>${prDescription}</DESCRIPTION>
 <CODE CHANGES>${diffFile}</CODE CHANGES>
 <TEST DIRECTORY>${testDirectory}</TEST DIRECTORY>
 <REPOSITORY PATH>${repositoryPath}</REPOSITORY PATH>
 
-Use the Skyramp MCP server tools for all tasks below.
-
-## Task 1: Recommend & Generate New Tests
-
-Read the diff at \`${diffFile}\`. Skip Task 1 if all changed files are non-application
-(CI/CD, docs, lock files, config). Otherwise proceed:
-
-### Steps
-
-1. Call \`skyramp_analyze_repository\` with \`repositoryPath\`: "${repositoryPath}", \`analysisScope\`: "current_branch_diff"${baseBranch ? `\n   , \`baseBranch\`: "${baseBranch}"` : ''}
-2. Call \`skyramp_recommend_tests\` with the returned \`sessionId\`.
-   It returns ${maxRecommendations} ranked recommendations. Walk through them in rank order and generate
-   up to ${maxGenerate} tests. Any recommendation you skip or cannot generate goes to
-   \`additionalRecommendations\`.
-
-3. **Generate** up to ${maxGenerate} tests by walking the ranked list top-to-bottom:
-   - If a recommendation can be generated, generate it and count it.
-   - If it cannot (e.g. E2E/UI without traces), move it to \`additionalRecommendations\`
-     and continue to the next recommendation.
-   - Stop once you have ${maxGenerate} generated tests OR exhaust all ${maxRecommendations} recommendations.
-   - All remaining (ungenerated) recommendations go to \`additionalRecommendations\`.
-   Keep a list of every file the CLI creates (test files AND scenario JSON files).
-
-   **Frontend-only PRs** (no backend/API changes): only generate tests if relevant
-   Playwright traces exist. If no traces are available, skip generation entirely and
-   move all ${maxRecommendations} recommendations to \`additionalRecommendations\` with scenario steps and
-   trace recording instructions. Do not generate integration tests for unchanged backend
-   APIs just to fill the quota — those tests don't validate the PR's changes.
-
-   **How to generate each type:**
-   - **Integration**: call \`skyramp_scenario_test_generation\` per step, then
-     \`skyramp_integration_test_generation\` with the scenario file.
-     The scenario JSON is written to the same \`outputDir\` as the test files
-     (e.g. \`tests/scenario_<name>.json\`), not \`.skyramp/\`.
-   - **Contract**: call \`skyramp_contract_test_generation\` with \`endpointURL\`, \`method\`,
-     and \`requestData\` for POST/PUT endpoints.
-     Pass \`apiSchema\` if an OpenAPI spec exists — it validates response structure.
-   - **Fuzz**: call \`skyramp_fuzz_test_generation\` with \`endpointURL\`, \`method\`, \`requestData\`.
-     Pass \`apiSchema\` if available — it generates smarter boundary values.
-   - **E2E/UI**: only generate when relevant Playwright traces exist (see step 5).
-     Without traces, move the test to \`additionalRecommendations\` with scenario steps
-     and trace recording instructions instead.
-   - Skip smoke tests entirely.
-
-   **Scenario quality:** Before generating, verify each step's preconditions are met by
-   prior steps. For example, you can't update a membership that was never created — check
-   the controller code for existence checks and ensure the scenario creates records first.
-
-   **Filenames:** Pass a descriptive \`--output\` name per test to avoid CLI overwrites.
-
-4. **Execute** the generated tests and record results.
-
-5. **Trace search** for E2E/UI: look in \`\${testDirectory}\`, repo root, and \`.skyramp/\` for
-   trace files (\`*trace*.json\`, \`*playwright*.zip\`). Only use a trace if it covers code
-   changed in this PR and targets localhost — skip traces for external hosts or unrelated code.
-
-   With relevant traces: backend + Playwright → \`skyramp_e2e_test_generation\`,
-   Playwright only → \`skyramp_ui_test_generation\`.
-
-**After generation, fix chaining only.** The CLI may use literal/hardcoded IDs instead
-of dynamic values from prior responses. Fix these two cases:
-1. **Path params:** variables like \`product_id = 'product_id'\` → use the response accessor
-   (e.g. \`getResponseValue(response, "response.id")\` in TS, \`skyramp.get_response_value(response, "id")\` in Python).
-2. **Request body refs:** hardcoded IDs in request bodies (e.g. \`"product_id": 1\`) → replace
-   with the dynamic ID extracted from the prior POST response (e.g. \`product_id\` variable or
-   \`dataOverride\`/\`data_override\` for the field).
-
-Change ONLY chaining-related values (path param assignments and body ID references).
-Preserve everything else exactly as the CLI generated it — headers, auth code, assertions,
-imports, and all other request body fields.
-
-## Task 2: Existing Test Maintenance
-
-Run this task regardless of Task 1 outcome — even if Task 1 was skipped or generated zero tests.
-
-1. Call \`skyramp_discover_tests\` with \`repositoryPath\`: "${repositoryPath}".
-2. If zero Skyramp tests found, report \`testMaintenance\` as an empty array.
-   This is expected for new repos — pass \`issuesFound\` as an empty array \`[]\`. Do not add a "no tests found" entry.
-3. If tests exist:
-   a. Baseline them (from CI status or by executing).
-   b. Run \`skyramp_analyze_test_drift\` → \`skyramp_calculate_health_scores\` → \`skyramp_actions\`.
-   c. Apply actions (path renames, schema updates) in-place. Do not regenerate.
-   d. Execute modified tests. Report before/after in \`testMaintenance\`.
-
-## Task 3: Submit Report
-
-Verify Tasks 1 and 2 are complete, then call \`skyramp_submit_report\` with
-\`summaryOutputFile\`: "${summaryOutputFile}".
+Use the Skyramp MCP server tools. Follow the steps below in order.
+
+---
+
+## Step 1: Analyze
+
+Read the diff at \`${diffFile}\`.
+If all changed files are non-application (CI/CD, docs, lock files, config only) → skip to Step 4 (Submit Report) with empty arrays.
+
+Otherwise:
+
+**Incremental mode:** Tests generated by prior bot runs on this PR are still in the
+working tree. Step 2/3 handles their maintenance (drift detection, health checks, fixes).
+Only generate tests for NEW endpoints or code paths not already covered by existing bot
+tests. The analyze tool uses PR comment history to avoid duplicates.
+
+1. Call \`skyramp_analyze_changes\` with \`repositoryPath\`: "${repositoryPath}", \`scope\`: "branch_diff", \`topN\`: ${maxRecommendations}${prNumber ? `, \`prNumber\`: ${prNumber}` : ""} — discovers existing Skyramp tests, scans endpoints changed in the diff, loads workspace config, and returns ${maxRecommendations} ranked ADD recommendations.${prNumber ? " Uses PR comment history to avoid re-recommending already-generated tests." : ""}
+2. Call \`skyramp_analyze_test_health\` with the \`stateFile\` from step 1 (skip if zero existing tests found) — scores each existing test for drift against the diff and assigns UPDATE / REGENERATE / VERIFY / ADD actions.
+
+---
+
+## Step 2: Decide — one action per affected test / endpoint
+
+Using the diff, the recommendations, and the health assessment, assign exactly one action to each item:
+
+### For each **existing Skyramp test**:
+- **UPDATE** — the diff touches the endpoint this test covers AND adds/changes fields the test should assert (e.g. new response field, changed status code, renamed path). The test still runs but has a coverage gap or will break.
+- **REGENERATE** — the endpoint was substantially restructured or the test is fundamentally broken by the diff.
+- **VERIFY** — the diff touches related code but the test is unaffected; no action needed.
+- **DELETE** — the endpoint the test covers was removed entirely.
+- **ADD** — existing tests for this endpoint do not capture a new scenario introduced by the diff (e.g. a new flow, a new field combination). A net-new test is needed alongside the existing ones.
+
+### For each **endpoint whose route definition is new in the diff** (no existing Skyramp test):
+- **ADD** — the diff introduced this route; generate a new test.
+- **VERIFY** — the endpoint existed before this diff (only a model/field change touched it); log as a coverage gap but do not generate a test.
+
+### Decision rules (apply in order):
+1. If the diff adds/removes/renames a field in a response this test asserts → **UPDATE** (not ADD).
+2. If the diff adds a **brand-new route definition** (e.g. a new \`@router.get\`, \`@app.route\`, \`router.get()\` line) → **ADD**.
+2.5. If the diff makes an **additive, non-breaking change** to an existing route (e.g. new optional query params, new optional request fields, new optional response fields) AND an existing test already covers that route → **UPDATE** that test to assert the new behavior. Do NOT create a new file.
+3. If an existing test covers the endpoint but the new behavior requires a **distinct setup or workflow** (e.g. a new auth path, a new multi-step flow, a new error/edge-case branch) → **ADD** (alongside the existing test).
+4. If the test is unrelated to the diff → **VERIFY** (no action).
+5. Only use **ADD** for endpoints whose route was introduced in this diff. An endpoint that existed before but now lacks a test is a pre-existing coverage gap — log it in \`additionalRecommendations\`, do NOT generate a test for it.
+6. Do NOT add a new test when an UPDATE to an existing test is the right fix.
+
+Output your decision table:
+\`\`\`
+Test/Endpoint | Action | Reason
+<file or METHOD /path> | <ACTION> | <1 sentence>
+\`\`\`
+
+---
+
+## Step 3: Act
+
+Execute the actions from Step 2. Limit total generated/updated tests to ${maxGenerate}.
+
+### UPDATE
+Edit the existing test file directly:
+- Add missing assertions for new response fields (e.g. \`assert "archived" in resp\` or \`assert resp["archived"] >= 0\`).
+- Fix path/method changes in the test.
+- Do not regenerate — only apply the minimal change needed.
+
+### REGENERATE
+Call the appropriate generation tool to replace the existing test from scratch.
+Use the same filename so it overwrites the old file.
+
+### ADD
+Generate a net-new test. Use a unique descriptive filename to avoid overwriting existing files.
+
+**How to generate each type (for ADD and REGENERATE):**
+- **Integration**: call \`skyramp_scenario_test_generation\` per step (sequentially), then \`skyramp_integration_test_generation\` with the scenario file.
+  Scenario JSON goes in the same \`outputDir\` (e.g. \`tests/scenario_<name>.json\`), not \`.skyramp/\`.
+- **Contract**: call \`skyramp_contract_test_generation\` with \`endpointURL\`, \`method\`, and \`requestData\` for POST/PUT/PATCH.
+  Pass \`apiSchema\` if an OpenAPI spec exists.
+- **Fuzz**: call \`skyramp_fuzz_test_generation\` with \`endpointURL\`, \`method\`, \`requestData\`.
+- **E2E/UI**: only if relevant Playwright traces exist in \`${testDirectory}\`, repo root, or \`.skyramp/\`.
+  Without traces, move to \`additionalRecommendations\` with scenario steps and trace recording instructions.
+- Skip smoke tests entirely.
+
+**Scenario quality:** Verify preconditions before each step (e.g. create before update).
+
+**After generation, fix chaining only:**
+- Path params like \`id = 'id'\` → \`skyramp.get_response_value(prev_response, "id")\`
+- Hardcoded IDs in request bodies → dynamic values from prior response
+- Change ONLY chaining values. Preserve everything else exactly as generated.
+
+After all actions, execute the changed/generated test files and record pass/fail.
+
+### VERIFY / DELETE
+- VERIFY: no file changes. Note in \`testMaintenance\`.
+- DELETE: remove the test file.
+
+---
+
+## Step 4: Submit Report
+
+Call \`skyramp_submit_report\` with \`summaryOutputFile\`: "${summaryOutputFile}".
 
 \`commitMessage\`: under 72 chars, e.g. "add integration tests for /products and /orders"
 
-**newTestsCreated** — list every generated test file (at most ${maxGenerate}):
-   \`testType\`, \`endpoint\`, \`fileName\`, \`description\`, \`scenarioFile\`, \`traceFile\`, \`frontendTrace\`
-   Use the actual file path returned by the generation tool for \`scenarioFile\`.
-   Include scenario JSON files in the git commit alongside test files.
-   Every test file in the commit should appear here. If you over-generated, delete extras first.
-   If no tests were generated (e.g. frontend-only PR without traces), pass an empty array.
+**newTestsCreated** — every file generated or updated (at most ${maxGenerate}):
+   \`testType\`, \`endpoint\`, \`fileName\`, \`description\`, \`scenarioFile\`
+   If action was UPDATE, set \`testType\` to \`"<type> (updated)"\`.
+   If no tests were generated or updated, pass an empty array.
 
-**additionalRecommendations** — remaining recommendations not generated:
-   \`testType\`, \`scenarioName\`, \`priority\`, \`description\`, \`steps\`, artifact paths
+**additionalRecommendations** — items you could not act on (quota exceeded, no traces, etc.):
+   \`testType\`, \`scenarioName\`, \`priority\`, \`description\`, \`steps\`
 
-**businessCaseAnalysis** — based only on PR data and tool outputs.`;
+**businessCaseAnalysis** — 2-3 sentences based only on the diff and tool outputs.`;
 }
 export function registerTestbotPrompt(server) {
     logger.info("Registering testbot prompt");
@@ -144,9 +152,13 @@ export function registerTestbotPrompt(server) {
                 .number()
                 .default(MAX_TESTS_TO_GENERATE)
                 .describe(`Maximum number of tests to generate.`),
+            prNumber: z
+                .number()
+                .optional()
+                .describe("GitHub PR number. Passed to skyramp_analyze_changes to fetch previous TestBot comments for recommendation consistency across commits."),
         },
     }, (args) => {
-        const prompt = getTestbotPrompt(args.prTitle, args.prDescription, args.diffFile, args.testDirectory, args.summaryOutputFile, args.repositoryPath, args.baseBranch, args.maxRecommendations, args.maxGenerate);
+        const prompt = getTestbotPrompt(args.prTitle, args.prDescription, args.diffFile, args.testDirectory, args.summaryOutputFile, args.repositoryPath, args.baseBranch, args.maxRecommendations, args.maxGenerate, args.prNumber);
         AnalyticsService.pushMCPToolEvent("skyramp_testbot_prompt", undefined, {}).catch(() => { });
         return {
             messages: [
@@ -179,7 +191,8 @@ export function registerTestbotResource(server) {
         const param = (name, fallback) => uri.searchParams.get(name) ?? fallback;
         const maxRec = parseInt(uri.searchParams.get("maxRecommendations") || "", 10);
         const maxGen = parseInt(uri.searchParams.get("maxGenerate") || "", 10);
-        const prompt = getTestbotPrompt(param("prTitle", ""), param("prDescription", ""), param("diffFile", ".skyramp_git_diff"), param("testDirectory", "tests"), param("summaryOutputFile", ""), param("repositoryPath", "."), uri.searchParams.get("baseBranch") || undefined, isNaN(maxRec) ? MAX_RECOMMENDATIONS : maxRec, isNaN(maxGen) ? MAX_TESTS_TO_GENERATE : maxGen);
+        const prNum = parseInt(uri.searchParams.get("prNumber") || "", 10);
+        const prompt = getTestbotPrompt(param("prTitle", ""), param("prDescription", ""), param("diffFile", ".skyramp_git_diff"), param("testDirectory", "tests"), param("summaryOutputFile", ""), param("repositoryPath", "."), uri.searchParams.get("baseBranch") || undefined, isNaN(maxRec) ? MAX_RECOMMENDATIONS : maxRec, isNaN(maxGen) ? MAX_TESTS_TO_GENERATE : maxGen, isNaN(prNum) ? undefined : prNum);
         AnalyticsService.pushMCPToolEvent("skyramp_testbot_prompt", undefined, {}).catch(() => { });
         return {
             contents: [

package/build/services/DriftAnalysisService.js

@@ -42,7 +42,7 @@ export class EnhancedDriftAnalysisService {
         if (!(await this.git.checkIsRepo())) {
             throw new Error(`Not a git repository: ${repositoryPath}`);
         }
-        let baseline = baselineCommit ||
+        const baseline = baselineCommit ||
             (await this.getTestBaselineCommit(testFile, repositoryPath));
         // Handle no git history case
         if (!baseline) {

package/build/services/ScenarioGenerationService.js

@@ -129,9 +129,13 @@ ${JSON.stringify(traceRequest, null, 2)}
             .some(v => v.includes("application/json"));
         const responseBody = params.responseBody || (isJsonResponse ? "{}" : "");
         const authHeaderName = params.authHeader || "Authorization";
+        let authValue = params.authToken ?? "";
+        if (!authValue && authHeaderName === "Authorization") {
+            authValue = "Bearer SKYRAMP_PLACEHOLDER_TOKEN";
+        }
         const requestHeaders = {
             "Content-Type": ["application/json"],
-            [authHeaderName]: [params.authToken ?? ""],
+            [authHeaderName]: [authValue],
         };
         return {
             Source: "192.168.65.1:39998",

package/build/services/TestExecutionService.js

@@ -4,6 +4,7 @@ import fs from "fs";
 import { Writable } from "stream";
 import { stripVTControlCharacters } from "util";
 import { logger } from "../utils/logger.js";
+import { buildContainerEnv } from "./containerEnv.js";
 const DEFAULT_TIMEOUT = 300000; // 5 minutes
 const MAX_CONCURRENT_EXECUTIONS = 5;
 export const EXECUTOR_DOCKER_IMAGE = "skyramp/executor:v1.3.13";
@@ -379,30 +380,7 @@ export class TestExecutionService {
                 mountedPaths.add(saveStorageTarget);
             }
         }
-        // Prepare environment variables
-        const env = [
-            `SKYRAMP_TEST_TOKEN=${options.token || ""}`,
-            "SKYRAMP_IN_DOCKER=true",
-        ];
-        // Skyramp-generated tests are standalone HTTP tests that never need host repo
-        // conftest.py files or pytest configuration. --noconftest prevents loading any
-        // conftest in the test directory tree (avoids missing deps like boto3, django).
-        // -c /dev/null overrides all config file discovery (pyproject.toml, pytest.ini,
-        // setup.cfg, tox.ini) so user-repo plugins (e.g. pytest-timeout) not installed
-        // in the executor container don't cause INTERNALERROR at collection time.
-        if (options.language === "python") {
-            env.push(`PYTEST_ADDOPTS=--noconftest -c /dev/null`);
-        }
-        // Add save storage path to environment if provided
-        if (saveStorageTargetPath) {
-            env.push(`PLAYWRIGHT_SAVE_STORAGE_PATH=${saveStorageTargetPath}`);
-        }
-        if (process.env.SKYRAMP_DEBUG) {
-            env.push(`SKYRAMP_DEBUG=${process.env.SKYRAMP_DEBUG}`);
-        }
-        if (process.env.API_KEY) {
-            env.push(`API_KEY=${process.env.API_KEY}`);
-        }
+        const env = buildContainerEnv(options, saveStorageTargetPath);
         // Capture output
         let output = "";
         class DockerStream extends Writable {

package/build/services/TestExecutionService.test.js

@@ -0,0 +1,167 @@
+import { buildContainerEnv } from "./containerEnv.js";
+// Mock dockerode before importing TestExecutionService
+const mockRun = jest.fn();
+const mockListImages = jest.fn();
+jest.mock("dockerode", () => {
+    return jest.fn().mockImplementation(() => ({
+        run: mockRun,
+        listImages: mockListImages,
+    }));
+});
+// Mock fs for workspace/file validation
+jest.mock("fs", () => ({
+    ...jest.requireActual("fs"),
+    accessSync: jest.fn(),
+    existsSync: jest.fn().mockReturnValue(true),
+    readdirSync: jest.fn().mockReturnValue(["test_file.py"]),
+    readFileSync: jest.fn().mockReturnValue(""),
+}));
+// Mock logger
+jest.mock("../utils/logger.js", () => ({
+    logger: {
+        debug: jest.fn(),
+        info: jest.fn(),
+        error: jest.fn(),
+        warning: jest.fn(),
+    },
+}));
+describe("buildContainerEnv", () => {
+    const baseOptions = { token: "test-token", language: "python" };
+    const emptyHostEnv = {};
+    it("always includes SKYRAMP_TEST_TOKEN and SKYRAMP_IN_DOCKER", () => {
+        const env = buildContainerEnv(baseOptions, undefined, emptyHostEnv);
+        expect(env).toContain("SKYRAMP_TEST_TOKEN=test-token");
+        expect(env).toContain("SKYRAMP_IN_DOCKER=true");
+    });
+    it("defaults token to empty string when not provided", () => {
+        const env = buildContainerEnv({ language: "typescript" }, undefined, emptyHostEnv);
+        expect(env).toContain("SKYRAMP_TEST_TOKEN=");
+    });
+    it("adds PYTEST_ADDOPTS for python language", () => {
+        const env = buildContainerEnv(baseOptions, undefined, emptyHostEnv);
+        expect(env).toContain("PYTEST_ADDOPTS=--noconftest");
+    });
+    it("does not add PYTEST_ADDOPTS for non-python language", () => {
+        const env = buildContainerEnv({ ...baseOptions, language: "typescript" }, undefined, emptyHostEnv);
+        expect(env.find((e) => e.startsWith("PYTEST_ADDOPTS="))).toBeUndefined();
+    });
+    it("adds PLAYWRIGHT_SAVE_STORAGE_PATH when saveStoragePath provided", () => {
+        const env = buildContainerEnv(baseOptions, "/tmp/storage", emptyHostEnv);
+        expect(env).toContain("PLAYWRIGHT_SAVE_STORAGE_PATH=/tmp/storage");
+    });
+    it("does not add PLAYWRIGHT_SAVE_STORAGE_PATH when not provided", () => {
+        const env = buildContainerEnv(baseOptions, undefined, emptyHostEnv);
+        expect(env.find((e) => e.startsWith("PLAYWRIGHT_SAVE_STORAGE_PATH="))).toBeUndefined();
+    });
+    describe("base URL env var forwarding", () => {
+        it("forwards SKYRAMP_TEST_BASE_URL from host env", () => {
+            const hostEnv = { SKYRAMP_TEST_BASE_URL: "http://localhost:8000" };
+            const env = buildContainerEnv(baseOptions, undefined, hostEnv);
+            expect(env).toContain("SKYRAMP_TEST_BASE_URL=http://localhost:8000");
+        });
+        it("forwards SKYRAMP_TEST_SERVICE_URL_* vars from host env", () => {
+            const hostEnv = {
+                SKYRAMP_TEST_SERVICE_URL_BACKEND: "http://localhost:8000",
+                SKYRAMP_TEST_SERVICE_URL_FRONTEND: "http://localhost:5173",
+            };
+            const env = buildContainerEnv(baseOptions, undefined, hostEnv);
+            expect(env).toContain("SKYRAMP_TEST_SERVICE_URL_BACKEND=http://localhost:8000");
+            expect(env).toContain("SKYRAMP_TEST_SERVICE_URL_FRONTEND=http://localhost:5173");
+        });
+        it("skips SKYRAMP_TEST_BASE_URL when value is empty", () => {
+            const hostEnv = { SKYRAMP_TEST_BASE_URL: "" };
+            const env = buildContainerEnv(baseOptions, undefined, hostEnv);
+            expect(env.find((e) => e.startsWith("SKYRAMP_TEST_BASE_URL="))).toBeUndefined();
+        });
+        it("does not forward unrelated SKYRAMP_ vars", () => {
+            const hostEnv = {
+                SKYRAMP_TEST_BASE_URL: "http://localhost:8000",
+                SKYRAMP_OTHER_VAR: "should-not-appear",
+                SKYRAMP_TEST_SERVICE_NOT_URL: "also-no",
+            };
+            const env = buildContainerEnv(baseOptions, undefined, hostEnv);
+            expect(env).toContain("SKYRAMP_TEST_BASE_URL=http://localhost:8000");
+            expect(env.find((e) => e.includes("SKYRAMP_OTHER_VAR"))).toBeUndefined();
+            expect(env.find((e) => e.includes("SKYRAMP_TEST_SERVICE_NOT_URL"))).toBeUndefined();
+        });
+    });
+    it("forwards SKYRAMP_DEBUG from host env", () => {
+        const hostEnv = { SKYRAMP_DEBUG: "true" };
+        const env = buildContainerEnv(baseOptions, undefined, hostEnv);
+        expect(env).toContain("SKYRAMP_DEBUG=true");
+    });
+    it("forwards API_KEY from host env", () => {
+        const hostEnv = { API_KEY: "my-key" };
+        const env = buildContainerEnv(baseOptions, undefined, hostEnv);
+        expect(env).toContain("API_KEY=my-key");
+    });
+});
+describe("TestExecutionService.executeTest - Docker env forwarding", () => {
+    // Import after mocks are set up
+    let TestExecutionService;
+    let EXECUTOR_DOCKER_IMAGE;
+    beforeAll(async () => {
+        const mod = await import("./TestExecutionService.js");
+        TestExecutionService = mod.TestExecutionService;
+        EXECUTOR_DOCKER_IMAGE = mod.EXECUTOR_DOCKER_IMAGE;
+    });
+    beforeEach(() => {
+        jest.clearAllMocks();
+        // Simulate image already cached
+        mockListImages.mockResolvedValue([
+            { RepoTags: [EXECUTOR_DOCKER_IMAGE] },
+        ]);
+    });
+    afterEach(() => {
+        // Clean up env vars we set during tests
+        delete process.env.SKYRAMP_TEST_BASE_URL;
+        delete process.env.SKYRAMP_TEST_SERVICE_URL_BACKEND;
+        delete process.env.SKYRAMP_TEST_SERVICE_URL_FRONTEND;
+    });
+    it("passes SKYRAMP_TEST_BASE_URL to Docker container Env", async () => {
+        process.env.SKYRAMP_TEST_BASE_URL = "http://external-host:8000";
+        const mockContainer = { remove: jest.fn().mockResolvedValue(undefined) };
+        mockRun.mockResolvedValue([{ StatusCode: 0 }, mockContainer]);
+        const service = new TestExecutionService();
+        await service.executeTest({
+            testFile: "/workspace/test_file.py",
+            workspacePath: "/workspace",
+            language: "python",
+            testType: "smoke",
+        });
+        expect(mockRun).toHaveBeenCalledTimes(1);
+        const dockerOptions = mockRun.mock.calls[0][3];
+        expect(dockerOptions.Env).toContain("SKYRAMP_TEST_BASE_URL=http://external-host:8000");
+    });
+    it("passes multiple SKYRAMP_TEST_SERVICE_URL_* to Docker container Env", async () => {
+        process.env.SKYRAMP_TEST_SERVICE_URL_BACKEND = "http://host1:8000";
+        process.env.SKYRAMP_TEST_SERVICE_URL_FRONTEND = "http://host2:5173";
+        const mockContainer = { remove: jest.fn().mockResolvedValue(undefined) };
+        mockRun.mockResolvedValue([{ StatusCode: 0 }, mockContainer]);
+        const service = new TestExecutionService();
+        await service.executeTest({
+            testFile: "/workspace/test_file.py",
+            workspacePath: "/workspace",
+            language: "python",
+            testType: "smoke",
+        });
+        const dockerOptions = mockRun.mock.calls[0][3];
+        expect(dockerOptions.Env).toContain("SKYRAMP_TEST_SERVICE_URL_BACKEND=http://host1:8000");
+        expect(dockerOptions.Env).toContain("SKYRAMP_TEST_SERVICE_URL_FRONTEND=http://host2:5173");
+    });
+    it("does not include base URL vars when not set in host env", async () => {
+        const mockContainer = { remove: jest.fn().mockResolvedValue(undefined) };
+        mockRun.mockResolvedValue([{ StatusCode: 0 }, mockContainer]);
+        const service = new TestExecutionService();
+        await service.executeTest({
+            testFile: "/workspace/test_file.py",
+            workspacePath: "/workspace",
+            language: "python",
+            testType: "smoke",
+        });
+        const dockerOptions = mockRun.mock.calls[0][3];
+        const envWithBaseUrl = dockerOptions.Env.filter((e) => e.startsWith("SKYRAMP_TEST_BASE_URL=") ||
+            e.startsWith("SKYRAMP_TEST_SERVICE_URL_"));
+        expect(envWithBaseUrl).toHaveLength(0);
+    });
+});

package/build/services/containerEnv.js

@@ -0,0 +1,35 @@
+/**
+ * Build the environment variable array for the Docker executor container.
+ */
+export function buildContainerEnv(options, saveStoragePath, hostEnv = process.env) {
+    const env = [
+        `SKYRAMP_TEST_TOKEN=${options.token || ""}`,
+        "SKYRAMP_IN_DOCKER=true",
+    ];
+    // Skyramp-generated tests are standalone HTTP tests that never need host repo
+    // conftest.py files. --noconftest prevents loading any conftest in the test
+    // directory tree (avoids missing deps like boto3, celery, django).
+    // Note: we intentionally omit -c /dev/null because it disrupts pytest's rootdir
+    // detection, causing flaky import failures depending on project structure.
+    if (options.language === "python") {
+        env.push(`PYTEST_ADDOPTS=--noconftest`);
+    }
+    if (saveStoragePath) {
+        env.push(`PLAYWRIGHT_SAVE_STORAGE_PATH=${saveStoragePath}`);
+    }
+    // Forward base URL env vars so generated tests can resolve URLs inside the container
+    for (const [key, val] of Object.entries(hostEnv)) {
+        if ((key === "SKYRAMP_TEST_BASE_URL" ||
+            key.startsWith("SKYRAMP_TEST_SERVICE_URL_")) &&
+            val) {
+            env.push(`${key}=${val}`);
+        }
+    }
+    if (hostEnv.SKYRAMP_DEBUG) {
+        env.push(`SKYRAMP_DEBUG=${hostEnv.SKYRAMP_DEBUG}`);
+    }
+    if (hostEnv.API_KEY) {
+        env.push(`API_KEY=${hostEnv.API_KEY}`);
+    }
+    return env;
+}

package/build/tools/generate-tests/generateScenarioRestTool.js

@@ -47,7 +47,13 @@ const scenarioTestSchema = {
     authToken: z
         .string()
         .optional()
-        .describe("Full auth token value to include in the request header. For Authorization headers, include the scheme prefix (e.g., 'Bearer my-token'). For Cookie headers, use the cookie string (e.g., 'session=abc123'). For API key headers, use the raw key. If omitted, the header value is left empty (the CLI injects the real token at runtime)."),
+        .describe("Full auth token value to include in the request header. "
+        + "IMPORTANT: You MUST provide a placeholder value — do NOT omit this parameter for authenticated endpoints. "
+        + "For Authorization headers, pass 'Bearer SKYRAMP_PLACEHOLDER_TOKEN'. "
+        + "For Cookie headers, pass 'session=SKYRAMP_PLACEHOLDER_TOKEN'. "
+        + "For X-API-Key headers, pass 'SKYRAMP_PLACEHOLDER_TOKEN'. "
+        + "The CLI replaces the placeholder with the real token (from SKYRAMP_TEST_TOKEN env var) at runtime. "
+        + "If this is left empty, the generated integration test will send requests without proper auth and fail with 401/403."),
     responseHeaders: z
         .record(z.array(z.string()))
         .optional()

package/build/tools/submitReportTool.js

@@ -63,17 +63,17 @@ export function registerSubmitReportTool(server) {
             newTestsCreated: z
                 .array(newTestSchema)
                 .describe("List of new tests created. Use empty array [] if none."),
+            additionalRecommendations: z
+                .array(additionalRecommendationSchema)
+                .optional()
+                .default([])
+                .describe("Recommended tests that were not generated (lower priority). Include the remaining recommendations from skyramp_recommend_tests that were not implemented."),
             testMaintenance: z
                 .array(testMaintenanceSchema)
                 .describe("List of existing test modifications with before/after execution results. Use empty array [] if none."),
             testResults: z
                 .array(testResultSchema)
                 .describe("List of ALL test execution results. One entry per test executed."),
-            additionalRecommendations: z
-                .array(additionalRecommendationSchema)
-                .optional()
-                .default([])
-                .describe("Recommended tests that were not generated (lower priority). Include the remaining recommendations from skyramp_recommend_tests that were not implemented."),
             issuesFound: z
                 .array(descriptionSchema)
                 .describe("List of issues, failures, or bugs found. Use empty array [] if none."),
@@ -93,9 +93,9 @@ export function registerSubmitReportTool(server) {
         const reportJson = JSON.stringify({
             businessCaseAnalysis: params.businessCaseAnalysis,
             newTestsCreated: params.newTestsCreated,
+            additionalRecommendations: params.additionalRecommendations ?? [],
             testMaintenance: params.testMaintenance,
             testResults: params.testResults,
-            additionalRecommendations: params.additionalRecommendations ?? [],
             issuesFound: params.issuesFound,
             commitMessage: (params.commitMessage ?? "").replace(/[\r\n]+/g, " ").trim().slice(0, 72) || DEFAULT_COMMIT_MESSAGE,
         }, null, 2);