@skyramp/mcp 0.2.8 → 0.2.10-rc.1

package/build/services/TestDiscoveryService.test.js

@@ -308,13 +308,12 @@ describe("TestDiscoveryService", () => {
             writeFile("test_products_api.py", 'import requests\nrequests.get("/api/products")');
             const result = await service.discoverTests(tmpDir, { changedResources: ["orders"] });
             const externalTests = result.tests.filter((t) => t.source === TestSource.External);
-            expect(externalTests.length).toBe(2);
+            // Only the relevant file (orders) is included — low-relevance files are excluded entirely.
+            expect(externalTests.length).toBe(1);
             const ordersTest = externalTests.find(t => t.testFile.includes("orders"));
-            const productsTest = externalTests.find(t => t.testFile.includes("products"));
-            // Relevant file gets endpoint extraction
             expect(ordersTest?.apiEndpoint).toContain("GET /api/orders");
-            // Low-relevance file gets name-only entry (no endpoint extraction)
-            expect(productsTest?.apiEndpoint).toBe("");
+            // Low-relevance products test is not in result.tests at all.
+            expect(externalTests.find(t => t.testFile.includes("products"))).toBeUndefined();
         });
         it("returns relevantExternalTestPaths including files where extraction may have failed", async () => {
             writeFile("test_orders_api.py", 'import requests\nrequests.get("/api/orders")');
@@ -380,27 +379,383 @@ describe("TestDiscoveryService", () => {
             // External tests suppressed in PR-mode-no-endpoints
             expect(externalTests.length).toBe(0);
         });
-        it("returns external tests as name-only with ['unknown'] sentinel (unresolvable resources)", async () => {
+        it("returns no external tests with ['unknown'] sentinel (unresolvable resources)", async () => {
             // When diff endpoints exist but all paths resolve to "unknown" (e.g. decorator-relative
-            // paths like "/{order_id}"), changedResources = ["unknown"]. External tests should be
-            // discovered (not skipped) but scored as low-relevance since "unknown" won't match filenames.
+            // paths like "/{order_id}"), changedResources = ["unknown"]. No external tests score > 0
+            // since "unknown" doesn't match any filename tokens — low-relevance files are excluded.
             writeFile("test_orders_api.py", 'import requests\nrequests.get("/api/orders")');
             writeFile("test_products_api.py", 'import requests\nrequests.get("/api/products")');
             const result = await service.discoverTests(tmpDir, { changedResources: ["unknown"] });
             const externalTests = result.tests.filter(t => t.source === TestSource.External);
-            // External tests discovered (not skipped like empty array)
-            expect(externalTests.length).toBe(2);
-            // But all are low-relevance (name-only) since "unknown" doesn't match any filename tokens
+            expect(externalTests.length).toBe(0);
             expect(result.relevantExternalTestPaths.length).toBe(0);
         });
-        it("low-relevance files have empty apiEndpoint and empty framework in PR mode", async () => {
+        it("low-relevance files are excluded from result.tests in PR mode", async () => {
             writeFile("test_orders_api.py", 'import requests\nrequests.get("/api/orders")');
             writeFile("test_products_api.py", 'import requests\nrequests.get("/api/products")');
             const result = await service.discoverTests(tmpDir, { changedResources: ["orders"] });
+            // products scored 0 — excluded entirely, not even as name-only entry
             const productsTest = result.tests.find(t => t.testFile.includes("test_products_api"));
-            expect(productsTest?.source).toBe(TestSource.External);
-            expect(productsTest?.apiEndpoint).toBe("");
-            expect(productsTest?.framework).toBe("");
+            expect(productsTest).toBeUndefined();
+        });
+    });
+});
+// ---------------------------------------------------------------------------
+// Framework detection ordering — private method behaviour tested via discoverTests
+// ---------------------------------------------------------------------------
+describe("TestDiscoveryService — detectExternalFramework ordering", () => {
+    let service;
+    let tmpDir;
+    beforeEach(() => {
+        service = new TestDiscoveryService();
+        tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "tds-fw-"));
+    });
+    afterEach(() => { fs.rmSync(tmpDir, { recursive: true, force: true }); });
+    function writeFile(rel, content) {
+        const fp = path.join(tmpDir, rel);
+        fs.mkdirSync(path.dirname(fp), { recursive: true });
+        fs.writeFileSync(fp, content, "utf-8");
+        return fp;
+    }
+    it("pytest-playwright file → playwright-ui (not pytest)", async () => {
+        writeFile("tests/test_login.py", "import pytest\nfrom playwright.sync_api import Page\ndef test_login(page: Page):\n  page.goto('http://localhost')");
+        const result = await service.discoverTests(tmpDir, { changedResources: ["login"] });
+        const t = result.tests.find(f => f.testFile.includes("test_login.py"));
+        expect(t?.framework).toBe("playwright-ui");
+    });
+    it("Playwright API-only test (request, no page) → playwright (not playwright-ui)", async () => {
+        writeFile("tests/test_api.py", "from playwright.sync_api import APIRequestContext\ndef test_orders(request: APIRequestContext):\n  request.get('/api/orders')");
+        const result = await service.discoverTests(tmpDir, { changedResources: ["orders"] });
+        const t = result.tests.find(f => f.testFile.includes("test_api.py"));
+        expect(t?.framework).toBe("playwright");
+    });
+    it("pure pytest file → pytest", async () => {
+        writeFile("tests/test_calc.py", "import pytest\ndef test_add():\n  assert 1 + 1 == 2");
+        const result = await service.discoverTests(tmpDir, { changedResources: ["calc"] });
+        const t = result.tests.find(f => f.testFile.includes("test_calc.py"));
+        expect(t?.framework).toBe("pytest");
+    });
+    it("RTL file with it()/expect() → rtl (not jest)", async () => {
+        writeFile("src/Cart.test.tsx", "import { render } from '@testing-library/react';\nit('renders', () => { expect(true).toBe(true); });");
+        const result = await service.discoverTests(tmpDir, { changedResources: ["cart"], hasFrontendChanges: true });
+        const t = result.tests.find(f => f.testFile.includes("Cart.test.tsx"));
+        expect(t?.framework).toBe("rtl");
+    });
+    it("vue-test-utils file → vue-test-utils (not jest)", async () => {
+        writeFile("src/Counter.test.ts", "import { mount } from '@testing-library/vue';\nit('works', () => { expect(true).toBe(true); });");
+        const result = await service.discoverTests(tmpDir, { changedResources: ["counter"], hasFrontendChanges: true });
+        const t = result.tests.find(f => f.testFile.includes("Counter.test.ts"));
+        expect(t?.framework).toBe("vue-test-utils");
+    });
+    it("JS Playwright file using page.goto() → playwright-ui", async () => {
+        writeFile("e2e/login.spec.ts", "import { test } from '@playwright/test';\ntest('login', async ({ page }) => { await page.goto('/'); });");
+        const result = await service.discoverTests(tmpDir, { changedResources: ["login"] });
+        const t = result.tests.find(f => f.testFile.includes("login.spec.ts"));
+        expect(t?.framework).toBe("playwright-ui");
+    });
+    it("JS Playwright using only request fixture → playwright", async () => {
+        writeFile("e2e/api.spec.ts", "import { test } from '@playwright/test';\ntest('api', async ({ request }) => { await request.get('/api'); });");
+        const result = await service.discoverTests(tmpDir, { changedResources: ["api"] });
+        const t = result.tests.find(f => f.testFile.includes("api.spec.ts"));
+        expect(t?.framework).toBe("playwright");
+    });
+});
+// ---------------------------------------------------------------------------
+// detectExternalTestType path rules
+// ---------------------------------------------------------------------------
+describe("TestDiscoveryService — detectExternalTestType path rules", () => {
+    let service;
+    let tmpDir;
+    beforeEach(() => {
+        service = new TestDiscoveryService();
+        tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "tds-type-"));
+    });
+    afterEach(() => { fs.rmSync(tmpDir, { recursive: true, force: true }); });
+    function writeFile(rel, content) {
+        const fp = path.join(tmpDir, rel);
+        fs.mkdirSync(path.dirname(fp), { recursive: true });
+        fs.writeFileSync(fp, content, "utf-8");
+        return fp;
+    }
+    it("file in playwright/ dir → testType=e2e", async () => {
+        writeFile("apps/web/playwright/login.spec.ts", "import { test } from '@playwright/test';\ntest('login', async ({ page }) => { await page.goto('/'); });");
+        const result = await service.discoverTests(tmpDir, { changedResources: ["login"] });
+        const t = result.tests.find(f => f.testFile.includes("login.spec.ts"));
+        expect(t?.testType).toBe("e2e");
+    });
+    it("file with e2e in filename inside e2e/ dir → testType=e2e", async () => {
+        // .e2e.ts in an e2e/ dir is recognized both by dir pattern and filename
+        writeFile("apps/web/e2e/auth.e2e.ts", "import { test } from '@playwright/test';\ntest('auth', async ({ page }) => { await page.goto('/'); });");
+        const result = await service.discoverTests(tmpDir, { changedResources: ["auth"] });
+        const t = result.tests.find(f => f.testFile.includes("auth.e2e.ts"));
+        expect(t?.testType).toBe("e2e");
+    });
+    it("file in api dir (no playwright in path) → NOT e2e", async () => {
+        writeFile("apps/api/v2/tests/login.spec.ts", "import { test } from '@jest/globals';\ntest('login', () => { expect(true).toBe(true); });");
+        const result = await service.discoverTests(tmpDir, { changedResources: ["login"] });
+        const t = result.tests.find(f => f.testFile.includes("login.spec.ts"));
+        expect(t?.testType).not.toBe("e2e");
+    });
+    it("generic spec file (no e2e/playwright signal) → NOT e2e", async () => {
+        writeFile("modules/bookings/login.spec.ts", "import { test } from '@jest/globals';\ntest('login', () => {});");
+        const result = await service.discoverTests(tmpDir, { changedResources: ["login"] });
+        const t = result.tests.find(f => f.testFile.includes("login.spec.ts"));
+        expect(t?.testType).not.toBe("e2e");
+    });
+});
+// ---------------------------------------------------------------------------
+// isFrontendFile edge cases
+// ---------------------------------------------------------------------------
+const { isFrontendFile } = await import("../prompts/test-recommendation/scopeAssessment.js");
+describe("isFrontendFile edge cases", () => {
+    it("pages/api/orders.tsx → false (Next.js API route, /api/ blocks all)", () => {
+        expect(isFrontendFile("pages/api/orders.tsx")).toBe(false);
+    });
+    it("routes/product-card.tsx → true (.tsx in routes/ is page component)", () => {
+        expect(isFrontendFile("routes/product-card.tsx")).toBe(true);
+    });
+    it("routes/orders.ts → false (.ts in routes/ is route handler)", () => {
+        expect(isFrontendFile("routes/orders.ts")).toBe(false);
+    });
+    it("src/components/Cart.tsx → true", () => {
+        expect(isFrontendFile("src/components/Cart.tsx")).toBe(true);
+    });
+    it("src/utils/helpers.ts → false (ambiguous extension, not in frontend dir)", () => {
+        expect(isFrontendFile("src/utils/helpers.ts")).toBe(false);
+    });
+    it("app/modules/booking/BookingList.tsx → true", () => {
+        expect(isFrontendFile("app/modules/booking/BookingList.tsx")).toBe(true);
+    });
+});
+// ---------------------------------------------------------------------------
+// Content-based relevance cap (MAX_CONTENT_PROMOTED = 5)
+// ---------------------------------------------------------------------------
+describe("TestDiscoveryService — content-based relevance cap", () => {
+    let service;
+    let tmpDir;
+    beforeEach(() => {
+        service = new TestDiscoveryService();
+        tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "tds-cap-"));
+    });
+    afterEach(() => { fs.rmSync(tmpDir, { recursive: true, force: true }); });
+    function writeFile(rel, content) {
+        const fp = path.join(tmpDir, rel);
+        fs.mkdirSync(path.dirname(fp), { recursive: true });
+        fs.writeFileSync(fp, content, "utf-8");
+        return fp;
+    }
+    it("promotes at most MAX_CONTENT_PROMOTED=5 files via content match", async () => {
+        // 10 test files that all reference /orders in their content
+        for (let i = 0; i < 10; i++) {
+            writeFile(`tests/test_feature_${i}.py`, `import requests\nrequests.get("/api/checkout_${i}")`);
+            // content-match: all contain "/orders"
+            fs.writeFileSync(path.join(tmpDir, `tests/test_feature_${i}.py`), `import requests\n# tests ordering\nrequests.get("/api/orders/item_${i}")`);
+        }
+        // changedResources=["unknown"] so all score 0 by filename; content scan fires
+        const result = await service.discoverTests(tmpDir, { changedResources: ["orders"] });
+        // At most 5 content-matched files should be in relevantExternalTestPaths
+        expect(result.relevantExternalTestPaths.length).toBeLessThanOrEqual(5);
+    });
+});
+// ---------------------------------------------------------------------------
+// scoreRelevanceByContent regex safety (dot/plus escaped)
+// ---------------------------------------------------------------------------
+describe("TestDiscoveryService — scoreRelevanceByContent regex safety", () => {
+    let service;
+    let tmpDir;
+    beforeEach(() => {
+        service = new TestDiscoveryService();
+        tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "tds-regex-"));
+    });
+    afterEach(() => { fs.rmSync(tmpDir, { recursive: true, force: true }); });
+    function writeFile(rel, content) {
+        const fp = path.join(tmpDir, rel);
+        fs.mkdirSync(path.dirname(fp), { recursive: true });
+        fs.writeFileSync(fp, content, "utf-8");
+        return fp;
+    }
+    it("resource with dot (order.items) does not match orderXitems via unescaped dot", async () => {
+        writeFile("tests/test_order.py", `import requests\nrequests.get("/api/orderXitems")`);
+        // "order.items" in changedResources — dot must be escaped
+        const result = await service.discoverTests(tmpDir, { changedResources: ["order.items"] });
+        // should NOT match (orderXitems ≠ order.items)
+        expect(result.relevantExternalTestPaths.some(p => p.includes("test_order.py"))).toBe(false);
+    });
+    it("resource with dot matches correctly when content has /order.items", async () => {
+        writeFile("tests/test_order_items.py", `import requests\nrequests.get("/api/order.items/list")`);
+        const result = await service.discoverTests(tmpDir, { changedResources: ["order.items"] });
+        expect(result.relevantExternalTestPaths.some(p => p.includes("test_order_items.py"))).toBe(true);
+    });
+    it("resource with plus does not throw SyntaxError", async () => {
+        writeFile("tests/test_v2.py", `import requests\nrequests.get("/api/v2/items")`);
+        // Should not throw
+        await expect(service.discoverTests(tmpDir, { changedResources: ["v2+api"] })).resolves.toBeDefined();
+    });
+});
+// ---------------------------------------------------------------------------
+// hasFrontendChanges=false: no .e2e.ts promotion
+// ---------------------------------------------------------------------------
+describe("TestDiscoveryService — hasFrontendChanges=false guard", () => {
+    let service;
+    let tmpDir;
+    beforeEach(() => {
+        service = new TestDiscoveryService();
+        tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "tds-nofe-"));
+    });
+    afterEach(() => { fs.rmSync(tmpDir, { recursive: true, force: true }); });
+    function writeFile(rel, content) {
+        const fp = path.join(tmpDir, rel);
+        fs.mkdirSync(path.dirname(fp), { recursive: true });
+        fs.writeFileSync(fp, content, "utf-8");
+        return fp;
+    }
+    it("does not promote .e2e.ts files when hasFrontendChanges=false", async () => {
+        writeFile("apps/web/playwright/onboarding.e2e.ts", "import { test } from '@playwright/test';\ntest('onboarding', async ({ page }) => { await page.goto('/onboarding'); });");
+        // changedResources=[] (no API endpoints) + hasFrontendChanges=false (config-only PR)
+        const result = await service.discoverTests(tmpDir, { changedResources: [], hasFrontendChanges: false });
+        expect(result.relevantExternalTestPaths.length).toBe(0);
+        expect(result.tests.length).toBe(0);
+    });
+    it("promotes .e2e.ts files when hasFrontendChanges=true", async () => {
+        writeFile("apps/web/playwright/onboarding.e2e.ts", "import { test } from '@playwright/test';\ntest('onboarding', async ({ page }) => { await page.goto('/onboarding'); });");
+        const result = await service.discoverTests(tmpDir, { changedResources: [], hasFrontendChanges: true });
+        // .e2e.ts should now be promoted (UI_TEST_EXT includes .e2e.ts)
+        expect(result.tests.some(t => t.testFile.includes("onboarding.e2e.ts"))).toBe(true);
+    });
+    // ── Step 3: API test promotion ─────────────────────────────────────────────
+    // Uses a minimal mirror of the prefect-testb file structure to validate that
+    // the content-scored promotion finds test_deployments.py (which imports from
+    // the changed module) but not unrelated schedule/events test files.
+    describe("Step 3 API promotion (hasApiChanges)", () => {
+        // Realistic import lines taken from prefect-testb actual files.
+        // test_deployments.py imports from schemas.actions — matches the import-context
+        // pattern "schemas[./]actions" built when schemas/actions.py changes.
+        const TEST_DEPLOYMENTS_CONTENT = `\
+import pytest
+from httpx import AsyncClient
+from prefect.server.schemas.actions import DeploymentCreate, DeploymentUpdate
+from prefect.server import models, schemas
+
+class TestCreateDeployment:
+    async def test_create_deployment(self, client):
+        response = await client.post("/api/deployments/", json={"name": "test"})
+        assert response.status_code == 201
+
+class TestCreateFlowRunFromDeployment:
+    async def test_create_flow_run(self, client, deployment):
+        response = await client.post(f"/api/deployments/{deployment.id}/create_flow_run", json={})
+        assert response.status_code == 201
+`;
+        // test_deployment_schedules.py imports from schedules, NOT schemas.actions
+        const TEST_DEPLOYMENT_SCHEDULES_CONTENT = `\
+import pytest
+from prefect.server import models, schemas
+from prefect.server.schemas.core import DeploymentSchedule
+
+class TestReadDeploymentSchedules:
+    async def test_read_schedules(self, client, deployment):
+        response = await client.get(f"/api/deployments/{deployment.id}/schedules")
+        assert response.status_code == 200
+`;
+        // events/actions file has no deployment schema imports at all
+        const TEST_EVENTS_ACTIONS_CONTENT = `\
+import pytest
+from prefect.server.events import actions
+from prefect.server.events.clients import AssertingEventsClient
+
+class TestPausingWorkQueue:
+    async def test_pausing(self, session):
+        action = actions.PauseWorkQueue(work_queue_id="test")
+        await action.act(event=None)
+`;
+        // test_schedules.py — no deployment imports
+        const TEST_SCHEDULES_CONTENT = `\
+import pytest
+from prefect.server.schemas.schedules import CronSchedule, IntervalSchedule
+
+class TestCronSchedule:
+    def test_cron_parse(self):
+        s = CronSchedule(cron="0 * * * *")
+        assert s.cron == "0 * * * *"
+`;
+        it("content-matches test_deployments.py when schemas/actions.py changed", async () => {
+            // Mirror the Prefect directory structure
+            writeFile("tests/server/orchestration/api/test_deployments.py", TEST_DEPLOYMENTS_CONTENT);
+            writeFile("tests/server/orchestration/api/test_deployment_schedules.py", TEST_DEPLOYMENT_SCHEDULES_CONTENT);
+            writeFile("tests/events/server/actions/test_pausing_resuming_work_queue.py", TEST_EVENTS_ACTIONS_CONTENT);
+            writeFile("tests/test_schedules.py", TEST_SCHEDULES_CONTENT);
+            // changedResources = ["unknown"] because "actions" is filtered by GENERIC_SCHEMA_NAMES
+            // in analyzeChangesTool. Step 3 is skipped for ["unknown"] sentinel.
+            // So we use the import-context pattern directly via hasApiChanges + a non-unknown resource.
+            // Simulate: schemas/actions.py changed, but it produced a non-generic parent resource.
+            // For this test, use changedResources derived from the parent path "schemas":
+            // "schemas" is in GENERIC_SCHEMA_NAMES so would be filtered → ["unknown"].
+            // Step 3 should be skipped → test_deployments.py not promoted via Step 3.
+            const resultUnknown = await service.discoverTests(tmpDir, {
+                changedResources: ["unknown"],
+            });
+            // Step 3 skipped for ["unknown"] — no content-matched files
+            const promotedUnknown = resultUnknown.relevantExternalTestPaths;
+            expect(promotedUnknown.some(p => p.includes("test_deployments.py"))).toBe(false);
+        });
+        it("content-matches test_deployments.py when api/deployments.py changed (TM17 scenario)", async () => {
+            writeFile("tests/server/orchestration/api/test_deployments.py", TEST_DEPLOYMENTS_CONTENT);
+            writeFile("tests/server/orchestration/api/test_deployment_schedules.py", TEST_DEPLOYMENT_SCHEDULES_CONTENT);
+            writeFile("tests/events/server/actions/test_pausing_resuming_work_queue.py", TEST_EVENTS_ACTIONS_CONTENT);
+            writeFile("tests/test_schedules.py", TEST_SCHEDULES_CONTENT);
+            // changedResources = ["deployments"] (from api/deployments.py route file)
+            // Content pattern: /import\s+[^\n]*["']\/api\/deployments/i or /deployments/ prefix
+            // test_deployments.py calls POST /api/deployments/ → content-matched in Step 3
+            const result = await service.discoverTests(tmpDir, {
+                changedResources: ["deployments"],
+            });
+            const promoted = result.relevantExternalTestPaths;
+            // test_deployments.py: calls /api/deployments/ → content-matched → promoted unconditionally
+            expect(promoted.some(p => p.includes("test_deployments.py"))).toBe(true);
+            // test_deployment_schedules.py: contains /api/deployments/ in URL calls → also content-matched
+            // (the word "deployments" appears in its URL strings even though it tests schedules)
+            // This is expected — it IS in scope for a deployment endpoint change.
+            // test_schedules.py: no /deployments/ URL → scores 0 → fallback only (may or may not be in promoted)
+            // test_pausing_resuming: no /deployments/ URL → fallback only
+        });
+        it("API external tests no longer pre-loaded (agent greps instead)", async () => {
+            // External API tests are discovered by the agent via the grep instruction in the
+            // drift analysis prompt. discoverTests in PR mode (changedResources=[]) returns
+            // only Skyramp tests; external tests are excluded entirely.
+            writeFile("tests/server/orchestration/api/test_deployments.py", [
+                "from prefect.server import utils",
+                "",
+                "async def test_max_active_runs(client, deployment):",
+                "    count = utils.calculate_max_runs(deployment.id, db)",
+                "    assert count >= 0",
+            ].join("\n"));
+            const result = await service.discoverTests(tmpDir, {
+                changedResources: [], // PR mode, no pre-loading of external API tests
+            });
+            // No external tests pre-loaded — agent discovers via grep
+            expect(result.relevantExternalTestPaths.some(p => p.includes("test_deployments.py"))).toBe(false);
+        });
+        it("Step 3 does not run when changedResources is empty (no API changes)", async () => {
+            writeFile("tests/server/orchestration/api/test_deployments.py", TEST_DEPLOYMENTS_CONTENT);
+            const result = await service.discoverTests(tmpDir, {
+                changedResources: [],
+            });
+            // changedResources=[] means PR mode with no endpoints — external tests excluded entirely
+            expect(result.relevantExternalTestPaths).toHaveLength(0);
+        });
+        it("Step 1 path-token scoring finds test_work_queues.py for changedResources=[work_queues]", async () => {
+            // Pattern Q scenario: work_queues.py changed → changedResources=["work_queues"]
+            // test_work_queues.py scores >0 by name (has "work_queues" token)
+            writeFile("tests/server/orchestration/api/test_work_queues.py", 'import pytest\nfrom prefect.server.api import work_queues\n\nasync def test_read_work_queue_status(client, work_queue):\n    r = await client.get(f"/api/work_queues/{work_queue.id}/status")\n    assert r.status_code == 200\n');
+            writeFile("tests/server/orchestration/api/test_flows.py", "import pytest\nfrom prefect.server import schemas\n\nasync def test_create_flow(client): pass\n");
+            const result = await service.discoverTests(tmpDir, {
+                changedResources: ["work_queues"],
+            });
+            const promoted = result.relevantExternalTestPaths;
+            // test_work_queues.py → Step 1 name-scores it → in relevantExternal before Step 3
+            expect(promoted.some(p => p.includes("test_work_queues.py"))).toBe(true);
+            // test_flows.py → scores 0 → not promoted via Step 1
+            // (may appear in Step 3 fallback but is not guaranteed)
         });
     });
 });

package/build/services/TestExecutionService.d.ts

@@ -1,5 +1,5 @@
 import { TestExecutionResult, BatchExecutionResult, TestExecutionOptions, ProgressCallback } from "../types/TestExecution.js";
-export declare const EXECUTOR_DOCKER_IMAGE = "skyramp/executor:v1.3.29";
+export declare const EXECUTOR_DOCKER_IMAGE = "skyramp/executor:v1.3.32";
 export declare const PLAYWRIGHT_CONFIG_FILES: string[];
 export declare const EXCLUDED_MOUNT_ITEMS: string[];
 export declare const MOUNT_NULL_ITEMS: string[];

package/build/services/TestExecutionService.js

@@ -84,14 +84,14 @@ function setupVideoCapture(options, workspacePath, containerMountPath) {
     };
     if (!browserTest)
         return setup;
-    // Per-execution video output directory on host (cleared so a retry never
-    // reports a stale recording from a prior run).
-    const videoSubdir = getVideoSubdir(options.testFile);
+    // Each execution gets a unique video directory so there is nothing to clean up
+    // between runs. Avoids EACCES when the Docker executor (which runs as root) has
+    // left root-owned files in a previous run's directory. The timestamp keeps the
+    // name readable/ordered; the random suffix guarantees uniqueness even when two
+    // executions land in the same millisecond (retries, parallel runs).
+    const videoSubdir = `${getVideoSubdir(options.testFile)}-${Date.now()}-${crypto.randomBytes(3).toString("hex")}`;
     setup.videoHostDir = path.join(workspacePath, ".skyramp", "videos", videoSubdir);
     setup.videoContainerDir = path.join(containerMountPath, ".skyramp", "videos", videoSubdir);
-    if (fs.existsSync(setup.videoHostDir)) {
-        fs.rmSync(setup.videoHostDir, { recursive: true, force: true });
-    }
     fs.mkdirSync(setup.videoHostDir, { recursive: true });
     // TypeScript/JavaScript: generate a per-execution Playwright config with video.
     if (options.language === "typescript" || options.language === "javascript") {
@@ -440,7 +440,7 @@ export class TestExecutionService {
         try {
             fs.accessSync(workspacePath, fs.constants.R_OK);
         }
-        catch (err) {
+        catch {
             throw new Error(`Workspace path does not exist or is not readable: ${workspacePath}`);
         }
         // Validate test file - use basename for safer filename extraction

package/build/services/TestExecutionService.test.js

@@ -521,7 +521,10 @@ describe("TestExecutionService.executeTest - Video capture for browser tests", (
         });
         const dockerOptions = mockRun.mock.calls[0][3];
         const videoSubdir = getVideoSubdir("/workspace/tests/login.spec.ts");
-        const videoMount = dockerOptions.HostConfig.Mounts.find((m) => m.Target === `/home/user/.skyramp/videos/${videoSubdir}`);
+        // Each execution gets a unique per-run dir: `${videoSubdir}-<timestamp>`,
+        // so match by prefix rather than exact equality.
+        const videoMountPrefix = `/home/user/.skyramp/videos/${videoSubdir}`;
+        const videoMount = dockerOptions.HostConfig.Mounts.find((m) => m.Target.startsWith(videoMountPrefix));
         expect(videoMount).toBeDefined();
         expect(videoMount.Type).toBe("bind");
         expect(videoMount.ReadOnly).toBe(false);

package/build/tools/executeSkyrampTestTool.js

@@ -1,4 +1,5 @@
 import { z } from "zod";
+import path from "path";
 import { stripVTControlCharacters } from "util";
 import { TestExecutionService } from "../services/TestExecutionService.js";
 import { AnalyticsService } from "../services/AnalyticsService.js";
@@ -7,43 +8,17 @@ import { getWorkspaceBaseUrl } from "../utils/workspaceAuth.js";
 import { ProgrammingLanguage, TestType } from "../types/TestTypes.js";
 import { StateManager } from "../utils/AnalysisStateManager.js";
 import { logger } from "../utils/logger.js";
+import { toolError } from "../utils/utils.js";
 const TOOL_NAME = "skyramp_execute_test";
 export function registerExecuteSkyrampTestTool(server) {
     server.registerTool(TOOL_NAME, {
-        description: `Execute a Skyramp-generated test in isolated containerized environments for reliable, deterministic testing.
-
-Skyramp is a comprehensive testing platform that generates reliable, out-of-the-box functional and performance tests with AI-powered scaffolding. The test execution engine runs your generated tests in controlled, isolated environments to ensure consistent results.
-
-KEY FEATURES:
-• Isolated Execution: Tests run in containerized environments for consistency
-• Multi-Language Support: Execute tests written in Python, Java, JavaScript, or TypeScript
-• Out-of-the-Box Execution: Generated tests work immediately without modification
-• StateFile Integration: Optionally write execution results back to stateFile for health analysis
-
-REQUIRED PARAMETERS:
-- language: Programming language of your test file (python, javascript, typescript, java)
-- testType: Type of test to execute (smoke, contract, fuzz, integration, load, ui, e2e)
-- testFile: Absolute path to the generated test file to execute
-- token: Authentication token for your service (use empty string if no authentication required)
-
-OPTIONAL PARAMETERS:
-- stateFile: Path to state file from skyramp_analyze_changes. When provided, execution results (passed/failed, errors, duration) will be written back to enrich test health analysis.
-
-AUTHENTICATION:
-Pass an empty string as the token parameter — the server automatically injects SKYRAMP_TEST_TOKEN from the environment. The executor applies it with the correct header defined in workspace.yml:
-- bearer / oauth / token (or custom via authScheme) → Authorization: <scheme> <token>
-- cookie / session                   → Cookie: <token>
-- apiKey                             → <authHeader>: <token>  (header name from workspace.yml)
-- basic                              → Authorization: Basic <token>
-Use an empty string only if the endpoint is confirmed unauthenticated.
-IMPORTANT NOTES:
-- First-time usage may take longer as Docker images are downloaded
-- Tests run in isolated containers for maximum reliability
-- Generated tests are designed to work out-of-the-box without modification
-- Results include detailed execution logs and test outcomes
-- When stateFile is provided, results are merged back for use by skyramp_actions
-
-For detailed documentation visit: https://www.skyramp.dev/docs/quickstart`,
+        description: `Execute a Skyramp-generated test in isolated containerized environments for reliable, deterministic testing. Call this once a test file exists on disk (from a skyramp_*_test_generation tool). First-time execution may take longer while Docker images download — this is expected, not a failure.`,
+        annotations: {
+            readOnlyHint: false,
+            destructiveHint: false,
+            idempotentHint: false,
+            openWorldHint: true,
+        },
         inputSchema: {
             workspacePath: z
                 .string()
@@ -53,13 +28,18 @@ For detailed documentation visit: https://www.skyramp.dev/docs/quickstart`,
                 .describe("Programming language of the test file to execute (e.g., python, javascript, typescript, java)"),
             testType: z
                 .nativeEnum(TestType)
-                .describe("Type of the test to execute."),
+                .describe("Type of the test to execute (e.g., smoke, contract, fuzz, integration, load, ui, e2e)."),
             testFile: z
                 .string()
-                .describe("ALWAYS USE ABSOLUTE PATH to the test file to execute"),
+                .describe("Absolute path to the test file to execute."),
             token: z
                 .string()
-                .describe("Skyramp authentication token for test execution. USE EMPTY STRING WHEN USER CONFIRMS 'No token required'"),
+                .optional()
+                .describe("Explicit Skyramp authentication token for test execution. Omit this parameter to use SKYRAMP_TEST_TOKEN from the environment. An empty string is passed through as a literal token value, not a 'no auth' signal — use `unauthenticated: true` for tests that must run without credentials."),
+            unauthenticated: z
+                .boolean()
+                .optional()
+                .describe("Set true to force this test execution to carry NO auth token, even if SKYRAMP_TEST_TOKEN is set in the environment or a token was passed. Use for tests that must assert 401/403 unauthenticated behavior."),
             playwrightSaveStoragePath: z
                 .string()
                 .optional()
@@ -106,6 +86,10 @@ For detailed documentation visit: https://www.skyramp.dev/docs/quickstart`,
         let didSetSkyrampBaseUrl = false;
         let useHostNetwork = false;
         try {
+            if (!path.isAbsolute(params.testFile)) {
+                errorResult = toolError(`testFile must be an absolute path, got: ${params.testFile}`);
+                return errorResult;
+            }
             // Send initial progress
             await sendProgress(0, 100, "Starting test execution...");
             // Always resolve workspace config for dockerNetwork (host networking)
@@ -119,25 +103,25 @@ For detailed documentation visit: https://www.skyramp.dev/docs/quickstart`,
                         didSetSkyrampBaseUrl = true;
                     }
                     else if (candidates.length > 0) {
-                        return {
-                            content: [{
-                                    type: "text",
-                                    text: [
-                                        `Cannot determine SKYRAMP_TEST_BASE_URL — test file matches multiple services:`,
-                                        ...candidates.map((c) => `  • ${c.serviceName}: ${c.baseUrl}`),
-                                        ``,
-                                        `Re-invoke with SKYRAMP_TEST_BASE_URL set to the correct service URL, or make each service's testDirectory unique in .skyramp/workspace.yml.`,
-                                    ].join("\n"),
-                                }],
-                            isError: true,
-                        };
+                        errorResult = toolError([
+                            `Cannot determine SKYRAMP_TEST_BASE_URL — test file matches multiple services:`,
+                            ...candidates.map((c) => `  • ${c.serviceName}: ${c.baseUrl}`),
+                            ``,
+                            `Re-invoke with SKYRAMP_TEST_BASE_URL set to the correct service URL, or make each service's testDirectory unique in .skyramp/workspace.yml.`,
+                        ].join("\n"));
+                        return errorResult;
                     }
                 }
             }
             const executionService = new TestExecutionService();
-            // Fall back to SKYRAMP_TEST_TOKEN when caller passes empty string or omits the token.
-            // This keeps secrets server-side and avoids embedding credential values in prompts.
-            const effectiveToken = params.token || process.env.SKYRAMP_TEST_TOKEN || "";
+            // `unauthenticated: true` forces no auth token regardless of what's passed or
+            // configured, so tests asserting 401/403 unauthenticated behavior can't be broken
+            // by an ambient SKYRAMP_TEST_TOKEN — see SKYR-3874. Otherwise fall back to the
+            // environment token when the caller omits `token`; this keeps secrets server-side
+            // and avoids embedding credential values in prompts.
+            const effectiveToken = params.unauthenticated
+                ? ""
+                : (params.token ?? process.env.SKYRAMP_TEST_TOKEN ?? "");
             // Execute test with progress callback - reports Docker cache/pull status
             const result = await executionService.executeTest({
                 testFile: params.testFile,
@@ -180,16 +164,9 @@ For detailed documentation visit: https://www.skyramp.dev/docs/quickstart`,
             // Progress is already reported by TestExecutionService
             // Only report final status if not already at 100%
             if (result.status !== TestExecutionStatus.Pass) {
-                errorResult = {
-                    content: [
-                        {
-                            type: "text",
-                            text: `Test execution failed: ${stripVTControlCharacters(result.output || "")}
-                **IMPORTANT: IF THE EXECUTION FAILED BECAUSE OF A 401 STATUS CODE, ASK THE USER TO PROVIDE THE AUTHENTICATION TOKEN TO THE TOOL OR A PLACE FROM WHERE WE CAN READ THE TOKEN. DO NOT UPDATE THE GENERATED TEST FILE.**`,
-                        },
-                    ],
-                    isError: true,
-                };
+                errorResult = toolError(`Test execution failed: ${stripVTControlCharacters(result.output || "")}
+
+If this failed with a 401 status code, ask the user for the authentication token (or where to read it from) and retry — leave the generated test file unchanged.`);
                 return errorResult;
             }
             // Success - progress already reported by TestExecutionService
@@ -206,15 +183,7 @@ For detailed documentation visit: https://www.skyramp.dev/docs/quickstart`,
             };
         }
         catch (err) {
-            errorResult = {
-                content: [
-                    {
-                        type: "text",
-                        text: `Test execution failed: ${err.message}`,
-                    },
-                ],
-                isError: true,
-            };
+            errorResult = toolError(`Test execution failed: ${err.message}`);
             return errorResult;
         }
         finally {