@skyramp/mcp 0.3.0-rc.1 → 0.3.0

package/build/services/TestDiscoveryService.test.js

@@ -326,6 +326,34 @@ describe("TestDiscoveryService", () => {
             // Products file should NOT be in relevantExternalTestPaths
             expect(result.relevantExternalTestPaths.some(p => p.includes("test_products_api"))).toBe(false);
         });
+        it("includes a file matched only by changedSymbols content grep (SKYR-3924 schema-only)", async () => {
+            // Path/URL do not token-match the resource; the file is relevant only because it
+            // references the changed symbol by name — the schema/DTO-only diff case.
+            writeFile("test_schema.py", "from app.schemas import DeploymentCreate\ndef test_ctor():\n    DeploymentCreate(name='x')");
+            writeFile("test_unrelated.py", "def test_noop(): pass");
+            const result = await service.discoverTests(tmpDir, {
+                changedResources: ["workpool"],
+                changedSymbols: ["DeploymentCreate"],
+            });
+            expect(result.relevantExternalTestPaths.some((p) => p.includes("test_schema"))).toBe(true);
+            expect(result.relevantExternalTestPaths.some((p) => p.includes("test_unrelated"))).toBe(false);
+        });
+        it("keeps direct endpoint-call matches uncapped when preciseResources is set", async () => {
+            // MAX_CONTENT_PROMOTED = 5; create 7 files that each POST the changed endpoint but whose
+            // names do not token-match the resource, so they route through endpoint-call matching.
+            for (let i = 0; i < 7; i++) {
+                writeFile(`test_case_${i}.py`, 'import requests\nrequests.post("/api/deployments/")');
+            }
+            const precise = await service.discoverTests(tmpDir, {
+                changedResources: ["deployments"],
+                preciseResources: true,
+            });
+            expect(precise.relevantExternalTestPaths.length).toBe(7);
+            // Without preciseResources the strong endpoint-call matches are capped together with
+            // loose mentions at MAX_CONTENT_PROMOTED.
+            const capped = await service.discoverTests(tmpDir, { changedResources: ["deployments"] });
+            expect(capped.relevantExternalTestPaths.length).toBe(5);
+        });
         it("does full extraction for all files when no changedResources provided", async () => {
             writeFile("test_orders_api.py", 'import requests\nrequests.get("/api/orders")');
             const result = await service.discoverTests(tmpDir);
@@ -437,13 +465,13 @@ describe("TestDiscoveryService — detectExternalFramework ordering", () => {
     });
     it("RTL file with it()/expect() → rtl (not jest)", async () => {
         writeFile("src/Cart.test.tsx", "import { render } from '@testing-library/react';\nit('renders', () => { expect(true).toBe(true); });");
-        const result = await service.discoverTests(tmpDir, { changedResources: ["cart"], hasFrontendChanges: true });
+        const result = await service.discoverTests(tmpDir, { changedResources: ["cart"], changedFrontendFiles: ["src/Cart.tsx"] });
         const t = result.tests.find(f => f.testFile.includes("Cart.test.tsx"));
         expect(t?.framework).toBe("rtl");
     });
     it("vue-test-utils file → vue-test-utils (not jest)", async () => {
         writeFile("src/Counter.test.ts", "import { mount } from '@testing-library/vue';\nit('works', () => { expect(true).toBe(true); });");
-        const result = await service.discoverTests(tmpDir, { changedResources: ["counter"], hasFrontendChanges: true });
+        const result = await service.discoverTests(tmpDir, { changedResources: ["counter"], changedFrontendFiles: ["src/Counter.tsx"] });
         const t = result.tests.find(f => f.testFile.includes("Counter.test.ts"));
         expect(t?.framework).toBe("vue-test-utils");
     });
@@ -593,9 +621,9 @@ describe("TestDiscoveryService — scoreRelevanceByContent regex safety", () =>
     });
 });
 // ---------------------------------------------------------------------------
-// hasFrontendChanges=false: no .e2e.ts promotion
+// UI promotion gating: no changed frontend files ⇒ no .e2e.ts promotion
 // ---------------------------------------------------------------------------
-describe("TestDiscoveryService — hasFrontendChanges=false guard", () => {
+describe("TestDiscoveryService — UI promotion gating (changedFrontendFiles)", () => {
     let service;
     let tmpDir;
     beforeEach(() => {
@@ -609,17 +637,17 @@ describe("TestDiscoveryService — hasFrontendChanges=false guard", () => {
         fs.writeFileSync(fp, content, "utf-8");
         return fp;
     }
-    it("does not promote .e2e.ts files when hasFrontendChanges=false", async () => {
+    it("does not promote .e2e.ts files when no changed frontend files are supplied", async () => {
         writeFile("apps/web/playwright/onboarding.e2e.ts", "import { test } from '@playwright/test';\ntest('onboarding', async ({ page }) => { await page.goto('/onboarding'); });");
-        // changedResources=[] (no API endpoints) + hasFrontendChanges=false (config-only PR)
-        const result = await service.discoverTests(tmpDir, { changedResources: [], hasFrontendChanges: false });
+        // changedResources=[] (no API endpoints) + no changedFrontendFiles (config-only PR) ⇒ Step 2 skipped
+        const result = await service.discoverTests(tmpDir, { changedResources: [] });
         expect(result.relevantExternalTestPaths.length).toBe(0);
         expect(result.tests.length).toBe(0);
     });
-    it("promotes .e2e.ts files when hasFrontendChanges=true", async () => {
+    it("promotes .e2e.ts files relevant to a changed frontend file", async () => {
         writeFile("apps/web/playwright/onboarding.e2e.ts", "import { test } from '@playwright/test';\ntest('onboarding', async ({ page }) => { await page.goto('/onboarding'); });");
-        const result = await service.discoverTests(tmpDir, { changedResources: [], hasFrontendChanges: true });
-        // .e2e.ts should now be promoted (UI_TEST_EXT includes .e2e.ts)
+        const result = await service.discoverTests(tmpDir, { changedResources: [], changedFrontendFiles: ["apps/web/onboarding.tsx"] });
+        // .e2e.ts should be promoted (UI_TEST_EXT includes .e2e.ts; "onboarding" token matches)
         expect(result.tests.some(t => t.testFile.includes("onboarding.e2e.ts"))).toBe(true);
     });
     // ── Step 3: API test promotion ─────────────────────────────────────────────
@@ -759,3 +787,197 @@ class TestCronSchedule:
         });
     });
 });
+// ---------------------------------------------------------------------------
+// SKYR-3941: UI-test promotion (Step 2) is scoped by relevance to the changed
+// frontend files — a frontend PR must not pull in the whole monorepo's UI suite.
+// ---------------------------------------------------------------------------
+describe("TestDiscoveryService — UI promotion scoping (SKYR-3941)", () => {
+    let service;
+    let tmpDir;
+    // Minimal RTL content so the file classifies as an external UI test.
+    const RTL = "import { render } from '@testing-library/react';\nit('renders', () => { expect(true).toBe(true); });";
+    beforeEach(() => {
+        service = new TestDiscoveryService();
+        tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "tds-fe-"));
+    });
+    afterEach(() => { fs.rmSync(tmpDir, { recursive: true, force: true }); });
+    function writeFile(rel, content) {
+        const fp = path.join(tmpDir, rel);
+        fs.mkdirSync(path.dirname(fp), { recursive: true });
+        fs.writeFileSync(fp, content, "utf-8");
+        return fp;
+    }
+    it("promotes UI tests relevant to the changed frontend files, not the whole suite", async () => {
+        writeFile("src/routes/orders/OrderSearch.test.tsx", RTL);
+        writeFile("packages/design-system/icons/__tests__/arrow-small-down.spec.tsx", RTL);
+        // No API resources (Step 1 promotes nothing) — promotion is purely Step 2.
+        const result = await service.discoverTests(tmpDir, {
+            changedResources: [], changedFrontendFiles: ["src/routes/orders/OrderSearch.tsx"],
+        });
+        const promoted = result.relevantExternalTestPaths;
+        expect(promoted.some(p => p.includes("OrderSearch.test.tsx"))).toBe(true);
+        expect(promoted.some(p => p.includes("arrow-small-down.spec.tsx"))).toBe(false);
+    });
+    it("matches a camelCase changed component against a hyphenated test name", async () => {
+        writeFile("src/order-search.spec.tsx", RTL);
+        const result = await service.discoverTests(tmpDir, {
+            changedResources: [], changedFrontendFiles: ["src/OrderSearch.tsx"],
+        });
+        expect(result.relevantExternalTestPaths.some(p => p.includes("order-search.spec.tsx"))).toBe(true);
+    });
+    it("uses the parent directory as the signal when the changed file's basename is generic (index.tsx)", async () => {
+        writeFile("src/orders/OrdersTable.test.tsx", RTL);
+        writeFile("src/billing/BillingPanel.test.tsx", RTL);
+        const result = await service.discoverTests(tmpDir, {
+            changedResources: [], changedFrontendFiles: ["src/orders/index.tsx"], // stem "index" generic → parent "orders" is the token
+        });
+        const promoted = result.relevantExternalTestPaths;
+        expect(promoted.some(p => p.includes("OrdersTable.test.tsx"))).toBe(true);
+        expect(promoted.some(p => p.includes("BillingPanel.test.tsx"))).toBe(false);
+    });
+    it("caps scoped promotion at the backstop even when many tests match", async () => {
+        for (let i = 0; i < 55; i++)
+            writeFile(`src/widgets/widget-${i}.test.tsx`, RTL);
+        const result = await service.discoverTests(tmpDir, {
+            changedResources: [], changedFrontendFiles: ["src/widgets/widget.tsx"], // token "widget" matches all 55
+        });
+        const promotedWidgets = result.relevantExternalTestPaths.filter(p => p.includes("widget-"));
+        expect(promotedWidgets.length).toBe(50); // MAX_UI_PROMOTED backstop
+    });
+    it("promotes a UI test that imports a changed component even when its name shares no token (import-edge recall)", async () => {
+        // "Checkout" shares no token with "PriceLabel" — only the import edge connects them.
+        writeFile("src/features/Checkout.test.tsx", "import { render } from '@testing-library/react';\nimport { PriceLabel } from '../components/PriceLabel';\nit('shows price', () => { render(<PriceLabel />); });");
+        writeFile("src/features/Billing.test.tsx", RTL); // no relevant import, no token overlap
+        const result = await service.discoverTests(tmpDir, {
+            changedResources: [], changedFrontendFiles: ["src/components/PriceLabel.tsx"],
+        });
+        const promoted = result.relevantExternalTestPaths;
+        expect(promoted.some(p => p.includes("Checkout.test.tsx"))).toBe(true); // via import edge
+        expect(promoted.some(p => p.includes("Billing.test.tsx"))).toBe(false);
+    });
+    it("does not let a generic-basename change (index.tsx) leak via import edge, but still matches on the parent dir", async () => {
+        writeFile("src/orders/OrdersList.test.tsx", RTL); // matches parent token "orders" (pass 1)
+        writeFile("src/widgets/Widget.test.tsx", "import { render } from '@testing-library/react';\nimport helpers from '../orders/index';\nit('x', () => {});"); // imports index — must NOT leak
+        const result = await service.discoverTests(tmpDir, {
+            changedResources: [], changedFrontendFiles: ["src/orders/index.tsx"],
+        });
+        const promoted = result.relevantExternalTestPaths;
+        expect(promoted.some(p => p.includes("OrdersList.test.tsx"))).toBe(true); // parent-dir token
+        expect(promoted.some(p => p.includes("Widget.test.tsx"))).toBe(false); // "/index'" signature skipped
+    });
+    it("falls back to capped promote-all when only generic entrypoints changed — never degenerates to zero (SKYR-3941)", async () => {
+        // src/app/index.tsx: stem "index" and parent "app" are both generic → no tokens, and
+        // "index" is skipped for import signatures → no signal at all. A root/entry change is
+        // broadly impactful, so promotion must fall back rather than promote nothing.
+        writeFile("src/routes/orders/OrderList.test.tsx", RTL);
+        writeFile("src/products/ProductList.test.tsx", RTL);
+        const result = await service.discoverTests(tmpDir, {
+            changedResources: [], changedFrontendFiles: ["src/app/index.tsx"],
+        });
+        const promoted = result.relevantExternalTestPaths;
+        // Neither test shares a token with "index"/"app" — without the fallback both would be
+        // dropped (score 0). The no-signal fallback promotes them instead.
+        expect(promoted.some(p => p.includes("OrderList.test.tsx"))).toBe(true);
+        expect(promoted.some(p => p.includes("ProductList.test.tsx"))).toBe(true);
+    });
+});
+// ---------------------------------------------------------------------------
+// Pass 3 — selector-edge promotion: page objects/specs couple to a changed
+// component by SELECTOR (data-testid / CSS class), not by filename or import.
+// ---------------------------------------------------------------------------
+describe("TestDiscoveryService — selector-edge promotion (changedSelectors)", () => {
+    let service;
+    let tmpDir;
+    beforeEach(() => {
+        service = new TestDiscoveryService();
+        tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "tds-sel-"));
+    });
+    afterEach(() => { fs.rmSync(tmpDir, { recursive: true, force: true }); });
+    function writeFile(rel, content) {
+        const fp = path.join(tmpDir, rel);
+        fs.mkdirSync(path.dirname(fp), { recursive: true });
+        fs.writeFileSync(fp, content, "utf-8");
+        return fp;
+    }
+    // A Playwright page object that selects a component by CSS class. It is NOT a
+    // *.test/*.spec file (so passes 1-2 never consider it) and shares no name token
+    // with the changed component, nor does it import it — the exact POM shape that
+    // regressed after SKYR-3941. It still holds the OLD (renamed-away) selector.
+    const POM_CONTENT = `\
+import { Locator, Page } from '@playwright/test';
+
+export class TableComponent {
+  readonly columnFilter: Locator;
+  constructor(private page: Page) {
+    this.columnFilter = page.locator('.t--table-filter-columns-dropdown');
+  }
+  async openFilter() { await this.columnFilter.click(); }
+}
+`;
+    it("promotes a page object that references a changed selector (not name/import matched)", async () => {
+        writeFile("app/client/playwright/page-objects/components/table.component.ts", POM_CONTENT);
+        const result = await service.discoverTests(tmpDir, {
+            changedResources: [],
+            // Changed component shares no token with "table.component" and the POM does not import it.
+            changedFrontendFiles: ["app/client/src/widgets/TableWidgetV2/component/header/actions/filter/CascadeFields.tsx"],
+            // Both the removed (old) and added (new) selector values, as the diff would carry them.
+            changedSelectors: ["t--table-filter-columns-dropdown", "t--tbl-filter-column-select"],
+        });
+        expect(result.tests.some(t => t.testFile.includes("table.component.ts"))).toBe(true);
+        expect(result.relevantExternalTestPaths.some(p => p.includes("table.component.ts"))).toBe(true);
+    });
+    it("does NOT promote the same page object when changedSelectors is absent (the regression baseline)", async () => {
+        writeFile("app/client/playwright/page-objects/components/table.component.ts", POM_CONTENT);
+        const result = await service.discoverTests(tmpDir, {
+            changedResources: [],
+            changedFrontendFiles: ["app/client/src/widgets/TableWidgetV2/component/header/actions/filter/CascadeFields.tsx"],
+            // no changedSelectors → passes 1-2 only → name/import miss → POM dropped
+        });
+        expect(result.tests.some(t => t.testFile.includes("table.component.ts"))).toBe(false);
+    });
+    it("does not promote a page object that references no changed selector", async () => {
+        writeFile("app/client/playwright/page-objects/components/unrelated.component.ts", "import { Page } from '@playwright/test';\nexport class Unrelated { constructor(p: Page) { p.locator('.t--some-other-widget-header'); } }");
+        const result = await service.discoverTests(tmpDir, {
+            changedResources: [],
+            changedFrontendFiles: ["app/client/src/widgets/TableWidgetV2/component/header/actions/filter/CascadeFields.tsx"],
+            changedSelectors: ["t--table-filter-columns-dropdown", "t--tbl-filter-column-select"],
+        });
+        expect(result.tests.some(t => t.testFile.includes("unrelated.component.ts"))).toBe(false);
+    });
+    it("promotes via a changed data-testid value (BEM/testid coupling)", async () => {
+        writeFile("e2e/pages/confirmation-modal.page.ts", "import { Page } from '@playwright/test';\nexport class ConfirmationModal { constructor(p: Page) { this.confirm = p.getByTestId('confirmation-modal-confirm-button'); } }");
+        const result = await service.discoverTests(tmpDir, {
+            changedResources: [],
+            changedFrontendFiles: ["src/modules/ui/layout/modal/components/ConfirmationModal.tsx"],
+            changedSelectors: ["confirmation-modal-confirm-button", "confirmation-modal-primary-button"],
+        });
+        expect(result.tests.some(t => t.testFile.includes("confirmation-modal.page.ts"))).toBe(true);
+    });
+    it("promotes a page object via a dynamic BEM base prefix (shared-component modifier)", async () => {
+        // The changed component composes its class dynamically (`ads-button--${kind}`), so only
+        // the base prefix `ads-button--` is extractable; the POM's concrete modifier classes
+        // (ads-button--primary/secondary) share that prefix and must still be promoted (SKYR-3943).
+        writeFile("app/client/playwright/page-objects/ads-button.page.ts", "import { Page } from '@playwright/test';\nexport class AdsButton { constructor(p: Page) { this.primary = p.locator('.ads-button--primary'); this.secondary = p.locator('.ads-button--secondary'); } }");
+        const result = await service.discoverTests(tmpDir, {
+            changedResources: [],
+            changedFrontendFiles: ["app/client/packages/design-system/ads/src/Button/Button.tsx"],
+            changedSelectors: ["ads-button--"],
+        });
+        expect(result.tests.some(t => t.testFile.includes("ads-button.page.ts"))).toBe(true);
+    });
+    it("caps pass-3 selector promotion at MAX_UI_PROMOTED (no re-flood)", async () => {
+        // 60 page objects all reference the same changed selector; pass 1/2 promote none of them
+        // (names/imports don't match), so pass 3 must cap total promotion at MAX_UI_PROMOTED (50).
+        for (let i = 0; i < 60; i++) {
+            writeFile(`app/client/playwright/page-objects/pom-${i}.page.ts`, `import { Page } from '@playwright/test';\nexport class P${i} { constructor(p: Page) { p.locator('.t--shared-widget-hook'); } }`);
+        }
+        const result = await service.discoverTests(tmpDir, {
+            changedResources: [],
+            changedFrontendFiles: ["app/client/src/widgets/SharedWidget.tsx"],
+            changedSelectors: ["t--shared-widget-hook"],
+        });
+        const promotedPoms = result.tests.filter(t => t.testFile.includes("/pom-")).length;
+        expect(promotedPoms).toBeGreaterThan(0);
+        expect(promotedPoms).toBeLessThanOrEqual(50);
+    });
+});

package/build/services/TestExecutionService.d.ts

@@ -1,5 +1,5 @@
 import { TestExecutionResult, BatchExecutionResult, TestExecutionOptions, ProgressCallback } from "../types/TestExecution.js";
-export declare const EXECUTOR_DOCKER_IMAGE = "skyramp/executor:v1.3.33";
+export declare const EXECUTOR_DOCKER_IMAGE = "skyramp/executor:v1.3.34";
 export declare const PLAYWRIGHT_CONFIG_FILES: string[];
 export declare const EXCLUDED_MOUNT_ITEMS: string[];
 export declare const MOUNT_NULL_ITEMS: string[];
@@ -12,6 +12,12 @@ export declare function isBrowserTest(testType: string): boolean;
  * based on the test file path
  */
 export declare function getVideoSubdir(testFile: string): string;
+/**
+ * Recursively scan a per-execution video output directory for the recorded
+ * `video.webm`. Returns its host path, or undefined if none was produced.
+ * Best-effort: scan errors are logged and treated as "no video".
+ */
+export declare function collectVideoPath(videoHostDir: string): string | undefined;
 /**
  * Detect session file paths referenced in test files
  * Looks for storageState patterns in TypeScript/JavaScript/Python/Java/C# test files
@@ -24,6 +30,11 @@ export declare function getVideoSubdir(testFile: string): string;
  * workspace path at runtime).
  */
 export declare function detectSessionFiles(testFilePath: string): string[];
+/**
+ * Recursively find all files/directories matching names in excludedItems within a directory.
+ * Skips recursing into directories that are themselves excluded.
+ */
+export declare function findExcludedPaths(dir: string, excludedItems: string[]): string[];
 export declare class TestExecutionService {
     private docker;
     private imageReady;

package/build/services/TestExecutionService.js

@@ -10,6 +10,7 @@ import { TestExecutionStatus, } from "../types/TestExecution.js";
 import { TestType } from "../types/TestTypes.js";
 import { buildContainerEnv } from "./containerEnv.js";
 import { SKYRAMP_IMAGE_VERSION } from "../utils/versions.js";
+import { walkDir } from "../utils/fileWalk.js";
 const DEFAULT_TIMEOUT = 300000; // 5 minutes
 const MAX_CONCURRENT_EXECUTIONS = 5;
 export const EXECUTOR_DOCKER_IMAGE = `skyramp/executor:${SKYRAMP_IMAGE_VERSION}`;
@@ -122,24 +123,17 @@ addopts = --video on --output ${setup.videoContainerDir}
  * `video.webm`. Returns its host path, or undefined if none was produced.
  * Best-effort: scan errors are logged and treated as "no video".
  */
-function collectVideoPath(videoHostDir) {
+export function collectVideoPath(videoHostDir) {
     if (!videoHostDir)
         return undefined;
-    const findVideo = (dir) => {
-        for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
-            const fullPath = path.join(dir, entry.name);
-            if (entry.isFile() && entry.name === "video.webm")
-                return fullPath;
-            if (entry.isDirectory()) {
-                const found = findVideo(fullPath);
-                if (found)
-                    return found;
+    try {
+        let videoPath;
+        for (const [entry, fullPath] of walkDir(videoHostDir)) {
+            if (entry.name === "video.webm") {
+                videoPath = fullPath; // first match wins
+                break;
             }
         }
-        return undefined;
-    };
-    try {
-        const videoPath = findVideo(videoHostDir);
         if (videoPath) {
             logger.info(`Video recording found: ${videoPath}`);
         }
@@ -331,24 +325,14 @@ export function detectSessionFiles(testFilePath) {
  * Recursively find all files/directories matching names in excludedItems within a directory.
  * Skips recursing into directories that are themselves excluded.
  */
-function findExcludedPaths(dir, excludedItems) {
+export function findExcludedPaths(dir, excludedItems) {
     const results = [];
-    let entries;
-    try {
-        entries = fs.readdirSync(dir, { withFileTypes: true });
-    }
-    catch {
-        return results;
-    }
-    for (const entry of entries) {
-        const fullPath = path.join(dir, entry.name);
+    const shouldSkipDir = (entry) => excludedItems.includes(entry.name) || EXCLUDED_MOUNT_ITEMS.includes(entry.name);
+    for (const [entry, fullPath] of walkDir(dir, { shouldSkipDir })) {
         // Only shadow files — mounting /dev/null to a directory target causes Docker errors
-        if (entry.isFile() && excludedItems.includes(entry.name)) {
+        if (excludedItems.includes(entry.name)) {
             results.push(fullPath);
         }
-        if (entry.isDirectory() && !excludedItems.includes(entry.name) && !EXCLUDED_MOUNT_ITEMS.includes(entry.name)) {
-            results.push(...findExcludedPaths(fullPath, excludedItems));
-        }
     }
     return results;
 }
@@ -672,6 +656,7 @@ export class TestExecutionService {
             // Start periodic progress reporter during execution
             // Runs in parallel with docker.run() to keep the AI agent informed
             let progressIntervalHandle;
+            let lastProgressPromise = Promise.resolve();
             if (onProgress) {
                 let progressTick = 0;
                 progressIntervalHandle = setInterval(() => {
@@ -680,7 +665,7 @@ export class TestExecutionService {
                     // Progress moves from 40% to 79% during execution phase
                     // Slowly increment to show activity without reaching 80% (reserved for processing)
                     const percent = Math.min(40 + progressTick * 2, 79);
-                    onProgress({
+                    lastProgressPromise = onProgress({
                         phase: "executing",
                         message: `Test running... (${elapsed}s elapsed)`,
                         percent,
@@ -753,12 +738,12 @@ export class TestExecutionService {
                 }
                 throw error; // Re-throw the original error
             }
-            // Report processing phase
-            await onProgress?.({
-                phase: "processing",
-                message: "Test execution completed, processing results...",
-                percent: 80,
-            });
+            // Drain any in-flight progress notification so it flushes to the
+            // transport before the tool result is written. A late notification
+            // races with the result on MCP stdio — if the result arrives first,
+            // Claude Code discards the progress token and treats the stale
+            // notification as a fatal transport error, killing the connection.
+            await lastProgressPromise;
             const duration = Date.now() - startTime;
             const cleanOutput = stripVTControlCharacters(output);
             logger.info("Test execution completed", {
@@ -784,12 +769,6 @@ export class TestExecutionService {
                 exitCode: statusCode,
                 videoPath,
             };
-            // Report completion
-            await onProgress?.({
-                phase: "processing",
-                message: `Test execution completed: ${result.status}`,
-                percent: 100,
-            });
             logger.debug(`Test ${result.status}: ${options.testFile}`);
             return result;
         }

package/build/services/TestGenerationService.d.ts

@@ -36,6 +36,14 @@ export interface BaseTestParams {
     mockPort?: number;
     optionalFields?: boolean;
 }
+/**
+ * Derive the framework actually used for generation from the (normalized)
+ * language + framework params. A blank/whitespace-only framework is treated as
+ * unset — ts/js then default to Playwright, everything else to "". Kept in ONE
+ * place so validateInputs and buildBaseGenerationOptions cannot drift on the
+ * empty-string edge (SKYR-3899).
+ */
+export declare function deriveEffectiveFramework(language: string | undefined, framework: string | undefined): string;
 export declare abstract class TestGenerationService {
     protected client: SkyrampClient;
     constructor();

package/build/services/TestGenerationService.js

@@ -10,6 +10,20 @@ import { logger } from "../utils/logger.js";
 import { normalizeLanguageParams } from "../utils/normalizeParams.js";
 import { stageGeneratedPaths, resolveOutputDir } from "../utils/gitStaging.js";
 import { getTestsRepoDir } from "../utils/AnalysisStateManager.js";
+/**
+ * Derive the framework actually used for generation from the (normalized)
+ * language + framework params. A blank/whitespace-only framework is treated as
+ * unset — ts/js then default to Playwright, everything else to "". Kept in ONE
+ * place so validateInputs and buildBaseGenerationOptions cannot drift on the
+ * empty-string edge (SKYR-3899).
+ */
+export function deriveEffectiveFramework(language, framework) {
+    const fw = (framework ?? "").trim().toLowerCase();
+    if (fw !== "")
+        return fw;
+    const lang = (language ?? "").trim().toLowerCase();
+    return lang === "typescript" || lang === "javascript" ? "playwright" : "";
+}
 export class TestGenerationService {
     client;
     constructor() {
@@ -128,19 +142,53 @@ The generated test file remains unchanged and ready to use as-is.
                 text: "Error: requestData must be either a valid JSON string or an absolute path to a file.",
             });
         }
-        const fw = (params.framework ?? "").toLowerCase();
-        if (fw === "playwright" && params.output && params.output !== "") {
-            const specPattern = /\.(spec|test)\.[tj]s$/;
-            if (!specPattern.test(params.output)) {
-                const parsed = path.parse(params.output);
-                const suggested = /\.[tj]s$/.test(parsed.ext)
-                    ? params.output.replace(/\.[tj]s$/, ".spec.ts")
-                    : params.output + ".spec.ts";
-                errList.content.push({
-                    type: "text",
-                    text: `Error: Playwright requires test files to match *.{spec}.{ts,js} (got "${params.output}"). ` +
-                        `Rename to e.g. ${suggested} so Playwright can discover it.`,
-                });
+        // Enforce output/language consistency using the EFFECTIVE framework — the same
+        // derivation buildBaseGenerationOptions applies (ts/js with no explicit framework
+        // defaults to playwright). Reading the raw params.framework here would miss the UI
+        // happy path, where a test is passed as language: "typescript" WITHOUT a framework
+        // and the TypeScript generator's output would then land in a mismatched output file
+        // (e.g. TypeScript written to ui_test.py — SKYR-3899).
+        if (params.output && params.output !== "") {
+            const lang = (params.language ?? "").trim().toLowerCase();
+            const isPlaywright = deriveEffectiveFramework(params.language, params.framework) ===
+                "playwright";
+            // Playwright JS/TS tests must use the *.spec.{ts,js} naming Playwright
+            // discovers. Python Playwright (pytest-playwright) and all non-playwright
+            // targets validate by their language's file extension instead — so an
+            // unset language under Playwright follows the js/ts spec rule, while an
+            // explicit python/java Playwright test is checked against .py/.java
+            // (SKYR-3899; Playwright supports Python via pytest-playwright).
+            if (isPlaywright && lang !== "python" && lang !== "java") {
+                const specPattern = /\.(spec|test)\.[tj]s$/;
+                if (!specPattern.test(params.output)) {
+                    const parsed = path.parse(params.output);
+                    const suggested = /\.[tj]s$/.test(parsed.ext)
+                        ? params.output.replace(/\.[tj]s$/, ".spec.ts")
+                        : params.output + ".spec.ts";
+                    errList.content.push({
+                        type: "text",
+                        text: `Error: Playwright requires test files to match *.{spec}.{ts,js} (got "${params.output}"). ` +
+                            `Rename to e.g. ${suggested} so Playwright can discover it.`,
+                    });
+                }
+            }
+            else {
+                const expectedExtByLanguage = {
+                    python: ".py",
+                    typescript: ".ts",
+                    javascript: ".js",
+                    java: ".java",
+                };
+                const expected = expectedExtByLanguage[lang || "python"];
+                if (expected && !params.output.toLowerCase().endsWith(expected)) {
+                    const parsed = path.parse(params.output);
+                    const suggested = path.join(parsed.dir, `${parsed.name}${expected}`);
+                    errList.content.push({
+                        type: "text",
+                        text: `Error: language "${lang || "python"}" requires a ${expected} output file (got "${params.output}"). ` +
+                            `Rename to e.g. ${suggested}.`,
+                    });
+                }
             }
         }
         return errList.content.length === 0
@@ -358,10 +406,7 @@ ${result}`;
             method: params.method,
             apiSchema: params.apiSchema ? [params.apiSchema] : [],
             language: params.language ?? "python",
-            framework: params.framework ??
-                (params.language === "typescript" || params.language === "javascript"
-                    ? "playwright"
-                    : ""),
+            framework: deriveEffectiveFramework(params.language, params.framework),
             output: params.output,
             outputDir: params.outputDir,
             force: params.force ?? true,

package/build/services/TestGenerationService.test.js

@@ -11,7 +11,7 @@ import path from "path";
 jest.unstable_mockModule("@skyramp/skyramp", () => ({
     SkyrampClient: jest.fn().mockImplementation(() => ({})),
 }));
-const { TestGenerationService, BaseTestParams } = await import("./TestGenerationService.js");
+const { TestGenerationService, deriveEffectiveFramework } = await import("./TestGenerationService.js");
 const { SkyrampClient } = await import("@skyramp/skyramp");
 class StubService extends TestGenerationService {
     buildGenerationOptions() {
@@ -87,6 +87,72 @@ describe("TestGenerationService — Playwright filename validation", () => {
         expect(playwrightError(validateOutput("Playwright", "bad.ts"))).toBeDefined();
     });
 });
+describe("TestGenerationService — language/output extension consistency (SKYR-3899)", () => {
+    function validateLangOutput(language, framework, output) {
+        const svc = new StubService();
+        return svc.validate({ ...BASE, language, framework, output });
+    }
+    function errorText(result) {
+        for (const c of result.content) {
+            if (c.type === "text")
+                return c.text;
+        }
+        return undefined;
+    }
+    it("rejects TypeScript output written to a .py file with no explicit framework (the reported bug)", () => {
+        // language typescript + no framework derives to playwright, so ui_test.py is
+        // rejected instead of emitting TypeScript into a Python file.
+        const result = validateLangOutput("typescript", "", "ui_test.py");
+        expect(result.isError).toBe(true);
+        expect(errorText(result)).toContain("Playwright requires");
+    });
+    it("accepts a TypeScript UI test named *.spec.ts with no explicit framework", () => {
+        const result = validateLangOutput("typescript", "", "ui_test.spec.ts");
+        expect(result.isError).toBe(false);
+    });
+    it("rejects a Python-language test written to a .ts file", () => {
+        const result = validateLangOutput("python", "", "my_test.ts");
+        expect(result.isError).toBe(true);
+        expect(errorText(result)).toContain('language "python" requires a .py');
+        expect(errorText(result)).toContain("my_test.py");
+    });
+    it("accepts a Python-language test written to a .py file", () => {
+        const result = validateLangOutput("python", "", "my_test.py");
+        expect(result.isError).toBe(false);
+    });
+    it("rejects a mismatched extension even when a non-playwright framework is explicit", () => {
+        const result = validateLangOutput("typescript", "pytest", "foo.py");
+        expect(result.isError).toBe(true);
+        expect(errorText(result)).toContain('language "typescript" requires a .ts');
+    });
+    it("accepts a Python Playwright test (pytest-playwright) written to a .py file", () => {
+        // Playwright supports Python via pytest-playwright — language: "python" +
+        // framework: "playwright" must validate against .py, not the js/ts spec rule.
+        const result = validateLangOutput("python", "playwright", "test_login.py");
+        expect(result.isError).toBe(false);
+    });
+    it("rejects a Python Playwright test written to a .spec.ts file", () => {
+        const result = validateLangOutput("python", "playwright", "foo.spec.ts");
+        expect(result.isError).toBe(true);
+        expect(errorText(result)).toContain('language "python" requires a .py');
+    });
+});
+describe("deriveEffectiveFramework — validation/generation share one derivation (SKYR-3899)", () => {
+    it("derives playwright for ts/js when framework is undefined", () => {
+        expect(deriveEffectiveFramework("typescript", undefined)).toBe("playwright");
+        expect(deriveEffectiveFramework("javascript", undefined)).toBe("playwright");
+    });
+    it("treats a blank/whitespace framework as unset (no drift vs undefined)", () => {
+        // The empty-string edge the two call sites used to disagree on: validation
+        // derived playwright while generation kept "". Both now agree.
+        expect(deriveEffectiveFramework("typescript", "")).toBe("playwright");
+        expect(deriveEffectiveFramework("typescript", "   ")).toBe("playwright");
+    });
+    it("keeps an explicit framework and defaults non-js/ts to empty", () => {
+        expect(deriveEffectiveFramework("typescript", "pytest")).toBe("pytest");
+        expect(deriveEffectiveFramework("python", undefined)).toBe("");
+    });
+});
 describe("TestGenerationService — authType/authScheme not passed to library", () => {
     let mockGenerateRestTest;
     beforeEach(() => {

package/build/tools/auth/loginTool.js

@@ -52,7 +52,7 @@ Logging into Skyramp provides access to additional platform features and service
             const recordParams = {
                 prompt: params.prompt,
             };
-            AnalyticsService.pushMCPToolEvent(TOOL_NAME, errorResult, recordParams);
+            AnalyticsService.pushMCPToolEvent(TOOL_NAME, errorResult, recordParams).catch((err) => { logger.warning("Analytics event failed", { error: String(err) }); });
         }
     });
 }