@empiricalrun/test-run 0.13.1 → 0.14.0

package/dist/bin/commands/optimize-shards.js

@@ -0,0 +1,400 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.registerOptimizeShardsCommand = registerOptimizeShardsCommand;
+const reporter_1 = require("@empiricalrun/reporter");
+const child_process_1 = require("child_process");
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const SUITES_DELIMITER = " › ";
+const BATCH_SIZE = 100;
+async function fetchTestHistoryBatch(testCaseIds, dashboardUrl, apiKey, cacheKey) {
+    const url = `${dashboardUrl}/api/test-cases/history/batch`;
+    const response = await fetch(url, {
+        method: "POST",
+        headers: {
+            "Content-Type": "application/json",
+            Authorization: `Bearer ${apiKey}`,
+        },
+        body: JSON.stringify({
+            test_case_ids: testCaseIds,
+            ...(cacheKey && { cache_key: cacheKey }),
+        }),
+    });
+    if (!response.ok) {
+        throw new Error(`Failed to fetch test history: ${response.status}`);
+    }
+    return response.json();
+}
+async function fetchTestHistory(testCaseIds, dashboardUrl, cacheKey) {
+    const apiKey = process.env.EMPIRICALRUN_API_KEY;
+    if (!apiKey) {
+        throw new Error("EMPIRICALRUN_API_KEY environment variable is required");
+    }
+    const result = {};
+    for (let i = 0; i < testCaseIds.length; i += BATCH_SIZE) {
+        const batch = testCaseIds.slice(i, i + BATCH_SIZE);
+        const batchResult = await fetchTestHistoryBatch(batch, dashboardUrl, apiKey, cacheKey);
+        Object.assign(result, batchResult);
+    }
+    return result;
+}
+function getPlaywrightTestList(shard) {
+    const shardArg = shard ? `--shard ${shard}` : "";
+    try {
+        const output = (0, child_process_1.execSync)(`npx playwright test ${shardArg} --list --reporter=json`.trim(), {
+            encoding: "utf-8",
+            stdio: ["pipe", "pipe", "pipe"],
+        });
+        return JSON.parse(output);
+    }
+    catch (error) {
+        if (error.stdout) {
+            try {
+                return JSON.parse(error.stdout);
+            }
+            catch {
+                throw new Error(`Failed to parse playwright JSON output: ${error.message}`);
+            }
+        }
+        throw new Error(`Failed to get test list from playwright: ${error.message}`);
+    }
+}
+function getShardInfoReporterPath() {
+    try {
+        const reporterPath = require.resolve("@empiricalrun/reporter/shard-info-reporter");
+        return reporterPath;
+    }
+    catch {
+        const fallbackPath = path_1.default.resolve(__dirname, "../../../../reporter/dist/shard-info-reporter.js");
+        if (fs_1.default.existsSync(fallbackPath)) {
+            return fallbackPath;
+        }
+        throw new Error("Could not resolve @empiricalrun/reporter/shard-info-reporter. " +
+            "Make sure @empiricalrun/reporter is installed.");
+    }
+}
+function getShardInfoReport() {
+    const reporterPath = getShardInfoReporterPath();
+    try {
+        const output = (0, child_process_1.execSync)(`npx playwright test --list --reporter=${reporterPath}`, {
+            encoding: "utf-8",
+            stdio: ["pipe", "pipe", "pipe"],
+        });
+        return JSON.parse(output);
+    }
+    catch (error) {
+        if (error.stdout) {
+            try {
+                return JSON.parse(error.stdout);
+            }
+            catch {
+                throw new Error(`Failed to parse shard info output: ${error.message}`);
+            }
+        }
+        throw new Error(`Failed to get shard info from playwright: ${error.message}`);
+    }
+}
+function extractSuiteMetadata(suites, configFullyParallel) {
+    const metadata = new Map();
+    function traverse(suite, parentParallelMode) {
+        let effectiveMode;
+        if (suite.parallelMode !== "none") {
+            effectiveMode = suite.parallelMode;
+        }
+        else if (parentParallelMode !== "none") {
+            effectiveMode = parentParallelMode;
+        }
+        else if (configFullyParallel) {
+            effectiveMode = "parallel";
+        }
+        else {
+            effectiveMode = "default";
+        }
+        const specIds = suite.specs.map((s) => s.id);
+        if (specIds.length > 0) {
+            const key = `${suite.file}:${suite.line}`;
+            metadata.set(key, {
+                file: suite.file,
+                line: suite.line,
+                parallelMode: effectiveMode,
+                hasBeforeAllHooks: suite.hasBeforeAllHooks,
+                hasAfterAllHooks: suite.hasAfterAllHooks,
+                specIds,
+            });
+        }
+        for (const child of suite.suites) {
+            traverse(child, effectiveMode);
+        }
+    }
+    for (const suite of suites) {
+        traverse(suite, "none");
+    }
+    return metadata;
+}
+function buildSpecToGroupKeyMap(suiteMetadata) {
+    const specToGroupKey = new Map();
+    for (const [suiteKey, meta] of suiteMetadata) {
+        for (const specId of meta.specIds) {
+            if (meta.parallelMode === "serial" || meta.parallelMode === "default") {
+                specToGroupKey.set(specId, `file:${meta.file}`);
+            }
+            else if (meta.hasBeforeAllHooks || meta.hasAfterAllHooks) {
+                specToGroupKey.set(specId, `hooks:${suiteKey}`);
+            }
+            else {
+                specToGroupKey.set(specId, `test:${specId}`);
+            }
+        }
+    }
+    return specToGroupKey;
+}
+function flattenedSpecsToTestInfos(specs, historyResponse, includeRetries, defaultDuration, specToGroupKey) {
+    const tests = [];
+    for (const spec of specs) {
+        for (const test of spec.tests) {
+            const history = historyResponse[spec.id] || [];
+            let estimatedDuration = defaultDuration;
+            if (history.length > 0) {
+                const durationField = includeRetries
+                    ? "duration_total"
+                    : "duration_per_retry";
+                const durations = history
+                    .map((r) => r[durationField])
+                    .sort((a, b) => a - b);
+                const p75Index = Math.floor(durations.length * 0.75);
+                estimatedDuration =
+                    durations[Math.min(p75Index, durations.length - 1)] ??
+                        defaultDuration;
+            }
+            const baseGroupKey = specToGroupKey?.get(spec.id) ?? `test:${spec.id}`;
+            const groupKey = `${test.projectName}:${baseGroupKey}`;
+            tests.push({
+                id: spec.id,
+                title: spec.title,
+                file: spec.file,
+                projectName: test.projectName,
+                nesting: spec.nesting,
+                suitesString: spec.suitesString,
+                estimatedDuration,
+                groupKey,
+            });
+        }
+    }
+    return tests;
+}
+function buildTestGroups(tests) {
+    const groupMap = new Map();
+    for (const test of tests) {
+        let group = groupMap.get(test.groupKey);
+        if (!group) {
+            group = {
+                key: test.groupKey,
+                tests: [],
+                totalDuration: 0,
+            };
+            groupMap.set(test.groupKey, group);
+        }
+        group.tests.push(test);
+        group.totalDuration += test.estimatedDuration;
+    }
+    return [...groupMap.values()];
+}
+function packGroupsIntoShards(groups, numShards) {
+    // Deterministic sort: by duration DESC, then by key ASC for stability
+    const sorted = [...groups].sort((a, b) => {
+        const durationDiff = b.totalDuration - a.totalDuration;
+        if (durationDiff !== 0)
+            return durationDiff;
+        return a.key.localeCompare(b.key);
+    });
+    const shards = Array.from({ length: numShards }, (_, i) => ({
+        index: i + 1,
+        groups: [],
+        tests: [],
+        totalDuration: 0,
+    }));
+    if (shards.length === 0) {
+        return shards;
+    }
+    for (const group of sorted) {
+        // Deterministic selection: pick lowest index shard when durations are equal
+        let targetShard = shards[0];
+        for (const shard of shards) {
+            if (shard.totalDuration < targetShard.totalDuration) {
+                targetShard = shard;
+            }
+        }
+        targetShard.groups.push(group);
+        targetShard.tests.push(...group.tests);
+        targetShard.totalDuration += group.totalDuration;
+    }
+    return shards;
+}
+function formatTestListLine(test) {
+    const suites = test.nesting.slice(1, -1);
+    const suitesAndTitle = suites.length > 0
+        ? [...suites, test.title].join(SUITES_DELIMITER)
+        : test.title;
+    return `[${test.projectName}] › ${test.file} › ${suitesAndTitle}`;
+}
+function generateTestListContent(shard, workers) {
+    const parallelizedDuration = shard.totalDuration / workers;
+    const totalSeconds = Math.round(parallelizedDuration / 1000);
+    const minutes = Math.floor(totalSeconds / 60);
+    const seconds = totalSeconds % 60;
+    const lines = [
+        `# Shard ${shard.index} - Optimized bin packing`,
+        `# Tests: ${shard.tests.length}`,
+        `# Total duration: ${Math.round(shard.totalDuration)}ms`,
+        `# Estimated with ${workers} workers: ${minutes}m ${seconds}s`,
+        `# Generated: ${new Date().toISOString()}`,
+        "",
+    ];
+    for (const test of shard.tests) {
+        lines.push(formatTestListLine(test));
+    }
+    return lines.join("\n");
+}
+function formatDuration(ms, workers) {
+    const parallelizedDuration = ms / workers;
+    const totalSeconds = Math.round(parallelizedDuration / 1000);
+    const minutes = Math.floor(totalSeconds / 60);
+    const seconds = totalSeconds % 60;
+    return `${minutes}m ${seconds}s`;
+}
+function registerOptimizeShardsCommand(program) {
+    program
+        .command("optimize-shards")
+        .description("Generate optimized shard test-list files using bin packing algorithm")
+        .requiredOption("--shards <count>", "Number of shards to create")
+        .option("--dashboard-url <url>", "Dashboard URL for fetching test history", process.env.DASHBOARD_DOMAIN || "https://dash.empirical.run")
+        .option("--workers <workers>", "Number of parallel workers per shard", "8")
+        .option("--include-retries", "Use total duration including retries (accounts for flakiness)", false)
+        .option("--default-duration <ms>", "Default duration for tests without history", "30000")
+        .option("--output-dir <dir>", "Output directory for test-list files", "./shards")
+        .option("--cache-key <key>", "Cache key for duration data (e.g., commit SHA). Ensures all workers see same data.")
+        .action(async (options) => {
+        const { shards: shardCountStr, dashboardUrl, includeRetries, outputDir, cacheKey, } = options;
+        const workers = parseInt(options.workers, 10);
+        const shardCount = parseInt(shardCountStr, 10);
+        const defaultDuration = parseInt(options.defaultDuration, 10);
+        if (shardCount < 1) {
+            console.error("Shard count must be at least 1");
+            process.exit(1);
+        }
+        console.log(`Fetching test metadata from playwright...`);
+        const shardInfoReport = getShardInfoReport();
+        const allTestsReport = getPlaywrightTestList();
+        const allSpecs = (0, reporter_1.getFlattenedTestList)(allTestsReport.suites);
+        if (allSpecs.length === 0) {
+            console.log("No tests found.");
+            process.exit(0);
+        }
+        const testCaseIds = allSpecs.map((spec) => spec.id);
+        console.log(`Found ${testCaseIds.length} tests. Fetching history...`);
+        let historyResponse = {};
+        try {
+            historyResponse = await fetchTestHistory(testCaseIds, dashboardUrl, cacheKey);
+        }
+        catch (error) {
+            console.error("Failed to fetch test history:", error.message);
+            process.exit(1);
+        }
+        const testsWithHistory = testCaseIds.filter((id) => (historyResponse[id]?.length ?? 0) > 0).length;
+        console.log(`History found for ${testsWithHistory}/${testCaseIds.length} tests`);
+        const configFullyParallel = shardInfoReport.config.fullyParallel;
+        console.log(`Config fullyParallel: ${configFullyParallel}`);
+        const suiteMetadata = extractSuiteMetadata(shardInfoReport.suites, configFullyParallel);
+        const specToGroupKey = buildSpecToGroupKeyMap(suiteMetadata);
+        const serialCount = [...specToGroupKey.values()].filter((k) => k.startsWith("file:")).length;
+        const parallelCount = [...specToGroupKey.values()].filter((k) => k.startsWith("test:")).length;
+        const hooksCount = [...specToGroupKey.values()].filter((k) => k.startsWith("hooks:")).length;
+        console.log(`Test grouping: ${parallelCount} parallel, ${serialCount} serial/default, ${hooksCount} with hooks`);
+        const allTests = flattenedSpecsToTestInfos(allSpecs, historyResponse, includeRetries, defaultDuration, specToGroupKey);
+        console.log(`\n--- Playwright Default Sharding ---`);
+        const playwrightShardEstimates = [];
+        for (let i = 1; i <= shardCount; i++) {
+            const shardReport = getPlaywrightTestList(`${i}/${shardCount}`);
+            const shardSpecs = (0, reporter_1.getFlattenedTestList)(shardReport.suites);
+            const shardTests = flattenedSpecsToTestInfos(shardSpecs, historyResponse, includeRetries, defaultDuration);
+            const totalDuration = shardTests.reduce((sum, t) => sum + t.estimatedDuration, 0);
+            playwrightShardEstimates.push({
+                shard: i,
+                tests: shardTests.length,
+                durationMs: totalDuration,
+            });
+            console.log(`  Shard ${i}/${shardCount}: ${shardTests.length} tests, ~${formatDuration(totalDuration, workers)}`);
+        }
+        const playwrightMaxDuration = Math.max(...playwrightShardEstimates.map((s) => s.durationMs));
+        const playwrightMinDuration = Math.min(...playwrightShardEstimates.map((s) => s.durationMs));
+        console.log(`  Makespan (max shard): ${formatDuration(playwrightMaxDuration, workers)}`);
+        console.log(`  Imbalance: ${formatDuration(playwrightMaxDuration - playwrightMinDuration, workers)}`);
+        console.log(`\n--- Optimized Bin Packing (LPT) ---`);
+        const testGroups = buildTestGroups(allTests);
+        console.log(`  Built ${testGroups.length} test groups from ${allTests.length} tests`);
+        const optimizedShards = packGroupsIntoShards(testGroups, shardCount);
+        for (const shard of optimizedShards) {
+            console.log(`  Shard ${shard.index}/${shardCount}: ${shard.tests.length} tests, ~${formatDuration(shard.totalDuration, workers)}`);
+        }
+        const optimizedMaxDuration = Math.max(...optimizedShards.map((s) => s.totalDuration));
+        const optimizedMinDuration = Math.min(...optimizedShards.map((s) => s.totalDuration));
+        console.log(`  Makespan (max shard): ${formatDuration(optimizedMaxDuration, workers)}`);
+        console.log(`  Imbalance: ${formatDuration(optimizedMaxDuration - optimizedMinDuration, workers)}`);
+        const improvement = ((playwrightMaxDuration - optimizedMaxDuration) /
+            playwrightMaxDuration) *
+            100;
+        console.log(`\n  Improvement: ${improvement.toFixed(1)}% faster makespan`);
+        const absoluteOutputDir = path_1.default.isAbsolute(outputDir)
+            ? outputDir
+            : path_1.default.join(process.cwd(), outputDir);
+        if (!fs_1.default.existsSync(absoluteOutputDir)) {
+            fs_1.default.mkdirSync(absoluteOutputDir, { recursive: true });
+        }
+        console.log(`\n--- Writing Test List Files ---`);
+        for (const shard of optimizedShards) {
+            const content = generateTestListContent(shard, workers);
+            const filePath = path_1.default.join(absoluteOutputDir, `shard-${shard.index}.txt`);
+            fs_1.default.writeFileSync(filePath, content, "utf-8");
+            console.log(`  Written: ${filePath}`);
+        }
+        const comparison = optimizedShards.map((os) => {
+            const pw = playwrightShardEstimates.find((p) => p.shard === os.index);
+            return {
+                shardIndex: os.index,
+                playwrightTestCount: pw.tests,
+                playwrightEstimatedMs: Math.round(pw.durationMs / workers),
+                optimizedTestCount: os.tests.length,
+                optimizedEstimatedMs: Math.round(os.totalDuration / workers),
+            };
+        });
+        const summaryPath = path_1.default.join(absoluteOutputDir, "summary.json");
+        fs_1.default.writeFileSync(summaryPath, JSON.stringify({
+            generatedAt: new Date().toISOString(),
+            shardCount,
+            workers,
+            totalTests: allTests.length,
+            testsWithHistory,
+            testsWithoutHistory: allTests.length - testsWithHistory,
+            includeRetries,
+            defaultDurationMs: defaultDuration,
+            playwright: {
+                makespanMs: Math.round(playwrightMaxDuration / workers),
+                imbalanceMs: Math.round((playwrightMaxDuration - playwrightMinDuration) / workers),
+            },
+            optimized: {
+                makespanMs: Math.round(optimizedMaxDuration / workers),
+                imbalanceMs: Math.round((optimizedMaxDuration - optimizedMinDuration) / workers),
+            },
+            improvementPercent: parseFloat(improvement.toFixed(1)),
+            shards: comparison,
+        }, null, 2), "utf-8");
+        console.log(`  Written: ${summaryPath}`);
+        console.log(`\nDone! Run shards with:`);
+        for (let i = 1; i <= shardCount; i++) {
+            console.log(`  npx playwright test --test-list ${path_1.default.join(outputDir, `shard-${i}.txt`)}`);
+        }
+    });
+}

package/dist/bin/commands/run.d.ts

@@ -0,0 +1,3 @@
+import type { Command } from "commander";
+export declare function registerRunCommand(program: Command): void;
+//# sourceMappingURL=run.d.ts.map

package/dist/bin/commands/run.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"run.d.ts","sourceRoot":"","sources":["../../../src/bin/commands/run.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAkBzC,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,OAAO,QAwJlD"}

package/dist/bin/commands/run.js

@@ -0,0 +1,132 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.registerRunCommand = registerRunCommand;
+const dashboard_1 = require("../../dashboard");
+const cmd_1 = require("../../lib/cmd");
+const run_all_tests_1 = require("../../lib/run-all-tests");
+const run_specific_test_1 = require("../../lib/run-specific-test");
+const types_1 = require("../../types");
+const utils_1 = require("../../utils");
+const config_parser_1 = require("../../utils/config-parser");
+function registerRunCommand(program) {
+    program
+        .command("run", { isDefault: true })
+        .description("Run Playwright tests")
+        .option("-n, --name <test-name>", "Name of the test to run")
+        .option("-s, --suites <suites>", "Suites under which the test is defined")
+        .option("-d, --dir <test-dir>", "Path to the test directory")
+        .option("-f, --file <test-file-path>", "Path to the test file")
+        .option("-p, --project <project-name...>", "Test projects to run")
+        .option("--payload <payload>", "Payload to run tests")
+        .option("--skip-teardown", "This options skips running teardown tests")
+        .option("--forbid-only", `This options forbids the use of ".only" in the test files`)
+        .allowUnknownOption()
+        .action(async (options, command) => {
+        const repoDir = process.cwd();
+        if (options.name && options.forbidOnly) {
+            console.error("--name and --forbid-only options cannot be used together");
+            process.exit(1);
+        }
+        options.project = options.project || ["*"];
+        const optionsToStrip = [
+            "-n",
+            "--name",
+            "--skip-teardown",
+            "-f",
+            "--file",
+            "-d",
+            "--dir",
+            "-p",
+            "--project",
+            "-s",
+            "--suites",
+            "--payload",
+            options.skipTeardown,
+            options.name,
+            options.dir,
+            options.file,
+            options.suites,
+            options.payload,
+            ...options.project,
+        ];
+        const pwOptions = command.args.filter((arg) => !optionsToStrip.includes(arg));
+        const projectName = process.env.PROJECT_NAME || (await (0, utils_1.pickNameFromPackageJson)());
+        if (!projectName) {
+            throw new Error("Project name is required");
+        }
+        const directory = options.dir || "tests";
+        const suites = options.suites && options.suites.trim() !== ""
+            ? options.suites?.split(",")
+            : undefined;
+        let tests = (0, config_parser_1.parseToken)(options.payload)?.tests ||
+            (options.name
+                ? [
+                    {
+                        name: options.name,
+                        dir: directory,
+                        filePath: options.file,
+                        suites,
+                    },
+                ]
+                : undefined);
+        const environmentSlug = process.env.TEST_RUN_ENVIRONMENT || "";
+        const environmentVariables = await (0, dashboard_1.fetchEnvironmentVariables)();
+        const envOverrides = {};
+        environmentVariables.forEach((envVar) => {
+            envOverrides[envVar.name] = envVar.value;
+        });
+        if (Object.keys(envOverrides).length > 0) {
+            console.log(`Loaded environment variables: ${Object.keys(envOverrides).join(", ")}`);
+        }
+        let environmentSpecificProjects = [];
+        let platform = types_1.Platform.WEB;
+        try {
+            if (environmentSlug) {
+                const { environment, build: latestBuild } = await (0, dashboard_1.fetchEnvironmentAndBuild)(projectName, environmentSlug);
+                platform = environment.platform;
+                environmentSpecificProjects = environment.playwright_projects;
+                if (!process.env.BUILD_URL) {
+                    process.env.BUILD_URL = latestBuild?.build_url;
+                }
+                const buildUrl = process.env.BUILD_URL;
+                await (0, utils_1.downloadBuild)(buildUrl);
+            }
+            const projectFilters = await (0, utils_1.generateProjectFilters)({
+                platform,
+                filteringSets: [...options.project, ...environmentSpecificProjects],
+                repoDir,
+            });
+            if (options.skipTeardown) {
+                await (0, utils_1.handleTeardownSkipFlag)(directory, repoDir);
+            }
+            const hasTestsFilter = tests && tests.length > 0;
+            let commandToRun;
+            if (hasTestsFilter) {
+                commandToRun = await (0, run_specific_test_1.runSpecificTestsCmd)({
+                    tests,
+                    projects: projectFilters,
+                    passthroughArgs: pwOptions.join(" "),
+                    platform,
+                    envOverrides,
+                    repoDir,
+                });
+            }
+            else {
+                commandToRun = (0, run_all_tests_1.runAllTestsCmd)({
+                    projects: projectFilters,
+                    passthroughArgs: pwOptions.join(" "),
+                    platform,
+                    envOverrides,
+                });
+            }
+            const { hasTestPassed } = await (0, cmd_1.runTestsForCmd)(commandToRun, repoDir);
+            if (!hasTestPassed) {
+                process.exit(1);
+            }
+        }
+        catch (error) {
+            console.error("Error while running playwright test:", error.message);
+            process.exit(1);
+        }
+    });
+}