@empiricalrun/test-run 0.13.0 → 0.13.2

package/CHANGELOG.md

@@ -1,5 +1,19 @@
 # @empiricalrun/test-run
 
+## 0.13.2
+
+### Patch Changes
+
+- Updated dependencies [1967ac3]
+  - @empiricalrun/r2-uploader@0.9.0
+
+## 0.13.1
+
+### Patch Changes
+
+- Updated dependencies [3ee1aec]
+  - @empiricalrun/r2-uploader@0.8.0
+
 ## 0.13.0
 
 ### Minor Changes

package/README.md

@@ -27,7 +27,15 @@ You can also use the CLI directly with `npx`.
 
 ## 3. Basic Usage
 
-The CLI entry point is defined in `package.json` under `"bin"`. You can invoke it using:
+The CLI entry point is defined in `package.json` under `"bin"`. The CLI provides three subcommands:
+
+```bash
+npx @empiricalrun/test-run run [flags] [Playwright/test arguments]  # Run tests (default)
+npx @empiricalrun/test-run merge [flags]                            # Merge blob reports
+npx @empiricalrun/test-run failed-list <run-id> [flags]             # Build test list from failed run
+```
+
+The `run` command is the default, so you can omit it:
 
 ```bash
 npx @empiricalrun/test-run [flags] [Playwright/test arguments]
@@ -45,7 +53,9 @@ Here, `--retries 0` (and any unrecognized options) get passed on to the underlyi
 
 ## 4. CLI Flags
 
-Below are the main CLI options implemented in `src/bin/index.ts`:
+### `run` command
+
+Below are the main CLI options for the `run` command (implemented in `src/bin/commands/run.ts`):
 | Flag/Option | Description |
 |-----------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------|
 | `-n, --name <test-name>` | The scenario name (title) of the specific test to run. If provided, only that particular test/scenario is run instead of all tests. |
@@ -59,6 +69,35 @@ Below are the main CLI options implemented in `src/bin/index.ts`:
 
 Other unrecognized flags on the command line will be appended as **Playwright arguments**.
 
+### `merge` command
+
+Merges blob reports from sharded test runs:
+
+| Flag/Option | Description |
+|---------------------------|-----------------------------------------------------------------------------|
+| `-b, --blob-dir <path>` | Path to the blob-report directory containing sharded reports. |
+| `-c, --cwd <path>` | Working directory for the merge operation. |
+
+```bash
+npx @empiricalrun/test-run merge --blob-dir ./blob-reports --cwd ./output
+```
+
+### `failed-list` command
+
+Builds a test list from a failed test run for retrying:
+
+| Flag/Option | Description |
+|---------------------------|-----------------------------------------------------------------------------|
+| `<run-id>` | (Required) Test run ID to fetch failed tests from. |
+| `-o, --output <path>` | Output file path for the test list. |
+| `-r, --repo-name <name>` | Repository name for project lookup. |
+| `-p, --repo-path <path>` | Path to local repo to expand serial blocks. |
+| `-v, --verbose` | Enable verbose logging. |
+
+```bash
+npx @empiricalrun/test-run failed-list abc123 --output ./failed-tests.json
+```
+
 ---
 
 ## 5. Environment Variables
@@ -161,4 +200,4 @@ This flow makes sure tests run against the relevant environment or a fresh build
 - Skipping teardown blocks.
 - Working with multiple or specialized Playwright projects.
 
-Explore the source (particularly `src/bin/index.ts` and `src/utils`) for deeper technical insights and advanced customization.
+Explore the source (particularly `src/bin/commands/` and `src/utils`) for deeper technical insights and advanced customization.

package/dist/bin/commands/estimate-time-shard.d.ts

@@ -0,0 +1,3 @@
+import type { Command } from "commander";
+export declare function registerEstimateTimeShardCommand(program: Command): void;
+//# sourceMappingURL=estimate-time-shard.d.ts.map

package/dist/bin/commands/estimate-time-shard.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"estimate-time-shard.d.ts","sourceRoot":"","sources":["../../../src/bin/commands/estimate-time-shard.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AA6DzC,wBAAgB,gCAAgC,CAAC,OAAO,EAAE,OAAO,QA6HhE"}

package/dist/bin/commands/estimate-time-shard.js

@@ -0,0 +1,122 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.registerEstimateTimeShardCommand = registerEstimateTimeShardCommand;
+const reporter_1 = require("@empiricalrun/reporter");
+const child_process_1 = require("child_process");
+async function fetchTestHistory(testCaseIds, dashboardUrl) {
+    const apiKey = process.env.EMPIRICALRUN_API_KEY;
+    if (!apiKey) {
+        throw new Error("EMPIRICALRUN_API_KEY environment variable is required");
+    }
+    const url = `${dashboardUrl}/api/test-cases/history/batch`;
+    const response = await fetch(url, {
+        method: "POST",
+        headers: {
+            "Content-Type": "application/json",
+            Authorization: `Bearer ${apiKey}`,
+        },
+        body: JSON.stringify({ test_case_ids: testCaseIds }),
+    });
+    if (!response.ok) {
+        throw new Error(`Failed to fetch test history: ${response.status}`);
+    }
+    return response.json();
+}
+function calculateEstimatedDuration(history, includeRetries) {
+    if (history.length === 0) {
+        return 0;
+    }
+    const durationField = includeRetries
+        ? "duration_total"
+        : "duration_per_retry";
+    const avgDuration = history.reduce((sum, r) => sum + r[durationField], 0) / history.length;
+    return avgDuration;
+}
+function registerEstimateTimeShardCommand(program) {
+    program
+        .command("estimate-time-shard")
+        .description("Estimate how long a shard will take based on historical data")
+        .requiredOption("--shard <shard>", "Shard identifier in N/M format (e.g., 1/2)")
+        .option("--dashboard-url <url>", "Dashboard URL for fetching test history", process.env.DASHBOARD_URL || "https://dash.empirical.run")
+        .option("--workers <workers>", "Number of parallel workers", "8")
+        .option("--include-retries", "Use total duration including retries (accounts for flakiness)", false)
+        .action(async (options) => {
+        const { shard, dashboardUrl, includeRetries } = options;
+        const workers = parseInt(options.workers, 10);
+        const shardMatch = shard.match(/^(\d+)\/(\d+)$/);
+        if (!shardMatch) {
+            console.error('Invalid shard format. Expected N/M format (e.g., "1/2")');
+            process.exit(1);
+        }
+        const shardIndex = parseInt(shardMatch[1], 10);
+        const totalShards = parseInt(shardMatch[2], 10);
+        if (shardIndex < 1 || shardIndex > totalShards) {
+            console.error(`Invalid shard index. Must be between 1 and ${totalShards}`);
+            process.exit(1);
+        }
+        let jsonReport;
+        try {
+            const output = (0, child_process_1.execSync)(`npx playwright test --shard ${shard} --list --reporter=json`, {
+                encoding: "utf-8",
+                stdio: ["pipe", "pipe", "pipe"],
+            });
+            jsonReport = JSON.parse(output);
+        }
+        catch (error) {
+            if (error.stdout) {
+                try {
+                    jsonReport = JSON.parse(error.stdout);
+                }
+                catch {
+                    console.error("Failed to parse playwright JSON output:", error.message);
+                    process.exit(1);
+                }
+            }
+            else {
+                console.error("Failed to get test list from playwright:", error.message);
+                process.exit(1);
+            }
+        }
+        const flattenedSpecs = (0, reporter_1.getFlattenedTestList)(jsonReport.suites);
+        if (flattenedSpecs.length === 0) {
+            console.log("No tests found for this shard.");
+            process.exit(0);
+        }
+        const testCaseIds = flattenedSpecs.map((spec) => spec.id);
+        let historyResponse;
+        try {
+            historyResponse = await fetchTestHistory(testCaseIds, dashboardUrl);
+        }
+        catch (error) {
+            console.error("Failed to fetch test history:", error.message);
+            process.exit(1);
+        }
+        let totalEstimatedDuration = 0;
+        let testsWithHistory = 0;
+        let testsWithoutHistory = 0;
+        for (const testId of testCaseIds) {
+            const history = historyResponse[testId] || [];
+            const estimatedDuration = calculateEstimatedDuration(history, includeRetries);
+            if (history.length > 0) {
+                testsWithHistory++;
+            }
+            else {
+                testsWithoutHistory++;
+            }
+            totalEstimatedDuration += estimatedDuration;
+        }
+        const parallelizedDuration = totalEstimatedDuration / workers;
+        const totalSeconds = Math.round(parallelizedDuration / 1000);
+        const minutes = Math.floor(totalSeconds / 60);
+        const seconds = totalSeconds % 60;
+        console.log(JSON.stringify({
+            shard,
+            totalTests: testCaseIds.length,
+            workers,
+            testsWithHistory,
+            testsWithoutHistory,
+            estimatedDurationMs: Math.round(parallelizedDuration),
+            estimatedDurationFormatted: `${minutes}m ${seconds}s`,
+        }));
+    });
+}

package/dist/bin/commands/failed-list.d.ts

@@ -0,0 +1,3 @@
+import type { Command } from "commander";
+export declare function registerFailedListCommand(program: Command): void;
+//# sourceMappingURL=failed-list.d.ts.map

package/dist/bin/commands/failed-list.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"failed-list.d.ts","sourceRoot":"","sources":["../../../src/bin/commands/failed-list.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIzC,wBAAgB,yBAAyB,CAAC,OAAO,EAAE,OAAO,QAoCzD"}

package/dist/bin/commands/failed-list.js

@@ -0,0 +1,34 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.registerFailedListCommand = registerFailedListCommand;
+const failed_test_list_1 = require("../../failed-test-list");
+function registerFailedListCommand(program) {
+    program
+        .command("failed-list")
+        .description("Build test list from failed test run")
+        .argument("<run-id>", "Test run ID to fetch failed tests from")
+        .option("-o, --output <path>", "Output file path for the test list")
+        .option("-r, --repo-name <name>", "Repository name for project lookup")
+        .option("-p, --repo-path <path>", "Path to local repo to expand serial blocks")
+        .option("-v, --verbose", "Enable verbose logging")
+        .action(async (runId, options) => {
+        if (!runId) {
+            console.error("Error: run-id argument is required");
+            process.exit(1);
+        }
+        try {
+            const result = await (0, failed_test_list_1.buildTestListFromFailedTestRun)(runId, {
+                outputPath: options.output,
+                repoName: options.repoName,
+                repoPath: options.repoPath,
+                verbose: options.verbose,
+            });
+            console.log(`Found ${result.failedTests.length} failed tests`);
+            console.log(`Test list written to: ${result.outputPath}`);
+        }
+        catch (error) {
+            console.error("Error:", error instanceof Error ? error.message : String(error));
+            process.exit(1);
+        }
+    });
+}

package/dist/bin/commands/merge.d.ts

@@ -0,0 +1,3 @@
+import type { Command } from "commander";
+export declare function registerMergeCommand(program: Command): void;
+//# sourceMappingURL=merge.d.ts.map

package/dist/bin/commands/merge.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"merge.d.ts","sourceRoot":"","sources":["../../../src/bin/commands/merge.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAIzC,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,OAAO,QAgBpD"}

package/dist/bin/commands/merge.js

@@ -0,0 +1,20 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.registerMergeCommand = registerMergeCommand;
+const merge_reports_1 = require("../../lib/merge-reports");
+function registerMergeCommand(program) {
+    program
+        .command("merge")
+        .description("Merge blob reports from sharded test runs")
+        .option("-b, --blob-dir <blob-dir>", "Path to the blob-report directory")
+        .option("-c, --cwd <cwd>", "Working directory")
+        .action(async (options) => {
+        const { success } = await (0, merge_reports_1.mergeReports)({
+            blobDir: options.blobDir,
+            cwd: options.cwd,
+        });
+        if (!success) {
+            process.exit(1);
+        }
+    });
+}

package/dist/bin/commands/optimize-shards.d.ts

@@ -0,0 +1,3 @@
+import type { Command } from "commander";
+export declare function registerOptimizeShardsCommand(program: Command): void;
+//# sourceMappingURL=optimize-shards.d.ts.map

package/dist/bin/commands/optimize-shards.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"optimize-shards.d.ts","sourceRoot":"","sources":["../../../src/bin/commands/optimize-shards.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAgZzC,wBAAgB,6BAA6B,CAAC,OAAO,EAAE,OAAO,QAyQ7D"}