@parity/product-lab 0.0.1-placeholder → 0.1.0

package/entrypoints/cli/src/test-command.js

@@ -0,0 +1,597 @@
+// Copyright 2026 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+import { randomBytes } from "node:crypto";
+import { mkdirSync, writeFileSync } from "node:fs";
+import { isAbsolute, join, relative, resolve } from "node:path";
+import { parseArgs } from "node:util";
+import { RaceJitter, RaceSeed } from "@parity/product-lab/domain";
+import { BASELINES_VARIABLE, CONTROL_RUN_VARIABLE, DEVICE_VARIABLE, slug } from "@parity/product-lab/domain/checks";
+import { PLAIN_OUTPUT_ENV } from "@parity/product-lab/domain/features";
+import { resolveRunVersions } from "./versions-command.js";
+import { runMutateCommand } from "./mutate-command.js";
+import { runTool } from "./tool-run.js";
+import { runEvidence } from "./reports/evidence-report.js";
+import { writeReports } from "./reports/write-reports.js";
+import { NO_VARIABLES } from "./environment.js";
+import { inWindowLine, inWindowRequest, windowRefusal } from "./test-in-window.js";
+import { controlRuns, NOT_RUN_EXIT, runEveryControl } from "./every-control.js";
+import { notForChain, sharedControls } from "./shared-controls.js";
+import { runSuiteCommand, videoEnv } from "./suite-run-command.js";
+import { failedRunsHostLogs, hostLogsWords, PlainOutputFilter, plainEnding, raceRunLines, recordingHostLogs, runRecording, workerLine, workersEnding } from "./plain-output.js";
+export const TEST_USAGE = `Usage: product-lab test [feature...] [--config lab.config.ts] [--cucumber <file>] [--tags <expr>] [--name <regex>]
+                        [--environment <id>] [--report-dir <dir>] [--dry-run] [--candidate <component>@<selector>…]
+                        [--negative-control] [--rebuild-fixtures]
+                        [--rerun-failures <n>] [--race <runs> [--race-max-ms <ms>] [--race-seed <seed>]] [--root-calls]
+                        [--workers <n>] [--video] [--verbose] [--own-lab]
+       product-lab test --mutate [feature...] [--tags <expr>] [--name <regex>] [--max <n>] [--plan] [--report-dir <dir>]
+
+Run the lab's feature files. With a Cucumber configuration next to lab.config.ts, that
+configuration and its step files; without one, the bundled run: product-lab's own Cucumber,
+steps generated from the lab's registered sentences, and the custom step() files its steps
+globs list (no cucumber.js to maintain). Each scenario is planned before any step runs.
+Feature files must be the lab's (its features globs, else every .feature under its
+directory); without any, the configuration's own selection runs, or in a bundled run every
+feature not tagged @negative-control.
+With this lab's window open (bun run lab), the run goes to the window's lab: in its process, on the
+environment it runs, with product-lab's bundled steps, so it shows there live while its output
+streams here. A flag that needs a lab of its own is refused there; --own-lab runs one instead.
+  --config <file|dir>      lab.config.ts (default: ./lab.config.ts)
+  --own-lab                a lab of its own even when this lab's window is open (it starts its own environment on
+                           the lab's ports, so close the window first)
+  --cucumber <file>        Cucumber configuration (default: cucumber.{ts,mjs,js,cjs} next to lab.config.ts)
+  --tags <expr>            narrow by a tag expression
+  --name <regex>           narrow by scenario name
+  --environment <id>       exported to the run as PRODUCT_LAB_ENVIRONMENT (a configured environment)
+  --keep                   keep chain and people as the last kept start left them (exported as PRODUCT_LAB_KEEP=1);
+                           without it a run starts clean
+  --report-dir <dir>       where the run keeps its report, log and attachments
+                           (default: <config dir>/.product-lab/test/<run id>)
+  --dry-run                discover and report without running steps or hooks; refused when product-lab lint
+                           reports an error in the selected features (a step that does not plan)
+  --candidate <c>@<sel>    try another version of a component for this run (repeatable); versions resolve
+                           once at run start and are recorded as a run manifest; lab.lock.json is untouched
+  --rebuild-fixtures       a scenario tagged @fixture:<name> starts from that saved fixture (restored before it,
+                           so no scenario sees another's changes; @fixture-shared on a feature or Rule restores
+                           once for its scenarios in a row); a missing or stale one refuses the scenario with its
+                           reasons, unless this flag: then its setup runs and is saved first (on a state nothing
+                           changed since the environment started). Without --environment, a run whose fixtures
+                           all belong to one environment starts that one
+  --verbose                print the run as Cucumber and the lab print it (progress dots, the lab's own lines, stack
+                           traces). Without it (backlog 186): the start, each scenario's steps with their time and what
+                           each checked on the screen and on chain, a failure with the lab's own answer, the result and
+                           what to look at; the lab's own lines stay in the run's log (cucumber.log). With
+                           GITHUB_ACTIONS=true the phases and scenarios are groups and a failed step an annotation.
+                           The same for each row of a --suite, each worker of --workers and each run of --race
+  --video                  record each identity's headless page as video into the recordings directory's videos/;
+                           each scenario's recording names its videos and when each page opened
+                           (PRODUCT_LAB_RECORDINGS_VIDEO=on; with --suite too; product-lab report comment)
+  --update-baselines       a scenario's visual check (@visual, or the lab's checks.visual.all) replaces its
+                           baselines beside the feature with this run's screenshots (PRODUCT_LAB_BASELINES=update;
+                           strict in CI fails a step without one; default: a missing baseline is written)
+  --mutate                 test the tests (backlog 38): each positive assertion step's value is changed on purpose (a
+                           string gets " (changed by product-lab)", an int 1 more) in a copy of its feature file, and
+                           the scenario is run as a negative control that must fail at that step: killed, survived (the
+                           assertion never checked the value: exit 1) or inconclusive. Absence checks ("do not
+                           include", "at most"), outputs, step groups, custom steps and outlines are not changed, each
+                           with why. --plan lists the mutants and stops; --max runs at most n; the report is
+                           mutations.md and mutations.json in --report-dir (default <lab>/.product-lab/mutants/<run>)
+  --devices <names|lab>    run the selection once per device profile, one after the other ("Pixel 7,Desktop", or
+                           "lab" for defineLab({ devices })): every identity's app emulates the device's viewport,
+                           pixel ratio, touch and user agent (PRODUCT_LAB_DEVICE), visual baselines are per device,
+                           and --report-dir gets one folder per device; passes only when every device passes
+  --workers <n>            run the feature files given on n workers at once, each with the lab's own variables
+                           for its ports, state and caches (defineLab({ workers })); passes only when every
+                           worker passes and no two reached the same address (record: product-lab-workers.json)
+  --negative-control       the run must fail at exactly one scenario step (not a hook) as its tags say: at
+                           its scenario's last step (@fails-at-step-N: that one), from that step's own check,
+                           never for an infrastructure reason (@fails-with:<reason>: for that reason only);
+                           the verdict shows the failure's reason and message. With no feature path: every
+                           tagged control, one run each
+                           (@runs-with:keep,rebuild-fixtures adds those flags), and a summary
+  --root-calls             allow this run's root calls (the chain's Root origin through its own Sudo pallet), for
+                           an environment the lab declares them for (defineLab({ rootCalls })); each is still
+                           decided and checked: never on a mainnet, only allowed calls, only with the chain's key
+  --rerun-failures <n>     flake detection: after a failed run, rerun each failed scenario up to n times (1-20)
+                           from the state it started on, through its run bundle (product-lab reproduce --reruns),
+                           and report it flaky (a rerun did not fail) or failing; one bundle per scenario keeps
+                           every run. The exit code is unchanged: flaky never counts as passing
+  --race <runs>            race hunting: run the selection <runs> times (1-100), each run with its own seed;
+                           before each step an identity takes after another identity's, the lab waits a time drawn
+                           from the seed (and in a run-together group before every step), so the actors interleave
+                           differently; one actor's steps never change order. Prints the seed of every run that
+                           failed: --race 1 --race-seed <seed> runs it again with the same waits, and its run
+                           bundle reproduces with them (product-lab reproduce). Passes only when every run passes
+  --race-max-ms <ms>       the longest wait before a step (default 300, at most 60000)
+  --race-seed <seed>       the first run's seed (8 hex digits; default: random); later runs' are drawn from it
+  Exit code: 0 when the run passed (a negative control: when it failed as intended), 1 otherwise.
+  A run in which no scenario ran fails.
+
+       product-lab test --mode <name> [--base <ref>] [--changed <path>…] [--row <id>…] [--plan]
+                        [--config lab.config.ts] [--report-dir <dir>] [--dry-run] [--rebuild-fixtures]
+
+Run one of the lab's modes (defineLab({ modes })): its environments × hosts × versions × devices
+(one row each, named by what varies), its tags, only the affected scenarios when onlyAffected
+(changes to feature files only select those files), its workers (rows at once; one combination
+shared out by feature file) and its budget. The same options and reports as --suite.
+
+       product-lab test --suite pr|merge|nightly [--base <ref>] [--changed <path>…] [--row <id>…] [--plan]
+                        [--config lab.config.ts] [--report-dir <dir>] [--dry-run] [--rebuild-fixtures]
+
+Run one of the lab's suites (defineLab({ suites })): pr (the affected scenarios of its tags,
+locked, within its budget), merge (every non-manual scenario, locked) or nightly (the
+host/runtime matrix rows it declares). @manual and @negative-control scenarios never run in a
+suite. Rows run one after the other; a row that cannot run here is reported unsupported
+(never passing) and a suite that selects nothing fails and says why.
+  --base <ref>             affected selection: diff against the merge base of <ref> and HEAD, plus
+                           uncommitted and untracked files (default $PRODUCT_LAB_BASE, else main)
+  --changed <path>         affected selection: these changed paths (relative to the lab), not git
+  --row <id>               only this row of the suite (repeat for more; an id it does not declare is refused)
+  --plan                   print what the suite would run and exit (0 when something would run)
+  Reports (every run): junit.xml and report.html (self-contained) beside product-lab-test.json;
+  a suite also writes product-lab-suite.json (with counts and measured timings) and summary.md,
+  each row's run in <report dir>/<row>.`;
+function summary(record) {
+    const steps = record.scenarios.flatMap((scenario) => scenario.steps.filter((step) => !step.hidden));
+    const count = (status) => steps.filter((step) => step.status === status).length;
+    const scenarios = record.scenarios.length;
+    const skipped = record.scenarios.filter((item) => item.status === "skipped").length;
+    return `${scenarios} scenario(s) (${record.scenarios.filter((item) => item.status === "passed").length} passed${skipped ? `, ${skipped} skipped` : ""}), ${steps.length} step(s) (${count("passed")} passed, ${count("failed")} failed, ${count("skipped")} skipped)`;
+}
+export async function runTestCommand(runtime, argv, signal = new AbortController().signal, out = (text) => process.stdout.write(text), 
+env = NO_VARIABLES, 
+callerEnv = {}, 
+shared) {
+    const { values, positionals } = parseArgs({
+        args: argv,
+        allowPositionals: true,
+        strict: true,
+        options: {
+            help: { type: "boolean", short: "h" },
+            config: { type: "string" },
+            cucumber: { type: "string" },
+            tags: { type: "string" },
+            name: { type: "string" },
+            environment: { type: "string" },
+            keep: { type: "boolean" },
+            "report-dir": { type: "string" },
+            "dry-run": { type: "boolean" },
+            "negative-control": { type: "boolean" },
+            "expect-step": { type: "string" },
+            "expect-marker": { type: "string" },
+            candidate: { type: "string", multiple: true },
+            suite: { type: "string" },
+            mode: { type: "string" },
+            base: { type: "string" },
+            changed: { type: "string", multiple: true },
+            row: { type: "string", multiple: true },
+            plan: { type: "boolean" },
+            "rebuild-fixtures": { type: "boolean" },
+            "root-calls": { type: "boolean" },
+            "rerun-failures": { type: "string" },
+            workers: { type: "string" },
+            race: { type: "string" },
+            "race-max-ms": { type: "string" },
+            "race-seed": { type: "string" },
+            video: { type: "boolean" },
+            verbose: { type: "boolean" },
+            "update-baselines": { type: "boolean" },
+            devices: { type: "string" },
+            mutate: { type: "boolean" },
+            max: { type: "string" },
+            "own-lab": { type: "boolean" },
+        },
+    });
+    if (values.mode !== undefined && values.suite !== undefined) {
+        console.error(`--mode and --suite both choose what runs; give one\n${TEST_USAGE}`);
+        return 2;
+    }
+    const suiteName = values.mode ?? values.suite;
+    const suiteFlag = values.mode !== undefined ? "--mode" : "--suite";
+    const runEnv = values.keep ? { ...callerEnv, PRODUCT_LAB_KEEP: "1" } : callerEnv;
+    const race = raceOptions(values);
+    if (typeof race === "string") {
+        console.error(`${race}\n${TEST_USAGE}`);
+        return 2;
+    }
+    const rerunFailures = values["rerun-failures"] === undefined ? undefined : Number(values["rerun-failures"]);
+    if (rerunFailures !== undefined && (!Number.isInteger(rerunFailures) || rerunFailures < 1 || rerunFailures > 20 || values["negative-control"] || values["dry-run"] || suiteName !== undefined)) {
+        console.error(`--rerun-failures takes a whole number from 1 to 20, and not with --negative-control, --dry-run, --suite or --mode\n${TEST_USAGE}`);
+        return 2;
+    }
+    if (values.help) {
+        out(`${TEST_USAGE}\n`);
+        return 0;
+    }
+    if (values["expect-step"] || values["expect-marker"]) {
+        console.error(`--expect-step and --expect-marker are gone: a control's tags say where and why it must fail (@negative-control, @fails-at-step-N, @fails-with:<reason>) and judge it\n${TEST_USAGE}`);
+        return 2;
+    }
+    await runtime.sync?.();
+    const window = runtime.attached;
+    if (window) {
+        const refused = windowRefusal(window, values, runtime.environmentState());
+        if (refused) {
+            console.error(refused);
+            return 2;
+        }
+        out(`${inWindowLine(window, runtime.environmentState())}\n`);
+    }
+    if (values.workers !== undefined && rerunFailures !== undefined) {
+        console.error(`--rerun-failures does not rerun a run on workers yet: drop one of --workers and --rerun-failures\n${TEST_USAGE}`);
+        return 2;
+    }
+    if (values.workers !== undefined && (suiteName !== undefined || values["negative-control"])) {
+        console.error(`--workers runs feature files; drop ${suiteName !== undefined ? suiteFlag : "--negative-control"}\n${TEST_USAGE}`);
+        return 2;
+    }
+    if (values.devices !== undefined) {
+        const conflicting = [suiteName !== undefined ? suiteFlag : "", values.workers !== undefined ? "--workers" : "", values.mutate ? "--mutate" : "", values["negative-control"] ? "--negative-control" : "", rerunFailures !== undefined ? "--rerun-failures" : "", values.race !== undefined ? "--race" : ""].filter(Boolean);
+        if (conflicting.length) {
+            console.error(`--devices repeats one plain run per device; drop ${conflicting.join(", ")}\n${TEST_USAGE}`);
+            return 2;
+        }
+        return runOnDevices(runtime, argv, values.devices, values["report-dir"], signal, out, env);
+    }
+    if (values["update-baselines"] && (suiteName !== undefined || values.workers !== undefined || values["negative-control"])) {
+        console.error(`--update-baselines rewrites baselines from one plain run; drop ${suiteName !== undefined ? suiteFlag : values.workers !== undefined ? "--workers" : "--negative-control"}\n${TEST_USAGE}`);
+        return 2;
+    }
+    if (values.max !== undefined && !values.mutate) {
+        console.error(`--max applies to --mutate only\n${TEST_USAGE}`);
+        return 2;
+    }
+    if (values.mutate) {
+        const conflicting = [suiteName !== undefined ? suiteFlag : "", values.workers !== undefined ? "--workers" : "", values["negative-control"] ? "--negative-control" : "", rerunFailures !== undefined ? "--rerun-failures" : "", values["dry-run"] ? "--dry-run" : "", values["rebuild-fixtures"] ? "--rebuild-fixtures" : "", values.video ? "--video" : "", values["update-baselines"] ? "--update-baselines" : ""].filter(Boolean);
+        const max = values.max === undefined ? undefined : Number(values.max);
+        if (conflicting.length || (max !== undefined && (!Number.isInteger(max) || max < 1))) {
+            console.error(`${conflicting.length ? `--mutate runs each mutant as its own negative control; drop ${conflicting.join(", ")}` : "--max takes a whole number of at least 1"}\n${TEST_USAGE}`);
+            return 2;
+        }
+    }
+    if (suiteName !== undefined) {
+        const conflicting = [positionals.length ? "feature paths" : "", values.tags ? "--tags" : "", values.name ? "--name" : "", values["negative-control"] ? "--negative-control" : "", values.candidate ? "--candidate" : "", values.environment ? "--environment" : "", values.cucumber ? "--cucumber" : ""].filter(Boolean);
+        if (conflicting.length) {
+            console.error(`${suiteFlag} decides the selection, environment and versions; drop ${conflicting.join(", ")}\n${TEST_USAGE}`);
+            return 2;
+        }
+        return runSuiteCommand(runtime, { suite: suiteName, ...(values.mode !== undefined ? { mode: true } : {}), base: values.base ?? env.get("PRODUCT_LAB_BASE"), changed: values.changed, rows: values.row, plan: !!values.plan, reportDir: values["report-dir"], dryRun: !!values["dry-run"], rebuildFixtures: !!values["rebuild-fixtures"], video: !!values.video, verbose: !!values.verbose }, signal, out);
+    }
+    if (values.base || values.changed || values.row || (values.plan && !values.mutate)) {
+        console.error(`--base, --changed, --row and --plan apply to --suite and --mode only (and --plan to --mutate)\n${TEST_USAGE}`);
+        return 2;
+    }
+    signal.throwIfAborted();
+    const lab = await runtime.load();
+    signal.throwIfAborted();
+    if (values["negative-control"] && !positionals.length)
+        return runEveryControl(controlRuns((await runtime.features({ read: true, inspect: true })).features, values.environment, { ...(values.tags !== undefined ? { tags: values.tags } : {}), ...(values.name !== undefined ? { name: values.name } : {}) }), argv, values["report-dir"], signal, out, (args) => runTestCommand(runtime, args, signal, out, env), sharedControls(runtime, values.environment, signal, out, (args, ran) => runTestCommand(runtime, args, signal, out, env, {}, { ran })));
+    const filter = values.verbose ? undefined : new PlainOutputFilter(out, undefined, !!window || !!shared);
+    const shown = filter?.write ?? out;
+    const versions = window || shared ? {} : await resolveRunVersions(runtime, { ...(values.environment ? { environment: values.environment } : {}), ...(values.candidate ? { candidates: values.candidate } : {}) }, (line) => out(`${line}\n`), !!filter);
+    signal.throwIfAborted();
+    const paths = positionals.map((path) => {
+        const absolute = resolve(path);
+        const inside = relative(lab.root, absolute);
+        return isAbsolute(path) || !inside.startsWith("..") ? inside : path;
+    });
+    if (values.workers !== undefined && values.video) {
+        console.error(`--video does not record a run on workers yet: drop one of --workers and --video\n${TEST_USAGE}`);
+        return 2;
+    }
+    if (values.mutate)
+        return runMutateCommand(runtime, {
+            paths,
+            plan: !!values.plan,
+            ...(values.tags ? { tags: values.tags } : {}),
+            ...(values.name ? { name: values.name } : {}),
+            ...(values.cucumber ? { configuration: resolve(values.cucumber) } : {}),
+            ...(values.environment ? { environment: values.environment } : {}),
+            ...(values.max !== undefined ? { max: Number(values.max) } : {}),
+            ...(values["report-dir"] ? { reportDir: resolve(values["report-dir"]) } : {}),
+            ...(Object.keys(versions).length ? { env: versions } : {}),
+        }, signal, out);
+    if (values.workers !== undefined)
+        return runWorkersCommand(runtime, { ...values, workers: values.workers }, paths, versions, lab.root, signal, out);
+    if (values["negative-control"] && values["dry-run"])
+        out(`Negative control ${paths.join(", ")}: a dry run plans it and runs no step, so it is not judged\n`);
+    const request = window ? inWindowRequest(paths, values) : shared ? { ...inWindowRequest(paths, values), ...(values["negative-control"] ? { control: {} } : {}) } : {
+        paths,
+        ...(values.tags ? { tags: values.tags } : {}),
+        ...(values.name ? { name: values.name } : {}),
+        ...(values.environment ? { environment: values.environment } : {}),
+        ...(values.cucumber ? { configuration: resolve(values.cucumber) } : {}),
+        ...(values["dry-run"] ? { dryRun: true } : {}),
+        ...(values["negative-control"] && !values["dry-run"] ? { control: {} } : {}),
+        requireScenarios: true,
+        label: "test",
+        ...(values["rebuild-fixtures"] ? { fixtures: "rebuild" } : {}),
+        ...(Object.keys(versions).length || values.video || values["root-calls"] || values["update-baselines"] || values["negative-control"] || Object.keys(runEnv).length || filter ? { env: { ...versions, ...videoEnv(values.video), ...(values["root-calls"] ? { PRODUCT_LAB_ROOT_CALLS: "1" } : {}), ...(values["update-baselines"] ? { [BASELINES_VARIABLE]: "update" } : {}), ...(values["negative-control"] && !values["dry-run"] ? { [CONTROL_RUN_VARIABLE]: "1" } : {}), ...runEnv, ...(filter ? { [PLAIN_OUTPUT_ENV]: "1" } : {}) } } : {}),
+    };
+    let passed;
+    let record;
+    if (race) {
+        const directory = resolve(values["report-dir"] ?? join(lab.root, ".product-lab", "test", `race-${new Date().toISOString().replace(/[:.]/g, "-")}`));
+        let last;
+        const failedRecords = [];
+        const { paths: selected, tags, name, environment, configuration, env: variables, fixtures } = request;
+        const view = await runTool(runtime, {
+            tool: "race",
+            options: { runs: race.runs, maxMs: race.maxMs, seedMode: "fixed", seed: race.seed.toString() },
+            scenarios: { selection: { paths: selected, ...(tags ? { tags } : {}), ...(name ? { name } : {}), ...(environment ? { environment } : {}), ...(configuration ? { configuration } : {}), ...(variables ? { env: variables } : {}), ...(fixtures ? { fixtures } : {}) }, directory, label: request.label, reports: true },
+            observer: {
+                started: (run) => out(`\nRace run ${run.index + 1} of ${race.runs}: seed ${run.seed}, waits up to ${race.maxMs} ms\n`),
+                output: shown,
+                ran: (_run, record) => {
+                    last = { record, passed: filter ? reportPlainRaceRun(runtime, record) : reportRun(record) };
+                    if (record.status !== "passed")
+                        failedRecords.push(record);
+                },
+            },
+        }, signal);
+        if (view.outcome?.tool !== "race") {
+            console.error(`product-lab test --race: ${view.error ?? `the race ended ${view.status} without its runs`}`);
+            return view.status === "refused" ? 2 : 1;
+        }
+        const runs = view.outcome.runs.map((item) => ({ run: item.run, seed: item.seed, status: item.passed ? "passed" : "failed", report: item.report }));
+        const failed = runs.filter((item) => item.status === "failed");
+        mkdirSync(directory, { recursive: true });
+        writeFileSync(join(directory, "product-lab-race.json"), JSON.stringify({ seed: race.seed.toString(), maxMs: race.maxMs, runs }, null, 2) + "\n");
+        console.log(`\nproduct-lab test --race ${race.runs}: ${failed.length ? `${failed.length} of ${runs.length} run(s) failed` : `all ${runs.length} run(s) passed`}${runs.length < race.runs ? ` (stopped after ${runs.length})` : ""}; record ${join(directory, "product-lab-race.json")}`);
+        for (const item of failed)
+            console.log(`  run ${item.run} failed with seed ${item.seed}: again with --race 1 --race-seed ${item.seed} --race-max-ms ${race.maxMs}; report ${item.report}`);
+        if (filter) {
+            let hostLogs = [];
+            try {
+                if (failedRecords.length)
+                    hostLogs = failedRunsHostLogs(await runtime.recordings(), failedRecords);
+            }
+            catch {
+            }
+            const look = [join(directory, "product-lab-race.json"), ...failed.map((item) => `run ${item.run}'s report: ${item.report}`), ...hostLogsWords(hostLogs), ...(filter.hidden ? ["the lab's own lines: each run's cucumber.log (--verbose prints them)"] : [])];
+            console.log(`Look at: ${look.join(" · ")}`);
+        }
+        passed = runs.length === race.runs && !failed.length;
+        if (!last) {
+            console.error("product-lab test --race: stopped before its first run");
+            return 1;
+        }
+        record = last.record;
+    }
+    else {
+        const where = values["report-dir"] ? { directory: resolve(values["report-dir"]), report: resolve(values["report-dir"], "product-lab-test.json") } : { runsDirectory: resolve(lab.root, ".product-lab", "test") };
+        try {
+            record = await testRun(runtime, { ...request, ...where }, signal, shown);
+        }
+        catch (error) {
+            const unmet = shared ? notForChain(error) : undefined;
+            if (unmet === undefined)
+                throw error;
+            console.error(`control ${paths.join(", ")}: not run here: ${unmet}; inconclusive: neither a pass nor a valid negative result`);
+            return NOT_RUN_EXIT;
+        }
+        shared?.ran(record);
+        passed = filter ? await reportPlainRun(runtime, record, filter) : reportRun(record);
+    }
+    if (rerunFailures !== undefined && !passed)
+        await (await import("./rerun-failures.js")).rerunFailures(runtime, record, rerunFailures, out);
+    if (!values["dry-run"] && !record.control)
+        await (await import("./journeys-after-test.js")).journeysAfterTest(runtime, out);
+    if (!passed && record.control?.verdict === "not-run")
+        return NOT_RUN_EXIT;
+    return passed ? 0 : 1;
+}
+function raceOptions(values) {
+    if (values.race === undefined)
+        return values["race-max-ms"] !== undefined || values["race-seed"] !== undefined ? "--race-max-ms and --race-seed apply to --race only" : undefined;
+    const runs = Number(values.race);
+    if (!Number.isInteger(runs) || runs < 1 || runs > 100)
+        return "--race takes a whole number of runs from 1 to 100";
+    const other = ["workers", "suite", "mode", "negative-control", "dry-run", "rerun-failures", "mutate"].find((flag) => values[flag] !== undefined && values[flag] !== false);
+    if (other)
+        return `--race runs the selection once per seed, one run at a time; drop --${other}`;
+    const maxMs = values["race-max-ms"] === undefined ? RaceJitter.DEFAULT_MAX_MS : Number(values["race-max-ms"]);
+    if (!Number.isInteger(maxMs) || maxMs < 1 || maxMs > 60_000)
+        return "--race-max-ms takes a whole number of milliseconds from 1 to 60000";
+    try {
+        return { runs, maxMs, seed: values["race-seed"] === undefined ? RaceSeed.of(randomBytes(4).readUInt32BE(0)) : RaceSeed.parse(String(values["race-seed"])) };
+    }
+    catch (error) {
+        return `--race-seed: ${error.message}`;
+    }
+}
+async function testRun(runtime, request, signal, out) {
+    const run = await runtime.runFeatures(request);
+    const off = run.onOutput(out);
+    let stopping;
+    let stopFailure;
+    const stop = () => { stopping ??= run.stop({ signal: "SIGINT", graceMs: 180_000 }).catch(error => { stopFailure = error; }); };
+    signal.addEventListener("abort", stop, { once: true });
+    if (signal.aborted)
+        stop();
+    let record;
+    try {
+        record = await run.completion;
+    }
+    finally {
+        off();
+        signal.removeEventListener("abort", stop);
+        await stopping;
+    }
+    if (stopFailure)
+        throw stopFailure;
+    return record;
+}
+function reportRun(record) {
+    fixtureLines(record);
+    console.log(`\n${recordLine(record)}`);
+    const { passed, written } = writeRunReports(record);
+    if (written)
+        console.log(`Reports: ${written.junit}, ${written.html}`);
+    controlLines(record);
+    return passed;
+}
+function fixtureLines(record) {
+    for (const scenario of record.scenarios)
+        if (scenario.fixture)
+            console.log(`  fixture ${scenario.fixture.name} ${scenario.fixture.action}${scenario.fixture.ms ? ` in ${(scenario.fixture.ms / 1000).toFixed(1)} s` : ""} before "${scenario.name}"`);
+}
+const recordLine = (record) => `product-lab test ${record.id}: ${record.status}${record.error ? ` (${record.error})` : ""}; ${summary(record)}; report ${record.report}`;
+function writeRunReports(record) {
+    const passed = record.control ? record.control.verdict === "failed-as-intended" && !record.error : record.status === "passed";
+    const reason = record.control ? `negative control: ${record.control.verdict} (${record.control.reason})` : `${record.status}${record.error ? `: ${record.error}` : ""}; ${summary(record)}`;
+    try {
+        return { passed, written: writeReports(record.directory, runEvidence(record, new Date(), passed, reason)) };
+    }
+    catch (error) {
+        console.error(`Could not write the JUnit and HTML reports: ${error instanceof Error ? error.message : String(error)}`);
+        return { passed };
+    }
+}
+async function reportPlainRun(runtime, record, filter) {
+    const { passed, written } = writeRunReports(record);
+    let explained;
+    try {
+        if (record.status !== "passed" || record.control)
+            explained = runtime.explainFailure({
+                runId: record.id,
+                kind: record.control ? "control" : "feature",
+                steps: record.steps,
+                ...(record.error ? { error: record.error } : {}),
+                ...(record.control ? { control: { verdict: record.control.verdict, why: record.control.reason, ...(record.control.expected ? { expected: record.control.expected } : {}), ...(record.control.failure ? { failure: record.control.failure } : {}) } } : {}),
+            });
+    }
+    catch {
+    }
+    let recording;
+    let hostLogs = [];
+    try {
+        const listed = await runtime.recordings();
+        recording = runRecording(record, listed.recordings);
+        hostLogs = recordingHostLogs(listed.recordings.find((item) => item.id === recording), listed.location);
+    }
+    catch {
+    }
+    const outsideSteps = !!record.error || !record.scenarios.length || record.steps.some((step) => step.hook && step.status === "failed");
+    const { result, lookAt } = plainEnding({ record, ...(explained ? { explained } : {}), ...(recording ? { recording } : {}), hostLogs, ...(written ? { reports: written } : {}), filter, outsideSteps });
+    console.log(`\n${result[0]}`);
+    fixtureLines(record);
+    if (result.length > 1)
+        console.log(result.slice(1).join("\n"));
+    console.log(recordLine(record));
+    controlLines(record);
+    console.log(lookAt);
+    return passed;
+}
+function reportPlainRaceRun(runtime, record) {
+    const { passed, written } = writeRunReports(record);
+    let explained;
+    try {
+        if (record.status !== "passed")
+            explained = runtime.explainFailure({ runId: record.id, kind: "feature", steps: record.steps, ...(record.error ? { error: record.error } : {}) });
+    }
+    catch {
+    }
+    console.log(`\n${recordLine(record)}`);
+    console.log(raceRunLines(record, explained, written).join("\n"));
+    return passed;
+}
+function controlLines(record) {
+    const selected = record.selection.paths.join(", ") || record.selection.tags || "the configuration's features";
+    if (!record.control)
+        return;
+    const failure = record.control.failure;
+    const failedAt = failure?.step ?? record.steps.find((step) => step.status === "failed")?.title;
+    const why = failure ? `\n  ${failure.reason}: ${failure.message.split("\n").slice(0, 3).join("\n  ")}` : "";
+    if (record.control.verdict === "not-run")
+        return console.error(`control ${selected}: ${record.control.reason}; inconclusive: neither a pass nor a valid negative result`);
+    if (record.control.verdict === "failed-as-intended")
+        console.log(`control ${selected}: failed as intended at "${failedAt}"${failure?.position !== undefined ? ` (step ${failure.position})` : ""}${record.control.expected?.marker ? ` with "${record.control.expected.marker}"` : ""}${why}`);
+    else
+        console.error(`control ${selected}: NOT a valid negative result${record.control.verdict === "wrong-reason" ? ": failed for another reason" : ""} (${record.control.verdict}: ${record.control.reason})${why}`);
+}
+async function runOnDevices(runtime, argv, list, reportDir, signal, out, env) {
+    if (!runtime.runDevices) {
+        console.error("This runtime cannot run on device profiles");
+        return 2;
+    }
+    await runtime.load();
+    const names = list.trim() === "lab" ? undefined : list.split(",").map((name) => name.trim()).filter(Boolean);
+    const { devices, problems } = await runtime.runDevices(names);
+    if (problems.length) {
+        console.error(`--devices: ${problems.join("; ")}\n${TEST_USAGE}`);
+        return 2;
+    }
+    const rest = [];
+    for (let index = 0; index < argv.length; index++) {
+        const arg = argv[index];
+        if (arg === "--devices" || arg === "--report-dir")
+            index++;
+        else if (!arg.startsWith("--devices=") && !arg.startsWith("--report-dir="))
+            rest.push(arg);
+    }
+    const results = [];
+    for (const device of devices) {
+        signal.throwIfAborted();
+        out(`\nDevice ${device} (${results.length + 1} of ${devices.length})\n`);
+        const dir = reportDir ? ["--report-dir", join(reportDir, deviceFolder(device))] : [];
+        results.push({ device, code: await runTestCommand(runtime, [...rest, ...dir], signal, out, env, { [DEVICE_VARIABLE]: device }) });
+    }
+    out(`\nDevices: ${results.map(({ device, code }) => `${device} ${code === 0 ? "passed" : "failed"}`).join(", ")}\n`);
+    return results.every(({ code }) => code === 0) ? 0 : 1;
+}
+const deviceFolder = (device) => slug(device);
+async function runWorkersCommand(runtime, values, paths, versions, root, signal, out) {
+    const count = Number(values.workers);
+    if (!runtime.runWorkers) {
+        console.error("This runtime cannot run workers");
+        return 2;
+    }
+    const filters = new Map();
+    const plain = !values.verbose;
+    const pending = new Map();
+    const onOutput = (worker, text) => {
+        if (plain) {
+            if (!filters.has(worker))
+                filters.set(worker, new PlainOutputFilter(out, workerLine(worker)));
+            return filters.get(worker).write(text);
+        }
+        const lines = `${pending.get(worker) ?? ""}${text}`.split("\n");
+        pending.set(worker, lines.pop() ?? "");
+        for (const line of lines)
+            out(`[worker ${worker}] ${line}\n`);
+    };
+    const record = await runtime.runWorkers({
+        count,
+        paths,
+        ...(values.tags ? { tags: values.tags } : {}),
+        ...(values.name ? { name: values.name } : {}),
+        ...(values.environment ? { environment: values.environment } : {}),
+        ...(values.cucumber ? { configuration: resolve(values.cucumber) } : {}),
+        directory: values["report-dir"] ? resolve(values["report-dir"]) : resolve(root, ".product-lab", "test", `workers-${new Date().toISOString().replace(/[:.]/g, "-")}`),
+        ...(values["dry-run"] ? { dryRun: true } : {}),
+        ...(values["rebuild-fixtures"] ? { fixtures: "rebuild" } : {}),
+        ...(Object.keys(versions).length || plain ? { env: { ...versions, ...(plain ? { [PLAIN_OUTPUT_ENV]: "1" } : {}) } } : {}),
+        onOutput,
+        signal,
+    });
+    for (const [worker, rest] of pending)
+        if (rest)
+            out(`[worker ${worker}] ${rest}\n`);
+    if (plain) {
+        let hostLogs = [];
+        try {
+            hostLogs = failedRunsHostLogs(await runtime.recordings(), record.workers.filter((worker) => worker.status === "failed").map((worker) => ({ startedAt: Date.parse(worker.startedAt), finishedAt: Date.parse(worker.endedAt) })));
+        }
+        catch {
+        }
+        const { result, lookAt } = workersEnding(record, filters, hostLogs);
+        console.log(`\n${result.join("\n")}`);
+        console.log(`product-lab test --workers ${record.workers.length}: ${record.status} (${record.reason}); together ${(record.overlapMs / 1000).toFixed(1)} s; record ${record.report}`);
+        console.log(lookAt);
+        return record.status === "passed" ? 0 : 1;
+    }
+    for (const worker of record.workers)
+        console.log(`  worker ${worker.index}: ${worker.status}${worker.reason ? ` (${worker.reason})` : ""} in ${(worker.durationMs / 1000).toFixed(1)} s; ${worker.features.join(", ")}; ${Object.values(worker.endpoints).join(", ") || "no endpoints recorded"}`);
+    console.log(`\nproduct-lab test --workers ${record.workers.length}: ${record.status} (${record.reason}); together ${(record.overlapMs / 1000).toFixed(1)} s; record ${record.report}`);
+    return record.status === "passed" ? 0 : 1;
+}

package/entrypoints/cli/src/test-in-window.d.ts

@@ -0,0 +1,11 @@
+import type { EnvironmentState, RunFeaturesRequest } from "@parity/product-lab/ports/driving";
+import type { RunningLab } from "@parity/product-lab/ports/storage";
+/** Why this run cannot go to the open window, or undefined when it can. */
+export declare function windowRefusal(window: RunningLab, values: Readonly<Record<string, unknown>>, state: EnvironmentState): string | undefined;
+/** The line that says where the run goes. */
+export declare function inWindowLine(window: RunningLab, state: EnvironmentState): string;
+/** The run's request in the window: the selection, in its process, never stopping at a breakpoint. */
+export declare function inWindowRequest(paths: readonly string[], values: {
+    readonly tags?: string;
+    readonly name?: string;
+}): RunFeaturesRequest;

package/entrypoints/cli/src/test-in-window.js

@@ -0,0 +1,27 @@
+// Copyright 2026 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+const OWN_LAB_FLAGS = ["workers", "race", "suite", "devices", "mutate", "negative-control", "dry-run", "candidate", "cucumber", "keep", "rebuild-fixtures", "rerun-failures", "root-calls", "video", "update-baselines"];
+export function windowRefusal(window, values, state) {
+    const flag = OWN_LAB_FLAGS.find((name) => values[name] !== undefined && values[name] !== false);
+    if (flag)
+        return `The lab window at ${window.window} is open, so product-lab test runs there, on its environment. --${flag} needs a lab of its own: add --own-lab (it uses this lab's ports, so close the window first), or drop --${flag}.`;
+    if (state.state !== "running")
+        return `The lab window at ${window.window} runs no environment${state.state === "starting" ? " yet (it is starting)" : ""}: start one there and run again, or add --own-lab for a lab of its own.`;
+    const environment = values.environment;
+    if (typeof environment === "string" && environment !== state.environment)
+        return `The lab window at ${window.window} runs ${state.environment}, not ${environment}: switch it there and run again, or drop --environment, or add --own-lab for a lab of its own.`;
+    return undefined;
+}
+export function inWindowLine(window, state) {
+    return `Running in the lab window at ${window.window} (${window.lab}${state.state === "running" ? `, ${state.environment}` : ""}): the run shows there live, with product-lab's bundled steps; --own-lab runs a lab of its own.`;
+}
+export function inWindowRequest(paths, values) {
+    return {
+        paths: [...paths],
+        ...(values.tags ? { tags: values.tags } : {}),
+        ...(values.name ? { name: values.name } : {}),
+        requireScenarios: true,
+        label: "test",
+        inProcess: { breakpoints: false, pauseAfterStep: false },
+    };
+}

package/entrypoints/cli/src/test-output.d.ts

@@ -0,0 +1,9 @@
+/** Whether this process is a `node --test` child, whose standard output belongs to the runner. */
+export declare function underTestRunner(env?: NodeJS.ProcessEnv): boolean;
+/** Where a lab's own lines go: standard output, standard error in a `node --test` child. */
+export declare function linesOut(): NodeJS.WriteStream;
+/**
+ * In a `node --test` child, `console.log`, `console.info` and `console.debug` write to standard error (the preloads
+ * every e2e script loads call it: own-running-record.ts). A test that replaces them (t.mock.method) still can.
+ */
+export declare function keepStdoutForTheRunner(): void;