@parity/product-lab 0.0.1-placeholder → 0.1.0

package/entrypoints/cli/src/reports/junit.js

@@ -0,0 +1,93 @@
+// Copyright 2026 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+import { hostBypassLines } from "./host-bypass.js";
+const xml = (value) => value
+    .replace(/[\u0000-\u0008\u000b\u000c\u000e-\u001f]/g, "")
+    .replace(/&/g, "&")
+    .replace(/</g, "&lt;")
+    .replace(/>/g, "&gt;")
+    .replace(/"/g, "&quot;");
+const seconds = (ms) => ((ms ?? 0) / 1000).toFixed(3);
+function scenarioCase(row, scenario) {
+    const steps = scenario.steps.filter((step) => !step.hidden);
+    const time = scenario.steps.reduce((sum, step) => sum + (step.durationMs ?? 0), 0);
+    const attrs = `classname="${xml(`${row.id}.${scenario.uri}`)}" name="${xml(scenario.name)}" time="${seconds(time)}"`;
+    const evidence = `<system-out>${xml([hostBypassLines(scenario.hostBypass).join("\n"), ...steps.map((step) => `${[step.keyword, step.name].filter(Boolean).join(" ")}\n${hostBypassLines(step.hostBypass).join("\n")}`)].join("\n"))}</system-out>`;
+    if (scenario.status === "passed")
+        return `    <testcase ${attrs}>${evidence}</testcase>`;
+    if (scenario.status === "skipped")
+        return `    <testcase ${attrs}><skipped/>${evidence}</testcase>`;
+    const failed = scenario.steps.find((step) => step.status === "failed") ?? steps.find((step) => step.status !== "passed");
+    const message = failed ? `${[failed.keyword, failed.name].filter(Boolean).join(" ")}: ${failed.status}` : scenario.status;
+    return `    <testcase ${attrs}><failure message="${xml(message)}">${xml([failed?.error ?? message, failed?.abortDetail].filter(Boolean).join("; "))}</failure>${evidence}</testcase>`;
+}
+function properties(row) {
+    const entries = [];
+    const { counts, timings } = row;
+    if (counts) {
+        entries.push(["product-lab.scenarios.planned", counts.planned], ["product-lab.scenarios.executed", counts.executed], ["product-lab.scenarios.unsupported", counts.unsupported]);
+        if (row.status !== "unsupported")
+            entries.push(["product-lab.scenarios.not-reported", counts.notReported], ["product-lab.checkpoints", counts.checkpoints.total]);
+    }
+    if (timings) {
+        if (timings.resolveMs !== undefined)
+            entries.push(["product-lab.resolve-ms", timings.resolveMs]);
+        if (timings.runMs !== undefined)
+            entries.push(["product-lab.run-ms", timings.runMs]);
+        timings.environmentStarts?.forEach((start, index) => {
+            const key = timings.environmentStarts.length > 1 ? `product-lab.environment-start.${index + 1}` : "product-lab.environment-start";
+            entries.push([`${key}.ms`, start.durationMs], [`${key}.state`, start.startup?.state ?? "not-declared"]);
+        });
+        if (timings.scenariosMs !== undefined)
+            entries.push(["product-lab.scenario-steps-ms", timings.scenariosMs]);
+        entries.push(["product-lab.row-ms", timings.totalMs]);
+    }
+    if (!entries.length)
+        return "";
+    return `    <properties>\n${entries.map(([name, value]) => `      <property name="${xml(name)}" value="${xml(String(value))}"/>`).join("\n")}\n    </properties>\n`;
+}
+function rowSuite(row) {
+    const name = xml(row.id);
+    const scenarios = row.run?.scenarios ?? [];
+    if (!scenarios.length) {
+        const reason = row.reason ?? row.run?.error ?? `row ${row.status}`;
+        const skipped = row.status === "unsupported";
+        const passed = row.status === "passed";
+        const body = passed
+            ? `    <testcase classname="${name}" name="${name}" time="${seconds(row.durationMs)}"/>`
+            : skipped
+                ? `    <testcase classname="${name}" name="${name}"><skipped message="${xml(`unsupported: ${reason}`)}"/></testcase>`
+                : `    <testcase classname="${name}" name="${name}" time="${seconds(row.durationMs)}"><failure message="${xml(`${row.status}: ${reason}`)}">${xml(reason)}</failure></testcase>`;
+        return {
+            xml: `  <testsuite name="${name}" tests="1" failures="${passed || skipped ? 0 : 1}" errors="0" skipped="${skipped ? 1 : 0}" time="${seconds(row.durationMs)}">\n${properties(row)}${body}\n  </testsuite>`,
+            tests: 1,
+            failures: passed || skipped ? 0 : 1,
+            skipped: skipped ? 1 : 0,
+        };
+    }
+    const cases = scenarios.map((scenario) => scenarioCase(row, scenario));
+    let failures = scenarios.filter((scenario) => scenario.status !== "passed" && scenario.status !== "skipped").length;
+    const skipped = scenarios.filter((scenario) => scenario.status === "skipped").length;
+    if (!failures && row.status !== "passed") {
+        cases.push(`    <testcase classname="${name}" name="${name} (run)"><failure message="${xml(`${row.status}: ${row.reason ?? row.run?.error ?? ""}`)}"/></testcase>`);
+        failures = 1;
+    }
+    return {
+        xml: `  <testsuite name="${name}" tests="${cases.length}" failures="${failures}" errors="0" skipped="${skipped}" time="${seconds(row.durationMs)}">\n${properties(row)}${cases.join("\n")}\n  </testsuite>`,
+        tests: cases.length,
+        failures,
+        skipped,
+    };
+}
+export function renderJunit(report) {
+    const suites = report.rows.map(rowSuite);
+    if (!report.rows.length)
+        suites.push({
+            xml: `  <testsuite name="${xml(report.title)}" tests="1" failures="1" errors="0" skipped="0" time="0.000">\n    <testcase classname="${xml(report.id)}" name="selection"><failure message="${xml(report.reason)}">${xml(report.reason)}</failure></testcase>\n  </testsuite>`,
+            tests: 1,
+            failures: 1,
+            skipped: 0,
+        });
+    const total = (key) => suites.reduce((sum, suite) => sum + suite[key], 0);
+    return `<?xml version="1.0" encoding="UTF-8"?>\n<testsuites name="${xml(report.title)}" tests="${total("tests")}" failures="${total("failures")}" errors="0" skipped="${total("skipped")}" time="${seconds(report.durationMs)}">\n${suites.map((suite) => suite.xml).join("\n")}\n</testsuites>\n`;
+}

package/entrypoints/cli/src/reports/markdown.d.ts

@@ -0,0 +1,2 @@
+import type { EvidenceReport } from "./evidence-report.js";
+export declare function renderMarkdownSummary(report: EvidenceReport): string;

package/entrypoints/cli/src/reports/markdown.js

@@ -0,0 +1,62 @@
+// Copyright 2026 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+import { hostBypassLines } from "./host-bypass.js";
+import { checkpointCounts, combinationEntries, duration, environmentStart, NOT_MEASURED, rowCountsText, scenarioCounts, startState, started } from "./metrics-text.js";
+const cell = (value) => value.replace(/\\/g, "\\\\").replace(/\|/g, "\\|").replace(/\r?\n/g, " ");
+const code = (value) => `\`${value.replace(/`/g, "'")}\``;
+function rowLine(row) {
+    const counts = row.counts;
+    const scenarios = !counts ? NOT_MEASURED : row.status === "unsupported" ? `${counts.planned} unsupported` : `${counts.executed}/${counts.planned} (${counts.passed} passed, ${counts.failed} failed${counts.notReported ? `, ${counts.notReported} not reported` : ""})`;
+    const checkpoints = !counts || row.status === "unsupported" ? "–" : `${counts.checkpoints.passed}/${counts.checkpoints.total}`;
+    const envStart = !started(row) ? "–" : row.timings?.environmentStarts?.length ? row.timings.environmentStarts.map((item) => duration(item.durationMs)).join(", ") : NOT_MEASURED;
+    const total = !started(row) ? "–" : duration(row.timings?.totalMs ?? row.durationMs);
+    return `| ${[row.id, row.status, row.environment ?? "", row.host ?? "lab default", startState(row), envStart, scenarios, checkpoints, total].map(cell).join(" | ")} |`;
+}
+export function renderMarkdownSummary(report) {
+    const lines = [`## ${report.title}: ${report.status}`, "", cell(report.reason), ""];
+    lines.push("| | |", "| --- | --- |");
+    lines.push(`| Suite run | ${cell(report.id)} |`);
+    if (report.counts) {
+        lines.push(`| Rows | ${cell(rowCountsText(report.counts.rows))} |`);
+        lines.push(`| Scenarios | ${cell(scenarioCounts(report.counts.scenarios))} |`);
+        lines.push(`| Checkpoints | ${cell(checkpointCounts(report.counts.scenarios))} |`);
+    }
+    else
+        lines.push(`| Counts | ${NOT_MEASURED} (the record has none) |`);
+    if (report.timings) {
+        lines.push(`| Planning | ${duration(report.timings.planMs)} |`);
+        lines.push(`| Critical path | ${duration(report.timings.criticalPathMs)} (rows run one after the other) |`);
+    }
+    lines.push(`| Total | ${duration(report.timings?.totalMs ?? report.durationMs)}${report.budgetMs !== undefined ? ` of a ${duration(report.budgetMs)} budget` : ""} |`);
+    lines.push("");
+    lines.push("| Row | Status | Environment | Host | Start | Environment start | Scenarios executed | Checkpoints passed | Row time |");
+    lines.push("| --- | --- | --- | --- | --- | --- | --- | --- | --- |");
+    for (const row of report.rows)
+        lines.push(rowLine(row));
+    const ran = report.rows.filter((row) => started(row));
+    const details = ran.filter((row) => row.timings?.environmentStarts?.length || Object.keys(row.combination ?? {}).length);
+    if (details.length) {
+        lines.push("", "### What each row ran with", "");
+        for (const row of details) {
+            lines.push(`- **${cell(row.id)}**: environment start ${cell(environmentStart(row))}`);
+            for (const [id, text] of combinationEntries(row.combination))
+                lines.push(`  - ${code(id)} ${cell(text)}`);
+        }
+    }
+    const notRun = report.rows.filter((row) => row.status !== "passed" && row.reason);
+    if (notRun.length) {
+        lines.push("", "### Rows that did not pass", "");
+        for (const row of notRun)
+            lines.push(`- **${cell(row.id)}** ${row.status}: ${cell(row.reason)}`);
+    }
+    for (const row of report.rows) {
+        for (const scenario of row.run?.scenarios ?? []) {
+            lines.push("", `### Host bypass: ${cell(row.id)} / ${cell(scenario.name)}`, "", ...hostBypassLines(scenario.hostBypass).map(cell));
+            for (const step of scenario.steps.filter((item) => !item.hidden))
+                lines.push("", `**${cell([step.keyword, step.name].filter(Boolean).join(" "))}**`, "", ...hostBypassLines(step.hostBypass).map(cell));
+        }
+    }
+    if (report.affected)
+        lines.push("", `Affected selection: ${cell(report.affected.reason)} (${cell(report.affected.source)})`);
+    return `${lines.join("\n")}\n`;
+}

package/entrypoints/cli/src/reports/metrics-text.d.ts

@@ -0,0 +1,19 @@
+import type { SuiteCombination } from "@parity/product-lab/domain/suites";
+import type { SuiteCounts, SuiteScenarioCounts } from "@parity/product-lab/ports/driving";
+import type { EvidenceRow } from "./evidence-report.js";
+export declare const NOT_MEASURED = "not measured";
+export declare const NOT_STARTED = "not started";
+export declare function duration(ms: number | undefined): string;
+/** Did the row start (a run, or at least a measured attempt)? Unsupported and empty rows never do. */
+export declare const started: (row: EvidenceRow) => boolean;
+/** The row's environment start(s): time and how each began, or why there is none. */
+export declare function environmentStart(row: EvidenceRow): string;
+/** Just cold/warm/attached for tables; `not declared` when a start was measured without it. */
+export declare function startState(row: EvidenceRow): string;
+export declare function scenarioCounts(counts: SuiteScenarioCounts | undefined, status?: string): string;
+export declare function checkpointCounts(counts: SuiteScenarioCounts | undefined): string;
+export declare function rowCountsText(counts: SuiteCounts["rows"]): string;
+/** The row's timings other than its environment start, in order. */
+export declare function rowTimings(row: EvidenceRow): [label: string, value: string][];
+/** Each component the row ran with: `version (origin)`, sorted by id. */
+export declare function combinationEntries(combination: SuiteCombination | undefined): [id: string, text: string][];

package/entrypoints/cli/src/reports/metrics-text.js

@@ -0,0 +1,72 @@
+// Copyright 2026 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+export const NOT_MEASURED = "not measured";
+export const NOT_STARTED = "not started";
+export function duration(ms) {
+    if (ms === undefined)
+        return NOT_MEASURED;
+    if (ms < 1000)
+        return `${Math.round(ms)} ms`;
+    if (ms < 120_000)
+        return `${(ms / 1000).toFixed(1)} s`;
+    const seconds = Math.round(ms / 1000);
+    return `${Math.floor(seconds / 60)} min${seconds % 60 ? ` ${seconds % 60} s` : ""}`;
+}
+export const started = (row) => row.status !== "unsupported" && row.status !== "empty" && (row.timings !== undefined || row.run !== undefined);
+export function environmentStart(row) {
+    if (!started(row))
+        return NOT_STARTED;
+    const starts = row.timings?.environmentStarts;
+    if (!starts?.length)
+        return NOT_MEASURED;
+    return starts
+        .map((item) => `${duration(item.durationMs)}, ${item.startup ? `${item.startup.state}${item.startup.detail ? ` (${item.startup.detail})` : ""}` : "cold/warm not declared by the environment"}`)
+        .join("; ");
+}
+export function startState(row) {
+    if (!started(row))
+        return NOT_STARTED;
+    const starts = row.timings?.environmentStarts;
+    if (!starts?.length)
+        return NOT_MEASURED;
+    return starts.map((item) => item.startup?.state ?? "not declared").join(", ");
+}
+export function scenarioCounts(counts, status) {
+    if (!counts)
+        return NOT_MEASURED;
+    if (status === "unsupported")
+        return `${counts.planned} planned, unsupported here (not run, not passed)`;
+    const parts = [`${counts.executed} of ${counts.planned} planned executed (${counts.passed} passed, ${counts.failed} failed)`];
+    if (counts.skipped)
+        parts.push(`${counts.skipped} skipped`);
+    if (counts.unsupported)
+        parts.push(`${counts.unsupported} unsupported`);
+    if (counts.notReported)
+        parts.push(`${counts.notReported} not reported`);
+    return parts.join(", ");
+}
+export function checkpointCounts(counts) {
+    if (!counts)
+        return NOT_MEASURED;
+    const { total, passed, failed, skipped } = counts.checkpoints;
+    return `${total} (${passed} passed, ${failed} failed, ${skipped} skipped)`;
+}
+export function rowCountsText(counts) {
+    return `${counts.total} in all: ${counts.passed} passed, ${counts.failed} failed, ${counts["over-budget"]} over budget, ${counts.unsupported} unsupported, ${counts.empty} empty`;
+}
+export function rowTimings(row) {
+    if (!started(row))
+        return [["time", NOT_STARTED]];
+    const timings = row.timings ?? {};
+    return [
+        ["version resolution", timings.resolveMs !== undefined ? duration(timings.resolveMs) : row.timings ? "none (the row resolved no versions)" : NOT_MEASURED],
+        ["run", duration(timings.runMs)],
+        ["scenario steps", duration(timings.scenariosMs)],
+        ["row total", duration(timings.totalMs ?? row.durationMs)],
+    ];
+}
+export function combinationEntries(combination) {
+    return Object.entries(combination ?? {})
+        .sort(([a], [b]) => a.localeCompare(b))
+        .map(([id, entry]) => [id, `${entry.version} (${entry.origin})`]);
+}

package/entrypoints/cli/src/reports/pr-comment.d.ts

@@ -0,0 +1,20 @@
+import type { RunRecording } from "@parity/product-lab/domain/recordings";
+import type { SuiteRunRecord } from "@parity/product-lab/ports/driving";
+export declare const PR_COMMENT_MARKER = "<!-- product-lab:pr-suite -->";
+export interface PrCommentOptions {
+    /** The heading's label, for example `pr-suite (notes)`. */
+    label: string;
+    /** Where the run's details are (the workflow run), linked under the table. */
+    runUrl?: string;
+    /** The artifact the videos are uploaded in (a zip: its offsets cannot be linked, so they are given as text). */
+    artifact?: string;
+    /**
+     * A direct address the recordings directory's `videos/` is served under. Only then does a video become a link that
+     * seeks to the scenario (`#t=<start>,<end>`); an artifact is a zip download, where `#t=` cannot seek.
+     */
+    videoBase?: string;
+}
+/** `m:ss` (or `h:mm:ss`) from milliseconds, rounded down to the second. */
+export declare function clock(ms: number): string;
+/** The comment's Markdown. `recordings` are every recording that may belong to the run; others are ignored. */
+export declare function renderPrComment(record: SuiteRunRecord, recordings: readonly RunRecording[], options: PrCommentOptions): string;

package/entrypoints/cli/src/reports/pr-comment.js

@@ -0,0 +1,61 @@
+// Copyright 2026 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+export const PR_COMMENT_MARKER = "<!-- product-lab:pr-suite -->";
+export function clock(ms) {
+    const total = Math.max(0, Math.floor(ms / 1000));
+    const [h, m, s] = [Math.floor(total / 3600), Math.floor((total % 3600) / 60), total % 60];
+    const two = (value) => String(value).padStart(2, "0");
+    return h ? `${h}:${two(m)}:${two(s)}` : `${m}:${two(s)}`;
+}
+const seconds = (ms) => (ms < 10_000 ? `${(ms / 1000).toFixed(1)} s` : `${Math.round(ms / 1000)} s`);
+const cell = (text) => text.replace(/\|/g, "\\|").replace(/\r?\n/g, " ");
+function videoCell(recording, names, options) {
+    if (!recording)
+        return "no recording";
+    const spans = recording.videoSpans();
+    if (!spans.length)
+        return "no video (run without `--video`)";
+    return spans
+        .map(({ video, startMs, endMs }) => {
+        const who = video.actor ? `${names.get(video.actor) ?? video.actor}${video.source === "detached-window" ? " (detached window)" : ""}` : "lab window";
+        const file = fileName(video);
+        if (options.videoBase) {
+            const href = `${options.videoBase.replace(/\/+$/, "")}/${encodeURIComponent(file)}#t=${Math.floor(startMs / 1000)},${Math.ceil(endMs / 1000)}`;
+            return `[${cell(who)}](${href}) ${clock(startMs)}–${clock(endMs)}`;
+        }
+        return `${cell(who)}: \`${file}\` starts at ${clock(startMs)} (to ${clock(endMs)})`;
+    })
+        .join("<br>");
+}
+const fileName = (video) => video.file.replace(/^videos\//, "");
+function recordingOf(run, scenario, recordings) {
+    const end = run.finishedAt ?? Number.POSITIVE_INFINITY;
+    return recordings
+        .filter((item) => item.data.title === scenario && (item.data.featureRun !== undefined ? item.data.featureRun === run.id : item.data.startedAt >= run.startedAt && item.data.startedAt <= end))
+        .sort((a, b) => b.data.startedAt - a.data.startedAt)[0];
+}
+export function renderPrComment(record, recordings, options) {
+    const rows = [];
+    let videos = 0;
+    for (const row of record.rows) {
+        for (const scenario of row.run?.scenarios ?? []) {
+            const recording = row.run ? recordingOf(row.run, scenario.name, recordings) : undefined;
+            const names = new Map((recording?.data.actors ?? []).map((actor) => [actor.id, actor.name]));
+            videos += recording?.data.capture.videos?.length ?? 0;
+            const ms = recording ? recording.data.finishedAt - recording.data.startedAt : scenario.steps.reduce((sum, step) => sum + (step.durationMs ?? 0), 0);
+            rows.push(`| ${cell(row.id)} | ${cell(scenario.feature)} › ${cell(scenario.name)} | ${scenario.status} | ${seconds(ms)} | ${videoCell(recording, names, options)} |`);
+        }
+    }
+    const lines = [PR_COMMENT_MARKER, `### product-lab ${options.label}: ${record.status}`, "", cell(record.reason), ""];
+    if (rows.length)
+        lines.push("| Row | Scenario | Result | Time | Video |", "| --- | --- | --- | --- | --- |", ...rows, "");
+    else
+        lines.push("No scenario ran.", "");
+    if (videos && !options.videoBase)
+        lines.push(`Videos: ${options.artifact ? `the \`${options.artifact}\` artifact` : "the recordings directory's `videos/`"}, one file per identity's page; each scenario starts at the time given.`);
+    if (videos)
+        lines.push("Videos show the apps as they painted themselves: a value typed into a field, a secret included, can be seen.");
+    if (options.runUrl)
+        lines.push(`Details in the job summary and the reports of the [workflow run](${options.runUrl}).`);
+    return `${lines.join("\n").trimEnd()}\n`;
+}

package/entrypoints/cli/src/reports/write-reports.d.ts

@@ -0,0 +1,8 @@
+import type { EvidenceReport } from "./evidence-report.js";
+export declare function writeReports(directory: string, report: EvidenceReport, options?: {
+    summary?: boolean;
+}): {
+    junit: string;
+    html: string;
+    summary?: string;
+};

package/entrypoints/cli/src/reports/write-reports.js

@@ -0,0 +1,30 @@
+// Copyright 2026 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+import { readFileSync, statSync, writeFileSync } from "node:fs";
+import { extname, join } from "node:path";
+import { renderHtmlReport } from "./html.js";
+import { renderJunit } from "./junit.js";
+import { renderMarkdownSummary } from "./markdown.js";
+const IMAGE_TYPES = { ".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".webp": "image/webp", ".gif": "image/gif" };
+function readImage(path) {
+    try {
+        const type = IMAGE_TYPES[extname(path).toLowerCase()];
+        if (!type || statSync(path).size > 2 * 1024 * 1024)
+            return undefined;
+        return `data:${type};base64,${readFileSync(path).toString("base64")}`;
+    }
+    catch {
+        return undefined;
+    }
+}
+export function writeReports(directory, report, options = {}) {
+    const junit = join(directory, "junit.xml");
+    const html = join(directory, "report.html");
+    writeFileSync(junit, renderJunit(report));
+    writeFileSync(html, renderHtmlReport(report, readImage));
+    if (!options.summary)
+        return { junit, html };
+    const summary = join(directory, "summary.md");
+    writeFileSync(summary, renderMarkdownSummary(report));
+    return { junit, html, summary };
+}

package/entrypoints/cli/src/reproduce-command.d.ts

@@ -0,0 +1,4 @@
+import type { LabRuntime, ReproduceReport } from "@parity/product-lab/ports/driving";
+export declare const REPRODUCE_USAGE = "Usage: product-lab reproduce <bundle.plbundle> [--reruns <n>] [--against-current] [--confirm <token>] [--config lab.config.ts] [--json]\n\nRebuild a bundled failing run here: the state it started on (its fixture imported, or the environment started fresh),\nthe same scenario run in this process, and its result judged against the bundled failure (the same step, the same\nstable error: reproduced). A bundle from another platform with native state, other app or lab source, a changed\nfeature file, a public network or unsaved state is refused, naming every reason, before anything starts.\n  --against-current     accept other app, lab or feature source: the run is a recheck against this code, never a\n                        reproduction\n  --reruns <n>          flake detection: rerun up to n times (default 1, at most 20), each from the same rebuilt state,\n                        stopping at the first rerun that does not fail; a failure that does not repeat is flaky\n  --confirm <token>     the running environment's reset confirmation, when one runs (reproducing replaces its state)\n  --config <file|dir>   lab.config.ts (default: ./lab.config.ts)\n  --json                print JSON\n  Exit code: 0 reproduced on every rerun (or, with --against-current, the same failure on this code); 1 otherwise\n  (flaky included: flaky never counts as passing) or refused.\nA bundle holding the bundled run and every rerun is written beside the extracted one.";
+export declare function renderReproduce(report: ReproduceReport): string;
+export declare function runReproduceCommand(runtime: LabRuntime, argv: string[], out?: (text: string) => void): Promise<number>;

package/entrypoints/cli/src/reproduce-command.js

@@ -0,0 +1,69 @@
+// Copyright 2026 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+import { parseArgs } from "node:util";
+export const REPRODUCE_USAGE = `Usage: product-lab reproduce <bundle.plbundle> [--reruns <n>] [--against-current] [--confirm <token>] [--config lab.config.ts] [--json]
+
+Rebuild a bundled failing run here: the state it started on (its fixture imported, or the environment started fresh),
+the same scenario run in this process, and its result judged against the bundled failure (the same step, the same
+stable error: reproduced). A bundle from another platform with native state, other app or lab source, a changed
+feature file, a public network or unsaved state is refused, naming every reason, before anything starts.
+  --against-current     accept other app, lab or feature source: the run is a recheck against this code, never a
+                        reproduction
+  --reruns <n>          flake detection: rerun up to n times (default 1, at most 20), each from the same rebuilt state,
+                        stopping at the first rerun that does not fail; a failure that does not repeat is flaky
+  --confirm <token>     the running environment's reset confirmation, when one runs (reproducing replaces its state)
+  --config <file|dir>   lab.config.ts (default: ./lab.config.ts)
+  --json                print JSON
+  Exit code: 0 reproduced on every rerun (or, with --against-current, the same failure on this code); 1 otherwise
+  (flaky included: flaky never counts as passing) or refused.
+A bundle holding the bundled run and every rerun is written beside the extracted one.`;
+export function renderReproduce(report) {
+    const word = {
+        reproduced: "Reproduced",
+        "rechecked-same-failure": "Rechecked: the same failure on this code (not a reproduction)",
+        "different-failure": "Not reproduced: it failed differently",
+        "did-not-fail": "Not reproduced: it did not fail",
+        invalid: "Not reproduced: the rerun cannot be judged",
+    };
+    return [
+        `${word[report.outcome]} · ${report.reason}`,
+        `Expected: fails at "${report.expected.step}"${report.expected.marker ? ` with "${report.expected.marker}"` : ""}`,
+        `Rerun: feature run ${report.rerun.featureRun} ${report.rerun.status}${report.rerun.recording ? ` · recording ${report.rerun.recording}` : ""}${report.rerun.error ? `\n  ${report.rerun.error.split("\n")[0]}` : ""}`,
+        ...(report.reruns.length > 1 ? report.reruns.map((rerun, index) => `  rerun ${index + 1}: ${rerun.status} · ${rerun.outcome}${rerun.error ? ` · ${rerun.error.split("\n")[0]}` : ""}`) : []),
+        ...report.rechecks.map((problem) => `Recheck: ${problem.message}`),
+        ...(report.race ? [`Race: scenario seed ${report.race.scenarioSeed}, waits up to ${report.race.maxMs} ms, applied again`] : []),
+        `Verdict: ${report.verdict}${report.verdict === "flaky" ? " (it failed, then did not fail on a rerun; flaky never counts as passing)" : ""}`,
+        `All runs: ${report.evidence} (${Math.round(report.ms / 1000)} s)`,
+    ].join("\n") + "\n";
+}
+export async function runReproduceCommand(runtime, argv, out = (text) => process.stdout.write(text)) {
+    const { values, positionals } = parseArgs({
+        args: argv,
+        allowPositionals: true,
+        strict: true,
+        options: { help: { type: "boolean", short: "h" }, config: { type: "string" }, json: { type: "boolean" }, "against-current": { type: "boolean" }, confirm: { type: "string" }, reruns: { type: "string" } },
+    });
+    const reruns = values.reruns === undefined ? undefined : Number(values.reruns);
+    if (values.help) {
+        out(REPRODUCE_USAGE + "\n");
+        return 0;
+    }
+    if (reruns !== undefined && (!Number.isInteger(reruns) || reruns < 1 || reruns > 20)) {
+        out(`--reruns takes a whole number from 1 to 20, not ${values.reruns}\n${REPRODUCE_USAGE}\n`);
+        return 2;
+    }
+    if (positionals.length !== 1) {
+        out(`${positionals.length ? `Unexpected arguments: ${positionals.slice(1).join(" ")}` : "Name the bundle to reproduce"}\n${REPRODUCE_USAGE}\n`);
+        return 2;
+    }
+    await runtime.load();
+    try {
+        const report = await runtime.runBundles().reproduce({ file: positionals[0], ...(values["against-current"] ? { againstCurrent: true } : {}), ...(values.confirm ? { confirmation: values.confirm } : {}), ...(reruns !== undefined ? { reruns } : {}) });
+        out(values.json ? JSON.stringify(report, null, 2) + "\n" : renderReproduce(report));
+        return report.verdict === "failed" && (report.outcome === "reproduced" || report.outcome === "rechecked-same-failure") ? 0 : 1;
+    }
+    finally {
+        if (runtime.environmentState().state !== "stopped")
+            await runtime.stopEnvironment({ interruptRun: true });
+    }
+}

package/entrypoints/cli/src/rerun-failures.d.ts

@@ -0,0 +1,8 @@
+import type { FeatureRunRecord, LabRuntime } from "@parity/product-lab/ports/driving";
+export interface RerunSummary {
+    readonly flaky: number;
+    readonly failing: number;
+    /** Failed scenarios that could not be rerun (no recording, a bundle that cannot be rebuilt), each said why. */
+    readonly notRerun: number;
+}
+export declare function rerunFailures(runtime: LabRuntime, record: FeatureRunRecord, reruns: number, out: (text: string) => void): Promise<RerunSummary>;

package/entrypoints/cli/src/rerun-failures.js

@@ -0,0 +1,59 @@
+// Copyright 2026 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+async function recordingOf(runtime, record, scenario) {
+    const { recordings } = await runtime.recordings();
+    const candidates = recordings.filter((item) => item.kind === "feature" && item.title === scenario.name && item.status === "failed" && item.startedAt >= record.startedAt);
+    for (const candidate of candidates.sort((a, b) => b.startedAt - a.startedAt)) {
+        const feature = (await runtime.recording(candidate.id)).recording.feature;
+        if (feature && feature.uri === scenario.uri && (scenario.line === undefined || feature.line === scenario.line))
+            return candidate.id;
+    }
+    return undefined;
+}
+export async function rerunFailures(runtime, record, reruns, out) {
+    const failed = record.scenarios.filter((scenario) => scenario.status !== "passed" && scenario.status !== "skipped" && scenario.uri);
+    let flaky = 0;
+    let failing = 0;
+    let notRerun = 0;
+    out(`\nRerunning ${failed.length} failed scenario(s), up to ${reruns} time(s) each, from the state each started on:\n`);
+    try {
+        for (const scenario of failed) {
+            const where = `${scenario.uri}${scenario.line !== undefined ? `:${scenario.line}` : ""}`;
+            try {
+                const recording = await recordingOf(runtime, record, scenario);
+                if (!recording) {
+                    notRerun++;
+                    out(`  not rerun: "${scenario.name}" (${where}): no recording of it from this run (recordings off, or it failed before its first step)\n`);
+                    continue;
+                }
+                const bundled = await runtime.runBundles().bundleRecording({ recording });
+                if (!bundled.reproducible.ok) {
+                    notRerun++;
+                    out(`  not rerun: "${scenario.name}" (${where}): ${bundled.reproducible.reasons.join("; ")} · bundle ${bundled.file}\n`);
+                    continue;
+                }
+                const report = await runtime.runBundles().reproduce({ file: bundled.file, reruns });
+                const statuses = report.reruns.map((rerun) => rerun.status).join(", ");
+                if (report.verdict === "flaky") {
+                    flaky++;
+                    out(`  flaky: "${scenario.name}" (${where}): failed, then did not fail on rerun ${report.reruns.length} of up to ${reruns} (${statuses}) · all runs ${report.evidence}\n`);
+                }
+                else {
+                    failing++;
+                    const same = report.reruns.every((rerun) => rerun.outcome === "reproduced");
+                    out(`  failing: "${scenario.name}" (${where}): failed on all ${report.reruns.length} rerun(s), ${same ? "the same way" : `not always the same way (${report.reason})`} · all runs ${report.evidence}\n`);
+                }
+            }
+            catch (error) {
+                notRerun++;
+                out(`  not rerun: "${scenario.name}" (${where}): ${error instanceof Error ? error.message : String(error)}\n`);
+            }
+        }
+    }
+    finally {
+        if (runtime.environmentState().state !== "stopped")
+            await runtime.stopEnvironment({ interruptRun: true });
+    }
+    out(`Reruns: ${flaky} flaky, ${failing} failing, ${notRerun} not rerun (flaky never counts as passing)\n`);
+    return { flaky, failing, notRerun };
+}

package/entrypoints/cli/src/rpc-flags.d.ts

@@ -0,0 +1,3 @@
+import type { EnvironmentRpcRequest } from "@parity/product-lab/ports/driving";
+/** The request `--rpc` and `--rpc-reason` give; undefined when neither is given. */
+export declare function rpcFromFlags(values: readonly string[] | undefined, reason: string | undefined): EnvironmentRpcRequest | undefined;

package/entrypoints/cli/src/rpc-flags.js

@@ -0,0 +1,23 @@
+// Copyright 2026 Parity Technologies (UK) Ltd.
+// SPDX-License-Identifier: Apache-2.0
+export function rpcFromFlags(values, reason) {
+    if (!values?.length) {
+        if (reason !== undefined)
+            throw new Error("--rpc-reason goes with --rpc <chain>=<url>: it says why the lab may use those URLs");
+        return undefined;
+    }
+    const chains = {};
+    for (const value of values) {
+        const index = value.indexOf("=");
+        const chain = index > 0 ? value.slice(0, index).trim() : "";
+        const url = index > 0 ? value.slice(index + 1).trim() : "";
+        if (!/^[A-Za-z0-9_-]+$/.test(chain) || !url)
+            throw new Error("--rpc takes <chain>=<url>, for example --rpc people=wss://rpc.example.org/people");
+        if (chains[chain] !== undefined)
+            throw new Error(`--rpc names ${chain} twice`);
+        chains[chain] = url;
+    }
+    if (reason !== undefined && !reason.trim())
+        throw new Error("--rpc-reason is empty: say why the lab may use those URLs");
+    return { chains, ...(reason?.trim() ? { reason: reason.trim() } : {}) };
+}

package/entrypoints/cli/src/setup-commands.d.ts

@@ -0,0 +1,36 @@
+import type { LabRuntime } from "@parity/product-lab/ports/driving";
+import type { SyncCommandRunner } from "@parity/product-lab/ports/system";
+import type { VariableReader } from "./environment.js";
+export declare const FETCH_USAGE = "Usage: product-lab fetch [--config lab.config.ts] [--environment <id>]... [--json]\n                         [--engine-repository <url|path>] [--engine-ref <ref>] [--no-engine-patch]\n\nDownload and verify every release file the lab's lab.lock.json pins for this machine (polkadot-omni-node,\nruntime Wasm) into the shared cache (PRODUCT_LAB_CACHE_DIR, default ~/.cache/product-lab). A file already\ncached and verified is not downloaded again.\n\nThen prepare, in the same cache, what the lab's default environment needs on this machine and never makes\nitself, and what each --environment needs (repeat it, or --environment all): a local copy of a network\n(paseo-copy, previewnet-copy) starts from an Engine checkout, cloned at the tested stock Engine release (or --engine-ref), with no lab patch,\nbuilt, with its network's binaries and the pinned dot CLI (<cache>/engine; Git and network access, a few\nminutes and about 900 MB the first time). One already prepared is only checked. Another Engine:\n  --engine-repository <url|path>  clone it from this Git URL or local repository instead of GitHub\n  --engine-ref <ref>              this commit, branch or tag instead of the tested release; starts use it as\n                                  prepared; doctor names it as not the tested release (cache:engine) and fails\n                                  when the copy is asked for (the default environment or --environment)\n  --no-engine-patch               compatibility flag; stock Engine is the default and the old patch is retired\n\nExit 0 when every file is cached and everything asked for is prepared, 1 otherwise.\nIn the product-lab checkout, `bun run setup` runs this for every example.";
+export declare const DOCTOR_USAGE = "Usage: product-lab doctor [--config lab.config.ts] [--environment <id>]... [--json]\n\nWhat this machine has and lacks to run the lab: Node.js, the shared cache, the lab's locked release files,\nthe selected browser (the full Chromium executable for the default launcher), what its environments need\nprepared (the Engine checkout of a local copy of a network: required for the default environment and each\n--environment, a note for the others), the Engine build the shared cache holds checked against the pinned\nrelease whatever the lab declares (cache:engine: another build, such as the retired patched one, fails where the lab\nwould start from it, as the Engine checkout line does, else a note; a cache holding no Engine says nothing) and the hosts its people run on (in a product, the truAPI WASM test host the package installs and the truAPI CLI host's locked official release:\ncached or not, what it provides; in the product-lab checkout, also Bun and its truAPI CLI host). Then what fits\nthis machine, from its cores, memory and free disk: the lab's local copy of a network as Full or Light, how\nmany people at once on each environment, and whether the cache is set up. Starts nothing, downloads nothing. Exit 0 when everything is\npresent, 1 otherwise, with the one command that fixes it; what fits never changes the exit code.";
+/** The command that fixes a missing piece, for the product-lab checkout. */
+export declare const SETUP_FIX = "bun run setup";
+/** The same in a product's own repository, where `bun run setup` does not exist: the locked files (its host comes with the package). */
+export declare const CONSUMER_SETUP_FIX = "npx --no-install product-lab fetch";
+export declare function runFetchCommand(runtime: LabRuntime, argv: string[], signal: AbortSignal): Promise<number>;
+interface Check {
+    name: string;
+    ok: boolean;
+    detail: string;
+    /** Informational only: never makes doctor fail. */
+    optional?: boolean;
+    /** The command that fixes it: the browser adapter's installer, or a cached tool's preparation. */
+    fix?: string;
+}
+/** The browser adapter owns resolution; an entry point only renders its prerequisite evidence. */
+export declare function browserCheck(runtime: LabRuntime, checkout: boolean): Promise<Check>;
+/**
+ * The truAPI WASM test host the package installs (`@parity/truapi-host`): its version, or why it is missing. Found as
+ * Node finds it from `from` (default: this CLI's folder): the nearest `node_modules/@parity/truapi-host` above it.
+ */
+export declare function wasmHostPackage(from?: string): {
+    version?: string;
+    problem?: string;
+};
+export declare function runDoctorCommand(runtime: LabRuntime, argv: string[], signal: AbortSignal, commands: SyncCommandRunner, env: VariableReader, 
+/** Where this runs (default: this CLI's own root) and where the WASM test host is looked for (default: this CLI's folder). */
+place?: {
+    readonly checkout?: boolean;
+    readonly wasmHostFrom?: string;
+}): Promise<number>;
+export {};