@hominis/fireforge 0.49.2 → 0.51.0

package/dist/src/commands/test-orchestrate.js

@@ -0,0 +1,240 @@
+// SPDX-License-Identifier: EUPL-1.2
+/**
+ * Multi-run orchestration for `fireforge test`, all under the one
+ * engine-session lock hold the command already has:
+ *
+ *  - mixed harness paths run as two consecutive mach invocations, xpcshell
+ *    first, with one pre-test build (when `--build`) packaging both halves;
+ *  - `--repeat <n>` runs the same selection n times, building once;
+ *  - `--negative-control <diff>` applies a reviewed mutation after the
+ *    build and right before each dispatch, and restores it after the run.
+ *
+ * Each sub-run is the ordinary single-run body. Its verdict line is
+ * captured rather than printed (see `setVerdictCapture`) and echoed as a
+ * `FIREFORGE-RUN:` line, and the invocation ends with one aggregate
+ * `FIREFORGE-VERDICT:` line. A request that needs none of this runs the
+ * body exactly as before.
+ */
+import { getProjectPaths } from '../core/config.js';
+import { applyNegativeControl, planNegativeControl, recoverPendingNegativeControl, restoreNegativeControl, } from '../core/negative-control.js';
+import { formatCompiledControlRefusal } from '../core/negative-control-compiled.js';
+import { getActiveRunLogPath, writeToActiveRunLog } from '../core/run-log.js';
+import { GeneralError, InvalidArgumentError, PreflightRefusalError } from '../errors/base.js';
+import { TestFailureError } from '../errors/build.js';
+import { toError } from '../utils/errors.js';
+import { info, notice, success } from '../utils/logger.js';
+import { stripEnginePrefix } from '../utils/paths.js';
+import { classifyTestHarnesses } from './test-modes.js';
+import { deriveControlOutcome, describeControlOutcome, formatAggregateBody, formatRunLine, parseCapturedVerdict, worstRun, } from './test-multi-verdict.js';
+import { emitAggregateVerdict, emitFailVerdict, setVerdictCapture, setVerdictRunAttribute, verdictEmitted, } from './test-verdict.js';
+function assertMultiRunUsage(testPaths, options) {
+    const flag = options.negativeControl !== undefined ? '--negative-control' : '--repeat';
+    if (testPaths.length === 0) {
+        throw new InvalidArgumentError(`${flag} needs explicit test paths.`, flag);
+    }
+    if (options.buildOnly === true || options.canary !== undefined || options.auto === true) {
+        throw new InvalidArgumentError(`${flag} runs tests, so it cannot be combined with --build-only, --canary or --auto.`, flag);
+    }
+}
+async function splitHarnesses(engineDir, testPaths, options) {
+    // `--build-only` packages a mixed union itself and dispatches nothing.
+    if (testPaths.length === 0 || options.buildOnly === true || options.canary !== undefined) {
+        return undefined;
+    }
+    const normalized = testPaths.map((path) => stripEnginePrefix(path).trim());
+    const classification = await classifyTestHarnesses(engineDir, normalized);
+    if (classification.xpcshell.length === 0 || classification.nonXpcshell.length === 0) {
+        return undefined;
+    }
+    notice(`Mixed harnesses: running ${classification.xpcshell.length} xpcshell path(s), then ` +
+        `${classification.nonXpcshell.length} browser/mochitest path(s), as two mach invocations ` +
+        'under this lock hold' +
+        (options.build === true ? ', after one pre-test build that packages both.' : '.'));
+    return [
+        { harness: 'xpcshell', paths: classification.xpcshell },
+        { harness: 'mochitest', paths: classification.nonXpcshell },
+    ];
+}
+/** Options for every run after the first: the build and doctor run once. */
+function laterRunOptions(options) {
+    const later = { ...options };
+    delete later.build;
+    delete later.extendCoverage;
+    delete later.refuseUnexportedDrift;
+    delete later.doctor;
+    return later;
+}
+async function planRuns(engineDir, testPaths, options) {
+    const repeat = options.repeat ?? 1;
+    const control = options.negativeControl !== undefined;
+    if (repeat > 1 || control)
+        assertMultiRunUsage(testPaths, options);
+    const halves = await splitHarnesses(engineDir, testPaths, options);
+    if (halves === undefined && repeat === 1 && !control)
+        return undefined;
+    const groups = halves ?? [
+        { paths: testPaths },
+    ];
+    const union = halves?.flatMap((half) => half.paths);
+    const runs = [];
+    for (let repetition = 1; repetition <= repeat; repetition += 1) {
+        for (const group of groups) {
+            const first = runs.length === 0;
+            runs.push({
+                repetition,
+                ...(group.harness === undefined ? {} : { harness: group.harness }),
+                paths: group.paths,
+                options: first ? options : laterRunOptions(options),
+                hooks: {
+                    repetition,
+                    ...(first && union !== undefined ? { coveragePaths: union } : {}),
+                },
+            });
+        }
+    }
+    return runs;
+}
+function writeRunLine(line) {
+    process.stdout.write(`${line}\n`);
+    writeToActiveRunLog(`${line}\n`);
+}
+function controlHook(paths, plan, slot) {
+    return async () => {
+        slot.applied = await applyNegativeControl(paths, plan);
+        notice(`Negative control applied to ${plan.targets.map((target) => `engine/${target.enginePath}`).join(', ')}; ` +
+            'it stays in place until this run has finished.');
+    };
+}
+/**
+ * Throws the invocation's error after the aggregate verdict, or returns
+ * when the invocation succeeded (all runs green, or a control caught).
+ */
+function finishInvocation(results, plannedTotal, control) {
+    if (control !== undefined)
+        info(describeControlOutcome(control));
+    if (control === 'caught') {
+        success('Negative control caught.');
+        return;
+    }
+    if (control === 'missed') {
+        throw new TestFailureError('Negative control stayed green: no run went red with the mutation applied, so the selected tests do not detect it.', 'mach test', undefined, getActiveRunLogPath());
+    }
+    const worst = worstRun(results);
+    if (worst === undefined)
+        return;
+    if (worst.reason === 'preflight' || worst.reason === 'inconclusive') {
+        throw worst.error instanceof Error
+            ? worst.error
+            : new GeneralError(`Run ${results.indexOf(worst) + 1} was refused.`);
+    }
+    const failing = results
+        .map((result, index) => ({ result, index }))
+        .filter(({ result }) => !result.passed)
+        .map(({ result, index }) => `run ${index + 1} (${result.reason ?? 'failed'})`);
+    throw new TestFailureError(`${failing.length} of ${plannedTotal} run(s) did not pass: ${failing.join(', ')}. See each FIREFORGE-RUN line above.`, 'mach test', undefined, getActiveRunLogPath());
+}
+/**
+ * Runs `fireforge test` as one or several sub-runs. Always recovers a
+ * negative control a killed earlier run left applied, first.
+ *
+ * @param projectRoot - Project root
+ * @param testPaths - The positional test paths
+ * @param options - Parsed test options
+ * @param body - The single-run body
+ * @param onRefusal - Renders a sub-run's preflight refusal and emits its
+ *   `FAIL reason=preflight` verdict (captured), as `testCommand` does for a
+ *   single run
+ */
+export async function orchestrateTestRuns(projectRoot, testPaths, options, body, onRefusal) {
+    const projectPaths = getProjectPaths(projectRoot);
+    const controlPaths = {
+        engineDir: projectPaths.engine,
+        fireforgeDir: projectPaths.fireforgeDir,
+    };
+    await recoverPendingNegativeControl(controlPaths);
+    const runs = await planRuns(projectPaths.engine, testPaths, options);
+    if (runs === undefined) {
+        await body(testPaths, options, {});
+        return;
+    }
+    const plan = options.negativeControl === undefined
+        ? undefined
+        : await planNegativeControl(projectRoot, projectPaths.engine, options.negativeControl, {
+            deferApplyCheck: options.build === true,
+        });
+    // A compiled target only reaches the binary through a build, so its
+    // control goes in before the first run's pre-test build.
+    const compiled = plan?.compiled;
+    const compiledControl = compiled !== undefined;
+    if (compiledControl) {
+        if (options.build !== true) {
+            throw new PreflightRefusalError(formatCompiledControlRefusal(compiled), 'negative-control');
+        }
+        setVerdictRunAttribute('control-build', 'compiled');
+    }
+    const results = [];
+    let restoreError;
+    for (const [index, run] of runs.entries()) {
+        notice(`Run ${index + 1}/${runs.length}${run.harness === undefined ? '' : ` (${run.harness})`}` +
+            `${plan === undefined ? '' : ', negative control'}: ${run.paths.join(' ')}`);
+        const capture = {};
+        const slot = {};
+        let error;
+        setVerdictCapture(capture);
+        try {
+            await body(run.paths, run.options, {
+                ...run.hooks,
+                ...(plan === undefined
+                    ? {}
+                    : compiledControl && index === 0
+                        ? { beforeBuild: controlHook(controlPaths, plan, slot) }
+                        : { beforeDispatch: controlHook(controlPaths, plan, slot) }),
+            });
+        }
+        catch (caught) {
+            error = caught;
+            if (capture.line === undefined)
+                onRefusal(caught);
+        }
+        finally {
+            setVerdictCapture(undefined);
+        }
+        // Only a kill writes a real verdict while a capture is installed (see
+        // emitKilledVerdict). The invocation is over: restore, start nothing
+        // more, print nothing after that line.
+        const killed = verdictEmitted();
+        if (slot.applied !== undefined) {
+            try {
+                const restored = await restoreNegativeControl(slot.applied);
+                info(`Negative control restored byte-exact: ${restored.restored.length} file(s).`);
+            }
+            catch (caught) {
+                restoreError = caught;
+            }
+        }
+        if (killed)
+            throw toError(restoreError ?? error ?? new GeneralError('The run was killed.'));
+        const result = {
+            repetition: run.repetition,
+            ...(run.harness === undefined ? {} : { harness: run.harness }),
+            ...parseCapturedVerdict(capture.line),
+            ...(error === undefined ? {} : { error }),
+        };
+        results.push(result);
+        writeRunLine(formatRunLine(result, index, runs.length));
+        if (restoreError !== undefined)
+            break;
+        if (result.reason === 'preflight' || result.reason === 'inconclusive')
+            break;
+    }
+    if (restoreError !== undefined) {
+        emitFailVerdict('inconclusive', 'negative-control-restore');
+        throw toError(restoreError);
+    }
+    const repetitions = options.repeat ?? 1;
+    const control = plan === undefined
+        ? undefined
+        : deriveControlOutcome(results, repetitions, results.length === runs.length);
+    emitAggregateVerdict(formatAggregateBody(results, runs.length, control));
+    finishInvocation(results, runs.length, control);
+}

package/dist/src/commands/test-perf-scope.d.ts

@@ -0,0 +1,50 @@
+/**
+ * Scope of the evidence a `--perf-samples` run reports and invalidates on.
+ *
+ * - A stale-build refusal on a perf run says which stale packaged files are
+ *   staging inputs of the sampled test and which lie elsewhere, so a sample
+ *   whose fixture is unaffected can proceed knowingly under
+ *   `--allow-stale-build`.
+ * - An engine edit that lands while a sample runs makes the verdict
+ *   `inconclusive` only when the edited path is something the running build
+ *   packages (an install-manifest source, or a test file). An edit to a
+ *   compiled source, a build file or anything else no manifest installs
+ *   cannot reach the running binary, so the sample stands with a warning.
+ */
+/**
+ * The perf-impact line of a stale-build refusal.
+ *
+ * @param engineDir - Absolute engine directory
+ * @param stalePaths - Stale packaged paths the refusal names
+ * @param requested - Normalized request paths of the run
+ */
+export declare function describePerfStaleImpact(engineDir: string, stalePaths: readonly string[], requested: readonly string[]): Promise<string>;
+/**
+ * Tolerance for `assertEngineGenerationUnchanged` on a `--perf-samples` run:
+ * true (with a warning naming the edits) when HEAD did not move and no
+ * changed path is packaged by the running build.
+ *
+ * @param engineDir - Absolute engine directory
+ */
+export declare function perfGenerationTolerance(engineDir: string): (before: string, after: string) => Promise<boolean>;
+/**
+ * Tolerance for `assertEngineGenerationUnchanged` on an ordinary run: true
+ * (with a warning, and `drift-tolerated=<n>` on the verdict) when HEAD did
+ * not move and every changed path is one the running browser cannot see.
+ *
+ * A peer deleting an unrelated chrome script used to make a whole run
+ * inconclusive. What a running browser can see mid-run is narrower than
+ * "anything under engine/": a file the objdir links to its source (`live`),
+ * one the harness stages at dispatch (`at-dispatch`), and the run's own
+ * test files and the staging inputs their manifests name. A compiled
+ * source, or a file the build copied into the package, is already fixed in
+ * the running binary or package; the next run's stale-build gate judges it.
+ *
+ * Perf runs keep {@link perfGenerationTolerance}: a sample is also void
+ * when a packaged path changes.
+ *
+ * @param engineDir - Absolute engine directory
+ * @param requested - Normalized request paths of the run
+ * @param onTolerated - Receives the tolerated path count
+ */
+export declare function runGenerationTolerance(engineDir: string, requested: readonly string[], onTolerated: (count: number) => void): (before: string, after: string) => Promise<boolean>;

package/dist/src/commands/test-perf-scope.js

@@ -0,0 +1,138 @@
+// SPDX-License-Identifier: EUPL-1.2
+/**
+ * Scope of the evidence a `--perf-samples` run reports and invalidates on.
+ *
+ * - A stale-build refusal on a perf run says which stale packaged files are
+ *   staging inputs of the sampled test and which lie elsewhere, so a sample
+ *   whose fixture is unaffected can proceed knowingly under
+ *   `--allow-stale-build`.
+ * - An engine edit that lands while a sample runs makes the verdict
+ *   `inconclusive` only when the edited path is something the running build
+ *   packages (an install-manifest source, or a test file). An edit to a
+ *   compiled source, a build file or anything else no manifest installs
+ *   cannot reach the running binary, so the sample stands with a warning.
+ */
+import { isTestPath } from '../core/build-audit-resolve.js';
+import { collectStagingInputs, isUnderPath } from '../core/coverage-records.js';
+import { generationDeltaPaths } from '../core/engine-session-lock.js';
+import { classifyRuntimeDelivery, installsFromDirectory, } from '../core/objdir-install-manifests.js';
+import { loadEngineInstallIndex } from '../core/test-stale-exempt.js';
+import { warn } from '../utils/logger.js';
+/** How many paths an impact line names per group. */
+const IMPACT_LIST_LIMIT = 5;
+function capList(paths) {
+    const head = paths.slice(0, IMPACT_LIST_LIMIT).join(', ');
+    const extra = paths.length - Math.min(paths.length, IMPACT_LIST_LIMIT);
+    return extra > 0 ? `${head}, … (+${extra} more)` : head;
+}
+/**
+ * The perf-impact line of a stale-build refusal.
+ *
+ * @param engineDir - Absolute engine directory
+ * @param stalePaths - Stale packaged paths the refusal names
+ * @param requested - Normalized request paths of the run
+ */
+export async function describePerfStaleImpact(engineDir, stalePaths, requested) {
+    const inputs = [];
+    for (const path of requested)
+        inputs.push(...(await collectStagingInputs(engineDir, path)));
+    const fixture = stalePaths.filter((path) => inputs.some((input) => isUnderPath(path, input)));
+    const elsewhere = stalePaths.filter((path) => !fixture.includes(path));
+    const fixturePart = fixture.length > 0
+        ? `${fixture.length} stale file(s) are staging inputs of the sampled test (${capList(fixture)}), so the sample would measure the old content`
+        : requested.length > 0
+            ? "the sampled test's own staging inputs are unaffected"
+            : 'no test paths were named, so every stale file may be measured';
+    const elsewherePart = elsewhere.length > 0 && requested.length > 0
+        ? `; ${elsewhere.length} stale packaged file(s) lie outside them (${capList(elsewhere)}) and affect the sample only if the measured path loads them`
+        : '';
+    return (`Perf sample impact: ${fixturePart}${elsewherePart}. Repackage first, or pass ` +
+        '--allow-stale-build to take the sample knowingly.');
+}
+/**
+ * Tolerance for `assertEngineGenerationUnchanged` on a `--perf-samples` run:
+ * true (with a warning naming the edits) when HEAD did not move and no
+ * changed path is packaged by the running build.
+ *
+ * @param engineDir - Absolute engine directory
+ */
+export function perfGenerationTolerance(engineDir) {
+    return async (before, after) => {
+        const delta = generationDeltaPaths(before, after);
+        if (delta.headMoved || delta.paths.length === 0)
+            return false;
+        const index = await loadEngineInstallIndex(engineDir);
+        if (index === undefined)
+            return false;
+        for (const path of delta.paths) {
+            if (isTestPath(path))
+                return false;
+            if (path.endsWith('/')) {
+                if (installsFromDirectory(index, path))
+                    return false;
+            }
+            else if ((await classifyRuntimeDelivery(index, engineDir, path)) !== 'unlisted') {
+                return false;
+            }
+        }
+        warn(`engine/ changed while this perf sample ran, but only in files the running build does not ` +
+            `package (${capList(delta.paths)}); the sample stands. Rebuild before the next sample if ` +
+            'they belong in it.');
+        return true;
+    };
+}
+/**
+ * Tolerance for `assertEngineGenerationUnchanged` on an ordinary run: true
+ * (with a warning, and `drift-tolerated=<n>` on the verdict) when HEAD did
+ * not move and every changed path is one the running browser cannot see.
+ *
+ * A peer deleting an unrelated chrome script used to make a whole run
+ * inconclusive. What a running browser can see mid-run is narrower than
+ * "anything under engine/": a file the objdir links to its source (`live`),
+ * one the harness stages at dispatch (`at-dispatch`), and the run's own
+ * test files and the staging inputs their manifests name. A compiled
+ * source, or a file the build copied into the package, is already fixed in
+ * the running binary or package; the next run's stale-build gate judges it.
+ *
+ * Perf runs keep {@link perfGenerationTolerance}: a sample is also void
+ * when a packaged path changes.
+ *
+ * @param engineDir - Absolute engine directory
+ * @param requested - Normalized request paths of the run
+ * @param onTolerated - Receives the tolerated path count
+ */
+export function runGenerationTolerance(engineDir, requested, onTolerated) {
+    return async (before, after) => {
+        const delta = generationDeltaPaths(before, after);
+        if (delta.headMoved || delta.paths.length === 0)
+            return false;
+        const index = await loadEngineInstallIndex(engineDir);
+        if (index === undefined)
+            return false;
+        const inputs = [...requested];
+        for (const path of requested)
+            inputs.push(...(await collectStagingInputs(engineDir, path)));
+        for (const path of delta.paths) {
+            const bare = path.endsWith('/') ? path.slice(0, -1) : path;
+            if (inputs.some((input) => isUnderPath(bare, input) || isUnderPath(input, bare))) {
+                return false;
+            }
+            // A path-less run requests every test, so any test file is its own.
+            if (requested.length === 0 && isTestPath(path))
+                return false;
+            if (path.endsWith('/')) {
+                if (installsFromDirectory(index, path))
+                    return false;
+                continue;
+            }
+            const delivery = await classifyRuntimeDelivery(index, engineDir, path);
+            if (delivery === 'live' || delivery === 'at-dispatch')
+                return false;
+        }
+        warn(`engine/ changed while this run was in flight, but only in files the running browser ` +
+            `cannot see (${capList(delta.paths)}); the verdict stands. The next run's stale-build ` +
+            'gate judges whether they need a rebuild.');
+        onTolerated(delta.paths.length);
+        return true;
+    };
+}

package/dist/src/commands/test-prelock.d.ts

@@ -0,0 +1,51 @@
+/**
+ * The `fireforge test` checks that run BEFORE the engine session lock.
+ *
+ * Everything here reads, never writes, so none of it needs the lock, and
+ * each check used to cost a full place in the lock queue before it could
+ * refuse:
+ *
+ *  - a missing launchable binary (a peer's interrupted build) refused only
+ *    after the wait, in every session, with nothing naming the cause;
+ *  - the named-file checkJs pass refused an `Element | null` after an
+ *    eighteen-minute queue;
+ *  - `--wait-browser` and `--wait-port` spent their budget holding the
+ *    lock, so every peer queued behind a run that was doing nothing.
+ *
+ * The in-lock ladder in `test.ts` still runs in full afterwards. That is
+ * the "re-check once under the lock": the waits this pass spent are
+ * stripped from the options it returns, so the in-lock browser and port
+ * checks probe once instead of waiting again.
+ */
+import type { TestOptions } from '../types/commands/index.js';
+/**
+ * Resolves the launchable binary a test run needs, or refuses naming why
+ * it is missing. The cause comes from the build run record, which any
+ * session's build writes, so a session that never ran the build can still
+ * say "a peer's build was interrupted" rather than only "the binary is
+ * missing".
+ *
+ * @param projectRoot - Project root
+ * @param engineDir - Absolute engine directory
+ * @param binaryName - Configured binary name
+ * @param objDir - Active objdir, when one exists
+ * @returns The engine-relative launchable path, or undefined without an objdir
+ */
+export declare function resolveLaunchablePathForTests(projectRoot: string, engineDir: string, binaryName: string, objDir: string | undefined): Promise<string | undefined>;
+/** Build-artifact preflight wording for `fireforge test`. */
+export declare const TEST_BUILD_PREFLIGHT: {
+    readonly label: "Tests";
+    readonly requirement: "Tests require a completed build.";
+    readonly remediation: "Run 'fireforge build' first, then run 'fireforge test'.";
+    readonly requireExisting: true;
+};
+/**
+ * Runs the lock-free checks and waits, and returns the options the in-lock
+ * run should use: with `waitBrowser`/`waitPort` removed when their budget
+ * was already spent here.
+ *
+ * @param projectRoot - Project root
+ * @param testPaths - Requested test paths
+ * @param options - Parsed test options
+ */
+export declare function runTestPreLockPreflight(projectRoot: string, testPaths: string[], options: TestOptions): Promise<TestOptions>;