@hominis/fireforge 0.55.0 → 0.60.0

package/dist/src/test/control.js

@@ -0,0 +1,236 @@
+// SPDX-License-Identifier: EUPL-1.2
+/**
+ * Negative controls: proof that tests catch what they claim to. A control
+ * applies a reviewed mutation to the engine, builds what the mutation needs,
+ * runs the tests, and requires them to go red exactly at the expected
+ * assertions. Then every byte is put back.
+ *
+ * A control means something only if the same tests are green without the
+ * mutation, so the unmutated tests run first, unless the workspace recorded
+ * a green run of exactly these tests on exactly this engine state.
+ *
+ * The mutation lives inside a journal transaction: a failure, a signal or a
+ * crash restores the engine (a crash on the next transaction over the same
+ * journal). When the mutation needed a build, the restored engine is built
+ * again so the workspace does not keep a mutated binary; if that build
+ * fails, the build's staleness still refuses any test of it.
+ */
+import { join, relative } from 'node:path';
+import { applyFileDiff, parsePatch } from '../diff/index.js';
+import { build } from '../engine/index.js';
+import { toError } from '../kernel/errors.js';
+import { isInside, readBytes, readBytesIfExists, resolveInside, toPosixPath, } from '../kernel/fs.js';
+import { transaction } from '../kernel/journal.js';
+import { info, warn } from '../kernel/log.js';
+import { refusal } from './plan.js';
+import { isRecordedGreen, runTests } from './run.js';
+/** A red at an assertion that should have passed, as opposed to an unexpected pass. */
+const isAssertionFailure = (entry) => ['FAIL', 'ERROR'].includes(entry.status) && ['PASS', 'OK'].includes(entry.expected);
+/**
+ * Whether a control's label names a red: its assertion (a mochitest's
+ * message, an xpcshell task), its whole xpcshell message, or that message
+ * without the ` - <actual> <operator> <expected>` details `Assert` adds.
+ *
+ * @param entry - The red
+ * @param label - The label
+ */
+export function matchesLabel(entry, label) {
+    if (entry.assertion === label)
+        return true;
+    return (entry.message !== undefined &&
+        (entry.message === label || entry.message.startsWith(`${label} - `)));
+}
+/**
+ * Judges a control's labels against the reds it produced: every label met
+ * and every red named.
+ *
+ * @param expected - The labels
+ * @param failures - The failing assertions
+ */
+export function judgeLabels(expected, failures) {
+    const missing = expected.filter((label) => !failures.some((entry) => matchesLabel(entry, label)));
+    const unrelated = failures.filter((entry) => !expected.some((label) => matchesLabel(entry, label)));
+    return {
+        matched: expected.length > 0 && missing.length === 0 && unrelated.length === 0,
+        missing,
+        unrelated,
+    };
+}
+/** Each red's labels, as a person would copy them into `--expect`. */
+export function describeLabels(failures) {
+    if (failures.length === 0)
+        return 'no failing assertion';
+    const shown = failures.map((entry) => entry.message === undefined
+        ? JSON.stringify(entry.assertion)
+        : `${JSON.stringify(entry.assertion)} (assertion ${JSON.stringify(entry.message)})`);
+    return [...new Set(shown)].join(', ');
+}
+/**
+ * Why a mutated run does not prove the labels, or undefined when it does:
+ * every label met and no red unnamed, over the run and in each repetition.
+ *
+ * @param expected - The labels
+ * @param run - The mutated run
+ */
+export function labelRefusal(expected, run) {
+    const failuresOf = (repetition) => run.dispatches
+        .filter((dispatch) => repetition === undefined || dispatch.run === repetition)
+        .flatMap((dispatch) => dispatch.files.flatMap((file) => file.failures.filter(isAssertionFailure)));
+    const wanted = expected.map((label) => JSON.stringify(label)).join(', ');
+    const all = failuresOf();
+    if (!judgeLabels(expected, all).matched) {
+        return `expected ${wanted}; the run was red at ${describeLabels(all)}`;
+    }
+    for (let repetition = 1; repetition <= run.plan.repeat; repetition += 1) {
+        const reds = failuresOf(repetition);
+        if (judgeLabels(expected, reds).missing.length > 0) {
+            return `expected ${wanted} in every repetition; repetition ${repetition} was red at ${describeLabels(reds)}`;
+        }
+    }
+    return undefined;
+}
+/**
+ * The engine file a diff section mutates. The old side names it: a diff made
+ * with `git diff --no-index <engine file> <edited copy>` names the copy on
+ * its new side. An absolute path loses its leading slash under `a/`, and a
+ * project-relative one starts with `engine/`.
+ */
+function enginePath(written, workspace, roots) {
+    const absolute = `/${written}`;
+    if (isInside(workspace.engineDir, absolute))
+        return toPosixPath(relative(workspace.engineDir, absolute));
+    // The longest directory wins: a component directory may lie inside another root.
+    const root = [...roots]
+        .sort((a, b) => b.directory.length - a.directory.length)
+        .find((candidate) => written.startsWith(`${candidate.directory}/`));
+    if (root !== undefined) {
+        const rest = written.slice(root.directory.length + 1);
+        return root.engine === '' ? rest : `${root.engine}/${rest}`;
+    }
+    return written.replace(/^engine\//, '');
+}
+async function readTargets(workspace, diff, roots) {
+    let patch;
+    try {
+        patch = parsePatch(await readBytes(diff));
+    }
+    catch (error) {
+        throw refusal('negative-control', `Cannot read the control diff ${diff}: ${toError(error).message}`);
+    }
+    if (patch.files.length === 0)
+        throw refusal('negative-control', `${diff} changes no file.`);
+    const targets = [];
+    for (const file of patch.files) {
+        if (file.status === 'renamed' || file.status === 'copied' || file.binary !== undefined) {
+            throw refusal('negative-control', `${diff}: ${file.path} is a rename, copy or binary change; a control edits text in place.`);
+        }
+        const path = enginePath(file.status === 'added' ? file.newPath : file.oldPath, workspace, roots);
+        const absolute = resolveInside(workspace.engineDir, path);
+        const original = await readBytesIfExists(absolute);
+        let mutated;
+        try {
+            mutated = applyFileDiff(file, original);
+        }
+        catch (error) {
+            throw refusal('negative-control', `The control does not apply to ${path}: ${toError(error).message}`);
+        }
+        targets.push({ path, absolute, original, mutated });
+    }
+    return targets;
+}
+/**
+ * Runs a negative control: the tests green without the mutation, then red
+ * exactly at the expected assertions with it. Call it while holding the
+ * workspace.
+ *
+ * @param workspace - The workspace
+ * @param plan - The tests that must catch the mutation
+ * @param request - The diff and the labels
+ * @param options - As for `runTests`; the mutated run always builds what it needs
+ * @throws `negative-control` when the diff cannot be read or applied, and what `runTests` throws
+ */
+export async function runControl(workspace, plan, request, options = {}) {
+    if (request.expect.length === 0)
+        throw refusal('negative-control', 'A control names at least one expected assertion.');
+    const targets = await readTargets(workspace, request.diff, request.roots ?? []);
+    const paths = targets.map((target) => target.path);
+    let baseline;
+    if (await isRecordedGreen(workspace, plan)) {
+        info('These tests are recorded green on this engine state; the unmutated run is not repeated.');
+    }
+    else {
+        info('Running the tests without the mutation first: a control needs them green.');
+        baseline = await runTests(workspace, plan, options);
+        if (baseline.reason === 'inconclusive') {
+            const why = 'the engine changed while the tests ran without the mutation';
+            return { outcome: 'inconclusive', why, baseline, targets: paths };
+        }
+        if (!baseline.pass) {
+            return {
+                outcome: 'baseline-red',
+                why: `the tests are not green without the mutation (reason=${baseline.reason ?? 'unknown'}), so a red says nothing about it`,
+                baseline,
+                targets: paths,
+            };
+        }
+    }
+    const journal = join(workspace.stateDir, 'journals', 'negative-control');
+    const mutated = await transaction(journal, 'a negative control', async (tx) => {
+        for (const target of targets) {
+            if (target.mutated === undefined)
+                await tx.remove(target.absolute);
+            else
+                await tx.write(target.absolute, target.mutated);
+        }
+        info(`Applied the control to ${paths.join(', ')}.`);
+        try {
+            return await runTests(workspace, plan, { ...options, build: true });
+        }
+        finally {
+            for (const target of targets) {
+                if (target.original === undefined)
+                    await tx.remove(target.absolute);
+                else
+                    await tx.write(target.absolute, target.original);
+            }
+            info('Restored the engine.');
+        }
+    });
+    const restoreBuild = mutated.built === undefined ? undefined : await rebuild(workspace, options);
+    const base = {
+        ...(baseline === undefined ? {} : { baseline }),
+        mutated,
+        targets: paths,
+        ...(restoreBuild === undefined ? {} : { restoreBuild }),
+    };
+    if (mutated.pass) {
+        return { ...base, outcome: 'missed', why: 'the tests stayed green with the mutation applied' };
+    }
+    if (mutated.reason === 'inconclusive') {
+        const why = 'the engine changed while the tests ran with the mutation';
+        return { ...base, outcome: 'inconclusive', why };
+    }
+    if (mutated.reason !== 'test-failures') {
+        const wanted = request.expect.map((label) => JSON.stringify(label)).join(', ');
+        return {
+            ...base,
+            outcome: 'wrong-red',
+            why: `expected ${wanted}; the run failed with reason=${mutated.reason ?? 'unknown'} instead of failing assertions`,
+        };
+    }
+    const refused = labelRefusal(request.expect, mutated);
+    return refused === undefined
+        ? { ...base, outcome: 'caught' }
+        : { ...base, outcome: 'wrong-red', why: refused };
+}
+async function rebuild(workspace, options) {
+    try {
+        info('Building the restored engine, so the workspace keeps no mutated build.');
+        await build(workspace, { ui: true, waitMs: options.waitMs });
+        return 'done';
+    }
+    catch (error) {
+        warn(`The build of the restored engine failed (${toError(error).message}); its staleness keeps the mutated build from being tested.`);
+        return 'failed';
+    }
+}

package/dist/src/test/host.js

@@ -0,0 +1,266 @@
+// SPDX-License-Identifier: EUPL-1.2
+/**
+ * The host around a test run: what a killed earlier run of this workspace
+ * left behind, the ports a browser harness binds, and the helpers this run
+ * started that escaped its process group.
+ *
+ * Authority comes from the workspace lease: while a command holds it, no
+ * other FireForge run uses this engine, so a harness process of this engine
+ * whose parent has died, or that carries this engine's run marker, is a
+ * leftover and is terminated by exact identity (pid and birth). Anything
+ * else (another workspace's live suite, an unrelated listener) is never
+ * signalled: it is waited for under `--wait-lock`, or the run is refused
+ * naming it.
+ *
+ * Every harness process inherits two environment markers: the run's id, and
+ * the engine it runs in. A helper that detaches into a session of its own
+ * (xpcshell's `moz-http2.js` server) leaves the process group but keeps its
+ * environment, which is how teardown still finds it.
+ */
+import { createServer } from 'node:net';
+import { PROGRESS_EVERY_MS, PROGRESS_FIRST_MS } from '../kernel/lock.js';
+import { info, verbose, warn } from '../kernel/log.js';
+import { commandNamesRoot, listProcesses, run, snapshotProcesses, terminateByIdentity, } from '../kernel/process/index.js';
+import { formatDuration, sleep } from '../kernel/time.js';
+import { refusal } from './plan.js';
+/** Environment variable naming the run a harness process belongs to. */
+export const RUN_MARKER = 'FIREFORGE_TEST_RUN';
+/** Environment variable naming the engine a harness process runs in. */
+export const ENGINE_MARKER = 'FIREFORGE_TEST_ENGINE';
+const GRACE_MS = 1_500;
+const HELPER_BINARY = /^(?:xpcshell|ssltunnel|http3server|moz-http2)(?:\.exe)?$/;
+const HELPER_INTERPRETER = /^(?:python[\d.]*|Python|node)(?:\.exe)?$/;
+const HELPER_SCRIPT = /(?:^|[/\\])(?:pywebsocket\w*\.py|websocket_server\.py|websocketprocessbridge\.py|moz-http2\.js|http2_server\.js|httpd\.js|server\.js|runtests\.py|runxpcshelltests\.py)$/;
+const basename = (token) => token.slice(Math.max(token.lastIndexOf('/'), token.lastIndexOf('\\')) + 1);
+/**
+ * Whether a command line is a harness process: a helper binary, an
+ * interpreter running a helper script, or a browser started for a harness.
+ *
+ * @param command - A process command line
+ */
+export function isHarnessCommand(command) {
+    const [exe, first] = command.trim().split(/\s+/);
+    if (exe === undefined)
+        return false;
+    if (HELPER_BINARY.test(basename(exe)))
+        return true;
+    if (first !== undefined && HELPER_INTERPRETER.test(basename(exe)) && HELPER_SCRIPT.test(first)) {
+        return true;
+    }
+    return (/\s-marionette(?:\s|$)/.test(command) ||
+        /\s-profile(?:\s|=)\S*(?:mochitest|rust_mozprofile|fireforge-)/.test(command));
+}
+/** Whether a listing row carries `NAME=value` in its environment. */
+export function carriesMarker(listing, name, value) {
+    const needle = ` ${name}=${value}`;
+    for (let at = listing.indexOf(needle); at !== -1; at = listing.indexOf(needle, at + 1)) {
+        const after = listing[at + needle.length];
+        if (after === undefined || /\s/.test(after))
+            return true;
+    }
+    return false;
+}
+const namesWorkspace = (command, workspace) => commandNamesRoot(command, workspace.engineDir) || commandNamesRoot(command, workspace.objDir);
+/** This process and its ancestors, which are never leftovers. */
+function ancestors(parents) {
+    const chain = new Set();
+    for (let pid = process.pid; pid !== undefined && pid > 1 && !chain.has(pid);) {
+        chain.add(pid);
+        pid = parents.get(pid);
+    }
+    return chain;
+}
+/**
+ * The live processes a predicate selects, with exact identities, from one
+ * snapshot and two listings: command lines alone, and with the environment.
+ */
+async function selectProcesses(select) {
+    if (process.platform === 'win32')
+        return [];
+    const snapshot = await snapshotProcesses();
+    if (snapshot === undefined)
+        return undefined;
+    const commands = new Map((await listProcesses(['pid', 'command'])).map((row) => [row.pid, row.command]));
+    const environments = new Map((await listProcesses(['pid', 'command'], { environment: true })).map((row) => [
+        row.pid,
+        row.command,
+    ]));
+    const excluded = ancestors(new Map([...snapshot].map(([pid, entry]) => [pid, entry.parent])));
+    const selected = [];
+    for (const [pid, entry] of snapshot) {
+        const command = commands.get(pid);
+        if (command === undefined || excluded.has(pid))
+            continue;
+        if (select({ parent: entry.parent, command, environment: environments.get(pid) ?? '' })) {
+            selected.push({ pid, start: entry.start, parent: entry.parent, command });
+        }
+    }
+    return selected;
+}
+/**
+ * Whether a process is a leftover of an earlier run of this workspace: it
+ * carries this engine's marker, or it is a harness process naming this
+ * workspace whose parent has died.
+ */
+function isLeftover(entry, workspace) {
+    if (carriesMarker(entry.environment, ENGINE_MARKER, workspace.engineDir))
+        return true;
+    return (entry.parent === 1 &&
+        isHarnessCommand(entry.command) &&
+        namesWorkspace(entry.command, workspace));
+}
+/**
+ * Terminates what killed earlier runs of this workspace left running. Call
+ * it only while holding the workspace.
+ *
+ * @param workspace - The engine and object directory
+ * @returns How many processes were terminated
+ */
+export async function reapLeftovers(workspace) {
+    const leftovers = await selectProcesses((entry) => isLeftover(entry, workspace));
+    if (leftovers === undefined) {
+        verbose('The process table could not be read; leftovers of earlier runs were not looked for.');
+        return 0;
+    }
+    if (leftovers.length === 0)
+        return 0;
+    const rows = leftovers.map((entry) => `PID ${entry.pid}: ${entry.command.slice(0, 120)}`);
+    info(`Terminating ${leftovers.length} process(es) an earlier test run of this workspace left behind:\n  ${rows.join('\n  ')}`);
+    const { signalled, survivors } = await terminateByIdentity(leftovers, GRACE_MS);
+    for (const survivor of survivors)
+        warn(`PID ${survivor.pid} survived SIGKILL; inspect it by hand.`);
+    return signalled.length - survivors.length;
+}
+/**
+ * Terminates what this run started that outlived its harness: processes
+ * carrying the run's marker whose command line names the workspace. Then
+ * checks that nothing carrying the marker survives.
+ *
+ * @param workspace - The engine and object directory
+ * @param runId - The run's marker value
+ * @returns How many were terminated, and the survivors
+ */
+export async function sweepRun(workspace, runId) {
+    const ours = (entry) => carriesMarker(entry.environment, RUN_MARKER, runId);
+    const strays = await selectProcesses((entry) => ours(entry) && namesWorkspace(entry.command, workspace));
+    const { signalled, survivors } = await terminateByIdentity(strays ?? [], GRACE_MS);
+    if (signalled.length > 0) {
+        verbose(`Terminated ${signalled.length} harness helper(s) that outlived the harness: PID ${signalled.map((entry) => entry.pid).join(', ')}.`);
+    }
+    const left = (await selectProcesses(ours)) ?? [];
+    for (const entry of left) {
+        warn(`A process of this test run is still alive: PID ${entry.pid} (${entry.command.slice(0, 120)}).`);
+    }
+    return { reaped: signalled.length - survivors.length, survivors: left };
+}
+/** Marionette's default control port. */
+const MARIONETTE_PORT = 2828;
+/** The ports desktop mochitest's servers bind: HTTP, ssltunnel, WebSocket. */
+const SERVER_PORTS = [8888, 4443, 9988];
+/**
+ * Reads the listener pids of `lsof -F p` output.
+ *
+ * @param output - What lsof printed
+ */
+export function parseLsofPids(output) {
+    return [
+        ...new Set(output
+            .split(/\r?\n/)
+            .filter((line) => /^p\d+$/.test(line))
+            .map((line) => Number(line.slice(1)))),
+    ];
+}
+/** The process listening on a loopback TCP port, or undefined when none is (or lsof is missing). */
+async function portHolder(port) {
+    let pids;
+    try {
+        const result = await run('lsof', ['-nP', `-iTCP:${port}`, '-sTCP:LISTEN', '-Fp'], {
+            timeoutMs: 10_000,
+        });
+        pids = parseLsofPids(result.stdout);
+    }
+    catch {
+        return undefined;
+    }
+    const pid = pids[0];
+    if (pid === undefined)
+        return undefined;
+    const snapshot = await snapshotProcesses();
+    const [row] = await listProcesses(['pid', 'ppid', 'command'], { pids: [pid] });
+    const start = snapshot?.get(pid)?.start;
+    if (row === undefined || start === undefined)
+        return undefined;
+    return { pid, start, parent: row.ppid, command: row.command };
+}
+/** Whether the engine's mochitest harness moves a busy server port to a free one by itself. */
+export function harnessFallsBack(runtests) {
+    const fallsBack = runtests !== undefined && /falling back to port/.test(runtests);
+    return {
+        servers: fallsBack && /\("httpPort",/.test(runtests) && /_port_in_use\(port\)/.test(runtests),
+        marionette: fallsBack && /_port_in_use\(marionette_port\)/.test(runtests),
+    };
+}
+/** A free loopback TCP port, chosen by the kernel. */
+export async function freePort() {
+    return new Promise((resolve, reject) => {
+        const server = createServer();
+        server.once('error', reject);
+        server.listen(0, '127.0.0.1', () => {
+            const address = server.address();
+            const port = typeof address === 'object' && address !== null ? address.port : 0;
+            server.close(() => {
+                resolve(port);
+            });
+        });
+    });
+}
+/**
+ * Makes the ports a browser harness binds usable: a leftover of this
+ * workspace is terminated, a foreign holder is waited for or, under
+ * `auto`, stepped around (Marionette moves to a free port FireForge picks;
+ * a server port only where the harness falls back by itself).
+ *
+ * @param workspace - The engine and object directory
+ * @param options - See {@link PortOptions}
+ * @returns The mach arguments that move Marionette, if it moved
+ * @throws `port-busy` naming the holder when the wait expires
+ */
+export async function preparePorts(workspace, options) {
+    const startedAt = Date.now();
+    const deadline = startedAt + options.waitMs;
+    let lastProgress = startedAt;
+    const args = [];
+    for (const port of [MARIONETTE_PORT, ...SERVER_PORTS]) {
+        for (;;) {
+            const holder = await portHolder(port);
+            if (holder === undefined)
+                break;
+            if (holder.parent === 1 && namesWorkspace(holder.command, workspace)) {
+                info(`Port ${port} is held by PID ${holder.pid}, left behind by an earlier run of this workspace; terminating it.`);
+                await terminateByIdentity([holder], GRACE_MS);
+                continue;
+            }
+            if (options.ports === 'auto' && port === MARIONETTE_PORT) {
+                const moved = await freePort();
+                info(`Marionette port ${port} is held by PID ${holder.pid}; this run uses port ${moved}.`);
+                args.push(`--setpref=marionette.port=${moved}`, `--marionette=127.0.0.1:${moved}`);
+                break;
+            }
+            if (options.ports === 'auto' && options.fallback.servers) {
+                info(`Port ${port} is held by PID ${holder.pid}; the harness moves this run to a free port.`);
+                break;
+            }
+            if (Date.now() >= deadline) {
+                throw refusal('port-busy', `Port ${port} is held by PID ${holder.pid}, which is not a leftover of this workspace: ${holder.command.slice(0, 200)}`, 'Stop it, allow a longer wait with --wait-lock <seconds>, or set test.mochitestPorts to "auto".');
+            }
+            if (Date.now() - lastProgress >=
+                (lastProgress === startedAt ? PROGRESS_FIRST_MS : PROGRESS_EVERY_MS)) {
+                lastProgress = Date.now();
+                info(`Waiting for port ${port}, held by PID ${holder.pid} (${holder.command.slice(0, 120)}): ` +
+                    `${formatDuration(Date.now() - startedAt)} of up to ${formatDuration(options.waitMs)}.`);
+            }
+            await sleep(Math.min(1_000, Math.max(0, deadline - Date.now())));
+        }
+    }
+    return args;
+}

package/dist/src/test/index.js

@@ -0,0 +1,11 @@
+// SPDX-License-Identifier: EUPL-1.2
+/**
+ * Test runs: resolving what to run, keeping the host clean around the
+ * harness, running mach's test harnesses in a workspace, judging their
+ * output into a verdict that a crash, a hang or a cut-off suite cannot fake,
+ * and negative controls that prove the tests catch what they claim to.
+ */
+export { runControl } from './control.js';
+export { envAssignments, planTests } from './plan.js';
+export { runTests, validatorEnvironment } from './run.js';
+export { controlReport, refusalLines, refusalVerdict, runReport, writeResults, } from './verdict.js';