@hominis/fireforge 0.47.0 → 0.47.1

package/dist/src/core/harness-helper-pids.d.ts

@@ -0,0 +1,98 @@
+/**
+ * Teardown reaper for the helper processes one mach test dispatch launched.
+ *
+ * The mochitest harness starts its httpd (`xpcshell … server.js`), the
+ * websocket server, ssltunnel and the websocket/process bridge as plain
+ * children, and the browser through mozprocess, which puts it in a process
+ * group of its own. It logs each launch as `runtests.py | <what> pid: <n>`.
+ * When mach dies before its own cleanup runs (a forwarded SIGTERM, a
+ * no-output timeout, a harness crash, a supervisor kill), the helpers do not
+ * necessarily die with it: the 2026-09-13 incident left the httpd reparented
+ * to launchd at 100% CPU for hours after a gate killed the run at its bound.
+ *
+ * The exec layer already sweeps the dispatch's process group after close.
+ * That reaches every helper that stayed in the group, and nothing else: the
+ * browser is in its own group, and a helper that re-grouped itself is not.
+ * This module closes that gap in two passes:
+ *
+ *  - {@link createHelperPidTracker} collects the announced pids from the
+ *    streamed output. {@link reapTrackedHarnessPids} runs after mach has
+ *    exited; a tracked pid still alive at that point is by definition a
+ *    survivor (its parent is gone), so no age heuristic is needed. Before
+ *    signalling, each pid is re-read from `ps` and must still be anchored to
+ *    the objdir and still look like the thing the harness launched: a pid
+ *    recycled onto an unrelated process between the launch line and
+ *    teardown is skipped.
+ *  - Not every helper is announced. mozserve starts `moz-http2` with
+ *    `start_new_session=True` and logs no pid, so it is in neither the
+ *    group nor the tracker, and it was the other survivor of the incident.
+ *    {@link snapshotObjdirHelpers} records which same-objdir helpers existed
+ *    before the dispatch spawned; after mach exits, a helper-shaped process
+ *    under THIS checkout's absolute objdir that is not in that baseline and
+ *    is reparented (its parent, the harness, is gone) is one this dispatch
+ *    launched, and is reaped too. The absolute-path anchor keeps a sibling
+ *    checkout's live harness out of it; the baseline keeps a survivor of an
+ *    earlier run out of it, so the preflight census's report-only posture
+ *    still decides that one's fate.
+ */
+/**
+ * What the harness said it launched, as it names it in the log line, or
+ * `unannounced` for a survivor found by the post-close objdir scan.
+ */
+export type TrackedHarnessKind = 'Server' | 'Websocket server' | 'SSL tunnel' | 'websocket/process bridge' | 'Application' | 'unannounced';
+/** One helper the harness launched. */
+export interface TrackedHarnessPid {
+    pid: number;
+    kind: TrackedHarnessKind;
+    /** Command line, when the entry came from `ps` rather than a log line. */
+    command?: string;
+}
+/** Which child stream a chunk came from. */
+export type OutputStreamName = 'stdout' | 'stderr';
+/** Collects harness launch announcements from streamed output chunks. */
+export interface HelperPidTracker {
+    /** Feed one chunk. Chunk boundaries need not align with lines. */
+    feed(stream: OutputStreamName, chunk: string): void;
+    /** Every pid announced so far, in announcement order, without duplicates. */
+    tracked(): TrackedHarnessPid[];
+}
+/**
+ * Creates a tracker for one dispatch. The two streams are buffered apart so
+ * a half-line on one cannot be glued to a half-line on the other.
+ */
+export declare function createHelperPidTracker(): HelperPidTracker;
+/**
+ * Decides whether a `ps` row still describes the process the harness
+ * announced. Exported for fixture tests.
+ *
+ * The helpers must match the census's helper pattern and the objdir anchor.
+ * The browser has no helper-shaped name, so the anchor alone attributes it:
+ * its binary lives under the objdir (`<objdir>/dist/<App>.app/…`).
+ */
+export declare function isStillTrackedProcess(tracked: TrackedHarnessPid, command: string, objDir: string | undefined): boolean;
+/**
+ * Records the same-objdir helpers alive before a dispatch spawns anything,
+ * so the post-close pass can tell this dispatch's unannounced helpers from
+ * an earlier run's survivors. Undefined when `ps` is unusable or no objdir
+ * is known; the post-close pass then reaps no unannounced process at all.
+ *
+ * @param objDir - This project's absolute objdir
+ * @returns The pids to leave alone at teardown, or undefined for "no baseline"
+ */
+export declare function snapshotObjdirHelpers(objDir: string | undefined): Promise<Set<number> | undefined>;
+/**
+ * Terminates every process this dispatch launched that is still alive after
+ * mach exited: the announced pids still attributable to the dispatch, and
+ * the unannounced same-objdir survivors that appeared since the baseline.
+ *
+ * Best-effort like the census: a host without `ps` logs at verbose and
+ * reaps nothing rather than signalling on the strength of a pid alone.
+ *
+ * @param tracked - The pids the harness announced during the dispatch
+ * @param objDir - This project's absolute objdir, for the anchor rule
+ * @param baseline - Same-objdir helpers alive before the dispatch spawned
+ *   ({@link snapshotObjdirHelpers}); never touched here. Undefined disables
+ *   the unannounced pass entirely
+ * @returns How many survivors are confirmed gone afterwards
+ */
+export declare function reapTrackedHarnessPids(tracked: readonly TrackedHarnessPid[], objDir: string | undefined, baseline?: ReadonlySet<number>): Promise<number>;

package/dist/src/core/harness-helper-pids.js

@@ -0,0 +1,240 @@
+// SPDX-License-Identifier: EUPL-1.2
+/**
+ * Teardown reaper for the helper processes one mach test dispatch launched.
+ *
+ * The mochitest harness starts its httpd (`xpcshell … server.js`), the
+ * websocket server, ssltunnel and the websocket/process bridge as plain
+ * children, and the browser through mozprocess, which puts it in a process
+ * group of its own. It logs each launch as `runtests.py | <what> pid: <n>`.
+ * When mach dies before its own cleanup runs (a forwarded SIGTERM, a
+ * no-output timeout, a harness crash, a supervisor kill), the helpers do not
+ * necessarily die with it: the 2026-09-13 incident left the httpd reparented
+ * to launchd at 100% CPU for hours after a gate killed the run at its bound.
+ *
+ * The exec layer already sweeps the dispatch's process group after close.
+ * That reaches every helper that stayed in the group, and nothing else: the
+ * browser is in its own group, and a helper that re-grouped itself is not.
+ * This module closes that gap in two passes:
+ *
+ *  - {@link createHelperPidTracker} collects the announced pids from the
+ *    streamed output. {@link reapTrackedHarnessPids} runs after mach has
+ *    exited; a tracked pid still alive at that point is by definition a
+ *    survivor (its parent is gone), so no age heuristic is needed. Before
+ *    signalling, each pid is re-read from `ps` and must still be anchored to
+ *    the objdir and still look like the thing the harness launched: a pid
+ *    recycled onto an unrelated process between the launch line and
+ *    teardown is skipped.
+ *  - Not every helper is announced. mozserve starts `moz-http2` with
+ *    `start_new_session=True` and logs no pid, so it is in neither the
+ *    group nor the tracker, and it was the other survivor of the incident.
+ *    {@link snapshotObjdirHelpers} records which same-objdir helpers existed
+ *    before the dispatch spawned; after mach exits, a helper-shaped process
+ *    under THIS checkout's absolute objdir that is not in that baseline and
+ *    is reparented (its parent, the harness, is gone) is one this dispatch
+ *    launched, and is reaped too. The absolute-path anchor keeps a sibling
+ *    checkout's live harness out of it; the baseline keeps a survivor of an
+ *    earlier run out of it, so the preflight census's report-only posture
+ *    still decides that one's fate.
+ */
+import { toError } from '../utils/errors.js';
+import { dispatchCompleteLines } from '../utils/line-dispatch.js';
+import { info, verbose } from '../utils/logger.js';
+import { exec } from '../utils/process.js';
+import { isObjdirAnchored, isPidAlive, matchesHarnessHelper, terminateHarnessProcesses, } from './harness-orphans.js';
+/**
+ * The launch announcements in `testing/mochitest/runtests.py`. mozlog puts a
+ * timestamp and level in front, so the match is unanchored.
+ */
+const HARNESS_PID_LINE = /runtests\.py \| (Server|Websocket server|SSL tunnel|websocket\/process bridge|Application) pid: (\d+)\b/;
+/**
+ * Creates a tracker for one dispatch. The two streams are buffered apart so
+ * a half-line on one cannot be glued to a half-line on the other.
+ */
+export function createHelperPidTracker() {
+    const seen = new Map();
+    const partial = { stdout: '', stderr: '' };
+    const onLine = (line) => {
+        const match = HARNESS_PID_LINE.exec(line);
+        if (!match)
+            return;
+        const pid = Number(match[2]);
+        if (!seen.has(pid)) {
+            seen.set(pid, { pid, kind: match[1] });
+        }
+    };
+    return {
+        feed(stream, chunk) {
+            partial[stream] = dispatchCompleteLines(partial[stream] + chunk, onLine);
+        },
+        tracked() {
+            return [...seen.values()];
+        },
+    };
+}
+/** Log prefix for every line this reaper writes. */
+const TEARDOWN_LABEL = 'harness teardown';
+/**
+ * Decides whether a `ps` row still describes the process the harness
+ * announced. Exported for fixture tests.
+ *
+ * The helpers must match the census's helper pattern and the objdir anchor.
+ * The browser has no helper-shaped name, so the anchor alone attributes it:
+ * its binary lives under the objdir (`<objdir>/dist/<App>.app/…`).
+ */
+export function isStillTrackedProcess(tracked, command, objDir) {
+    if (!isObjdirAnchored(command, objDir))
+        return false;
+    return tracked.kind === 'Application' || matchesHarnessHelper(command);
+}
+/** Lists every process. Throws when `ps` is unusable, like the census. */
+async function listAllProcesses() {
+    const result = await exec('ps', ['-axo', 'pid=,ppid=,command='], { timeout: 10000 });
+    if (result.exitCode !== 0)
+        throw new Error(`ps exited ${result.exitCode}`);
+    const rows = [];
+    for (const line of result.stdout.split('\n')) {
+        const match = /^\s*(\d+)\s+(\d+)\s+(.+)$/.exec(line);
+        if (match) {
+            rows.push({
+                pid: Number(match[1]),
+                ppid: Number(match[2]),
+                command: (match[3] ?? '').trim(),
+            });
+        }
+    }
+    return rows;
+}
+/**
+ * A helper-shaped process under this checkout's objdir. The anchor here is
+ * the absolute objdir path alone, on purpose: the census's generic `obj-…`
+ * fallback would also match a sibling checkout's live harness.
+ */
+function isThisObjdirHelper(command, objDir) {
+    return command.includes(objDir) && matchesHarnessHelper(command);
+}
+/**
+ * Records the same-objdir helpers alive before a dispatch spawns anything,
+ * so the post-close pass can tell this dispatch's unannounced helpers from
+ * an earlier run's survivors. Undefined when `ps` is unusable or no objdir
+ * is known; the post-close pass then reaps no unannounced process at all.
+ *
+ * @param objDir - This project's absolute objdir
+ * @returns The pids to leave alone at teardown, or undefined for "no baseline"
+ */
+export async function snapshotObjdirHelpers(objDir) {
+    if (process.platform === 'win32' || objDir === undefined)
+        return undefined;
+    try {
+        const rows = await listAllProcesses();
+        return new Set(rows.filter((r) => isThisObjdirHelper(r.command, objDir)).map((r) => r.pid));
+    }
+    catch (error) {
+        verbose(`${TEARDOWN_LABEL}: could not snapshot helpers (${toError(error).message}).`);
+        return undefined;
+    }
+}
+/**
+ * Helpers this dispatch launched without announcing them: helper-shaped,
+ * under this checkout's objdir, not in the pre-dispatch baseline, and
+ * reparented to pid 1 because the harness that started them is gone.
+ * Returns nothing when `ps` is unusable rather than guessing. Without a
+ * baseline (a snapshot that failed) it also returns nothing: an earlier
+ * run's survivor must stay the census's call.
+ */
+async function findUnannouncedSurvivors(objDir, baseline, exclude) {
+    if (baseline === undefined)
+        return [];
+    let rows;
+    try {
+        rows = await listAllProcesses();
+    }
+    catch (error) {
+        verbose(`${TEARDOWN_LABEL}: could not scan for unannounced helpers (${toError(error).message}).`);
+        return [];
+    }
+    return rows
+        .filter((r) => r.ppid === 1 &&
+        r.pid !== process.pid &&
+        !baseline.has(r.pid) &&
+        !exclude.has(r.pid) &&
+        isThisObjdirHelper(r.command, objDir))
+        .map((r) => ({ pid: r.pid, kind: 'unannounced', command: r.command }));
+}
+/** Reads `pid command` rows for the given pids. Missing pids simply produce no row. */
+async function listCommands(pids) {
+    const result = await exec('ps', ['-o', 'pid=,command=', '-p', pids.join(',')], {
+        timeout: 10000,
+    });
+    const rows = new Map();
+    // `ps -p` exits 1 when any listed pid is gone, while still printing the
+    // rest, so the exit code is not a failure signal here.
+    for (const line of result.stdout.split('\n')) {
+        const match = /^\s*(\d+)\s+(.+)$/.exec(line);
+        if (match)
+            rows.set(Number(match[1]), (match[2] ?? '').trim());
+    }
+    return rows;
+}
+/**
+ * Re-reads the announced pids that are still alive and keeps the ones still
+ * attributable to this dispatch. Empty when `ps` is unusable.
+ */
+async function survivingTrackedPids(tracked, objDir) {
+    const alive = tracked.filter((entry) => isPidAlive(entry.pid));
+    if (alive.length === 0)
+        return [];
+    let commands;
+    try {
+        commands = await listCommands(alive.map((entry) => entry.pid));
+    }
+    catch (error) {
+        verbose(`${TEARDOWN_LABEL}: could not read ps for tracked helpers (${toError(error).message}); skipping.`);
+        return [];
+    }
+    const targets = [];
+    for (const entry of alive) {
+        const command = commands.get(entry.pid);
+        if (command === undefined)
+            continue; // exited between the probe and ps
+        if (!isStillTrackedProcess(entry, command, objDir)) {
+            verbose(`${TEARDOWN_LABEL}: PID ${entry.pid} (announced as ${entry.kind}) is now "${command.slice(0, 120)}", ` +
+                'not attributable to this run; left alone.');
+            continue;
+        }
+        targets.push(entry);
+    }
+    return targets;
+}
+function describeTarget(target) {
+    const excerpt = target.command === undefined ? '' : ` "${target.command.slice(0, 80)}"`;
+    return `${target.kind} ${target.pid}${excerpt}`;
+}
+/**
+ * Terminates every process this dispatch launched that is still alive after
+ * mach exited: the announced pids still attributable to the dispatch, and
+ * the unannounced same-objdir survivors that appeared since the baseline.
+ *
+ * Best-effort like the census: a host without `ps` logs at verbose and
+ * reaps nothing rather than signalling on the strength of a pid alone.
+ *
+ * @param tracked - The pids the harness announced during the dispatch
+ * @param objDir - This project's absolute objdir, for the anchor rule
+ * @param baseline - Same-objdir helpers alive before the dispatch spawned
+ *   ({@link snapshotObjdirHelpers}); never touched here. Undefined disables
+ *   the unannounced pass entirely
+ * @returns How many survivors are confirmed gone afterwards
+ */
+export async function reapTrackedHarnessPids(tracked, objDir, baseline) {
+    if (process.platform === 'win32')
+        return 0;
+    const targets = await survivingTrackedPids(tracked, objDir);
+    if (objDir !== undefined) {
+        const known = new Set(tracked.map((t) => t.pid));
+        targets.push(...(await findUnannouncedSurvivors(objDir, baseline, known)));
+    }
+    if (targets.length === 0)
+        return 0;
+    info(`${TEARDOWN_LABEL}: ${targets.length} harness process(es) outlived mach ` +
+        `(${targets.map(describeTarget).join(', ')}); terminating.`);
+    return terminateHarnessProcesses(targets, TEARDOWN_LABEL);
+}

package/dist/src/core/harness-orphans.d.ts

@@ -21,10 +21,15 @@
  *    for the same reason. FireForge has no business reporting, let alone
  *    terminating, a process it cannot attribute to the harness.
  *
- * `--reap-orphans` opts into termination. Without it the census is
- * report-only, exactly like the `Orphaned harness workers` doctor check
- * (which covers a different shape: reparented Python multiprocessing
- * workers, matched on PPID 1 and accumulated CPU time).
+ * `--reap-orphans` (or `test.reapOrphans: "reap"` in `fireforge.json`) opts
+ * into termination. Without it the census is report-only, exactly like the
+ * `Orphaned harness workers` doctor check (which covers a different shape:
+ * reparented Python multiprocessing workers, matched on PPID 1 and
+ * accumulated CPU time).
+ *
+ * The anchoring and termination primitives are exported for the teardown
+ * reaper in `harness-helper-pids.ts`, which applies the same objdir rule to
+ * the helpers THIS run launched, after mach has exited.
  */
 /** One surviving harness helper process. */
 export interface OrphanedHarnessProcess {
@@ -36,15 +41,32 @@ export interface OrphanedHarnessProcess {
     elapsedSeconds: number;
     command: string;
 }
+/**
+ * True when `command` IS a harness helper: its executable is a helper
+ * binary, or an interpreter whose first argument is a helper script.
+ *
+ * Structural on purpose. An earlier version matched the helper names
+ * anywhere in the command line, which is right for the httpd (`xpcshell -g
+ * … -f …/server.js`) and wrong for everything that merely mentions such a
+ * path: an editor opened on `server.js`, a shell whose command text quotes
+ * it, a `grep` over the objdir. Report-only, that printed a misleading
+ * `kill`; under the reap posture it terminated the operator's shell.
+ */
+export declare function matchesHarnessHelper(command: string): boolean;
+/** Objdir provenance: an explicit objdir path, a `obj-…` path segment, or `_tests/`. */
+export declare function isObjdirAnchored(command: string, objDir: string | undefined): boolean;
 /**
  * Scans `ps -axo pid=,ppid=,etime=,command=` output for surviving harness
  * helpers. Pure, so it is fixture-testable without spawning anything.
  *
+ * FireForge's own process, its direct children and its ancestors are never
+ * candidates: the terminal shell FireForge runs under can carry the helper
+ * paths in its own command text, and must never be offered for a kill.
+ *
  * @param psOutput - Raw `ps` output
  * @param objDir - Absolute objdir of this project, when known. Widens the
  *   provenance test beyond the generic `obj-…` path segment
- * @param selfPid - This process's pid, excluded so FireForge cannot report
- *   itself
+ * @param selfPid - This process's pid
  */
 export declare function findOrphanedHarnessProcesses(psOutput: string, objDir?: string, selfPid?: number): OrphanedHarnessProcess[];
 /**
@@ -66,8 +88,31 @@ export declare function formatOrphanReport(orphans: readonly OrphanedHarnessProc
  * @param objDir - Absolute objdir of this project, when known
  * @param options - `reap` terminates each recognized survivor (SIGTERM,
  *   then SIGKILL for anything that stays)
- * @returns The census, empty when nothing was found or nothing could be probed
+ * @returns The census (empty when nothing was found or nothing could be
+ *   probed) and how many of them are confirmed gone after reaping (0 in
+ *   report-only mode)
  */
 export declare function reportOrphanedHarnessProcesses(objDir: string | undefined, options?: {
     reap?: boolean;
-}): Promise<OrphanedHarnessProcess[]>;
+}): Promise<OrphanCensus>;
+/** Result of {@link reportOrphanedHarnessProcesses}. */
+export interface OrphanCensus {
+    orphans: OrphanedHarnessProcess[];
+    /** Survivors confirmed gone after the reap. Always 0 in report-only mode. */
+    reaped: number;
+}
+/** Liveness probe: `kill(pid, 0)` throws for a pid that is gone. */
+export declare function isPidAlive(pid: number): boolean;
+/**
+ * Terminates each entry: SIGTERM first, then SIGKILL for anything still
+ * alive after the grace period. Failures are reported, never thrown. A
+ * process that exited between the listing and the signal is the common case
+ * rather than an error.
+ *
+ * @param targets - Processes to terminate, already attributed to the harness
+ *   by the caller (objdir anchor rule)
+ * @param label - Prefix for the log lines, naming which path is reaping
+ *   (`--reap-orphans` at preflight, `teardown` after mach exits)
+ * @returns How many targets are confirmed gone afterwards
+ */
+export declare function terminateHarnessProcesses(targets: readonly Pick<OrphanedHarnessProcess, 'pid'>[], label: string): Promise<number>;