@hominis/fireforge 0.33.0 → 0.34.1

package/dist/src/core/mach-resource-shim.js

@@ -1,92 +1,350 @@
 // SPDX-License-Identifier: EUPL-1.2
 /**
- * Resource-monitor degrade shim for `mach build`.
+ * Resource-monitor degrade guard for mach dispatches.
  *
  * mozbuild's build resource monitor calls `psutil.virtual_memory()` at
- * startup (`start_resource_recording`). On some hosts (psutil vs Darwin 27)
- * that raises `RuntimeError: host_statistics64(HOST_VM_INFO64) syscall
- * failed`, which aborts `mach build` / `mach build faster` before any work
- * happens — so a raw `./mach build faster` dies while the build never
- * reaches the compiler. The suite-specific test commands sidestep the
- * monitor entirely, but the build commands have no such escape.
+ * startup (`start_resource_recording`) and again later from
+ * `mozbuild.base._run_make`. On some hosts (psutil vs Darwin 27) those
+ * raise `RuntimeError: host_statistics64(HOST_VM_INFO64) syscall failed`,
+ * and a `SystemResourceMonitor.__init__` that fails partway leaves an
+ * instance without `poll_interval`, so later polling dies with
+ * `AttributeError: ... poll_interval`. Any of these aborts `mach build` /
+ * `mach build faster` before the compiler ever runs.
  *
- * FireForge writes a tiny `sitecustomize.py` and points the mach
- * subprocess's `PYTHONPATH` at it. Python imports `sitecustomize` at
- * interpreter startup (before mach runs), so the shim wraps
- * `psutil.virtual_memory` / `swap_memory` to downgrade the host syscall
- * failure into a non-fatal `UserWarning` and return a zeroed reading. The
- * build then proceeds regardless of which mach entry point spawned it. When
- * psutil works normally the wrapper is a transparent pass-through.
+ * 0.33.0 injected a `sitecustomize.py` via the mach subprocess PYTHONPATH.
+ * Field report (0.34.0 cycle): mach re-execs itself into its private
+ * virtualenv and drops PYTHONPATH, so the env-var route never loads and
+ * the crash family escaped on every protected entry point. FireForge now
+ * owns the guard in-process on its mach dispatches by installing a
+ * `fireforge_mach_guard.pth` + module pair directly into every discovered
+ * mach virtualenv site-packages directory: `.pth` files execute at
+ * interpreter startup from the venv's own site-packages, which survives
+ * the re-exec. The PYTHONPATH `sitecustomize.py` is retained only as a
+ * belt-and-suspenders for the pre-venv bootstrap phase (first build, no
+ * `_virtualenvs` on disk yet).
+ *
+ * The guard itself covers the whole crash family:
+ *  - wraps `psutil.virtual_memory` / `swap_memory` / `cpu_percent` /
+ *    `cpu_times` / `disk_io_counters` module-wide, so every call site —
+ *    including the direct `psutil.virtual_memory()` in
+ *    `mozbuild.base._run_make` — degrades to a `UserWarning` and a zeroed
+ *    reading instead of aborting. Downstream report (0.34.0 cycle):
+ *    mozsystemmonitor subscripts (`_build_meta` does
+ *    `psutil.virtual_memory()[0]`) and iterates/unpacks the reading, so the
+ *    zeroed fallback is a full namedtuple duck type — a zeroed instance of
+ *    psutil's own result namedtuple where resolvable without touching the
+ *    failing syscall, else `_DegradedReading` (subscriptable, iterable, with
+ *    `len()`, `_fields`, `_asdict()`); `cpu_percent` degrades to `0.0`
+ *    because callers do arithmetic on it;
+ *  - guards `SystemResourceMonitor` CONSTRUCTION via an import hook:
+ *    `poll_interval` is pre-populated before the real `__init__` runs, a
+ *    failing `__init__` marks the instance degraded instead of raising,
+ *    and every monitor method no-ops on a degraded instance — so a
+ *    partially-constructed monitor can never surface the
+ *    `AttributeError: poll_interval` variant.
  */
-import { tmpdir } from 'node:os';
+import { readdir } from 'node:fs/promises';
+import { homedir, tmpdir } from 'node:os';
 import { delimiter, join } from 'node:path';
-import { ensureDir, writeText } from '../utils/fs.js';
-/** Directory name (under the OS temp dir) holding the generated shim. */
+import { toError } from '../utils/errors.js';
+import { ensureDir, pathExists, writeText } from '../utils/fs.js';
+import { verbose } from '../utils/logger.js';
+/** Directory name (under the OS temp dir) holding the PYTHONPATH fallback. */
 const RESOURCE_SHIM_DIRNAME = 'fireforge-mach-resource-shim';
+/** Basename (sans extension) of the in-venv guard module. */
+const GUARD_MODULE_NAME = 'fireforge_mach_guard';
 /**
- * `sitecustomize.py` body. Defensive: a missing/broken psutil leaves mach
- * untouched, and the wrapper only intercepts the failure path.
+ * Python guard body, importable both as the in-venv `fireforge_mach_guard`
+ * module (loaded by the sibling `.pth` at interpreter startup) and as the
+ * PYTHONPATH `sitecustomize` fallback. Defensive throughout: a missing or
+ * broken psutil/mozsystemmonitor leaves mach untouched, and every wrapper
+ * only intercepts the failure path. Exported for the Python-executing
+ * regression test (mach-resource-shim.python.test.ts), which asserts the
+ * degraded readings' runtime behavior rather than string-matching the source.
  */
-const SITECUSTOMIZE_SOURCE = `# Generated by FireForge — do not edit.
-# Degrades a broken host resource monitor (psutil vs Darwin) from a fatal
-# RuntimeError in start_resource_recording into a non-fatal warning, so
-# \`mach build\` does not abort before the build runs. Loaded via PYTHONPATH
-# so it patches psutil before mach's resource monitor starts.
+export const GUARD_PYTHON_SOURCE = `# Generated by FireForge - do not edit.
+# Degrades the broken host resource monitor family (psutil vs Darwin) from
+# fatal startup errors into non-fatal warnings, so mach builds and tests
+# proceed. Installed into the mach virtualenv site-packages via a .pth file
+# (survives mach's venv re-exec, which drops PYTHONPATH).
+import builtins
+import sys
 import warnings
 
-try:
-    import psutil
-except Exception:
-    psutil = None
 
-if psutil is not None:
-    class _DegradedReading(object):
-        total = available = used = free = active = inactive = wired = 0
-        percent = 0.0
+def _fireforge_degraded_notice(exc):
+    warnings.warn(
+        "FireForge: host resource monitor degraded (%s); continuing without resource monitoring." % exc
+    )
+
+
+class _DegradedReading(object):
+    # Duck type of psutil's namedtuple results (svmem field order):
+    # mozsystemmonitor subscripts, iterates, and unpacks readings, so the
+    # degraded fallback must survive r[0], list(r), len(r), r._fields and
+    # r._asdict(), not just attribute access.
+    _fields = ("total", "available", "percent", "used", "free", "active", "inactive", "wired")
+    total = available = used = free = active = inactive = wired = 0
+    percent = 0.0
+
+    def __getattr__(self, _name):
+        return 0
+
+    def _values(self):
+        return tuple(getattr(self, _field) for _field in self._fields)
+
+    def __getitem__(self, index):
+        return self._values()[index]
+
+    def __iter__(self):
+        return iter(self._values())
+
+    def __len__(self):
+        return len(self._fields)
+
+    def _asdict(self):
+        return dict(zip(self._fields, self._values()))
+
+
+# psutil result namedtuple per wrapped function, resolvable via getattr from
+# the platform module / _common without invoking the failing syscall.
+_PSUTIL_RESULT_CLASSES = {
+    "virtual_memory": "svmem",
+    "swap_memory": "sswap",
+    "cpu_times": "scputimes",
+    "disk_io_counters": "sdiskio",
+}
 
-        def __getattr__(self, _name):
-            return 0
 
-    def _guard(orig):
+def _degraded_result_factory(psutil, name):
+    # cpu_percent returns a plain float callers do arithmetic on.
+    if name == "cpu_percent":
+        return lambda: 0.0
+    _cls_name = _PSUTIL_RESULT_CLASSES.get(name)
+    if _cls_name is not None:
+        for _mod in (getattr(psutil, "_psplatform", None), getattr(psutil, "_common", None)):
+            _cls = getattr(_mod, _cls_name, None) if _mod is not None else None
+            _fields = getattr(_cls, "_fields", None)
+            if _cls is not None and _fields:
+                return lambda _cls=_cls, _n=len(_fields): _cls(*([0] * _n))
+    return _DegradedReading
+
+
+def _guard_psutil():
+    try:
+        import psutil
+    except Exception:
+        return
+
+    def _wrap(orig, fallback):
         def wrapper(*args, **kwargs):
             try:
                 return orig(*args, **kwargs)
             except Exception as exc:  # noqa: BLE001
-                warnings.warn(
-                    "FireForge: host resource monitor degraded (%s); build continues." % exc
-                )
-                return _DegradedReading()
+                _fireforge_degraded_notice(exc)
+                try:
+                    return fallback()
+                except Exception:  # noqa: BLE001
+                    return _DegradedReading()
 
         return wrapper
 
-    for _name in ("virtual_memory", "swap_memory"):
+    for _name in ("virtual_memory", "swap_memory", "cpu_percent", "cpu_times", "disk_io_counters"):
         _orig = getattr(psutil, _name, None)
+        if _orig is not None and not getattr(_orig, "_fireforge_guarded", False):
+            try:
+                _fallback = _degraded_result_factory(psutil, _name)
+            except Exception:  # noqa: BLE001
+                _fallback = _DegradedReading
+            _wrapped = _wrap(_orig, _fallback)
+            _wrapped._fireforge_guarded = True
+            setattr(psutil, _name, _wrapped)
+
+
+_MONITOR_METHOD_NAMES = (
+    "start",
+    "stop",
+    "record_event",
+    "record_marker",
+    "begin_phase",
+    "finish_phase",
+    "aggregate_cpu_percent",
+    "aggregate_cpu_times",
+    "aggregate_io",
+    "min_memory_available",
+    "as_dict",
+)
+
+
+def _patch_monitor_class(cls):
+    if cls is None or getattr(cls, "_fireforge_guarded", False):
+        return
+    orig_init = cls.__init__
+
+    def guarded_init(self, *args, **kwargs):
+        # Pre-populate the attributes a partially-constructed instance is
+        # later polled for, so a failing __init__ can never surface
+        # AttributeError: poll_interval.
+        self.poll_interval = kwargs.get("poll_interval") or 1.0
+        self._fireforge_degraded = False
+        try:
+            orig_init(self, *args, **kwargs)
+        except Exception as exc:  # noqa: BLE001
+            self._fireforge_degraded = True
+            _fireforge_degraded_notice(exc)
+
+    cls.__init__ = guarded_init
+
+    def _wrap_method(orig):
+        def wrapper(self, *args, **kwargs):
+            if getattr(self, "_fireforge_degraded", False):
+                return None
+            try:
+                return orig(self, *args, **kwargs)
+            except Exception as exc:  # noqa: BLE001
+                self._fireforge_degraded = True
+                _fireforge_degraded_notice(exc)
+                return None
+
+        return wrapper
+
+    for _name in _MONITOR_METHOD_NAMES:
+        _orig = getattr(cls, _name, None)
         if _orig is not None:
-            setattr(psutil, _name, _guard(_orig))
+            setattr(cls, _name, _wrap_method(_orig))
+    cls._fireforge_guarded = True
+
+
+_MONITOR_MODULES = ("mozsystemmonitor.resourcemonitor", "mozbuild.resources")
+
+
+def _patch_loaded_monitor_modules():
+    for _mod_name in _MONITOR_MODULES:
+        _mod = sys.modules.get(_mod_name)
+        if _mod is not None:
+            try:
+                _patch_monitor_class(getattr(_mod, "SystemResourceMonitor", None))
+            except Exception:  # noqa: BLE001
+                pass
+
+
+def _install_import_hook():
+    _orig_import = builtins.__import__
+    if getattr(_orig_import, "_fireforge_guarded", False):
+        return
+
+    def _guarding_import(name, *args, **kwargs):
+        _module = _orig_import(name, *args, **kwargs)
+        try:
+            _patch_loaded_monitor_modules()
+        except Exception:  # noqa: BLE001
+            pass
+        return _module
+
+    _guarding_import._fireforge_guarded = True
+    builtins.__import__ = _guarding_import
+
+
+try:
+    _guard_psutil()
+    _patch_loaded_monitor_modules()
+    _install_import_hook()
+except Exception:  # noqa: BLE001
+    pass
 `;
+/** Lists subdirectory names of `dir`, tolerating a missing directory. */
+async function listSubdirs(dir) {
+    try {
+        const entries = await readdir(dir, { withFileTypes: true });
+        return entries.filter((entry) => entry.isDirectory()).map((entry) => entry.name);
+    }
+    catch {
+        return [];
+    }
+}
+/**
+ * Discovers `site-packages` directories of the mach virtualenvs under one
+ * `_virtualenvs` root: `<root>/<venv>/lib/python<N.M>/site-packages`.
+ */
+async function discoverVenvSitePackages(virtualenvsDir) {
+    const found = [];
+    for (const venvName of await listSubdirs(virtualenvsDir)) {
+        const libDir = join(virtualenvsDir, venvName, 'lib');
+        for (const pythonName of await listSubdirs(libDir)) {
+            if (!pythonName.startsWith('python'))
+                continue;
+            const sitePackages = join(libDir, pythonName, 'site-packages');
+            if (await pathExists(sitePackages)) {
+                found.push(sitePackages);
+            }
+        }
+    }
+    return found;
+}
+/**
+ * Discovers every mach virtualenv site-packages dir relevant to an engine
+ * checkout: the objdir venvs (under `<engineDir>/obj-&lt;x&gt;/_virtualenvs`)
+ * and the mach state-dir venvs (under `$MOZBUILD_STATE_PATH` or
+ * `~/.mozbuild`, in `srcdirs/&lt;hash&gt;/_virtualenvs`, where mach keeps the
+ * `mach` command venv itself).
+ */
+async function discoverMachSitePackages(engineDir) {
+    const found = [];
+    for (const name of await listSubdirs(engineDir)) {
+        if (!name.startsWith('obj-'))
+            continue;
+        found.push(...(await discoverVenvSitePackages(join(engineDir, name, '_virtualenvs'))));
+    }
+    const stateDir = process.env['MOZBUILD_STATE_PATH'] ?? join(homedir(), '.mozbuild');
+    for (const srcdirName of await listSubdirs(join(stateDir, 'srcdirs'))) {
+        found.push(...(await discoverVenvSitePackages(join(stateDir, 'srcdirs', srcdirName, '_virtualenvs'))));
+    }
+    return found;
+}
 /**
- * Writes the `sitecustomize.py` degrade shim into a stable FireForge-owned
- * temp directory and returns that directory. Idempotent — overwriting the
- * file each call keeps it in sync with this source if FireForge upgrades.
+ * Writes the PYTHONPATH `sitecustomize.py` fallback into a stable
+ * FireForge-owned temp directory and returns that directory. Idempotent —
+ * overwriting each call keeps it in sync with this source across upgrades.
  */
-async function ensureMachResourceShim() {
+async function ensureSitecustomizeFallback() {
     const dir = join(tmpdir(), RESOURCE_SHIM_DIRNAME);
     await ensureDir(dir);
-    await writeText(join(dir, 'sitecustomize.py'), SITECUSTOMIZE_SOURCE);
+    await writeText(join(dir, 'sitecustomize.py'), GUARD_PYTHON_SOURCE);
     return dir;
 }
 /**
  * Builds the `env` overlay (merged over `process.env` by the exec layer)
- * that prepends the shim directory to `PYTHONPATH`, so the mach subprocess
- * imports the degrade `sitecustomize` without clobbering an existing
- * `PYTHONPATH`.
+ * that prepends the fallback shim directory to `PYTHONPATH` without
+ * clobbering an existing `PYTHONPATH`.
  */
 function machResourceShimEnv(shimDir, baseEnv = process.env) {
     const existing = baseEnv['PYTHONPATH'];
     return { PYTHONPATH: existing ? `${shimDir}${delimiter}${existing}` : shimDir };
 }
-/** Ensures the shim exists and returns the mach `env` overlay that enables it. */
-export async function resolveMachBuildEnv() {
-    return machResourceShimEnv(await ensureMachResourceShim());
+/**
+ * Installs the resource-monitor degrade guard for a mach dispatch:
+ * `fireforge_mach_guard.pth` + `fireforge_mach_guard.py` into every
+ * discovered mach virtualenv site-packages (survives mach's venv re-exec),
+ * plus the PYTHONPATH sitecustomize fallback for the pre-venv bootstrap
+ * phase. Idempotent and re-run before EVERY protected dispatch (including
+ * each retry) so a venv created by a crashed first attempt is guarded on
+ * the next one instead of re-dying on the same wedged state.
+ */
+export async function installMachResourceGuard(engineDir) {
+    const sitePackagesDirs = await discoverMachSitePackages(engineDir);
+    for (const sitePackages of sitePackagesDirs) {
+        try {
+            await writeText(join(sitePackages, `${GUARD_MODULE_NAME}.py`), GUARD_PYTHON_SOURCE);
+            await writeText(join(sitePackages, `${GUARD_MODULE_NAME}.pth`), `import ${GUARD_MODULE_NAME}\n`);
+        }
+        catch (error) {
+            // A read-only or vanished venv must not block the dispatch; the
+            // PYTHONPATH fallback still applies and other venvs may have taken
+            // the guard.
+            verbose(`Could not install mach resource guard into ${sitePackages}: ${toError(error).message}`);
+        }
+    }
+    const env = machResourceShimEnv(await ensureSitecustomizeFallback());
+    return { env, sitePackagesDirs };
 }
 //# sourceMappingURL=mach-resource-shim.js.map

package/dist/src/core/mach.d.ts

@@ -1,4 +1,5 @@
 import { type SmokeLineCallback, type SmokeRunResult } from '../utils/process.js';
+import { type HarnessCrashSignature } from './test-harness-crash.js';
 export { attemptMozinfoRewrite, buildArtifactMismatchMessage, hasBuildArtifacts, hasRunnableBundle, } from './mach-build-artifacts.js';
 export { generateMozconfig } from './mach-mozconfig.js';
 export { ensurePython, resetResolvedPython } from './mach-python.js';
@@ -57,12 +58,52 @@ export declare function bootstrap(engineDir: string): Promise<number>;
  * @returns Captured output and exit code
  */
 export declare function bootstrapWithOutput(engineDir: string): Promise<MachCommandResult>;
+/** Which mach build entry a protected dispatch runs. */
+export type ProtectedBuildKind = 'full' | 'faster';
+/** Options for {@link runProtectedMachBuild}. */
+export interface ProtectedMachBuildOptions {
+    /** Parallel jobs for the full build (ignored for `faster`). */
+    jobs?: number | undefined;
+    /** Recognized-crash retry budget (0 disables retries). */
+    retries?: number | undefined;
+    /** Called before each retry with the detected crash signature. */
+    onRetry?: (signature: HarnessCrashSignature, nextAttempt: number, maxAttempts: number) => void;
+}
+/** Captured result of a protected (guarded, retried) mach build dispatch. */
+export interface ProtectedMachBuildResult extends MachCommandResult {
+    /** How many mach processes were spawned (1 = no retry needed). */
+    attempts: number;
+    /** Crash signature of the final failed attempt, when recognized. */
+    crashSignature?: HarnessCrashSignature;
+}
+/**
+ * The single protected path every FireForge mach build dispatch routes
+ * through (field report: `build --ui` and the pre-test `--build` step died
+ * on the resource-monitor crash family while plain `build` survived,
+ * because only some entries were protected and retry budgets differed):
+ *
+ *  1. installs the resource-monitor degrade guard IN the mach virtualenvs
+ *     (plus the PYTHONPATH fallback) — re-installed before every attempt,
+ *     so a venv materialized by a crashed first attempt is guarded on the
+ *     next one instead of every retry dying on the same wedged state;
+ *  2. spawns a fresh mach process per attempt;
+ *  3. retries ONLY the recognized harness-crash family (resource monitor /
+ *     psutil startup tracebacks) up to the uniform budget — an ordinary
+ *     compile error is never retried.
+ *
+ * The protected path never runs `mach configure`, never clobbers, and
+ * never widens the requested build kind — a `faster` dispatch retries as
+ * `mach build faster`, so it cannot invalidate more of the objdir than the
+ * command the operator asked for.
+ */
+export declare function runProtectedMachBuild(kind: ProtectedBuildKind, engineDir: string, options?: ProtectedMachBuildOptions): Promise<ProtectedMachBuildResult>;
 /**
- * Runs a full mach build. On a non-zero exit, any matched error hints are
- * surfaced on top of the raw mach output so operators get an actionable
- * nudge alongside the cryptic mozbuild traceback. Returns the captured
- * result so the caller (e.g. `fireforge build`) can inspect the tail
- * for post-build diagnostics that mach prints AFTER "Your build was
+ * Runs a full mach build through the protected dispatch path (resource
+ * guard + recognized-crash retries). On a non-zero exit, any matched error
+ * hints are surfaced on top of the raw mach output so operators get an
+ * actionable nudge alongside the cryptic mozbuild traceback. Returns the
+ * captured result so the caller (e.g. `fireforge build`) can inspect the
+ * tail for post-build diagnostics that mach prints AFTER "Your build was
  * successful!" — notably the stale `config.status is out of date`
  * notice that mach emits when a tool-managed edit landed on
  * `moz.configure` before the build.
@@ -70,15 +111,15 @@ export declare function bootstrapWithOutput(engineDir: string): Promise<MachComm
  * @param jobs - Number of parallel jobs (optional)
  * @returns Captured mach result (stdout tail, stderr tail, exit code)
  */
-export declare function build(engineDir: string, jobs?: number): Promise<MachCommandResult>;
+export declare function build(engineDir: string, jobs?: number): Promise<ProtectedMachBuildResult>;
 /**
- * Runs a fast UI-only build. On a non-zero exit, any matched error hints are
- * surfaced on top of the raw mach output. See {@link build} for why the
- * full captured result is returned rather than just the exit code.
+ * Runs a fast UI-only build through the same protected dispatch path as
+ * {@link build}. See {@link build} for why the full captured result is
+ * returned rather than just the exit code.
  * @param engineDir - Path to the engine directory
  * @returns Captured mach result
  */
-export declare function buildUI(engineDir: string): Promise<MachCommandResult>;
+export declare function buildUI(engineDir: string): Promise<ProtectedMachBuildResult>;
 /**
  * Runs an operation while holding a sidecar build lock keyed on the
  * project root. Concurrent `fireforge build` / `fireforge build --ui`

package/dist/src/core/mach.js

@@ -8,7 +8,8 @@ import { createSiblingLockPath, withFileLock } from './file-lock.js';
 import { ensureFirefoxIgnorefileCompatibility } from './firefox-ignorefile.js';
 import { explainMachError } from './mach-error-hints.js';
 import { getPython } from './mach-python.js';
-import { resolveMachBuildEnv } from './mach-resource-shim.js';
+import { installMachResourceGuard } from './mach-resource-shim.js';
+import { detectHarnessCrashSignature } from './test-harness-crash.js';
 // Re-export sub-modules so existing `from './mach.js'` imports keep working.
 export { attemptMozinfoRewrite, buildArtifactMismatchMessage, hasBuildArtifacts, hasRunnableBundle, } from './mach-build-artifacts.js';
 export { generateMozconfig } from './mach-mozconfig.js';
@@ -140,11 +141,63 @@ function surfaceMachErrorHints(result) {
     }
 }
 /**
- * Runs a full mach build. On a non-zero exit, any matched error hints are
- * surfaced on top of the raw mach output so operators get an actionable
- * nudge alongside the cryptic mozbuild traceback. Returns the captured
- * result so the caller (e.g. `fireforge build`) can inspect the tail
- * for post-build diagnostics that mach prints AFTER "Your build was
+ * Uniform recognized-crash retry budget for the protected mach build
+ * dispatches (`fireforge build`, `build --ui`, and the pre-test `--build`
+ * step). Matches the test harness default so every mach dispatch retries
+ * the same crash family the same number of times.
+ */
+const DEFAULT_BUILD_CRASH_RETRIES = 2;
+/**
+ * The single protected path every FireForge mach build dispatch routes
+ * through (field report: `build --ui` and the pre-test `--build` step died
+ * on the resource-monitor crash family while plain `build` survived,
+ * because only some entries were protected and retry budgets differed):
+ *
+ *  1. installs the resource-monitor degrade guard IN the mach virtualenvs
+ *     (plus the PYTHONPATH fallback) — re-installed before every attempt,
+ *     so a venv materialized by a crashed first attempt is guarded on the
+ *     next one instead of every retry dying on the same wedged state;
+ *  2. spawns a fresh mach process per attempt;
+ *  3. retries ONLY the recognized harness-crash family (resource monitor /
+ *     psutil startup tracebacks) up to the uniform budget — an ordinary
+ *     compile error is never retried.
+ *
+ * The protected path never runs `mach configure`, never clobbers, and
+ * never widens the requested build kind — a `faster` dispatch retries as
+ * `mach build faster`, so it cannot invalidate more of the objdir than the
+ * command the operator asked for.
+ */
+export async function runProtectedMachBuild(kind, engineDir, options = {}) {
+    const args = kind === 'faster' ? ['build', 'faster'] : ['build'];
+    if (kind === 'full' && options.jobs !== undefined) {
+        args.push('-j', String(options.jobs));
+    }
+    const maxAttempts = Math.max(1, (options.retries ?? DEFAULT_BUILD_CRASH_RETRIES) + 1);
+    for (let attempt = 1;; attempt += 1) {
+        const { env } = await installMachResourceGuard(engineDir);
+        const result = await runMachInheritCapture(args, engineDir, { env });
+        if (result.exitCode === 0) {
+            return { ...result, attempts: attempt };
+        }
+        const signature = detectHarnessCrashSignature(`${result.stdout}\n${result.stderr}`);
+        if (signature && attempt < maxAttempts) {
+            options.onRetry?.(signature, attempt + 1, maxAttempts);
+            warn(`mach ${kind === 'faster' ? 'build faster' : 'build'} hit a recognized harness crash ` +
+                `(${signature.reason}): ${signature.line}\n` +
+                `Retrying with a fresh process (attempt ${attempt + 1} of ${maxAttempts})...`);
+            continue;
+        }
+        surfaceMachErrorHints(result);
+        return { ...result, attempts: attempt, ...(signature ? { crashSignature: signature } : {}) };
+    }
+}
+/**
+ * Runs a full mach build through the protected dispatch path (resource
+ * guard + recognized-crash retries). On a non-zero exit, any matched error
+ * hints are surfaced on top of the raw mach output so operators get an
+ * actionable nudge alongside the cryptic mozbuild traceback. Returns the
+ * captured result so the caller (e.g. `fireforge build`) can inspect the
+ * tail for post-build diagnostics that mach prints AFTER "Your build was
  * successful!" — notably the stale `config.status is out of date`
  * notice that mach emits when a tool-managed edit landed on
  * `moz.configure` before the build.
@@ -153,35 +206,17 @@ function surfaceMachErrorHints(result) {
  * @returns Captured mach result (stdout tail, stderr tail, exit code)
  */
 export async function build(engineDir, jobs) {
-    const args = ['build'];
-    if (jobs !== undefined) {
-        args.push('-j', String(jobs));
-    }
-    // Inject the resource-monitor degrade shim so a host psutil failure does
-    // not abort the build regardless of which mach entry FireForge spawns.
-    const env = await resolveMachBuildEnv();
-    const result = await runMachInheritCapture(args, engineDir, { env });
-    if (result.exitCode !== 0) {
-        surfaceMachErrorHints(result);
-    }
-    return result;
+    return runProtectedMachBuild('full', engineDir, { jobs });
 }
 /**
- * Runs a fast UI-only build. On a non-zero exit, any matched error hints are
- * surfaced on top of the raw mach output. See {@link build} for why the
- * full captured result is returned rather than just the exit code.
+ * Runs a fast UI-only build through the same protected dispatch path as
+ * {@link build}. See {@link build} for why the full captured result is
+ * returned rather than just the exit code.
  * @param engineDir - Path to the engine directory
  * @returns Captured mach result
  */
 export async function buildUI(engineDir) {
-    // Same resource-monitor degrade shim as the full build: raw
-    // `./mach build faster` aborts on the host psutil monitor without it.
-    const env = await resolveMachBuildEnv();
-    const result = await runMachInheritCapture(['build', 'faster'], engineDir, { env });
-    if (result.exitCode !== 0) {
-        surfaceMachErrorHints(result);
-    }
-    return result;
+    return runProtectedMachBuild('faster', engineDir);
 }
 /**
  * Runs an operation while holding a sidecar build lock keyed on the
@@ -325,7 +360,10 @@ export async function test(engineDir, testPaths = [], args = []) {
  *   `fireforge test --perf-samples` to publish the artifact-path contract.
  */
 export async function testWithOutput(engineDir, testPaths = [], args = [], env) {
-    return runMachCapture(['test', ...testPaths, ...args], engineDir, env ? { env } : {});
+    const guard = await installMachResourceGuard(engineDir);
+    return runMachCapture(['test', ...testPaths, ...args], engineDir, {
+        env: { ...guard.env, ...env },
+    });
 }
 /**
  * Runs `mach xpcshell-test` (the suite-specific xpcshell command) while
@@ -338,7 +376,10 @@ export async function testWithOutput(engineDir, testPaths = [], args = [], env)
  * the dispatch path.
  */
 export async function xpcshellTestWithOutput(engineDir, testPaths = [], args = [], env) {
-    return runMachCapture(['xpcshell-test', ...testPaths, ...args], engineDir, env ? { env } : {});
+    const guard = await installMachResourceGuard(engineDir);
+    return runMachCapture(['xpcshell-test', ...testPaths, ...args], engineDir, {
+        env: { ...guard.env, ...env },
+    });
 }
 /**
  * Runs `mach mochitest` (covers browser-chrome / mochitest flavors) while
@@ -346,6 +387,9 @@ export async function xpcshellTestWithOutput(engineDir, testPaths = [], args = [
  * for non-xpcshell single-suite runs — see {@link xpcshellTestWithOutput}.
  */
 export async function mochitestWithOutput(engineDir, testPaths = [], args = [], env) {
-    return runMachCapture(['mochitest', ...testPaths, ...args], engineDir, env ? { env } : {});
+    const guard = await installMachResourceGuard(engineDir);
+    return runMachCapture(['mochitest', ...testPaths, ...args], engineDir, {
+        env: { ...guard.env, ...env },
+    });
 }
 //# sourceMappingURL=mach.js.map

package/dist/src/core/manifest-helpers.d.ts

@@ -1,4 +1,17 @@
 import type { JarMnToken, MozBuildToken } from './manifest-tokenizers.js';
+/**
+ * Ordering comparator matching mozbuild's `StrictOrderingOnAppendList`
+ * (`mozbuild.util.UnsortedError`): entries are compared **case-insensitively**,
+ * so `HominisAppearanceController` (`appe`) sorts before
+ * `HominisAppMenuIntegration` (`appm`) even though a raw byte comparison
+ * places the uppercase `M` (0x4D) before the lowercase `e` (0x65).
+ *
+ * A naive `a > b` insertion landed the new entry in byte order, which
+ * `mach configure` then rejected with `UnsortedError`, aborting the build.
+ * Ties on the lower-cased key fall back to byte order so the comparison is
+ * total and stable.
+ */
+export declare function mozbuildSortCompare(a: string, b: string): number;
 /**
  * Inserts a line into an array of lines in alphabetical order within a
  * specified range. The comparison key is extracted from each line.