@hominis/fireforge 0.54.0 → 0.60.0

package/dist/src/engine/resourceGuard.js

@@ -0,0 +1,582 @@
+// SPDX-License-Identifier: EUPL-1.2
+/**
+ * The resource-monitor guard for `mach`.
+ *
+ * mozbuild's resource monitor calls `psutil.virtual_memory()` at startup and
+ * again from `mozbuild.base._run_make`. On some hosts those raise
+ * `RuntimeError: host_statistics64(HOST_VM_INFO64) syscall failed`, and a
+ * `SystemResourceMonitor.__init__` that fails partway leaves an instance
+ * without `poll_interval`, so later polling dies with an `AttributeError`.
+ * Either aborts `mach build` before the compiler runs.
+ *
+ * The guard is a `fireforge_mach_guard.pth` and module pair in the
+ * site-packages of every mach virtualenv. A `sitecustomize.py` on PYTHONPATH
+ * is not enough: mach re-executes itself inside its private virtualenv and
+ * drops PYTHONPATH. A `.pth` file runs at interpreter startup from the
+ * virtualenv's own site-packages, which survives that. The PYTHONPATH copy
+ * covers only the first run, before any virtualenv exists.
+ *
+ * What the Python guards, and why each shape matters:
+ *
+ *  - `psutil.virtual_memory`, `swap_memory`, `cpu_percent`, `cpu_times` and
+ *    `disk_io_counters` are wrapped module-wide, so every call site degrades
+ *    to a warning and a zeroed reading. The fallback has the arity of that
+ *    function's own result: mozsystemmonitor captures each reading's type at
+ *    construction and rebuilds collector samples with `type(reading)(*values)`,
+ *    so a memory-shaped reading in the swap position makes the parent reject
+ *    every sample, the pipe fills, the collector child blocks in `send()`,
+ *    and mach hangs in its exit-time `waitpid`. psutil's own result
+ *    namedtuple is used where it resolves, otherwise a module-level one of
+ *    the guard's (module-level so readings pickle across the collector pipe).
+ *  - `SystemResourceMonitor` construction is guarded through an import hook:
+ *    `poll_interval` is set before the real `__init__`, a failing `__init__`
+ *    marks the instance degraded, and every monitor method is inert on a
+ *    degraded instance. Aggregate methods return zeroed shapes rather than
+ *    `None` where callers subscript the result.
+ *  - Once any degradation was seen in the process, monitors stay inert and a
+ *    live collector child is terminated and its pipe drained, so a malformed
+ *    sample stream cannot wedge mach's shutdown.
+ *  - `BuildMonitor.log_resource_usage` warns and continues on any exception,
+ *    so end-of-build reporting cannot fail a build whose artifacts are complete.
+ */
+import { homedir, tmpdir, userInfo } from 'node:os';
+import { basename, delimiter, join, resolve } from 'node:path';
+import { ExitCode, FireForgeError, toError } from '../kernel/errors.js';
+import { ensureDir, exists, listDir, pathInfo, realPath, setMode, writeFileIfChanged, } from '../kernel/fs.js';
+import { hash } from '../kernel/hash.js';
+import { verbose } from '../kernel/log.js';
+/** Module name of the guard inside a virtualenv's site-packages. */
+export const GUARD_MODULE_NAME = 'fireforge_mach_guard';
+/**
+ * The Python guard, importable both as the in-virtualenv module the `.pth`
+ * loads and as a PYTHONPATH `sitecustomize`. Defensive throughout: a missing
+ * or broken psutil or mozsystemmonitor leaves mach untouched, and every
+ * wrapper intercepts only the failure path.
+ */
+export const GUARD_PYTHON_SOURCE = `# Generated by FireForge - do not edit.
+# Degrades the broken host resource monitor family (psutil vs Darwin) from
+# fatal startup errors into non-fatal warnings, so mach builds and tests
+# proceed. Installed into the mach virtualenv site-packages via a .pth file
+# (survives mach's venv re-exec, which drops PYTHONPATH).
+import builtins
+import collections
+import inspect
+import sys
+import warnings
+
+# Any psutil degradation observed in this process. Once set, monitors are
+# kept inert (start never spawns the collector child) — on a flapping host a
+# healthy-looking moment must not be trusted to start a sample stream that a
+# later degraded reading could malform.
+_fireforge_host_degraded = [False]
+
+
+def _fireforge_degraded_notice(exc):
+    _fireforge_host_degraded[0] = True
+    warnings.warn(
+        "FireForge: host resource monitor degraded (%s); continuing without resource monitoring." % exc
+    )
+
+
+class _DegradedReading(object):
+    # Last-resort duck type of psutil's namedtuple results (svmem field
+    # order): mozsystemmonitor subscripts, iterates, and unpacks readings,
+    # so the degraded fallback must survive r[0], list(r), len(r), r._fields
+    # and r._asdict(), not just attribute access. The parent also reconstructs
+    # readings via type(reading)(*values), so the constructor must tolerate
+    # the full positional field list. Normal
+    # degraded paths use the per-function namedtuple fallbacks below; this
+    # class only backstops a fallback that itself raised.
+    _fields = ("total", "available", "percent", "used", "free", "active", "inactive", "wired")
+    total = available = used = free = active = inactive = wired = 0
+    percent = 0.0
+
+    def __init__(self, *_args, **_kwargs):
+        pass
+
+    def __getattr__(self, _name):
+        return 0
+
+    def _values(self):
+        return tuple(getattr(self, _field) for _field in self._fields)
+
+    def __getitem__(self, index):
+        return self._values()[index]
+
+    def __iter__(self):
+        return iter(self._values())
+
+    def __len__(self):
+        return len(self._fields)
+
+    def _asdict(self):
+        return dict(zip(self._fields, self._values()))
+
+
+# Per-function fallback result classes with the exact macOS field
+# orders/arities. Module-level on purpose: readings cross the
+# mozsystemmonitor collector pipe via pickle (by reference), and the parent
+# rebuilds each sample with type(reading)(*values) — so a swap fallback must
+# be 6-field sswap-shaped in every position. An svmem-shaped fallback there
+# makes the parent reject every sample, fill the pipe, block the collector
+# child in send(), and wedge mach's atexit join forever.
+_FallbackVmem = collections.namedtuple(
+    "_FallbackVmem",
+    ("total", "available", "percent", "used", "free", "active", "inactive", "wired"),
+)
+_FallbackSwap = collections.namedtuple(
+    "_FallbackSwap", ("total", "used", "free", "percent", "sin", "sout")
+)
+_FallbackCpuTimes = collections.namedtuple("_FallbackCpuTimes", ("user", "nice", "system", "idle"))
+_FallbackDiskIO = collections.namedtuple(
+    "_FallbackDiskIO",
+    ("read_count", "write_count", "read_bytes", "write_bytes", "read_time", "write_time"),
+)
+
+_FALLBACK_CLASSES = {
+    "virtual_memory": _FallbackVmem,
+    "swap_memory": _FallbackSwap,
+    "cpu_times": _FallbackCpuTimes,
+    "disk_io_counters": _FallbackDiskIO,
+}
+
+# psutil result namedtuple per wrapped function, resolvable via getattr from
+# the platform module / _common without invoking the failing syscall.
+_PSUTIL_RESULT_CLASSES = {
+    "virtual_memory": "svmem",
+    "swap_memory": "sswap",
+    "cpu_times": "scputimes",
+    "disk_io_counters": "sdiskio",
+}
+
+
+def _zeroed(cls):
+    return cls(*([0] * len(cls._fields)))
+
+
+def _percpu_requested(args, kwargs, positional_index):
+    if "percpu" in kwargs:
+        return bool(kwargs["percpu"])
+    return len(args) > positional_index and bool(args[positional_index])
+
+
+def _degraded_result_factory(psutil, name):
+    # cpu_percent returns a plain float (or a per-CPU list) callers do
+    # arithmetic on.
+    if name == "cpu_percent":
+        def _cpu_percent_fallback(*args, **kwargs):
+            # cpu_percent(interval=None, percpu=False)
+            if _percpu_requested(args, kwargs, 1):
+                return []
+            return 0.0
+
+        return _cpu_percent_fallback
+    # The guard-owned class is always shape-correct for this function;
+    # psutil's own result namedtuple is still preferred when resolvable so
+    # type() captures stay identical to real readings on a flapping host.
+    _cls = _FALLBACK_CLASSES.get(name, _DegradedReading)
+    _cls_name = _PSUTIL_RESULT_CLASSES.get(name)
+    if _cls_name is not None:
+        for _mod in (getattr(psutil, "_psplatform", None), getattr(psutil, "_common", None)):
+            _real = getattr(_mod, _cls_name, None) if _mod is not None else None
+            if _real is not None and getattr(_real, "_fields", None):
+                _cls = _real
+                break
+    if name == "cpu_times":
+        def _cpu_times_fallback(*args, **kwargs):
+            # cpu_times(percpu=False); the collector child samples per-CPU.
+            if _percpu_requested(args, kwargs, 0):
+                return []
+            return _zeroed(_cls)
+
+        return _cpu_times_fallback
+
+    def _reading_fallback(*_args, **_kwargs):
+        return _zeroed(_cls)
+
+    return _reading_fallback
+
+
+def _guard_psutil():
+    try:
+        import psutil
+    except Exception:
+        return
+
+    def _wrap(orig, fallback):
+        def wrapper(*args, **kwargs):
+            try:
+                return orig(*args, **kwargs)
+            except Exception as exc:  # noqa: BLE001
+                _fireforge_degraded_notice(exc)
+                try:
+                    return fallback(*args, **kwargs)
+                except Exception:  # noqa: BLE001
+                    return _DegradedReading()
+
+        return wrapper
+
+    for _name in ("virtual_memory", "swap_memory", "cpu_percent", "cpu_times", "disk_io_counters"):
+        _orig = getattr(psutil, _name, None)
+        if _orig is not None and not getattr(_orig, "_fireforge_guarded", False):
+            try:
+                _fallback = _degraded_result_factory(psutil, _name)
+            except Exception:  # noqa: BLE001
+                _fallback = _DegradedReading
+            _wrapped = _wrap(_orig, _fallback)
+            _wrapped._fireforge_guarded = True
+            setattr(psutil, _name, _wrapped)
+
+
+_MONITOR_METHOD_NAMES = (
+    "start",
+    "stop",
+    "record_event",
+    "record_marker",
+    "begin_phase",
+    "finish_phase",
+    "aggregate_cpu_percent",
+    "aggregate_cpu_times",
+    "aggregate_io",
+    "min_memory_available",
+    "as_dict",
+)
+
+
+def _fireforge_stop_collector(monitor):
+    # Best-effort: terminate a live collector child and drain the parent end
+    # of the pipe, so a child blocked in send() on a full pipe can never
+    # keep mach's atexit join (os.waitpid) waiting forever. Attribute names
+    # follow upstream resourcemonitor.py (self._process / self._pipe); a
+    # miss on a refactored upstream is harmless — the shape-correct
+    # fallbacks above still keep the sample stream well-formed.
+    try:
+        proc = getattr(monitor, "_process", None)
+        if proc is not None and getattr(proc, "is_alive", lambda: False)():
+            proc.terminate()
+            proc.join(1)
+    except Exception:  # noqa: BLE001
+        pass
+    try:
+        pipe = getattr(monitor, "_pipe", None)
+        if pipe is not None:
+            while pipe.poll(0):
+                pipe.recv()
+    except Exception:  # noqa: BLE001
+        pass
+
+
+def _monitor_per_cpu_requested(args, kwargs):
+    # aggregate_cpu_percent/aggregate_cpu_times(start=None, end=None,
+    # phase=None, per_cpu=True) — self already stripped from args.
+    if "per_cpu" in kwargs:
+        return bool(kwargs["per_cpu"])
+    if len(args) > 3:
+        return bool(args[3])
+    return True
+
+
+def _monitor_degraded_result(name, args, kwargs):
+    # Degraded aggregate methods return zeroed shapes, not None, where
+    # callers subscript the result: mozbuild's log_resource_usage does
+    # usage["io"].read_bytes after a successful compile, so None there fails
+    # the build with complete artifacts already on disk.
+    if name == "aggregate_io":
+        return _zeroed(_FallbackDiskIO)
+    if name == "aggregate_cpu_percent":
+        return [] if _monitor_per_cpu_requested(args, kwargs) else 0.0
+    if name == "aggregate_cpu_times":
+        return [] if _monitor_per_cpu_requested(args, kwargs) else _zeroed(_FallbackCpuTimes)
+    return None
+
+
+def _patch_monitor_class(cls):
+    if cls is None or getattr(cls, "_fireforge_guarded", False):
+        return
+    orig_init = cls.__init__
+
+    def guarded_init(self, *args, **kwargs):
+        # Pre-populate the attributes a partially-constructed instance is
+        # later polled for, so a failing __init__ can never surface
+        # AttributeError: poll_interval.
+        self.poll_interval = kwargs.get("poll_interval") or 1.0
+        self._fireforge_degraded = False
+        try:
+            orig_init(self, *args, **kwargs)
+        except Exception as exc:  # noqa: BLE001
+            self._fireforge_degraded = True
+            _fireforge_degraded_notice(exc)
+        if _fireforge_host_degraded[0] and not self._fireforge_degraded:
+            # Host already degraded once: keep the monitor inert so the
+            # collector child never spawns on a flapping host.
+            self._fireforge_degraded = True
+        if self._fireforge_degraded:
+            _fireforge_stop_collector(self)
+
+    cls.__init__ = guarded_init
+
+    def _wrap_method(name, orig):
+        def wrapper(self, *args, **kwargs):
+            if (
+                name == "start"
+                and _fireforge_host_degraded[0]
+                and not getattr(self, "_fireforge_degraded", False)
+            ):
+                self._fireforge_degraded = True
+                _fireforge_stop_collector(self)
+            if getattr(self, "_fireforge_degraded", False):
+                if name == "stop":
+                    _fireforge_stop_collector(self)
+                return _monitor_degraded_result(name, args, kwargs)
+            try:
+                return orig(self, *args, **kwargs)
+            except Exception as exc:  # noqa: BLE001
+                self._fireforge_degraded = True
+                _fireforge_degraded_notice(exc)
+                # A raising stop (or any degraded transition) must still
+                # take the collector child down, or the parent's atexit
+                # join hangs on a child blocked writing to a full pipe.
+                _fireforge_stop_collector(self)
+                return _monitor_degraded_result(name, args, kwargs)
+
+        return wrapper
+
+    def _wrap_unbound(name, func):
+        # Upstream declares record_event/record_marker as @staticmethods (and
+        # may declare others as @classmethods). Those take no receiver, so
+        # they cannot use the per-instance degraded flag and must not have one
+        # threaded in — see the descriptor dispatch below.
+        def wrapper(*args, **kwargs):
+            if _fireforge_host_degraded[0]:
+                return _monitor_degraded_result(name, args, kwargs)
+            try:
+                return func(*args, **kwargs)
+            except Exception as exc:  # noqa: BLE001
+                _fireforge_degraded_notice(exc)
+                return _monitor_degraded_result(name, args, kwargs)
+
+        return wrapper
+
+    for _name in _MONITOR_METHOD_NAMES:
+        # getattr_static returns the raw class attribute, so a staticmethod
+        # is still a staticmethod object here. Plain getattr would hand back
+        # the underlying function, and re-setting THAT on the class rebinds
+        # it as an instance method: the receiver is then passed through as an
+        # extra leading positional and every call raises
+        # "takes N positional arguments but N+1 were given". Re-apply the
+        # descriptor the attribute actually had.
+        try:
+            _raw = inspect.getattr_static(cls, _name)
+        except AttributeError:
+            continue
+        if isinstance(_raw, staticmethod):
+            setattr(cls, _name, staticmethod(_wrap_unbound(_name, _raw.__func__)))
+            continue
+        if isinstance(_raw, classmethod):
+            setattr(cls, _name, classmethod(_wrap_unbound(_name, _raw.__func__)))
+            continue
+        _orig = getattr(cls, _name, None)
+        if _orig is not None:
+            setattr(cls, _name, _wrap_method(_name, _orig))
+    cls._fireforge_guarded = True
+
+
+def _patch_build_monitor_class(cls):
+    # mozbuild's BuildMonitor is not a SystemResourceMonitor; only
+    # log_resource_usage needs guarding — end-of-build resource reporting
+    # must never fail a build whose artifacts are complete. No __init__
+    # replacement, no degraded-instance gating.
+    if cls is None or getattr(cls, "_fireforge_guarded", False):
+        return
+    _orig = getattr(cls, "log_resource_usage", None)
+    if _orig is not None:
+        def _guarded_log_resource_usage(self, *args, **kwargs):
+            try:
+                return _orig(self, *args, **kwargs)
+            except Exception as exc:  # noqa: BLE001
+                _fireforge_degraded_notice(exc)
+                return None
+
+        cls.log_resource_usage = _guarded_log_resource_usage
+    cls._fireforge_guarded = True
+
+
+_MONITOR_CLASS_PATCHES = (
+    ("mozsystemmonitor.resourcemonitor", "SystemResourceMonitor", _patch_monitor_class),
+    ("mozbuild.resources", "SystemResourceMonitor", _patch_monitor_class),
+    ("mozbuild.controller.building", "BuildMonitor", _patch_build_monitor_class),
+)
+
+
+def _patch_loaded_monitor_modules():
+    for _mod_name, _cls_name, _patcher in _MONITOR_CLASS_PATCHES:
+        _mod = sys.modules.get(_mod_name)
+        if _mod is not None:
+            try:
+                _patcher(getattr(_mod, _cls_name, None))
+            except Exception:  # noqa: BLE001
+                pass
+
+
+def _install_import_hook():
+    _orig_import = builtins.__import__
+    if getattr(_orig_import, "_fireforge_guarded", False):
+        return
+
+    def _guarding_import(name, *args, **kwargs):
+        _module = _orig_import(name, *args, **kwargs)
+        try:
+            _patch_loaded_monitor_modules()
+        except Exception:  # noqa: BLE001
+            pass
+        return _module
+
+    _guarding_import._fireforge_guarded = True
+    builtins.__import__ = _guarding_import
+
+
+try:
+    _guard_psutil()
+    _patch_loaded_monitor_modules()
+    _install_import_hook()
+except Exception:  # noqa: BLE001
+    pass
+`;
+async function subdirectories(directory) {
+    try {
+        return (await listDir(directory))
+            .filter((entry) => entry.kind === 'directory')
+            .map((entry) => entry.name);
+    }
+    catch {
+        // An unreadable directory holds no virtualenv this process could guard.
+        return [];
+    }
+}
+/** The site-packages of the virtualenvs under one `_virtualenvs` directory. */
+async function sitePackagesUnder(virtualenvs) {
+    const found = [];
+    for (const venv of await subdirectories(virtualenvs)) {
+        const lib = join(virtualenvs, venv, 'lib');
+        for (const python of await subdirectories(lib)) {
+            const sitePackages = join(lib, python, 'site-packages');
+            if (python.startsWith('python') && (await exists(sitePackages)))
+                found.push(sitePackages);
+        }
+    }
+    return found;
+}
+/**
+ * Refuses a directory that is not a private directory of the current user.
+ * A symlink is not followed: one planted at the path would make the check
+ * describe its target. Ownership is proven by re-applying the directory's
+ * own permission bits, which only its owner may do.
+ *
+ * @param directory - The directory mach will import `sitecustomize.py` from
+ * @throws `resource-guard-unsafe` (exit code 5) when it is missing, a symlink
+ *   or file, another user's, or writable by group or others
+ */
+export async function assertPrivateDirectory(directory) {
+    const refuse = (reason) => new FireForgeError('resource-guard-unsafe', `Refusing to use the mach guard directory ${directory}: ${reason}`, {
+        exitCode: ExitCode.BUILD_ERROR,
+        hint: 'Another local account may have planted it. Remove it and retry.',
+    });
+    const info = await pathInfo(directory);
+    if (info === undefined)
+        throw refuse('it does not exist');
+    if (info.kind !== 'directory')
+        throw refuse('it is not a directory (symlink or file)');
+    if (process.platform === 'win32')
+        return;
+    if ((info.mode & 0o022) !== 0) {
+        throw refuse(`mode ${(info.mode & 0o777).toString(8)} is group/world-writable`);
+    }
+    try {
+        await setMode(directory, info.mode);
+    }
+    catch (error) {
+        throw refuse(`it belongs to another user (${toError(error).message})`);
+    }
+}
+/** The numeric uid, or on Windows the user name, that keeps one user's directory apart from another's. */
+function directoryOwner(env) {
+    const uid = process.getuid?.();
+    if (uid !== undefined)
+        return String(uid);
+    let name;
+    try {
+        name = userInfo().username;
+    }
+    catch {
+        // A service account can have no user record.
+        name = env['USERNAME'] ?? env['USER'];
+    }
+    return (name ?? 'unknown-user').replace(/[^A-Za-z0-9_.-]/g, '_');
+}
+/** Writes the PYTHONPATH `sitecustomize.py` into a per-user private directory and returns it. */
+async function writeFallback(env) {
+    const directory = join(tmpdir(), `fireforge-mach-guard-${directoryOwner(env)}`);
+    if (!(await exists(directory))) {
+        await ensureDir(directory);
+        // Whoever created the directory in the meantime owns it, and this then fails.
+        if (process.platform !== 'win32')
+            await setMode(directory, 0o700).catch(() => undefined);
+    }
+    await assertPrivateDirectory(directory);
+    await writeFileIfChanged(join(directory, 'sitecustomize.py'), GUARD_PYTHON_SOURCE, {
+        durable: false,
+    });
+    return directory;
+}
+/**
+ * The names mach gives this engine's directory under its state directory's
+ * `srcdirs/`: the source directory's name and the start of the SHA-256 of
+ * its path, for either spelling of a path behind a symlink. Other source
+ * trees on the host keep their own virtualenvs, which are left alone.
+ */
+async function srcdirStateNames(engineDir) {
+    const spellings = new Set([resolve(engineDir), await realPath(engineDir).catch(() => engineDir)]);
+    return [...spellings].map((path) => `${basename(path)}-${hash(path).slice(0, 12)}`);
+}
+/**
+ * Installs the guard for the next `mach` run in an engine: into the
+ * site-packages of every virtualenv mach keeps for that engine (under each
+ * object directory, and under the engine's own entry in the mach state
+ * directory, `MOZBUILD_STATE_PATH` or `~/.mozbuild`), and as a PYTHONPATH fallback for a run that has no
+ * virtualenv yet. Idempotent; call it before every run, since a run can
+ * create a virtualenv the next one needs guarded.
+ *
+ * @param engineDir - The engine
+ * @param options - See {@link ResourceGuardOptions}
+ * @returns The environment to merge into mach's
+ * @throws `resource-guard-unsafe` (exit code 5) when the fallback directory is not private
+ */
+export async function installResourceGuard(engineDir, options = {}) {
+    const env = options.env ?? process.env;
+    const objDirs = (await subdirectories(engineDir))
+        .filter((name) => name.startsWith('obj-'))
+        .map((name) => join(engineDir, name));
+    const virtualenvs = [...objDirs, ...(options.objDirs ?? [])].map((objDir) => join(objDir, '_virtualenvs'));
+    const sources = join(env['MOZBUILD_STATE_PATH'] ?? join(homedir(), '.mozbuild'), 'srcdirs');
+    for (const name of await srcdirStateNames(engineDir)) {
+        virtualenvs.push(join(sources, name, '_virtualenvs'));
+    }
+    for (const directory of virtualenvs) {
+        for (const sitePackages of await sitePackagesUnder(directory)) {
+            try {
+                await writeFileIfChanged(join(sitePackages, `${GUARD_MODULE_NAME}.py`), GUARD_PYTHON_SOURCE, { durable: false });
+                await writeFileIfChanged(join(sitePackages, `${GUARD_MODULE_NAME}.pth`), `import ${GUARD_MODULE_NAME}\n`, { durable: false });
+            }
+            catch (error) {
+                // A read-only or vanished virtualenv must not stop the run: the
+                // fallback still applies and other virtualenvs took the guard.
+                verbose(`Could not install the mach guard into ${sitePackages}: ${toError(error).message}`);
+            }
+        }
+    }
+    const fallback = await writeFallback(env);
+    const existing = env['PYTHONPATH'];
+    return {
+        PYTHONPATH: existing === undefined || existing === '' ? fallback : `${fallback}${delimiter}${existing}`,
+    };
+}