@hominis/fireforge 0.54.0 → 0.60.0

package/dist/src/kernel/process/spawn.js

@@ -0,0 +1,430 @@
+// SPDX-License-Identifier: EUPL-1.2
+/**
+ * Running other programs. Every child goes through {@link run} (or
+ * {@link runBytes}), so each one is known to the interrupt pipeline: a signal
+ * is forwarded where the child would not get it by itself, and the process
+ * does not exit while a child's teardown is still running.
+ *
+ * Arguments are an argv array, never a shell string.
+ */
+import { spawn } from 'node:child_process';
+import { constants as fsConstants } from 'node:fs';
+import { access } from 'node:fs/promises';
+import { constants as osConstants } from 'node:os';
+import { delimiter, join } from 'node:path';
+import { StringDecoder } from 'node:string_decoder';
+import { errnoCode, ExitCode, FireForgeError, toError } from '../errors.js';
+import { verbose, warn } from '../log.js';
+import { onForceExit, onShutdown } from '../signals.js';
+import { lineSplitter, tailLines } from '../text.js';
+import { settlesWithin } from '../time.js';
+import { ownProcessTree, signalChild, } from './tree.js';
+/**
+ * Cap on each captured stream. Past it the capture is cut and marked, so a
+ * runaway build cannot exhaust memory. Mirrors and callbacks still see everything.
+ */
+const MAX_CAPTURE = 50 * 1024 * 1024;
+/** Default grace between SIGTERM and SIGKILL. */
+const DEFAULT_GRACE_MS = 1_500;
+/** How long after exit the output pipes get to close before they are abandoned. */
+const PIPE_DRAIN_MS = 1_000;
+/** Upper bound on a caller's `afterExit`. */
+const AFTER_EXIT_LIMIT_MS = 5_000;
+/** How long an interrupt waits for children at least, whatever their grace. */
+const SHUTDOWN_WAIT_MS = 12_000;
+function childEnv(overrides) {
+    const env = { ...process.env };
+    for (const [key, value] of Object.entries(overrides ?? {})) {
+        if (value === undefined)
+            Reflect.deleteProperty(env, key);
+        else
+            env[key] = value;
+    }
+    return env;
+}
+function exitCodeOf(code, signal) {
+    if (code !== null)
+        return code;
+    const number = signal === null ? undefined : osConstants.signals[signal];
+    return number === undefined ? 1 : 128 + number;
+}
+function capture(stream, mirror, onLine) {
+    // One decoder per stream keeps a multibyte character split across chunks intact.
+    const decoder = new StringDecoder('utf8');
+    const lines = onLine === undefined
+        ? undefined
+        : lineSplitter((line) => {
+            onLine(line, stream);
+        });
+    const chunks = [];
+    let length = 0;
+    let cut = false;
+    const accept = (text) => {
+        if (text.length === 0)
+            return;
+        mirror?.write(text);
+        lines?.push(text);
+        if (cut)
+            return;
+        const room = MAX_CAPTURE - length;
+        chunks.push(text.length > room ? text.slice(0, room) : text);
+        length += Math.min(text.length, room);
+        if (text.length > room) {
+            chunks.push('\n[output truncated]');
+            cut = true;
+        }
+    };
+    return {
+        write: (chunk) => {
+            accept(decoder.write(chunk));
+        },
+        end: () => {
+            accept(decoder.end());
+            lines?.flush();
+        },
+        text: () => chunks.join(''),
+        truncated: () => cut,
+    };
+}
+const tracked = new Set();
+let hooked = false;
+async function stopChildren() {
+    let wait = SHUTDOWN_WAIT_MS;
+    for (const child of tracked) {
+        if (!child.forward)
+            continue;
+        child.signal('SIGTERM');
+        setTimeout(() => {
+            if (tracked.has(child))
+                child.signal('SIGKILL');
+        }, child.graceMs).unref();
+        wait = Math.max(wait, child.graceMs + 2_000);
+    }
+    // Holding the exit here is what lets the escalation above actually run.
+    await settlesWithin(Promise.allSettled([...tracked].map((child) => child.closed)), wait);
+}
+function track(child) {
+    if (!hooked) {
+        hooked = true;
+        onShutdown('stop', stopChildren, { timeoutMs: 120_000 });
+        onForceExit(() => {
+            for (const entry of tracked)
+                if (entry.forward)
+                    entry.signal('SIGKILL');
+        });
+    }
+    let settle = () => undefined;
+    const entry = {
+        ...child,
+        closed: new Promise((resolve) => {
+            settle = resolve;
+        }),
+    };
+    tracked.add(entry);
+    return () => {
+        tracked.delete(entry);
+        settle();
+    };
+}
+const groupListeners = new Set();
+/**
+ * Subscribes to the process groups this process starts. A group whose
+ * teardown stayed inconclusive is never reported as ended, so a subscriber
+ * errs toward treating it as alive.
+ *
+ * @param listener - See {@link ProcessGroupListener}
+ */
+export function onProcessGroup(listener) {
+    groupListeners.add(listener);
+}
+function reportTeardown(teardown) {
+    if (teardown === undefined)
+        return;
+    if (teardown.signalled.length > 0) {
+        warn(`${teardown.signalled.length} process(es) outlived their command and were terminated: ` +
+            `PID ${teardown.signalled.map((entry) => entry.pid).join(', ')}.`);
+    }
+    if (teardown.detached.length > 0) {
+        const rows = teardown.detached.map((entry) => `${entry.pid} (${entry.command?.slice(0, 80) ?? 'command unavailable'})`);
+        verbose(`Left running, detached and not attributable to this run: ${rows.join(', ')}.`);
+    }
+    const uncertainty = teardown.uncertain ??
+        (teardown.stillAlive.length > 0
+            ? `PID ${teardown.stillAlive.map((entry) => entry.pid).join(', ')} survived SIGKILL`
+            : undefined);
+    if (uncertainty !== undefined) {
+        warn(`Process cleanup is inconclusive: ${uncertainty}. No unowned process was signalled.`);
+    }
+}
+/**
+ * Follows a spawned child to the end of its teardown.
+ *
+ * `exit` starts the teardown. `close` only means the pipes drained: a
+ * background process holding a pipe must not hold up the result or change
+ * it, so the wait for it is bounded. Rejects only with the child's own
+ * failure (a failed spawn); a survivor or an uncertain teardown is a
+ * diagnostic.
+ */
+async function follow(child, owner, options) {
+    let failure;
+    let escalation;
+    let hasExited = false;
+    const closed = new Promise((resolve) => {
+        child.once('close', () => {
+            resolve();
+        });
+    });
+    const exit = await new Promise((resolve) => {
+        const onExit = (code, signal) => {
+            hasExited = true;
+            resolve({ code, signal });
+        };
+        child.once('exit', onExit);
+        // A failed spawn emits `error` and `close` with no `exit`.
+        child.once('close', onExit);
+        child.on('error', (error) => {
+            if (hasExited)
+                return;
+            failure ??= error;
+            if (child.pid === undefined) {
+                onExit(null, null);
+                return;
+            }
+            // Something went wrong around a live child; nothing else will end it.
+            owner?.signal('SIGTERM');
+            escalation ??= setTimeout(() => {
+                if (owner !== undefined)
+                    owner.signal('SIGKILL');
+                else
+                    signalChild(child, 'SIGKILL');
+            }, options.graceMs);
+            escalation.unref();
+        });
+    });
+    clearTimeout(escalation);
+    const teardown = await owner?.finish(options.graceMs);
+    if (owner !== undefined && teardown?.uncertain === undefined && !teardown?.stillAlive.length) {
+        for (const listener of groupListeners)
+            listener.ended(owner.pid);
+    }
+    const abandoned = !(await settlesWithin(closed, PIPE_DRAIN_MS));
+    if (abandoned) {
+        child.stdout?.destroy();
+        child.stderr?.destroy();
+        child.stdin?.destroy();
+        child.unref();
+        verbose(`PID ${child.pid ?? '?'} exited, but a background process still holds its pipes.`);
+    }
+    if (options.afterExit !== undefined) {
+        let sweepFailure;
+        const sweep = options.afterExit().catch((error) => {
+            sweepFailure = error;
+        });
+        if (!(await settlesWithin(sweep, AFTER_EXIT_LIMIT_MS))) {
+            sweepFailure = new Error(`exceeded ${AFTER_EXIT_LIMIT_MS} ms`);
+        }
+        if (sweepFailure !== undefined) {
+            warn(`Cleanup after PID ${child.pid ?? '?'} failed: ${toError(sweepFailure).message}`);
+        }
+    }
+    reportTeardown(teardown);
+    if (failure !== undefined)
+        throw failure;
+    const outlived = (teardown?.signalled.length ?? 0) - (teardown?.stillAlive.length ?? 0);
+    return { ...exit, abandoned, outlived: Math.max(0, outlived) };
+}
+function spawnFailure(command, error) {
+    if (errnoCode(error) !== 'ENOENT')
+        return toError(error);
+    return new FireForgeError('command-not-found', `Could not run "${command}": it is not installed`, {
+        exitCode: ExitCode.MISSING_DEPENDENCY,
+        hint: `Install ${command} and make sure it is on PATH.`,
+        cause: error,
+    });
+}
+function timeoutError(command, args, timeoutMs, result) {
+    const stream = result.stderr.trim().length > 0 ? 'stderr' : 'stdout';
+    const tail = tailLines(result[stream], 20);
+    const output = tail.length === 0 ? '' : `\nLast output (${stream}):\n${tail.map((l) => `  ${l}`).join('\n')}`;
+    return new FireForgeError('command-timeout', `Command timed out after ${Math.round(timeoutMs / 1000)}s: ${[command, ...args].join(' ')}${output}`, {
+        hint: 'It was stopped for exceeding its time budget. On a loaded host, retrying may succeed.',
+    });
+}
+/** Spawns the child and wires it up, before any of its events can fire. */
+function start(command, args, options) {
+    const output = options.output ?? 'capture';
+    const useGroup = options.processGroup === true && process.platform !== 'win32';
+    const graceMs = options.graceMs ?? DEFAULT_GRACE_MS;
+    const stdin = options.input !== undefined ? 'pipe' : output === 'capture' ? 'ignore' : 'inherit';
+    const sink = output === 'inherit' ? 'inherit' : 'pipe';
+    const child = spawn(command, [...args], {
+        cwd: options.cwd,
+        env: childEnv(options.env),
+        detached: useGroup,
+        shell: false,
+        stdio: [stdin, sink, sink],
+    });
+    const owner = useGroup ? ownProcessTree(child) : undefined;
+    const signal = (value) => {
+        if (owner !== undefined)
+            owner.signal(value);
+        else
+            signalChild(child, value);
+    };
+    const untrack = track({
+        forward: options.forwardSignals ?? (useGroup || output !== 'capture'),
+        graceMs,
+        signal,
+    });
+    const live = output === 'tee' ? { stdout: process.stdout, stderr: process.stderr } : undefined;
+    const out = capture('stdout', options.mirror?.stdout ?? live?.stdout, options.onLine);
+    const err = capture('stderr', options.mirror?.stderr ?? live?.stderr, options.onLine);
+    child.stdout?.on('data', out.write);
+    child.stderr?.on('data', err.write);
+    if (options.input !== undefined) {
+        // A child that exits before reading all of stdin makes the write fail
+        // with EPIPE. Its exit status reports the real outcome.
+        child.stdin?.on('error', () => undefined);
+        child.stdin?.end(options.input);
+    }
+    if (owner !== undefined) {
+        for (const listener of groupListeners)
+            listener.started(owner.pid);
+        options.onProcessGroup?.(owner);
+    }
+    return { child, owner, signal, graceMs, out, err, untrack };
+}
+/** Arms the deadline: SIGTERM when it passes, SIGKILL after the grace. */
+function armDeadline(timeoutMs, started) {
+    let passed = false;
+    let escalation;
+    const deadline = timeoutMs === undefined
+        ? undefined
+        : setTimeout(() => {
+            passed = true;
+            started.signal('SIGTERM');
+            escalation = setTimeout(() => {
+                started.signal('SIGKILL');
+            }, started.graceMs);
+        }, timeoutMs);
+    return {
+        passed: () => passed,
+        clear: () => {
+            clearTimeout(deadline);
+            clearTimeout(escalation);
+        },
+    };
+}
+/**
+ * Runs a program to completion.
+ *
+ * @param command - The executable (looked up on PATH)
+ * @param args - Its arguments
+ * @param options - See {@link RunOptions}
+ * @returns How it ended. A non-zero exit resolves; only a failed spawn or an
+ *   expired deadline rejects
+ */
+export async function run(command, args, options = {}) {
+    const started = start(command, args, options);
+    const deadline = armDeadline(options.timeoutMs, started);
+    const { out, err } = started;
+    try {
+        const exit = await follow(started.child, started.owner, {
+            graceMs: started.graceMs,
+            afterExit: options.afterExit,
+        });
+        out.end();
+        err.end();
+        const result = {
+            exitCode: exitCodeOf(exit.code, exit.signal),
+            stdout: out.text(),
+            stderr: err.text(),
+            truncated: out.truncated() || err.truncated() || exit.abandoned,
+            timedOut: deadline.passed(),
+            outlived: exit.outlived,
+        };
+        if (result.timedOut && options.timeoutIsResult !== true) {
+            throw timeoutError(command, args, options.timeoutMs ?? 0, result);
+        }
+        return result;
+    }
+    catch (error) {
+        throw error instanceof FireForgeError ? error : spawnFailure(command, error);
+    }
+    finally {
+        deadline.clear();
+        started.untrack();
+    }
+}
+/**
+ * Runs a program and returns stdout as raw, uncapped bytes. For protocols
+ * whose framing counts bytes (`git cat-file --batch`), where decoding or
+ * truncating would corrupt the stream. The caller bounds the output.
+ *
+ * @param command - The executable
+ * @param args - Its arguments
+ * @param options - Working directory, environment overrides, and stdin content
+ */
+export async function runBytes(command, args, options = {}) {
+    const child = spawn(command, [...args], {
+        cwd: options.cwd,
+        env: childEnv(options.env),
+        shell: false,
+        stdio: ['pipe', 'pipe', 'pipe'],
+    });
+    const untrack = track({
+        forward: false,
+        graceMs: DEFAULT_GRACE_MS,
+        signal: (value) => signalChild(child, value),
+    });
+    const out = [];
+    const err = [];
+    child.stdout.on('data', (chunk) => out.push(chunk));
+    child.stderr.on('data', (chunk) => err.push(chunk));
+    child.stdin.on('error', () => undefined);
+    child.stdin.end(options.input ?? '');
+    try {
+        const exit = await follow(child, undefined, {
+            graceMs: DEFAULT_GRACE_MS,
+            afterExit: undefined,
+        });
+        if (exit.abandoned) {
+            throw new FireForgeError('command-output-incomplete', `${command} exited, but a process it left behind still holds its output; the output is incomplete.`, { hint: 'Find and stop the leftover process, then retry.' });
+        }
+        return {
+            exitCode: exitCodeOf(exit.code, exit.signal),
+            stdout: Buffer.concat(out),
+            stderr: Buffer.concat(err).toString('utf8'),
+        };
+    }
+    catch (error) {
+        throw error instanceof FireForgeError ? error : spawnFailure(command, error);
+    }
+    finally {
+        untrack();
+    }
+}
+/**
+ * Finds an executable on PATH without running anything.
+ *
+ * @param name - The program name
+ * @returns Its full path, or undefined when it is not installed
+ */
+export async function findExecutable(name) {
+    const extensions = process.platform === 'win32'
+        ? ['', ...(process.env['PATHEXT'] ?? '.EXE;.CMD').split(';')]
+        : [''];
+    for (const directory of (process.env['PATH'] ?? '').split(delimiter).filter(Boolean)) {
+        for (const extension of extensions) {
+            const candidate = join(directory, `${name}${extension}`);
+            try {
+                await access(candidate, fsConstants.X_OK);
+                return candidate;
+            }
+            catch {
+                // Not here; keep looking.
+            }
+        }
+    }
+    return undefined;
+}