@procedures/shell 0.0.0-stage → 0.1.0

package/src/command/processes.ts

@@ -0,0 +1,271 @@
+/**
+ * The registry of long-running processes (deep dive WRP-4, WRP-10).
+ *
+ * Some procedures start a process and return while it runs: `vitest.watch`, `node.spawn`,
+ * `vite.dev`. Before, they started it detached and forgot it, so nothing could stop it, and it
+ * kept running after the host ended. The registry gives each process:
+ * - a record (id, pid, status, exit code), which `list` and `get` give;
+ * - a `stop` that kills the whole process tree;
+ * - the end of its output (the last 64 KiB), which also keeps its pipes from filling;
+ * - an end with the host: the host does not wait for the process, and kills it when it ends.
+ *
+ * The registry keeps the records of ended processes, up to `keepExited` of them.
+ */
+
+import type { ChildProcess } from "node:child_process";
+import { killTree } from "./tree-kill.js";
+import { OutputCollector } from "./output.js";
+import { startChild, startError, type SpawnOptions } from "./spawn.js";
+
+export interface ManagedProcessInfo {
+  id: string;
+  pid: number;
+  command: string;
+  args: string[];
+  cwd?: string;
+  /** The kind of process, set by the procedure that started it (for example "vitest"). */
+  group?: string;
+  /** A name for people. */
+  label?: string;
+  status: "running" | "exited" | "error";
+  startedAt: string;
+  exitedAt?: string;
+  exitCode?: number;
+  signal?: string;
+  /** The error that kept the program from starting. */
+  error?: string;
+}
+
+export interface StartProcessOptions extends SpawnOptions {
+  group?: string | undefined;
+  label?: string | undefined;
+}
+
+export interface StopResult {
+  /** True when this call ended a running process. */
+  stopped: boolean;
+  exitCode?: number;
+  signal?: string;
+}
+
+/** The end of the output that the registry keeps for each process. */
+const OUTPUT_TAIL_BYTES = 64 * 1024;
+/** How long `stop` waits after the first kill, before it kills again with SIGKILL. */
+const STOP_GRACE_MS = 5000;
+
+// The CSI and OSC escape sequences of terminals (colors, styles, links)
+const ANSI = /\x1b\[[0-?]*[ -/]*[@-~]|\x1b\][^\x07\x1b]*(?:\x07|\x1b\\)/g;
+
+/** This function removes the ANSI escape sequences from a text. */
+export function stripAnsi(text: string): string {
+  return text.replace(ANSI, "");
+}
+
+interface Entry {
+  info: ManagedProcessInfo;
+  child: ChildProcess;
+  output: OutputCollector;
+  /** Called after each output chunk and at the end. */
+  watchers: Set<() => void>;
+  done: Promise<void>;
+}
+
+export class ProcessRegistry {
+  private readonly entries = new Map<string, Entry>();
+  private readonly keepExited: number;
+  private counter = 0;
+
+  constructor(options: { keepExited?: number } = {}) {
+    this.keepExited = options.keepExited ?? 50;
+  }
+
+  /** This function starts a program and records it. The program inherits no stdin. */
+  start(command: string, options: StartProcessOptions = {}): ManagedProcessInfo {
+    const child = startChild(command, options, "ignore");
+    const id = `${options.group ?? "process"}-${Date.now().toString(36)}-${++this.counter}`;
+    const info: ManagedProcessInfo = {
+      id,
+      pid: child.pid ?? 0,
+      command,
+      args: [...(options.args ?? [])],
+      status: "running",
+      startedAt: new Date().toISOString(),
+    };
+    if (options.cwd !== undefined) info.cwd = options.cwd;
+    if (options.group !== undefined) info.group = options.group;
+    if (options.label !== undefined) info.label = options.label;
+
+    const output = new OutputCollector(OUTPUT_TAIL_BYTES, "tail");
+    const watchers = new Set<() => void>();
+    const changed = (): void => {
+      for (const watcher of [...watchers]) watcher();
+    };
+    child.stdout!.on("data", (chunk: Buffer) => {
+      output.add(chunk);
+      changed();
+    });
+    child.stderr!.on("data", (chunk: Buffer) => {
+      output.add(chunk);
+      changed();
+    });
+
+    const done = new Promise<void>((resolve) => {
+      let finished = false;
+      const finish = (): void => {
+        if (finished) return;
+        finished = true;
+        info.exitedAt = new Date().toISOString();
+        resolve();
+        changed();
+        this.reap();
+      };
+      child.on("error", (error) => {
+        info.error = startError(error);
+        if (child.pid === undefined) {
+          info.status = "error";
+          finish();
+        }
+      });
+      child.on("close", (code, signal) => {
+        info.status = info.error ? "error" : "exited";
+        if (code !== null) info.exitCode = code;
+        if (signal) info.signal = signal;
+        finish();
+      });
+    });
+
+    // The host does not wait for the process: it ends when the host ends (see tree-kill.ts)
+    child.unref();
+    for (const stream of [child.stdout, child.stderr]) {
+      (stream as unknown as { unref?: () => void } | null)?.unref?.();
+    }
+
+    this.entries.set(id, { info, child, output, watchers, done });
+    this.reap();
+    return { ...info };
+  }
+
+  /** This function gives a copy of the record of a process. */
+  get(id: string): ManagedProcessInfo | undefined {
+    const entry = this.entries.get(id);
+    return entry ? { ...entry.info } : undefined;
+  }
+
+  /** This function gives copies of the records, oldest first. */
+  list(filter: { group?: string | undefined } = {}): ManagedProcessInfo[] {
+    const result: ManagedProcessInfo[] = [];
+    for (const entry of this.entries.values()) {
+      if (filter.group !== undefined && entry.info.group !== filter.group) continue;
+      result.push({ ...entry.info });
+    }
+    return result;
+  }
+
+  /** This function gives the end of the output of a process (stdout and stderr together). */
+  output(id: string): string {
+    return this.entries.get(id)?.output.text() ?? "";
+  }
+
+  /** This function waits until the process ends, then gives its record. */
+  async exited(id: string): Promise<ManagedProcessInfo | undefined> {
+    const entry = this.entries.get(id);
+    if (!entry) return undefined;
+    await entry.done;
+    return { ...entry.info };
+  }
+
+  /**
+   * This function waits until the output of the process matches `pattern`, and gives the match.
+   * The pattern sees the output without ANSI escape codes: a program can color its output when
+   * the environment has `FORCE_COLOR`. It rejects when the process ends first, or after
+   * `timeout` milliseconds.
+   */
+  waitFor(id: string, pattern: RegExp, timeout = 30_000): Promise<RegExpMatchArray> {
+    const entry = this.entries.get(id);
+    if (!entry) return Promise.reject(new Error(`No process ${id}`));
+    return new Promise((resolve, reject) => {
+      const check = (): void => {
+        const match = stripAnsi(entry.output.text()).match(pattern);
+        if (match) {
+          stop();
+          resolve(match);
+        } else if (entry.info.status !== "running" || entry.info.exitedAt) {
+          stop();
+          const code = entry.info.exitCode ?? entry.info.signal ?? entry.info.error;
+          reject(new Error(`Process ${id} exited (${code}) before its output matched ${pattern}`));
+        }
+      };
+      const timer = setTimeout(() => {
+        stop();
+        reject(new Error(`Timeout after ${timeout} ms: the output of process ${id} did not match ${pattern}`));
+      }, timeout);
+      const stop = (): void => {
+        clearTimeout(timer);
+        entry.watchers.delete(check);
+      };
+      entry.watchers.add(check);
+      check();
+    });
+  }
+
+  /**
+   * This function kills the process tree and waits for the end. When the process does not end
+   * within the grace time, it kills again with SIGKILL.
+   */
+  async stop(id: string, signal: NodeJS.Signals = "SIGTERM"): Promise<StopResult> {
+    const entry = this.entries.get(id);
+    if (!entry || entry.info.exitedAt) {
+      return entry ? this.stopResult(entry, false) : { stopped: false };
+    }
+    killTree(entry.child, signal);
+    if (!(await this.endsWithin(entry, STOP_GRACE_MS))) {
+      killTree(entry.child, "SIGKILL");
+      await this.endsWithin(entry, STOP_GRACE_MS);
+    }
+    return this.stopResult(entry, true);
+  }
+
+  /** This function stops every running process of the registry (or of one group). */
+  async stopAll(filter: { group?: string | undefined } = {}): Promise<number> {
+    const running = this.list(filter).filter((info) => info.status === "running");
+    await Promise.all(running.map((info) => this.stop(info.id)));
+    return running.length;
+  }
+
+  /** This function removes the record of an ended process. It keeps the record of a running one. */
+  forget(id: string): boolean {
+    const entry = this.entries.get(id);
+    if (!entry || !entry.info.exitedAt) return false;
+    return this.entries.delete(id);
+  }
+
+  /** This function removes the oldest records of ended processes past `keepExited`. */
+  reap(): number {
+    const ended = [...this.entries.values()].filter((entry) => entry.info.exitedAt);
+    const excess = ended.length - this.keepExited;
+    for (let i = 0; i < excess; i++) this.entries.delete(ended[i]!.info.id);
+    return Math.max(0, excess);
+  }
+
+  private stopResult(entry: Entry, stopped: boolean): StopResult {
+    const result: StopResult = { stopped };
+    if (entry.info.exitCode !== undefined) result.exitCode = entry.info.exitCode;
+    if (entry.info.signal !== undefined) result.signal = entry.info.signal;
+    return result;
+  }
+
+  private async endsWithin(entry: Entry, ms: number): Promise<boolean> {
+    let timer: NodeJS.Timeout | undefined;
+    const ended = await Promise.race([
+      entry.done.then(() => true),
+      new Promise<boolean>((resolve) => {
+        timer = setTimeout(() => resolve(false), ms);
+      }),
+    ]);
+    clearTimeout(timer);
+    return ended;
+  }
+}
+
+/** The registry of the host process. The procedures of the wrapper packages use it. */
+export const processes: ProcessRegistry = new ProcessRegistry();

package/src/command/run.ts

@@ -0,0 +1,127 @@
+/**
+ * runCommand: run a program to its end and give its output (roadmap 2.2).
+ *
+ * This is the one way the wrapper packages run a program. Each wrapper had its own way: shell
+ * strings (`docker`, `pnpm`), synchronous calls that blocked the event loop and ignored the
+ * signal (`git`, `cue`), and `spawn` calls that killed only the direct child.
+ */
+
+import { killTree } from "./tree-kill.js";
+import { DEFAULT_MAX_OUTPUT_BYTES, OutputCollector } from "./output.js";
+import { startChild, startError, type SpawnOptions } from "./spawn.js";
+
+export interface CommandOptions extends SpawnOptions {
+  /** Text or bytes to write to stdin. Without it, stdin is empty (the program reads its end at once). */
+  input?: string | Buffer | undefined;
+  /** After this many milliseconds, the process tree is killed. */
+  timeout?: number | undefined;
+  /** When the signal aborts, the process tree is killed. An aborted signal starts nothing. */
+  signal?: AbortSignal | undefined;
+  /** The most bytes kept of each output stream (default: 64 MiB). */
+  maxOutputBytes?: number | undefined;
+  /** Past the limit, keep the first bytes ("head", the default) or the last bytes ("tail"). */
+  keep?: "head" | "tail" | undefined;
+  /** The encoding of the output (default: utf8). */
+  encoding?: BufferEncoding | undefined;
+}
+
+export interface CommandResult {
+  /** The exit code. It is 1 when the process ended on a signal or did not start. */
+  exitCode: number;
+  stdout: string;
+  stderr: string;
+  /** True when the exit code is 0. */
+  success: boolean;
+  /** The time from the start to the end, in milliseconds. */
+  duration: number;
+  /** The signal that ended the process. When the helper killed the tree, it is "SIGTERM". */
+  signal?: string;
+  /** True when the timeout killed the process. */
+  timedOut?: boolean;
+  /** True when the signal aborted the command. */
+  aborted?: boolean;
+  /** Present when an output stream was longer than `maxOutputBytes`: which one lost bytes. */
+  truncated?: { stdout: boolean; stderr: boolean };
+  /** The error that kept the program from starting, for example `spawn x ENOENT`. */
+  error?: string;
+}
+
+/** This function runs `command` with `options.args`. It does not throw: a failure is in the result. */
+export function runCommand(command: string, options: CommandOptions = {}): Promise<CommandResult> {
+  const started = Date.now();
+  if (options.signal?.aborted) {
+    return Promise.resolve({ exitCode: 1, stdout: "", stderr: "", success: false, duration: 0, aborted: true });
+  }
+  const encoding = options.encoding ?? "utf8";
+  const limit = options.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES;
+  const stdout = new OutputCollector(limit, options.keep);
+  const stderr = new OutputCollector(limit, options.keep);
+
+  return new Promise((resolve) => {
+    const child = startChild(command, options, options.input !== undefined ? "pipe" : "ignore");
+    let timedOut = false;
+    let aborted = false;
+    let settled = false;
+
+    const timer = options.timeout !== undefined
+      ? setTimeout(() => {
+          timedOut = true;
+          killTree(child);
+        }, options.timeout)
+      : undefined;
+    const onAbort = (): void => {
+      aborted = true;
+      killTree(child);
+    };
+    options.signal?.addEventListener("abort", onAbort, { once: true });
+
+    const finish = (result: CommandResult): void => {
+      if (settled) return;
+      settled = true;
+      if (timer) clearTimeout(timer);
+      options.signal?.removeEventListener("abort", onAbort);
+      resolve(result);
+    };
+
+    child.stdout!.on("data", (chunk: Buffer) => stdout.add(chunk));
+    child.stderr!.on("data", (chunk: Buffer) => stderr.add(chunk));
+    if (options.input !== undefined) {
+      // A program that ends before it reads its input closes the pipe: that is not an error here
+      child.stdin!.on("error", () => {});
+      child.stdin!.end(options.input);
+    }
+
+    child.on("error", (error) => {
+      // The program did not start: no "close" follows
+      if (child.pid === undefined) {
+        finish({
+          exitCode: 1,
+          stdout: "",
+          stderr: startError(error),
+          success: false,
+          duration: Date.now() - started,
+          error: startError(error),
+        });
+      }
+    });
+
+    child.on("close", (code, signal) => {
+      const result: CommandResult = {
+        exitCode: code ?? 1,
+        stdout: stdout.text(encoding),
+        stderr: stderr.text(encoding),
+        success: code === 0 && !timedOut && !aborted,
+        duration: Date.now() - started,
+      };
+      // On Windows, taskkill ends the process with an exit code and no signal
+      const endSignal = signal ?? (timedOut || aborted ? "SIGTERM" : undefined);
+      if (endSignal) result.signal = endSignal;
+      if (timedOut) result.timedOut = true;
+      if (aborted) result.aborted = true;
+      if (stdout.truncated || stderr.truncated) {
+        result.truncated = { stdout: stdout.truncated, stderr: stderr.truncated };
+      }
+      finish(result);
+    });
+  });
+}

package/src/command/spawn.ts

@@ -0,0 +1,44 @@
+/**
+ * Start a child process the one way the helpers allow: an argument list, no shell, no window.
+ */
+
+import { spawn, type ChildProcess } from "node:child_process";
+import { OWN_PROCESS_GROUP, track } from "./tree-kill.js";
+
+export interface SpawnOptions {
+  /** The arguments. Each one goes to the program as it is: no shell reads it. */
+  args?: readonly string[] | undefined;
+  /** The working directory (default: the current directory of the host). */
+  cwd?: string | undefined;
+  /** Environment variables to add to the environment of the host. */
+  env?: Record<string, string | undefined> | undefined;
+  /**
+   * On Windows, pass the arguments without quotes. Only `shell.exec` sets it, for the command
+   * line of `cmd.exe`.
+   */
+  windowsVerbatimArguments?: boolean | undefined;
+}
+
+/**
+ * This function starts `command` with `shell: false` and `windowsHide`. On POSIX the child gets
+ * its own process group, so `killTree` reaches its descendants. The child is recorded until it
+ * ends, so the end of the host kills it.
+ */
+export function startChild(command: string, options: SpawnOptions, stdin: "pipe" | "ignore"): ChildProcess {
+  const child = spawn(command, [...(options.args ?? [])], {
+    cwd: options.cwd,
+    env: options.env ? { ...process.env, ...options.env } : process.env,
+    shell: false,
+    windowsHide: true,
+    detached: OWN_PROCESS_GROUP,
+    stdio: [stdin, "pipe", "pipe"],
+    ...(options.windowsVerbatimArguments ? { windowsVerbatimArguments: true } : {}),
+  });
+  track(child);
+  return child;
+}
+
+/** The text of an error that kept a program from starting, for example `spawn x ENOENT`. */
+export function startError(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}

package/src/command/stream.ts

@@ -0,0 +1,186 @@
+/**
+ * streamCommand: run a program and give each output line while it runs.
+ *
+ * The queue between the program and the reader has a limit (deep dive WRP-9). When the queue
+ * is full, the helper stops reading the pipes of the program. The pipes fill, and the program
+ * waits in its next write. When the reader takes items again, the helper reads again. Before,
+ * the queue had no limit: a program with a large output filled the memory of the host.
+ */
+
+import { StringDecoder } from "node:string_decoder";
+import { killTree } from "./tree-kill.js";
+import { startChild, type SpawnOptions } from "./spawn.js";
+
+/** One item of a stream: an output line, or the exit of the program (the last item). */
+export type CommandStreamItem =
+  | { type: "stdout" | "stderr"; line: string }
+  | {
+      type: "exit";
+      exitCode: number;
+      duration: number;
+      signal?: string;
+      timedOut?: boolean;
+      aborted?: boolean;
+    };
+
+export interface StreamCommandOptions extends SpawnOptions {
+  /** Text or bytes to write to stdin. Without it, stdin is empty. */
+  input?: string | Buffer | undefined;
+  /** After this many milliseconds, the process tree is killed. */
+  timeout?: number | undefined;
+  /** When the signal aborts, the process tree is killed. */
+  signal?: AbortSignal | undefined;
+  /** The most items that wait for the reader before the helper stops reading (default: 1000). */
+  highWaterMark?: number | undefined;
+  /** A longer line arrives in parts of this many characters (default: 1 MiB). */
+  maxLineLength?: number | undefined;
+  /** The encoding of the output (default: utf8). */
+  encoding?: BufferEncoding | undefined;
+}
+
+const DEFAULT_HIGH_WATER_MARK = 1000;
+const DEFAULT_MAX_LINE_LENGTH = 1024 * 1024;
+
+/**
+ * This function runs `command` and yields each line of stdout and stderr, then one exit item.
+ * When the reader stops early, or the signal aborts, the process tree is killed. A program that
+ * cannot start makes the generator throw.
+ */
+export async function* streamCommand(
+  command: string,
+  options: StreamCommandOptions = {},
+): AsyncGenerator<CommandStreamItem, void, undefined> {
+  const started = Date.now();
+  const highWaterMark = Math.max(1, options.highWaterMark ?? DEFAULT_HIGH_WATER_MARK);
+  const lowWaterMark = Math.floor(highWaterMark / 2);
+  const maxLineLength = Math.max(1, options.maxLineLength ?? DEFAULT_MAX_LINE_LENGTH);
+  const encoding = options.encoding ?? "utf8";
+
+  const items: CommandStreamItem[] = [];
+  let ended = false;
+  let failure: { error: Error } | undefined;
+  let wake: (() => void) | undefined;
+  const notify = (): void => {
+    const resume = wake;
+    wake = undefined;
+    resume?.();
+  };
+
+  if (options.signal?.aborted) {
+    yield { type: "exit", exitCode: 1, duration: 0, aborted: true };
+    return;
+  }
+
+  const child = startChild(command, options, options.input !== undefined ? "pipe" : "ignore");
+  let paused = false;
+  let timedOut = false;
+  let aborted = false;
+
+  const pause = (): void => {
+    if (paused) return;
+    paused = true;
+    child.stdout!.pause();
+    child.stderr!.pause();
+  };
+  const resume = (): void => {
+    if (!paused) return;
+    paused = false;
+    child.stdout!.resume();
+    child.stderr!.resume();
+  };
+  const push = (item: CommandStreamItem): void => {
+    items.push(item);
+    if (items.length >= highWaterMark) pause();
+    notify();
+  };
+
+  // A decoder for each stream keeps a character whose bytes arrive in two chunks (deep dive WRP-2)
+  const partial = { stdout: "", stderr: "" };
+  const decoders = { stdout: new StringDecoder(encoding), stderr: new StringDecoder(encoding) };
+  const emit = (type: "stdout" | "stderr", text: string): void => {
+    if (text.length <= maxLineLength) {
+      push({ type, line: text });
+      return;
+    }
+    for (let at = 0; at < text.length; at += maxLineLength) {
+      push({ type, line: text.slice(at, at + maxLineLength) });
+    }
+  };
+  const onData = (type: "stdout" | "stderr") => (data: Buffer): void => {
+    const lines = (partial[type] + decoders[type].write(data)).split(/\r?\n/);
+    let rest = lines.pop() ?? "";
+    for (const line of lines) emit(type, line);
+    // A line without its end yet: give the full parts now, keep the remainder
+    if (rest.length >= maxLineLength) {
+      const whole = rest.length - (rest.length % maxLineLength);
+      emit(type, rest.slice(0, whole));
+      rest = rest.slice(whole);
+    }
+    partial[type] = rest;
+  };
+  child.stdout!.on("data", onData("stdout"));
+  child.stderr!.on("data", onData("stderr"));
+
+  if (options.input !== undefined) {
+    child.stdin!.on("error", () => {});
+    child.stdin!.end(options.input);
+  }
+
+  const timer = options.timeout !== undefined
+    ? setTimeout(() => {
+        timedOut = true;
+        killTree(child);
+      }, options.timeout)
+    : undefined;
+  const onAbort = (): void => {
+    aborted = true;
+    killTree(child);
+  };
+  options.signal?.addEventListener("abort", onAbort, { once: true });
+
+  child.on("error", (error) => {
+    if (child.pid === undefined) {
+      failure = { error };
+      notify();
+    }
+  });
+  child.on("close", (code, signal) => {
+    for (const type of ["stdout", "stderr"] as const) {
+      const rest = partial[type] + decoders[type].end();
+      if (rest) emit(type, rest);
+    }
+    const exit: CommandStreamItem = { type: "exit", exitCode: code ?? 1, duration: Date.now() - started };
+    const endSignal = signal ?? (timedOut || aborted ? "SIGTERM" : undefined);
+    if (endSignal) exit.signal = endSignal;
+    if (timedOut) exit.timedOut = true;
+    if (aborted) exit.aborted = true;
+    items.push(exit);
+    ended = true;
+    notify();
+  });
+
+  try {
+    for (;;) {
+      if (items.length > 0) {
+        const item = items.shift()!;
+        if (items.length <= lowWaterMark) resume();
+        yield item;
+        continue;
+      }
+      if (failure) throw failure.error;
+      if (ended) return;
+      await new Promise<void>((resolve) => {
+        wake = resolve;
+      });
+    }
+  } finally {
+    if (timer) clearTimeout(timer);
+    options.signal?.removeEventListener("abort", onAbort);
+    // The reader stopped early: end the program and its descendants. The pipes read again, so
+    // they can close.
+    if (!ended) {
+      resume();
+      killTree(child);
+    }
+  }
+}