@jmcombs/pi-steward 0.0.0

package/server/host-collector.ts

@@ -0,0 +1,272 @@
+/**
+ * The Node body behind {@link HostMetricsProvider}: it spawns the operator's
+ * declared collector command and drains a persistent NDJSON stream from its
+ * stdout, keeping the latest validated reading for the live source to overlay.
+ *
+ * This is the long-lived process that clears the per-poll-exec timeout blocker —
+ * Steward spawns the collector once and reads it, rather than shelling out on
+ * every snapshot. The hazards it guards against were all verified real on this
+ * hardware (plan H1):
+ *
+ *   - It spawns `detached: true` and, on close, kills the whole PROCESS GROUP
+ *     (`process.kill(-pid)`). A plain `child.kill()` orphans a collector that is
+ *     itself a shell pipeline (`macmon | jq …`) — each respawn would leak one.
+ *   - It reads stdout line-by-line and validates every line against the schema
+ *     (see `core/host-metrics.ts`); malformed, oversized, or non-UTF8 lines are
+ *     dropped, never turned into an all-`null` sample.
+ *   - A producer that exits is respawned with exponential backoff, under a
+ *     failure CAP so a command that cannot stay up (or a `jq` that block-buffers
+ *     and never emits) fails honestly instead of fork-bombing. A spawn that does
+ *     produce a valid sample resets the streak, so an occasional restart over a
+ *     long healthy run never accretes toward the cap.
+ *   - stderr is inherited (the operator's terminal), never treated as data.
+ *
+ * Node-only (spawns processes); injected into the otherwise Node-free
+ * {@link LlamaSource}. Staleness is judged by the overlay, not here — this module
+ * only stamps each sample's arrival time.
+ */
+
+import { type ChildProcess, spawn as nodeSpawn } from "node:child_process";
+import {
+  type HostMetricsProvider,
+  type HostSample,
+  parseHostMetricsLine,
+} from "../core/host-metrics.js";
+
+/** The child-spawn surface this module uses; injected in tests for observability. */
+export type SpawnHostCollector = (command: string, args: string[]) => ChildProcess;
+
+export interface HostCollectorOptions {
+  /**
+   * How many consecutive failed (re)starts to tolerate before giving up. A start
+   * that yields at least one valid sample resets the count. Default 5.
+   */
+  maxRespawns?: number;
+  /** First backoff after a failure, ms; doubles each further failure. Default 500. */
+  minBackoffMs?: number;
+  /** Ceiling for the backoff, ms. Default 5000. */
+  maxBackoffMs?: number;
+  /** Grace after SIGTERM before a SIGKILL escalation on close. Default 750. */
+  killEscalationMs?: number;
+  /** Arrival clock for each sample. Injected in tests; defaults to `Date.now`. */
+  now?: () => number;
+  /** The spawner. Injected in tests; defaults to a `detached` `node:child_process` spawn. */
+  spawn?: SpawnHostCollector;
+}
+
+/**
+ * Longest stdout line we will assemble. A producer emitting a newline-less flood
+ * must never buffer without bound (Node's own line readers have no such cap), so
+ * the splitter below discards a run that exceeds this and resyncs at the next
+ * newline — memory stays bounded whatever the producer emits.
+ */
+const MAX_LINE_LENGTH = 64 * 1024;
+
+/** Grace after SIGTERM before escalating to SIGKILL on a producer that ignores it. */
+const KILL_ESCALATION_MS = 750;
+
+const DEFAULT_MAX_RESPAWNS = 5;
+const DEFAULT_MIN_BACKOFF_MS = 500;
+const DEFAULT_MAX_BACKOFF_MS = 5000;
+
+/** Feeds decoded stdout chunks in; hands complete, within-cap lines out. */
+export interface LineSplitter {
+  push(chunk: string): void;
+}
+
+/**
+ * A bounded NDJSON line splitter. It accumulates decoded chunks and emits each
+ * complete line (a trailing `\r` stripped, for CRLF producers) to `onLine`,
+ * reassembling lines split across chunks. Crucially, it NEVER holds more than
+ * `maxLineLength` of a single unterminated line: once the bytes since the last
+ * newline exceed the cap it discards the buffer and enters a resync — dropping
+ * everything until the next newline — so an unterminated flood cannot grow
+ * memory without bound. An over-cap line that does eventually terminate is
+ * dropped whole; the following line parses normally.
+ */
+export function createLineSplitter(
+  maxLineLength: number,
+  onLine: (line: string) => void,
+): LineSplitter {
+  // Bytes accumulated since the last newline, always ≤ maxLineLength.
+  let pending = "";
+  // True while discarding the tail of an over-cap run, until the next newline.
+  let overflowed = false;
+
+  return {
+    push(chunk: string): void {
+      let start = 0;
+      for (;;) {
+        const newline = chunk.indexOf("\n", start);
+        if (newline === -1) {
+          // No line terminator in the remainder: buffer it, unless doing so
+          // would breach the cap — in which case drop and resync, never store it.
+          const rest = chunk.slice(start);
+          if (!overflowed) {
+            if (pending.length + rest.length > maxLineLength) {
+              pending = "";
+              overflowed = true;
+            } else {
+              pending += rest;
+            }
+          }
+          return;
+        }
+
+        let segment = chunk.slice(start, newline);
+        if (segment.endsWith("\r")) segment = segment.slice(0, -1);
+        if (overflowed) {
+          // This newline ends the over-cap run; resume normal buffering after it.
+          overflowed = false;
+          pending = "";
+        } else if (pending.length + segment.length > maxLineLength) {
+          // A completed but over-cap line — dropped whole, like any bad line.
+          pending = "";
+        } else {
+          const line = pending + segment;
+          pending = "";
+          onLine(line);
+        }
+        start = newline + 1;
+      }
+    },
+  };
+}
+
+/** The default spawner: detached (its own group), stdout piped, stderr inherited. */
+function defaultSpawn(command: string, args: string[]): ChildProcess {
+  return nodeSpawn(command, args, {
+    detached: true,
+    // stdin ignored, stdout is the data channel, stderr goes to the terminal.
+    stdio: ["ignore", "pipe", "inherit"],
+  });
+}
+
+/**
+ * Spawns `command` and streams host readings off its stdout. `intervalMs` is the
+ * collector's declared cadence (from `steward.json`); the overlay uses it for
+ * staleness, and it is reported here only if the collector is given up on.
+ */
+export function createHostCollector(
+  command: string[],
+  intervalMs: number,
+  options: HostCollectorOptions = {},
+): HostMetricsProvider {
+  const program = command[0] ?? "";
+  const args = command.slice(1);
+  const maxRespawns = options.maxRespawns ?? DEFAULT_MAX_RESPAWNS;
+  const minBackoffMs = options.minBackoffMs ?? DEFAULT_MIN_BACKOFF_MS;
+  const maxBackoffMs = options.maxBackoffMs ?? DEFAULT_MAX_BACKOFF_MS;
+  const killEscalationMs = options.killEscalationMs ?? KILL_ESCALATION_MS;
+  const now = options.now ?? Date.now;
+  const spawn = options.spawn ?? defaultSpawn;
+
+  let sample: HostSample | null = null;
+  let child: ChildProcess | null = null;
+  let backoffTimer: ReturnType<typeof setTimeout> | null = null;
+  let failures = 0;
+  let closed = false;
+
+  function handleLine(line: string): void {
+    // The splitter guarantees `line` is within the cap; malformed/foreign lines
+    // simply fail validation and are dropped.
+    const reading = parseHostMetricsLine(line);
+    if (reading === null) return;
+    // A real reading means this spawn is healthy: forgive its restart history so
+    // an occasional respawn over a long, working run never reaches the cap.
+    failures = 0;
+    sample = { reading, receivedAt: now() };
+  }
+
+  function scheduleRespawn(): void {
+    failures += 1;
+    if (failures > maxRespawns) {
+      console.warn(
+        `[steward] host collector gave up after ${failures} failed starts ` +
+          `(command: ${program}, intervalMs: ${intervalMs})`,
+      );
+      return;
+    }
+    const backoff = Math.min(maxBackoffMs, minBackoffMs * 2 ** (failures - 1));
+    backoffTimer = setTimeout(spawnOnce, backoff);
+  }
+
+  function spawnOnce(): void {
+    if (closed) return;
+    backoffTimer = null;
+
+    let proc: ChildProcess;
+    try {
+      proc = spawn(program, args);
+    } catch {
+      // A synchronous spawn throw (rare) is just another failed start.
+      scheduleRespawn();
+      return;
+    }
+    child = proc;
+
+    // exit and error can both fire for one spawn (e.g. ENOENT); settle once so a
+    // single failed start counts once toward the cap.
+    let settled = false;
+    const settle = (): void => {
+      if (settled) return;
+      settled = true;
+      if (child === proc) child = null;
+      if (!closed) scheduleRespawn();
+    };
+    // A listener is required or an 'error' would throw as unhandled.
+    proc.once("error", settle);
+    proc.once("exit", settle);
+
+    if (proc.stdout !== null) {
+      // Own the line-splitting (with a hard cap) rather than delegating to a
+      // reader that would buffer an unterminated flood without bound.
+      proc.stdout.setEncoding("utf8");
+      const splitter = createLineSplitter(MAX_LINE_LENGTH, handleLine);
+      proc.stdout.on("data", (chunk: string) => splitter.push(chunk));
+    }
+  }
+
+  spawnOnce();
+
+  return {
+    latest(): HostSample | null {
+      return sample;
+    },
+
+    close(): void {
+      if (closed) return;
+      closed = true;
+      if (backoffTimer !== null) {
+        clearTimeout(backoffTimer);
+        backoffTimer = null;
+      }
+      const proc = child;
+      child = null;
+      if (proc === null || proc.pid === undefined) return;
+      const pid = proc.pid;
+
+      // Signal the whole group: the collector may be a shell pipeline, and a
+      // direct kill would orphan the producer feeding it. Fall back to a direct
+      // kill when there is no group (already gone, or negative pids unsupported).
+      const signalGroup = (signal: NodeJS.Signals): void => {
+        try {
+          process.kill(-pid, signal);
+        } catch {
+          try {
+            proc.kill(signal);
+          } catch {
+            // Already dead — nothing to do.
+          }
+        }
+      };
+
+      signalGroup("SIGTERM");
+      // A producer that traps or ignores SIGTERM would otherwise linger; escalate
+      // to an uncatchable SIGKILL after a grace period. The timer is unref'd so it
+      // never keeps the process alive, and is cleared if it lands before firing.
+      const escalation = setTimeout(() => signalGroup("SIGKILL"), killEscalationMs);
+      escalation.unref();
+    },
+  };
+}

package/server/index.ts

@@ -0,0 +1,228 @@
+/**
+ * The Steward server.
+ *
+ * A `node:http` server bound to loopback that does two things: serve the
+ * dashboard's own files (see `./assets.ts`) and expose one
+ * {@link StewardDataSource} over HTTP (see `./api.ts`). It holds the single
+ * source instance for the process, which is why swapping the mock for a live
+ * `llama-server` reader is a change here and nowhere else.
+ *
+ * It is never exposed beyond `127.0.0.1`: it can start and stop a local
+ * service and has no authentication of its own.
+ */
+
+import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http";
+import { createDisconnectedSource } from "../core/disconnected-source.js";
+import type { StewardDataSource } from "../core/source.js";
+import { handleApiRequest } from "./api.js";
+import { assertTypeStripping, readAsset } from "./assets.js";
+
+/** The port Steward asks for unless told otherwise. */
+export const DEFAULT_PORT = 8788;
+
+/** Loopback only, deliberately: see the module comment. */
+const HOST = "127.0.0.1";
+
+export interface StewardServerOptions {
+  /** Port to bind. `0` picks a free one, which {@link StewardServer.port} reports. */
+  port?: number;
+  /**
+   * The source to serve. Defaults to the simulated one, which is built on the
+   * first {@link StewardServer.start} rather than here, so a server that never
+   * binds never starts a simulation. The server takes ownership either way: it
+   * closes the source when it stops, and when a start fails.
+   */
+  source?: StewardDataSource;
+}
+
+export interface StewardServer {
+  /**
+   * Binds the socket and resolves with the URL to open, port included.
+   * Resolves with the same URL if it is already bound, and joins an in-flight
+   * start rather than binding twice.
+   *
+   * A start that fails releases everything it created, which includes closing
+   * the data source — so a rejected start, like {@link StewardServer.stop},
+   * spends the instance. Calling `start` on a spent instance rejects.
+   */
+  start(): Promise<string>;
+  /**
+   * Closes live connections, the socket, and the data source, waiting for an
+   * in-flight {@link StewardServer.start} first so it cannot leave a socket
+   * bound behind the stop.
+   *
+   * Terminal: repeat calls resolve without doing more, and the instance cannot
+   * be started again — its source has been closed and closed sources do not
+   * come back.
+   */
+  stop(): Promise<void>;
+  /** The bound URL, or `null` before {@link start} resolves. */
+  readonly url: string | null;
+  /** The bound port, or `null` before {@link start} resolves. */
+  readonly port: number | null;
+}
+
+function sendPlain(res: ServerResponse, status: number, body: string): void {
+  res.writeHead(status, {
+    "Content-Type": "text/plain; charset=utf-8",
+    "Content-Length": Buffer.byteLength(body),
+    "Cache-Control": "no-store",
+  });
+  res.end(body);
+}
+
+/** Closes any connection still open, so `close()` is not held up by an SSE client. */
+function closeConnections(server: Server): void {
+  if (typeof server.closeAllConnections === "function") server.closeAllConnections();
+}
+
+export function createStewardServer(options: StewardServerOptions = {}): StewardServer {
+  const requestedPort = options.port ?? DEFAULT_PORT;
+
+  let source: StewardDataSource | null = options.source ?? null;
+  let boundPort: number | null = null;
+  let boundUrl: string | null = null;
+  /** Set once the instance is spent, by a stop or by a start that failed. */
+  let spent = false;
+  let starting: Promise<string> | null = null;
+  let stopping: Promise<void> | null = null;
+
+  async function handleRequest(req: IncomingMessage, res: ServerResponse): Promise<void> {
+    // Requests only arrive between a resolved start and a stop, so the source
+    // is there — but a stop that lands mid-request must not be a crash.
+    const active = source;
+    if (active === null) {
+      sendPlain(res, 503, "Steward is shutting down\n");
+      return;
+    }
+
+    // The base is a formality: only the path and query of `req.url` are real.
+    const { pathname } = new URL(req.url ?? "/", `http://${HOST}`);
+
+    if (await handleApiRequest(req, res, active, pathname)) return;
+
+    if (req.method === "GET") {
+      const asset = await readAsset(pathname);
+      if (asset !== null) {
+        res.writeHead(200, {
+          "Content-Type": asset.contentType,
+          "Content-Length": Buffer.byteLength(asset.body),
+          "Cache-Control": "no-store",
+        });
+        res.end(asset.body);
+        return;
+      }
+    }
+
+    sendPlain(res, 404, "Not found\n");
+  }
+
+  const server = createServer((req, res) => {
+    handleRequest(req, res).catch((error: unknown) => {
+      const detail = error instanceof Error ? error.message : String(error);
+      // The detail goes to the operator's terminal, not to the page: it can
+      // name absolute paths, and the browser has no use for it either way.
+      console.error(`[steward] ${req.method ?? "GET"} ${req.url ?? "/"} failed: ${detail}`);
+      if (!res.headersSent) sendPlain(res, 500, "Internal error\n");
+      else res.end();
+    });
+  });
+
+  /** Releases every resource the instance holds. Safe to call more than once. */
+  function release(): Promise<void> {
+    const active = source;
+    source = null;
+    boundUrl = null;
+    boundPort = null;
+    if (active !== null) active.close();
+    if (!server.listening) return Promise.resolve();
+    return new Promise<void>((resolve, reject) => {
+      server.close((error) => {
+        if (error) reject(error);
+        else resolve();
+      });
+      closeConnections(server);
+    });
+  }
+
+  function bind(): Promise<string> {
+    return new Promise<string>((resolve, reject) => {
+      // Every browser module is type-stripped on demand, so a runtime that
+      // cannot strip serves the shell and nothing that makes it work. Fail
+      // here, where whoever asked for the dashboard is still listening.
+      assertTypeStripping();
+      source = source ?? createDisconnectedSource();
+
+      const onError = (error: Error) => {
+        server.removeListener("listening", onListening);
+        reject(error);
+      };
+      const onListening = () => {
+        server.removeListener("error", onError);
+        const address = server.address();
+        if (address === null || typeof address === "string") {
+          reject(new Error("Steward server bound to an unexpected address"));
+          return;
+        }
+        boundPort = address.port;
+        boundUrl = `http://${HOST}:${address.port}`;
+        resolve(boundUrl);
+      };
+      server.once("error", onError);
+      server.once("listening", onListening);
+      server.listen(requestedPort, HOST);
+    });
+  }
+
+  return {
+    get url(): string | null {
+      return boundUrl;
+    },
+
+    get port(): number | null {
+      return boundPort;
+    },
+
+    start(): Promise<string> {
+      if (spent) {
+        return Promise.reject(
+          new Error("Steward server is spent: start a new one rather than reviving this one"),
+        );
+      }
+      if (boundUrl !== null) return Promise.resolve(boundUrl);
+      if (starting !== null) return starting;
+
+      const attempt = bind().then(
+        (url) => {
+          starting = null;
+          return url;
+        },
+        async (error: unknown) => {
+          // A start owns everything it built, whether or not it got as far as
+          // binding: the source's tickers are running by now and only this
+          // path can still reach them.
+          starting = null;
+          spent = true;
+          await release();
+          throw error;
+        },
+      );
+      starting = attempt;
+      return attempt;
+    },
+
+    stop(): Promise<void> {
+      if (stopping !== null) return stopping;
+      spent = true;
+      const pending = starting;
+      const attempt = (async () => {
+        // A stop that lands mid-start must wait for the socket the start is
+        // about to bind, or it would close nothing and leave it listening.
+        if (pending !== null) await pending.catch(() => undefined);
+        await release();
+      })();
+      stopping = attempt;
+      return attempt;
+    },
+  };
+}