@vellumai/assistant 0.10.4 → 0.10.5-dev.202607022329.d89bf2f

package/src/monitoring/__tests__/control.test.ts

@@ -0,0 +1,235 @@
+/**
+ * Tests for the resource monitor process control surface.
+ *
+ * Focuses on the readiness wait (which gates whether `assistant monitoring
+ * start` reports success and flips `monitoring.enabled`) and the PID-file
+ * liveness probe, mirroring the memory worker's worker-control tests.
+ */
+
+import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
+
+import * as realLogger from "../../util/logger.js";
+import * as realPlatform from "../../util/platform.js";
+
+let tmpDir: string;
+let pidPath: string;
+
+mock.module("../../util/platform.js", () => ({
+  ...realPlatform,
+  getMonitoringPidPath: () => pidPath,
+}));
+
+const noopLogger = {
+  info: () => {},
+  warn: () => {},
+  error: () => {},
+  debug: () => {},
+};
+mock.module("../../util/logger.js", () => ({
+  ...realLogger,
+  getLogger: () => noopLogger,
+  getCliLogger: () => noopLogger,
+  getCurrentLogFilePath: () => join(tmpDir, "monitoring-test.log"),
+}));
+
+const {
+  spawnMonitoringWorkerProcess,
+  MonitoringWorkerSpawnError,
+  probeMonitoringWorker,
+  stopMonitoringWorkerProcess,
+} = await import("../control.js");
+
+function stubProcessKill(
+  livePids: Set<number>,
+  permissionErrorPids: Set<number> = new Set(),
+): () => void {
+  const original = process.kill.bind(process);
+  process.kill = ((pid: number, signal?: string | number) => {
+    if ((signal ?? 0) !== 0) return true;
+    if (permissionErrorPids.has(pid)) {
+      const err = new Error("kill EPERM") as NodeJS.ErrnoException;
+      err.code = "EPERM";
+      throw err;
+    }
+    if (livePids.has(pid)) return true;
+    const err = new Error("kill ESRCH") as NodeJS.ErrnoException;
+    err.code = "ESRCH";
+    throw err;
+  }) as typeof process.kill;
+  return () => {
+    process.kill = original;
+  };
+}
+
+function stubBunSpawn(
+  makeChild: () => {
+    unref: () => void;
+    kill: () => void;
+    pid: number;
+    exited?: Promise<unknown>;
+  },
+): () => void {
+  const original = Bun.spawn;
+  (Bun as { spawn: typeof Bun.spawn }).spawn = (() =>
+    makeChild()) as unknown as typeof Bun.spawn;
+  return () => {
+    (Bun as { spawn: typeof Bun.spawn }).spawn = original;
+  };
+}
+
+function neverExits(): Promise<unknown> {
+  return new Promise(() => {});
+}
+
+beforeEach(() => {
+  tmpDir = mkdtempSync(join(tmpdir(), "monitoring-control-test-"));
+  pidPath = join(tmpDir, "monitoring.pid");
+});
+
+afterEach(() => {
+  rmSync(tmpDir, { recursive: true, force: true });
+});
+
+describe("spawnMonitoringWorkerProcess", () => {
+  test("returns the PID when the monitor writes its file immediately", async () => {
+    const restore = stubBunSpawn(() => {
+      writeFileSync(pidPath, "4242");
+      return {
+        unref: () => {},
+        kill: () => {},
+        pid: 4242,
+        exited: neverExits(),
+      };
+    });
+    try {
+      const result = await spawnMonitoringWorkerProcess({
+        pidWaitTimeoutMs: 1_000,
+        pidPollIntervalMs: 10,
+      });
+      expect(result).toEqual({ pid: 4242, alreadyRunning: false });
+    } finally {
+      restore();
+    }
+  });
+
+  test("fails fast when the child exits during startup", async () => {
+    let killed = false;
+    const restore = stubBunSpawn(() => ({
+      unref: () => {},
+      kill: () => {
+        killed = true;
+      },
+      pid: 99,
+      exited: Promise.resolve(1),
+    }));
+    try {
+      await expect(
+        spawnMonitoringWorkerProcess({
+          pidWaitTimeoutMs: 10_000,
+          pidPollIntervalMs: 10,
+          terminateOnTimeout: true,
+        }),
+      ).rejects.toBeInstanceOf(MonitoringWorkerSpawnError);
+      expect(killed).toBe(false);
+    } finally {
+      restore();
+    }
+  });
+
+  test("terminates a still-alive child on timeout when asked", async () => {
+    let killed = false;
+    const restore = stubBunSpawn(() => ({
+      unref: () => {},
+      kill: () => {
+        killed = true;
+      },
+      pid: 55,
+      exited: neverExits(),
+    }));
+    try {
+      await expect(
+        spawnMonitoringWorkerProcess({
+          pidWaitTimeoutMs: 150,
+          pidPollIntervalMs: 20,
+          terminateOnTimeout: true,
+        }),
+      ).rejects.toBeInstanceOf(MonitoringWorkerSpawnError);
+      expect(killed).toBe(true);
+    } finally {
+      restore();
+    }
+  });
+
+  test("reuses an already-running monitor without spawning", async () => {
+    writeFileSync(pidPath, String(process.pid));
+    let spawned = false;
+    const restore = stubBunSpawn(() => {
+      spawned = true;
+      return { unref: () => {}, kill: () => {}, pid: 1, exited: neverExits() };
+    });
+    try {
+      const result = await spawnMonitoringWorkerProcess({
+        pidWaitTimeoutMs: 100,
+      });
+      expect(result).toEqual({ pid: process.pid, alreadyRunning: true });
+      expect(spawned).toBe(false);
+    } finally {
+      restore();
+    }
+  });
+});
+
+describe("probeMonitoringWorker", () => {
+  test("reports not_running when no PID file exists", () => {
+    expect(probeMonitoringWorker()).toEqual({ status: "not_running" });
+  });
+
+  test("cleans up a stale PID file pointing at a dead process", () => {
+    writeFileSync(pidPath, "999999");
+    const restore = stubProcessKill(new Set());
+    try {
+      expect(probeMonitoringWorker()).toEqual({ status: "not_running" });
+    } finally {
+      restore();
+    }
+  });
+
+  test("reports running (not throws) when the process exists but is not signalable (EPERM)", () => {
+    writeFileSync(pidPath, "4321");
+    const restore = stubProcessKill(new Set(), new Set([4321]));
+    try {
+      expect(probeMonitoringWorker()).toEqual({ status: "running", pid: 4321 });
+    } finally {
+      restore();
+    }
+  });
+});
+
+describe("stopMonitoringWorkerProcess", () => {
+  test("is a no-op when the monitor is not running", () => {
+    expect(stopMonitoringWorkerProcess()).toEqual({ status: "not_running" });
+  });
+
+  test("signals a running monitor and reports its prior state", () => {
+    writeFileSync(pidPath, "4321");
+    const signalled: Array<[number, string | number | undefined]> = [];
+    const original = process.kill.bind(process);
+    process.kill = ((pid: number, signal?: string | number) => {
+      if ((signal ?? 0) === 0) return true; // liveness probe
+      signalled.push([pid, signal]);
+      return true;
+    }) as typeof process.kill;
+    try {
+      expect(stopMonitoringWorkerProcess()).toEqual({
+        status: "running",
+        pid: 4321,
+      });
+      expect(signalled).toEqual([[4321, "SIGTERM"]]);
+    } finally {
+      process.kill = original;
+    }
+  });
+});

package/src/monitoring/__tests__/sample-ring-buffer.test.ts

@@ -0,0 +1,96 @@
+/**
+ * Tests for the on-disk sample ring buffer.
+ *
+ * The buffer is the resource monitor's crash-durable record, so the contract
+ * that matters is: appends bound at capacity via rotation (not unbounded
+ * growth), reads return chronological order across the rotation boundary, and a
+ * half-written trailing line (as a SIGKILL would leave) is skipped rather than
+ * throwing.
+ */
+
+import { appendFileSync, mkdtempSync, readdirSync, rmSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+
+import { SampleRingBuffer } from "../sample-ring-buffer.js";
+
+let tmpDir: string;
+let bufPath: string;
+
+beforeEach(() => {
+  tmpDir = mkdtempSync(join(tmpdir(), "ring-buffer-test-"));
+  bufPath = join(tmpDir, "samples.jsonl");
+});
+
+afterEach(() => {
+  rmSync(tmpDir, { recursive: true, force: true });
+});
+
+describe("SampleRingBuffer", () => {
+  test("readRecent returns records oldest-first", () => {
+    const buf = new SampleRingBuffer<{ n: number }>(bufPath, 100);
+    for (let n = 0; n < 5; n++) buf.append({ n });
+    expect(buf.readRecent().map((r) => r.n)).toEqual([0, 1, 2, 3, 4]);
+    expect(buf.readLast()).toEqual({ n: 4 });
+  });
+
+  test("readRecent honours the limit, keeping the most recent", () => {
+    const buf = new SampleRingBuffer<{ n: number }>(bufPath, 100);
+    for (let n = 0; n < 10; n++) buf.append({ n });
+    expect(buf.readRecent(3).map((r) => r.n)).toEqual([7, 8, 9]);
+  });
+
+  test("rotates at capacity and keeps history across the boundary", () => {
+    // capacity 3: after 3 appends the active file rotates to <name>.1.
+    const buf = new SampleRingBuffer<{ n: number }>(bufPath, 3);
+    for (let n = 0; n < 5; n++) buf.append({ n });
+
+    // Two files exist at most: active + rotated.
+    const files = readdirSync(tmpDir).sort();
+    expect(files).toEqual(["samples.jsonl", "samples.jsonl.1"]);
+
+    // History spans the rotation: 0,1,2 in the rotated file, 3,4 in the active.
+    expect(buf.readRecent().map((r) => r.n)).toEqual([0, 1, 2, 3, 4]);
+    expect(buf.readLast()).toEqual({ n: 4 });
+  });
+
+  test("never keeps more than two files, bounding disk use", () => {
+    const buf = new SampleRingBuffer<{ n: number }>(bufPath, 2);
+    for (let n = 0; n < 20; n++) buf.append({ n });
+    expect(readdirSync(tmpDir).length).toBeLessThanOrEqual(2);
+    // The most recent record is always retained.
+    expect(buf.readLast()).toEqual({ n: 19 });
+  });
+
+  test("skips a malformed trailing line (as a SIGKILL mid-write would leave)", () => {
+    const buf = new SampleRingBuffer<{ n: number }>(bufPath, 100);
+    buf.append({ n: 1 });
+    // Simulate a half-written line with no trailing newline.
+    appendFileSync(bufPath, '{"n":2,"partia');
+    expect(buf.readRecent().map((r) => r.n)).toEqual([1]);
+    expect(buf.readLast()).toEqual({ n: 1 });
+  });
+
+  test("resumes line accounting from an existing file so it still rotates", () => {
+    const first = new SampleRingBuffer<{ n: number }>(bufPath, 3);
+    first.append({ n: 0 });
+    first.append({ n: 1 });
+
+    // A fresh instance over the same path must count the two existing lines and
+    // rotate on the next append rather than growing to 5.
+    const second = new SampleRingBuffer<{ n: number }>(bufPath, 3);
+    second.append({ n: 2 });
+    second.append({ n: 3 });
+
+    const files = readdirSync(tmpDir).sort();
+    expect(files).toEqual(["samples.jsonl", "samples.jsonl.1"]);
+    expect(second.readRecent().map((r) => r.n)).toEqual([0, 1, 2, 3]);
+  });
+
+  test("readLast returns null on an empty buffer", () => {
+    const buf = new SampleRingBuffer<{ n: number }>(bufPath, 10);
+    expect(buf.readLast()).toBeNull();
+    expect(buf.readRecent()).toEqual([]);
+  });
+});

package/src/monitoring/control.ts

@@ -0,0 +1,105 @@
+/**
+ * Shared control surface for the resource monitor *process* — the background OS
+ * process whose entry point is `worker.ts`.
+ *
+ * Both the `assistant monitoring` CLI and the daemon lifecycle (when
+ * `monitoring.enabled` is set) need to probe, spawn, and stop this process.
+ * The generic PID-file mechanics live in `util/worker-process.ts`; this module
+ * binds them to the monitor's PID path and entry point.
+ */
+
+import { getConfig } from "../config/loader.js";
+import { getLogger } from "../util/logger.js";
+import { getMonitoringPidPath } from "../util/platform.js";
+import {
+  probeWorkerPidFile,
+  spawnWorkerProcess,
+  type SpawnWorkerProcessOptions,
+  stopWorkerProcess,
+  WorkerProcessSpawnError,
+  type WorkerProcessStatus,
+} from "../util/worker-process.js";
+
+const log = getLogger("monitoring-control");
+
+/**
+ * Read the PID file and report liveness. A missing or malformed file reports
+ * not_running; a file pointing at a dead process is cleaned up and reported as
+ * not_running.
+ */
+export function probeMonitoringWorker(): WorkerProcessStatus {
+  return probeWorkerPidFile(getMonitoringPidPath());
+}
+
+export class MonitoringWorkerSpawnError extends WorkerProcessSpawnError {}
+
+/**
+ * Spawn the resource monitor as a background process. If a monitor is already
+ * running, returns its PID with `alreadyRunning: true` rather than spawning a
+ * second one. Throws {@link MonitoringWorkerSpawnError} if the child crashes
+ * during startup or never writes its PID file within the wait window.
+ */
+export async function spawnMonitoringWorkerProcess(
+  opts: SpawnWorkerProcessOptions = {},
+): Promise<{ pid: number; alreadyRunning: boolean }> {
+  try {
+    return await spawnWorkerProcess({
+      pidPath: getMonitoringPidPath(),
+      entry: new URL("./worker.ts", import.meta.url),
+      workerLabel: "Resource monitor",
+      options: opts,
+    });
+  } catch (err) {
+    if (err instanceof WorkerProcessSpawnError) {
+      throw new MonitoringWorkerSpawnError(err.message);
+    }
+    throw err;
+  }
+}
+
+/**
+ * Send SIGTERM to the monitor process if it is actually running. Returns the
+ * status observed before signalling. Only throws if `process.kill` itself fails
+ * (e.g. EPERM) — a not-running monitor is a no-op.
+ */
+export function stopMonitoringWorkerProcess(): WorkerProcessStatus {
+  return stopWorkerProcess(getMonitoringPidPath());
+}
+
+/**
+ * Daemon-lifecycle entry point: spawn the monitor as a child of the daemon
+ * (`detached: false`, so it appears in `assistant ps` and is torn down on
+ * shutdown) when `monitoring.enabled` is set. Fire-and-forget — a monitor
+ * failure must never block boot.
+ */
+export function startMonitoring(): void {
+  if (!getConfig().monitoring.enabled) {
+    return;
+  }
+  void spawnMonitoringWorkerProcess({ detached: false })
+    .then((r) =>
+      log.info(
+        { pid: r.pid, alreadyRunning: r.alreadyRunning },
+        "Resource monitor started at boot",
+      ),
+    )
+    .catch((err) =>
+      log.warn({ err }, "Failed to start resource monitor at boot"),
+    );
+}
+
+/**
+ * Daemon-lifecycle entry point: SIGTERM the monitor process if it is running.
+ * Keyed off live state rather than config: it may have been spawned at startup
+ * or out of band via `assistant monitoring start`. Never throws.
+ */
+export function stopMonitoring(): void {
+  try {
+    const status = stopMonitoringWorkerProcess();
+    if (status.status === "running") {
+      log.info({ pid: status.pid }, "Sent SIGTERM to resource monitor process");
+    }
+  } catch (err) {
+    log.warn({ err }, "Failed to stop resource monitor process (non-fatal)");
+  }
+}

package/src/monitoring/resource-sampler.ts

@@ -0,0 +1,258 @@
+/**
+ * The resource monitor's sampling loop.
+ *
+ * On a fast timer (default 250ms) it reads the container's own cgroup memory
+ * accounting and workspace-disk usage and appends a compact sample to an
+ * on-disk ring buffer. When memory crosses a configurable fraction of the
+ * container limit it also captures a one-off "high-memory" snapshot — cgroup
+ * stats plus the live process tree with per-process RSS — so a spike leaves
+ * behind a record of *what was running* when it happened.
+ *
+ * Everything here runs in the resource monitor's own OS process, off the
+ * assistant's main event loop, so it keeps sampling even while that loop is
+ * frozen mid-allocation and the samples survive an OOM SIGKILL.
+ */
+
+import {
+  mkdirSync,
+  readdirSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from "node:fs";
+import { join } from "node:path";
+
+import type { MonitoringConfig } from "../config/schemas/monitoring.js";
+import {
+  type ContainerMemoryEvents,
+  getContainerMemoryEvents,
+  getContainerMemoryLimitBytes,
+  getContainerMemoryPeakBytes,
+  getContainerMemoryUsageBytes,
+} from "../util/cgroup-memory.js";
+import { getDiskUsageInfo } from "../util/disk-usage.js";
+import { getLogger } from "../util/logger.js";
+import { getMonitoringDataDir } from "../util/platform.js";
+import { buildProcessTree, listProcesses } from "../util/process-tree.js";
+import { SampleRingBuffer } from "./sample-ring-buffer.js";
+
+const log = getLogger("resource-sampler");
+
+const SAMPLES_FILE = "samples.jsonl";
+const SNAPSHOTS_DIR = "snapshots";
+/** Cap on retained high-memory snapshots so forensics can't fill the volume. */
+const MAX_SNAPSHOTS = 20;
+/** Linux page size assumed when converting `/proc/<pid>/statm` pages to bytes. */
+const PAGE_SIZE_BYTES = 4096;
+
+export interface ResourceSampleMemory {
+  currentBytes: number;
+  limitBytes: number | null;
+  peakBytes: number | null;
+  /** current / limit, or null when the limit is unknown. */
+  ratio: number | null;
+}
+
+export interface ResourceSampleDisk {
+  path: string;
+  usedMb: number;
+  totalMb: number;
+  freeMb: number;
+}
+
+export interface ResourceSample {
+  ts: number;
+  memory: ResourceSampleMemory | null;
+  events: ContainerMemoryEvents | null;
+  disk: ResourceSampleDisk | null;
+}
+
+/** Take a single point-in-time sample of memory + disk. */
+export function takeSample(now: number): ResourceSample {
+  const currentBytes = getContainerMemoryUsageBytes();
+  const limitBytes = getContainerMemoryLimitBytes();
+  const memory: ResourceSampleMemory | null =
+    currentBytes != null
+      ? {
+          currentBytes,
+          limitBytes,
+          peakBytes: getContainerMemoryPeakBytes(),
+          ratio: limitBytes != null ? currentBytes / limitBytes : null,
+        }
+      : null;
+
+  const disk = getDiskUsageInfo();
+
+  return {
+    ts: now,
+    memory,
+    events: getContainerMemoryEvents(),
+    disk: disk
+      ? {
+          path: disk.path,
+          usedMb: disk.usedMb,
+          totalMb: disk.totalMb,
+          freeMb: disk.freeMb,
+        }
+      : null,
+  };
+}
+
+interface ProcessRss {
+  pid: number;
+  command: string;
+  rssBytes: number;
+}
+
+/**
+ * Best-effort per-process RSS read from `/proc/<pid>/statm` (resident pages,
+ * field 2). Returns the top `limit` processes by RSS, largest first. Empty when
+ * `/proc` is unavailable (e.g. macOS) — the snapshot's process *tree* still
+ * captures what was running in that case.
+ */
+function topProcessesByRss(limit: number): ProcessRss[] {
+  let pids: string[];
+  try {
+    pids = readdirSync("/proc").filter((e) => /^\d+$/.test(e));
+  } catch {
+    return [];
+  }
+
+  const rows: ProcessRss[] = [];
+  for (const entry of pids) {
+    const pid = Number(entry);
+    try {
+      const statm = readFileSync(`/proc/${pid}/statm`, "utf-8").trim();
+      const residentPages = parseInt(statm.split(/\s+/)[1], 10);
+      if (!Number.isFinite(residentPages)) continue;
+      const raw = readFileSync(`/proc/${pid}/cmdline`, "utf-8");
+      const command = raw.split("\0").filter(Boolean).join(" ") || `pid ${pid}`;
+      rows.push({ pid, command, rssBytes: residentPages * PAGE_SIZE_BYTES });
+    } catch {
+      // Process exited between readdir and read — skip.
+    }
+  }
+
+  rows.sort((a, b) => b.rssBytes - a.rssBytes);
+  return rows.slice(0, limit);
+}
+
+/**
+ * Capture a high-memory snapshot to the snapshots directory: the triggering
+ * sample, the cgroup memory.events counters, the top processes by RSS, and the
+ * full process tree of the container. Prunes to {@link MAX_SNAPSHOTS} newest.
+ */
+export async function writeHighMemSnapshot(
+  dataDir: string,
+  sample: ResourceSample,
+): Promise<void> {
+  const snapshotsDir = join(dataDir, SNAPSHOTS_DIR);
+  // The ring buffer only creates the data dir for samples.jsonl; the nested
+  // snapshots dir may not exist yet on the first threshold crossing.
+  mkdirSync(snapshotsDir, { recursive: true });
+
+  let tree: unknown = null;
+  try {
+    const procs = await listProcesses();
+    // pid 1 is the container init; its subtree is the whole container.
+    tree = buildProcessTree(procs, 1);
+  } catch (err) {
+    log.warn({ err }, "Failed to enumerate process tree for snapshot");
+  }
+
+  const snapshot = {
+    ts: sample.ts,
+    sample,
+    topProcessesByRss: topProcessesByRss(15),
+    processTree: tree,
+  };
+
+  const filename = `snapshot-${sample.ts}.json`;
+  writeFileSync(
+    join(snapshotsDir, filename),
+    JSON.stringify(snapshot, null, 2),
+  );
+  pruneSnapshots(snapshotsDir);
+  log.warn(
+    {
+      currentBytes: sample.memory?.currentBytes,
+      limitBytes: sample.memory?.limitBytes,
+      ratio: sample.memory?.ratio,
+      file: filename,
+    },
+    "Captured high-memory snapshot",
+  );
+}
+
+function pruneSnapshots(snapshotsDir: string): void {
+  let files: string[];
+  try {
+    files = readdirSync(snapshotsDir).filter(
+      (f) => f.startsWith("snapshot-") && f.endsWith(".json"),
+    );
+  } catch {
+    return;
+  }
+  if (files.length <= MAX_SNAPSHOTS) return;
+  // Filenames embed the millisecond timestamp, so lexical sort is chronological.
+  files.sort();
+  for (const stale of files.slice(0, files.length - MAX_SNAPSHOTS)) {
+    try {
+      rmSync(join(snapshotsDir, stale));
+    } catch {
+      // best-effort
+    }
+  }
+}
+
+export interface ResourceSamplerHandle {
+  stop: () => void;
+}
+
+/**
+ * Start the sampling loop. Returns a handle whose `stop()` clears the timer.
+ * The timer is intentionally *not* unref'd: in the standalone monitor process
+ * it is the keep-alive that holds the process open.
+ */
+export function startResourceSampler(
+  config: MonitoringConfig,
+  clock: () => number = Date.now,
+): ResourceSamplerHandle {
+  const dataDir = getMonitoringDataDir();
+  const buffer = new SampleRingBuffer<ResourceSample>(
+    join(dataDir, SAMPLES_FILE),
+    config.ringBufferSize,
+  );
+
+  let lastSnapshotAt = 0;
+
+  const tick = () => {
+    const now = clock();
+    let sample: ResourceSample;
+    try {
+      sample = takeSample(now);
+      buffer.append(sample);
+    } catch (err) {
+      log.warn({ err }, "Resource sample failed");
+      return;
+    }
+
+    const ratio = sample.memory?.ratio;
+    if (
+      ratio != null &&
+      ratio >= config.highMemThresholdRatio &&
+      now - lastSnapshotAt >= config.snapshotCooldownMs
+    ) {
+      lastSnapshotAt = now;
+      void writeHighMemSnapshot(dataDir, sample).catch((err) =>
+        log.warn({ err }, "Failed to write high-memory snapshot"),
+      );
+    }
+  };
+
+  const timer = setInterval(tick, config.sampleIntervalMs);
+
+  return {
+    stop: () => clearInterval(timer),
+  };
+}