harnery 0.45.2 → 0.46.0

package/src/lib/admission-history.ts

@@ -0,0 +1,267 @@
+// Append-only history of admission slot use. The queue files in admission.ts
+// describe only the present: a waiter or holder disappears the moment it is
+// admitted, released, or pruned. This module keeps one JSON line per finished
+// ticket so a later reader can answer how long work queued, behind what, and
+// how long each holder kept its slot.
+//
+// Layout: <history dir>/<resource>.jsonl, rotated to <resource>.1.jsonl once
+// it passes HISTORY_ROTATE_BYTES. Each line is a complete record appended in a
+// single write, so concurrent writers on one machine never interleave.
+//
+// Toolkit tier: this module must not import src/core (layering check).
+
+import {
+  appendFileSync,
+  mkdirSync,
+  readdirSync,
+  readFileSync,
+  renameSync,
+  statSync,
+} from "node:fs";
+import { homedir } from "node:os";
+import { basename, join } from "node:path";
+import { coordEnv } from "./env.ts";
+
+export const ADMISSION_HISTORY_SCHEMA_VERSION = 1;
+
+/** Rotate a resource's history once it passes this size; one older file is kept. */
+const HISTORY_ROTATE_BYTES = 8 * 1024 * 1024;
+
+/**
+ * How a ticket ended.
+ * - `released`: the holder ran and gave the slot back.
+ * - `timeout`: no slot freed before the waiter's deadline.
+ * - `abandoned`: the process died while waiting or holding; a peer pruned it.
+ * - `expired`: the entry outlived the queue TTL and a peer pruned it.
+ */
+export type AdmissionHistoryOutcome = "released" | "timeout" | "abandoned" | "expired";
+
+export interface AdmissionHistoryRecord {
+  v: typeof ADMISSION_HISTORY_SCHEMA_VERSION;
+  resource: string;
+  label: string;
+  pid: number;
+  /** Working directory of the process that queued, when known. */
+  cwd?: string;
+  capacity?: number;
+  outcome: AdmissionHistoryOutcome;
+  enqueued_at: string;
+  acquired_at?: string;
+  /** When the slot was released, the wait gave up, or a peer pruned the entry. */
+  ended_at: string;
+  /** Time spent in the queue. For an entry pruned while waiting, an upper bound. */
+  wait_ms: number;
+  /** Time the slot was held. For an entry pruned while holding, an upper bound. */
+  hold_ms?: number;
+  /** Labels of the holders when this ticket first had to wait. */
+  waited_behind?: string[];
+  /** Time spent deferring for memory pressure before joining the queue. */
+  pressure_wait_ms?: number;
+  exit_code?: number | null;
+  signal?: string | null;
+}
+
+/** Outcome details a caller can attach when it releases its slot. */
+export interface AdmissionReleaseResult {
+  exitCode?: number | null;
+  signal?: string | null;
+}
+
+/**
+ * Persistent machine-wide history directory used for the default admission
+ * queue root. The queues live in tmpdir and correctly vanish at reboot; the
+ * history is evidence and must survive one.
+ */
+export function defaultAdmissionHistoryDir(): string {
+  const stateHome = process.env.XDG_STATE_HOME || join(homedir(), ".local", "state");
+  return join(stateHome, "harnery", "admission-history");
+}
+
+/** Directory name of a machine-wide queue root (see admissionBaseDir). */
+const SHARED_QUEUE_DIRNAME = "harnery-admission";
+
+/**
+ * Where history for a given queue root is written, or null when disabled.
+ * HARNERY_ADMISSION_HISTORY=off disables recording; HARNERY_ADMISSION_HISTORY_DIR
+ * overrides the location. A machine-wide queue root, named `harnery-admission`
+ * under any temp directory, records to the persistent state directory. Any
+ * other root (an isolated test queue, for example) records beside itself as
+ * `<root>-history`, so isolated queues never write into the
+ * machine-wide history and the history directory is never listed as a resource.
+ */
+export function resolveAdmissionHistoryDir(queueDir: string): string | null {
+  const toggle = coordEnv("ADMISSION_HISTORY")?.trim().toLowerCase();
+  if (toggle === "off" || toggle === "0" || toggle === "false") return null;
+  const override = coordEnv("ADMISSION_HISTORY_DIR");
+  if (override) return override;
+  return basename(queueDir) === SHARED_QUEUE_DIRNAME
+    ? defaultAdmissionHistoryDir()
+    : `${queueDir}-history`;
+}
+
+function historyFile(dir: string, resource: string, generation = 0): string {
+  return join(dir, generation === 0 ? `${resource}.jsonl` : `${resource}.${generation}.jsonl`);
+}
+
+/**
+ * Append one record. History is diagnostic: a write failure never fails the
+ * caller's admission, so errors are swallowed.
+ */
+export function appendAdmissionHistory(dir: string | null, record: AdmissionHistoryRecord): void {
+  if (!dir) return;
+  try {
+    mkdirSync(dir, { recursive: true });
+    const path = historyFile(dir, record.resource);
+    try {
+      if (statSync(path).size > HISTORY_ROTATE_BYTES) {
+        renameSync(path, historyFile(dir, record.resource, 1));
+      }
+    } catch {
+      // no file yet, or a peer rotated it first
+    }
+    appendFileSync(path, `${JSON.stringify(record)}\n`);
+  } catch {
+    // diagnostic only
+  }
+}
+
+/** Resources with recorded history in a history directory. */
+export function listAdmissionHistoryResources(dir: string): string[] {
+  try {
+    const names = new Set<string>();
+    for (const name of readdirSync(dir)) {
+      const match = /^(.+?)(?:\.1)?\.jsonl$/.exec(name);
+      if (match?.[1]) names.add(match[1]);
+    }
+    return [...names].sort();
+  } catch {
+    return [];
+  }
+}
+
+function parseRecord(line: string): AdmissionHistoryRecord | undefined {
+  try {
+    const parsed: unknown = JSON.parse(line);
+    if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return undefined;
+    const record = parsed as Partial<AdmissionHistoryRecord>;
+    if (
+      typeof record.resource !== "string" ||
+      typeof record.outcome !== "string" ||
+      typeof record.enqueued_at !== "string" ||
+      typeof record.ended_at !== "string" ||
+      typeof record.wait_ms !== "number"
+    ) {
+      return undefined;
+    }
+    return record as AdmissionHistoryRecord;
+  } catch {
+    return undefined;
+  }
+}
+
+export interface AdmissionHistoryQuery {
+  resource?: string;
+  /** Keep records that ended at or after this epoch-ms. */
+  sinceMs?: number;
+  /** Keep records whose label contains this text (case-insensitive). */
+  labelContains?: string;
+}
+
+/** Read history records, oldest first, across the current and rotated files. */
+export function readAdmissionHistory(
+  dir: string | null,
+  query: AdmissionHistoryQuery = {},
+): AdmissionHistoryRecord[] {
+  if (!dir) return [];
+  const resources =
+    query.resource !== undefined ? [query.resource] : listAdmissionHistoryResources(dir);
+  const needle = query.labelContains?.toLowerCase();
+  const records: AdmissionHistoryRecord[] = [];
+  for (const resource of resources) {
+    for (const path of [historyFile(dir, resource, 1), historyFile(dir, resource)]) {
+      let text: string;
+      try {
+        text = readFileSync(path, "utf8");
+      } catch {
+        continue;
+      }
+      for (const line of text.split("\n")) {
+        if (!line) continue;
+        const record = parseRecord(line);
+        if (!record) continue;
+        if (query.sinceMs !== undefined && Date.parse(record.ended_at) < query.sinceMs) continue;
+        if (needle && !record.label.toLowerCase().includes(needle)) continue;
+        records.push(record);
+      }
+    }
+  }
+  return records.sort((a, b) => Date.parse(a.ended_at) - Date.parse(b.ended_at));
+}
+
+export interface AdmissionHistoryGroup {
+  resource: string;
+  label: string;
+  count: number;
+  /** Records that waited at least one second for a slot. */
+  queued: number;
+  wait_ms: { p50: number; p90: number; max: number; total: number };
+  /** Hold statistics over released records only. */
+  hold_ms: { p50: number; p90: number; max: number } | null;
+  outcomes: Partial<Record<AdmissionHistoryOutcome, number>>;
+}
+
+function percentile(sorted: number[], p: number): number {
+  if (sorted.length === 0) return 0;
+  return sorted[Math.min(sorted.length - 1, Math.ceil(p * sorted.length) - 1)] ?? 0;
+}
+
+const QUEUED_THRESHOLD_MS = 1_000;
+
+/**
+ * Group records by resource and label, most total wait first, so the work
+ * that spends the longest queueing leads the report.
+ */
+export function summarizeAdmissionHistory(
+  records: AdmissionHistoryRecord[],
+): AdmissionHistoryGroup[] {
+  const groups = new Map<string, AdmissionHistoryRecord[]>();
+  for (const record of records) {
+    const key = `${record.resource}\u0000${record.label}`;
+    const bucket = groups.get(key);
+    if (bucket) bucket.push(record);
+    else groups.set(key, [record]);
+  }
+  const summaries: AdmissionHistoryGroup[] = [];
+  for (const bucket of groups.values()) {
+    const first = bucket[0] as AdmissionHistoryRecord;
+    const waits = bucket.map((record) => record.wait_ms).sort((a, b) => a - b);
+    const holds = bucket
+      .filter((record) => record.outcome === "released" && typeof record.hold_ms === "number")
+      .map((record) => record.hold_ms as number)
+      .sort((a, b) => a - b);
+    const outcomes: Partial<Record<AdmissionHistoryOutcome, number>> = {};
+    for (const record of bucket) outcomes[record.outcome] = (outcomes[record.outcome] ?? 0) + 1;
+    summaries.push({
+      resource: first.resource,
+      label: first.label,
+      count: bucket.length,
+      queued: waits.filter((wait) => wait >= QUEUED_THRESHOLD_MS).length,
+      wait_ms: {
+        p50: percentile(waits, 0.5),
+        p90: percentile(waits, 0.9),
+        max: waits[waits.length - 1] ?? 0,
+        total: waits.reduce((sum, wait) => sum + wait, 0),
+      },
+      hold_ms:
+        holds.length > 0
+          ? {
+              p50: percentile(holds, 0.5),
+              p90: percentile(holds, 0.9),
+              max: holds[holds.length - 1] ?? 0,
+            }
+          : null,
+      outcomes,
+    });
+  }
+  return summaries.sort((a, b) => b.wait_ms.total - a.wait_ms.total);
+}

package/src/lib/admission.ts

@@ -18,6 +18,9 @@
 // file; process death leaves a dead-PID file for the next participant to
 // prune.
 //
+// Every ticket that ends (released, timed out, or pruned after its process
+// died) appends one record to the admission history (admission-history.ts).
+//
 // Toolkit tier: this module must not import src/core (layering check).
 
 import { randomUUID } from "node:crypto";
@@ -31,8 +34,17 @@ import {
 } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
+import {
+  ADMISSION_HISTORY_SCHEMA_VERSION,
+  type AdmissionHistoryOutcome,
+  type AdmissionReleaseResult,
+  appendAdmissionHistory,
+  resolveAdmissionHistoryDir,
+} from "./admission-history.ts";
 import { coordEnv } from "./env.ts";
 
+export type { AdmissionReleaseResult } from "./admission-history.ts";
+
 export interface AdmissionConfig {
   /** Base directory holding one subdirectory per resource. */
   dir: string;
@@ -44,6 +56,11 @@ export interface AdmissionConfig {
   pollMs?: number;
   /** Entries older than this are pruned regardless of PID (default 6h). */
   ttlMs?: number;
+  /**
+   * Directory for the slot-use history. Omitted: resolved from the queue
+   * directory (see resolveAdmissionHistoryDir). null: record nothing.
+   */
+  historyDir?: string | null;
 }
 
 export interface AdmissionEntry {
@@ -52,6 +69,12 @@ export interface AdmissionEntry {
   created_at: string;
   /** Set on holders once admitted. */
   acquired_at?: string;
+  /** Working directory of the queueing process. */
+  cwd?: string;
+  /** Capacity the queueing process asked for. */
+  capacity?: number;
+  /** Set on holders that had to wait: the holder labels they first waited behind. */
+  waited_behind?: string[];
 }
 
 export interface AdmissionWaitInfo {
@@ -67,6 +90,8 @@ export interface AdmissionAcquireOptions {
   timeoutMs?: number;
   /** Progress callback, invoked at most once per poll while waiting. */
   onWait?: (info: AdmissionWaitInfo) => void;
+  /** Time the caller already spent deferring for memory pressure, for the history. */
+  pressureWaitMs?: number;
 }
 
 export interface AdmissionHandle {
@@ -74,8 +99,11 @@ export interface AdmissionHandle {
   entry: string;
   /** Milliseconds spent waiting in the queue. */
   waitedMs: number;
-  /** Give the slot back. Safe to call more than once. */
-  release: () => void;
+  /**
+   * Give the slot back and record the slot use in the history. Safe to call
+   * more than once; only the first call records.
+   */
+  release: (result?: AdmissionReleaseResult) => void;
 }
 
 export class AdmissionTimeoutError extends Error {
@@ -143,7 +171,13 @@ interface LiveEntry {
 /** List live entries in lexical (= enqueue) order, pruning dead-PID, expired,
  * and torn files as a side effect. Every participant prunes, so a crashed
  * holder's slot frees as soon as anyone else looks. */
-function listLive(dir: string, ttlMs: number): LiveEntry[] {
+type PruneReason = Extract<AdmissionHistoryOutcome, "abandoned" | "expired">;
+
+function listLive(
+  dir: string,
+  ttlMs: number,
+  onPrune?: (entry: AdmissionEntry, reason: PruneReason) => void,
+): LiveEntry[] {
   let names: string[];
   try {
     names = readdirSync(dir)
@@ -167,6 +201,15 @@ function listLive(dir: string, ttlMs: number): LiveEntry[] {
             label: typeof record.label === "string" ? record.label : "",
             created_at: record.created_at,
             ...(typeof record.acquired_at === "string" ? { acquired_at: record.acquired_at } : {}),
+            ...(typeof record.cwd === "string" ? { cwd: record.cwd } : {}),
+            ...(typeof record.capacity === "number" ? { capacity: record.capacity } : {}),
+            ...(Array.isArray(record.waited_behind)
+              ? {
+                  waited_behind: record.waited_behind.filter(
+                    (label): label is string => typeof label === "string",
+                  ),
+                }
+              : {}),
           };
         }
       }
@@ -188,12 +231,16 @@ function listLive(dir: string, ttlMs: number): LiveEntry[] {
       continue;
     }
     const age = now - Date.parse(entry.created_at);
-    if (!pidAlive(entry.pid) || Number.isNaN(age) || age > ttlMs) {
+    const dead = !pidAlive(entry.pid);
+    if (dead || Number.isNaN(age) || age > ttlMs) {
+      let pruned = false;
       try {
         unlinkSync(path);
+        pruned = true;
       } catch {
-        // a peer pruned it first
+        // a peer pruned it first, and records it
       }
+      if (pruned) onPrune?.(entry, dead ? "abandoned" : "expired");
       continue;
     }
     live.push({ name, entry });
@@ -201,6 +248,42 @@ function listLive(dir: string, ttlMs: number): LiveEntry[] {
   return live;
 }
 
+function historyDirFor(config: Pick<AdmissionConfig, "dir" | "historyDir">): string | null {
+  return config.historyDir !== undefined
+    ? config.historyDir
+    : resolveAdmissionHistoryDir(config.dir);
+}
+
+/** Record an entry a participant pruned because its process died or it expired. */
+function recordPruned(
+  historyDir: string | null,
+  resource: string,
+  entry: AdmissionEntry,
+  reason: PruneReason,
+): void {
+  if (!historyDir) return;
+  const now = Date.now();
+  const enqueuedMs = Date.parse(entry.created_at);
+  const acquiredMs = entry.acquired_at ? Date.parse(entry.acquired_at) : Number.NaN;
+  appendAdmissionHistory(historyDir, {
+    v: ADMISSION_HISTORY_SCHEMA_VERSION,
+    resource,
+    label: entry.label,
+    pid: entry.pid,
+    ...(entry.cwd ? { cwd: entry.cwd } : {}),
+    ...(entry.capacity !== undefined ? { capacity: entry.capacity } : {}),
+    outcome: reason,
+    enqueued_at: entry.created_at,
+    ...(entry.acquired_at ? { acquired_at: entry.acquired_at } : {}),
+    ended_at: new Date(now).toISOString(),
+    wait_ms: Number.isNaN(enqueuedMs)
+      ? 0
+      : Math.max(0, (Number.isNaN(acquiredMs) ? now : acquiredMs) - enqueuedMs),
+    ...(Number.isNaN(acquiredMs) ? {} : { hold_ms: Math.max(0, now - acquiredMs) }),
+    ...(entry.waited_behind?.length ? { waited_behind: entry.waited_behind } : {}),
+  });
+}
+
 function sleep(ms: number): Promise<void> {
   return new Promise((resolvePromise) => setTimeout(resolvePromise, ms));
 }
@@ -224,11 +307,18 @@ export async function acquireAdmission(
   const tickets = ticketsDir(config);
   const held = heldDir(config);
 
+  const historyDir = historyDirFor(config);
+  const onPrune = (entry: AdmissionEntry, reason: PruneReason): void =>
+    recordPruned(historyDir, config.resource, entry, reason);
   const myEntry: AdmissionEntry = {
     pid: process.pid,
     label: options.label,
     created_at: new Date().toISOString(),
+    cwd: process.cwd(),
+    capacity: config.capacity,
   };
+  let waitedBehind: string[] | undefined;
+  const pressureWait = options.pressureWaitMs ? { pressure_wait_ms: options.pressureWaitMs } : {};
   const myName = `${String(Date.now()).padStart(13, "0")}-${process.pid}-${randomUUID().slice(0, 8)}.json`;
   writeEntry(tickets, myName, myEntry);
 
@@ -236,8 +326,8 @@ export async function acquireAdmission(
   const deadline = startedWaiting + timeoutMs;
   try {
     while (true) {
-      const liveHeld = listLive(held, ttlMs);
-      const liveTickets = listLive(tickets, ttlMs);
+      const liveHeld = listLive(held, ttlMs, onPrune);
+      const liveTickets = listLive(tickets, ttlMs, onPrune);
       const myIndex = liveTickets.findIndex((ticket) => ticket.name === myName);
       if (myIndex === -1) {
         // Pruned by a peer (should not happen while we are alive) or lost to
@@ -253,7 +343,7 @@ export async function acquireAdmission(
         // between two waiters' listings. Lexical order (= enqueue order)
         // decides who stays; the loser rejoins the queue with its original
         // priority because the filename keeps its enqueue timestamp.
-        const heldNow = listLive(held, ttlMs);
+        const heldNow = listLive(held, ttlMs, onPrune);
         if (heldNow.length > config.capacity) {
           const keepers = new Set(
             heldNow.slice(0, config.capacity).map((holderEntry) => holderEntry.name),
@@ -264,12 +354,14 @@ export async function acquireAdmission(
             continue;
           }
         }
-        writeEntry(held, myName, { ...myEntry, acquired_at: new Date().toISOString() });
+        const acquiredAt = new Date();
+        writeEntry(held, myName, { ...myEntry, acquired_at: acquiredAt.toISOString() });
+        const waitedMs = acquiredAt.getTime() - startedWaiting;
         let released = false;
         return {
           entry: myName,
-          waitedMs: Date.now() - startedWaiting,
-          release: () => {
+          waitedMs,
+          release: (result?: AdmissionReleaseResult) => {
             if (released) return;
             released = true;
             try {
@@ -277,14 +369,55 @@ export async function acquireAdmission(
             } catch {
               // already pruned
             }
+            const endedAt = new Date();
+            appendAdmissionHistory(historyDir, {
+              v: ADMISSION_HISTORY_SCHEMA_VERSION,
+              resource: config.resource,
+              label: myEntry.label,
+              pid: myEntry.pid,
+              ...(myEntry.cwd ? { cwd: myEntry.cwd } : {}),
+              capacity: config.capacity,
+              outcome: "released",
+              enqueued_at: myEntry.created_at,
+              acquired_at: acquiredAt.toISOString(),
+              ended_at: endedAt.toISOString(),
+              wait_ms: waitedMs,
+              hold_ms: endedAt.getTime() - acquiredAt.getTime(),
+              ...(waitedBehind?.length ? { waited_behind: waitedBehind } : {}),
+              ...pressureWait,
+              ...(result?.exitCode !== undefined ? { exit_code: result.exitCode } : {}),
+              ...(result?.signal !== undefined ? { signal: result.signal } : {}),
+            });
           },
         };
       }
+      if (waitedBehind === undefined) {
+        // Persist who this ticket waits behind, so the record survives even
+        // if this process dies in the queue and a peer prunes the ticket.
+        waitedBehind = liveHeld.map((holderEntry) => holderEntry.entry.label);
+        myEntry.waited_behind = waitedBehind;
+        writeEntry(tickets, myName, myEntry);
+      }
       options.onWait?.({
         position: myIndex + 1,
         holders: liveHeld.map((holderEntry) => holderEntry.entry),
       });
       if (Date.now() >= deadline) {
+        const endedAt = new Date();
+        appendAdmissionHistory(historyDir, {
+          v: ADMISSION_HISTORY_SCHEMA_VERSION,
+          resource: config.resource,
+          label: myEntry.label,
+          pid: myEntry.pid,
+          ...(myEntry.cwd ? { cwd: myEntry.cwd } : {}),
+          capacity: config.capacity,
+          outcome: "timeout",
+          enqueued_at: myEntry.created_at,
+          ended_at: endedAt.toISOString(),
+          wait_ms: endedAt.getTime() - startedWaiting,
+          ...(waitedBehind?.length ? { waited_behind: waitedBehind } : {}),
+          ...pressureWait,
+        });
         throw new AdmissionTimeoutError(
           config.resource,
           timeoutMs,
@@ -313,25 +446,90 @@ export interface AdmissionStatus {
 
 /** Snapshot the queue for one resource, pruning dead entries on the way. */
 export function admissionStatus(
-  config: Pick<AdmissionConfig, "dir" | "resource" | "ttlMs">,
+  config: Pick<AdmissionConfig, "dir" | "resource" | "ttlMs" | "historyDir">,
 ): AdmissionStatus {
   const ttlMs = config.ttlMs ?? DEFAULT_TTL_MS;
   const full: AdmissionConfig = { ...config, capacity: 1 };
+  const historyDir = historyDirFor(config);
+  const onPrune = (entry: AdmissionEntry, reason: PruneReason): void =>
+    recordPruned(historyDir, config.resource, entry, reason);
   return {
     resource: config.resource,
-    holders: listLive(heldDir(full), ttlMs).map((holderEntry) => holderEntry.entry),
-    waiters: listLive(ticketsDir(full), ttlMs).map((ticket) => ticket.entry),
+    holders: listLive(heldDir(full), ttlMs, onPrune).map((holderEntry) => holderEntry.entry),
+    waiters: listLive(ticketsDir(full), ttlMs, onPrune).map((ticket) => ticket.entry),
   };
 }
 
 /**
- * Base directory for admission queue state. tmpdir clears on reboot, which
- * is correct — stale queue state must not outlive the boot. Override with
- * HARNERY_ADMISSION_DIR. Host CLIs and harn commands must share this helper
- * so they join the same machine-wide queues.
+ * Base directory for admission queue state. The temp directory clears on
+ * reboot, which is correct: stale queue state must not outlive the boot.
+ * Override with HARNERY_ADMISSION_DIR. Host CLIs and harn commands must share
+ * this helper so they join the same machine-wide queues.
+ *
+ * On POSIX the default is the literal /tmp, not os.tmpdir(): agent sandboxes
+ * often set a private TMPDIR per session, and a queue under it would admit
+ * every session on its own.
  */
 export function admissionBaseDir(): string {
-  return coordEnv("ADMISSION_DIR") ?? join(tmpdir(), "harnery-admission");
+  return (
+    coordEnv("ADMISSION_DIR") ??
+    (process.platform === "win32" ? join(tmpdir(), "harnery-admission") : "/tmp/harnery-admission")
+  );
+}
+
+/**
+ * Linux memory pressure: the `full avg10` figure from /proc/pressure/memory,
+ * the share of the last ten seconds in which every runnable task stalled on
+ * memory. Undefined where pressure stall information is unavailable.
+ */
+export function memoryPressureFullAvg10(): number | undefined {
+  try {
+    const line = readFileSync("/proc/pressure/memory", "utf8")
+      .split("\n")
+      .find((candidate) => candidate.startsWith("full"));
+    const match = line ? /avg10=([0-9.]+)/.exec(line) : null;
+    return match?.[1] !== undefined ? Number(match[1]) : undefined;
+  } catch {
+    return undefined;
+  }
+}
+
+export interface PressureWaitOptions {
+  /** Defer while `full avg10` is above this value. */
+  threshold: number;
+  /** Give up deferring after this long and let the caller start anyway. */
+  maxWaitMs: number;
+  pollMs?: number;
+  /** Pressure reader; tests inject one. */
+  read?: () => number | undefined;
+  /** Called once when deferral starts and once if it gives up. */
+  onWait?: (event: { pressure: number; waitedMs: number; gaveUp: boolean }) => void;
+}
+
+/**
+ * Wait, holding no slot, while the machine is already stalling on memory, so
+ * a heavy job does not start into a storm. Never blocks forever: after
+ * maxWaitMs it returns and the caller starts anyway. Returns the time waited.
+ */
+export async function waitForMemoryPressure(options: PressureWaitOptions): Promise<number> {
+  const read = options.read ?? memoryPressureFullAvg10;
+  const pollMs = options.pollMs ?? 5_000;
+  const started = Date.now();
+  let announced = false;
+  while (true) {
+    const pressure = read();
+    if (pressure === undefined || pressure <= options.threshold) return Date.now() - started;
+    const waitedMs = Date.now() - started;
+    if (waitedMs + pollMs > options.maxWaitMs) {
+      options.onWait?.({ pressure, waitedMs, gaveUp: true });
+      return waitedMs;
+    }
+    if (!announced) {
+      announced = true;
+      options.onWait?.({ pressure, waitedMs, gaveUp: false });
+    }
+    await sleep(pollMs);
+  }
 }
 
 /** Resources present under an admission base directory. */