@saadnvd1/agent-os 0.3.616 → 0.3.624

package/lib/health/ci.ts

@@ -0,0 +1,90 @@
+/**
+ * How long CI takes, learned from what the orchestrator already sees. Each
+ * diff it reads every open PR's verdict; the first time a head is pending is
+ * when its run started (to within a diff), and the first time it isn't is
+ * when it settled. No extra gh call.
+ */
+
+import { db } from "../db";
+import type { SessionFacts } from "../orchestrator/facts";
+import { MIN, type CiFact } from "./types";
+
+const BASELINE_RUNS = 20;
+// A pending row older than this is a lost one, not a slow run.
+const LOST_AFTER_MS = 24 * 60 * MIN;
+
+export function observeCi(
+  workspaceId: string,
+  facts: Pick<SessionFacts, "project" | "task">[],
+  now: number
+): void {
+  db.transaction(() => {
+    for (const f of facts) {
+      const pr = f.task?.pr;
+      const project = f.project;
+      if (!pr || !project) continue;
+      const head = pr.head;
+      if (pr.state !== "OPEN" || !head) {
+        db.prepare(
+          `DELETE FROM ci_runs WHERE project_id = ? AND pr_number = ?
+           AND settled_at IS NULL`
+        ).run(project, pr.number);
+        continue;
+      }
+      if (pr.checks === "pending") {
+        // A newer head replaces the run still pending on an older one.
+        db.prepare(
+          `DELETE FROM ci_runs WHERE project_id = ? AND pr_number = ?
+           AND head != ? AND settled_at IS NULL`
+        ).run(project, pr.number, head);
+        db.prepare(
+          `INSERT OR IGNORE INTO ci_runs
+             (project_id, workspace_id, pr_number, head, pending_since)
+           VALUES (?, ?, ?, ?, ?)`
+        ).run(project, workspaceId, pr.number, head, now);
+      } else if (pr.checks === "pass" || pr.checks === "fail") {
+        db.prepare(
+          `UPDATE ci_runs SET settled_at = ?
+           WHERE project_id = ? AND pr_number = ? AND head = ?
+           AND settled_at IS NULL`
+        ).run(now, project, pr.number, head);
+      }
+    }
+  })();
+}
+
+export function pendingCi(now: number): CiFact[] {
+  const rows = db
+    .prepare(
+      `SELECT * FROM ci_runs WHERE settled_at IS NULL AND pending_since >= ?`
+    )
+    .all(now - LOST_AFTER_MS) as {
+    project_id: string;
+    workspace_id: string | null;
+    pr_number: number;
+    head: string;
+    pending_since: number;
+  }[];
+  return rows.map((r) => ({
+    projectId: r.project_id,
+    workspaceId: r.workspace_id,
+    prNumber: r.pr_number,
+    head: r.head,
+    pendingSince: r.pending_since,
+    baseline: baseline(r.project_id),
+  }));
+}
+
+export function baseline(projectId: string): number[] {
+  return (
+    db
+      .prepare(
+        `SELECT settled_at - pending_since AS ms FROM ci_runs
+         WHERE project_id = ? AND settled_at IS NOT NULL
+         ORDER BY settled_at DESC LIMIT ?`
+      )
+      .all(projectId, BASELINE_RUNS) as { ms: number }[]
+  )
+    .map((r) => r.ms)
+    .filter((ms) => ms > 0);
+}

package/lib/health/config.ts

@@ -0,0 +1,79 @@
+// The thresholds, all editable in Settings → Health. A setting holds a
+// number as text under health.<name>; anything missing or invalid is the
+// default, so a bad edit can't turn a check off by accident.
+
+import { getSetting, putSetting } from "../settings-store";
+
+export const HEALTH_DEFAULTS = {
+  loadRedMinutes: 10,
+  queueHeldMinutes: 60,
+  bgQuietMinutes: 20,
+  pieceDupOverlap: 0.4,
+  restartsMax: 2,
+  restartsWindowMinutes: 60,
+  ciSlowFallbackMinutes: 20,
+  armedStuckMinutes: 120,
+  serverErrorsMin: 20,
+  serverErrorsFactor: 5,
+  // The watchers stop waking past this share of the 5-hour usage window.
+  wakeUsageCeilingPct: 90,
+  // 0 turns the watchers off: anomalies still open and reach the chat.
+  watchersOn: 1,
+} as const;
+
+export type HealthKey = keyof typeof HEALTH_DEFAULTS;
+export type HealthConfig = Record<HealthKey, number>;
+
+export const HEALTH_LABELS: Record<HealthKey, string> = {
+  loadRedMinutes: "Machine load red for (minutes)",
+  queueHeldMinutes: "Queued task held for (minutes)",
+  bgQuietMinutes: "Background job quiet for (minutes)",
+  pieceDupOverlap: "Plan piece matches merged work (0 to 1 overlap)",
+  restartsMax: "Server restarts allowed per window",
+  restartsWindowMinutes: "Restart window (minutes)",
+  ciSlowFallbackMinutes: "CI pending, when no history yet (minutes)",
+  armedStuckMinutes: "PR armed without merging for (minutes)",
+  serverErrorsMin: "Server errors per 10 minutes, at least",
+  serverErrorsFactor: "Server errors, times the usual rate",
+  wakeUsageCeilingPct: "Stop waking watchers above usage (%)",
+  watchersOn: "Watchers on (1) or off (0)",
+};
+
+const settingKey = (k: HealthKey) => `health.${k}`;
+
+export function healthConfig(
+  read: (key: string) => string | null = getSetting
+): HealthConfig {
+  const out = {} as HealthConfig;
+  for (const k of Object.keys(HEALTH_DEFAULTS) as HealthKey[]) {
+    const raw = read(settingKey(k));
+    const n = raw === null || raw.trim() === "" ? NaN : Number(raw);
+    out[k] = Number.isFinite(n) && n >= 0 ? n : HEALTH_DEFAULTS[k];
+  }
+  return out;
+}
+
+// Stores a patch; null puts the default back. Returns what it refused.
+export function setHealthConfig(patch: Record<string, unknown>): {
+  errors: string[];
+} {
+  const errors: string[] = [];
+  for (const [k, v] of Object.entries(patch)) {
+    if (!(k in HEALTH_DEFAULTS)) {
+      errors.push(`Unknown setting ${k}`);
+      continue;
+    }
+    const key = k as HealthKey;
+    if (v === null) {
+      putSetting(settingKey(key), null);
+      continue;
+    }
+    const n = typeof v === "number" ? v : Number(v);
+    if (!Number.isFinite(n) || n < 0) {
+      errors.push(`${HEALTH_LABELS[key]} must be a number, 0 or more`);
+      continue;
+    }
+    putSetting(settingKey(key), String(n));
+  }
+  return { errors };
+}

package/lib/health/detectors.ts

@@ -0,0 +1,227 @@
+/**
+ * The checks. Each is a pure function of one snapshot (facts.ts) and the
+ * thresholds (config.ts): no I/O, no model, no tokens. A check that finds
+ * something returns a Trip; store.ts decides whether that's news.
+ */
+
+import type { HealthConfig } from "./config";
+import { duration } from "./view";
+import {
+  MIN,
+  type BgFact,
+  type HealthFacts,
+  type Severity,
+  type Trip,
+} from "./types";
+
+export { duration };
+
+const trip = (
+  t: Omit<Trip, "workspaceId" | "detail"> &
+    Partial<Pick<Trip, "workspaceId" | "detail">>
+): Trip => ({ workspaceId: null, detail: {}, ...t });
+
+// More than this many times the threshold reads as high.
+const HIGH_AFTER = 3;
+const severityOf = (age: number, limit: number): Severity =>
+  limit > 0 && age >= limit * HIGH_AFTER ? "high" : "warn";
+
+export function loadRed(f: HealthFacts, c: HealthConfig): Trip[] {
+  const limit = c.loadRedMinutes * MIN;
+  return f.machines
+    .filter((m) => m.redSince !== null && f.now - m.redSince >= limit)
+    .map((m) => {
+      const age = f.now - (m.redSince as number);
+      return trip({
+        check: "load-red",
+        subject: m.id,
+        severity: severityOf(age, limit),
+        line: `${m.label} load has been red for ${duration(age)}.`,
+        detail: { load1: m.load1 ?? null, cores: m.cores ?? null },
+      });
+    });
+}
+
+export function queueHeld(f: HealthFacts, c: HealthConfig): Trip[] {
+  const limit = c.queueHeldMinutes * MIN;
+  return f.queue
+    .filter((q) => f.now - q.createdAt >= limit)
+    .map((q) => {
+      const age = f.now - q.createdAt;
+      return trip({
+        check: "queue-held",
+        subject: q.id,
+        severity: severityOf(age, limit),
+        line: `Queued task "${q.name ?? q.id}" has been held ${duration(age)}: ${q.heldBy}.`,
+        workspaceId: q.workspaceId,
+        detail: { heldBy: q.heldBy },
+      });
+    });
+}
+
+const bgLine = (b: BgFact, quietMs: number) =>
+  b.outputAt === null
+    ? `Background job "${b.description}" in ${b.sessionName} has printed nothing in ${duration(quietMs)}.`
+    : `Background job "${b.description}" in ${b.sessionName} has been quiet ${duration(quietMs)}.`;
+
+export function bgQuiet(f: HealthFacts, c: HealthConfig): Trip[] {
+  const limit = c.bgQuietMinutes * MIN;
+  return f.bg
+    .map((b) => ({ b, quietMs: f.now - (b.outputAt ?? b.startedAt) }))
+    .filter(({ quietMs }) => quietMs >= limit)
+    .map(({ b, quietMs }) =>
+      trip({
+        check: "bg-quiet",
+        subject: `${b.sessionId}:${b.taskId}`,
+        severity: severityOf(quietMs, limit),
+        line: bgLine(b, quietMs),
+        workspaceId: b.workspaceId,
+        detail: {
+          sessionId: b.sessionId,
+          taskId: b.taskId,
+          neverPrinted: b.outputAt === null,
+        },
+      })
+    );
+}
+
+export function restarts(f: HealthFacts, c: HealthConfig): Trip[] {
+  const recent = f.boots.filter(
+    (t) => f.now - t <= c.restartsWindowMinutes * MIN
+  );
+  if (recent.length <= c.restartsMax) return [];
+  return [
+    trip({
+      check: "restarts",
+      subject: "server",
+      severity: "high",
+      line: `AgentOS restarted ${recent.length} times in the last ${c.restartsWindowMinutes} minutes.`,
+      detail: { boots: recent },
+    }),
+  ];
+}
+
+// The oldest a CI run may be and still be usual: twice the median, or the
+// slowest tenth plus five minutes, whichever is longer. With too little
+// history, the fallback.
+export function ciLimit(baseline: number[], fallbackMs: number): number {
+  if (baseline.length < 5) return fallbackMs;
+  const sorted = [...baseline].sort((a, b) => a - b);
+  const at = (p: number) =>
+    sorted[Math.min(sorted.length - 1, Math.floor(p * sorted.length))];
+  return Math.max(2 * at(0.5), at(0.9) + 5 * MIN);
+}
+
+export function ciSlow(f: HealthFacts, c: HealthConfig): Trip[] {
+  return f.ci
+    .map((r) => ({
+      r,
+      limit: ciLimit(r.baseline, c.ciSlowFallbackMinutes * MIN),
+    }))
+    .filter(({ r, limit }) => f.now - r.pendingSince > limit)
+    .map(({ r, limit }) => {
+      const age = f.now - r.pendingSince;
+      return trip({
+        check: "ci-slow",
+        subject: `${r.projectId}#${r.prNumber}:${r.head.slice(0, 8)}`,
+        severity: severityOf(age, limit),
+        line: `CI on PR #${r.prNumber} has been pending ${duration(age)}; it usually settles within ${duration(limit)}.`,
+        workspaceId: r.workspaceId,
+        detail: { prNumber: r.prNumber, head: r.head, limitMs: limit },
+      });
+    });
+}
+
+export function armedStuck(f: HealthFacts, c: HealthConfig): Trip[] {
+  const limit = c.armedStuckMinutes * MIN;
+  return f.armed
+    .filter((a) => f.now - a.armedAt >= limit)
+    .map((a) => {
+      const age = f.now - a.armedAt;
+      return trip({
+        check: "armed-stuck",
+        subject: a.sessionId,
+        severity: severityOf(age, limit),
+        line: `PR${a.prNumber ? ` #${a.prNumber}` : ""} of ${a.name} has been armed ${duration(age)} without merging.`,
+        workspaceId: a.workspaceId,
+        detail: { prNumber: a.prNumber },
+      });
+    });
+}
+
+export function serverErrors(f: HealthFacts, c: HealthConfig): Trip[] {
+  const limit = Math.max(
+    c.serverErrorsMin,
+    c.serverErrorsFactor * f.errors.baselineMean
+  );
+  if (f.errors.recent < limit) return [];
+  return [
+    trip({
+      check: "server-errors",
+      subject: "server",
+      severity: f.errors.recent >= limit * 2 ? "high" : "warn",
+      line: `${f.errors.recent} server errors in the last 10 minutes (usually ${Math.round(f.errors.baselineMean)}).`,
+      detail: { recent: f.errors.recent },
+    }),
+  ];
+}
+
+export function runDetectors(f: HealthFacts, c: HealthConfig): Trip[] {
+  return [
+    ...loadRed(f, c),
+    ...queueHeld(f, c),
+    ...bgQuiet(f, c),
+    ...restarts(f, c),
+    ...ciSlow(f, c),
+    ...armedStuck(f, c),
+    ...serverErrors(f, c),
+  ];
+}
+
+// --- piece-dup: raised where a piece is about to start, not on the tick ---
+
+const STOP = new Set(
+  "a an and the of to for in on with from by is are as at be it this that into add fix feat feature update make use new piece slice".split(
+    " "
+  )
+);
+
+export function words(text: string): Set<string> {
+  const out = new Set<string>();
+  for (const w of text.toLowerCase().match(/[a-z][a-z0-9]{2,}/g) ?? []) {
+    if (STOP.has(w)) continue;
+    out.add(w.replace(/(ing|ed|es|s)$/, ""));
+  }
+  return out;
+}
+
+// How much of the piece's words the merged work's words cover, 0 to 1.
+export function overlap(piece: string, merged: string): number {
+  const a = words(piece);
+  if (!a.size) return 0;
+  const b = words(merged);
+  let hit = 0;
+  for (const w of a) if (b.has(w)) hit++;
+  return hit / a.size;
+}
+
+export interface MergedWork {
+  sessionId: string;
+  prNumber: number | null;
+  text: string;
+}
+
+export function matchingMerge(
+  piece: { name: string; slice: string },
+  merged: MergedWork[],
+  threshold: number
+): { work: MergedWork; score: number } | null {
+  const text = `${piece.name} ${piece.slice}`;
+  let best: { work: MergedWork; score: number } | null = null;
+  for (const work of merged) {
+    const score = overlap(text, work.text);
+    if (score >= threshold && (!best || score > best.score))
+      best = { work, score };
+  }
+  return best;
+}

package/lib/health/errors.ts

@@ -0,0 +1,90 @@
+/**
+ * Server errors, counted. console.error calls go into ten-minute
+ * buckets (a day of them is the baseline), and the last lines
+ * are kept, redacted, for a watcher that asks to see them.
+ */
+
+import { redact } from "../models/secrets";
+import { MIN } from "./types";
+
+const BUCKET_MS = 10 * MIN;
+const BUCKETS = 144;
+const KEEP_LINES = 200;
+// Fewer full buckets than this is too little to call a baseline.
+const MIN_HISTORY = 6;
+
+const buckets = new Map<number, number>();
+const lines: { at: number; text: string }[] = [];
+let installed = false;
+
+const bucketOf = (t: number) => Math.floor(t / BUCKET_MS);
+
+export function countError(text: string, now = Date.now()): void {
+  const b = bucketOf(now);
+  buckets.set(b, (buckets.get(b) ?? 0) + 1);
+  for (const k of buckets.keys()) if (k <= b - BUCKETS) buckets.delete(k);
+  lines.push({ at: now, text: redact(text).slice(0, 400) });
+  if (lines.length > KEEP_LINES) lines.splice(0, lines.length - KEEP_LINES);
+}
+
+// Errors in the current and previous bucket, so "the last 10 minutes" never
+// reads as zero right after a bucket turns over; and the mean per bucket
+// before those.
+export function errorRate(now = Date.now()): {
+  recent: number;
+  baselineMean: number;
+} {
+  const b = bucketOf(now);
+  const recent = (buckets.get(b) ?? 0) + (buckets.get(b - 1) ?? 0);
+  const before = [...buckets.entries()].filter(([k]) => k < b - 1);
+  if (before.length < MIN_HISTORY) return { recent, baselineMean: 0 };
+  const sum = before.reduce((n, [, v]) => n + v, 0);
+  // Buckets with no errors aren't stored: the mean is over the whole span.
+  const span = BUCKETS - 2;
+  return { recent, baselineMean: sum / span };
+}
+
+export const recentErrorLines = (max = 30, since = Date.now() - 60 * MIN) =>
+  lines
+    .filter((l) => l.at >= since)
+    .slice(-max)
+    .map((l) => l.text);
+
+export function resetErrors(): void {
+  buckets.clear();
+  lines.length = 0;
+}
+
+function text(args: unknown[]): string {
+  return args
+    .map((a) =>
+      a instanceof Error
+        ? (a.stack ?? a.message)
+        : typeof a === "string"
+          ? a
+          : safeJson(a)
+    )
+    .join(" ");
+}
+
+function safeJson(v: unknown): string {
+  try {
+    return JSON.stringify(v) ?? String(v);
+  } catch {
+    return String(v);
+  }
+}
+
+export function installErrorCounter(): void {
+  if (installed) return;
+  installed = true;
+  const original = console.error.bind(console);
+  console.error = (...args: unknown[]) => {
+    try {
+      countError(text(args));
+    } catch {
+      // Counting must never break logging.
+    }
+    original(...args);
+  };
+}

package/lib/health/evidence.ts

@@ -0,0 +1,172 @@
+/**
+ * What a watcher is shown: the anomaly, a snapshot of its whole area (so it
+ * can name a neighbour no check was written for), and a bounded excerpt of
+ * the relevant logs. Built here, deterministically, so the model needs no
+ * shell, and everything a third party wrote is fenced as data.
+ */
+
+import { db } from "../db";
+import { taskOutputTail } from "../chat/task-output";
+import { itemsOfKind } from "../chat/store";
+import { untrusted } from "../orchestrator/untrusted";
+import { recentBoots } from "./boots";
+import { recentErrorLines } from "./errors";
+import { duration, ciLimit } from "./detectors";
+import { MIN, type AnomalyRow, type Area, type HealthFacts } from "./types";
+
+// About 5k tokens.
+export const EVIDENCE_CHARS = 18_000;
+const TAIL_CHARS = 3_000;
+
+export const PROBES = [
+  "server_log",
+  "task_output",
+  "queue_state",
+  "boots",
+] as const;
+export type Probe = (typeof PROBES)[number];
+export const isProbe = (v: unknown): v is Probe =>
+  typeof v === "string" && (PROBES as readonly string[]).includes(v);
+
+const ago = (now: number, t: number) => `${duration(Math.max(0, now - t))} ago`;
+
+function detailOf(row: AnomalyRow): Record<string, unknown> {
+  try {
+    return JSON.parse(row.detail) as Record<string, unknown>;
+  } catch {
+    return {};
+  }
+}
+
+function anomalySection(row: AnomalyRow, now: number): string {
+  return [
+    `Anomaly: ${row.check_id} on ${row.subject}`,
+    `Severity ${row.severity}, open ${duration(now - row.opened_at)}, seen ${row.hits} time(s).`,
+    `Detector said: ${row.line}`,
+  ].join("\n");
+}
+
+function schedulerSnapshot(f: HealthFacts): string {
+  const lines = ["Machines:"];
+  for (const m of f.machines)
+    lines.push(
+      `- ${m.label}: ${m.level ?? "unknown"}${m.redSince ? ` (red for ${duration(f.now - m.redSince)})` : ""}${m.load1 !== undefined ? `, load ${m.load1} on ${m.cores} cores` : ""}`
+    );
+  lines.push("Queued tasks:");
+  for (const q of f.queue.slice(0, 12))
+    lines.push(
+      `- ${q.name ?? q.id}: queued ${duration(f.now - q.createdAt)}, ${q.heldBy}`
+    );
+  if (!f.queue.length) lines.push("- none");
+  const running = db
+    .prepare(
+      `SELECT s.name, s.pause_state, s.plan_only, s.plan_phase, s.task_status
+       FROM sessions s WHERE s.task_status IN ('running', 'moving')
+         AND s.archived_at IS NULL AND s.quick_of IS NULL LIMIT 20`
+    )
+    .all() as {
+    name: string;
+    pause_state: string | null;
+    plan_only: number | null;
+    plan_phase: string | null;
+    task_status: string;
+  }[];
+  lines.push("Tasks holding or near a slot:");
+  for (const r of running)
+    lines.push(
+      `- ${r.name}: ${r.task_status}${r.pause_state === "paused" ? ", paused" : ""}${r.plan_only ? `, plan-only (${r.plan_phase ?? "planning"})` : ""}`
+    );
+  if (!running.length) lines.push("- none");
+  return lines.join("\n");
+}
+
+function pipelineSnapshot(f: HealthFacts): string {
+  const lines = ["Armed PRs:"];
+  for (const a of f.armed.slice(0, 10))
+    lines.push(
+      `- ${a.name}${a.prNumber ? ` #${a.prNumber}` : ""}: armed ${duration(f.now - a.armedAt)}`
+    );
+  if (!f.armed.length) lines.push("- none");
+  lines.push("CI pending:");
+  for (const c of f.ci.slice(0, 10))
+    lines.push(
+      `- PR #${c.prNumber}: pending ${duration(f.now - c.pendingSince)} (usual limit ${duration(ciLimit(c.baseline, 20 * MIN))}, ${c.baseline.length} past runs)`
+    );
+  if (!f.ci.length) lines.push("- none");
+  lines.push("Recent server starts:");
+  for (const b of recentBoots(6))
+    lines.push(
+      `- ${ago(f.now, b.started_at)}${b.head ? ` at ${b.head}` : ""}${b.prev_seen_at ? `; the previous run was last seen ${ago(f.now, b.prev_seen_at)}` : ""}`
+    );
+  lines.push(
+    `Server errors in the last 10 minutes: ${f.errors.recent} (usual ${Math.round(f.errors.baselineMean)}).`
+  );
+  return lines.join("\n");
+}
+
+function tasksSnapshot(f: HealthFacts): string {
+  const lines = ["Background jobs running longer than the quiet limit:"];
+  for (const b of f.bg.slice(0, 12))
+    lines.push(
+      `- "${b.description}" in ${b.sessionName}: started ${ago(f.now, b.startedAt)}, ${b.outputAt ? `last output ${ago(f.now, b.outputAt)}` : "no output yet"}`
+    );
+  if (!f.bg.length) lines.push("- none");
+  return lines.join("\n");
+}
+
+const SNAPSHOTS: Record<Area, (f: HealthFacts) => string> = {
+  scheduler: schedulerSnapshot,
+  pipeline: pipelineSnapshot,
+  tasks: tasksSnapshot,
+};
+
+// The output of the background job an anomaly names.
+function taskTail(row: AnomalyRow, chars: number): string | null {
+  const d = detailOf(row);
+  const sessionId = typeof d.sessionId === "string" ? d.sessionId : null;
+  const taskId = typeof d.taskId === "string" ? d.taskId : null;
+  if (!sessionId || !taskId) return null;
+  const item = itemsOfKind(sessionId, "task").find((t) => t.taskId === taskId);
+  const tail = taskOutputTail(taskId, item?.outputFile);
+  if (!tail?.text)
+    return "(the job has printed nothing, or its output file is gone)";
+  return tail.text.slice(-chars);
+}
+
+export function evidencePack(row: AnomalyRow, f: HealthFacts): string {
+  const parts = [anomalySection(row, f.now), SNAPSHOTS[row.area](f)];
+  const tail = row.check_id === "bg-quiet" ? taskTail(row, TAIL_CHARS) : null;
+  if (tail) parts.push(untrusted("background job output", tail));
+  if (row.check_id === "server-errors" || row.check_id === "restarts")
+    parts.push(
+      untrusted("server error log", recentErrorLines(15).join("\n") || "(none)")
+    );
+  return parts.join("\n\n").slice(0, EVIDENCE_CHARS);
+}
+
+// What a watcher asked to see next. Always bounded, always fenced.
+export function probe(name: Probe, row: AnomalyRow, f: HealthFacts): string {
+  const cap = 4_000;
+  switch (name) {
+    case "server_log":
+      return untrusted(
+        "server error log",
+        recentErrorLines(40).join("\n") || "(no errors recorded)"
+      ).slice(0, cap);
+    case "task_output":
+      return untrusted(
+        "background job output",
+        taskTail(row, cap) ?? "(this anomaly names no background job)"
+      );
+    case "queue_state":
+      return schedulerSnapshot(f).slice(0, cap);
+    case "boots":
+      return recentBoots(12)
+        .map(
+          (b) =>
+            `${ago(f.now, b.started_at)} head ${b.head ?? "?"}; previous run last seen ${b.prev_seen_at ? ago(f.now, b.prev_seen_at) : "unknown"}; background jobs then: ${untrusted("boot record", b.prev_bg ?? "[]")}`
+        )
+        .join("\n")
+        .slice(0, cap);
+  }
+}