@saadnvd1/agent-os 0.3.620 → 0.3.624

package/lib/health/store.ts

@@ -0,0 +1,212 @@
+/**
+ * Anomalies: one open row per check and subject. A trip that is already open
+ * only bumps it; one that comes back within REOPEN_MS of clearing reopens the
+ * same row instead of starting a second one (and so wakes nobody again);
+ * only a rise from warn to high is news while open.
+ */
+
+import { db } from "../db";
+import {
+  AREA_OF,
+  MIN,
+  type AnomalyRow,
+  type CheckId,
+  type Severity,
+  type Trip,
+  type Verdict,
+} from "./types";
+
+// Absent this long before it clears: a blip in a check isn't an end.
+export const CLEAR_AFTER_MS = 5 * MIN;
+export const REOPEN_MS = 30 * MIN;
+export const MUTE_MS = 24 * 60 * MIN;
+
+export interface RecordResult {
+  opened: AnomalyRow[];
+  raised: AnomalyRow[];
+  closed: AnomalyRow[];
+}
+
+const rowOf = (id: number) =>
+  db.prepare(`SELECT * FROM health_anomalies WHERE id = ?`).get(id) as
+    | AnomalyRow
+    | undefined;
+
+export function anomaly(id: number): AnomalyRow | null {
+  return rowOf(id) ?? null;
+}
+
+function record(trip: Trip, now: number, out: RecordResult): void {
+  const muted = db
+    .prepare(
+      `SELECT 1 FROM health_anomalies WHERE check_id = ? AND subject = ?
+       AND muted_until > ? LIMIT 1`
+    )
+    .get(trip.check, trip.subject, now);
+  if (muted) return;
+
+  const open = db
+    .prepare(
+      `SELECT * FROM health_anomalies WHERE check_id = ? AND subject = ?
+       AND closed_at IS NULL`
+    )
+    .get(trip.check, trip.subject) as AnomalyRow | undefined;
+  const detail = JSON.stringify(trip.detail);
+  if (open) {
+    const rises = open.severity === "warn" && trip.severity === "high";
+    db.prepare(
+      `UPDATE health_anomalies SET last_seen_at = ?, hits = hits + 1, line = ?,
+         detail = ?, severity = ? WHERE id = ?`
+    ).run(now, trip.line, detail, rises ? "high" : open.severity, open.id);
+    if (rises) out.raised.push(rowOf(open.id)!);
+    return;
+  }
+
+  const recent = db
+    .prepare(
+      `SELECT * FROM health_anomalies WHERE check_id = ? AND subject = ?
+       AND close_reason = 'cleared' AND closed_at >= ?
+       ORDER BY closed_at DESC LIMIT 1`
+    )
+    .get(trip.check, trip.subject, now - REOPEN_MS) as AnomalyRow | undefined;
+  if (recent) {
+    db.prepare(
+      `UPDATE health_anomalies SET closed_at = NULL, close_reason = NULL,
+         last_seen_at = ?, hits = hits + 1, line = ?, detail = ?,
+         severity = CASE WHEN severity = 'high' THEN 'high' ELSE ? END
+       WHERE id = ?`
+    ).run(now, trip.line, detail, trip.severity, recent.id);
+    const after = rowOf(recent.id)!;
+    if (after.severity === "high" && recent.severity === "warn")
+      out.raised.push(after);
+    return;
+  }
+
+  const info = db
+    .prepare(
+      `INSERT INTO health_anomalies
+         (check_id, subject, area, severity, line, detail, workspace_id,
+          opened_at, last_seen_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`
+    )
+    .run(
+      trip.check,
+      trip.subject,
+      AREA_OF[trip.check],
+      trip.severity,
+      trip.line,
+      detail,
+      trip.workspaceId,
+      now,
+      now
+    );
+  out.opened.push(rowOf(Number(info.lastInsertRowid))!);
+}
+
+// Records what the checks found. `evaluated` is the checks that ran, so an
+// anomaly of a check that didn't run (piece-dup, raised elsewhere) isn't
+// cleared for being absent.
+export function recordTrips(
+  trips: Trip[],
+  evaluated: readonly CheckId[],
+  now: number
+): RecordResult {
+  const out: RecordResult = { opened: [], raised: [], closed: [] };
+  db.transaction(() => {
+    for (const t of trips) record(t, now, out);
+    const open = db
+      .prepare(`SELECT * FROM health_anomalies WHERE closed_at IS NULL`)
+      .all() as AnomalyRow[];
+    for (const row of open) {
+      if (!evaluated.includes(row.check_id)) continue;
+      if (now - row.last_seen_at < CLEAR_AFTER_MS) continue;
+      if (row.last_seen_at === now) continue;
+      close(row.id, "cleared", now);
+      out.closed.push(rowOf(row.id)!);
+    }
+  })();
+  return out;
+}
+
+export function close(id: number, reason: string, now: number): void {
+  db.prepare(
+    `UPDATE health_anomalies SET closed_at = ?, close_reason = ?
+     WHERE id = ? AND closed_at IS NULL`
+  ).run(now, reason, id);
+}
+
+// A person's dismissal: closes it and mutes that subject for a day.
+export function dismiss(id: number, now = Date.now()): boolean {
+  const row = rowOf(id);
+  if (!row) return false;
+  db.prepare(
+    `UPDATE health_anomalies SET closed_at = COALESCE(closed_at, ?),
+       close_reason = COALESCE(close_reason, 'dismissed'), muted_until = ?
+     WHERE id = ?`
+  ).run(now, now + MUTE_MS, id);
+  return true;
+}
+
+export const openAnomalies = () =>
+  db
+    .prepare(
+      `SELECT * FROM health_anomalies WHERE closed_at IS NULL
+       ORDER BY CASE severity WHEN 'high' THEN 0 ELSE 1 END, opened_at`
+    )
+    .all() as AnomalyRow[];
+
+export const closedSince = (since: number, limit = 200) =>
+  db
+    .prepare(
+      `SELECT * FROM health_anomalies WHERE closed_at IS NOT NULL
+       AND closed_at >= ? ORDER BY closed_at DESC LIMIT ?`
+    )
+    .all(since, limit) as AnomalyRow[];
+
+export const allSince = (since: number) =>
+  db
+    .prepare(`SELECT * FROM health_anomalies WHERE opened_at >= ?`)
+    .all(since) as AnomalyRow[];
+
+// Whether a watcher should look: never judged, or it got worse since.
+export function needsWake(row: AnomalyRow): boolean {
+  if (row.closed_at !== null) return false;
+  if (row.woken_severity === null) return true;
+  return row.woken_severity === "warn" && row.severity === "high";
+}
+
+export function markWoken(id: number, severity: Severity, now: number): void {
+  db.prepare(
+    `UPDATE health_anomalies SET woken_severity = ?, woken_at = ? WHERE id = ?`
+  ).run(severity, now, id);
+}
+
+export function saveVerdict(
+  id: number,
+  verdict: Verdict,
+  tokens: { input: number; output: number } | null
+): void {
+  db.prepare(
+    `UPDATE health_anomalies SET verdict = ?,
+       tokens_in = COALESCE(tokens_in, 0) + ?,
+       tokens_out = COALESCE(tokens_out, 0) + ? WHERE id = ?`
+  ).run(JSON.stringify(verdict), tokens?.input ?? 0, tokens?.output ?? 0, id);
+}
+
+export const markDelivered = (id: number, now: number) =>
+  db
+    .prepare(`UPDATE health_anomalies SET delivered_at = ? WHERE id = ?`)
+    .run(now, id);
+
+export const markPhoned = (id: number, now: number) =>
+  db
+    .prepare(`UPDATE health_anomalies SET phoned_at = ? WHERE id = ?`)
+    .run(now, id);
+
+export const verdictOf = (row: Pick<AnomalyRow, "verdict">): Verdict | null => {
+  try {
+    return row.verdict ? (JSON.parse(row.verdict) as Verdict) : null;
+  } catch {
+    return null;
+  }
+};

package/lib/health/tick.ts

@@ -0,0 +1,149 @@
+/**
+ * The health tick: once a minute, on the orchestrator watcher's own tick,
+ * read one snapshot, run the checks, record what they found, and wake a
+ * watcher for each anomaly that is new or got worse. All of it but the wake
+ * is plain code; with nothing wrong, a tick costs no tokens.
+ */
+
+import { runClaude } from "../orchestrator/claude-cli";
+import { leaveHeartbeat } from "./boots";
+import { healthConfig, type HealthConfig } from "./config";
+import { runDetectors } from "./detectors";
+import { gatherFacts, liveBackgroundJobs } from "./facts";
+import { addNote } from "./notes";
+import { closeResolvedPieces } from "./pieces";
+import { deliver, deliverClosed, type RouteDeps, realRoute } from "./route";
+import {
+  markWoken,
+  needsWake,
+  openAnomalies,
+  recordTrips,
+  saveVerdict,
+  anomaly,
+} from "./store";
+import {
+  currentUsagePct,
+  unjudged,
+  wake,
+  type WakeResult,
+  type WatchDeps,
+} from "./watchers";
+import {
+  CHECKS,
+  MIN,
+  type AnomalyRow,
+  type CheckId,
+  type HealthFacts,
+} from "./types";
+
+export const EVERY_MS = MIN;
+// Watchers running at once; the rest wait for the next tick.
+export const MAX_WAKES = 2;
+// A wake that never came back (a restart cut it off) is given up on.
+export const WAKE_GIVE_UP_MS = 3 * MIN;
+
+// The checks the tick evaluates: piece-dup is raised where a piece starts.
+export const TICK_CHECKS: readonly CheckId[] = CHECKS.filter(
+  (c) => c !== "piece-dup"
+);
+
+export interface TickDeps {
+  config: () => HealthConfig;
+  facts: (now: number, c: HealthConfig) => HealthFacts;
+  watch: WatchDeps;
+  route: RouteDeps;
+  wake: typeof wake;
+}
+
+export const realTick: TickDeps = {
+  config: healthConfig,
+  facts: gatherFacts,
+  watch: { run: runClaude, usagePct: currentUsagePct },
+  route: realRoute,
+  wake,
+};
+
+const inFlight = new Set<number>();
+let last = 0;
+
+export function resetTick(): void {
+  last = 0;
+  inFlight.clear();
+}
+
+function finish(
+  row: AnomalyRow,
+  result: WakeResult,
+  now: number,
+  deps: TickDeps
+): void {
+  saveVerdict(row.id, result.verdict, result.tokens);
+  const fresh = anomaly(row.id);
+  if (!fresh) return;
+  // A verdict that says it's normal teaches the area what normal is.
+  if (!result.verdict.real && result.verdict.note)
+    addNote(row.area, result.verdict.note, "watcher", now);
+  deliver(fresh, result.verdict, now, deps.route);
+}
+
+async function wakeOne(
+  row: AnomalyRow,
+  facts: HealthFacts,
+  c: HealthConfig,
+  deps: TickDeps
+): Promise<void> {
+  inFlight.add(row.id);
+  try {
+    const result = await deps.wake(row, facts, c, deps.watch);
+    finish(row, result, Date.now(), deps);
+  } catch (error) {
+    console.error(`[health] watcher for #${row.id} failed:`, error);
+    finish(
+      row,
+      { verdict: unjudged(row, "the watcher failed"), tokens: null },
+      Date.now(),
+      deps
+    );
+  } finally {
+    inFlight.delete(row.id);
+  }
+}
+
+// Anomalies whose wake was cut off get their own words delivered.
+function giveUpStale(now: number, deps: TickDeps): void {
+  for (const row of openAnomalies()) {
+    if (row.woken_at === null || row.verdict !== null) continue;
+    if (inFlight.has(row.id) || now - row.woken_at < WAKE_GIVE_UP_MS) continue;
+    finish(
+      row,
+      { verdict: unjudged(row, "the watcher was cut off"), tokens: null },
+      now,
+      deps
+    );
+  }
+}
+
+// Returns once the synchronous part is done; wakes run on in the background.
+export function healthTick(
+  now = Date.now(),
+  deps: TickDeps = realTick
+): Promise<void>[] {
+  if (now - last < EVERY_MS) return [];
+  last = now;
+  const c = deps.config();
+  const facts = deps.facts(now, c);
+  const found = recordTrips(runDetectors(facts, c), TICK_CHECKS, now);
+  for (const row of found.closed) deliverClosed(row, now, deps.route);
+  closeResolvedPieces(now);
+  leaveHeartbeat(now, liveBackgroundJobs());
+  giveUpStale(now, deps);
+
+  const running: Promise<void>[] = [];
+  for (const row of openAnomalies()) {
+    if (!needsWake(row) || inFlight.has(row.id)) continue;
+    if (inFlight.size >= MAX_WAKES) break;
+    markWoken(row.id, row.severity, now);
+    running.push(wakeOne(row, facts, c, deps));
+  }
+  return running;
+}

package/lib/health/types.ts

@@ -0,0 +1,148 @@
+// What the health watcher knows and says. Plain data, no I/O: the detectors
+// are pure functions of HealthFacts, and a trip is what one returns.
+
+export const AREAS = ["scheduler", "pipeline", "tasks"] as const;
+export type Area = (typeof AREAS)[number];
+
+export const AREA_LABELS: Record<Area, string> = {
+  scheduler: "Scheduler",
+  pipeline: "Pipeline",
+  tasks: "Tasks",
+};
+
+export const CHECKS = [
+  "load-red",
+  "queue-held",
+  "bg-quiet",
+  "piece-dup",
+  "restarts",
+  "ci-slow",
+  "armed-stuck",
+  "server-errors",
+] as const;
+export type CheckId = (typeof CHECKS)[number];
+
+export const AREA_OF: Record<CheckId, Area> = {
+  "load-red": "scheduler",
+  "queue-held": "scheduler",
+  "bg-quiet": "tasks",
+  "piece-dup": "pipeline",
+  restarts: "pipeline",
+  "ci-slow": "pipeline",
+  "armed-stuck": "pipeline",
+  "server-errors": "pipeline",
+};
+
+// High can only be reached from warn while an anomaly is open.
+export type Severity = "warn" | "high";
+
+export interface Trip {
+  check: CheckId;
+  // What it is about: a machine, queue row, session and task, PR, piece or
+  // "server". One open anomaly per check and subject.
+  subject: string;
+  severity: Severity;
+  line: string;
+  detail: Record<string, unknown>;
+  workspaceId: string | null;
+}
+
+export interface MachineFact {
+  id: string;
+  label: string;
+  level: "green" | "amber" | "red" | null;
+  // When it turned red, if it still is.
+  redSince: number | null;
+  load1?: number;
+  cores?: number;
+}
+
+export interface QueueFact {
+  id: string;
+  name: string | null;
+  projectId: string;
+  workspaceId: string | null;
+  createdAt: number;
+  // Why it hasn't started, in words.
+  heldBy: string;
+}
+
+export interface BgFact {
+  sessionId: string;
+  sessionName: string;
+  workspaceId: string | null;
+  taskId: string;
+  description: string;
+  startedAt: number;
+  // Null when the task has printed nothing at all.
+  outputAt: number | null;
+}
+
+export interface ArmedFact {
+  sessionId: string;
+  name: string;
+  workspaceId: string | null;
+  armedAt: number;
+  prNumber: number | null;
+}
+
+export interface CiFact {
+  projectId: string;
+  workspaceId: string | null;
+  prNumber: number;
+  head: string;
+  pendingSince: number;
+  // Settled durations of this project's recent runs, ms.
+  baseline: number[];
+}
+
+export interface HealthFacts {
+  now: number;
+  machines: MachineFact[];
+  queue: QueueFact[];
+  bg: BgFact[];
+  armed: ArmedFact[];
+  ci: CiFact[];
+  // Server start times inside the restart window.
+  boots: number[];
+  errors: { recent: number; baselineMean: number };
+}
+
+export interface Verdict {
+  real: boolean;
+  cause: string;
+  action: string;
+  saadOnly: boolean;
+  // Said by the model, so shown as its word, never as a fact.
+  probe?: string | null;
+  note?: string | null;
+  // Set when no model judged it, and why.
+  unjudged?: boolean;
+  why?: string | null;
+}
+
+export interface AnomalyRow {
+  id: number;
+  check_id: CheckId;
+  subject: string;
+  area: Area;
+  severity: Severity;
+  line: string;
+  detail: string;
+  workspace_id: string | null;
+  opened_at: number;
+  last_seen_at: number;
+  closed_at: number | null;
+  close_reason: string | null;
+  muted_until: number | null;
+  hits: number;
+  woken_severity: Severity | null;
+  woken_at: number | null;
+  verdict: string | null;
+  tokens_in: number | null;
+  tokens_out: number | null;
+  delivered_at: number | null;
+  phoned_at: number | null;
+}
+
+export const MIN = 60 * 1000;

package/lib/health/view.ts

@@ -0,0 +1,120 @@
+// What the UI gets of an anomaly. Plain data, no Node imports: the panel and
+// the routes share it.
+
+import type { AnomalyRow, Area, CheckId, Severity, Verdict } from "./types";
+
+// "11m", "2h 5m".
+export function duration(ms: number): string {
+  const m = Math.round(ms / 60_000);
+  return m >= 60 ? `${Math.floor(m / 60)}h ${m % 60}m` : `${m}m`;
+}
+
+export interface AnomalyView {
+  id: number;
+  check: CheckId;
+  area: Area;
+  severity: Severity;
+  subject: string;
+  line: string;
+  openedAt: number;
+  closedAt: number | null;
+  closeReason: string | null;
+  hits: number;
+  verdict: Verdict | null;
+  // A watcher was woken and hasn't answered yet.
+  judging: boolean;
+  tokens: number;
+  needsYou: boolean;
+  phoned: boolean;
+}
+
+export interface CheckSummary {
+  check: CheckId;
+  opened: number;
+  real: number;
+  medianOpenMs: number | null;
+  wakes: number;
+  tokens: number;
+}
+
+export interface HealthView {
+  // When this was read, so "open so far" is computed from the server's clock.
+  now: number;
+  open: AnomalyView[];
+  history: AnomalyView[];
+  summary: CheckSummary[];
+  needsYou: number;
+}
+
+const parse = (raw: string | null): Verdict | null => {
+  try {
+    return raw ? (JSON.parse(raw) as Verdict) : null;
+  } catch {
+    return null;
+  }
+};
+
+export function toView(row: AnomalyRow): AnomalyView {
+  const verdict = parse(row.verdict);
+  return {
+    id: row.id,
+    check: row.check_id,
+    area: row.area,
+    severity: row.severity,
+    subject: row.subject,
+    line: row.line,
+    openedAt: row.opened_at,
+    closedAt: row.closed_at,
+    closeReason: row.close_reason,
+    hits: row.hits,
+    verdict,
+    judging:
+      row.woken_at !== null && verdict === null && row.closed_at === null,
+    tokens: (row.tokens_in ?? 0) + (row.tokens_out ?? 0),
+    needsYou: verdict?.real === true && verdict.saadOnly === true,
+    phoned: row.phoned_at !== null,
+  };
+}
+
+const median = (xs: number[]) => {
+  if (!xs.length) return null;
+  const s = [...xs].sort((a, b) => a - b);
+  return s[Math.floor(s.length / 2)];
+};
+
+export function summarize(rows: AnomalyRow[]): CheckSummary[] {
+  const by = new Map<CheckId, AnomalyRow[]>();
+  for (const r of rows) by.set(r.check_id, [...(by.get(r.check_id) ?? []), r]);
+  return [...by.entries()]
+    .map(([check, rs]) => ({
+      check,
+      opened: rs.length,
+      real: rs.filter((r) => parse(r.verdict)?.real === true).length,
+      medianOpenMs: median(
+        rs
+          .filter((r) => r.closed_at !== null)
+          .map((r) => (r.closed_at as number) - r.opened_at)
+      ),
+      wakes: rs.filter((r) => r.woken_at !== null).length,
+      tokens: rs.reduce(
+        (n, r) => n + (r.tokens_in ?? 0) + (r.tokens_out ?? 0),
+        0
+      ),
+    }))
+    .sort((a, b) => b.opened - a.opened);
+}
+
+export function healthView(
+  open: AnomalyRow[],
+  closed: AnomalyRow[],
+  week: AnomalyRow[]
+): HealthView {
+  const views = open.map(toView);
+  return {
+    now: Date.now(),
+    open: views,
+    history: closed.map(toView),
+    summary: summarize(week),
+    needsYou: views.filter((v) => v.needsYou).length,
+  };
+}