@saadnvd1/agent-os 0.3.616 → 0.3.624

package/lib/health/facts.ts

@@ -0,0 +1,159 @@
+/**
+ * One snapshot of what the detectors need, read cheaply: database rows, the
+ * load monitor's last sample, a file's mtime. Nothing here calls gh, a
+ * model or a process listing, because it runs every minute.
+ */
+
+import { db } from "../db";
+import { itemsOfKind } from "../chat/store";
+import { taskOutputTail } from "../chat/task-output";
+import { runningWorkers } from "../chat/worker/client";
+import { localFacts } from "../load/machines";
+import { listQueue } from "../tasks/queue";
+import { pendingCi } from "./ci";
+import { bootsSince } from "./boots";
+import { errorRate } from "./errors";
+import type { HealthConfig } from "./config";
+import {
+  MIN,
+  type ArmedFact,
+  type BgFact,
+  type HealthFacts,
+  type MachineFact,
+  type QueueFact,
+} from "./types";
+
+const sqliteMs = (v: string | null) =>
+  v ? Date.parse(v.includes("T") ? v : `${v.replace(" ", "T")}Z`) || 0 : 0;
+
+// When each machine turned red, kept while it stays red. A restart forgets
+// it, which only delays a trip by the threshold.
+const redSince = new Map<string, number>();
+
+export function trackRed(
+  id: string,
+  level: MachineFact["level"],
+  now: number
+): number | null {
+  if (level !== "red") {
+    redSince.delete(id);
+    return null;
+  }
+  if (!redSince.has(id)) redSince.set(id, now);
+  return redSince.get(id) ?? null;
+}
+
+export const resetRedTracking = () => redSince.clear();
+
+function machines(now: number): MachineFact[] {
+  const l = localFacts();
+  return [
+    {
+      id: "local",
+      label: "This machine",
+      level: l.level,
+      redSince: trackRed("local", l.level, now),
+      load1: l.load1,
+      cores: l.cores,
+    },
+  ];
+}
+
+function queue(): QueueFact[] {
+  return listQueue()
+    .filter((q) => q.status === "queued")
+    .map((q) => ({
+      id: q.id,
+      name: q.name,
+      projectId: q.projectId,
+      workspaceId: q.workspaceId,
+      createdAt: sqliteMs(q.createdAt),
+      heldBy:
+        q.note ??
+        (q.after ? `waiting for ${q.after}` : "waiting for a free slot"),
+    }));
+}
+
+// Background jobs still running in a chat whose agent is alive, that have
+// been going long enough to be worth a look at their output.
+function backgroundJobs(now: number, c: HealthConfig): BgFact[] {
+  const live = new Set(runningWorkers());
+  if (!live.size) return [];
+  const sessions = db
+    .prepare(
+      `SELECT s.id, s.name, p.workspace_id FROM sessions s
+       LEFT JOIN projects p ON p.id = s.project_id
+       WHERE s.archived_at IS NULL AND s.view = 'chat'`
+    )
+    .all() as { id: string; name: string; workspace_id: string | null }[];
+  const out: BgFact[] = [];
+  for (const s of sessions) {
+    if (!live.has(s.id)) continue;
+    for (const t of itemsOfKind(s.id, "task")) {
+      if (t.status !== "running" || t.ambient) continue;
+      if (now - t.createdAt < c.bgQuietMinutes * MIN) continue;
+      const tail = taskOutputTail(t.taskId, t.outputFile);
+      out.push({
+        sessionId: s.id,
+        sessionName: s.name,
+        workspaceId: s.workspace_id,
+        taskId: t.taskId,
+        description: t.description,
+        startedAt: t.createdAt,
+        outputAt: tail?.updatedAt ?? null,
+      });
+    }
+  }
+  return out;
+}
+
+// Every running background job, for the heartbeat a restart reads back.
+export function liveBackgroundJobs(): string[] {
+  const live = new Set(runningWorkers());
+  const out: string[] = [];
+  for (const id of live) {
+    for (const t of itemsOfKind(id, "task"))
+      if (t.status === "running" && !t.ambient) out.push(t.description);
+  }
+  return out;
+}
+
+function armed(): ArmedFact[] {
+  return (
+    db
+      .prepare(
+        `SELECT s.id, s.name, s.merge_armed_at, s.pr_number, p.workspace_id
+         FROM sessions s LEFT JOIN projects p ON p.id = s.project_id
+         WHERE s.merge_armed_at IS NOT NULL AND s.archived_at IS NULL
+           AND COALESCE(s.task_status, 'running') = 'running'`
+      )
+      .all() as {
+      id: string;
+      name: string;
+      merge_armed_at: string;
+      pr_number: number | null;
+      workspace_id: string | null;
+    }[]
+  ).map((r) => ({
+    sessionId: r.id,
+    name: r.name,
+    workspaceId: r.workspace_id,
+    armedAt: sqliteMs(r.merge_armed_at),
+    prNumber: r.pr_number,
+  }));
+}
+
+export function gatherFacts(now: number, c: HealthConfig): HealthFacts {
+  return {
+    now,
+    machines: machines(now),
+    queue: queue(),
+    bg: backgroundJobs(now, c),
+    armed: armed(),
+    ci: pendingCi(now),
+    boots: bootsSince(now - c.restartsWindowMinutes * MIN).map(
+      (b) => b.started_at
+    ),
+    errors: errorRate(now),
+  };
+}

package/lib/health/fixtures.ts

@@ -0,0 +1,30 @@
+import { HEALTH_DEFAULTS, type HealthConfig } from "./config";
+import type { HealthFacts, Trip } from "./types";
+
+export const NOW = Date.parse("2026-10-14T18:00:00Z");
+export const config = (over: Partial<HealthConfig> = {}): HealthConfig => ({
+  ...HEALTH_DEFAULTS,
+  ...over,
+});
+
+export const facts = (over: Partial<HealthFacts> = {}): HealthFacts => ({
+  now: NOW,
+  machines: [],
+  queue: [],
+  bg: [],
+  armed: [],
+  ci: [],
+  boots: [],
+  errors: { recent: 0, baselineMean: 0 },
+  ...over,
+});
+
+export const trip = (over: Partial<Trip> = {}): Trip => ({
+  check: "load-red",
+  subject: "local",
+  severity: "warn",
+  line: "This machine load has been red for 11m.",
+  detail: {},
+  workspaceId: null,
+  ...over,
+});

package/lib/health/notes.ts

@@ -0,0 +1,76 @@
+/**
+ * What normal looks like, per area: short lines a watcher reads before it
+ * judges. Seeds are generic and ship with the code; anything machine- or
+ * person-specific is learned at run time and lives only in the local
+ * database, where Saad can edit or delete it.
+ */
+
+import { db } from "../db";
+import { AREAS, type Area } from "./types";
+
+export const SEED_NOTES: Record<Area, string[]> = {
+  scheduler: [
+    "Load spikes while a task runs a simulator, a full build or a test suite, and is back to normal within minutes.",
+    "A queued task waits while the load gate is closed or the workspace's running-task slots are full; plan-only tasks never hold a slot.",
+    "A paused task gives its slot back.",
+  ],
+  pipeline: [
+    "CI duration is per repository; a run slightly over its usual time is normal, one at several times it usually is not.",
+    "An armed PR waits for CI, the independent review and the scope check; an armed PR whose gate keeps failing is waiting for its task, not stuck.",
+    "A restart during a deploy is expected; three or more within an hour is not.",
+  ],
+  tasks: [
+    "Builds and test suites can print nothing for a few minutes; a job quiet for much longer, or that never printed anything, deserves a look at whether its process is still running.",
+    "A background monitor waits for a marker line; if the job it watches failed, the marker never comes.",
+  ],
+};
+
+export interface NoteRow {
+  id: number;
+  area: Area;
+  line: string;
+  source: "watcher" | "saad";
+  created_at: number;
+}
+
+export const MAX_NOTE = 200;
+const SHOWN = 40;
+
+export function addNote(
+  area: Area,
+  line: string,
+  source: NoteRow["source"],
+  now = Date.now()
+): number | null {
+  const text = line.replace(/\s+/g, " ").trim().slice(0, MAX_NOTE);
+  if (!text || !AREAS.includes(area)) return null;
+  const same = db
+    .prepare(`SELECT 1 FROM watcher_notes WHERE area = ? AND line = ?`)
+    .get(area, text);
+  if (same) return null;
+  const info = db
+    .prepare(
+      `INSERT INTO watcher_notes (area, line, source, created_at)
+       VALUES (?, ?, ?, ?)`
+    )
+    .run(area, text, source, now);
+  return Number(info.lastInsertRowid);
+}
+
+export const listNotes = (area?: Area) =>
+  (area
+    ? db
+        .prepare(`SELECT * FROM watcher_notes WHERE area = ? ORDER BY id`)
+        .all(area)
+    : db.prepare(`SELECT * FROM watcher_notes ORDER BY id`).all()) as NoteRow[];
+
+export const deleteNote = (id: number): boolean =>
+  db.prepare(`DELETE FROM watcher_notes WHERE id = ?`).run(id).changes > 0;
+
+// The seeds, then the newest learned lines.
+export function notesFor(area: Area): string[] {
+  const learned = listNotes(area)
+    .slice(-SHOWN)
+    .map((n) => n.line);
+  return [...SEED_NOTES[area], ...learned];
+}

package/lib/health/pieces.ts

@@ -0,0 +1,120 @@
+/**
+ * piece-dup: a plan piece about to start when matching work has already
+ * merged. The release guard (lib/plans/release.ts) only knows PRs from the
+ * same feature; this reads every merged task in the piece's project since
+ * the feature began, so work that merged through a separately started task
+ * is seen. Report only: the piece still starts.
+ */
+
+import { db } from "../db";
+import { close, openAnomalies, recordTrips } from "./store";
+import { healthConfig, type HealthConfig } from "./config";
+import { matchingMerge, type MergedWork } from "./detectors";
+import type { Trip } from "./types";
+
+interface PieceRef {
+  id: number;
+  name: string;
+  slice: string;
+  project_id: string;
+}
+
+interface FeatureRef {
+  id: string;
+  name: string;
+  workspace_id: string;
+  plan_session_id: string;
+  created_at: string;
+}
+
+export function mergedSince(
+  feature: FeatureRef,
+  piece: PieceRef,
+  ownSessionIds: string[]
+): MergedWork[] {
+  const own = new Set([feature.plan_session_id, ...ownSessionIds]);
+  const since = feature.created_at.includes("T")
+    ? feature.created_at
+    : `${feature.created_at.replace(" ", "T")}Z`;
+  return (
+    db
+      .prepare(
+        `SELECT id, name, pr_number, task_prompt, updated_at FROM sessions
+         WHERE project_id = ? AND task_status = 'merged'
+           AND updated_at >= datetime(?)`
+      )
+      .all(piece.project_id, since) as {
+      id: string;
+      name: string;
+      pr_number: number | null;
+      task_prompt: string | null;
+    }[]
+  )
+    .filter((s) => !own.has(s.id))
+    .map((s) => ({
+      sessionId: s.id,
+      prNumber: s.pr_number,
+      text: `${s.name} ${(s.task_prompt ?? "").slice(0, 600)}`,
+    }));
+}
+
+export function duplicateTrip(
+  feature: FeatureRef,
+  piece: PieceRef,
+  merged: MergedWork[],
+  c: HealthConfig
+): Trip | null {
+  const hit = matchingMerge(piece, merged, c.pieceDupOverlap);
+  if (!hit) return null;
+  const pct = Math.round(hit.score * 100);
+  return {
+    check: "piece-dup",
+    subject: `piece:${piece.id}`,
+    severity: "warn",
+    line: `Plan piece "${piece.name}" of feature "${feature.name}" is starting, but ${hit.work.prNumber ? `PR #${hit.work.prNumber}` : "a task"} already merged in the same project and shares ${pct}% of its wording.`,
+    detail: {
+      mergedSession: hit.work.sessionId,
+      prNumber: hit.work.prNumber,
+      pct,
+    },
+    workspaceId: feature.workspace_id,
+  };
+}
+
+// Called as a piece is released. Never throws: it must not hold a start.
+export function reportDuplicatePiece(
+  feature: FeatureRef,
+  piece: PieceRef,
+  ownSessionIds: string[],
+  now = Date.now()
+): void {
+  try {
+    const c = healthConfig();
+    const trip = duplicateTrip(
+      feature,
+      piece,
+      mergedSince(feature, piece, ownSessionIds),
+      c
+    );
+    if (trip) recordTrips([trip], [], now);
+  } catch (error) {
+    console.error("[health] piece check failed:", error);
+  }
+}
+
+// A piece that merged or was dropped no longer needs the look.
+export function closeResolvedPieces(now: number): number {
+  let closed = 0;
+  for (const row of openAnomalies()) {
+    if (row.check_id !== "piece-dup") continue;
+    const id = Number(row.subject.replace("piece:", ""));
+    const piece = db
+      .prepare(`SELECT status FROM feature_pieces WHERE id = ?`)
+      .get(id) as { status: string } | undefined;
+    if (!piece || piece.status === "merged" || piece.status === "dropped") {
+      close(row.id, "resolved", now);
+      closed++;
+    }
+  }
+  return closed;
+}

package/lib/health/route.ts

@@ -0,0 +1,113 @@
+/**
+ * Where an anomaly is heard: one line in the orchestrator chat, and the
+ * phone only when the watcher says it needs the person. Each is sent once
+ * per anomaly (keys carry the anomaly id and severity), so one that keeps
+ * tripping is one line.
+ */
+
+import { notifyPhone } from "../notify";
+import { listOrchestrators } from "../orchestrator/home";
+import { queueEventOnce } from "../orchestrator/events";
+import { redact, untrusted } from "../orchestrator/untrusted";
+import { duration } from "./detectors";
+import { markDelivered, markPhoned, verdictOf } from "./store";
+import { db } from "../db";
+import { AREA_LABELS, MIN, type AnomalyRow, type Verdict } from "./types";
+
+// Phone pushes a day, whatever the watchers say.
+export const PHONE_CAP_PER_DAY = 5;
+
+export interface RouteDeps {
+  queue: typeof queueEventOnce;
+  phone: typeof notifyPhone;
+  workspaces: () => string[];
+}
+
+export const realRoute: RouteDeps = {
+  queue: queueEventOnce,
+  phone: notifyPhone,
+  workspaces: () =>
+    listOrchestrators()
+      .filter((o) => !o.archived_at && o.workspace_id)
+      .map((o) => o.workspace_id as string),
+};
+
+export const phoneLine = (line: string): string => {
+  const safe = redact(line).replace(/\s+/g, " ").trim();
+  return safe.length > 160 ? `${safe.slice(0, 159)}…` : safe;
+};
+
+export function chatLine(row: AnomalyRow, v: Verdict): string {
+  const head = `Health (${AREA_LABELS[row.area].toLowerCase()}${v.unjudged ? ", unjudged" : ""}):`;
+  // A detector line can carry a job description or a piece name an agent wrote.
+  const line = untrusted("health detector", row.line);
+  if (v.unjudged) return `${head} ${line}`;
+  const said = [v.cause, v.action ? `Suggest: ${v.action}` : ""]
+    .filter(Boolean)
+    .join(" ");
+  return `${head} ${line} ${untrusted("health watcher", said)}`;
+}
+
+function phonedToday(now: number): number {
+  return (
+    db
+      .prepare(
+        `SELECT COUNT(*) AS n FROM health_anomalies WHERE phoned_at >= ?`
+      )
+      .get(now - 24 * 60 * MIN) as { n: number }
+  ).n;
+}
+
+export function deliver(
+  row: AnomalyRow,
+  v: Verdict,
+  now: number,
+  deps: RouteDeps = realRoute
+): void {
+  // The watcher judged it normal: it's in the panel, not in the chat.
+  if (!v.real) {
+    markDelivered(row.id, now);
+    return;
+  }
+  const targets = row.workspace_id ? [row.workspace_id] : deps.workspaces();
+  const urgent = v.saadOnly || row.severity === "high";
+  const line = chatLine(row, v);
+  for (const ws of targets)
+    deps.queue(
+      ws,
+      `health:${row.id}:${row.severity}`,
+      row.subject,
+      line,
+      now,
+      !urgent
+    );
+  markDelivered(row.id, now);
+  if (v.saadOnly && !row.phoned_at && phonedToday(now) < PHONE_CAP_PER_DAY) {
+    // Nothing a model wrote; the detector's line, redacted and clipped.
+    deps.phone(
+      "health",
+      `${phoneLine(row.line)} It needs you; the details are in the orchestrator chat.`,
+      {
+        category: "system",
+        key: `health:${row.id}`,
+        collapse: `health:${row.id}`,
+      }
+    );
+    markPhoned(row.id, now);
+  }
+}
+
+// After it clears, one short line, only if the chat ever heard of it.
+export function deliverClosed(
+  row: AnomalyRow,
+  now: number,
+  deps: RouteDeps = realRoute
+): void {
+  if (!row.delivered_at || row.close_reason !== "cleared") return;
+  const v = verdictOf(row);
+  if (!v?.real) return;
+  const text = `Health (${AREA_LABELS[row.area].toLowerCase()}): cleared after ${duration((row.closed_at ?? now) - row.opened_at)}: ${row.check_id} on ${row.subject}.`;
+  const targets = row.workspace_id ? [row.workspace_id] : deps.workspaces();
+  for (const ws of targets)
+    deps.queue(ws, `health:${row.id}:closed`, row.subject, text, now, true);
+}