@saadnvd1/agent-os 0.3.654 → 0.3.657

package/lib/health/config.ts

@@ -15,6 +15,11 @@ export const HEALTH_DEFAULTS = {
   armedStuckMinutes: 120,
   serverErrorsMin: 20,
   serverErrorsFactor: 5,
+  // An environment failure reaches the orchestrator when this many sessions
+  // hit one class inside the window, or when it returns after the quiet days.
+  envBurstSessions: 3,
+  envBurstHours: 24,
+  envQuietDays: 7,
   // The watchers stop waking past this share of the 5-hour usage window.
   wakeUsageCeilingPct: 90,
   // 0 turns the watchers off: anomalies still open and reach the chat.
@@ -35,6 +40,9 @@ export const HEALTH_LABELS: Record<HealthKey, string> = {
   armedStuckMinutes: "PR armed without merging for (minutes)",
   serverErrorsMin: "Server errors per 10 minutes, at least",
   serverErrorsFactor: "Server errors, times the usual rate",
+  envBurstSessions: "Environment failure: sessions in the window to alert",
+  envBurstHours: "Environment failure window (hours)",
+  envQuietDays: "Environment failure back after quiet for (days)",
   wakeUsageCeilingPct: "Stop waking watchers above usage (%)",
   watchersOn: "Watchers on (1) or off (0)",
 };

package/lib/health/env/classes.ts

@@ -0,0 +1,121 @@
+/**
+ * The known environment failures: a pattern, a class and the fix to send.
+ * Adding a class is one entry here, with a positive and a negative case in
+ * classify.test.ts. The first class that matches wins, so a narrower one
+ * sits above the broader one it would otherwise be mistaken for.
+ */
+
+import type { EnvClass } from "./types";
+
+const OLD_PYTHON =
+  /python3?\.[0-9]\b|Python3?\.framework\/Versions\/3\.[0-9]\b|Python\/3\.[0-9]\b/i;
+const USE_ENV =
+  "run it through the repo's own environment (`uv run …` when it has a pyproject.toml or uv.lock, else its .venv/bin/python), not a bare `python3`";
+
+export const ENV_CLASSES: readonly EnvClass[] = [
+  {
+    id: "python-version",
+    label: "Python too old for the code",
+    scope: "shell",
+    match: [
+      /No module named '?"?tomllib/,
+      /^SyntaxError:/,
+      /unsupported operand type\(s\) for \|/,
+      /requires a different Python/,
+      /Python 3\.\d+ or (?:later|newer|above) is required/i,
+    ],
+    requires:
+      /tomllib|requires a different Python|Python 3\.\d+ or |unsupported operand|python3?\.[0-9]\b|Versions\/3\.[0-9]\b|Python\/3\.[0-9]\b/i,
+    fix: `This interpreter is too old for the code: ${USE_ENV}.`,
+  },
+  {
+    id: "python-externally-managed",
+    label: "pip into a managed Python",
+    scope: "shell",
+    match: [/externally-managed-environment/],
+    fix: "Don't `pip install` into a system or Homebrew Python: use the repo's environment (`uv run`, `uv pip`, or its .venv).",
+  },
+  {
+    id: "python-missing-module",
+    label: "Python module not found",
+    scope: "shell",
+    match: [/^(?:ModuleNotFoundError|ImportError): No module named/],
+    fix: `Each Python on this machine has different packages: ${USE_ENV}.`,
+  },
+  {
+    id: "python-not-found",
+    label: "python not found",
+    scope: "shell",
+    match: [
+      /command not found: python/,
+      /^(?:\S+: )?(?:line \d+: )?python[0-9.]*: (?:command )?not found/,
+    ],
+    fix: "There is no bare `python` here: call `python3`, or better `uv run` in the repo.",
+  },
+  {
+    id: "macos-automation",
+    label: "macOS Automation denied",
+    scope: "shell",
+    match: [
+      /\(-1743\)/,
+      /Not authorized to send Apple events/,
+      /errAEEventNotPermitted/,
+    ],
+    fix: "macOS Automation (Apple events) permission was refused for this program. It's a machine setting, not something to retry: tell the orchestrator.",
+  },
+  {
+    id: "local-permission",
+    label: "Local permission denied",
+    scope: "shell",
+    match: [/Permission denied/, /\bEACCES\b/, /Operation not permitted/],
+    except: {
+      command: /\b(?:ssh|scp|sftp|rsync|kubectl|docker\s+exec)\b/,
+      output: /\(publickey/,
+    },
+    fix: null,
+  },
+  {
+    id: "guard-git-write",
+    label: "Guard: git write refused",
+    scope: "guard",
+    match: [
+      /^Refused: git (?:pull|stash|checkout|merge|rebase|reset)\b/,
+      /is inside .*, a project repository/,
+    ],
+    fix: "Orchestrators don't pull or switch a project checkout: `git fetch`, then read the latest with `git show origin/main:<path>`.",
+  },
+  {
+    id: "guard-write",
+    label: "Guard: write refused",
+    scope: "guard",
+    match: [
+      /orchestrators never write/,
+      /you may write only in your folder/,
+      /Change code there through a task/,
+    ],
+    fix: "Write only in your own folder; change repository code through a task (`start_task`) or a quick change.",
+  },
+  {
+    id: "guard-shell",
+    label: "Guard: command refused",
+    scope: "guard",
+    match: [/^Refused: /],
+    fix: null,
+  },
+];
+
+export const classById = (id: string): EnvClass | undefined =>
+  ENV_CLASSES.find((c) => c.id === id);
+
+export const SHELL_TOOLS = new Set([
+  "bash",
+  "shell",
+  "exec_command",
+  "run_shell_command",
+  "local_shell",
+]);
+
+// Commands that show text somebody else wrote: an error in their output is
+// in the data, not something this run hit.
+export const READERS =
+  /^(?:grep|egrep|rg|ag|cat|head|tail|less|more|sed\s+-n|awk|jq|sqlite3|journalctl|echo|printf|aos\s+(?:history|inbox)|sm\s+logs|gh\s+run\s+view|git\s+(?:log|show|diff|grep))\b/;

package/lib/health/env/classify.ts

@@ -0,0 +1,67 @@
+import {
+  normalizeCommand,
+  scrub,
+  genericPath,
+} from "@/lib/session-audit/normalize";
+import { ENV_CLASSES, READERS, SHELL_TOOLS } from "./classes";
+import type { EnvHit } from "./types";
+
+export interface ToolResult {
+  tool: string;
+  input: unknown;
+  output: string;
+  status: string;
+}
+
+// How much of a long output is read: the error is at the end of a traceback.
+export const TAIL = 32_000;
+
+function commandOf(tool: string, input: unknown): string {
+  const i = (input && typeof input === "object" ? input : {}) as Record<
+    string,
+    unknown
+  >;
+  const c = i.command ?? i.cmd;
+  if (typeof c === "string") return c;
+  if (Array.isArray(c)) return c.join(" ");
+  const file = i.file_path ?? i.path;
+  return typeof file === "string" ? `${tool} ${genericPath(file)}` : tool;
+}
+
+// True when every part of the command only reads (after `cd`).
+function onlyReads(command: string): boolean {
+  const parts = command
+    .split(/\n|&&|\|\||;|\|/)
+    .map((p) => p.trim())
+    .filter((p) => p && !/^cd\s/.test(p));
+  return parts.length > 0 && parts.every((p) => READERS.test(p));
+}
+
+export function classifyToolResult(r: ToolResult): EnvHit | null {
+  if (!r.output) return null;
+  const shell = SHELL_TOOLS.has(r.tool.toLowerCase());
+  const command = commandOf(r.tool, r.input);
+  const text = r.output.length > TAIL ? r.output.slice(-TAIL) : r.output;
+  const lines = text.split("\n");
+  const reads = shell && onlyReads(command);
+  for (const c of ENV_CLASSES) {
+    if (c.scope === "shell" ? !shell || reads : r.status !== "error") continue;
+    const line = lines.find((l) => c.match.some((m) => m.test(l.trim())));
+    if (line === undefined) continue;
+    if (c.requires && !c.requires.test(text)) continue;
+    if (c.except?.command?.test(command) || c.except?.output?.test(text))
+      continue;
+    // The command names the very text it found: a search, not a failure.
+    if (shell && command.toLowerCase().includes(line.trim().toLowerCase()))
+      continue;
+    const hit = c.match.map((m) => m.exec(line.trim())?.[0]).find(Boolean);
+    if (shell && hit && command.toLowerCase().includes(hit.toLowerCase()))
+      continue;
+    return {
+      class: c.id,
+      line: scrub(line.trim()).slice(0, 200),
+      command: normalizeCommand(command),
+    };
+  }
+  return null;
+}

package/lib/health/env/envTick.ts

@@ -0,0 +1,46 @@
+import { sendMessage } from "@/lib/bus";
+import { registry } from "@/lib/chat/registry";
+import { chatNeed } from "@/lib/needs-you";
+import { queueEventOnce } from "@/lib/orchestrator/events";
+import { listOrchestrators } from "@/lib/orchestrator/home";
+import type { HealthConfig } from "../config";
+import { raiseAlerts, type AlertDeps } from "./raiseAlerts";
+import { sendFixNotes, type NoteDeps } from "./sendFixNotes";
+
+export interface EnvDeps {
+  notes: NoteDeps;
+  alerts: AlertDeps;
+}
+
+export const realEnv: EnvDeps = {
+  notes: {
+    send: async (sessionId, body) => {
+      await sendMessage({
+        fromId: null,
+        to: sessionId,
+        body,
+        fromLabel: "AgentOS health",
+        origin: { kind: "system", label: "Environment", body },
+      });
+    },
+    // Open asks don't count: an orchestrator with some still hears it.
+    blocked: (id, role) =>
+      chatNeed({ id, role }, registry.live.get(id)?.state ?? null, 0) !== null,
+  },
+  alerts: {
+    queue: queueEventOnce,
+    workspaces: () =>
+      listOrchestrators()
+        .filter((o) => !o.archived_at && o.workspace_id)
+        .map((o) => o.workspace_id as string),
+  },
+};
+
+export async function envTick(
+  now: number,
+  c: HealthConfig,
+  deps: EnvDeps = realEnv
+): Promise<void> {
+  raiseAlerts(now, c, deps.alerts);
+  await sendFixNotes(now, deps.notes);
+}

package/lib/health/env/fixText.ts

@@ -0,0 +1,24 @@
+import { getSetting, putSetting } from "@/lib/settings-store";
+import { classById } from "./classes";
+
+export const MAX_FIX = 500;
+const key = (id: string) => `health.envFix.${id}`;
+
+// The fix a session is sent: this machine's own text if one was set (it
+// stays in the local database, never in the repository), else the default.
+export function fixFor(id: string): string | null {
+  const own = getSetting(key(id));
+  if (own !== null && own.trim() !== "") return own;
+  return classById(id)?.fix ?? null;
+}
+
+export const hasOwnFix = (id: string): boolean =>
+  (getSetting(key(id)) ?? "").trim() !== "";
+
+// Null puts the default back. False for a class that doesn't exist.
+export function setFix(id: string, text: string | null): boolean {
+  if (!classById(id)) return false;
+  const t = text?.replace(/\s+/g, " ").trim().slice(0, MAX_FIX) ?? "";
+  putSetting(key(id), t === "" ? null : t);
+  return true;
+}

package/lib/health/env/helpers.test-util.ts

@@ -0,0 +1,21 @@
+import { db } from "@/lib/db";
+import { updateSessionRow } from "@/lib/db/session-parts/rows";
+
+export function seedSession(
+  id: string,
+  patch: Record<string, string | number | null> = {}
+): void {
+  db.prepare(
+    `INSERT OR REPLACE INTO sessions (id, name, tmux_name, working_directory)
+     VALUES (?, ?, ?, '/repo')`
+  ).run(id, `session ${id}`, `t-${id}`);
+  updateSessionRow(db, id, { view: "chat", ...patch });
+}
+
+export function resetEnv(): void {
+  db.exec(`DELETE FROM env_failures`);
+  for (const r of db
+    .prepare(`SELECT id FROM sessions WHERE id LIKE 'env-%'`)
+    .all() as { id: string }[])
+    db.prepare(`DELETE FROM sessions WHERE id = ?`).run(r.id);
+}

package/lib/health/env/index.ts

@@ -0,0 +1,6 @@
+export { ENV_CLASSES, classById } from "./classes";
+export { classifyToolResult } from "./classify";
+export { noteToolResult, recordHit } from "./recordHit";
+export { envTick, realEnv, type EnvDeps } from "./envTick";
+export { envSummary, envSessions } from "./summarize";
+export { fixFor, setFix, MAX_FIX } from "./fixText";

package/lib/health/env/raiseAlerts.ts

@@ -0,0 +1,84 @@
+import { db } from "@/lib/db";
+import type { queueEventOnce } from "@/lib/orchestrator/events";
+import { untrusted } from "@/lib/orchestrator/untrusted";
+import type { HealthConfig } from "../config";
+import { MIN } from "../types";
+import { classById } from "./classes";
+import { fixFor } from "./fixText";
+import type { EnvFailureRow } from "./types";
+
+const DAY = 24 * 60 * MIN;
+
+export interface AlertDeps {
+  queue: typeof queueEventOnce;
+  workspaces: () => string[];
+}
+
+const one = (sql: string, ...args: unknown[]) =>
+  (db.prepare(sql).get(...args) as { n: number }).n;
+
+// The orchestrator hears of a class when it comes back after a quiet spell
+// (or is new), or when enough sessions hit it inside a window. One event
+// per row, and a burst at most once per window: the root cause is the thing
+// to fix, so a hot class is one message, not one per session.
+export function raiseAlerts(
+  now: number,
+  c: HealthConfig,
+  deps: AlertDeps
+): number {
+  const window = c.envBurstHours * 60 * MIN;
+  const quiet = c.envQuietDays * DAY;
+  const fresh = db
+    .prepare(
+      `SELECT * FROM env_failures WHERE alerted IS NULL AND first_at >= ?
+       ORDER BY first_at, id`
+    )
+    .all(now - window) as EnvFailureRow[];
+  let raised = 0;
+  for (const r of fresh) {
+    const recent = one(
+      `SELECT COUNT(*) AS n FROM env_failures WHERE class = ? AND first_at >= ?`,
+      r.class,
+      now - window
+    );
+    const bursting =
+      recent >= c.envBurstSessions &&
+      one(
+        `SELECT COUNT(*) AS n FROM env_failures WHERE class = ?
+         AND alerted = 'burst' AND alerted_at >= ?`,
+        r.class,
+        now - window
+      ) === 0;
+    const before = one(
+      `SELECT COUNT(*) AS n FROM env_failures WHERE class = ? AND id != ?
+       AND first_at < ? AND first_at >= ?`,
+      r.class,
+      r.id,
+      r.first_at,
+      r.first_at - quiet
+    );
+    const kind = bursting ? "burst" : before === 0 ? "recur" : null;
+    if (!kind) continue;
+    db.prepare(
+      `UPDATE env_failures SET alerted = ?, alerted_at = ? WHERE id = ?`
+    ).run(kind, now, r.id);
+    const label = classById(r.class)?.label ?? r.class;
+    const what =
+      kind === "burst"
+        ? `${recent} sessions in ${c.envBurstHours}h`
+        : "back after a quiet spell";
+    const line = `Environment failure (${label}): ${what}. Latest: ${untrusted("session command", r.command ?? r.class)}. A fix note ${fixFor(r.class) ? "goes to each session" : "is not defined"}; fix the root cause rather than keep nudging.`;
+    const targets = r.workspace_id ? [r.workspace_id] : deps.workspaces();
+    for (const ws of targets)
+      deps.queue(
+        ws,
+        `env:${r.class}:${kind}:${r.id}`,
+        `env:${r.class}`,
+        line,
+        now,
+        false
+      );
+    raised++;
+  }
+  return raised;
+}

package/lib/health/env/recordHit.ts

@@ -0,0 +1,62 @@
+import { db } from "@/lib/db";
+import type { ChatItem } from "@/lib/chat/events";
+import { classifyToolResult } from "./classify";
+import type { EnvHit } from "./types";
+
+type ToolItem = Extract<ChatItem, { kind: "tool" }>;
+
+// One row per session and class; the same item seen again doesn't count.
+export function recordHit(
+  sessionId: string,
+  itemId: string,
+  hit: EnvHit,
+  now = Date.now()
+): void {
+  const s = db
+    .prepare(
+      `SELECT r.project_id, COALESCE(r.workspace_id, p.workspace_id) AS ws
+       FROM session_rows r LEFT JOIN projects p ON p.id = r.project_id
+       WHERE r.id = ?`
+    )
+    .get(sessionId) as
+    | { project_id: string | null; ws: string | null }
+    | undefined;
+  db.prepare(
+    `INSERT INTO env_failures
+       (session_id, class, project_id, workspace_id, command, line,
+        last_item_id, first_at, last_at)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+     ON CONFLICT(session_id, class) DO UPDATE SET
+       hits = hits + (excluded.last_item_id IS NOT last_item_id),
+       last_at = excluded.last_at, last_item_id = excluded.last_item_id,
+       command = excluded.command, line = excluded.line`
+  ).run(
+    sessionId,
+    hit.class,
+    s?.project_id ?? null,
+    s?.ws ?? null,
+    hit.command,
+    hit.line,
+    itemId,
+    now,
+    now
+  );
+}
+
+// The runner's entry point for every tool item. It must never throw into
+// the chat, and a call still running has no result to read yet.
+export function noteToolResult(sessionId: string, item: ToolItem): void {
+  if (item.status !== "done" && item.status !== "error") return;
+  if (!item.output) return;
+  try {
+    const hit = classifyToolResult({
+      tool: item.name,
+      input: item.input,
+      output: item.output,
+      status: item.status,
+    });
+    if (hit) recordHit(sessionId, item.id, hit);
+  } catch (error) {
+    console.error(`[env-failures] ${sessionId}:`, error);
+  }
+}

package/lib/health/env/sendFixNotes.ts

@@ -0,0 +1,80 @@
+import { db } from "@/lib/db";
+import { MIN } from "../types";
+import { fixFor } from "./fixText";
+import { classById } from "./classes";
+import type { EnvFailureRow } from "./types";
+
+// A note this late is about something the session has moved on from.
+export const STALE_MS = 120 * MIN;
+
+export interface NoteDeps {
+  send: (sessionId: string, body: string) => Promise<void>;
+  // Waiting on a person: an approval or a question.
+  blocked: (sessionId: string, role: "orchestrator" | null) => boolean;
+}
+
+interface Target {
+  role: "orchestrator" | null;
+  archived_at: string | null;
+  task_status: string | null;
+  pause_state: string | null;
+}
+
+const settle = (id: number, why: string) =>
+  db
+    .prepare(`UPDATE env_failures SET nudge_skipped = ? WHERE id = ?`)
+    .run(why, id);
+
+export const noteBody = (r: EnvFailureRow, fix: string): string => {
+  const label = classById(r.class)?.label ?? r.class;
+  const cmd = r.command ? ` (\`${r.command.slice(0, 100)}\`)` : "";
+  return `Environment: ${label}${cmd}. ${fix}`;
+};
+
+// Each session hears each class's fix once: the row is claimed before the
+// send, so a failed send is lost rather than repeated. A paused session or
+// one waiting on a person is looked at again next tick.
+export async function sendFixNotes(
+  now: number,
+  deps: NoteDeps
+): Promise<number> {
+  const rows = db
+    .prepare(
+      `SELECT * FROM env_failures WHERE nudged_at IS NULL AND nudge_skipped IS NULL
+       ORDER BY id`
+    )
+    .all() as EnvFailureRow[];
+  let sent = 0;
+  for (const r of rows) {
+    const fix = fixFor(r.class);
+    if (!fix) {
+      settle(r.id, "no fix");
+      continue;
+    }
+    const s = db
+      .prepare(
+        `SELECT role, archived_at, task_status, pause_state FROM session_rows WHERE id = ?`
+      )
+      .get(r.session_id) as Target | undefined;
+    if (!s || s.archived_at || (s.task_status && s.task_status !== "running")) {
+      settle(r.id, "gone");
+      continue;
+    }
+    if (now - r.first_at > STALE_MS) {
+      settle(r.id, "stale");
+      continue;
+    }
+    if (s.pause_state || deps.blocked(r.session_id, s.role)) continue;
+    db.prepare(`UPDATE env_failures SET nudged_at = ? WHERE id = ?`).run(
+      now,
+      r.id
+    );
+    try {
+      await deps.send(r.session_id, noteBody(r, fix));
+      sent++;
+    } catch (error) {
+      console.error(`[env-failures] note to ${r.session_id} failed:`, error);
+    }
+  }
+  return sent;
+}

package/lib/health/env/summarize.ts

@@ -0,0 +1,94 @@
+import { db } from "@/lib/db";
+import { MIN } from "../types";
+import { ENV_CLASSES } from "./classes";
+import { fixFor, hasOwnFix } from "./fixText";
+import type { EnvSessionView, EnvSummary } from "./types";
+
+const DAY = 24 * 60 * MIN;
+
+interface Group {
+  class: string;
+  project_id: string | null;
+  name: string | null;
+  s7: number;
+  s30: number;
+}
+
+// Sessions that hit each class in the last 7 and 30 days, by project.
+export function envSummary(now = Date.now()): EnvSummary {
+  const groups = db
+    .prepare(
+      `SELECT e.class, e.project_id, p.name,
+         SUM(e.last_at >= ?) AS s7, COUNT(*) AS s30
+       FROM env_failures e LEFT JOIN projects p ON p.id = e.project_id
+       WHERE e.last_at >= ? GROUP BY e.class, e.project_id`
+    )
+    .all(now - 7 * DAY, now - 30 * DAY) as Group[];
+  return {
+    now,
+    classes: ENV_CLASSES.map((c) => {
+      const mine = groups.filter((g) => g.class === c.id);
+      return {
+        id: c.id,
+        label: c.label,
+        fix: fixFor(c.id),
+        fixOverridden: hasOwnFix(c.id),
+        sessions7: mine.reduce((n, g) => n + g.s7, 0),
+        sessions30: mine.reduce((n, g) => n + g.s30, 0),
+        projects: mine
+          .map((g) => ({
+            id: g.project_id,
+            name: g.name ?? "No project",
+            sessions7: g.s7,
+            sessions30: g.s30,
+          }))
+          .sort((a, b) => b.sessions30 - a.sessions30),
+      };
+    }).sort((a, b) => b.sessions30 - a.sessions30),
+  };
+}
+
+interface Row {
+  session_id: string;
+  name: string | null;
+  project: string | null;
+  command: string | null;
+  line: string | null;
+  hits: number;
+  first_at: number;
+  last_at: number;
+  nudged_at: number | null;
+  nudge_skipped: string | null;
+}
+
+export function envSessions(
+  cls: string,
+  days: number,
+  now = Date.now()
+): EnvSessionView[] {
+  const rows = db
+    .prepare(
+      `SELECT e.session_id, s.name, p.name AS project, e.command, e.line,
+         e.hits, e.first_at, e.last_at, e.nudged_at, e.nudge_skipped
+       FROM env_failures e
+       LEFT JOIN session_rows s ON s.id = e.session_id
+       LEFT JOIN projects p ON p.id = e.project_id
+       WHERE e.class = ? AND e.last_at >= ? ORDER BY e.last_at DESC LIMIT 100`
+    )
+    .all(cls, now - days * DAY) as Row[];
+  return rows.map((r) => ({
+    sessionId: r.session_id,
+    name: r.name ?? r.session_id.slice(0, 8),
+    project: r.project,
+    command: r.command,
+    line: r.line,
+    hits: r.hits,
+    firstAt: r.first_at,
+    lastAt: r.last_at,
+    fix: r.nudged_at
+      ? "sent"
+      : r.nudge_skipped
+        ? `not sent (${r.nudge_skipped})`
+        : "waiting to send",
+  }));
+}