@saadnvd1/agent-os 0.3.678 → 0.3.680

package/lib/agents/brief.ts

@@ -1,3 +1,8 @@
+// What every agent AgentOS starts is told about waiting; the note sent after
+// AgentOS stops a spinning background job repeats it (lib/health/spin.ts).
+export const WAIT_ADVICE =
+  "To wait for a long command, run it with run_in_background and end your turn; you're woken when it exits. Don't start a second job to poll for the first. Any wait loop must sleep on each pass and test a condition that can actually become true (pgrep for the exact process, a log line), never a fixed sleep.";
+
 // What every agent AgentOS starts is told about the network it's on.
 export const BUS_BRIEF = `You are one of several coding agents running in AgentOS. Other sessions may be working on related projects at the same time, and you can talk to them with the \`aos\` command:
 
@@ -18,4 +23,6 @@ Messages arrive in your terminal as lines starting with "[AgentOS message from .
 
 AgentOS itself is running your session, on this machine. Never stop processes with pkill/killall by pattern. Stop your own dev server with \`npm run dev:stop\` (or kill the PID you started). Never stop, kill or restart the AgentOS server, or whatever listens on its port: that ends every session, yours included. If it needs a restart, make that the very last thing you do and say so first, or ask the user to do it.
 
+${WAIT_ADVICE}
+
 Python: run a repo's Python through its own tooling: \`uv run …\` where it has a pyproject.toml or uv.lock (\`uv\` is at ~/.local/bin/uv), the repo's documented test script (for example \`npm run pytest\` in dashboards), or its venv. Never a bare \`python3\` for repo code, and never \`pip install\` into Homebrew or system Python. Make temp files with \`mktemp\`, not \`$TMPDIR/…\`.`;

package/lib/demo-reseed.ts

@@ -57,6 +57,10 @@ export function restoreDemo(db: Database.Database, file: string): void {
     db.transaction(() => {
       for (const { name } of tables) {
         if (KEEP_TABLES.has(name)) continue;
+        // The search index (an FTS table, its shadow tables, its keys) is
+        // kept up to date by triggers on chat_items; copying it would
+        // fight them, and shadow tables can't be written at all.
+        if (name.startsWith("chat_search")) continue;
         // Stored columns only: a generated one can't be written.
         const cols = (
           db.prepare(`SELECT * FROM pragma_table_xinfo(?)`).all(name) as {

package/lib/health/facts.ts

@@ -76,7 +76,12 @@ function queue(): QueueFact[] {
 
 // Background jobs still running in a chat whose agent is alive, that have
 // been going long enough to be worth a look at their output.
-function backgroundJobs(now: number, c: HealthConfig): BgFact[] {
+export interface BgTask extends BgFact {
+  outputFile?: string;
+  taskType?: string;
+}
+
+export function runningBgTasks(now: number, minAgeMs: number): BgTask[] {
   const live = new Set(runningWorkers());
   if (!live.size) return [];
   const sessions = db
@@ -86,12 +91,12 @@ function backgroundJobs(now: number, c: HealthConfig): BgFact[] {
        WHERE s.archived_at IS NULL AND s.view = 'chat'`
     )
     .all() as { id: string; name: string; workspace_id: string | null }[];
-  const out: BgFact[] = [];
+  const out: BgTask[] = [];
   for (const s of sessions) {
     if (!live.has(s.id)) continue;
     for (const t of itemsOfKind(s.id, "task")) {
       if (t.status !== "running" || t.ambient) continue;
-      if (now - t.createdAt < c.bgQuietMinutes * MIN) continue;
+      if (now - t.createdAt < minAgeMs) continue;
       const tail = taskOutputTail(t.taskId, t.outputFile);
       out.push({
         sessionId: s.id,
@@ -101,12 +106,17 @@ function backgroundJobs(now: number, c: HealthConfig): BgFact[] {
         description: t.description,
         startedAt: t.createdAt,
         outputAt: tail?.updatedAt ?? null,
+        outputFile: t.outputFile,
+        taskType: t.taskType,
       });
     }
   }
   return out;
 }
 
+const backgroundJobs = (now: number, c: HealthConfig): BgFact[] =>
+  runningBgTasks(now, c.bgQuietMinutes * MIN);
+
 // Every running background job, for the heartbeat a restart reads back.
 export function liveBackgroundJobs(): string[] {
   const live = new Set(runningWorkers());

package/lib/health/spin-tick.ts

@@ -0,0 +1,293 @@
+/**
+ * The I/O half of bg-spin (spin.ts decides): once a minute, for each old
+ * enough background job of a live chat, find its process tree, sample the CPU
+ * of its shells, and stop a job that spins. A stop is by PID, the job's own
+ * tree only: SIGTERM, then SIGKILL for what is still the same process after a
+ * grace. Never a pattern, never a group, never AgentOS, tmux or the agent.
+ * With no old background job there is no process I/O at all.
+ */
+
+import { execFile } from "child_process";
+import fs from "fs";
+import { sendMessage } from "../bus";
+import { findOutputFile } from "../chat/task-output";
+import { queueEventOnce } from "../orchestrator/events";
+import { untrusted } from "../orchestrator/untrusted";
+import { processTable, type ProcRow } from "../process-table";
+import { runningBgTasks, type BgTask } from "./facts";
+import { close, recordTrips } from "./store";
+import {
+  KILL_GRACE_MS,
+  SPIN_KEEP_MS,
+  SPIN_MIN_AGE_MS,
+  baseOf,
+  classifySpin,
+  jobTree,
+  killPlan,
+  parseCpuTime,
+  pickRoot,
+  safeToKill,
+  shellPids,
+  spinEventLine,
+  spinNote,
+  type SpinSample,
+} from "./spin";
+import type { Trip } from "./types";
+
+export interface SpinDeps {
+  tasks: (now: number) => BgTask[];
+  // Output file -> the PIDs holding it open.
+  holders: (files: string[]) => Promise<Map<string, number[]>>;
+  table: () => Promise<ProcRow[] | null>;
+  cpuSeconds: (pids: number[]) => Promise<Map<number, number>>;
+  kill: (pid: number, signal: "SIGTERM" | "SIGKILL") => void;
+  sleep: (ms: number) => Promise<void>;
+  tell: (sessionId: string, body: string) => Promise<void>;
+  event: (
+    workspaceId: string,
+    key: string,
+    subject: string,
+    line: string,
+    now: number
+  ) => void;
+  record: (trip: Trip, now: number) => void;
+}
+
+const run = (cmd: string, args: string[]): Promise<string> =>
+  new Promise((resolve) =>
+    execFile(cmd, args, { timeout: 5000, maxBuffer: 4 << 20 }, (_err, out) =>
+      resolve(out ?? "")
+    )
+  );
+
+// `lsof -F pn`: a `p<pid>` line, then `n<name>` for each file it holds.
+export function parseHolders(out: string): Map<string, number[]> {
+  const map = new Map<string, number[]>();
+  let pid = 0;
+  for (const line of out.split("\n")) {
+    if (line.startsWith("p")) pid = Number(line.slice(1)) || 0;
+    else if (line.startsWith("n") && pid) {
+      const list = map.get(line.slice(1)) ?? [];
+      if (!list.includes(pid)) list.push(pid);
+      map.set(line.slice(1), list);
+    }
+  }
+  return map;
+}
+
+// Linux counts reaped children in /proc; macOS ps's TIME already includes them.
+function procCpu(pid: number): number | null {
+  try {
+    const stat = fs.readFileSync(`/proc/${pid}/stat`, "utf8");
+    const f = stat.slice(stat.lastIndexOf(")") + 2).split(" ");
+    return (
+      (Number(f[11]) + Number(f[12]) + Number(f[13]) + Number(f[14])) / 100
+    );
+  } catch {
+    return null;
+  }
+}
+
+async function cpuSeconds(pids: number[]): Promise<Map<number, number>> {
+  const out = new Map<number, number>();
+  if (!pids.length) return out;
+  if (process.platform === "linux") {
+    for (const pid of pids) {
+      const sec = procCpu(pid);
+      if (sec !== null) out.set(pid, sec);
+    }
+    return out;
+  }
+  const text = await run("ps", ["-o", "pid=,time=", "-p", pids.join(",")]);
+  for (const line of text.split("\n")) {
+    const m = /^\s*(\d+)\s+(\S+)\s*$/.exec(line);
+    const sec = m ? parseCpuTime(m[2]) : null;
+    if (m && sec !== null) out.set(Number(m[1]), sec);
+  }
+  return out;
+}
+
+export const realSpin: SpinDeps = {
+  tasks: (now) =>
+    runningBgTasks(now, SPIN_MIN_AGE_MS).map((t) => ({
+      ...t,
+      outputFile: t.outputFile ?? findOutputFile(t.taskId) ?? undefined,
+    })),
+  holders: async (files) =>
+    parseHolders(await run("lsof", ["-F", "pn", "--", ...files])),
+  table: () => processTable(0),
+  cpuSeconds,
+  kill: (pid, signal) => process.kill(pid, signal),
+  sleep: (ms) => new Promise((r) => setTimeout(r, ms)),
+  tell: async (sessionId, body) => {
+    await sendMessage({
+      fromId: null,
+      to: sessionId,
+      body,
+      fromLabel: "AgentOS health",
+      origin: { kind: "system", label: "Health", body },
+    });
+  },
+  event: (ws, key, subject, line, now) => {
+    queueEventOnce(ws, key, subject, line, now);
+  },
+  record: (trip, now) => {
+    const { opened } = recordTrips([trip], [], now);
+    // History only: nothing to wake a watcher for.
+    for (const row of opened) close(row.id, "stopped", now);
+  },
+};
+
+interface Track {
+  root: number;
+  comm: string;
+  samples: SpinSample[];
+}
+
+const tracks = new Map<string, Track>();
+let running = false;
+
+export function resetSpin(): void {
+  tracks.clear();
+  running = false;
+}
+
+const keyOf = (t: BgTask) => `${t.sessionId}:${t.taskId}`;
+
+// SIGTERM the whole job, root first; SIGKILL what is still the same process.
+async function stopTree(
+  root: number,
+  table: ProcRow[],
+  deps: SpinDeps
+): Promise<number[]> {
+  const plan = killPlan(root, table);
+  const comm = new Map(table.map((r) => [r.pid, r.comm]));
+  const signal = (pid: number, sig: "SIGTERM" | "SIGKILL") => {
+    try {
+      deps.kill(pid, sig);
+    } catch {
+      // Already gone.
+    }
+  };
+  for (const pid of plan) signal(pid, "SIGTERM");
+  await deps.sleep(KILL_GRACE_MS);
+  const after = new Map((await deps.table())?.map((r) => [r.pid, r.comm]));
+  for (const pid of plan)
+    if (after.has(pid) && after.get(pid) === comm.get(pid))
+      signal(pid, "SIGKILL");
+  return plan;
+}
+
+async function sampleAll(
+  jobs: BgTask[],
+  now: number,
+  deps: SpinDeps
+): Promise<{ job: BgTask; track: Track; table: ProcRow[] }[]> {
+  const table = await deps.table();
+  if (!table) return [];
+  const need = jobs.filter((j) => !tracks.get(keyOf(j)) && j.outputFile);
+  if (need.length) {
+    const held = await deps.holders(need.map((j) => j.outputFile as string));
+    for (const j of need) {
+      const root = pickRoot(held.get(j.outputFile as string) ?? [], table);
+      const row = table.find((r) => r.pid === root);
+      if (row)
+        tracks.set(keyOf(j), { root: row.pid, comm: row.comm, samples: [] });
+    }
+  }
+  const live: { job: BgTask; track: Track; table: ProcRow[] }[] = [];
+  for (const j of jobs) {
+    const track = tracks.get(keyOf(j));
+    if (!track) continue;
+    const row = table.find((r) => r.pid === track.root);
+    // Gone, or the PID is someone else's now.
+    if (!row || row.comm !== track.comm) {
+      tracks.delete(keyOf(j));
+      continue;
+    }
+    live.push({ job: j, track, table });
+  }
+  const pids = live.flatMap((l) => shellPids(jobTree(l.track.root, table)));
+  const cpu = await deps.cpuSeconds(pids);
+  for (const { job, track } of live) {
+    const sec = shellPids(jobTree(track.root, table)).reduce(
+      (sum, pid) => sum + (cpu.get(pid) ?? 0),
+      0
+    );
+    track.samples.push({ at: now, cpuSec: sec, outputAt: job.outputAt });
+    track.samples = track.samples.filter((s) => now - s.at <= SPIN_KEEP_MS);
+  }
+  return live;
+}
+
+export async function spinTick(
+  now = Date.now(),
+  deps: SpinDeps = realSpin
+): Promise<void> {
+  if (running) return;
+  running = true;
+  try {
+    // Shell jobs only: a subagent task has no shell to spin.
+    const jobs = deps
+      .tasks(now)
+      .filter((t) => !t.taskType || t.taskType === "local_bash");
+    const names = new Set(jobs.map(keyOf));
+    for (const k of tracks.keys()) if (!names.has(k)) tracks.delete(k);
+    if (!jobs.length) return;
+    for (const { job, track, table } of await sampleAll(jobs, now, deps)) {
+      if (classifySpin(track.samples) !== "spinning") continue;
+      if (!safeToKill(track.root, table)) {
+        console.warn(
+          `[health] bg-spin: not stopping pid ${track.root} (${baseOf(track.comm)}) in ${job.sessionName}: it is not a plain job of the agent`
+        );
+        continue;
+      }
+      const pids = await stopTree(track.root, table, deps);
+      tracks.delete(keyOf(job));
+      console.warn(
+        `[health] bg-spin: stopped "${job.description}" in ${job.sessionName} (session ${job.sessionId}): pid ${track.root}, tree ${pids.join(",")}`
+      );
+      await announce(job, track.root, now, deps);
+    }
+  } finally {
+    running = false;
+  }
+}
+
+async function announce(
+  job: BgTask,
+  pid: number,
+  now: number,
+  deps: SpinDeps
+): Promise<void> {
+  const line = spinEventLine(job.sessionName, job.description, pid);
+  try {
+    deps.record(
+      {
+        check: "bg-spin",
+        subject: keyOf(job),
+        severity: "high",
+        line,
+        detail: { sessionId: job.sessionId, taskId: job.taskId, pid },
+        workspaceId: job.workspaceId,
+      },
+      now
+    );
+  } catch (error) {
+    console.error("[health] bg-spin record:", error);
+  }
+  if (job.workspaceId)
+    deps.event(
+      job.workspaceId,
+      `bg-spin:${keyOf(job)}`,
+      job.sessionId,
+      // The description is the agent's own text.
+      `Health (tasks): ${untrusted("health detector", line)}`,
+      now
+    );
+  try {
+    await deps.tell(job.sessionId, spinNote(job.description, pid));
+  } catch (error) {
+    console.error("[health] bg-spin note:", error);
+  }
+}

package/lib/health/spin.ts

@@ -0,0 +1,158 @@
+/**
+ * bg-spin: a background job whose shell burns a core with nothing printed is a
+ * wait loop that can never exit (`until grep -q "" /dev/null; do :; done`).
+ * The pure half: deciding it, picking what to stop, and the words about it.
+ * No I/O here; spin-tick.ts reads the processes and stops them.
+ *
+ * Only the CPU of shells in the job's tree is counted. A loop's work happens
+ * in the shell itself (builtins, or children reaped into its total), while a
+ * build's lives in node, cc or cargo, so a silent heavy build never reads as
+ * spinning.
+ */
+
+import path from "path";
+import { WAIT_ADVICE } from "../agents/brief";
+import type { ProcRow } from "../process-table";
+import { MIN } from "./types";
+
+// Share of one core, every interval of the window.
+export const SPIN_CORES = 0.8;
+// How long it has to hold, with nothing new in the output.
+export const SPIN_WINDOW_MS = 4 * MIN;
+// Younger jobs aren't sampled.
+export const SPIN_MIN_AGE_MS = SPIN_WINDOW_MS;
+// A longer gap between samples (missed ticks, a sleeping laptop) starts over.
+export const SPIN_MAX_GAP_MS = 3 * MIN;
+// SIGTERM, then SIGKILL for whatever is still there.
+export const KILL_GRACE_MS = 3000;
+// Samples older than this are no use to the decision.
+export const SPIN_KEEP_MS = SPIN_WINDOW_MS + 2 * MIN;
+
+const SHELLS = new Set(["sh", "bash", "zsh", "dash", "fish", "ksh"]);
+const AGENTS = new Set(["claude"]);
+// Never part of a job's tree to stop.
+const NEVER = new Set(["tmux", "claude"]);
+
+export const baseOf = (comm: string): string =>
+  path.basename(comm).replace(/^-/, "");
+
+export interface SpinSample {
+  at: number;
+  // Cumulative CPU seconds of the tree's shells.
+  cpuSec: number;
+  // The output file's last write, if it has one.
+  outputAt: number | null;
+}
+
+export type SpinClass =
+  | "warming"
+  | "sleeping"
+  | "busy-with-output"
+  | "spinning";
+
+export function classifySpin(samples: SpinSample[]): SpinClass {
+  if (samples.length < 2) return "warming";
+  // The unbroken run of samples ending now.
+  let from = samples.length - 1;
+  while (from > 0 && samples[from].at - samples[from - 1].at <= SPIN_MAX_GAP_MS)
+    from--;
+  const run = samples.slice(from);
+  const last = run[run.length - 1];
+  // Start at the newest sample at least a window old.
+  let start = -1;
+  for (let i = 0; i < run.length; i++)
+    if (last.at - run[i].at >= SPIN_WINDOW_MS) start = i;
+  if (start < 0) return "warming";
+  const win = run.slice(start);
+  for (let i = 1; i < win.length; i++) {
+    const wall = (win[i].at - win[i - 1].at) / 1000;
+    const cpu = Math.max(0, win[i].cpuSec - win[i - 1].cpuSec);
+    if (wall <= 0 || cpu / wall < SPIN_CORES) return "sleeping";
+  }
+  const wrote = win.some((s) => s.outputAt !== null && s.outputAt > win[0].at);
+  return wrote ? "busy-with-output" : "spinning";
+}
+
+// Of the processes holding a task's output file open, the job's root: the
+// one whose parent doesn't hold it too.
+export function pickRoot(holders: number[], table: ProcRow[]): number | null {
+  const held = new Set(holders);
+  const roots = table.filter((r) => held.has(r.pid) && !held.has(r.ppid));
+  return roots.length === 1 ? roots[0].pid : null;
+}
+
+export function jobTree(root: number, table: ProcRow[]): ProcRow[] {
+  const kids = new Map<number, ProcRow[]>();
+  for (const r of table) {
+    const list = kids.get(r.ppid);
+    if (list) list.push(r);
+    else kids.set(r.ppid, [r]);
+  }
+  const byPid = new Map(table.map((r) => [r.pid, r]));
+  const out: ProcRow[] = [];
+  const seen = new Set<number>();
+  const stack = [byPid.get(root)].filter((r): r is ProcRow => !!r);
+  while (stack.length) {
+    const at = stack.shift() as ProcRow;
+    if (seen.has(at.pid)) continue;
+    seen.add(at.pid);
+    out.push(at);
+    stack.push(...(kids.get(at.pid) ?? []));
+  }
+  return out;
+}
+
+export const shellPids = (tree: ProcRow[]): number[] =>
+  tree.filter((r) => SHELLS.has(baseOf(r.comm))).map((r) => r.pid);
+
+// Whether this tree is a background job of an agent and nothing else: a
+// shell directly under the agent, with nothing of AgentOS, tmux or the agent
+// inside it.
+export function safeToKill(
+  root: number,
+  table: ProcRow[],
+  protectedPids: number[] = [process.pid, process.ppid]
+): boolean {
+  const head = table.find((r) => r.pid === root);
+  if (!head || root <= 1) return false;
+  if (!SHELLS.has(baseOf(head.comm))) return false;
+  const parent = table.find((r) => r.pid === head.ppid);
+  if (!parent || !AGENTS.has(baseOf(parent.comm))) return false;
+  return !jobTree(root, table).some(
+    (r) =>
+      protectedPids.includes(r.pid) ||
+      r.pid === parent.pid ||
+      NEVER.has(baseOf(r.comm))
+  );
+}
+
+// The PIDs to signal: the root first, so a loop can't respawn children, then
+// everything under it.
+export const killPlan = (root: number, table: ProcRow[]): number[] =>
+  jobTree(root, table).map((r) => r.pid);
+
+// ps's TIME column: [[D-]H:]M:SS[.cc] to seconds.
+export function parseCpuTime(text: string): number | null {
+  const m = /^\s*(?:(\d+)-)?(?:(\d+):)?(\d+):(\d+(?:\.\d+)?)\s*$/.exec(text);
+  if (!m) return null;
+  return (
+    Number(m[1] ?? 0) * 86400 +
+    Number(m[2] ?? 0) * 3600 +
+    Number(m[3]) * 60 +
+    Number(m[4])
+  );
+}
+
+export function spinNote(description: string, pid: number): string {
+  const mins = Math.round(SPIN_WINDOW_MS / MIN);
+  return `AgentOS stopped your background job "${description}" (PID ${pid}): its shell used about a core for ${mins}+ minutes with no output, which is a wait loop that can't exit. ${WAIT_ADVICE}`;
+}
+
+export function spinEventLine(
+  sessionName: string,
+  description: string,
+  pid: number
+): string {
+  const mins = Math.round(SPIN_WINDOW_MS / MIN);
+  return `Stopped background job "${description}" (PID ${pid}) in ${sessionName}: a shell wait loop holding a core for ${mins}+ minutes with no output. The task was told.`;
+}

package/lib/health/tick.ts

@@ -11,6 +11,7 @@ import { healthConfig, type HealthConfig } from "./config";
 import { runDetectors } from "./detectors";
 import { gatherFacts, liveBackgroundJobs } from "./facts";
 import { envTick, realEnv, type EnvDeps } from "./env/envTick";
+import { spinTick } from "./spin-tick";
 import { addNote } from "./notes";
 import { closeResolvedPieces } from "./pieces";
 import { deliver, deliverClosed, type RouteDeps, realRoute } from "./route";
@@ -43,9 +44,10 @@ export const MAX_WAKES = 2;
 // A wake that never came back (a restart cut it off) is given up on.
 export const WAKE_GIVE_UP_MS = 3 * MIN;
 
-// The checks the tick evaluates: piece-dup is raised where a piece starts.
+// The checks the tick evaluates: piece-dup is raised where a piece starts,
+// bg-spin by the process step (spin-tick.ts).
 export const TICK_CHECKS: readonly CheckId[] = CHECKS.filter(
-  (c) => c !== "piece-dup"
+  (c) => c !== "piece-dup" && c !== "bg-spin"
 );
 
 export interface TickDeps {
@@ -54,6 +56,8 @@ export interface TickDeps {
   watch: WatchDeps;
   route: RouteDeps;
   env: EnvDeps;
+  // Stops background jobs that spin (spin-tick.ts).
+  spin: (now: number) => Promise<void>;
   wake: typeof wake;
 }
 
@@ -63,6 +67,7 @@ export const realTick: TickDeps = {
   watch: { run: runClaude, usagePct: currentUsagePct },
   route: realRoute,
   env: realEnv,
+  spin: spinTick,
   wake,
 };
 
@@ -145,6 +150,9 @@ export function healthTick(
   closeResolvedPieces(now);
   leaveHeartbeat(now, liveBackgroundJobs());
   giveUpStale(now, deps);
+  void deps
+    .spin(now)
+    .catch((error) => console.error("[health] spin check:", error));
   void envTick(now, c, deps.env).catch((error) =>
     console.error("[health] environment failures:", error)
   );

package/lib/health/types.ts

@@ -14,6 +14,7 @@ export const CHECKS = [
   "load-red",
   "queue-held",
   "bg-quiet",
+  "bg-spin",
   "piece-dup",
   "restarts",
   "ci-slow",
@@ -26,6 +27,7 @@ export const AREA_OF: Record<CheckId, Area> = {
   "load-red": "scheduler",
   "queue-held": "scheduler",
   "bg-quiet": "tasks",
+  "bg-spin": "tasks",
   "piece-dup": "pipeline",
   restarts: "pipeline",
   "ci-slow": "pipeline",

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@saadnvd1/agent-os",
-  "version": "0.3.678",
+  "version": "0.3.680",
   "description": "Self-hosted web UI for managing Claude Code sessions",
   "bin": {
     "agent-os": "scripts/agent-os"