@saadnvd1/agent-os 0.3.609 → 0.3.611

package/app/api/sessions/[id]/simulator/run/route.ts

@@ -20,6 +20,7 @@ export async function POST(
         stderr: "",
         code: 0,
         screenshot: null,
+        read: null,
       });
     if (
       !Array.isArray(body.args) ||

package/lib/load/alert.ts

@@ -18,9 +18,14 @@ export function alertText(view: LoadView): string {
       : "";
     return `${s.name} ${s.cores.toFixed(1)} cores, ${gb(s.rssBytes)}${heavy}`;
   });
+  const others = (view.unattributed ?? []).map(
+    (g) =>
+      `${g.name}${g.count > 1 ? ` ×${g.count}` : ""} ${g.cores.toFixed(1)} cores, ${gb(g.rssBytes)}`
+  );
   return [
     `Machine load red for 2+ minutes: ${view.load1} on ${view.cores} cores${memory}.`,
     top.length ? `Top: ${top.join("; ")}.` : "",
+    others.length ? `Not from a session: ${others.join("; ")}.` : "",
   ]
     .filter(Boolean)
     .join(" ");

package/lib/load/attribute.ts

@@ -0,0 +1,217 @@
+// Who is using the machine: every process goes to the session whose tree it
+// is in, or, for a heavy one that left its tree (a build daemon reparented to
+// init), to the session whose worktree it runs in. What's left is grouped by
+// name so the orchestrator can tell its tasks from everything else.
+import { execFile } from "child_process";
+import { existsSync } from "fs";
+import { readlink } from "fs/promises";
+import path from "path";
+import type { ProcRow } from "../process-table";
+import { workerTmuxName } from "../chat/worker/protocol";
+import { claimTrees, type Usage } from "./usage";
+
+// A row shows its usage, and a group is listed, from here up.
+export const BUSY_CORES = 0.5;
+const HOT_PCPU = 20;
+const MAX_LOOKUPS = 40;
+const MAX_HOT = 20;
+const MAX_GROUPS = 3;
+const BIG_RSS = 2 ** 30;
+
+export interface SessionLoad extends Usage {
+  sessionId: string;
+  name: string;
+  heavy: string[];
+}
+
+export interface ProcGroup {
+  name: string;
+  count: number;
+  cores: number;
+  rssBytes: number;
+}
+
+export interface SessionRow {
+  id: string;
+  name: string;
+  tmux_name: string;
+  worktree_path: string | null;
+  working_directory: string;
+}
+
+const round1 = (n: number) => Math.round(n * 10) / 10;
+const baseName = (comm: string) =>
+  comm.split("/").pop()?.replace(/^-/, "") || comm;
+
+// Each session's own tmux session, plus its chat worker's.
+export function claimSessions(
+  procs: ProcRow[],
+  panes: { tmux: string; pid: number }[],
+  rows: SessionRow[]
+): Map<number, string> {
+  const roots: { key: string; pid: number }[] = [];
+  for (const row of rows)
+    for (const p of panes)
+      if (p.tmux === row.tmux_name || p.tmux === workerTmuxName(row.id))
+        roots.push({ key: row.id, pid: p.pid });
+  return claimTrees(procs, roots);
+}
+
+// The directories a process's cwd can place it by, longest first. The
+// worktree when there is one; otherwise the working directory, but only when
+// it is specific to the session.
+export function sessionPaths(
+  rows: SessionRow[],
+  home: string
+): { sessionId: string; path: string }[] {
+  const count = new Map<string, number>();
+  for (const r of rows)
+    count.set(r.working_directory, (count.get(r.working_directory) ?? 0) + 1);
+  const out: { sessionId: string; path: string }[] = [];
+  for (const r of rows) {
+    let dir = r.worktree_path;
+    if (!dir) {
+      dir = r.working_directory.replace(/^~(?=$|\/)/, home);
+      if (
+        !path.isAbsolute(dir) ||
+        dir === "/" ||
+        dir === home ||
+        count.get(r.working_directory)! > 1
+      )
+        continue;
+    }
+    out.push({ sessionId: r.id, path: dir.replace(/\/+$/, "") });
+  }
+  return out.sort((a, b) => b.path.length - a.path.length);
+}
+
+export function matchSession(
+  cwd: string,
+  paths: { sessionId: string; path: string }[]
+): string | null {
+  const hit = paths.find((p) => cwd === p.path || cwd.startsWith(p.path + "/"));
+  return hit?.sessionId ?? null;
+}
+
+// The heavy processes no session owns, and their unowned ancestors (the
+// daemon that spawned the workers), whose cwd is worth looking up.
+export function cwdCandidates(
+  procs: ProcRow[],
+  owner: Map<number, string>
+): number[] {
+  const byPid = new Map(procs.map((p) => [p.pid, p]));
+  const hot = procs
+    .filter((p) => !owner.has(p.pid) && p.pcpu >= HOT_PCPU)
+    .sort((a, b) => b.pcpu - a.pcpu)
+    .slice(0, MAX_HOT);
+  const out = new Set<number>();
+  for (const p of hot) {
+    let at: ProcRow | undefined = p;
+    while (at && at.pid > 1 && !owner.has(at.pid) && out.size < MAX_LOOKUPS) {
+      out.add(at.pid);
+      at = byPid.get(at.ppid);
+    }
+  }
+  return [...out];
+}
+
+// `lsof -a -d cwd -Fpn`: a `p<pid>` line, then `n<path>` for its cwd.
+export function parseLsofCwd(out: string): Map<number, string> {
+  const cwds = new Map<number, string>();
+  let pid = 0;
+  for (const line of out.split("\n")) {
+    if (line.startsWith("p")) pid = Number(line.slice(1)) || 0;
+    else if (line.startsWith("n") && pid) cwds.set(pid, line.slice(1));
+  }
+  return cwds;
+}
+
+const LSOF = ["/usr/sbin/lsof", "/usr/bin/lsof"].find((p) => existsSync(p));
+
+export async function readCwds(pids: number[]): Promise<Map<number, string>> {
+  const cwds = new Map<number, string>();
+  if (!pids.length) return cwds;
+  if (process.platform === "linux") {
+    await Promise.all(
+      pids.map((pid) =>
+        readlink(`/proc/${pid}/cwd`).then(
+          (dir) => void cwds.set(pid, dir),
+          () => undefined
+        )
+      )
+    );
+    return cwds;
+  }
+  return new Promise((resolve) =>
+    execFile(
+      LSOF ?? "lsof",
+      ["-a", "-d", "cwd", "-Fpn", "-p", pids.join(",")],
+      { timeout: 2000, maxBuffer: 1 << 20 },
+      // lsof exits 1 when some pids are gone but still prints the rest.
+      (_err, stdout) => resolve(parseLsofCwd(stdout ?? ""))
+    )
+  );
+}
+
+export function attributeLoad(input: {
+  procs: ProcRow[];
+  owner: Map<number, string>;
+  rows: SessionRow[];
+  cwds: Map<number, string>;
+  home: string;
+}): { sessions: Record<string, SessionLoad>; unattributed: ProcGroup[] } {
+  const { procs, owner, rows, cwds, home } = input;
+  const byPid = new Map(procs.map((p) => [p.pid, p]));
+  const paths = sessionPaths(rows, home);
+  // Memoised: the session an unowned process belongs to, by its own cwd or
+  // its nearest unowned ancestor's.
+  const placed = new Map<number, string | null>();
+  const place = (p: ProcRow, depth = 0): string | null => {
+    if (placed.has(p.pid)) return placed.get(p.pid)!;
+    const cwd = cwds.get(p.pid);
+    let id = cwd ? matchSession(cwd, paths) : null;
+    const parent = byPid.get(p.ppid);
+    if (!id && parent && parent.pid > 1 && depth < 64 && !owner.has(parent.pid))
+      id = place(parent, depth + 1);
+    placed.set(p.pid, id);
+    return id;
+  };
+
+  const raw = new Map<string, Usage>();
+  const groups = new Map<string, ProcGroup>();
+  for (const p of procs) {
+    const id = owner.get(p.pid) ?? place(p);
+    if (id) {
+      const u = raw.get(id) ?? { cores: 0, rssBytes: 0 };
+      u.cores += p.pcpu / 100;
+      u.rssBytes += p.rss * 1024;
+      raw.set(id, u);
+      continue;
+    }
+    const name = baseName(p.comm);
+    const g = groups.get(name) ?? { name, count: 0, cores: 0, rssBytes: 0 };
+    g.count++;
+    g.cores += p.pcpu / 100;
+    g.rssBytes += p.rss * 1024;
+    groups.set(name, g);
+  }
+
+  const sessions: Record<string, SessionLoad> = {};
+  for (const row of rows) {
+    const u = raw.get(row.id);
+    if (!u) continue;
+    sessions[row.id] = {
+      sessionId: row.id,
+      name: row.name,
+      cores: round1(u.cores),
+      rssBytes: u.rssBytes,
+      heavy: [],
+    };
+  }
+  const unattributed = [...groups.values()]
+    .filter((g) => g.cores >= BUSY_CORES || g.rssBytes >= BIG_RSS)
+    .sort((a, b) => b.cores - a.cores)
+    .slice(0, MAX_GROUPS)
+    .map((g) => ({ ...g, cores: round1(g.cores) }));
+  return { sessions, unattributed };
+}

package/lib/load/monitor.ts

@@ -9,21 +9,24 @@ import { LoadAlarm } from "./alarm";
 import { HeavyRegistry, type HeavyRun } from "./heavy";
 import { nextLevel, type LoadLevel, type Pressure } from "./level";
 import { readPressure } from "./pressure";
-import { readUsage, type Usage } from "./usage";
-import { workerTmuxName } from "../chat/worker/protocol";
+import type { Usage } from "./usage";
+import { readSnapshot } from "./usage";
+import {
+  attributeLoad,
+  BUSY_CORES,
+  claimSessions,
+  cwdCandidates,
+  readCwds,
+  type ProcGroup,
+  type SessionLoad,
+  type SessionRow,
+} from "./attribute";
 import { alertText, raiseAlert } from "./alert";
 import { LOCAL, feedGate, gateOpen } from "./start-gate";
 
 export const SAMPLE_MS = 10_000;
 export const USAGE_MS = 30_000;
-// A row shows its usage from here up.
-const BUSY_CORES = 0.5;
-
-export interface SessionLoad extends Usage {
-  sessionId: string;
-  name: string;
-  heavy: string[];
-}
+export type { SessionLoad, ProcGroup };
 
 export interface LoadView {
   level: LoadLevel;
@@ -32,6 +35,8 @@ export interface LoadView {
   memUsedPct: number;
   pressure: Pressure;
   top: SessionLoad[];
+  // Heavy processes that belong to no session, grouped by name.
+  unattributed: ProcGroup[];
   sessions: Record<string, Usage>;
 }
 
@@ -39,6 +44,7 @@ interface State {
   level: LoadLevel | null;
   view: LoadView | null;
   usage: Record<string, SessionLoad>;
+  unattributed: ProcGroup[];
   registry: HeavyRegistry;
   alarm: LoadAlarm;
   timers: ReturnType<typeof setInterval>[];
@@ -50,6 +56,7 @@ const state: State = (g.__agentosLoad ??= {
   level: null,
   view: null,
   usage: {},
+  unattributed: [],
   registry: new HeavyRegistry(),
   alarm: new LoadAlarm(),
   timers: [],
@@ -107,6 +114,7 @@ async function sample(): Promise<void> {
     memUsedPct: Math.round(((total - os.freemem()) / total) * 100),
     pressure,
     top: topSessions(),
+    unattributed: state.unattributed,
     sessions,
   };
   state.view = view;
@@ -145,39 +153,24 @@ export function loadAlarmFromDb(): void {
   state.alarm = new LoadAlarm(readArmed());
 }
 
-type UsageRow = { id: string; name: string; tmux_name: string };
-
-// Each session's share of the usage: its own tmux session's, plus its chat
-// worker's, where a chat's agent runs.
-export function sessionUsage(
-  byTmux: Record<string, Usage>,
-  rows: UsageRow[]
-): Record<string, SessionLoad> {
-  const usage: Record<string, SessionLoad> = {};
-  for (const row of rows) {
-    const parts = [byTmux[row.tmux_name], byTmux[workerTmuxName(row.id)]];
-    const found = parts.filter((u): u is Usage => !!u);
-    if (!found.length) continue;
-    const cores = found.reduce((n, u) => n + u.cores, 0);
-    usage[row.id] = {
-      sessionId: row.id,
-      name: row.name,
-      cores: Math.round(cores * 10) / 10,
-      rssBytes: found.reduce((n, u) => n + u.rssBytes, 0),
-      heavy: [],
-    };
-  }
-  return usage;
-}
-
 async function sampleUsage(): Promise<void> {
-  const byTmux = await readUsage();
+  const { procs, panes } = await readSnapshot();
   const rows = db
     .prepare(
-      `SELECT id, name, tmux_name FROM sessions WHERE archived_at IS NULL AND host_id = 'local'`
+      `SELECT id, name, tmux_name, worktree_path, working_directory FROM sessions WHERE archived_at IS NULL AND host_id = 'local'`
     )
-    .all() as UsageRow[];
-  state.usage = sessionUsage(byTmux, rows);
+    .all() as SessionRow[];
+  const owner = claimSessions(procs, panes, rows);
+  const cwds = await readCwds(cwdCandidates(procs, owner));
+  const { sessions, unattributed } = attributeLoad({
+    procs,
+    owner,
+    rows,
+    cwds,
+    home: os.homedir(),
+  });
+  state.usage = sessions;
+  state.unattributed = unattributed;
 }
 
 // Off the hot path: each tick is async, never overlaps the last, and logs

package/lib/load/usage.ts

@@ -1,7 +1,8 @@
 // What each session's processes use: one `ps` for the whole machine and one
-// `tmux list-panes`, then each pane's process tree summed. Never a call per
+// `tmux list-panes`, then each pane's process tree walked. Never a call per
 // session.
 import { execFile } from "child_process";
+import type { ProcRow } from "../process-table";
 
 export interface Proc {
   pid: number;
@@ -37,36 +38,45 @@ export function parsePanes(out: string): { tmux: string; pid: number }[] {
     .map(([tmux, pid]) => ({ tmux, pid: Number(pid) }));
 }
 
-// Each key's panes and everything under them. A process is counted once,
-// for the first key that reaches it.
-export function sumTrees(
+// The key that owns each process: its panes and everything under them. A
+// process goes to the first key that reaches it.
+export function claimTrees(
   procs: Proc[],
   roots: { key: string; pid: number }[]
-): Record<string, Usage> {
-  const byPid = new Map(procs.map((p) => [p.pid, p]));
+): Map<number, string> {
   const children = new Map<number, number[]>();
   for (const p of procs) {
     const list = children.get(p.ppid);
     if (list) list.push(p.pid);
     else children.set(p.ppid, [p.pid]);
   }
-  const seen = new Set<number>();
-  const out: Record<string, Usage> = {};
+  const owner = new Map<number, string>();
   for (const { key, pid } of roots) {
-    const usage = (out[key] ??= { cores: 0, rssBytes: 0 });
     const stack = [pid];
     while (stack.length) {
       const at = stack.pop()!;
-      if (seen.has(at)) continue;
-      seen.add(at);
-      const p = byPid.get(at);
-      if (p) {
-        usage.cores += p.pcpu / 100;
-        usage.rssBytes += p.rss * 1024;
-      }
+      if (owner.has(at)) continue;
+      owner.set(at, key);
       stack.push(...(children.get(at) ?? []));
     }
   }
+  return owner;
+}
+
+// Each key's panes and everything under them, summed.
+export function sumTrees(
+  procs: Proc[],
+  roots: { key: string; pid: number }[]
+): Record<string, Usage> {
+  const byPid = new Map(procs.map((p) => [p.pid, p]));
+  const out: Record<string, Usage> = {};
+  for (const { key } of roots) out[key] ??= { cores: 0, rssBytes: 0 };
+  for (const [pid, key] of claimTrees(procs, roots)) {
+    const p = byPid.get(pid);
+    if (!p) continue;
+    out[key].cores += p.pcpu / 100;
+    out[key].rssBytes += p.rss * 1024;
+  }
   return out;
 }
 
@@ -77,11 +87,13 @@ const run = (cmd: string, args: string[], timeout: number) =>
     )
   );
 
-// Usage by tmux session name. Empty when ps or tmux can't be read. The
-// process table is the one shared with the rest of the server.
-export async function readUsage(
-  timeoutMs = 3000
-): Promise<Record<string, Usage>> {
+// The process table and the tmux panes, for attributing load to sessions.
+// Empty when ps or tmux can't be read. The table is the one shared with the
+// rest of the server.
+export async function readSnapshot(timeoutMs = 3000): Promise<{
+  procs: ProcRow[];
+  panes: { tmux: string; pid: number }[];
+}> {
   const { processTable } = await import("../process-table");
   const [procs, panes] = await Promise.all([
     processTable(),
@@ -91,8 +103,5 @@ export async function readUsage(
       timeoutMs
     ),
   ]);
-  return sumTrees(
-    procs ?? [],
-    parsePanes(panes).map((p) => ({ key: p.tmux, pid: p.pid }))
-  );
+  return { procs: procs ?? [], panes: parsePanes(panes) };
 }

package/lib/project-config/brief.ts

@@ -50,6 +50,10 @@ export function runningBrief(
     lines.push(
       `- \`${browse.map}\` maps every page to its URL: navigate by URL, don't explore.`
     );
+  if (browse)
+    lines.push(
+      "- Check state with the browser's text snapshot, not screenshots. Screenshot only the final changed screen, and just the changed element (its uid) rather than the full page where the tool allows; every image stays in your context."
+    );
   const sim = simulatorBrief(config.simulator);
   if (sim) lines.push(sim);
   for (const note of notesOf(config.notes))

package/lib/simulator/brief.ts

@@ -12,7 +12,8 @@ export function simulatorBrief(
     return "- The iOS simulator tools are not available on this machine; skip simulator checks and say so in the PR.";
   return [
     "- iOS simulator (optional; it never gates a PR or a build, and a failure is reported, not a blocker): `aos sim open` makes your own simulator and installs this worktree's build (run it with run_in_background, a cold build takes minutes; `aos sim status` shows where it is).",
-    '  Then `aos sim snapshot -i` (accessibility tree with @refs), `aos sim click @e3`, `aos sim fill @e5 "text"`, `aos sim screenshot` (prints a PNG path; read it), `aos sim logs --last 2m`. Refs are only valid from the latest snapshot.',
+    '  Drive and check state with `aos sim snapshot -i` (accessibility tree with @refs, text, cheap), `aos sim click @e3`, `aos sim fill @e5 "text"`, `aos sim logs --last 2m`. Refs are only valid from the latest snapshot.',
+    "  Every image you read stays in your context for every later turn, so screenshot only the final changed screens, and prefer `aos sim screenshot --region @e3` (one element from the latest snapshot) for a single row or banner. It prints two paths: read the `read:` one (reduced to 800px); attach the `full-size` file to the PR.",
     "  Screenshots are written outside the worktree: don't put them in commits or the PR body unless they belong there. `aos sim close` when you're done; it is closed for you when the task ends.",
   ].join("\n");
 }

package/lib/simulator/index.ts

@@ -24,6 +24,13 @@ import {
   simulatorDir,
   type CallResult,
 } from "./agent-device";
+import {
+  RegionError,
+  cropToFrame,
+  frameOf,
+  reducedCopy,
+  takeRegion,
+} from "./screenshot";
 import {
   allLeases,
   bootedCount,
@@ -320,7 +327,7 @@ async function wake(sessionId: string, lease: Lease): Promise<void> {
 export async function runSimulator(
   sessionId: string,
   args: string[]
-): Promise<CallResult & { screenshot: string | null }> {
+): Promise<CallResult & { screenshot: string | null; read: string | null }> {
   const { config, cwd } = declared(sessionId);
   let lease = getLease(sessionId);
   if (!lease || lease.state === "failed")
@@ -333,23 +340,58 @@ export async function runSimulator(
     await wake(sessionId, lease);
     lease = getLease(sessionId)!;
   }
+  let region: string | null = null;
+  if (args[0] === "screenshot") {
+    try {
+      ({ args, region } = takeRegion(args));
+    } catch (error) {
+      throw new SimulatorError((error as Error).message, 400);
+    }
+  }
+  const target = { udid: lease.udid!, sessionId, app: config.app, cwd };
   let argv: string[];
   try {
-    argv = buildArgs(args, {
-      udid: lease.udid!,
-      sessionId,
-      app: config.app,
-      cwd,
-    });
+    argv = buildArgs(args, target);
   } catch (error) {
     throw new SimulatorError((error as Error).message, 400);
   }
   const shot = screenshotPath(argv);
   if (shot) fs.mkdirSync(path.dirname(shot), { recursive: true });
   touchLease(sessionId);
+  // The element's frame comes from a fresh snapshot, before the shot, so a
+  // bad ref fails without writing anything.
+  let located: ReturnType<typeof frameOf> | null = null;
+  if (region) {
+    const snap = await callTool(
+      buildArgs(["snapshot", "-i", "--json"], target)
+    );
+    try {
+      if (snap.code) throw new RegionError(snap.stderr || "snapshot failed");
+      located = frameOf(snap.stdout, region);
+    } catch (error) {
+      if (error instanceof RegionError)
+        throw new SimulatorError(error.message, 400);
+      throw error;
+    }
+  }
   const result = await callTool(argv);
+  const taken = shot && fs.existsSync(shot) ? shot : null;
+  let note = "";
+  if (taken && located) {
+    try {
+      await cropToFrame(taken, located.frame, located.screen);
+    } catch (error) {
+      note = `region crop failed (${(error as Error).message}); kept the full screen\n`;
+    }
+  }
+  const read = taken ? await reducedCopy(taken) : null;
   touchLease(sessionId);
-  return { ...result, screenshot: shot && fs.existsSync(shot) ? shot : null };
+  return {
+    ...result,
+    stderr: result.stderr + note,
+    screenshot: taken,
+    read,
+  };
 }
 
 export async function simulatorLogs(sessionId: string, last: string) {

package/lib/simulator/screenshot.ts

@@ -0,0 +1,180 @@
+/**
+ * Cheaper screenshots for the agent: every image it reads costs tokens on
+ * every later turn, so it reads a copy capped at READ_SIZE px (or just one
+ * element of the screen) while the full-size file stays for the PR.
+ */
+
+import { execFile } from "child_process";
+import fs from "fs";
+import path from "path";
+import { promisify } from "util";
+
+const execFileAsync = promisify(execFile);
+
+export const READ_SIZE = 800;
+// Points of context kept around a cropped element.
+export const REGION_MARGIN = 8;
+
+export type Sips = (args: string[]) => Promise<string>;
+
+const realSips: Sips = async (args) =>
+  (await execFileAsync("sips", args, { timeout: 30_000 })).stdout;
+let sips: Sips = realSips;
+export const setSips = (fn: Sips | null) => {
+  sips = fn ?? realSips;
+};
+
+export interface Frame {
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+}
+export interface Crop {
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+}
+
+export class RegionError extends Error {}
+
+// Where the agent reads from: a copy under shots/read, so the shots strip and
+// the PR files stay the full-size ones.
+export const readCopyPath = (full: string) =>
+  path.join(path.dirname(full), "read", path.basename(full));
+
+async function pixelSize(file: string) {
+  const out = await sips(["-g", "pixelWidth", "-g", "pixelHeight", file]);
+  const w = Number(/pixelWidth:\s*(\d+)/.exec(out)?.[1]);
+  const h = Number(/pixelHeight:\s*(\d+)/.exec(out)?.[1]);
+  if (!w || !h) throw new Error("cannot read image size");
+  return { width: w, height: h };
+}
+
+// The reduced copy, or the full file when resizing fails.
+export async function reducedCopy(full: string): Promise<string> {
+  const out = readCopyPath(full);
+  try {
+    fs.mkdirSync(path.dirname(out), { recursive: true });
+    await sips(["-Z", String(READ_SIZE), full, "--out", out]);
+    return fs.existsSync(out) ? out : full;
+  } catch {
+    return full;
+  }
+}
+
+// An element's frame (points) to a pixel rectangle of the image, with a
+// margin, clamped to the image. `screen` is the size of the frame's space.
+export function cropRect(
+  frame: Frame,
+  screen: { width: number; height: number },
+  image: { width: number; height: number },
+  margin = REGION_MARGIN
+): Crop {
+  const sx = image.width / screen.width;
+  const sy = image.height / screen.height;
+  const x0 = Math.max(0, Math.floor((frame.x - margin) * sx));
+  const y0 = Math.max(0, Math.floor((frame.y - margin) * sy));
+  const x1 = Math.min(
+    image.width,
+    Math.ceil((frame.x + frame.width + margin) * sx)
+  );
+  const y1 = Math.min(
+    image.height,
+    Math.ceil((frame.y + frame.height + margin) * sy)
+  );
+  if (x1 <= x0 || y1 <= y0)
+    throw new RegionError("that element is off the screen");
+  return { x: x0, y: y0, width: x1 - x0, height: y1 - y0 };
+}
+
+type Node = { ref?: unknown; rect?: Frame };
+
+// Every node with a ref and a rect, wherever the JSON nests them.
+function nodesOf(value: unknown, out: Node[] = []): Node[] {
+  if (Array.isArray(value)) value.forEach((v) => nodesOf(v, out));
+  else if (value && typeof value === "object") {
+    const n = value as Node;
+    if (typeof n.ref === "string" && n.rect) out.push(n);
+    for (const v of Object.values(value)) nodesOf(v, out);
+  }
+  return out;
+}
+
+const isFrame = (r: unknown): r is Frame =>
+  !!r &&
+  ["x", "y", "width", "height"].every(
+    (k) => typeof (r as Record<string, unknown>)[k] === "number"
+  );
+
+// The frame of `ref` in a `snapshot -i --json` output, and the screen size
+// (the widest, tallest frame: the app's window).
+export function frameOf(
+  json: string,
+  ref: string
+): { frame: Frame; screen: { width: number; height: number } } {
+  let nodes: Node[];
+  try {
+    nodes = nodesOf(JSON.parse(json)).filter((n) => isFrame(n.rect));
+  } catch {
+    throw new RegionError(
+      "could not read the snapshot; run `aos sim snapshot -i`"
+    );
+  }
+  const want = ref.replace(/^@/, "");
+  const hit = nodes.find((n) => String(n.ref).replace(/^@/, "") === want);
+  if (!hit)
+    throw new RegionError(
+      `unknown or stale ref @${want}: run \`aos sim snapshot -i\` and use a ref from it`
+    );
+  const screen = nodes.reduce(
+    (s, n) => ({
+      width: Math.max(s.width, n.rect!.x + n.rect!.width),
+      height: Math.max(s.height, n.rect!.y + n.rect!.height),
+    }),
+    { width: 0, height: 0 }
+  );
+  return { frame: hit.rect!, screen };
+}
+
+// Crops `file` in place to the rectangle.
+export async function cropInPlace(file: string, c: Crop): Promise<void> {
+  await sips([
+    "--cropToHeightWidth",
+    String(c.height),
+    String(c.width),
+    "--cropOffset",
+    String(c.y),
+    String(c.x),
+    file,
+  ]);
+}
+
+export async function cropToFrame(
+  file: string,
+  frame: Frame,
+  screen: { width: number; height: number }
+): Promise<void> {
+  await cropInPlace(file, cropRect(frame, screen, await pixelSize(file)));
+}
+
+// `--region <ref>` / `--region=<ref>` out of a screenshot's args.
+export function takeRegion(args: string[]): {
+  args: string[];
+  region: string | null;
+} {
+  const rest: string[] = [];
+  let region: string | null = null;
+  for (let i = 0; i < args.length; i++) {
+    const a = args[i];
+    if (a === "--region") {
+      region = args[++i] ?? "";
+      if (!region) throw new RegionError("--region needs a ref, like @e3");
+    } else if (a.startsWith("--region=")) region = a.slice(9);
+    else rest.push(a);
+  }
+  if (region !== null && !/^@?e?\d+$/.test(region) && !/^@\S+$/.test(region))
+    throw new RegionError("--region takes a ref from `aos sim snapshot -i`");
+  return { args: rest, region };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@saadnvd1/agent-os",
-  "version": "0.3.609",
+  "version": "0.3.611",
   "description": "Self-hosted web UI for managing Claude Code sessions",
   "bin": {
     "agent-os": "scripts/agent-os"