@saadnvd1/agent-os 0.3.610 → 0.3.611

package/app/api/sessions/[id]/simulator/run/route.ts

@@ -20,6 +20,7 @@ export async function POST(
         stderr: "",
         code: 0,
         screenshot: null,
+        read: null,
       });
     if (
       !Array.isArray(body.args) ||

package/lib/project-config/brief.ts

@@ -50,6 +50,10 @@ export function runningBrief(
     lines.push(
       `- \`${browse.map}\` maps every page to its URL: navigate by URL, don't explore.`
     );
+  if (browse)
+    lines.push(
+      "- Check state with the browser's text snapshot, not screenshots. Screenshot only the final changed screen, and just the changed element (its uid) rather than the full page where the tool allows; every image stays in your context."
+    );
   const sim = simulatorBrief(config.simulator);
   if (sim) lines.push(sim);
   for (const note of notesOf(config.notes))

package/lib/simulator/brief.ts

@@ -12,7 +12,8 @@ export function simulatorBrief(
     return "- The iOS simulator tools are not available on this machine; skip simulator checks and say so in the PR.";
   return [
     "- iOS simulator (optional; it never gates a PR or a build, and a failure is reported, not a blocker): `aos sim open` makes your own simulator and installs this worktree's build (run it with run_in_background, a cold build takes minutes; `aos sim status` shows where it is).",
-    '  Then `aos sim snapshot -i` (accessibility tree with @refs), `aos sim click @e3`, `aos sim fill @e5 "text"`, `aos sim screenshot` (prints a PNG path; read it), `aos sim logs --last 2m`. Refs are only valid from the latest snapshot.',
+    '  Drive and check state with `aos sim snapshot -i` (accessibility tree with @refs, text, cheap), `aos sim click @e3`, `aos sim fill @e5 "text"`, `aos sim logs --last 2m`. Refs are only valid from the latest snapshot.',
+    "  Every image you read stays in your context for every later turn, so screenshot only the final changed screens, and prefer `aos sim screenshot --region @e3` (one element from the latest snapshot) for a single row or banner. It prints two paths: read the `read:` one (reduced to 800px); attach the `full-size` file to the PR.",
     "  Screenshots are written outside the worktree: don't put them in commits or the PR body unless they belong there. `aos sim close` when you're done; it is closed for you when the task ends.",
   ].join("\n");
 }

package/lib/simulator/index.ts

@@ -24,6 +24,13 @@ import {
   simulatorDir,
   type CallResult,
 } from "./agent-device";
+import {
+  RegionError,
+  cropToFrame,
+  frameOf,
+  reducedCopy,
+  takeRegion,
+} from "./screenshot";
 import {
   allLeases,
   bootedCount,
@@ -320,7 +327,7 @@ async function wake(sessionId: string, lease: Lease): Promise<void> {
 export async function runSimulator(
   sessionId: string,
   args: string[]
-): Promise<CallResult & { screenshot: string | null }> {
+): Promise<CallResult & { screenshot: string | null; read: string | null }> {
   const { config, cwd } = declared(sessionId);
   let lease = getLease(sessionId);
   if (!lease || lease.state === "failed")
@@ -333,23 +340,58 @@ export async function runSimulator(
     await wake(sessionId, lease);
     lease = getLease(sessionId)!;
   }
+  let region: string | null = null;
+  if (args[0] === "screenshot") {
+    try {
+      ({ args, region } = takeRegion(args));
+    } catch (error) {
+      throw new SimulatorError((error as Error).message, 400);
+    }
+  }
+  const target = { udid: lease.udid!, sessionId, app: config.app, cwd };
   let argv: string[];
   try {
-    argv = buildArgs(args, {
-      udid: lease.udid!,
-      sessionId,
-      app: config.app,
-      cwd,
-    });
+    argv = buildArgs(args, target);
   } catch (error) {
     throw new SimulatorError((error as Error).message, 400);
   }
   const shot = screenshotPath(argv);
   if (shot) fs.mkdirSync(path.dirname(shot), { recursive: true });
   touchLease(sessionId);
+  // The element's frame comes from a fresh snapshot, before the shot, so a
+  // bad ref fails without writing anything.
+  let located: ReturnType<typeof frameOf> | null = null;
+  if (region) {
+    const snap = await callTool(
+      buildArgs(["snapshot", "-i", "--json"], target)
+    );
+    try {
+      if (snap.code) throw new RegionError(snap.stderr || "snapshot failed");
+      located = frameOf(snap.stdout, region);
+    } catch (error) {
+      if (error instanceof RegionError)
+        throw new SimulatorError(error.message, 400);
+      throw error;
+    }
+  }
   const result = await callTool(argv);
+  const taken = shot && fs.existsSync(shot) ? shot : null;
+  let note = "";
+  if (taken && located) {
+    try {
+      await cropToFrame(taken, located.frame, located.screen);
+    } catch (error) {
+      note = `region crop failed (${(error as Error).message}); kept the full screen\n`;
+    }
+  }
+  const read = taken ? await reducedCopy(taken) : null;
   touchLease(sessionId);
-  return { ...result, screenshot: shot && fs.existsSync(shot) ? shot : null };
+  return {
+    ...result,
+    stderr: result.stderr + note,
+    screenshot: taken,
+    read,
+  };
 }
 
 export async function simulatorLogs(sessionId: string, last: string) {

package/lib/simulator/screenshot.ts

@@ -0,0 +1,180 @@
+/**
+ * Cheaper screenshots for the agent: every image it reads costs tokens on
+ * every later turn, so it reads a copy capped at READ_SIZE px (or just one
+ * element of the screen) while the full-size file stays for the PR.
+ */
+
+import { execFile } from "child_process";
+import fs from "fs";
+import path from "path";
+import { promisify } from "util";
+
+const execFileAsync = promisify(execFile);
+
+export const READ_SIZE = 800;
+// Points of context kept around a cropped element.
+export const REGION_MARGIN = 8;
+
+export type Sips = (args: string[]) => Promise<string>;
+
+const realSips: Sips = async (args) =>
+  (await execFileAsync("sips", args, { timeout: 30_000 })).stdout;
+let sips: Sips = realSips;
+export const setSips = (fn: Sips | null) => {
+  sips = fn ?? realSips;
+};
+
+export interface Frame {
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+}
+export interface Crop {
+  x: number;
+  y: number;
+  width: number;
+  height: number;
+}
+
+export class RegionError extends Error {}
+
+// Where the agent reads from: a copy under shots/read, so the shots strip and
+// the PR files stay the full-size ones.
+export const readCopyPath = (full: string) =>
+  path.join(path.dirname(full), "read", path.basename(full));
+
+async function pixelSize(file: string) {
+  const out = await sips(["-g", "pixelWidth", "-g", "pixelHeight", file]);
+  const w = Number(/pixelWidth:\s*(\d+)/.exec(out)?.[1]);
+  const h = Number(/pixelHeight:\s*(\d+)/.exec(out)?.[1]);
+  if (!w || !h) throw new Error("cannot read image size");
+  return { width: w, height: h };
+}
+
+// The reduced copy, or the full file when resizing fails.
+export async function reducedCopy(full: string): Promise<string> {
+  const out = readCopyPath(full);
+  try {
+    fs.mkdirSync(path.dirname(out), { recursive: true });
+    await sips(["-Z", String(READ_SIZE), full, "--out", out]);
+    return fs.existsSync(out) ? out : full;
+  } catch {
+    return full;
+  }
+}
+
+// An element's frame (points) to a pixel rectangle of the image, with a
+// margin, clamped to the image. `screen` is the size of the frame's space.
+export function cropRect(
+  frame: Frame,
+  screen: { width: number; height: number },
+  image: { width: number; height: number },
+  margin = REGION_MARGIN
+): Crop {
+  const sx = image.width / screen.width;
+  const sy = image.height / screen.height;
+  const x0 = Math.max(0, Math.floor((frame.x - margin) * sx));
+  const y0 = Math.max(0, Math.floor((frame.y - margin) * sy));
+  const x1 = Math.min(
+    image.width,
+    Math.ceil((frame.x + frame.width + margin) * sx)
+  );
+  const y1 = Math.min(
+    image.height,
+    Math.ceil((frame.y + frame.height + margin) * sy)
+  );
+  if (x1 <= x0 || y1 <= y0)
+    throw new RegionError("that element is off the screen");
+  return { x: x0, y: y0, width: x1 - x0, height: y1 - y0 };
+}
+
+type Node = { ref?: unknown; rect?: Frame };
+
+// Every node with a ref and a rect, wherever the JSON nests them.
+function nodesOf(value: unknown, out: Node[] = []): Node[] {
+  if (Array.isArray(value)) value.forEach((v) => nodesOf(v, out));
+  else if (value && typeof value === "object") {
+    const n = value as Node;
+    if (typeof n.ref === "string" && n.rect) out.push(n);
+    for (const v of Object.values(value)) nodesOf(v, out);
+  }
+  return out;
+}
+
+const isFrame = (r: unknown): r is Frame =>
+  !!r &&
+  ["x", "y", "width", "height"].every(
+    (k) => typeof (r as Record<string, unknown>)[k] === "number"
+  );
+
+// The frame of `ref` in a `snapshot -i --json` output, and the screen size
+// (the widest, tallest frame: the app's window).
+export function frameOf(
+  json: string,
+  ref: string
+): { frame: Frame; screen: { width: number; height: number } } {
+  let nodes: Node[];
+  try {
+    nodes = nodesOf(JSON.parse(json)).filter((n) => isFrame(n.rect));
+  } catch {
+    throw new RegionError(
+      "could not read the snapshot; run `aos sim snapshot -i`"
+    );
+  }
+  const want = ref.replace(/^@/, "");
+  const hit = nodes.find((n) => String(n.ref).replace(/^@/, "") === want);
+  if (!hit)
+    throw new RegionError(
+      `unknown or stale ref @${want}: run \`aos sim snapshot -i\` and use a ref from it`
+    );
+  const screen = nodes.reduce(
+    (s, n) => ({
+      width: Math.max(s.width, n.rect!.x + n.rect!.width),
+      height: Math.max(s.height, n.rect!.y + n.rect!.height),
+    }),
+    { width: 0, height: 0 }
+  );
+  return { frame: hit.rect!, screen };
+}
+
+// Crops `file` in place to the rectangle.
+export async function cropInPlace(file: string, c: Crop): Promise<void> {
+  await sips([
+    "--cropToHeightWidth",
+    String(c.height),
+    String(c.width),
+    "--cropOffset",
+    String(c.y),
+    String(c.x),
+    file,
+  ]);
+}
+
+export async function cropToFrame(
+  file: string,
+  frame: Frame,
+  screen: { width: number; height: number }
+): Promise<void> {
+  await cropInPlace(file, cropRect(frame, screen, await pixelSize(file)));
+}
+
+// `--region <ref>` / `--region=<ref>` out of a screenshot's args.
+export function takeRegion(args: string[]): {
+  args: string[];
+  region: string | null;
+} {
+  const rest: string[] = [];
+  let region: string | null = null;
+  for (let i = 0; i < args.length; i++) {
+    const a = args[i];
+    if (a === "--region") {
+      region = args[++i] ?? "";
+      if (!region) throw new RegionError("--region needs a ref, like @e3");
+    } else if (a.startsWith("--region=")) region = a.slice(9);
+    else rest.push(a);
+  }
+  if (region !== null && !/^@?e?\d+$/.test(region) && !/^@\S+$/.test(region))
+    throw new RegionError("--region takes a ref from `aos sim snapshot -i`");
+  return { args: rest, region };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@saadnvd1/agent-os",
-  "version": "0.3.610",
+  "version": "0.3.611",
   "description": "Self-hosted web UI for managing Claude Code sessions",
   "bin": {
     "agent-os": "scripts/agent-os"