@saadnvd1/agent-os 0.3.521 → 0.3.524

package/lib/chat/settings.ts

@@ -75,6 +75,7 @@ export function emitCapabilities(session: Session, listener?: Listener): void {
     model: chatModel(session),
     access: session.chat_access,
     plan: canPlan(session) ? !!session.chat_plan : null,
+    background: chatDriverFor(session.agent_type)?.backgroundTasks === true,
   };
   if (listener) listener(m);
   else emit(session.id, m);

package/lib/chat/socket.ts

@@ -12,6 +12,7 @@ import {
   carryOutPlan,
   chatFileSuggestions,
   chatHistory,
+  backgroundChatTask,
   chatTaskOutput,
   chatToolBody,
   deleteQueuedChat,
@@ -183,6 +184,14 @@ function onChatMessage(
     run(carryOutPlan(sessionId, String(msg.id)));
   else if (msg.type === "respond") respondChat(sessionId, msg.id, msg);
   else if (msg.type === "stop_task") stopChatTask(sessionId, msg.taskId);
+  else if (msg.type === "background_task")
+    run(
+      backgroundChatTask(
+        sessionId,
+        typeof msg.toolUseId === "string" ? msg.toolUseId : undefined,
+        reply
+      )
+    );
   else if (msg.type === "task_output")
     reply({
       type: "task_output",

package/lib/chat/worker/client.ts

@@ -41,6 +41,11 @@ export class WorkerClient {
 
   private files = new Map<string, (r: FileSuggestion[] | null) => void>();
 
+  private backgrounds = new Map<
+    string,
+    (r: { ok?: boolean; error?: string }) => void
+  >();
+
   private constructor(
     private socket: net.Socket,
     private handlers: WorkerHandlers
@@ -55,6 +60,9 @@ export class WorkerClient {
       this.undos.clear();
       for (const f of this.files.values()) f(null);
       this.files.clear();
+      for (const b of this.backgrounds.values())
+        b({ error: "The chat worker went away" });
+      this.backgrounds.clear();
       handlers.onClose(this.detached);
     });
     socket.on("error", () => socket.destroy());
@@ -101,6 +109,11 @@ export class WorkerClient {
       else u?.reject(new Error(e.error ?? "Undo failed"));
       return;
     }
+    if (e.type === "background_result") {
+      this.backgrounds.get(e.reqId)?.({ ok: e.ok, error: e.error });
+      this.backgrounds.delete(e.reqId);
+      return;
+    }
     if (e.type === "files_result") {
       this.files.get(e.reqId)?.(e.files ?? null);
       this.files.delete(e.reqId);
@@ -121,6 +134,26 @@ export class WorkerClient {
     });
   }
 
+  // Moves a blocking call (or all of them) to the background: whether
+  // anything moved, or why not (an error, or no answer in time).
+  background(
+    toolUseId?: string,
+    timeoutMs = 5000
+  ): Promise<{ ok?: boolean; error?: string }> {
+    const reqId = randomUUID();
+    return new Promise((resolve) => {
+      const timer = setTimeout(() => {
+        this.backgrounds.delete(reqId);
+        resolve({ error: "timeout" });
+      }, timeoutMs);
+      this.backgrounds.set(reqId, (r) => {
+        clearTimeout(timer);
+        resolve(r);
+      });
+      this.command({ type: "background_task", reqId, toolUseId });
+    });
+  }
+
   // The agent's own matches for an @mention; null when it can't say (an
   // older worker, an error, no answer in time).
   fileSuggestions(

package/lib/chat/worker/host.ts

@@ -712,6 +712,19 @@ export class ChatHost {
         return this.conversation.respond(cmd.id, cmd);
       case "stop_task":
         return this.conversation.stopTask(cmd.taskId);
+      case "background_task":
+        try {
+          if (!this.conversation.backgroundTask) throw new Error("unsupported");
+          const ok = await this.conversation.backgroundTask(cmd.toolUseId);
+          this.emit({ type: "background_result", reqId: cmd.reqId, ok });
+        } catch (error) {
+          this.emit({
+            type: "background_result",
+            reqId: cmd.reqId,
+            error: error instanceof Error ? error.message : String(error),
+          });
+        }
+        return;
       case "undo":
         try {
           const result = await this.conversation.undo(

package/lib/chat/worker/main.ts

@@ -76,7 +76,7 @@ async function main(sessionId: string) {
         state: host.state,
         compacting: host.compactingNow,
         streaming: [...host.streaming.values()],
-        caps: ["plan", "queue", "retire", "fresh"],
+        caps: ["plan", "queue", "retire", "fresh", "background"],
       } satisfies WorkerEvent)}\n`
     );
     client.on(

package/lib/chat/worker/protocol.ts

@@ -55,6 +55,7 @@ export type WorkerCommand =
   | ({ type: "respond"; id: string } & ApprovalDecision)
   | { type: "undo"; reqId: string; checkpoint: string; dryRun: boolean }
   | { type: "stop_task"; taskId: string }
+  | { type: "background_task"; reqId: string; toolUseId?: string }
   // It runs an older build: it takes no new message into its agent (they
   // wait in the queue) and closes at its next turn boundary, so a current
   // worker resumes the conversation with today's tools.
@@ -99,6 +100,7 @@ export type WorkerEvent =
   // The agent's guess at the next message, or null once it's stale.
   | { type: "suggestion"; text: string | null }
   | { type: "undo_result"; reqId: string; result?: UndoResult; error?: string }
+  | { type: "background_result"; reqId: string; ok?: boolean; error?: string }
   | {
       type: "files_result";
       reqId: string;

package/lib/db/migrations.ts

@@ -1768,6 +1768,25 @@ const migrations: Migration[] = [
       );
     },
   },
+  {
+    id: 77,
+    name: "simulator_leases",
+    up: (db) => {
+      // One iOS simulator a task holds (lib/simulator): the device AgentOS
+      // made for it, and where it is in coming up.
+      db.exec(
+        `CREATE TABLE IF NOT EXISTS simulator_leases (
+           session_id TEXT PRIMARY KEY,
+           udid TEXT,
+           name TEXT NOT NULL,
+           state TEXT NOT NULL,
+           error TEXT,
+           created_at INTEGER NOT NULL,
+           last_used_at INTEGER NOT NULL
+         )`
+      );
+    },
+  },
 ];
 
 // `upTo`: stop after this id (tests that start from an older database).

package/lib/project-config/agentos.schema.json

@@ -132,6 +132,25 @@
       },
       "additionalProperties": false
     },
+    "simulator": {
+      "type": "object",
+      "properties": {
+        "start": {
+          "type": "string",
+          "minLength": 1
+        },
+        "app": {
+          "type": "string",
+          "minLength": 1
+        },
+        "device": {
+          "type": "string",
+          "minLength": 1
+        }
+      },
+      "required": ["start"],
+      "additionalProperties": false
+    },
     "database": {
       "type": "object",
       "properties": {

package/lib/project-config/brief.ts

@@ -13,6 +13,7 @@ import {
   type ProjectConfig,
 } from "./index";
 import { databaseBrief, type SessionDatabase } from "./database";
+import { simulatorBrief } from "../simulator/brief";
 
 export function runningBrief(
   config: ProjectConfig,
@@ -49,6 +50,8 @@ export function runningBrief(
     lines.push(
       `- \`${browse.map}\` maps every page to its URL: navigate by URL, don't explore.`
     );
+  const sim = simulatorBrief(config.simulator);
+  if (sim) lines.push(sim);
   for (const note of notesOf(config.notes))
     lines.push(`- ${withPorts(note, ports)}`);
   return lines.length ? `## Running this project\n\n${lines.join("\n")}` : "";

package/lib/project-config/schema.ts

@@ -60,6 +60,15 @@ const browse = z.strictObject({
   port: envName.optional(),
 });
 
+// An iOS simulator tasks may drive on a Mac (lib/simulator). `start` runs in
+// the task's worktree with SIMULATOR_UDID set, builds and installs the app
+// on that device and exits 0 once it is up.
+const simulator = z.strictObject({
+  start: z.string().min(1),
+  app: z.string().min(1).optional(),
+  device: z.string().min(1).optional(),
+});
+
 const database = z.strictObject({
   from: z.string().min(1),
   env: projectEnvName.optional(),
@@ -109,6 +118,7 @@ const shape = {
   test: z.string().min(1).optional(),
   notes: notes.optional(),
   browse: browse.optional(),
+  simulator: simulator.optional(),
   database: database.optional(),
   mcp: z.array(z.string().min(1)).optional(),
   model: z.string().min(1).optional(),

package/lib/security/auth.ts

@@ -162,8 +162,18 @@ export function authorize(req: AuthRequest, policy: AuthPolicy): AuthResult {
 
 function decide(req: AuthRequest, policy: AuthPolicy): AuthResult {
   const trusted = req.tunnel ? null : networkTrust(req, policy);
-  if (trusted) return { ok: true, via: trusted };
   const token = readToken(req.headers);
+  if (trusted) {
+    // Identity only: network trust already decided access, so a token that
+    // is unknown, revoked or fails to resolve never turns this into a refusal.
+    let deviceId: string | undefined;
+    try {
+      deviceId = (token ? policy.lookup(token) : null)?.id;
+    } catch {}
+    return deviceId
+      ? { ok: true, via: trusted, deviceId }
+      : { ok: true, via: trusted };
+  }
   const device = token ? policy.lookup(token) : null;
   if (device) return { ok: true, via: "device", deviceId: device.id };
   // AGENTOS_AUTH=off is for a login proxy on this side of the network.

package/lib/sessions/end.ts

@@ -2,6 +2,7 @@ import type { Session } from "../db";
 import { stopChat } from "../chat/runner";
 import { waitForExit } from "../chat/worker/client";
 import { endTmuxSession } from "../tmux/end";
+import { releaseOnEnd } from "../simulator";
 
 type Agent = Pick<Session, "id" | "host_id" | "tmux_name">;
 
@@ -10,6 +11,8 @@ type Agent = Pick<Session, "id" | "host_id" | "tmux_name">;
 // finish share.
 export async function endSessionAgent(s: Agent): Promise<void> {
   stopChat(s.id);
+  // The task's simulator goes with it; no row, no work.
+  releaseOnEnd(s.id, { remove: true });
   void waitForExit(s.id).catch(() => {});
   await endTmuxSession(s.host_id, s.tmux_name);
 }

package/lib/sessions/stop.ts

@@ -3,6 +3,7 @@ import { hostLink } from "../hosts/remote-api";
 import { stopChatTurn } from "../chat/stop";
 import type { Session } from "../db";
 import { stopHeldTask } from "../tasks/held-remote";
+import { releaseOnEnd } from "../simulator";
 
 // Stops a session's agent and keeps everything else, the same for every
 // caller (the orchestrator's stop tool, `aos stop`): a chat's turn ends
@@ -33,10 +34,12 @@ export async function stopSession(
     const out = await stopChatTurn(s.id);
     if (!out.stopped)
       throw new Error(`Couldn't stop ${s.name}: ${out.reason}.`);
+    releaseOnEnd(s.id, { remove: false });
     return out.killed
       ? `Stopped ${s.name}: its agent didn't answer the interrupt, so it was ended. Its conversation and worktree are kept; your next message runs as its next turn.`
       : `Stopped ${s.name}'s turn. Your next message runs as its next turn.`;
   }
   await endTmuxSession(s.host_id, s.tmux_name);
+  releaseOnEnd(s.id, { remove: false });
   return `Stopped ${s.name}: its agent is ended, its worktree and branch are kept.`;
 }

package/lib/simulator/agent-device.ts

@@ -0,0 +1,258 @@
+/**
+ * Driving a simulator: the `agent-device` CLI (an XCTest runner behind an
+ * accessibility-tree snapshot with element refs), pinned and installed on
+ * first use. The task never names a device or session: they are injected
+ * here, so it can only drive its own simulator.
+ */
+
+import { execFile } from "child_process";
+import fs from "fs";
+import os from "os";
+import path from "path";
+import { promisify } from "util";
+
+const execFileAsync = promisify(execFile);
+
+export const AGENT_DEVICE_VERSION = "0.21.27";
+
+const root = () => path.join(os.homedir(), ".agent-os");
+export const toolDir = () =>
+  path.join(root(), "tools", `agent-device-${AGENT_DEVICE_VERSION}`);
+export const toolBin = () =>
+  path.join(toolDir(), "node_modules/.bin/agent-device");
+// Its daemon, device claims and XCTest runner cache stay apart from any other
+// copy of the tool on this Mac.
+export const stateDir = () => path.join(root(), "agent-device");
+export const simulatorDir = (sessionId: string) =>
+  path.join(root(), "simulators", sessionId);
+export const shotsDir = (sessionId: string) =>
+  path.join(simulatorDir(sessionId), "shots");
+
+export const ALLOWED = new Set([
+  "snapshot",
+  "click",
+  "press",
+  "longpress",
+  "fill",
+  "type",
+  "scroll",
+  "swipe",
+  "back",
+  "home",
+  "find",
+  "get",
+  "wait",
+  "screenshot",
+  "apps",
+  "relaunch",
+]);
+
+// Flags that pick a device, a session or where the tool keeps its state.
+const REFUSED = new Set([
+  "--udid",
+  "--device",
+  "--serial",
+  "--session",
+  "--state-dir",
+  "--platform",
+  "--ios-simulator-device-set",
+  "--ios-xctest-derived-data-path",
+]);
+
+export interface Target {
+  udid: string;
+  sessionId: string;
+  app?: string | null;
+  // The task's worktree: with the shots dir, the only place a screenshot may
+  // be written. Relative paths are taken from here.
+  cwd?: string | null;
+}
+
+// The tool's session on the task's device, attached to the app.
+export function openArgs(target: Target): string[] {
+  return [
+    "open",
+    target.app ?? "",
+    "--platform",
+    "ios",
+    "--udid",
+    target.udid,
+    "--session",
+    agentSession(target.sessionId),
+  ];
+}
+
+export const agentSession = (sessionId: string) =>
+  `aos-${sessionId.replace(/-/g, "").slice(0, 8)}`;
+
+// The argv for one call, or why it is refused.
+export function buildArgs(
+  args: string[],
+  target: Target,
+  now = () => Date.now()
+): string[] {
+  const [sub, ...rest] = args;
+  if (!sub || !ALLOWED.has(sub))
+    throw new Error(
+      `aos sim ${sub ?? ""}: not allowed. Use: ${[...ALLOWED].join(", ")}`
+    );
+  for (const a of rest) {
+    const flag = a.split("=")[0];
+    if (REFUSED.has(flag))
+      throw new Error(
+        `${flag} is set for you: the simulator is this task's own`
+      );
+  }
+  // `aos sim open` is AgentOS's own; the tool's `open` is how a session is
+  // attached to the app, which bring-up does and `relaunch` repeats.
+  const out =
+    sub === "relaunch"
+      ? ["open", ...(target.app ? [target.app] : []), "--relaunch", ...rest]
+      : [sub, ...rest];
+  if (sub === "screenshot") {
+    const at = shotArg(out);
+    if (!at) out.push(path.join(shotsDir(target.sessionId), `${now()}.png`));
+    else
+      out[at.index] =
+        at.prefix + safeShotPath(out[at.index].slice(at.prefix.length), target);
+  }
+  return [
+    ...out,
+    "--platform",
+    "ios",
+    "--udid",
+    target.udid,
+    "--session",
+    agentSession(target.sessionId),
+  ];
+}
+
+const within = (dir: string, file: string) => {
+  const rel = path.relative(dir, file);
+  return rel !== "" && !rel.startsWith("..") && !path.isAbsolute(rel);
+};
+
+// Resolves through symlinks as far as the path exists, so a link inside the
+// worktree can't lead the write elsewhere.
+function realish(file: string): string {
+  const rest: string[] = [];
+  let dir = file;
+  for (;;) {
+    try {
+      return path.join(fs.realpathSync(dir), ...rest.reverse());
+    } catch {
+      const up = path.dirname(dir);
+      if (up === dir) return file;
+      rest.push(path.basename(dir));
+      dir = up;
+    }
+  }
+}
+
+// A screenshot goes under the session's shots dir or the task's worktree, as
+// a .png; anything else is refused.
+function safeShotPath(given: string, target: Target): string {
+  const base = target.cwd ?? shotsDir(target.sessionId);
+  const full = realish(path.resolve(base, given));
+  const roots = [shotsDir(target.sessionId), target.cwd].flatMap((r) =>
+    r ? [realish(r)] : []
+  );
+  if (!full.endsWith(".png") || !roots.some((r) => within(r, full)))
+    throw new Error(
+      "screenshot: the path must be a .png under this task's worktree or its simulator shots folder"
+    );
+  return full;
+}
+
+const VALUE_FLAGS = new Set(["--crop-on", "--pixel-density", "--scale"]);
+
+// Where a screenshot argv names its file: a positional path or --out.
+function shotArg(argv: string[]): { index: number; prefix: string } | null {
+  for (let i = 1; i < argv.length; i++) {
+    const a = argv[i];
+    if (a === "--out")
+      return argv[i + 1] === undefined ? null : { index: i + 1, prefix: "" };
+    if (a.startsWith("--out=")) return { index: i, prefix: "--out=" };
+    if (VALUE_FLAGS.has(a)) i++;
+    else if (!a.startsWith("-")) return { index: i, prefix: "" };
+  }
+  return null;
+}
+
+// The screenshot path a built argv will write, if it is one.
+export const screenshotPath = (argv: string[]): string | null => {
+  if (argv[0] !== "screenshot") return null;
+  const at = shotArg(argv);
+  return at ? argv[at.index].slice(at.prefix.length) : null;
+};
+
+// What the tool and its install get: no tokens or project secrets from the
+// server's environment, only what it needs to find Xcode and a home.
+const ENV_KEYS = ["PATH", "HOME", "TMPDIR", "LANG", "DEVELOPER_DIR"];
+export const toolEnv = (): NodeJS.ProcessEnv =>
+  ({
+    ...Object.fromEntries(
+      ENV_KEYS.filter((k) => process.env[k]).map((k) => [k, process.env[k]])
+    ),
+    AGENT_DEVICE_STATE_DIR: stateDir(),
+  }) as unknown as NodeJS.ProcessEnv;
+
+let installing: Promise<void> | null = null;
+
+export function ensureInstalled(): Promise<void> {
+  if (fs.existsSync(toolBin())) return Promise.resolve();
+  installing ??= (async () => {
+    fs.mkdirSync(toolDir(), { recursive: true });
+    await execFileAsync(
+      "npm",
+      [
+        "install",
+        "--prefix",
+        toolDir(),
+        "--no-audit",
+        "--no-fund",
+        // agent-device has no install scripts; none are run for it or its deps.
+        "--ignore-scripts",
+        `agent-device@${AGENT_DEVICE_VERSION}`,
+      ],
+      { timeout: 300_000, env: toolEnv() }
+    );
+  })().finally(() => {
+    installing = null;
+  });
+  return installing;
+}
+
+export interface CallResult {
+  stdout: string;
+  stderr: string;
+  code: number;
+}
+
+export async function callTool(
+  argv: string[],
+  timeoutMs = 120_000
+): Promise<CallResult> {
+  await ensureInstalled();
+  fs.mkdirSync(stateDir(), { recursive: true });
+  try {
+    const { stdout, stderr } = await execFileAsync(toolBin(), argv, {
+      env: toolEnv(),
+      timeout: timeoutMs,
+      maxBuffer: 16 * 1024 * 1024,
+    });
+    return { stdout, stderr, code: 0 };
+  } catch (error) {
+    const e = error as {
+      stdout?: string;
+      stderr?: string;
+      code?: number | string;
+      message: string;
+    };
+    return {
+      stdout: e.stdout ?? "",
+      stderr: e.stderr || e.message,
+      code: typeof e.code === "number" ? e.code : 1,
+    };
+  }
+}

package/lib/simulator/available.ts

@@ -0,0 +1,5 @@
+import fs from "fs";
+
+// xcrun only exists on a Mac with Xcode.
+export const simulatorAvailable = (): boolean =>
+  process.platform === "darwin" && fs.existsSync("/usr/bin/xcrun");

package/lib/simulator/brief.ts

@@ -0,0 +1,18 @@
+import type { ProjectConfig } from "../project-config/schema";
+import { simulatorAvailable } from "./available";
+
+// What a task is told about the simulator: nothing when the project declares
+// none, one line when this machine can't run one.
+export function simulatorBrief(
+  sim: ProjectConfig["simulator"],
+  available = simulatorAvailable()
+): string {
+  if (!sim) return "";
+  if (!available)
+    return "- The iOS simulator tools are not available on this machine; skip simulator checks and say so in the PR.";
+  return [
+    "- iOS simulator (optional; it never gates a PR or a build, and a failure is reported, not a blocker): `aos sim open` makes your own simulator and installs this worktree's build (run it with run_in_background, a cold build takes minutes; `aos sim status` shows where it is).",
+    '  Then `aos sim snapshot -i` (accessibility tree with @refs), `aos sim click @e3`, `aos sim fill @e5 "text"`, `aos sim screenshot` (prints a PNG path; read it), `aos sim logs --last 2m`. Refs are only valid from the latest snapshot.',
+    "  Screenshots are written outside the worktree: don't put them in commits or the PR body unless they belong there. `aos sim close` when you're done; it is closed for you when the task ends.",
+  ].join("\n");
+}

package/lib/simulator/http.ts

@@ -0,0 +1,16 @@
+import { NextResponse } from "next/server";
+import { SimulatorError } from "./index";
+
+// A simulator call's failure as a response: its own errors carry their status.
+export function simulatorFailure(error: unknown) {
+  if (error instanceof SimulatorError)
+    return NextResponse.json(
+      { error: error.message },
+      { status: error.status }
+    );
+  console.error("[simulator]", error);
+  return NextResponse.json(
+    { error: error instanceof Error ? error.message : String(error) },
+    { status: 500 }
+  );
+}