@opengeni/runtime 0.2.2 → 0.3.0

package/src/sandbox-computer.ts

@@ -67,10 +67,16 @@ const SCROLL_NOTCH_PIXELS = 100;
 const SCROLL_MAX_CLICKS = 15;
 // screenshot() never hands the model an empty image_url (the SDK turns "" into
 // `image_url: ''`, which the model API 400s). A cold/not-yet-painting :0 can yield
-// a zero-byte frame on the first scrot; bounded retries with a short pause let a
-// momentarily-unpainted-but-live display self-heal before we FAIL LOUD.
-const SCREENSHOT_MAX_ATTEMPTS = 3;
-const SCREENSHOT_RETRY_DELAY_MS = 400;
+// zero-byte frames for the WHOLE warm-up window of a freshly cold-booted box — Xvfb
+// + XFCE + dbus + font-cache under gVisor routinely take 20s+, and the recovery path
+// after a machine→sandbox swap ALWAYS hits a stone-cold Modal box on its first turn.
+// So we retry across a bounded WALL-CLOCK budget (not a tiny fixed attempt count) with
+// a short pause between tries, so that first post-cold / post-swap screenshot self-heals
+// as the display warms — then FAIL LOUD once the budget is genuinely spent (a display
+// that is dead, not merely warming). ~800ms of retries (the prior 3×400ms) was far too
+// short to ride out a cold gVisor XFCE boot, so the turn failed loud on a transient.
+const SCREENSHOT_WARMUP_BUDGET_MS = 30_000;
+const SCREENSHOT_RETRY_DELAY_MS = 750;
 
 export type SandboxComputerOptions = {
   display?: string; // ":0"
@@ -79,6 +85,11 @@ export type SandboxComputerOptions = {
   typeDelayMs?: number; // xdotool type --delay (default 12ms)
   readOnly?: boolean; // when true, every WRITE action throws ComputerReadOnlyError
   screenshotTmpDir?: string; // "/tmp"
+  // How long screenshot() keeps retrying an empty (still-warming) frame before it
+  // FAILS LOUD, and the pause between tries. Defaults to the cold-boot warm-up budget;
+  // exposed mainly so tests can shrink it (a real caller wants the full budget).
+  screenshotWarmupBudgetMs?: number;
+  screenshotRetryDelayMs?: number;
 };
 
 // X keysym map for keypress(): model key names → xdotool keysyms.
@@ -144,6 +155,8 @@ export class SandboxComputer implements Computer {
   private readonly typeDelayMs: number;
   private readonly readOnly: boolean;
   private readonly tmp: string;
+  private readonly screenshotWarmupBudgetMs: number;
+  private readonly screenshotRetryDelayMs: number;
 
   constructor(session: SandboxSessionLike, opts: SandboxComputerOptions = {}) {
     this.session = session as unknown as ComputerSession;
@@ -155,6 +168,8 @@ export class SandboxComputer implements Computer {
     this.typeDelayMs = opts.typeDelayMs ?? 12;
     this.readOnly = opts.readOnly ?? false;
     this.tmp = opts.screenshotTmpDir ?? "/tmp";
+    this.screenshotWarmupBudgetMs = opts.screenshotWarmupBudgetMs ?? SCREENSHOT_WARMUP_BUDGET_MS;
+    this.screenshotRetryDelayMs = opts.screenshotRetryDelayMs ?? SCREENSHOT_RETRY_DELAY_MS;
   }
 
   /** Rebind to a freshly resumed-by-id session after a box rollover / re-establish. */
@@ -231,17 +246,23 @@ export class SandboxComputer implements Computer {
     // but momentarily not painting (XFCE/dbus still warming) recovers without
     // failing the turn.
     let lastError: unknown;
-    for (let attempt = 0; attempt < SCREENSHOT_MAX_ATTEMPTS; attempt++) {
+    const deadline = Date.now() + this.screenshotWarmupBudgetMs;
+    let attempt = 0;
+    // Retry across a WALL-CLOCK budget (not a fixed count): a stone-cold box on the
+    // first post-swap / post-cold turn can take 20s+ to paint, and a zero-byte frame
+    // is a KNOWN transient during that warm-up — not a reason to fail the turn.
+    while (true) {
       if (attempt > 0) {
-        await new Promise((r) => setTimeout(r, SCREENSHOT_RETRY_DELAY_MS));
+        await new Promise((r) => setTimeout(r, this.screenshotRetryDelayMs));
       }
+      attempt++;
       const f = `${this.tmp}/og-shot-${Date.now()}-${Math.random().toString(36).slice(2)}.png`;
       try {
         await this.x(`scrot --pointer --overwrite ${f}`);
         const bytes = await this.readScreenshotBytes(f);
         if (bytes.length === 0) {
           // A cold/not-yet-painting :0 yields a zero-byte frame. Retry rather than
-          // hand the model an empty image_url; throw on the final attempt.
+          // hand the model an empty image_url; throw once the budget is spent.
           throw new ComputerUnavailableError("scrot produced an empty screenshot (display not up?)");
         }
         return Buffer.from(bytes).toString("base64");
@@ -252,9 +273,15 @@ export class SandboxComputer implements Computer {
         // screenshot result.
         await this.x(`rm -f ${f}`).catch(() => undefined);
       }
+      // Stop once the warm-up budget is spent — the NEXT sleep would push us past it.
+      if (Date.now() + this.screenshotRetryDelayMs >= deadline) {
+        break;
+      }
     }
-    // Exhausted retries: FAIL LOUD. A clear throw is the only acceptable outcome —
-    // returning "" here would surface to the model as an invalid empty image_url.
+    // Exhausted the warm-up budget: FAIL LOUD. A clear throw is the only acceptable
+    // outcome — returning "" here would surface to the model as an invalid empty
+    // image_url. Reaching here means the display was still dead after ~30s, not merely
+    // warming, so a hard action failure is correct.
     if (lastError instanceof Error) {
       throw lastError;
     }
@@ -358,7 +385,11 @@ export class SandboxComputer implements Computer {
  *  leaf; the duck-typed `isNativeDesktopSession` probe (below) selects on it. */
 export type NativeDesktopSession = {
   desktopInput(event: DesktopInputRequest["event"]): Promise<void>;
-  screenshot(): Promise<{ png: Uint8Array; width: number; height: number }>;
+  // `nativeWidth`/`nativeHeight` are the PRE-DOWNSCALE capture geometry (equal to
+  // width/height when the agent did not have to shrink the PNG to fit the transport
+  // budget). NativeDesktopComputer scales model clicks from the ENCODED pixel space
+  // back to native pixels using their ratio before injecting.
+  screenshot(): Promise<{ png: Uint8Array; width: number; height: number; nativeWidth: number; nativeHeight: number }>;
 };
 
 /** Model `Button` → wire `PointerButton`. The proto has no back/forward button, so
@@ -397,6 +428,15 @@ export class NativeDesktopComputer implements Computer {
   readonly dimensions: [number, number];
   private session: NativeDesktopSession;
   private readonly readOnly: boolean;
+  // The ENCODED vs NATIVE geometry of the MOST RECENT screenshot the model saw. The
+  // model computes click coordinates in the encoded-pixel space of that screenshot;
+  // when the agent downscaled the PNG to fit the transport budget, encoded < native,
+  // so we scale coordinates back up to native pixels before injecting (the agent's
+  // native inject — macOS CGEvent / Linux XTEST — expects native-pixel coordinates,
+  // exactly as it received them pre-downscale). Null until the first screenshot;
+  // equal encoded==native (or absent) ⇒ scale factor 1.0 ⇒ byte-identical behavior.
+  private lastEncoded: [number, number] | null = null;
+  private lastNative: [number, number] | null = null;
 
   constructor(session: NativeDesktopSession, opts: NativeDesktopComputerOptions = {}) {
     this.session = session;
@@ -414,17 +454,34 @@ export class NativeDesktopComputer implements Computer {
     if (this.readOnly) throw new ComputerReadOnlyError();
   }
 
+  /** Scale a coordinate the model expressed in the MOST RECENT screenshot's
+   *  ENCODED pixel space back to NATIVE pixels. When the last frame was not
+   *  downscaled (encoded == native), or no screenshot has been taken yet, this is a
+   *  1:1 identity — the byte-identical current behavior. The agent then applies its
+   *  own platform mapping (macOS divides native pixels by the backing scale to reach
+   *  CGEvent points; Linux XTEST is 1:1) exactly as it did pre-downscale. */
+  private toNative(x: number, y: number): { x: number; y: number } {
+    const enc = this.lastEncoded;
+    const nat = this.lastNative;
+    if (!enc || !nat || enc[0] <= 0 || enc[1] <= 0) return { x, y };
+    if (enc[0] === nat[0] && enc[1] === nat[1]) return { x, y };
+    return {
+      x: Math.round((x * nat[0]) / enc[0]),
+      y: Math.round((y * nat[1]) / enc[1]),
+    };
+  }
+
   private async pointer(x: number, y: number, action: PointerAction, button: PointerButton): Promise<void> {
-    // COORDINATE SEAM — TODO(verify e2e on macOS): the model computes x/y against the
-    // pixels of the screenshot it just saw, and the agent's macOS CGEvent inject
-    // treats x/y as raw screen coordinates. On a Retina Mac, ScreenCaptureKit may
-    // capture at 2× the logical POINT space while CGEvent expects logical points — a
-    // potential 2× mismatch between the coords the model derives and the coords the
-    // inject applies. This MUST be measured on a real Retina Mac (compare the
-    // screenshot's reported width/height against the logical display bounds) before
-    // any DPR scaling is added. Do NOT add scaling speculatively. Self-hosted Linux
-    // (XTEST/x11) is 1:1 and unaffected.
-    await this.session.desktopInput({ $case: "pointer", pointer: { x, y, action, button } });
+    // COORDINATE SEAM: the model computes x/y against the pixels of the screenshot it
+    // just saw — which the agent may have DOWNSCALED to fit the transport's max
+    // payload (a full-res Retina/busy screen exceeds NATS's 1 MiB default). We scale
+    // those encoded-pixel coordinates back to native pixels here, using the native
+    // geometry the last screenshot reported, so the agent's native inject lands the
+    // click where the model intended. When no downscale occurred the factor is 1.0
+    // and the coordinates pass through unchanged. Self-hosted Linux (XTEST/x11) and
+    // any non-downscaled frame are 1:1 and unaffected.
+    const n = this.toNative(x, y);
+    await this.session.desktopInput({ $case: "pointer", pointer: { x: n.x, y: n.y, action, button } });
   }
 
   async screenshot(): Promise<string> {
@@ -434,10 +491,14 @@ export class NativeDesktopComputer implements Computer {
     // missing/empty frame is therefore a THROW, never a silent "". Native capture
     // (ScreenCaptureKit / x11) does not have the cold-scrot warm-up the xdotool path
     // retries around, so a single capture + a hard empty-guard is sufficient.
-    const { png } = await this.session.screenshot();
+    const { png, width, height, nativeWidth, nativeHeight } = await this.session.screenshot();
     if (png.length === 0) {
       throw new ComputerUnavailableError("native desktop screenshot returned an empty frame (display not up?)");
     }
+    // Record the encoded (what the model sees) vs native geometry of THIS frame so
+    // the next click/move/scroll/drag scales its coordinates back to native pixels.
+    this.lastEncoded = [width, height];
+    this.lastNative = [nativeWidth || width, nativeHeight || height];
     return Buffer.from(png).toString("base64");
   }
 
@@ -455,11 +516,12 @@ export class NativeDesktopComputer implements Computer {
   }
   async scroll(x: number, y: number, sx: number, sy: number) {
     this.guardWrite();
-    // The model's scroll deltas are PIXELS — forward them straight to the agent as a
-    // ScrollEvent{x,y,deltaX,deltaY} and let the native inject translate to wheel
-    // events per platform. No xdotool "notch" quantization here (that is an
-    // xdotool-specific artifact); the agent owns the platform-appropriate scaling.
-    await this.session.desktopInput({ $case: "scroll", scroll: { x, y, deltaX: sx, deltaY: sy } });
+    // The scroll ANCHOR (x,y) is a screenshot-pixel position → scale to native like a
+    // click. The deltas (sx,sy) are relative scroll AMOUNTS, not positions; the agent
+    // owns the platform-appropriate wheel translation, so they pass through unscaled
+    // (no xdotool "notch" quantization here — that is an xdotool-specific artifact).
+    const n = this.toNative(x, y);
+    await this.session.desktopInput({ $case: "scroll", scroll: { x: n.x, y: n.y, deltaX: sx, deltaY: sy } });
   }
   async type(text: string) {
     this.guardWrite();