@opengeni/runtime 0.2.3 → 0.3.0

package/src/sandbox/providers/modal.ts

@@ -1,9 +1,53 @@
 import { ModalImageSelector, ModalSandboxClient } from "@openai/agents-extensions/sandbox/modal";
 import { effectiveModalIdleTimeoutSeconds } from "@opengeni/config";
+import type { Settings } from "@opengeni/config";
 import { CAPABILITY_DESCRIPTORS } from "../capabilities";
 import { SandboxConfigError } from "../errors";
 import type { ProviderRegistration } from "./types";
 
+const MODAL_ORPHAN_SWEEP_LIMIT = 50;
+const MODAL_UNATTRIBUTED_ORPHAN_GRACE_MS = 30 * 60_000;
+
+export type ModalSandboxAttribution = {
+  leaseId: string;
+  workspaceId: string;
+  sandboxGroupId: string;
+};
+
+export type LiveModalSandboxLeaseAttribution = ModalSandboxAttribution & {
+  instanceId: string | null;
+  liveness?: string;
+};
+
+export type ModalOrphanSweepTermination = {
+  sandboxId: string;
+  reason: "stale_attribution" | "unattributed";
+  tags: Record<string, string>;
+};
+
+export type ModalOrphanSweepResult = {
+  examined: number;
+  terminated: ModalOrphanSweepTermination[];
+  skipped: number;
+};
+
+export function modalSandboxAttributionEnvironment(input: ModalSandboxAttribution): Record<string, string> {
+  return {
+    OPENGENI_SANDBOX_LEASE_ID: input.leaseId,
+    OPENGENI_SANDBOX_GROUP_ID: input.sandboxGroupId,
+    OPENGENI_WORKSPACE_ID: input.workspaceId,
+  };
+}
+
+export function modalSandboxAttributionTags(input: ModalSandboxAttribution): Record<string, string> {
+  return {
+    opengeni: "true",
+    opengeni_lease_id: input.leaseId,
+    opengeni_workspace_id: input.workspaceId,
+    opengeni_sandbox_group_id: input.sandboxGroupId,
+  };
+}
+
 export const modalProvider: ProviderRegistration = {
   backend: "modal",
   descriptor: CAPABILITY_DESCRIPTORS.modal,
@@ -23,8 +67,13 @@ export const modalProvider: ProviderRegistration = {
     const options: NonNullable<ConstructorParameters<typeof ModalSandboxClient>[0]> = {
       appName: settings.modalAppName,
       timeoutMs: settings.modalTimeoutSeconds * 1000,
+      sandboxCreateTimeoutS: Math.ceil(settings.sandboxWarmingTimeoutMs / 1000),
       exposedPorts,
       env: environment,
+      // The Modal JS SDK's sandbox default command already sleeps until timeout
+      // or explicit termination. Do not let the Agents extension stamp a separate
+      // hardcoded sleep command; OPENGENI_MODAL_TIMEOUT_SECONDS owns lifetime.
+      useSleepCmd: false,
     };
     // gap-fill (module 03 §4.1): these SDK options were previously unmapped.
     // ALWAYS pin idleTimeoutMs (sandbox-file-persistence): an UNSET idle timeout
@@ -53,3 +102,179 @@ export const modalProvider: ProviderRegistration = {
     return new ModalSandboxClient(options);
   },
 };
+
+type ModalModule = typeof import("modal");
+type ModalClientLike = InstanceType<ModalModule["ModalClient"]>;
+
+function modalClientOptions(settings: Settings): ConstructorParameters<ModalModule["ModalClient"]>[0] {
+  return {
+    ...(settings.modalTokenId ? { tokenId: settings.modalTokenId } : {}),
+    ...(settings.modalTokenSecret ? { tokenSecret: settings.modalTokenSecret } : {}),
+    ...(settings.modalEnvironment ? { environment: settings.modalEnvironment } : {}),
+    ...(settings.modalTimeoutSeconds ? { timeoutMs: settings.modalTimeoutSeconds * 1000 } : {}),
+  };
+}
+
+async function createModalClient(settings: Settings): Promise<ModalClientLike> {
+  const modal = await import("modal");
+  return new modal.ModalClient(modalClientOptions(settings));
+}
+
+export async function tagModalSandbox(
+  settings: Settings,
+  sandboxId: string,
+  attribution: ModalSandboxAttribution,
+): Promise<boolean> {
+  if (!sandboxId) {
+    return false;
+  }
+  const modal = await createModalClient(settings);
+  try {
+    const sandbox = await modal.sandboxes.fromId(sandboxId);
+    await sandbox.setTags(modalSandboxAttributionTags(attribution));
+    return true;
+  } finally {
+    modal.close();
+  }
+}
+
+export async function terminateModalSandboxById(settings: Settings, sandboxId: string): Promise<boolean> {
+  if (!sandboxId) {
+    return true;
+  }
+  const modal = await createModalClient(settings);
+  try {
+    const sandbox = await modal.sandboxes.fromId(sandboxId);
+    await sandbox.terminate();
+    return true;
+  } finally {
+    modal.close();
+  }
+}
+
+type ModalSandboxInfo = {
+  id: string;
+  createdAt?: number;
+  tags?: Array<{ tagName?: string; tagValue?: string }>;
+};
+
+type ModalCpListClient = ModalClientLike & {
+  cpClient: {
+    sandboxList(input: {
+      appId?: string;
+      beforeTimestamp?: number;
+      environmentName?: string;
+      includeFinished?: boolean;
+      tags?: Array<{ tagName: string; tagValue: string }>;
+    }): Promise<{ sandboxes?: ModalSandboxInfo[] }>;
+  };
+};
+
+function tagsFromInfo(info: ModalSandboxInfo): Record<string, string> {
+  const tags: Record<string, string> = {};
+  for (const tag of info.tags ?? []) {
+    if (typeof tag.tagName === "string" && typeof tag.tagValue === "string") {
+      tags[tag.tagName] = tag.tagValue;
+    }
+  }
+  return tags;
+}
+
+function sandboxCreatedAtMs(info: ModalSandboxInfo): number | null {
+  if (typeof info.createdAt !== "number" || !Number.isFinite(info.createdAt) || info.createdAt <= 0) {
+    return null;
+  }
+  // Modal protobuf timestamps in this SDK are seconds as doubles.
+  return info.createdAt < 10_000_000_000 ? Math.floor(info.createdAt * 1000) : Math.floor(info.createdAt);
+}
+
+function attributionKey(input: Pick<ModalSandboxAttribution, "leaseId" | "workspaceId" | "sandboxGroupId">): string {
+  return `${input.workspaceId}:${input.sandboxGroupId}:${input.leaseId}`;
+}
+
+export async function sweepModalOrphanSandboxes(
+  settings: Settings,
+  liveLeases: LiveModalSandboxLeaseAttribution[],
+  options: {
+    now?: Date;
+    maxTerminations?: number;
+    unattributedGraceMs?: number;
+    client?: ModalClientLike;
+  } = {},
+): Promise<ModalOrphanSweepResult> {
+  const nowMs = options.now?.getTime() ?? Date.now();
+  const maxTerminations = options.maxTerminations ?? MODAL_ORPHAN_SWEEP_LIMIT;
+  const unattributedGraceMs = options.unattributedGraceMs ?? MODAL_UNATTRIBUTED_ORPHAN_GRACE_MS;
+  const liveByAttribution = new Map(liveLeases.map((lease) => [attributionKey(lease), lease]));
+  const ownedClient = options.client ? null : await createModalClient(settings);
+  const modal = (options.client ?? ownedClient)! as ModalCpListClient;
+  try {
+    const app = await modal.apps.fromName(settings.modalAppName, {
+      createIfMissing: false,
+      ...(settings.modalEnvironment ? { environment: settings.modalEnvironment } : {}),
+    });
+    const appId = app.appId;
+    if (!appId) {
+      return { examined: 0, terminated: [], skipped: 0 };
+    }
+
+    let examined = 0;
+    let skipped = 0;
+    const terminated: ModalOrphanSweepTermination[] = [];
+    let beforeTimestamp: number | undefined;
+    while (terminated.length < maxTerminations) {
+      const response = await modal.cpClient.sandboxList({
+        appId,
+        ...(beforeTimestamp !== undefined ? { beforeTimestamp } : {}),
+        includeFinished: false,
+        ...(settings.modalEnvironment ? { environmentName: settings.modalEnvironment } : {}),
+        tags: [],
+      });
+      const sandboxes = response.sandboxes ?? [];
+      if (sandboxes.length === 0) {
+        break;
+      }
+      for (const info of sandboxes) {
+        examined += 1;
+        const tags = tagsFromInfo(info);
+        const leaseId = tags.opengeni_lease_id;
+        const workspaceId = tags.opengeni_workspace_id;
+        const sandboxGroupId = tags.opengeni_sandbox_group_id;
+        let reason: ModalOrphanSweepTermination["reason"] | null = null;
+        if (leaseId && workspaceId && sandboxGroupId) {
+          const live = liveByAttribution.get(attributionKey({ leaseId, workspaceId, sandboxGroupId }));
+          if (!live || (live.instanceId && live.instanceId !== info.id)) {
+            reason = "stale_attribution";
+          }
+        } else {
+          const createdAtMs = sandboxCreatedAtMs(info);
+          if (createdAtMs !== null && nowMs - createdAtMs >= unattributedGraceMs) {
+            reason = "unattributed";
+          }
+        }
+
+        if (!reason) {
+          skipped += 1;
+          continue;
+        }
+        try {
+          const sandbox = await modal.sandboxes.fromId(info.id);
+          await sandbox.terminate();
+          terminated.push({ sandboxId: info.id, reason, tags });
+        } catch {
+          skipped += 1;
+        }
+        if (terminated.length >= maxTerminations) {
+          break;
+        }
+      }
+      beforeTimestamp = sandboxes[sandboxes.length - 1]?.createdAt;
+      if (beforeTimestamp === undefined) {
+        break;
+      }
+    }
+    return { examined, terminated, skipped };
+  } finally {
+    ownedClient?.close();
+  }
+}

package/src/sandbox/routing/routing-session.ts

@@ -74,7 +74,7 @@ export interface RoutableBackendSession {
   // `event` is kept `unknown` (mirroring the interface's structural style + avoiding
   // a proto import into the leaf); the SelfhostedSession takes `DesktopInputRequest["event"]`.
   desktopInput?(event: unknown): Promise<void>;
-  screenshot?(): Promise<{ png: Uint8Array; width: number; height: number }>;
+  screenshot?(): Promise<{ png: Uint8Array; width: number; height: number; nativeWidth: number; nativeHeight: number }>;
 }
 
 /** The resolved active backend for an epoch: the live session + the sandbox id it
@@ -189,7 +189,7 @@ export class RoutingSandboxSession implements RoutableBackendSession {
   // that cannot serve them. So the constructor assigns them ONLY when the
   // construction-time default backend actually implements the native surface (below).
   desktopInput?: (event: unknown) => Promise<void>;
-  screenshot?: () => Promise<{ png: Uint8Array; width: number; height: number }>;
+  screenshot?: () => Promise<{ png: Uint8Array; width: number; height: number; nativeWidth: number; nativeHeight: number }>;
 
   constructor(deps: RoutingSandboxSessionDeps) {
     this.deps = deps;

package/src/sandbox/selfhosted/session.ts

@@ -545,16 +545,32 @@ export class SelfhostedSession {
   /** Computer-use VIEW op: capture a single PNG screenshot of the machine's desktop
    *  plus its geometry (via ScreenCaptureKit / x11). NOT consent-gated (a view op —
    *  the view/control decoupling), so it works with a display but no screen-control
-   *  consent. Returns the raw encoded bytes + width/height. */
-  async screenshot(): Promise<{ png: Uint8Array; width: number; height: number }> {
+   *  consent. Returns the raw encoded bytes + the ENCODED width/height, plus the
+   *  NATIVE (pre-downscale) geometry: when the agent had to downscale the PNG to fit
+   *  the transport's max payload, `nativeWidth`/`nativeHeight` carry the original
+   *  capture size so the computer-use layer can scale model clicks (in encoded-pixel
+   *  space) back to native pixels. An older agent leaves them 0 → read as "same as
+   *  width/height" (no downscale). */
+  async screenshot(): Promise<{
+    png: Uint8Array;
+    width: number;
+    height: number;
+    nativeWidth: number;
+    nativeHeight: number;
+  }> {
     const result = await this.call({ $case: "desktopScreenshot", desktopScreenshot: {} });
     if (result.$case !== "desktopScreenshot") {
       throw new Error(`selfhosted screenshot: unexpected result ${result.$case}`);
     }
+    const s = result.desktopScreenshot;
+    // Back-compat: an agent predating the native-geometry fields sends 0 → treat the
+    // encoded geometry AS the native geometry (scale factor 1.0, no coordinate shift).
     return {
-      png: result.desktopScreenshot.png,
-      width: result.desktopScreenshot.width,
-      height: result.desktopScreenshot.height,
+      png: s.png,
+      width: s.width,
+      height: s.height,
+      nativeWidth: s.nativeWidth || s.width,
+      nativeHeight: s.nativeHeight || s.height,
     };
   }
 

package/src/sandbox-computer.ts

@@ -385,7 +385,11 @@ export class SandboxComputer implements Computer {
  *  leaf; the duck-typed `isNativeDesktopSession` probe (below) selects on it. */
 export type NativeDesktopSession = {
   desktopInput(event: DesktopInputRequest["event"]): Promise<void>;
-  screenshot(): Promise<{ png: Uint8Array; width: number; height: number }>;
+  // `nativeWidth`/`nativeHeight` are the PRE-DOWNSCALE capture geometry (equal to
+  // width/height when the agent did not have to shrink the PNG to fit the transport
+  // budget). NativeDesktopComputer scales model clicks from the ENCODED pixel space
+  // back to native pixels using their ratio before injecting.
+  screenshot(): Promise<{ png: Uint8Array; width: number; height: number; nativeWidth: number; nativeHeight: number }>;
 };
 
 /** Model `Button` → wire `PointerButton`. The proto has no back/forward button, so
@@ -424,6 +428,15 @@ export class NativeDesktopComputer implements Computer {
   readonly dimensions: [number, number];
   private session: NativeDesktopSession;
   private readonly readOnly: boolean;
+  // The ENCODED vs NATIVE geometry of the MOST RECENT screenshot the model saw. The
+  // model computes click coordinates in the encoded-pixel space of that screenshot;
+  // when the agent downscaled the PNG to fit the transport budget, encoded < native,
+  // so we scale coordinates back up to native pixels before injecting (the agent's
+  // native inject — macOS CGEvent / Linux XTEST — expects native-pixel coordinates,
+  // exactly as it received them pre-downscale). Null until the first screenshot;
+  // equal encoded==native (or absent) ⇒ scale factor 1.0 ⇒ byte-identical behavior.
+  private lastEncoded: [number, number] | null = null;
+  private lastNative: [number, number] | null = null;
 
   constructor(session: NativeDesktopSession, opts: NativeDesktopComputerOptions = {}) {
     this.session = session;
@@ -441,17 +454,34 @@ export class NativeDesktopComputer implements Computer {
     if (this.readOnly) throw new ComputerReadOnlyError();
   }
 
+  /** Scale a coordinate the model expressed in the MOST RECENT screenshot's
+   *  ENCODED pixel space back to NATIVE pixels. When the last frame was not
+   *  downscaled (encoded == native), or no screenshot has been taken yet, this is a
+   *  1:1 identity — the byte-identical current behavior. The agent then applies its
+   *  own platform mapping (macOS divides native pixels by the backing scale to reach
+   *  CGEvent points; Linux XTEST is 1:1) exactly as it did pre-downscale. */
+  private toNative(x: number, y: number): { x: number; y: number } {
+    const enc = this.lastEncoded;
+    const nat = this.lastNative;
+    if (!enc || !nat || enc[0] <= 0 || enc[1] <= 0) return { x, y };
+    if (enc[0] === nat[0] && enc[1] === nat[1]) return { x, y };
+    return {
+      x: Math.round((x * nat[0]) / enc[0]),
+      y: Math.round((y * nat[1]) / enc[1]),
+    };
+  }
+
   private async pointer(x: number, y: number, action: PointerAction, button: PointerButton): Promise<void> {
-    // COORDINATE SEAM — TODO(verify e2e on macOS): the model computes x/y against the
-    // pixels of the screenshot it just saw, and the agent's macOS CGEvent inject
-    // treats x/y as raw screen coordinates. On a Retina Mac, ScreenCaptureKit may
-    // capture at 2× the logical POINT space while CGEvent expects logical points — a
-    // potential 2× mismatch between the coords the model derives and the coords the
-    // inject applies. This MUST be measured on a real Retina Mac (compare the
-    // screenshot's reported width/height against the logical display bounds) before
-    // any DPR scaling is added. Do NOT add scaling speculatively. Self-hosted Linux
-    // (XTEST/x11) is 1:1 and unaffected.
-    await this.session.desktopInput({ $case: "pointer", pointer: { x, y, action, button } });
+    // COORDINATE SEAM: the model computes x/y against the pixels of the screenshot it
+    // just saw — which the agent may have DOWNSCALED to fit the transport's max
+    // payload (a full-res Retina/busy screen exceeds NATS's 1 MiB default). We scale
+    // those encoded-pixel coordinates back to native pixels here, using the native
+    // geometry the last screenshot reported, so the agent's native inject lands the
+    // click where the model intended. When no downscale occurred the factor is 1.0
+    // and the coordinates pass through unchanged. Self-hosted Linux (XTEST/x11) and
+    // any non-downscaled frame are 1:1 and unaffected.
+    const n = this.toNative(x, y);
+    await this.session.desktopInput({ $case: "pointer", pointer: { x: n.x, y: n.y, action, button } });
   }
 
   async screenshot(): Promise<string> {
@@ -461,10 +491,14 @@ export class NativeDesktopComputer implements Computer {
     // missing/empty frame is therefore a THROW, never a silent "". Native capture
     // (ScreenCaptureKit / x11) does not have the cold-scrot warm-up the xdotool path
     // retries around, so a single capture + a hard empty-guard is sufficient.
-    const { png } = await this.session.screenshot();
+    const { png, width, height, nativeWidth, nativeHeight } = await this.session.screenshot();
     if (png.length === 0) {
       throw new ComputerUnavailableError("native desktop screenshot returned an empty frame (display not up?)");
     }
+    // Record the encoded (what the model sees) vs native geometry of THIS frame so
+    // the next click/move/scroll/drag scales its coordinates back to native pixels.
+    this.lastEncoded = [width, height];
+    this.lastNative = [nativeWidth || width, nativeHeight || height];
     return Buffer.from(png).toString("base64");
   }
 
@@ -482,11 +516,12 @@ export class NativeDesktopComputer implements Computer {
   }
   async scroll(x: number, y: number, sx: number, sy: number) {
     this.guardWrite();
-    // The model's scroll deltas are PIXELS — forward them straight to the agent as a
-    // ScrollEvent{x,y,deltaX,deltaY} and let the native inject translate to wheel
-    // events per platform. No xdotool "notch" quantization here (that is an
-    // xdotool-specific artifact); the agent owns the platform-appropriate scaling.
-    await this.session.desktopInput({ $case: "scroll", scroll: { x, y, deltaX: sx, deltaY: sy } });
+    // The scroll ANCHOR (x,y) is a screenshot-pixel position → scale to native like a
+    // click. The deltas (sx,sy) are relative scroll AMOUNTS, not positions; the agent
+    // owns the platform-appropriate wheel translation, so they pass through unscaled
+    // (no xdotool "notch" quantization here — that is an xdotool-specific artifact).
+    const n = this.toNative(x, y);
+    await this.session.desktopInput({ $case: "scroll", scroll: { x: n.x, y: n.y, deltaX: sx, deltaY: sy } });
   }
   async type(text: string) {
     this.guardWrite();