@opengeni/runtime 0.2.3 → 0.3.1

package/src/sandbox/index.ts

@@ -29,6 +29,7 @@ import type {
 import { PROVIDER_REGISTRY } from "./providers";
 import { SandboxConfigError } from "./errors";
 import { isSelfhostedProviderNotFoundError } from "./selfhosted/session";
+import type { RuntimeMetricsHooks } from "../metrics";
 
 // Re-export the config-owned environment/port helpers from the leaf so the
 // API-direct control plane can pull its full sandbox-construction surface from
@@ -53,6 +54,16 @@ export {
   type ProviderRegistration,
   type ProviderConstructionContext,
 } from "./providers";
+export {
+  modalSandboxAttributionEnvironment,
+  modalSandboxAttributionTags,
+  sweepModalOrphanSandboxes,
+  tagModalSandbox,
+  terminateModalSandboxById,
+  type LiveModalSandboxLeaseAttribution,
+  type ModalOrphanSweepResult,
+  type ModalSandboxAttribution,
+} from "./providers/modal";
 export {
   selectBackend,
   backendSupportsOs,
@@ -540,6 +551,8 @@ export type EstablishedSandboxSession = {
   backendId: string;
 };
 
+export type SandboxCreatedCallback = (established: EstablishedSandboxSession) => Promise<void>;
+
 // The structural slice we need from a provider SandboxClient to resume by id and
 // cold-restore. Narrowed (not the full agent-loop SandboxClient) so the leaf
 // stays agent-loop-free.
@@ -616,6 +629,16 @@ function readInstanceId(session: unknown): string {
   return typeof candidate === "string" && candidate.length > 0 ? candidate : "";
 }
 
+async function terminateCreatedSandbox(client: ResumeCapableClient, session: unknown, sessionState: unknown): Promise<void> {
+  const clientWithDelete = client as { delete?: (state: unknown) => Promise<unknown> };
+  if (typeof clientWithDelete.delete === "function" && sessionState !== undefined) {
+    try { await clientWithDelete.delete(sessionState); } catch { /* best-effort */ }
+    return;
+  }
+  const sess = session as { close?: () => Promise<unknown>; terminate?: () => Promise<unknown>; kill?: () => Promise<unknown> };
+  try { await (sess.terminate ?? sess.kill ?? sess.close)?.(); } catch { /* best-effort */ }
+}
+
 /**
  * Resume the one box by id from its recovery envelope, or cold-restore from the
  * snapshot when the provider reports it gone. The envelope is the lease's
@@ -633,7 +656,13 @@ function readInstanceId(session: unknown): string {
 export async function establishSandboxSessionFromEnvelope(
   settings: Settings,
   envelope: Record<string, unknown> | null,
-  opts: { sessionId: string; backendOverride?: SandboxBackend; environment?: Record<string, string> },
+  opts: {
+    sessionId: string;
+    backendOverride?: SandboxBackend;
+    environment?: Record<string, string>;
+    onSandboxCreated?: SandboxCreatedCallback;
+    metrics?: RuntimeMetricsHooks;
+  },
 ): Promise<EstablishedSandboxSession> {
   const envelopeBackend = typeof envelope?.backendId === "string" ? (envelope.backendId as SandboxBackend) : undefined;
   const backend = (opts.backendOverride ?? envelopeBackend ?? (settings.sandboxBackend as SandboxBackend));
@@ -679,7 +708,31 @@ export async function establishSandboxSessionFromEnvelope(
   // SOLE archive-replay seam, shared by the NotFound warm-reattach path AND the
   // cold-restore branch (b) below.
   const coldRestore = async (resumeFallbackState?: unknown): Promise<EstablishedSandboxSession> => {
-    const restored = await client.create!({ manifest: createManifest });
+    const createStarted = Date.now();
+    let restored: Awaited<ReturnType<NonNullable<typeof client.create>>>;
+    try {
+      restored = await client.create!({ manifest: createManifest });
+      recordSandboxCreateMetric(opts.metrics, client.backendId, "completed", createStarted);
+    } catch (error) {
+      recordSandboxCreateMetric(opts.metrics, client.backendId, "failed", createStarted);
+      throw error;
+    }
+    let restoredState = (restored as { state?: unknown }).state;
+    let established: EstablishedSandboxSession = {
+      client,
+      session: restored,
+      sessionState: restoredState ?? resumeFallbackState,
+      instanceId: readInstanceId(restored),
+      backendId: client.backendId,
+    };
+    if (opts.onSandboxCreated) {
+      try {
+        await opts.onSandboxCreated(established);
+      } catch (createCallbackError) {
+        await terminateCreatedSandbox(client, restored, restoredState);
+        throw createCallbackError;
+      }
+    }
     if (workspaceArchive) {
       const hydrate = (restored as { hydrateWorkspace?: (data: Uint8Array) => Promise<void> }).hydrateWorkspace;
       if (typeof hydrate === "function") {
@@ -696,21 +749,38 @@ export async function establishSandboxSessionFromEnvelope(
           // re-throwing so no box leaks. The original error semantics are preserved
           // (the re-throw propagates to the caller). This mirrors the reaper's
           // discipline: NEVER leave an orphaned box running.
-          const restoredState = (restored as { state?: unknown }).state;
-          const clientWithDelete = client as { delete?: (state: unknown) => Promise<unknown> };
-          if (typeof clientWithDelete.delete === "function" && restoredState !== undefined) {
-            try { await clientWithDelete.delete(restoredState); } catch { /* best-effort; re-throw the hydrate error below */ }
-          } else {
-            // No delete() — try a session-level close/terminate as a fallback.
-            const sess = restored as { close?: () => Promise<unknown>; terminate?: () => Promise<unknown> };
-            try { await (sess.terminate ?? sess.close)?.(); } catch { /* best-effort */ }
-          }
+          await terminateCreatedSandbox(client, restored, restoredState);
           throw hydrateError;
         }
+        const hydratedState = (restored as { state?: unknown }).state;
+        const hydratedInstanceId = readInstanceId(restored);
+        if (hydratedInstanceId && hydratedInstanceId !== established.instanceId) {
+          established = {
+            client,
+            session: restored,
+            sessionState: hydratedState ?? resumeFallbackState,
+            instanceId: hydratedInstanceId,
+            backendId: client.backendId,
+          };
+          if (opts.onSandboxCreated) {
+            try {
+              await opts.onSandboxCreated(established);
+            } catch (createCallbackError) {
+              await terminateCreatedSandbox(client, restored, hydratedState);
+              throw createCallbackError;
+            }
+          }
+        }
       }
     }
-    const restoredState = (restored as { state?: unknown }).state;
-    return { client, session: restored, sessionState: restoredState ?? resumeFallbackState, instanceId: readInstanceId(restored), backendId: client.backendId };
+    restoredState = (restored as { state?: unknown }).state;
+    return {
+      client,
+      session: restored,
+      sessionState: restoredState ?? resumeFallbackState,
+      instanceId: readInstanceId(restored),
+      backendId: client.backendId,
+    };
   };
 
   // Does the envelope carry a RESUMABLE box id (warm reattach), or only a
@@ -770,6 +840,23 @@ export async function establishSandboxSessionFromEnvelope(
   return await coldRestore();
 }
 
+function recordSandboxCreateMetric(
+  metrics: RuntimeMetricsHooks | undefined,
+  backend: string,
+  outcome: "completed" | "failed",
+  startedMs: number,
+): void {
+  try {
+    metrics?.onSandboxCreate?.({
+      backend,
+      outcome,
+      durationSeconds: Math.max(0, (Date.now() - startedMs) / 1000),
+    });
+  } catch {
+    // Metrics emission must not affect sandbox lifecycle.
+  }
+}
+
 // A client that can SERIALIZE a live session state back to the persistable
 // envelope form (the inverse of deserializeSessionState). Narrowed so the leaf
 // stays agent-loop-free.

package/src/sandbox/providers/modal.ts

@@ -1,9 +1,53 @@
 import { ModalImageSelector, ModalSandboxClient } from "@openai/agents-extensions/sandbox/modal";
 import { effectiveModalIdleTimeoutSeconds } from "@opengeni/config";
+import type { Settings } from "@opengeni/config";
 import { CAPABILITY_DESCRIPTORS } from "../capabilities";
 import { SandboxConfigError } from "../errors";
 import type { ProviderRegistration } from "./types";
 
+const MODAL_ORPHAN_SWEEP_LIMIT = 50;
+const MODAL_UNATTRIBUTED_ORPHAN_GRACE_MS = 30 * 60_000;
+
+export type ModalSandboxAttribution = {
+  leaseId: string;
+  workspaceId: string;
+  sandboxGroupId: string;
+};
+
+export type LiveModalSandboxLeaseAttribution = ModalSandboxAttribution & {
+  instanceId: string | null;
+  liveness?: string;
+};
+
+export type ModalOrphanSweepTermination = {
+  sandboxId: string;
+  reason: "stale_attribution" | "unattributed";
+  tags: Record<string, string>;
+};
+
+export type ModalOrphanSweepResult = {
+  examined: number;
+  terminated: ModalOrphanSweepTermination[];
+  skipped: number;
+};
+
+export function modalSandboxAttributionEnvironment(input: ModalSandboxAttribution): Record<string, string> {
+  return {
+    OPENGENI_SANDBOX_LEASE_ID: input.leaseId,
+    OPENGENI_SANDBOX_GROUP_ID: input.sandboxGroupId,
+    OPENGENI_WORKSPACE_ID: input.workspaceId,
+  };
+}
+
+export function modalSandboxAttributionTags(input: ModalSandboxAttribution): Record<string, string> {
+  return {
+    opengeni: "true",
+    opengeni_lease_id: input.leaseId,
+    opengeni_workspace_id: input.workspaceId,
+    opengeni_sandbox_group_id: input.sandboxGroupId,
+  };
+}
+
 export const modalProvider: ProviderRegistration = {
   backend: "modal",
   descriptor: CAPABILITY_DESCRIPTORS.modal,
@@ -23,8 +67,13 @@ export const modalProvider: ProviderRegistration = {
     const options: NonNullable<ConstructorParameters<typeof ModalSandboxClient>[0]> = {
       appName: settings.modalAppName,
       timeoutMs: settings.modalTimeoutSeconds * 1000,
+      sandboxCreateTimeoutS: Math.ceil(settings.sandboxWarmingTimeoutMs / 1000),
       exposedPorts,
       env: environment,
+      // The Modal JS SDK's sandbox default command already sleeps until timeout
+      // or explicit termination. Do not let the Agents extension stamp a separate
+      // hardcoded sleep command; OPENGENI_MODAL_TIMEOUT_SECONDS owns lifetime.
+      useSleepCmd: false,
     };
     // gap-fill (module 03 §4.1): these SDK options were previously unmapped.
     // ALWAYS pin idleTimeoutMs (sandbox-file-persistence): an UNSET idle timeout
@@ -53,3 +102,179 @@ export const modalProvider: ProviderRegistration = {
     return new ModalSandboxClient(options);
   },
 };
+
+type ModalModule = typeof import("modal");
+type ModalClientLike = InstanceType<ModalModule["ModalClient"]>;
+
+function modalClientOptions(settings: Settings): ConstructorParameters<ModalModule["ModalClient"]>[0] {
+  return {
+    ...(settings.modalTokenId ? { tokenId: settings.modalTokenId } : {}),
+    ...(settings.modalTokenSecret ? { tokenSecret: settings.modalTokenSecret } : {}),
+    ...(settings.modalEnvironment ? { environment: settings.modalEnvironment } : {}),
+    ...(settings.modalTimeoutSeconds ? { timeoutMs: settings.modalTimeoutSeconds * 1000 } : {}),
+  };
+}
+
+async function createModalClient(settings: Settings): Promise<ModalClientLike> {
+  const modal = await import("modal");
+  return new modal.ModalClient(modalClientOptions(settings));
+}
+
+export async function tagModalSandbox(
+  settings: Settings,
+  sandboxId: string,
+  attribution: ModalSandboxAttribution,
+): Promise<boolean> {
+  if (!sandboxId) {
+    return false;
+  }
+  const modal = await createModalClient(settings);
+  try {
+    const sandbox = await modal.sandboxes.fromId(sandboxId);
+    await sandbox.setTags(modalSandboxAttributionTags(attribution));
+    return true;
+  } finally {
+    modal.close();
+  }
+}
+
+export async function terminateModalSandboxById(settings: Settings, sandboxId: string): Promise<boolean> {
+  if (!sandboxId) {
+    return true;
+  }
+  const modal = await createModalClient(settings);
+  try {
+    const sandbox = await modal.sandboxes.fromId(sandboxId);
+    await sandbox.terminate();
+    return true;
+  } finally {
+    modal.close();
+  }
+}
+
+type ModalSandboxInfo = {
+  id: string;
+  createdAt?: number;
+  tags?: Array<{ tagName?: string; tagValue?: string }>;
+};
+
+type ModalCpListClient = ModalClientLike & {
+  cpClient: {
+    sandboxList(input: {
+      appId?: string;
+      beforeTimestamp?: number;
+      environmentName?: string;
+      includeFinished?: boolean;
+      tags?: Array<{ tagName: string; tagValue: string }>;
+    }): Promise<{ sandboxes?: ModalSandboxInfo[] }>;
+  };
+};
+
+function tagsFromInfo(info: ModalSandboxInfo): Record<string, string> {
+  const tags: Record<string, string> = {};
+  for (const tag of info.tags ?? []) {
+    if (typeof tag.tagName === "string" && typeof tag.tagValue === "string") {
+      tags[tag.tagName] = tag.tagValue;
+    }
+  }
+  return tags;
+}
+
+function sandboxCreatedAtMs(info: ModalSandboxInfo): number | null {
+  if (typeof info.createdAt !== "number" || !Number.isFinite(info.createdAt) || info.createdAt <= 0) {
+    return null;
+  }
+  // Modal protobuf timestamps in this SDK are seconds as doubles.
+  return info.createdAt < 10_000_000_000 ? Math.floor(info.createdAt * 1000) : Math.floor(info.createdAt);
+}
+
+function attributionKey(input: Pick<ModalSandboxAttribution, "leaseId" | "workspaceId" | "sandboxGroupId">): string {
+  return `${input.workspaceId}:${input.sandboxGroupId}:${input.leaseId}`;
+}
+
+export async function sweepModalOrphanSandboxes(
+  settings: Settings,
+  liveLeases: LiveModalSandboxLeaseAttribution[],
+  options: {
+    now?: Date;
+    maxTerminations?: number;
+    unattributedGraceMs?: number;
+    client?: ModalClientLike;
+  } = {},
+): Promise<ModalOrphanSweepResult> {
+  const nowMs = options.now?.getTime() ?? Date.now();
+  const maxTerminations = options.maxTerminations ?? MODAL_ORPHAN_SWEEP_LIMIT;
+  const unattributedGraceMs = options.unattributedGraceMs ?? MODAL_UNATTRIBUTED_ORPHAN_GRACE_MS;
+  const liveByAttribution = new Map(liveLeases.map((lease) => [attributionKey(lease), lease]));
+  const ownedClient = options.client ? null : await createModalClient(settings);
+  const modal = (options.client ?? ownedClient)! as ModalCpListClient;
+  try {
+    const app = await modal.apps.fromName(settings.modalAppName, {
+      createIfMissing: false,
+      ...(settings.modalEnvironment ? { environment: settings.modalEnvironment } : {}),
+    });
+    const appId = app.appId;
+    if (!appId) {
+      return { examined: 0, terminated: [], skipped: 0 };
+    }
+
+    let examined = 0;
+    let skipped = 0;
+    const terminated: ModalOrphanSweepTermination[] = [];
+    let beforeTimestamp: number | undefined;
+    while (terminated.length < maxTerminations) {
+      const response = await modal.cpClient.sandboxList({
+        appId,
+        ...(beforeTimestamp !== undefined ? { beforeTimestamp } : {}),
+        includeFinished: false,
+        ...(settings.modalEnvironment ? { environmentName: settings.modalEnvironment } : {}),
+        tags: [],
+      });
+      const sandboxes = response.sandboxes ?? [];
+      if (sandboxes.length === 0) {
+        break;
+      }
+      for (const info of sandboxes) {
+        examined += 1;
+        const tags = tagsFromInfo(info);
+        const leaseId = tags.opengeni_lease_id;
+        const workspaceId = tags.opengeni_workspace_id;
+        const sandboxGroupId = tags.opengeni_sandbox_group_id;
+        let reason: ModalOrphanSweepTermination["reason"] | null = null;
+        if (leaseId && workspaceId && sandboxGroupId) {
+          const live = liveByAttribution.get(attributionKey({ leaseId, workspaceId, sandboxGroupId }));
+          if (!live || (live.instanceId && live.instanceId !== info.id)) {
+            reason = "stale_attribution";
+          }
+        } else {
+          const createdAtMs = sandboxCreatedAtMs(info);
+          if (createdAtMs !== null && nowMs - createdAtMs >= unattributedGraceMs) {
+            reason = "unattributed";
+          }
+        }
+
+        if (!reason) {
+          skipped += 1;
+          continue;
+        }
+        try {
+          const sandbox = await modal.sandboxes.fromId(info.id);
+          await sandbox.terminate();
+          terminated.push({ sandboxId: info.id, reason, tags });
+        } catch {
+          skipped += 1;
+        }
+        if (terminated.length >= maxTerminations) {
+          break;
+        }
+      }
+      beforeTimestamp = sandboxes[sandboxes.length - 1]?.createdAt;
+      if (beforeTimestamp === undefined) {
+        break;
+      }
+    }
+    return { examined, terminated, skipped };
+  } finally {
+    ownedClient?.close();
+  }
+}

package/src/sandbox/routing/routing-session.ts

@@ -74,7 +74,7 @@ export interface RoutableBackendSession {
   // `event` is kept `unknown` (mirroring the interface's structural style + avoiding
   // a proto import into the leaf); the SelfhostedSession takes `DesktopInputRequest["event"]`.
   desktopInput?(event: unknown): Promise<void>;
-  screenshot?(): Promise<{ png: Uint8Array; width: number; height: number }>;
+  screenshot?(): Promise<{ png: Uint8Array; width: number; height: number; nativeWidth: number; nativeHeight: number }>;
 }
 
 /** The resolved active backend for an epoch: the live session + the sandbox id it
@@ -189,7 +189,7 @@ export class RoutingSandboxSession implements RoutableBackendSession {
   // that cannot serve them. So the constructor assigns them ONLY when the
   // construction-time default backend actually implements the native surface (below).
   desktopInput?: (event: unknown) => Promise<void>;
-  screenshot?: () => Promise<{ png: Uint8Array; width: number; height: number }>;
+  screenshot?: () => Promise<{ png: Uint8Array; width: number; height: number; nativeWidth: number; nativeHeight: number }>;
 
   constructor(deps: RoutingSandboxSessionDeps) {
     this.deps = deps;

package/src/sandbox/selfhosted/session.ts

@@ -545,16 +545,32 @@ export class SelfhostedSession {
   /** Computer-use VIEW op: capture a single PNG screenshot of the machine's desktop
    *  plus its geometry (via ScreenCaptureKit / x11). NOT consent-gated (a view op —
    *  the view/control decoupling), so it works with a display but no screen-control
-   *  consent. Returns the raw encoded bytes + width/height. */
-  async screenshot(): Promise<{ png: Uint8Array; width: number; height: number }> {
+   *  consent. Returns the raw encoded bytes + the ENCODED width/height, plus the
+   *  NATIVE (pre-downscale) geometry: when the agent had to downscale the PNG to fit
+   *  the transport's max payload, `nativeWidth`/`nativeHeight` carry the original
+   *  capture size so the computer-use layer can scale model clicks (in encoded-pixel
+   *  space) back to native pixels. An older agent leaves them 0 → read as "same as
+   *  width/height" (no downscale). */
+  async screenshot(): Promise<{
+    png: Uint8Array;
+    width: number;
+    height: number;
+    nativeWidth: number;
+    nativeHeight: number;
+  }> {
     const result = await this.call({ $case: "desktopScreenshot", desktopScreenshot: {} });
     if (result.$case !== "desktopScreenshot") {
       throw new Error(`selfhosted screenshot: unexpected result ${result.$case}`);
     }
+    const s = result.desktopScreenshot;
+    // Back-compat: an agent predating the native-geometry fields sends 0 → treat the
+    // encoded geometry AS the native geometry (scale factor 1.0, no coordinate shift).
     return {
-      png: result.desktopScreenshot.png,
-      width: result.desktopScreenshot.width,
-      height: result.desktopScreenshot.height,
+      png: s.png,
+      width: s.width,
+      height: s.height,
+      nativeWidth: s.nativeWidth || s.width,
+      nativeHeight: s.nativeHeight || s.height,
     };
   }