@opengeni/runtime 0.6.0 → 0.7.0

package/src/sandbox/op-correlation.ts

@@ -0,0 +1,70 @@
+// Durable op-identity correlation (op-stream ruling B1): the DURABLE op id a
+// sandbox transport op carries is `{sdk_tool_call_id}:{ordinal}` — minted ABOVE
+// the transport, at the semantic layer. The tool call id lives in the model's
+// function_call (persisted run history), so a re-dispatched turn re-executes the
+// SAME function_call with the SAME call id, and the transport's idempotent
+// OpStart turns an at-least-once re-execution into an ATTACH instead of a
+// re-run. The ordinal is the deterministic position of the physical sub-op
+// inside one tool invocation (a tool that performs read→write→rm mints :0, :1,
+// :2), so a re-executed invocation re-mints identical ids.
+//
+// The correlation rides an AsyncLocalStorage bound around the SDK tool's
+// `execute` (see `withExecOpCorrelation` in the runtime barrel): the SDK's tool
+// machinery passes `details.toolCall` (with `callId`) into every function-tool
+// invocation, and the shell capability's `configureTools` hook lets us wrap
+// exec_command without touching the SDK. Per-async-chain storage keeps PARALLEL
+// tool calls correctly separated.
+//
+// This module is deliberately dependency-free (node:async_hooks only) so the
+// agent-loop-free sandbox leaf may import it: the leaf READS the context; only
+// the runtime barrel (which owns the SDK imports) BINDS it.
+
+import { AsyncLocalStorage } from "node:async_hooks";
+
+interface ToolCallCorrelation {
+  /** The sanitized sdk tool call id (a legal NATS subject-token fragment). */
+  callId: string;
+  /** The next sub-op ordinal within this tool invocation (mutable). */
+  ordinal: number;
+}
+
+const storage = new AsyncLocalStorage<ToolCallCorrelation>();
+
+/**
+ * Sanitize a tool call id into a legal NATS subject TOKEN fragment. The op id
+ * is interpolated into the per-op frame subject, whose tokens must not contain
+ * whitespace/control characters or the subject-structure characters (`.`,
+ * `*`, `>`); the runner refuses illegal ids loudly. The mapping is INJECTIVE
+ * (each disallowed char becomes `_<hex>_`), so two distinct call ids can never
+ * collide into one op id — a collision would merge two different execs through
+ * the idempotent-OpStart dedup and return the wrong result.
+ */
+export function sanitizeOpIdToken(raw: string): string {
+  return raw.replace(/[^A-Za-z0-9_-]/g, (c) => `_${c.charCodeAt(0).toString(16)}_`);
+}
+
+/**
+ * Bind a tool-call correlation context around `fn` (the tool's execute). Every
+ * durable op id minted inside — however deep in the transport — is
+ * `{callId}:{ordinal}` with ordinals starting at 0 per invocation.
+ */
+export function runWithToolCallCorrelation<T>(callId: string, fn: () => T): T {
+  return storage.run({ callId: sanitizeOpIdToken(callId), ordinal: 0 }, fn);
+}
+
+/**
+ * Mint the next durable op id for the current tool invocation, or null when no
+ * correlation context is bound (a non-tool caller, e.g. Channel-A structural
+ * exec). Callers fall back to a random unique id — safe (never collides, never
+ * wrongly dedups), merely not stable across a turn re-dispatch, which degrades
+ * that one op to today's at-least-once semantics.
+ */
+export function nextDurableOpId(): string | null {
+  const context = storage.getStore();
+  if (!context) {
+    return null;
+  }
+  const ordinal = context.ordinal;
+  context.ordinal += 1;
+  return `${context.callId}:${ordinal}`;
+}

package/src/sandbox/providers/index.ts

@@ -66,16 +66,22 @@ export function assertProviderRegistryInvariants(): void {
   for (const backend of SandboxBackend.options) {
     const registration = PROVIDER_REGISTRY[backend];
     if (registration.backend !== backend) {
-      throw new Error(`PROVIDER_REGISTRY["${backend}"].backend mismatch (got "${registration.backend}")`);
+      throw new Error(
+        `PROVIDER_REGISTRY["${backend}"].backend mismatch (got "${registration.backend}")`,
+      );
     }
     if (registration.descriptor.backend !== backend) {
-      throw new Error(`PROVIDER_REGISTRY["${backend}"].descriptor.backend mismatch (got "${registration.descriptor.backend}")`);
+      throw new Error(
+        `PROVIDER_REGISTRY["${backend}"].descriptor.backend mismatch (got "${registration.descriptor.backend}")`,
+      );
     }
     if (backend === "none") {
       // "none" has no SDK client (build returns undefined); the descriptor
       // backendId "none" is self-consistent.
       if (registration.descriptor.backendId !== "none") {
-        throw new Error(`"none" descriptor.backendId must be "none" (got "${registration.descriptor.backendId}")`);
+        throw new Error(
+          `"none" descriptor.backendId must be "none" (got "${registration.descriptor.backendId}")`,
+        );
       }
       continue;
     }

package/src/sandbox/providers/modal.ts

@@ -6,7 +6,14 @@ import { SandboxConfigError } from "../errors";
 import type { ProviderRegistration } from "./types";
 
 const MODAL_ORPHAN_SWEEP_LIMIT = 50;
-const MODAL_UNATTRIBUTED_ORPHAN_GRACE_MS = 30 * 60_000;
+// A provider box is invisible to the lease until Modal create + manifest
+// materialization returns and the creation callback records its instance id.
+// Production baseline (2026-07-15, all 8 turn workers): 155/155 completed creates
+// finished under 10s (154 under 2.5s). Two minutes is a 12x observed-max buffer,
+// while avoiding the former 30-minute retention of boxes abandoned by a rolling
+// worker restart. The live-instance guard below remains authoritative: once a box
+// is recorded by any lease, age and missing/stale tags can never terminate it.
+const MODAL_UNATTRIBUTED_ORPHAN_GRACE_MS = 2 * 60_000;
 
 export type ModalSandboxAttribution = {
   leaseId: string;
@@ -31,7 +38,9 @@ export type ModalOrphanSweepResult = {
   skipped: number;
 };
 
-export function modalSandboxAttributionEnvironment(input: ModalSandboxAttribution): Record<string, string> {
+export function modalSandboxAttributionEnvironment(
+  input: ModalSandboxAttribution,
+): Record<string, string> {
   return {
     OPENGENI_SANDBOX_LEASE_ID: input.leaseId,
     OPENGENI_SANDBOX_GROUP_ID: input.sandboxGroupId,
@@ -39,7 +48,9 @@ export function modalSandboxAttributionEnvironment(input: ModalSandboxAttributio
   };
 }
 
-export function modalSandboxAttributionTags(input: ModalSandboxAttribution): Record<string, string> {
+export function modalSandboxAttributionTags(
+  input: ModalSandboxAttribution,
+): Record<string, string> {
   return {
     opengeni: "true",
     opengeni_lease_id: input.leaseId,
@@ -70,10 +81,11 @@ export const modalProvider: ProviderRegistration = {
       sandboxCreateTimeoutS: Math.ceil(settings.sandboxWarmingTimeoutMs / 1000),
       exposedPorts,
       env: environment,
-      // The Modal JS SDK's sandbox default command already sleeps until timeout
-      // or explicit termination. Do not let the Agents extension stamp a separate
-      // hardcoded sleep command; OPENGENI_MODAL_TIMEOUT_SECONDS owns lifetime.
-      useSleepCmd: false,
+      // A registry image's own CMD is not a sandbox keepalive contract (for
+      // example, python:3.12-slim can exit immediately). Keep the provider's
+      // control process alive so exec/resume remains available; Modal's hard
+      // timeout and explicit OpenGeni teardown still own the box lifetime.
+      useSleepCmd: true,
     };
     // gap-fill (module 03 §4.1): these SDK options were previously unmapped.
     // ALWAYS pin idleTimeoutMs (sandbox-file-persistence): an UNSET idle timeout
@@ -115,14 +127,13 @@ type ModalClientLike = InstanceType<ModalModule["ModalClient"]>;
 // `fromRegistry(tag, secret)` image ONCE per process, then hand the provider `build`
 // a `ModalImageSelector.fromImage(...)`. `build` is synchronous and modal is imported
 // lazily (never loaded for non-modal backends), so resolution can't happen inside
-// `build`; the worker boot awaits `ensureModalRegistryImage` first and `build` reads
-// the settled result. Modal images are lazy, workspace-scoped definitions, so an image
-// built by this module's client is usable by the ModalSandboxClient's own client.
+// `build`; the worker awaits `ensureModalRegistryImage` at boot for global refs and
+// at turn time for pack-scoped refs, then `build` reads the settled result. Modal
+// images are lazy, workspace-scoped definitions, so an image built by this module's
+// client is usable by the ModalSandboxClient's own client.
 
 /** Loader seam so unit tests can inject a fake modal module. */
-export type ModalModuleLoader = () => Promise<
-  Pick<ModalModule, "ModalClient" | "Secret">
->;
+export type ModalModuleLoader = () => Promise<Pick<ModalModule, "ModalClient">>;
 
 const defaultModalLoader: ModalModuleLoader = () => import("modal");
 
@@ -143,8 +154,9 @@ function registryImageCacheKey(settings: Settings): string {
  * Resolve + cache the private-registry Modal image. No-op unless BOTH
  * `modalImageRef` and `modalImageRegistrySecret` are set. Memoized per
  * (imageRef, secret, environment) so it runs once per worker process. Awaited at
- * worker boot BEFORE the first sandbox is created; `build` then reads the resolved
- * image and otherwise falls back to the public `fromTag` path.
+ * worker boot for the deployment-global image and at turn time for pack-scoped
+ * images BEFORE the first sandbox using that ref is created; `build` then reads the
+ * resolved image and otherwise falls back to the public `fromTag` path.
  */
 export async function ensureModalRegistryImage(
   settings: Settings,
@@ -162,7 +174,12 @@ export async function ensureModalRegistryImage(
     pending = (async () => {
       const modal = await loadModal();
       const client = new modal.ModalClient(modalClientOptions(settings));
-      const secret = await modal.Secret.fromName(
+      // Resolve the Secret via the AUTHENTICATED client (client.secrets.fromName),
+      // NOT the static `modal.Secret.fromName`, which resolves against
+      // `getDefaultClient()` — i.e. the standard MODAL_TOKEN_ID/MODAL_TOKEN_SECRET env
+      // or ~/.modal.toml — and so would throw "Profile is missing token_id" in any host
+      // that supplies the token only through OpenGeni settings (OPENGENI_MODAL_TOKEN_ID).
+      const secret = await client.secrets.fromName(
         settings.modalImageRegistrySecret!,
         settings.modalEnvironment ? { environment: settings.modalEnvironment } : undefined,
       );
@@ -189,7 +206,7 @@ function cachedModalRegistryImage(settings: Settings): unknown | undefined {
 /**
  * Choose the image selector for a Modal sandbox client from settings. Returns:
  *  - `fromImage(resolved)` when a private-registry secret is configured AND the
- *    image has been resolved (ensureModalRegistryImage ran at worker boot);
+ *    image has been resolved (ensureModalRegistryImage ran before create);
  *  - `fromTag(modalImageRef)` for the public path (no secret, or cold cache — the
  *    resume/attach paths never pull an image so the tag branch is harmless there);
  *  - `undefined` when no image ref is set (Modal uses its default image).
@@ -201,7 +218,9 @@ export function resolveModalImageSelector(settings: Settings): ModalImageSelecto
   }
   const registryImage = cachedModalRegistryImage(settings);
   return registryImage
-    ? ModalImageSelector.fromImage(registryImage as Parameters<typeof ModalImageSelector.fromImage>[0])
+    ? ModalImageSelector.fromImage(
+        registryImage as Parameters<typeof ModalImageSelector.fromImage>[0],
+      )
     : ModalImageSelector.fromTag(settings.modalImageRef);
 }
 
@@ -211,7 +230,9 @@ export function __resetModalRegistryImageCacheForTest(): void {
   inFlightRegistryImages.clear();
 }
 
-function modalClientOptions(settings: Settings): ConstructorParameters<ModalModule["ModalClient"]>[0] {
+function modalClientOptions(
+  settings: Settings,
+): ConstructorParameters<ModalModule["ModalClient"]>[0] {
   return {
     ...(settings.modalTokenId ? { tokenId: settings.modalTokenId } : {}),
     ...(settings.modalTokenSecret ? { tokenSecret: settings.modalTokenSecret } : {}),
@@ -243,7 +264,10 @@ export async function tagModalSandbox(
   }
 }
 
-export async function terminateModalSandboxById(settings: Settings, sandboxId: string): Promise<boolean> {
+export async function terminateModalSandboxById(
+  settings: Settings,
+  sandboxId: string,
+): Promise<boolean> {
   if (!sandboxId) {
     return true;
   }
@@ -286,14 +310,22 @@ function tagsFromInfo(info: ModalSandboxInfo): Record<string, string> {
 }
 
 function sandboxCreatedAtMs(info: ModalSandboxInfo): number | null {
-  if (typeof info.createdAt !== "number" || !Number.isFinite(info.createdAt) || info.createdAt <= 0) {
+  if (
+    typeof info.createdAt !== "number" ||
+    !Number.isFinite(info.createdAt) ||
+    info.createdAt <= 0
+  ) {
     return null;
   }
   // Modal protobuf timestamps in this SDK are seconds as doubles.
-  return info.createdAt < 10_000_000_000 ? Math.floor(info.createdAt * 1000) : Math.floor(info.createdAt);
+  return info.createdAt < 10_000_000_000
+    ? Math.floor(info.createdAt * 1000)
+    : Math.floor(info.createdAt);
 }
 
-function attributionKey(input: Pick<ModalSandboxAttribution, "leaseId" | "workspaceId" | "sandboxGroupId">): string {
+function attributionKey(
+  input: Pick<ModalSandboxAttribution, "leaseId" | "workspaceId" | "sandboxGroupId">,
+): string {
   return `${input.workspaceId}:${input.sandboxGroupId}:${input.leaseId}`;
 }
 
@@ -311,6 +343,18 @@ export async function sweepModalOrphanSandboxes(
   const maxTerminations = options.maxTerminations ?? MODAL_ORPHAN_SWEEP_LIMIT;
   const unattributedGraceMs = options.unattributedGraceMs ?? MODAL_UNATTRIBUTED_ORPHAN_GRACE_MS;
   const liveByAttribution = new Map(liveLeases.map((lease) => [attributionKey(lease), lease]));
+  // LIVE-INSTANCE GUARD: a box that any live lease's envelope points at is NEVER
+  // an orphan, whatever its tags say. Tags are best-effort attribution (setTags
+  // is a separate call after create and can fail or lag); the lease envelope is
+  // the source of truth the turn path actually resumes by. Judging by tags alone
+  // terminated a LIVE box mid-turn at exactly creation+30min (staging session
+  // e644e8a8, 2026-07-06) — the box's unpushed work was unrecoverable because
+  // nothing outside the reaper drain persists /workspace.
+  const liveByInstanceId = new Map(
+    liveLeases
+      .filter((lease) => lease.instanceId)
+      .map((lease) => [lease.instanceId as string, lease]),
+  );
   const ownedClient = options.client ? null : await createModalClient(settings);
   const modal = (options.client ?? ownedClient)! as ModalCpListClient;
   try {
@@ -345,9 +389,40 @@ export async function sweepModalOrphanSandboxes(
         const leaseId = tags.opengeni_lease_id;
         const workspaceId = tags.opengeni_workspace_id;
         const sandboxGroupId = tags.opengeni_sandbox_group_id;
+        const liveByInstance = info.id ? liveByInstanceId.get(info.id) : undefined;
+        if (liveByInstance) {
+          // Live-instance guard (see above): a live lease resumes this exact box
+          // by id — hard-skip it, and HEAL its attribution tags when they are
+          // missing/stale so it stops looking sweep-eligible. Best-effort: a
+          // failed re-tag must never fail the sweep (the guard, not the tags,
+          // is what protects the box now).
+          if (
+            leaseId !== liveByInstance.leaseId ||
+            workspaceId !== liveByInstance.workspaceId ||
+            sandboxGroupId !== liveByInstance.sandboxGroupId
+          ) {
+            try {
+              const sandbox = await modal.sandboxes.fromId(info.id);
+              await sandbox.setTags(
+                modalSandboxAttributionTags({
+                  leaseId: liveByInstance.leaseId,
+                  workspaceId: liveByInstance.workspaceId,
+                  sandboxGroupId: liveByInstance.sandboxGroupId,
+                }),
+              );
+            } catch {
+              // Tag healing is opportunistic; the instance guard already
+              // protects this box on every future sweep pass.
+            }
+          }
+          skipped += 1;
+          continue;
+        }
         let reason: ModalOrphanSweepTermination["reason"] | null = null;
         if (leaseId && workspaceId && sandboxGroupId) {
-          const live = liveByAttribution.get(attributionKey({ leaseId, workspaceId, sandboxGroupId }));
+          const live = liveByAttribution.get(
+            attributionKey({ leaseId, workspaceId, sandboxGroupId }),
+          );
           if (!live || (live.instanceId && live.instanceId !== info.id)) {
             reason = "stale_attribution";
           }

package/src/sandbox/providers/selfhosted.ts

@@ -20,10 +20,7 @@ import {
   type ControlRpc,
   type NatsRequestConnection,
 } from "../selfhosted/control-rpc";
-import {
-  SelfhostedSandboxClient,
-  type SelfhostedRelayConfig,
-} from "../selfhosted/session";
+import { SelfhostedSandboxClient, type SelfhostedRelayConfig } from "../selfhosted/session";
 import type { ProviderRegistration } from "./types";
 
 /**