@opengeni/worker-bundle 1.0.0-canary.0 → 1.0.0

package/src/activities/agent-turn/sandbox-establish.ts

@@ -1,4 +1,5 @@
 import {
+  authorizeAutomaticSandboxCheckpointRecovery,
   getEnrollment,
   getSandbox,
   readActiveSandbox,
@@ -6,7 +7,10 @@ import {
   assertPersonalMachineForAttempt,
   type SandboxRecord,
 } from "@opengeni/db";
-import { sandboxOperationMetricObserver } from "@opengeni/observability";
+import {
+  sandboxOperationMetricObserver,
+  sandboxCaptureWaitMetricObserver,
+} from "@opengeni/observability";
 import {
   withRunCredentialsSession,
   runOwnedSandboxSetup,
@@ -29,10 +33,14 @@ import {
   type MintedRunGitCredentials,
   type SandboxCodemodeAuthority,
 } from "../environment";
-import { rigProviderImageSourceImage } from "../packs";
+import { rigProviderImageSourceImage } from "../sandbox-images";
 import type { TurnActivityServices as ActivityServices, RunAgentTurnInput } from "../types";
 import type { currentActivityContext } from "../streaming";
-import { resumeBoxForTurn, type ResumedTurnSandbox } from "../../sandbox-resume";
+import {
+  createFreshSandboxReadinessReplacementBudget,
+  resumeBoxForTurn,
+  type ResumedTurnSandbox,
+} from "../../sandbox-resume";
 import {
   wrapTurnBoxWithRouting,
   wrapLazyTurnBoxWithRouting,
@@ -41,6 +49,8 @@ import {
 } from "../../sandbox-routing";
 import {
   makeMachineOpObserver,
+  recordSandboxAutomaticRecoverySelected,
+  sandboxAutomaticRecoveryOutcome,
   recordSandboxLogicalProvision,
   recordSandboxProvisionAttempt,
   recordSandboxSharedPreparation,
@@ -92,6 +102,7 @@ export type SandboxRouteOk = {
   machinePrimary: boolean;
   groupBoxBackend: Settings["sandboxBackend"];
   groupBoxImage: ReturnType<typeof rigProviderImageSourceImage>;
+  groupBoxImagePolicy: "new_creates_only";
   sandboxCreationBackend: Settings["sandboxBackend"];
   effectiveRunCredentialBackend: Settings["sandboxBackend"];
 };
@@ -289,12 +300,16 @@ export async function resolveSandboxRoute(deps: SandboxRouteDeps): Promise<Sandb
     machinePrimary,
     groupBoxBackend,
     groupBoxImage,
+    groupBoxImagePolicy: "new_creates_only",
     sandboxCreationBackend,
     effectiveRunCredentialBackend,
   };
 }
 
 export async function establishTurnSandbox(deps: EstablishTurnSandboxDeps): Promise<void> {
+  // One fresh-box readiness replacement per turn attempt, shared by the eager
+  // establish and every lazy provisioner retry of this attempt.
+  const freshSandboxReadinessReplacementBudget = createFreshSandboxReadinessReplacementBudget();
   const {
     input,
     settings,
@@ -328,6 +343,7 @@ export async function establishTurnSandbox(deps: EstablishTurnSandboxDeps): Prom
     machinePrimary,
     groupBoxBackend,
     groupBoxImage,
+    groupBoxImagePolicy,
     rigVersion,
     turnResources,
   } = deps;
@@ -368,7 +384,54 @@ export async function establishTurnSandbox(deps: EstablishTurnSandboxDeps): Prom
   ) {
     const sandboxEstablishStartedAt = performance.now();
     let sandboxEstablishOutcome: "completed" | "failed" = "completed";
+    // A committed system recovery decision is rematerialized in this turn even
+    // when the sandbox is otherwise on-demand: the decision pins group
+    // membership until publication, and the next turn must not dead-end.
+    let automaticRecoveryPending = false;
     try {
+      if (!machinePrimary && groupBoxBackend === "modal" && activeSandboxBackend !== "selfhosted") {
+        // This happens before buildTurnAgent reads the durable instruction
+        // tail, even for an on-demand sandbox. The DB admits only a provider-
+        // lost group with no unresolved workspace writers in ANY member: the
+        // latest verified checkpoint, or a new empty workspace when none is
+        // usable. Every group member receives its durable warning receipt.
+        const fallback = await authorizeAutomaticSandboxCheckpointRecovery(db, {
+          accountId: input.accountId,
+          workspaceId: input.workspaceId,
+          sessionId: input.sessionId,
+          attemptId: input.attemptId,
+        });
+        automaticRecoveryPending = fallback.status !== "not_eligible";
+        if (fallback.status === "authorized") {
+          try {
+            recordSandboxAutomaticRecoverySelected(
+              observability,
+              "modal",
+              sandboxAutomaticRecoveryOutcome(fallback),
+            );
+          } catch {
+            // Telemetry must not turn a committed recovery into another failure.
+          }
+          if (fallback.lane === "checkpoint") {
+            observability.warn("managed sandbox selected an older verified checkpoint", {
+              backend: "modal",
+              workspaceId: input.workspaceId,
+              sessionId: input.sessionId,
+              archiveGeneration: fallback.selection.archiveGeneration,
+              workspaceGeneration: fallback.selection.workspaceGeneration,
+              groupSessionCount: fallback.groupSessionCount,
+            });
+          } else {
+            observability.warn("managed sandbox continues on an empty workspace after loss", {
+              backend: "modal",
+              workspaceId: input.workspaceId,
+              sessionId: input.sessionId,
+              reason: fallback.reason,
+              groupSessionCount: fallback.groupSessionCount,
+            });
+          }
+        }
+      }
       const managedOwnership = managedSandboxOwnershipForTurn(
         machinePrimary,
         input.attemptId,
@@ -444,6 +507,7 @@ export async function establishTurnSandbox(deps: EstablishTurnSandboxDeps): Prom
             bus,
             onOp: machineOpObserver.observer,
             onSandboxOperation: sandboxOperationObserver,
+            onSandboxCaptureWait: sandboxCaptureWaitMetricObserver(observability),
             opJournal,
           },
           {
@@ -515,6 +579,7 @@ export async function establishTurnSandbox(deps: EstablishTurnSandboxDeps): Prom
               opJournal,
               onOp: machineOpObserver.observer,
               onSandboxOperation: sandboxOperationObserver,
+              onSandboxCaptureWait: sandboxCaptureWaitMetricObserver(observability),
               onHomeSandboxRebound,
               ...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
             },
@@ -587,8 +652,11 @@ export async function establishTurnSandbox(deps: EstablishTurnSandboxDeps): Prom
                 logicalFallbackSettings: logicalSandboxSettings,
                 cancellationSignal: sandboxResumeSignal,
                 sandboxMetrics: runtimeMetricsHooksForObservability(observability),
+                observability,
+                freshSandboxReadinessReplacementBudget,
                 onSandboxLost: publishSandboxLost,
                 objectStorage,
+                bus,
               },
               {
                 accountId: input.accountId,
@@ -601,6 +669,7 @@ export async function establishTurnSandbox(deps: EstablishTurnSandboxDeps): Prom
                 ...(groupBoxImage
                   ? {
                       image: groupBoxImage,
+                      imagePolicy: groupBoxImagePolicy,
                     }
                   : {}),
                 // The lazy acquire must enforce the same frozen rig authority
@@ -619,6 +688,7 @@ export async function establishTurnSandbox(deps: EstablishTurnSandboxDeps): Prom
               hasRepositoryResources: turnResources.some(
                 (resource) => resource.kind === "repository",
               ),
+              automaticRecoveryPending,
             })
           ) {
             startRunGitCredentialsMint();
@@ -662,8 +732,11 @@ export async function establishTurnSandbox(deps: EstablishTurnSandboxDeps): Prom
                 logicalFallbackSettings: logicalSandboxSettings,
                 cancellationSignal: sandboxResumeSignal,
                 sandboxMetrics: runtimeMetricsHooksForObservability(observability),
+                observability,
+                freshSandboxReadinessReplacementBudget,
                 onSandboxLost: publishSandboxLost,
                 objectStorage,
+                bus,
               },
               {
                 accountId: input.accountId,
@@ -677,17 +750,17 @@ export async function establishTurnSandbox(deps: EstablishTurnSandboxDeps): Prom
                 backend: groupBoxBackend,
                 os: groupBoxOs,
                 environment: sandboxEnvironment,
-                // IMAGE IS SHARED STATE (B3): the container image
-                // this run resolves. The lease stamps it + conflicts on a live shared box
-                // running a DIFFERENT image (solo → durable rotation; N-holders →
-                // SandboxImageConflictError surfaced as an actionable turn error). Select
-                // the image for the actual group-box backend; a configured Modal image must
+                // Deployment/workspace pins select NEW creates only. The lease
+                // retains a live group's image through between-turn repins, even
+                // with other holders; only rotation/reaping elects a new image.
+                // Select the image for the actual group-box backend; a configured Modal image must
                 // never override a Docker run. The selfhosted branch
                 // (establishSelfhostedTurnSession) NEVER passes
                 // an image — B3 lives only on this managed-box branch.
                 ...(groupBoxImage
                   ? {
                       image: groupBoxImage,
+                      imagePolicy: groupBoxImagePolicy,
                     }
                   : {}),
                 // RIG IS SHARED STATE (M3): stamp the frozen rig version so the lease
@@ -755,6 +828,7 @@ export async function establishTurnSandbox(deps: EstablishTurnSandboxDeps): Prom
               bus,
               opJournal,
               onSandboxOperation: sandboxOperationObserver,
+              onSandboxCaptureWait: sandboxCaptureWaitMetricObserver(observability),
               onHomeSandboxLost: publishSandboxLost,
               onHomeSandboxRebound,
               ...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),
@@ -1004,6 +1078,7 @@ export async function bindLazySandboxProvisioner(
                 {
                   settings: runSettings,
                   environment: sandboxEnvironment,
+                  recordLazyManifest: true,
                   onRuntimeEvent: async (event) => {
                     await eventing.publish?.([{ type: event.type, payload: event.payload }], true);
                   },
@@ -1122,6 +1197,7 @@ export async function bindLazySandboxProvisioner(
         opJournal,
         onOp: machineOpObserver.observer,
         onSandboxOperation: sandboxOperationObserver,
+        onSandboxCaptureWait: sandboxCaptureWaitMetricObserver(observability),
         onHomeSandboxLost: publishSandboxLost,
         onHomeSandboxRebound,
         ...(runtimeCancellationSignal ? { waitSignal: runtimeCancellationSignal } : {}),

package/src/activities/agent-turn/sandbox-provision.ts

@@ -570,8 +570,10 @@ export function createTurnSandboxProvisioner<T>(
  *   - legacy/built-in OpenAI or Azure Responses fallback (resolvedModel null);
  *   - resolved built-in OpenAI/Azure providers;
  *   - ChatGPT/Codex subscription backend (its strict allowlist permits the field).
+ *   - native Anthropic Messages (maps the internal key to Claude session identity;
+ *     the OpenAI field is never sent on that wire).
  *
- * Registry API-key providers are intentionally excluded. Fireworks' prompt-cache
+ * Other registry API-key providers are intentionally excluded. Fireworks' prompt-cache
  * docs prescribe `user` or `x-session-affinity`, not `prompt_cache_key`; Z.AI/GLM
  * documents automatic context caching plus `user_id`. Sending OpenAI-only fields
  * to unknown OpenAI-compatible providers risks unsupported-parameter 400s.

package/src/activities/agent-turn/sandbox-route.ts

@@ -142,13 +142,16 @@ export function shouldPrefetchManagedSandbox(input: {
   machinePrimary: boolean;
   groupBoxBackend: Settings["sandboxBackend"];
   hasRepositoryResources: boolean;
+  /** A committed automatic post-loss recovery decision still awaits its box.
+   * Rematerialize now instead of leaving the group pinned until a tool call. */
+  automaticRecoveryPending?: boolean;
 }): boolean {
   if (input.establishPolicy !== "on-demand") return false;
   if (input.machinePrimary) return false;
   if (input.groupBoxBackend === "none" || input.groupBoxBackend === "selfhosted") {
     return false;
   }
-  return input.hasRepositoryResources;
+  return input.hasRepositoryResources || input.automaticRecoveryPending === true;
 }
 
 /**
@@ -268,15 +271,6 @@ export async function reconcileActiveSandboxPointer(
   return { pointer: reread, record: rereadRecord };
 }
 
-/**
- * Warm the Modal private-registry image for the image ref this turn actually
- * resolved, not only the deployment-global OPENGENI_MODAL_IMAGE_REF warmed at
- * worker boot. A provider-native modalImageId bypasses registry import and is
- * resolved by ModalImageSelector.fromId during create. Otherwise packs can
- * override `modalImageRef` per workspace/turn, so a private pack image must be
- * resolved before sandbox creation or Modal falls back to the unauthenticated
- * `fromTag` path.
- */
 export async function ensureTurnModalRegistryImage(
   runSettings: Settings,
   sandboxCreationBackend: Settings["sandboxBackend"] | undefined,
@@ -302,17 +296,6 @@ export type SandboxArtifactRuntimeAdmission = Readonly<{
   environment: Readonly<Record<string, string>>;
 }>;
 
-/**
- * Admit the optional native standalone-file runtime only for the deployment's
- * exact base sandbox image contract. A pack/rig image override is an
- * independent filesystem and therefore fails closed even when the deployment
- * base image is capable. Collaborative artifact skills are admitted separately
- * from the frozen canonical tool catalog.
- *
- * This keeps lazy provisioning intact: CI/release proves the image closure,
- * while a before-agent-start doctor verifies the actual box before any model
- * call. No speculative sandbox is created merely to populate the skill index.
- */
 export function sandboxArtifactRuntimeAdmission(
   deploymentSettings: Settings,
   runSettings: Settings,