@opengeni/worker-bundle 2.3.1 → 2.3.2-canary.37090756480001

package/dist/{activities-control-3HAH4FMG.js → activities-control-3BY4IJKL.js}

@@ -24,6 +24,7 @@ import {
   notifyParentOfChildIdle,
   persistWorkspaceArchiveCandidate,
   providerRecoveryCountFromMetadata,
+  publishDurableSessionEvents,
   putVersion1TarArchiveOrInline,
   reconcileAutomaticSessionTitleFanout,
   reconcilePendingParentSystemUpdates,
@@ -34,7 +35,7 @@ import {
   warnDrainSnapshotFailure,
   withFirstPartyTools,
   workflowIdForSession
-} from "./chunk-3K3ULWEE.js";
+} from "./chunk-PA7Z55P4.js";
 import {
   inspectOpenSandboxKubernetesInventory,
   recordActiveUserGauges,
@@ -47,6 +48,7 @@ import {
   recordRetainedProcessReconciliation,
   recordSandboxCheckpointArtifactGauges,
   recordSandboxCheckpointArtifactOutcome,
+  recordSandboxCommandContainment,
   recordSandboxDeadlineRotationsRequested,
   recordSandboxInventoryProjectionFailure,
   recordSandboxInventoryProjectionSuccess,
@@ -60,7 +62,7 @@ import {
   recordVerifiedSignupTrialSwitchGauge,
   recordWorkerDeathRecoveryMetrics,
   runtimeMetricsHooksForObservability
-} from "./chunk-YG27NEMX.js";
+} from "./chunk-YOI3Z3GF.js";
 import "./chunk-PZ5AY32C.js";
 
 // src/activities/knowledge-indexing.ts
@@ -1156,6 +1158,85 @@ function createFileUploadReaperActivities(services, options = {}) {
   return { reapExpiredFileUploads, recoverVideoGenerationWorkflows };
 }
 
+// src/activities/model-call-fact-reconciler.ts
+import { randomUUID as randomUUID2 } from "crypto";
+import { listWorkspaceIdsAfter, reconcileModelCallFacts } from "@opengeni/db";
+var MODEL_CALL_FACT_RECONCILE_LOOKBACK_MS = 2 * 60 * 60 * 1e3;
+var MODEL_CALL_FACT_RECONCILE_SETTLE_MS = 10 * 60 * 1e3;
+var MODEL_CALL_FACT_RECONCILE_WORKSPACE_LIMIT = 500;
+var MODEL_CALL_FACT_RECONCILE_RUN_BUDGET_MS = 3 * 60 * 1e3;
+var WORKSPACE_PAGE_SIZE = 200;
+function createModelCallFactReconcilerActivities(services, options = {}) {
+  const now = options.now ?? Date.now;
+  const runBudgetMs = options.runBudgetMs ?? MODEL_CALL_FACT_RECONCILE_RUN_BUDGET_MS;
+  const startAfterWorkspaceId = options.startAfterWorkspaceId ?? randomUUID2;
+  const reconcile = options.reconcile ?? reconcileModelCallFacts;
+  const listWorkspaces = options.listWorkspaces ?? listWorkspaceIdsAfter;
+  async function reconcileRecentModelCallFacts() {
+    const { db, observability } = await services();
+    const startedAt = now();
+    const until = new Date(startedAt - MODEL_CALL_FACT_RECONCILE_SETTLE_MS);
+    const since = new Date(until.getTime() - MODEL_CALL_FACT_RECONCILE_LOOKBACK_MS);
+    const result = {
+      workspaces: 0,
+      missing: 0,
+      repaired: 0,
+      unrepaired: 0,
+      truncatedWorkspaces: 0,
+      failedWorkspaces: 0,
+      budgetExhausted: false
+    };
+    const start = startAfterWorkspaceId();
+    let cursor = start;
+    let wrapped = false;
+    while (true) {
+      const page = await listWorkspaces(db, { afterId: cursor, limit: WORKSPACE_PAGE_SIZE });
+      const ids = wrapped ? page.filter((id) => id <= start) : page;
+      for (const workspaceId of ids) {
+        if (now() - startedAt >= runBudgetMs) {
+          result.budgetExhausted = true;
+          break;
+        }
+        result.workspaces += 1;
+        try {
+          const outcome = await reconcile(db, {
+            workspaceId,
+            since,
+            until,
+            limit: MODEL_CALL_FACT_RECONCILE_WORKSPACE_LIMIT
+          });
+          result.missing += outcome.missing;
+          result.repaired += outcome.repaired;
+          result.unrepaired += outcome.unrepaired;
+          if (outcome.truncated) result.truncatedWorkspaces += 1;
+        } catch (error) {
+          result.failedWorkspaces += 1;
+          observability.warn("model call fact reconciliation failed for a workspace", {
+            workspaceId,
+            errorName: error instanceof Error ? error.name : "unknown"
+          });
+        }
+      }
+      if (result.budgetExhausted) break;
+      if (wrapped && ids.length < page.length) break;
+      if (page.length < WORKSPACE_PAGE_SIZE) {
+        if (wrapped) break;
+        wrapped = true;
+        cursor = null;
+        continue;
+      }
+      cursor = page[page.length - 1];
+    }
+    if (result.failedWorkspaces > 0 || result.truncatedWorkspaces > 0 || result.budgetExhausted) {
+      observability.warn("model call fact reconciliation incomplete", { ...result });
+    } else if (result.missing > 0) {
+      observability.info("model call fact reconciliation", { ...result });
+    }
+    return result;
+  }
+  return { reconcileRecentModelCallFacts };
+}
+
 // src/activities/retained-screenshot-reaper.ts
 import {
   claimRetainedScreenshotMaintenance,
@@ -2671,7 +2752,7 @@ function retainedProcessDeadlineRetryMs(process, lease, settings, retryAfterMs,
 }
 
 // src/activities/sandbox-lease.ts
-import { createHash, randomUUID as randomUUID2 } from "crypto";
+import { createHash, randomUUID as randomUUID3 } from "crypto";
 import { requestRetainedProcessDeadlineCancellation } from "@opengeni/db/retained-provider-commands";
 import { Context as Context2 } from "@temporalio/activity";
 import { OpLostReason, OpState } from "@opengeni/agent-proto";
@@ -2721,6 +2802,7 @@ import {
   workspaceArchiveCaptureDeadlineElapsed,
   retainedProcessReconciliationProof,
   retainedProcessSettlementIdentity,
+  recoverManagedSessionBackgroundCommand,
   settleClaimedConnectedMachineBackgroundCommand,
   settleSandboxCheckpointArtifactGc,
   rlsContextForWorkspace,
@@ -2887,14 +2969,18 @@ function createSandboxLeaseActivities(services, options = {}) {
     try {
       const { db, settings, observability } = await services();
       const timing = sandboxDrainTiming(settings);
-      const onUnobservableCommandDrainError = (error) => {
-        observability.warn("sandbox reaper: unobservable command drain inspection failed", {
-          error: error instanceof Error ? error.message : String(error)
-        });
+      const commandContainment = {
+        idleCommandContainmentMs: settings.sandboxIdleCommandContainmentMs,
+        onCommandContainment: (outcome) => recordSandboxCommandContainment(observability, outcome),
+        onCommandContainmentError: (error) => {
+          observability.warn("sandbox reaper: retained command containment inspection failed", {
+            error: error instanceof Error ? error.message : String(error)
+          });
+        }
       };
       if (!settings.sandboxOwnershipEnabled) {
         const drainableInventory2 = await reapStaleLeaseHoldersGlobal(db, {
-          onUnobservableCommandDrainError,
+          ...commandContainment,
           viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
           turnHolderTtlMs: settings.sandboxLeaseTtlMs,
           interactionHolderTtlMs: settings.sandboxInteractionHolderTtlMs,
@@ -2921,7 +3007,7 @@ function createSandboxLeaseActivities(services, options = {}) {
       });
       recordSandboxDeadlineRotationsRequested(observability, rotationsRequested);
       const drainableInventory = await reapStaleLeaseHoldersGlobal(db, {
-        onUnobservableCommandDrainError,
+        ...commandContainment,
         viewerHolderTtlMs: settings.sandboxViewerHolderTtlMs,
         // Dead-worker turn holders: a live holder is touched every 10s from the
         // moment it is registered (resumeBoxForTurn's holder-liveness loop covers
@@ -2962,7 +3048,7 @@ function createSandboxLeaseActivities(services, options = {}) {
       instanceId: input.target.instanceId
     });
     try {
-      const { db, settings, observability, objectStorage } = await services();
+      const { db, settings, observability, objectStorage, bus } = await services();
       assertSandboxDrainInputTiming(input);
       const drainSettings = settings.sandboxSnapshotTimeoutMs === input.snapshotTimeoutMs ? settings : { ...settings, sandboxSnapshotTimeoutMs: input.snapshotTimeoutMs };
       const row = input.target;
@@ -2982,7 +3068,8 @@ function createSandboxLeaseActivities(services, options = {}) {
           terminateBox,
           probeDrainableProvider,
           captureAttempt,
-          objectStorage
+          objectStorage,
+          bus
         );
         return { status: drainedCold ? "terminated" : "skipped" };
       } catch (error) {
@@ -3178,7 +3265,7 @@ function createSandboxLeaseActivities(services, options = {}) {
           timeoutClass: plan.timeoutClass,
           snapshotTimeoutMs: plan.snapshotTimeoutMs,
           captureTimeoutMs: plan.captureTimeoutMs,
-          operationId: randomUUID2()
+          operationId: randomUUID3()
         })
       )
     );
@@ -3617,6 +3704,24 @@ async function reconcileTerminalRetainedProcesses(db, settings, observability, p
       sessionId: process.sessionId,
       processId: process.id
     };
+    if (process.ownerActorKind === "turn" && process.providerBackend === "modal" && !proof && !claim.ownerState.startsWith("background_")) {
+      try {
+        const command = await recoverManagedSessionBackgroundCommand(db, {
+          ...processScope,
+          expected,
+          reconciliationClaimId: claim.claimId
+        });
+        if (command) {
+          claim.ownerState = `background_${command.state}`;
+          recordRetainedProcessReconciliation(observability, "background_recovered");
+        }
+      } catch (error) {
+        observability.warn("sandbox reaper: retained-command background recovery failed", {
+          processId: process.id,
+          error: error instanceof Error ? error.message : String(error)
+        });
+      }
+    }
     const storedCommandPersistence = retainedProviderCommandPersistence(
       db,
       processScope,
@@ -4608,7 +4713,7 @@ function sandboxDrainReason(lease) {
   if (lease.rotationRequestedAt === null) return "idle";
   return lease.rotationReason ?? "rotation_requested";
 }
-async function terminateDrainableBox(db, settings, row, observability, terminateBox, probeDrainableProvider, attempt, objectStorage) {
+async function terminateDrainableBox(db, settings, row, observability, terminateBox, probeDrainableProvider, attempt, objectStorage, bus = null) {
   const { accountId } = await rlsContextForWorkspace(db, row.workspaceId);
   const lease = await readLease2(db, row.workspaceId, row.sandboxGroupId);
   if (!lease) {
@@ -4922,18 +5027,31 @@ async function terminateDrainableBox(db, settings, row, observability, terminate
     }
   }
   const providerMissing = providerMissingBeforeCapture || (typeof termination === "boolean" ? false : termination.providerMissingBeforeCapture);
-  const { wentCold } = await confirmDrainCold(db, {
+  const { wentCold, backgroundCommandEvents } = await confirmDrainCold(db, {
     accountId,
     workspaceId: row.workspaceId,
     sandboxGroupId: row.sandboxGroupId,
     expectedEpoch: row.leaseEpoch,
     ...captureClaim ? { expectedCaptureId: captureClaim.id } : {},
-    providerMissingBeforeCapture: providerMissing
+    providerMissingBeforeCapture: providerMissing,
+    idleCommandContainmentMs: settings.sandboxIdleCommandContainmentMs
+  });
+  await publishDurableSessionEvents(bus, row.workspaceId, backgroundCommandEvents, (error) => {
+    observability.warn("sandbox reaper: contained command event fanout failed", {
+      sandboxGroupId: row.sandboxGroupId,
+      error: error instanceof Error ? error.message : String(error)
+    });
   });
   if (wentCold) {
     if (providerMissing) {
       recordSandboxProviderMissingBeforeCapture(observability, backend);
     }
+    if (lease.unobservableCommandDrainIds?.length) {
+      recordSandboxCommandContainment(
+        observability,
+        providerMissing ? "provider_missing" : "contained"
+      );
+    }
     try {
       await appendSessionEventToSandboxGroup(db, row.workspaceId, row.sandboxGroupId, {
         type: "sandbox.box.terminated",
@@ -5123,7 +5241,7 @@ async function terminateProviderBox(settings, lease, observability, persistArchi
         session,
         Date.now(),
         {
-          requestId: providerCaptureRequestId ?? randomUUID2(),
+          requestId: providerCaptureRequestId ?? randomUUID3(),
           strategy: liveCapturePolicy.strategy
         },
         diskBackedArchives
@@ -5317,7 +5435,7 @@ import {
   upsertScheduledSessionGoalForRun,
   withSessionActivityRlsContext
 } from "@opengeni/db";
-import { publishDurableSessionEvents } from "@opengeni/events";
+import { publishDurableSessionEvents as publishDurableSessionEvents2 } from "@opengeni/events";
 import {
   allowedFirstPartyMcpToolsForSession,
   resolveFirstPartyMcpToolPolicy,
@@ -6458,7 +6576,7 @@ function createScheduledTaskActivities(services) {
                     events: started.events
                   });
                 } else {
-                  await publishDurableSessionEvents(
+                  await publishDurableSessionEvents2(
                     bus,
                     task.workspaceId,
                     session.id,
@@ -6501,7 +6619,7 @@ function createScheduledTaskActivities(services) {
                 if (deferPublications) {
                   deferredEvents.push({ sessionId: session.id, events: started.events });
                 } else {
-                  await publishDurableSessionEvents(
+                  await publishDurableSessionEvents2(
                     bus,
                     task.workspaceId,
                     session.id,
@@ -6530,7 +6648,7 @@ function createScheduledTaskActivities(services) {
               if (deferPublications) {
                 deferredEvents.push({ sessionId: session.id, events: titleEvents });
               } else {
-                await publishDurableSessionEvents(bus, task.workspaceId, session.id, titleEvents);
+                await publishDurableSessionEvents2(bus, task.workspaceId, session.id, titleEvents);
               }
             }
             const scheduledUpdate = await addSessionSystemUpdateWithSourceMutation(
@@ -6617,7 +6735,7 @@ function createScheduledTaskActivities(services) {
                   events: scheduledUpdate.events
                 });
               } else {
-                await publishDurableSessionEvents(
+                await publishDurableSessionEvents2(
                   bus,
                   task.workspaceId,
                   session.id,
@@ -6744,7 +6862,7 @@ function createScheduledTaskActivities(services) {
                   events: bundled.events
                 });
               } else {
-                await publishDurableSessionEvents(
+                await publishDurableSessionEvents2(
                   bus,
                   task.workspaceId,
                   session.id,
@@ -6826,7 +6944,7 @@ function createScheduledTaskActivities(services) {
         });
       }
       for (const deferred of deferredEvents) {
-        await publishDurableSessionEvents(
+        await publishDurableSessionEvents2(
           bus,
           task.workspaceId,
           deferred.sessionId,
@@ -7058,7 +7176,7 @@ async function seedScheduledGeneratedSessionRoute(input) {
     if (input.deferPublications) {
       input.deferredEvents.push({ sessionId: input.session.id, events: failed.events });
     } else {
-      await publishDurableSessionEvents(
+      await publishDurableSessionEvents2(
         input.bus,
         input.task.workspaceId,
         input.session.id,
@@ -7328,7 +7446,7 @@ async function recoverBoundScheduledTaskDispatch(input) {
     });
     workflowId = started.temporalWorkflowId;
     if (started.events.length > 0) {
-      await publishDurableSessionEvents(input.bus, task.workspaceId, session.id, started.events);
+      await publishDurableSessionEvents2(input.bus, task.workspaceId, session.id, started.events);
     }
     if (task.runMode === "reusable_session") {
       await materializeScheduledTaskReusableSessionFromRun(input.db, {
@@ -7347,7 +7465,7 @@ async function recoverBoundScheduledTaskDispatch(input) {
       taskName: task.name
     });
     if (titleEvents.length > 0) {
-      await publishDurableSessionEvents(input.bus, task.workspaceId, session.id, titleEvents);
+      await publishDurableSessionEvents2(input.bus, task.workspaceId, session.id, titleEvents);
     }
   }
   const scheduledUpdate = await addSessionSystemUpdateWithSourceMutation(
@@ -7440,7 +7558,7 @@ async function recoverBoundScheduledTaskDispatch(input) {
     recordUserMessageAccepted(input.observability, { surface: "scheduled" });
   }
   if (scheduledUpdate.added && scheduledUpdate.events.length > 0) {
-    await publishDurableSessionEvents(
+    await publishDurableSessionEvents2(
       input.bus,
       task.workspaceId,
       session.id,
@@ -7796,9 +7914,10 @@ import {
   expireSessionHumanInputRequest,
   markSessionAttemptQuiesced,
   requireSession as requireSession2,
-  settleSessionIdleWithParentOutbox
+  settleSessionIdleWithParentOutbox,
+  SANDBOX_SETUP_RECOVERY_LIMIT
 } from "@opengeni/db";
-import { publishDurableSessionEvents as publishDurableSessionEvents2 } from "@opengeni/events";
+import { publishDurableSessionEvents as publishDurableSessionEvents3 } from "@opengeni/events";
 import { CancelledFailure } from "@temporalio/activity";
 var WORKER_DEATH_MAX_REDISPATCHES = 3;
 function createSessionStateActivities(services, overrides = {}) {
@@ -7822,7 +7941,7 @@ function createSessionStateActivities(services, overrides = {}) {
   const requireSessionFn = overrides.requireSession ?? requireSession2;
   const settleSessionIdleWithParentOutboxFn = overrides.settleSessionIdleWithParentOutbox ?? settleSessionIdleWithParentOutbox;
   const markSessionAttemptQuiescedFn = overrides.markSessionAttemptQuiesced ?? markSessionAttemptQuiesced;
-  const publishDurableSessionEventsFn = overrides.publishDurableSessionEvents ?? publishDurableSessionEvents2;
+  const publishDurableSessionEventsFn = overrides.publishDurableSessionEvents ?? publishDurableSessionEvents3;
   const deliverFailedChildTurnToParentFn = overrides.deliverFailedChildTurnToParent ?? deliverFailedChildTurnToParent;
   const notifyParentOfChildIdleFn = overrides.notifyParentOfChildIdle ?? notifyParentOfChildIdle;
   const recordTurnsQueuedGaugeFn = overrides.recordTurnsQueuedGauge ?? recordTurnsQueuedGauge;
@@ -7911,12 +8030,16 @@ function createSessionStateActivities(services, overrides = {}) {
     const postClaimIdentityMatches = Boolean(
       postClaimRecovery && turn.id === postClaimRecovery.turnId && turn.triggerEventId === postClaimRecovery.triggerEventId && turn.executionGeneration === postClaimRecovery.executionGeneration
     );
+    if (postClaimRecovery?.sandboxSetupRecoveryExhausted && !postClaimIdentityMatches) {
+      return { action: "stale" };
+    }
     const providerRecoveryCount = postClaimIdentityMatches ? postClaimRecovery?.providerRecoveryCount : void 0;
     const providerFailureCode = postClaimIdentityMatches ? postClaimRecovery?.providerFailureCode : void 0;
     const hasProviderRecoveryCount = providerRecoveryCount !== void 0;
     const hasProviderFailureCode = providerFailureCode !== void 0;
     const setupOutcomeUnknown = postClaimIdentityMatches && postClaimRecovery?.sandboxSetupOutcomeUnknown === true;
-    if (setupOutcomeUnknown && (hasProviderRecoveryCount || hasProviderFailureCode)) {
+    const setupRecoveryExhausted = postClaimIdentityMatches && postClaimRecovery?.sandboxSetupRecoveryExhausted === true;
+    if (setupOutcomeUnknown && setupRecoveryExhausted || (setupOutcomeUnknown || setupRecoveryExhausted) && (hasProviderRecoveryCount || hasProviderFailureCode) || setupRecoveryExhausted && turn.metadata?.providerRecoveryCount !== SANDBOX_SETUP_RECOVERY_LIMIT) {
       return { action: "stale" };
     }
     if (hasProviderRecoveryCount !== hasProviderFailureCode) {
@@ -7932,13 +8055,20 @@ function createSessionStateActivities(services, overrides = {}) {
         turnId: turn.id,
         triggerEventId: turn.triggerEventId,
         attemptId: input.attemptId,
-        reason: setupOutcomeUnknown ? "sandbox_command_start_outcome_unknown" : providerFailureCode ?? "claimed_attempt_database_failure",
+        reason: setupOutcomeUnknown ? "sandbox_command_start_outcome_unknown" : setupRecoveryExhausted ? "sandbox_command_start_recovery_exhausted" : providerFailureCode ?? "claimed_attempt_database_failure",
         ...setupOutcomeUnknown ? { sandboxSetupOutcomeUnknown: true } : {},
+        ...setupRecoveryExhausted ? { sandboxSetupRecoveryExhausted: true } : {},
         ...providerRecoveryCount !== void 0 ? { providerRecoveryCount } : {},
         detail: {
-          code: setupOutcomeUnknown ? "sandbox_command_start_outcome_unknown" : providerFailureCode ?? recoveredClaimCode,
-          retryable: !setupOutcomeUnknown,
+          code: setupOutcomeUnknown ? "sandbox_command_start_outcome_unknown" : setupRecoveryExhausted ? "sandbox_command_start_recovery_exhausted" : providerFailureCode ?? recoveredClaimCode,
+          retryable: !setupOutcomeUnknown && !setupRecoveryExhausted,
           ...setupOutcomeUnknown ? { setupOutcome: "unknown", replay: "blocked" } : {},
+          ...setupRecoveryExhausted ? {
+            setupOutcome: "not_started",
+            replay: "blocked",
+            recoveryExhausted: true,
+            providerRecoveryCount: SANDBOX_SETUP_RECOVERY_LIMIT
+          } : {},
           ...providerRecoveryCount !== void 0 ? {
             databaseFailureCode: recoveredClaimCode,
             providerRecoveryCount
@@ -8374,6 +8504,7 @@ function createControlActivitiesFromServices(services, resolveDocumentServices)
     ...createBrowserStateArtifactMaintenanceActivities(services),
     ...createBrowserDeadlineCheckpointActivities(services),
     ...createFileUploadReaperActivities(services),
+    ...createModelCallFactReconcilerActivities(services),
     ...createRetainedScreenshotMaintenanceActivities(services),
     ...createWorkflowWakeActivities(services),
     ...createSandboxLeaseActivities(services)
@@ -8394,4 +8525,4 @@ export {
   createControlActivities,
   createControlActivitiesFromServices
 };
-//# sourceMappingURL=activities-control-3HAH4FMG.js.map
+//# sourceMappingURL=activities-control-3BY4IJKL.js.map