@opengeni/worker-bundle 1.4.0 → 1.4.4

package/dist/{chunk-BMTWULF7.js → chunk-BICGFGO7.js}

@@ -1,6 +1,6 @@
 import {
   observabilityEventBusOptions
-} from "./chunk-FF4HPY7L.js";
+} from "./chunk-HVNNCWEU.js";
 
 // src/activity-services.ts
 import {
@@ -1565,7 +1565,7 @@ var GoogleDriveProviderTransportError = class extends Error {
 };
 async function fetchGoogleDriveProvider(input) {
   const now = input.now ?? Date.now;
-  const sleep3 = input.sleep ?? Bun.sleep;
+  const sleep2 = input.sleep ?? Bun.sleep;
   let retryDelaySpentMs = 0;
   for (let attempt = 1; attempt <= input.policy.attempts; attempt += 1) {
     let response;
@@ -1586,7 +1586,7 @@ async function fetchGoogleDriveProvider(input) {
       recordRetry(input.observability, input.operation, "transport", attempt, delayMs2);
       retryDelaySpentMs += delayMs2;
       input.heartbeat?.({ operation: input.operation, attempt, phase: "retry_delay_start" });
-      await sleep3(delayMs2);
+      await sleep2(delayMs2);
       input.heartbeat?.({ operation: input.operation, attempt, phase: "retry_delay_end" });
       continue;
     }
@@ -1616,7 +1616,7 @@ async function fetchGoogleDriveProvider(input) {
     );
     retryDelaySpentMs += delayMs;
     input.heartbeat?.({ operation: input.operation, attempt, phase: "retry_delay_start" });
-    await sleep3(delayMs);
+    await sleep2(delayMs);
     input.heartbeat?.({ operation: input.operation, attempt, phase: "retry_delay_end" });
   }
   throw new GoogleDriveProviderTransportError();
@@ -4998,6 +4998,7 @@ import {
   readActiveSandbox,
   resolvePersonalMachineConnectionForAttempt,
   SandboxRetainedProcessPromotionFencedError,
+  SandboxRetainedProcessTerminalError,
   SandboxWorkspaceMutationOutputRejectedError
 } from "@opengeni/db";
 import {
@@ -5217,9 +5218,6 @@ var SandboxLeaseInstanceLostError = class extends SandboxLeaseSupersededError {
   }
 };
 var WARMING_POLL_INTERVAL_MS = 250;
-async function sleep(ms) {
-  await new Promise((resolve) => setTimeout(resolve, ms));
-}
 async function waitForSandboxExecReadiness(established, timeoutMs = MODAL_EXEC_READINESS_TIMEOUT_MS, identity = {}, signal) {
   try {
     await verifySandboxExecReadiness(established, timeoutMs, signal);
@@ -5949,6 +5947,7 @@ async function resumeBoxForTurnOnce(services, ids, kind, holderId) {
   if (acquired.role === "spawner") {
     const expectedEpoch = acquired.lease.leaseEpoch;
     let createdEstablished = null;
+    let published = false;
     let providerCreateOperationId;
     let providerCreateBindingKey;
     let rematerialization = null;
@@ -6271,13 +6270,19 @@ async function resumeBoxForTurnOnce(services, ids, kind, holderId) {
         await release();
         throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
       }
+      published = true;
       holderLeaseHeartbeat = {
         expectedEpoch: committed.lease.leaseEpoch,
         leaseTtlMs
       };
+      await services.onSpawnedSandboxPublished?.();
       throwIfReleasedOrCancelled();
       return { established, leaseEpoch: committed.lease.leaseEpoch, release };
     } catch (error) {
+      if (published) {
+        await release();
+        throw error;
+      }
       if (error instanceof SandboxLeaseSupersededError) {
         await terminateEstablishedSandbox(createdEstablished);
         await release();
@@ -6402,7 +6407,9 @@ async function waitForWarm(services, ids) {
   const deadline = Date.now() + settings.sandboxWarmingTimeoutMs;
   let instanceId = null;
   while (Date.now() < deadline) {
-    await sleep(WARMING_POLL_INTERVAL_MS);
+    if (!await sleepUnlessCancelled(WARMING_POLL_INTERVAL_MS, services.cancellationSignal)) {
+      throw services.cancellationSignal?.reason instanceof Error ? services.cancellationSignal.reason : new Error("Sandbox warming wait was cancelled with its owning turn attempt");
+    }
     const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
     if (!lease) {
       throw new SandboxLeaseSupersededError(ids.sandboxGroupId, 0);
@@ -6732,6 +6739,12 @@ function afterRetainedProcessMutation(services, ids) {
         operation: op,
         outcome
       })) {
+        if (error.retainedProcessTerminal) {
+          throw new SandboxRetainedProcessTerminalError(
+            error.retainedProcessTerminal.state,
+            error.retainedProcessTerminal.exitCode
+          );
+        }
         throw new RoutingMutationOutputRejectedError(op, error.code, { cause: error });
       }
       throw error;
@@ -8544,7 +8557,30 @@ async function waitForTurnOperation(operation, signal, disposeLateResult) {
     signal.removeEventListener("abort", cancel);
   }
 }
-function sleep2(ms) {
+var PHYSICAL_SANDBOX_RESUME_DRAIN_TIMEOUT_MS = 6e4;
+function trackPhysicalSandboxResume(inFlight, operation) {
+  inFlight.add(operation);
+  const settle = () => {
+    inFlight.delete(operation);
+  };
+  operation.then(settle, settle);
+  return operation;
+}
+async function drainPhysicalSandboxResumes(inFlight, timeoutMs = PHYSICAL_SANDBOX_RESUME_DRAIN_TIMEOUT_MS) {
+  if (inFlight.size === 0) return "none";
+  let timer;
+  try {
+    return await Promise.race([
+      Promise.allSettled([...inFlight]).then(() => "settled"),
+      new Promise((resolve) => {
+        timer = setTimeout(() => resolve("timed_out"), Math.max(1, timeoutMs));
+      })
+    ]);
+  } finally {
+    if (timer) clearTimeout(timer);
+  }
+}
+function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 function sandboxProvisionErrorChain(error) {
@@ -8769,7 +8805,7 @@ function createTurnSandboxProvisioner(establish, options = {}) {
         if (!retrying) {
           throw error;
         }
-        await sleep2(backoffMs * internalAttempts);
+        await sleep(backoffMs * internalAttempts);
       }
     }
   };
@@ -8904,7 +8940,7 @@ async function persistOrSignalSessionAttemptQuiescence(input) {
         cause: receiptError
       });
     }
-    const delay = input.sleep ?? sleep2;
+    const delay = input.sleep ?? sleep;
     let retryMs = QUIESCENCE_PROOF_SIGNAL_INITIAL_RETRY_MS;
     let attempt = 1;
     for (; ; ) {
@@ -8953,7 +8989,7 @@ async function drainAttemptOwnedSandboxWriters(input) {
 async function releaseTurnSandboxAfterWriterDrain(sandbox, options = {}) {
   const maxAttempts = Math.max(1, options.maxAttempts ?? 3);
   const retryDelayMs2 = Math.max(0, options.retryDelayMs ?? 50);
-  const wait = options.wait ?? sleep2;
+  const wait = options.wait ?? sleep;
   let lastError;
   for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
     try {
@@ -9143,6 +9179,7 @@ import {
   RoutingWorkspaceRootChangedError,
   RoutingBackendRecoveryRequiredError as RoutingBackendRecoveryRequiredError2,
   ResponsesStreamingTerminalError,
+  classifyModelStreamIdleTimeoutError,
   SandboxMaterializationVerificationError,
   materializationVerificationDiagnostic,
   PROVIDER_QUOTA_EXHAUSTED_CODE,
@@ -10018,8 +10055,9 @@ function retryableDatabaseFailureCode(error, requireDatabaseProvenance = false)
       boundaries.add(node);
       const sqlState = typed ? node.details.sqlState : typeof record.code === "string" ? record.code : null;
       const connectionOutage = requireDatabaseProvenance ? isRunningTurnDatabaseConnectionSqlState(sqlState) : isDatabaseConnectionSqlState(sqlState);
-      const code = typed ? retryablePersistenceFailureCode(sqlState) : connectionOutage || isRetryablePersistenceSqlState(sqlState) ? databaseFailureCode(sqlState) : null;
-      if (!code || requireDatabaseProvenance && !connectionOutage && !(sqlState === null && hasOwnTransport(node)))
+      const rolledBack = isRetryablePersistenceSqlState(sqlState);
+      const code = typed ? retryablePersistenceFailureCode(sqlState) : connectionOutage || rolledBack ? databaseFailureCode(sqlState) : null;
+      if (!code || requireDatabaseProvenance && !connectionOutage && !rolledBack && !(sqlState === null && hasOwnTransport(node)))
         return null;
       if (!requireDatabaseProvenance || ownDatabaseNodes.has(node)) codes.add(code);
     }
@@ -10062,8 +10100,7 @@ function retryablePersistenceFailureCode(sqlState) {
 }
 function postClaimDatabaseRecoveryFailure(input) {
   const code = retryableDatabaseFailureCode(input.error, input.requireDatabaseProvenance);
-  if (!code || input.executionGeneration < 1 || input.requireDatabaseProvenance && code !== "db_failure")
-    return null;
+  if (!code || input.executionGeneration < 1) return null;
   if (input.sandboxSetupOutcomeUnknown && input.sandboxSetupRecoveryExhausted || (input.sandboxSetupOutcomeUnknown || input.sandboxSetupRecoveryExhausted) && input.providerRecovery) {
     return null;
   }
@@ -10358,6 +10395,20 @@ function providerHttpStatus(error) {
   }
   return void 0;
 }
+function isTransportTimeoutError(error) {
+  let current = error;
+  const seen = /* @__PURE__ */ new Set();
+  for (let depth = 0; depth < 8 && current && typeof current === "object"; depth += 1) {
+    if (seen.has(current)) break;
+    seen.add(current);
+    const value = current;
+    if (value.name === "TimeoutError" || value.message === "The operation timed out.") {
+      return true;
+    }
+    current = value.cause;
+  }
+  return false;
+}
 function isTransientProviderError(error) {
   if (error instanceof ResponsesStreamingTerminalError) {
     return error.category === "unavailable";
@@ -10721,6 +10772,28 @@ function baseAgentRunFailurePayload(error, options = {}) {
       ...mcpTransportDiagnostic ? { mcpTransportDiagnostic } : {}
     };
   }
+  const streamIdle = classifyModelStreamIdleTimeoutError(error);
+  if (streamIdle) {
+    const seconds = streamIdle.idleTimeoutMs !== null ? Math.round(streamIdle.idleTimeoutMs / 1e3) : null;
+    const window = seconds !== null ? ` for ${seconds}s` : "";
+    return {
+      error: streamIdle.kind === "progress" ? `The model provider stream stalled: only keepalive traffic and no response progress${window}. The same turn will retry after a short delay.` : `The model provider stopped sending response data${window}. The same turn will retry after a short delay.`,
+      code: "provider_unavailable",
+      retryable: true,
+      timeoutClass: streamIdle.kind === "progress" ? "progress_stream" : "idle_stream",
+      responseObserved: true,
+      detail: streamIdle.message
+    };
+  }
+  if (status === void 0 && isTransportTimeoutError(error)) {
+    return {
+      error: "The model provider connection timed out. The same turn will retry after a short delay.",
+      code: "provider_unavailable",
+      retryable: true,
+      timeoutClass: "transport",
+      detail: message
+    };
+  }
   if (code === UNKNOWN_MODEL_FINISH_REASON_CODE) {
     return {
       error: "The model provider ended its response ambiguously. Partial output was not accepted as complete; the same turn will retry from durable history.",
@@ -10957,6 +11030,8 @@ export {
   turnOperationCancellationFailure,
   throwIfTurnOperationCancelled,
   waitForTurnOperation,
+  trackPhysicalSandboxResume,
+  drainPhysicalSandboxResumes,
   classifySandboxLogicalProvisionFailure,
   isLazySandboxProvisionRetryable,
   sandboxDeadlineRotationRecoveryDelayMs,
@@ -11009,4 +11084,4 @@ export {
   CODEX_USAGE_LIMIT_MAX_RESUME_MS,
   classifyClaudeCredentialFailure
 };
-//# sourceMappingURL=chunk-BMTWULF7.js.map
+//# sourceMappingURL=chunk-BICGFGO7.js.map