@opengeni/worker-bundle 1.3.0 → 1.4.2

package/dist/{chunk-QUX5LT2B.js → chunk-WVZQJBGD.js}

@@ -1,10 +1,11 @@
 import {
   observabilityEventBusOptions
-} from "./chunk-XONQEVAD.js";
+} from "./chunk-DCQ7XTXW.js";
 
 // src/activity-services.ts
 import {
   codeSearchDeploymentPolicy,
+  resolveWebSearchProvider,
   dbSearchPath,
   getSettings,
   resolveNatsControlPlaneAuth
@@ -413,6 +414,12 @@ function createSharedActivityServices(dependencies) {
       const resolvedSettings = (await resolveCatalogSettings(db, settings)).settings;
       configureChildLifecycleNotices({ enabled: settings.childLifecycleNoticesEnabled });
       configureCodeSearchDeploymentPolicy(codeSearchDeploymentPolicy(settings));
+      const webSearchProvider = resolveWebSearchProvider(settings);
+      if (webSearchProvider.status === "invalid") {
+        observability.warn("web search provider is misconfigured; web_search stays off", {
+          reason: webSearchProvider.reason
+        });
+      }
       return {
         settings: resolvedSettings,
         catalogSourceSettings: settings,
@@ -1558,7 +1565,7 @@ var GoogleDriveProviderTransportError = class extends Error {
 };
 async function fetchGoogleDriveProvider(input) {
   const now = input.now ?? Date.now;
-  const sleep3 = input.sleep ?? Bun.sleep;
+  const sleep2 = input.sleep ?? Bun.sleep;
   let retryDelaySpentMs = 0;
   for (let attempt = 1; attempt <= input.policy.attempts; attempt += 1) {
     let response;
@@ -1579,7 +1586,7 @@ async function fetchGoogleDriveProvider(input) {
       recordRetry(input.observability, input.operation, "transport", attempt, delayMs2);
       retryDelaySpentMs += delayMs2;
       input.heartbeat?.({ operation: input.operation, attempt, phase: "retry_delay_start" });
-      await sleep3(delayMs2);
+      await sleep2(delayMs2);
       input.heartbeat?.({ operation: input.operation, attempt, phase: "retry_delay_end" });
       continue;
     }
@@ -1609,7 +1616,7 @@ async function fetchGoogleDriveProvider(input) {
     );
     retryDelaySpentMs += delayMs;
     input.heartbeat?.({ operation: input.operation, attempt, phase: "retry_delay_start" });
-    await sleep3(delayMs);
+    await sleep2(delayMs);
     input.heartbeat?.({ operation: input.operation, attempt, phase: "retry_delay_end" });
   }
   throw new GoogleDriveProviderTransportError();
@@ -4990,7 +4997,9 @@ import {
   readLease as readLease2,
   readActiveSandbox,
   resolvePersonalMachineConnectionForAttempt,
-  SandboxRetainedProcessPromotionFencedError
+  SandboxRetainedProcessPromotionFencedError,
+  SandboxRetainedProcessTerminalError,
+  SandboxWorkspaceMutationOutputRejectedError
 } from "@opengeni/db";
 import {
   observeSessionBackgroundCommandCompletion,
@@ -5008,6 +5017,7 @@ import {
   NatsOpStreamTransport,
   RoutingBackendRecoveryRequiredError,
   RoutingSandboxSession,
+  RoutingMutationOutputRejectedError,
   resolveModalCheckpointProviderBindingForSession as resolveModalCheckpointProviderBindingForSession2,
   resolveConnectedMachineWorkspaceRoot,
   sandboxProviderInstanceIdFromEnvelope,
@@ -5208,9 +5218,6 @@ var SandboxLeaseInstanceLostError = class extends SandboxLeaseSupersededError {
   }
 };
 var WARMING_POLL_INTERVAL_MS = 250;
-async function sleep(ms) {
-  await new Promise((resolve) => setTimeout(resolve, ms));
-}
 async function waitForSandboxExecReadiness(established, timeoutMs = MODAL_EXEC_READINESS_TIMEOUT_MS, identity = {}, signal) {
   try {
     await verifySandboxExecReadiness(established, timeoutMs, signal);
@@ -5940,6 +5947,7 @@ async function resumeBoxForTurnOnce(services, ids, kind, holderId) {
   if (acquired.role === "spawner") {
     const expectedEpoch = acquired.lease.leaseEpoch;
     let createdEstablished = null;
+    let published = false;
     let providerCreateOperationId;
     let providerCreateBindingKey;
     let rematerialization = null;
@@ -6262,13 +6270,19 @@ async function resumeBoxForTurnOnce(services, ids, kind, holderId) {
         await release();
         throw new SandboxLeaseSupersededError(ids.sandboxGroupId, expectedEpoch);
       }
+      published = true;
       holderLeaseHeartbeat = {
         expectedEpoch: committed.lease.leaseEpoch,
         leaseTtlMs
       };
+      await services.onSpawnedSandboxPublished?.();
       throwIfReleasedOrCancelled();
       return { established, leaseEpoch: committed.lease.leaseEpoch, release };
     } catch (error) {
+      if (published) {
+        await release();
+        throw error;
+      }
       if (error instanceof SandboxLeaseSupersededError) {
         await terminateEstablishedSandbox(createdEstablished);
         await release();
@@ -6393,7 +6407,9 @@ async function waitForWarm(services, ids) {
   const deadline = Date.now() + settings.sandboxWarmingTimeoutMs;
   let instanceId = null;
   while (Date.now() < deadline) {
-    await sleep(WARMING_POLL_INTERVAL_MS);
+    if (!await sleepUnlessCancelled(WARMING_POLL_INTERVAL_MS, services.cancellationSignal)) {
+      throw services.cancellationSignal?.reason instanceof Error ? services.cancellationSignal.reason : new Error("Sandbox warming wait was cancelled with its owning turn attempt");
+    }
     const lease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
     if (!lease) {
       throw new SandboxLeaseSupersededError(ids.sandboxGroupId, 0);
@@ -6655,21 +6671,34 @@ function afterPersistableHomeMutation(services, ids, home) {
     }
     if (outcome === "outcome_unknown")
       throw new Error("Outcome-unknown command settlement requires its exact retained invocation");
-    await verifyWorkspaceMutationSettlement(services.db, {
-      accountId: fence.accountId,
-      workspaceId: ids.workspaceId,
-      sessionId: ids.sessionId,
-      turnId: fence.turnId,
-      executionGeneration: fence.executionGeneration,
-      attemptId: fence.attemptId,
-      holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
-      sandboxGroupId: home.sandboxGroupId,
-      expectedEpoch: backend.leaseEpoch,
-      expectedInstanceId: backend.providerInstanceId,
-      admission: exactAdmission,
-      operation: op,
-      outcome
-    });
+    try {
+      await verifyWorkspaceMutationSettlement(services.db, {
+        accountId: fence.accountId,
+        workspaceId: ids.workspaceId,
+        sessionId: ids.sessionId,
+        turnId: fence.turnId,
+        executionGeneration: fence.executionGeneration,
+        attemptId: fence.attemptId,
+        holderId: sandboxLeaseHolderIdForAttempt(fence.attemptId),
+        sandboxGroupId: home.sandboxGroupId,
+        expectedEpoch: backend.leaseEpoch,
+        expectedInstanceId: backend.providerInstanceId,
+        admission: exactAdmission,
+        operation: op,
+        outcome
+      });
+    } catch (error) {
+      if (error instanceof SandboxWorkspaceMutationOutputRejectedError && error.matchesPhysicalSettlement({
+        accountId: fence.accountId,
+        workspaceId: ids.workspaceId,
+        admission: exactAdmission,
+        operation: op,
+        outcome
+      })) {
+        throw new RoutingMutationOutputRejectedError(op, error.code, { cause: error });
+      }
+      throw error;
+    }
   };
 }
 function beforeRetainedProcessMutation(services, ids) {
@@ -6692,15 +6721,34 @@ function afterRetainedProcessMutation(services, ids) {
     if (!admission || typeof admission !== "object" || typeof admission.id !== "string" || typeof admission.workspaceGeneration !== "number") {
       throw new Error("Retained-process mutation settlement lacked its exact admission");
     }
-    await verifyRetainedProcessMutationSettlement(services.db, {
-      accountId: fence.accountId,
-      workspaceId: ids.workspaceId,
-      sessionId: ids.sessionId,
-      processId: process2.id,
-      admission,
-      operation: op,
-      outcome
-    });
+    try {
+      await verifyRetainedProcessMutationSettlement(services.db, {
+        accountId: fence.accountId,
+        workspaceId: ids.workspaceId,
+        sessionId: ids.sessionId,
+        processId: process2.id,
+        admission,
+        operation: op,
+        outcome
+      });
+    } catch (error) {
+      if (error instanceof SandboxWorkspaceMutationOutputRejectedError && error.matchesPhysicalSettlement({
+        accountId: fence.accountId,
+        workspaceId: ids.workspaceId,
+        admission,
+        operation: op,
+        outcome
+      })) {
+        if (error.retainedProcessTerminal) {
+          throw new SandboxRetainedProcessTerminalError(
+            error.retainedProcessTerminal.state,
+            error.retainedProcessTerminal.exitCode
+          );
+        }
+        throw new RoutingMutationOutputRejectedError(op, error.code, { cause: error });
+      }
+      throw error;
+    }
   };
 }
 function captureConnectedCommandOutput(db, ids, bus) {
@@ -8509,7 +8557,30 @@ async function waitForTurnOperation(operation, signal, disposeLateResult) {
     signal.removeEventListener("abort", cancel);
   }
 }
-function sleep2(ms) {
+var PHYSICAL_SANDBOX_RESUME_DRAIN_TIMEOUT_MS = 6e4;
+function trackPhysicalSandboxResume(inFlight, operation) {
+  inFlight.add(operation);
+  const settle = () => {
+    inFlight.delete(operation);
+  };
+  operation.then(settle, settle);
+  return operation;
+}
+async function drainPhysicalSandboxResumes(inFlight, timeoutMs = PHYSICAL_SANDBOX_RESUME_DRAIN_TIMEOUT_MS) {
+  if (inFlight.size === 0) return "none";
+  let timer;
+  try {
+    return await Promise.race([
+      Promise.allSettled([...inFlight]).then(() => "settled"),
+      new Promise((resolve) => {
+        timer = setTimeout(() => resolve("timed_out"), Math.max(1, timeoutMs));
+      })
+    ]);
+  } finally {
+    if (timer) clearTimeout(timer);
+  }
+}
+function sleep(ms) {
   return new Promise((resolve) => setTimeout(resolve, ms));
 }
 function sandboxProvisionErrorChain(error) {
@@ -8734,7 +8805,7 @@ function createTurnSandboxProvisioner(establish, options = {}) {
         if (!retrying) {
           throw error;
         }
-        await sleep2(backoffMs * internalAttempts);
+        await sleep(backoffMs * internalAttempts);
       }
     }
   };
@@ -8869,7 +8940,7 @@ async function persistOrSignalSessionAttemptQuiescence(input) {
         cause: receiptError
       });
     }
-    const delay = input.sleep ?? sleep2;
+    const delay = input.sleep ?? sleep;
     let retryMs = QUIESCENCE_PROOF_SIGNAL_INITIAL_RETRY_MS;
     let attempt = 1;
     for (; ; ) {
@@ -8918,7 +8989,7 @@ async function drainAttemptOwnedSandboxWriters(input) {
 async function releaseTurnSandboxAfterWriterDrain(sandbox, options = {}) {
   const maxAttempts = Math.max(1, options.maxAttempts ?? 3);
   const retryDelayMs2 = Math.max(0, options.retryDelayMs ?? 50);
-  const wait = options.wait ?? sleep2;
+  const wait = options.wait ?? sleep;
   let lastError;
   for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
     try {
@@ -9101,8 +9172,12 @@ import {
   isModalTaskExecStartPreDispatchUnavailableError,
   isModalCommandStartOutcomeUnknownError,
   isProviderCommandObservationUnavailableError,
+  ProviderCommandInputOutcomeUnknownError,
+  ProviderCommandStartOutcomeUnknownError,
   isRoutingMutationOutcomeUnknownError,
+  isRoutingMutationOutputRejectedError,
   RoutingWorkspaceRootChangedError,
+  RoutingBackendRecoveryRequiredError as RoutingBackendRecoveryRequiredError2,
   ResponsesStreamingTerminalError,
   SandboxMaterializationVerificationError,
   materializationVerificationDiagnostic,
@@ -9900,7 +9975,7 @@ function preClaimAdmissionFailure(error) {
     details: [detail]
   });
 }
-function databaseRecoveryCauseGraph(error) {
+function structuredRecoveryCauseGraph(error) {
   const graph = /* @__PURE__ */ new Map();
   const queue = [];
   let links = 0;
@@ -9928,7 +10003,7 @@ function databaseRecoveryCauseGraph(error) {
 }
 function retryableDatabaseFailureCode(error, requireDatabaseProvenance = false) {
   try {
-    const graph = databaseRecoveryCauseGraph(error);
+    const graph = structuredRecoveryCauseGraph(error);
     if (!graph) return null;
     const transports = /* @__PURE__ */ new Set();
     const boundaries = /* @__PURE__ */ new Set();
@@ -9940,7 +10015,8 @@ function retryableDatabaseFailureCode(error, requireDatabaseProvenance = false)
       [...graph.get(node)].filter((child) => child === node.cause)
     ) : [...graph.get(node)];
     for (const node of graph.keys()) {
-      if (isRoutingMutationOutcomeUnknownError(node)) return null;
+      if (isRoutingMutationOutcomeUnknownError(node) || isRoutingMutationOutputRejectedError(node))
+        return null;
       const record = node;
       if (requireDatabaseProvenance ? isRunningTurnDatabaseTransportFailure(record) : isRetryableDatabaseTransportFailure({ code: record.code, errno: record.errno }))
         transports.add(node);
@@ -9978,8 +10054,9 @@ function retryableDatabaseFailureCode(error, requireDatabaseProvenance = false)
       boundaries.add(node);
       const sqlState = typed ? node.details.sqlState : typeof record.code === "string" ? record.code : null;
       const connectionOutage = requireDatabaseProvenance ? isRunningTurnDatabaseConnectionSqlState(sqlState) : isDatabaseConnectionSqlState(sqlState);
-      const code = typed ? retryablePersistenceFailureCode(sqlState) : connectionOutage || isRetryablePersistenceSqlState(sqlState) ? databaseFailureCode(sqlState) : null;
-      if (!code || requireDatabaseProvenance && !connectionOutage && !(sqlState === null && hasOwnTransport(node)))
+      const rolledBack = isRetryablePersistenceSqlState(sqlState);
+      const code = typed ? retryablePersistenceFailureCode(sqlState) : connectionOutage || rolledBack ? databaseFailureCode(sqlState) : null;
+      if (!code || requireDatabaseProvenance && !connectionOutage && !rolledBack && !(sqlState === null && hasOwnTransport(node)))
         return null;
       if (!requireDatabaseProvenance || ownDatabaseNodes.has(node)) codes.add(code);
     }
@@ -10022,8 +10099,7 @@ function retryablePersistenceFailureCode(sqlState) {
 }
 function postClaimDatabaseRecoveryFailure(input) {
   const code = retryableDatabaseFailureCode(input.error, input.requireDatabaseProvenance);
-  if (!code || input.executionGeneration < 1 || input.requireDatabaseProvenance && code !== "db_failure")
-    return null;
+  if (!code || input.executionGeneration < 1) return null;
   if (input.sandboxSetupOutcomeUnknown && input.sandboxSetupRecoveryExhausted || (input.sandboxSetupOutcomeUnknown || input.sandboxSetupRecoveryExhausted) && input.providerRecovery) {
     return null;
   }
@@ -10095,7 +10171,29 @@ function modelPreparationFailureEventPayload(error, durationMs) {
     } : {}
   };
 }
+function isPreDispatchHomeBackendRecoveryRequired(error) {
+  const graph = structuredRecoveryCauseGraph(error);
+  if (!graph) return false;
+  let found = false;
+  try {
+    for (const node of graph.keys()) {
+      if (isRoutingMutationOutcomeUnknownError(node) || isRoutingMutationOutputRejectedError(node) || isRawProviderCommandOutcomeUnknown(node) || isModalCommandStartOutcomeUnknownError(node) || isProviderCommandObservationUnavailableError(node))
+        return false;
+      if (node instanceof RoutingBackendRecoveryRequiredError2) {
+        if (node.op !== "resolve_home_backend" || !Number.isSafeInteger(node.leaseEpoch) || node.leaseEpoch < 0 || !node.retryable || node.recovery !== "pending" && node.recovery !== "superseded")
+          return false;
+        found = true;
+      }
+    }
+  } catch {
+    return false;
+  }
+  return found;
+}
 function sandboxRouteTransitionCode(error) {
+  if (isPreDispatchHomeBackendRecoveryRequired(error)) {
+    return "home_backend_recovery_pending";
+  }
   const pending = [error];
   const seen = /* @__PURE__ */ new WeakSet();
   let inspected = 0;
@@ -10385,7 +10483,28 @@ function anthropicRequestDiagnostic(error) {
   if (error instanceof AnthropicRequestError) return error;
   return error instanceof Error && error.cause instanceof AnthropicRequestError ? error.cause : void 0;
 }
+function isRawProviderCommandOutcomeUnknown(error) {
+  try {
+    return error instanceof ProviderCommandInputOutcomeUnknownError || error instanceof ProviderCommandStartOutcomeUnknownError;
+  } catch {
+    return false;
+  }
+}
 function agentRunFailurePayload(error, options = {}) {
+  const graph = structuredRecoveryCauseGraph(error);
+  const nodes = graph ? [...graph.keys()] : [];
+  const outputRejected = nodes.find(isRoutingMutationOutputRejectedError);
+  if (outputRejected) {
+    const unknown = nodes.find(
+      (node) => isRoutingMutationOutcomeUnknownError(node) || isModalCommandStartOutcomeUnknownError(node) || isProviderCommandObservationUnavailableError(node) || isRawProviderCommandOutcomeUnknown(node)
+    );
+    if (unknown) return { ...baseAgentRunFailurePayload(unknown, options), retryable: false };
+    return {
+      error: outputRejected.message,
+      code: outputRejected.code,
+      retryable: false
+    };
+  }
   const failure = baseAgentRunFailurePayload(error, options);
   const diagnostic = materializationVerificationDiagnostic(error);
   const anthropic = anthropicRequestDiagnostic(error);
@@ -10874,6 +10993,8 @@ export {
   turnOperationCancellationFailure,
   throwIfTurnOperationCancelled,
   waitForTurnOperation,
+  trackPhysicalSandboxResume,
+  drainPhysicalSandboxResumes,
   classifySandboxLogicalProvisionFailure,
   isLazySandboxProvisionRetryable,
   sandboxDeadlineRotationRecoveryDelayMs,
@@ -10926,4 +11047,4 @@ export {
   CODEX_USAGE_LIMIT_MAX_RESUME_MS,
   classifyClaudeCredentialFailure
 };
-//# sourceMappingURL=chunk-QUX5LT2B.js.map
+//# sourceMappingURL=chunk-WVZQJBGD.js.map