@opengeni/worker-bundle 0.28.3 → 0.28.4-canary.0

package/dist/{chunk-NNXLW6V7.js → chunk-ZNIKZQMN.js}

@@ -1,6 +1,6 @@
 import {
   observabilityEventLogger
-} from "./chunk-BARTLVBK.js";
+} from "./chunk-A4XQWVZK.js";
 
 // src/activity-services.ts
 import { dbSearchPath, getSettings, resolveNatsControlPlaneAuth } from "@opengeni/config";
@@ -1646,6 +1646,8 @@ async function nextStreamEvent(iterator, context) {
 }
 
 // src/sandbox-archive-storage.ts
+import { uploadWorkspaceArchiveSpool } from "@opengeni/storage";
+import { randomUUID as randomUUID2 } from "crypto";
 import {
   parseWorkspaceArchiveObjectRef,
   workspaceArchiveObjectKey
@@ -1657,47 +1659,74 @@ var WorkspaceArchiveObjectStorageRequiredError = class extends Error {
     this.name = "WorkspaceArchiveObjectStorageRequiredError";
   }
 };
-function collectWorkspaceArchiveObjectKeys(resumeState) {
-  const sessionState = resumeState?.sessionState && typeof resumeState.sessionState === "object" ? resumeState.sessionState : null;
-  const keys = /* @__PURE__ */ new Set();
-  for (const value of [sessionState?.workspaceArchiveRef, sessionState?.workspaceArchivePrevRef]) {
-    const ref = parseWorkspaceArchiveObjectRef(value);
-    if (ref) keys.add(ref.key);
-  }
-  return keys;
-}
 async function putTarWorkspaceArchiveObject(input) {
-  if (input.archive.descriptor.version !== 1) {
+  const descriptor = structuredClone(input.archive.descriptor);
+  if (descriptor.version !== 1) {
     throw new Error("Object-storage workspace archives are portable tar only");
   }
   const key = workspaceArchiveObjectKey({
     accountId: input.accountId,
     workspaceId: input.workspaceId,
     sandboxGroupId: input.sandboxGroupId,
-    revision: input.archive.descriptor.revision
-  });
-  await input.objectStorage.putObject({
-    key,
-    contentType: "application/x-tar",
-    body: input.archive.bytes,
-    sha256: input.archive.descriptor.archiveSha256
+    revision: descriptor.revision,
+    uploadId: randomUUID2()
   });
+  const archive = input.archive;
+  if ("spool" in archive) {
+    if (archive.spool.byteSize !== descriptor.archiveBytes || archive.spool.sha256 !== descriptor.archiveSha256) {
+      throw new Error("Workspace archive spool does not match its verified descriptor");
+    }
+  }
+  const source = "spool" in archive ? archive.spool : {
+    path: "",
+    byteSize: archive.bytes.byteLength,
+    sha256: descriptor.archiveSha256,
+    async *open() {
+      for (let offset = 0; offset < archive.bytes.length; offset += 1024 * 1024)
+        yield archive.bytes.subarray(offset, offset + 1024 * 1024);
+    },
+    async dispose() {
+    }
+  };
+  try {
+    await uploadWorkspaceArchiveSpool(input.objectStorage, key, {
+      ...source,
+      byteSize: descriptor.archiveBytes,
+      sha256: descriptor.archiveSha256,
+      open: source.open.bind(source)
+    });
+  } catch (error) {
+    await input.objectStorage.deleteObject(key).catch(() => void 0);
+    throw error;
+  }
   return {
     schema: "sandbox_archive_object_v1",
     key,
-    sha256: input.archive.descriptor.archiveSha256,
-    bytes: input.archive.descriptor.archiveBytes,
+    sha256: descriptor.archiveSha256,
+    bytes: descriptor.archiveBytes,
     backend: input.objectStorage.backend
   };
 }
+async function persistWorkspaceArchiveCandidate(input) {
+  const objectStorage = input.objectStorage;
+  const ref = input.ref ? { ...input.ref } : void 0;
+  const result = await input.persist();
+  if (objectStorage && ref && result.candidateDisposition === "unused") {
+    await deleteUnpublishedWorkspaceArchiveObject(objectStorage, ref);
+  }
+  return result;
+}
 async function putVersion1TarArchiveOrInline(input) {
   if (input.archive.descriptor.version !== 1) {
+    if ("spool" in input.archive) throw new Error("Native snapshots cannot use a portable spool");
     return { workspaceArchive: input.archive.base64 };
   }
   if (input.backend === "opensandbox" && !input.objectStorage) {
     throw new WorkspaceArchiveObjectStorageRequiredError("opensandbox");
   }
   if (!input.objectStorage) {
+    if ("spool" in input.archive)
+      throw new WorkspaceArchiveObjectStorageRequiredError(input.backend);
     return { workspaceArchive: input.archive.base64 };
   }
   try {
@@ -1706,7 +1735,7 @@ async function putVersion1TarArchiveOrInline(input) {
       accountId: input.accountId,
       workspaceId: input.workspaceId,
       sandboxGroupId: input.sandboxGroupId,
-      archive: { bytes: input.archive.bytes, descriptor: input.archive.descriptor }
+      archive: input.archive
     });
     try {
       input.metrics?.onWorkspaceArchiveObject?.({
@@ -1747,11 +1776,6 @@ async function deleteUnpublishedWorkspaceArchiveObject(objectStorage, ref, metri
     console.error("unpublished workspace archive object delete failed", { key: ref.key });
   }
 }
-async function deleteWorkspaceArchiveObjectKeys(objectStorage, keys) {
-  for (const key of keys) {
-    await objectStorage.deleteObject(key);
-  }
-}
 
 // src/sandbox-routing.ts
 import { sandboxLifecycleTransitionWaitMs as sandboxLifecycleTransitionWaitMs2 } from "@opengeni/config";
@@ -1806,7 +1830,7 @@ import {
   sandboxArchiveCaptureTimeoutMs,
   sandboxLifecycleTransitionWaitMs
 } from "@opengeni/config";
-import { randomUUID as randomUUID2 } from "crypto";
+import { randomUUID as randomUUID3 } from "crypto";
 import {
   acquireLease,
   adoptLegacyModalCheckpointArtifact,
@@ -1833,7 +1857,8 @@ import {
   SandboxLeaseTransitionError
 } from "@opengeni/db";
 import {
-  captureVerifiedWorkspaceArchive,
+  captureWorkspaceArchiveForStorage,
+  disposeWorkspaceArchive,
   describeLegacyNativeSnapshotArchive,
   inlineWorkspaceArchiveForRestore,
   MODAL_EXEC_READINESS_TIMEOUT_MS,
@@ -1854,6 +1879,10 @@ import {
   verifySandboxExecReadiness,
   withoutSandboxProviderIdentity
 } from "@opengeni/runtime";
+import {
+  downloadWorkspaceArchiveSpool,
+  WorkspaceArchiveStorageError
+} from "@opengeni/storage";
 import { parseWorkspaceArchiveObjectRef as parseWorkspaceArchiveObjectRef2 } from "@opengeni/contracts";
 function sandboxLeaseHolderIdForAttempt(attemptId) {
   const normalized = attemptId.trim();
@@ -2109,6 +2138,10 @@ async function materializeSpawnEnvelopeArchive(envelope, objectStorage) {
       "workspace archive object storage is not configured"
     );
   }
+  const descriptor = parseWorkspaceArchiveDescriptor(sessionState.workspaceArchiveMeta);
+  if (process.platform === "linux" && descriptor?.version === 1 && descriptor.workspace.projection === "sdk_local_archive_v1") {
+    return envelope;
+  }
   return {
     ...record,
     sessionState: await inlineWorkspaceArchiveForRestore(
@@ -2206,7 +2239,7 @@ async function persistWarmWorkspaceSnapshot(services, ids, session, leaseEpoch,
     const nativeModalPersistence = workspacePersistence === "snapshot_filesystem" || workspacePersistence === "snapshot_directory";
     const checkpointBinding = nativeModalPersistence ? await resolveModalCheckpointBindingBeforeCapture(settings, persistable, signal) : null;
     if (signal?.aborted) return false;
-    const captureId = randomUUID2();
+    const captureId = randomUUID3();
     const captureTimeoutMs = sandboxArchiveCaptureTimeoutMs(settings);
     const claimed = await claimWorkspaceArchiveCapture(db, {
       accountId: ids.accountId,
@@ -2239,7 +2272,7 @@ async function persistWarmWorkspaceSnapshot(services, ids, session, leaseEpoch,
     let timeout;
     const capturedAtMs = Date.now();
     const registerCandidate = async (archive) => {
-      if (archive.descriptor.version !== 2 || archive.descriptor.provider !== "modal_snapshot_filesystem" && archive.descriptor.provider !== "modal_snapshot_directory") {
+      if (archive.kind === "host_spool" || archive.descriptor.version !== 2 || archive.descriptor.provider !== "modal_snapshot_filesystem" && archive.descriptor.provider !== "modal_snapshot_directory") {
         return null;
       }
       if (!checkpointBinding) {
@@ -2268,80 +2301,64 @@ async function persistWarmWorkspaceSnapshot(services, ids, session, leaseEpoch,
       }).catch(() => void 0);
     };
     const captureAndPublish = (async () => {
+      let archive;
       let candidate = null;
-      let workspaceArchiveRef;
+      let publicationAttempted = false;
       try {
-        const archive = await captureVerifiedWorkspaceArchive(session, capturedAtMs, {
-          requestId: claimed.claim.providerRequestId,
-          strategy: capturePolicy.strategy
-        });
-        candidate = await registerCandidate(archive);
-        const priorLease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
-        const priorKeys = collectWorkspaceArchiveObjectKeys(
-          priorLease?.resumeState ?? null
+        archive = await captureWorkspaceArchiveForStorage(
+          session,
+          capturedAtMs,
+          {
+            requestId: claimed.claim.providerRequestId,
+            strategy: capturePolicy.strategy
+          },
+          Boolean(services.objectStorage)
         );
+        candidate = await registerCandidate(archive);
+        const archiveDescriptor = archive.descriptor;
         const published = await putVersion1TarArchiveOrInline({
           backend: lease.backend,
           objectStorage: services.objectStorage,
           accountId: ids.accountId,
           workspaceId: ids.workspaceId,
           sandboxGroupId: ids.sandboxGroupId,
-          archive: {
-            bytes: archive.bytes,
-            descriptor: archive.descriptor,
-            base64: archive.base64
-          },
+          archive,
           ...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {}
         });
-        workspaceArchiveRef = published.workspaceArchiveRef;
-        const { wrote } = await persistWarmSnapshot(db, {
-          accountId: ids.accountId,
-          workspaceId: ids.workspaceId,
-          sessionId: ids.sessionId,
-          turnId: ids.turnId,
-          attemptId: ids.attemptId,
-          sandboxGroupId: ids.sandboxGroupId,
-          expectedEpoch: leaseEpoch,
-          expectedInstanceId: instanceId,
-          expectedWorkspaceGeneration: claimed.claim.workspaceGeneration,
-          captureId,
-          workspaceArchiveMeta: archive.descriptor,
-          ...published,
-          checkpointArtifactId: candidate?.id ?? null,
-          minIntervalMs: force ? 0 : intervalMs,
-          capturedAtMs
+        publicationAttempted = true;
+        const { wrote } = await persistWorkspaceArchiveCandidate({
+          ...services.objectStorage ? { objectStorage: services.objectStorage } : {},
+          ...published.workspaceArchiveRef ? { ref: published.workspaceArchiveRef } : {},
+          persist: () => persistWarmSnapshot(db, {
+            accountId: ids.accountId,
+            workspaceId: ids.workspaceId,
+            sessionId: ids.sessionId,
+            turnId: ids.turnId,
+            attemptId: ids.attemptId,
+            sandboxGroupId: ids.sandboxGroupId,
+            expectedEpoch: leaseEpoch,
+            expectedInstanceId: instanceId,
+            expectedWorkspaceGeneration: claimed.claim.workspaceGeneration,
+            captureId,
+            workspaceArchiveMeta: archiveDescriptor,
+            ...published,
+            checkpointArtifactId: candidate?.id ?? null,
+            minIntervalMs: force ? 0 : intervalMs,
+            capturedAtMs
+          })
         });
-        if (published.workspaceArchiveRef && services.objectStorage) {
-          if (!wrote) {
-            await deleteUnpublishedWorkspaceArchiveObject(
-              services.objectStorage,
-              published.workspaceArchiveRef,
-              services.sandboxMetrics
-            );
-          } else {
-            const afterLease = await readLease(db, ids.workspaceId, ids.sandboxGroupId);
-            const afterKeys = collectWorkspaceArchiveObjectKeys(
-              afterLease?.resumeState ?? null
-            );
-            await deleteWorkspaceArchiveObjectKeys(
-              services.objectStorage,
-              [...priorKeys].filter((key) => !afterKeys.has(key))
-            ).catch(() => void 0);
-          }
-        }
         if (!wrote && candidate) {
           await abandonCandidate(candidate.id, "snapshot_publication_fenced");
         }
         return wrote;
       } catch (error) {
-        await deleteUnpublishedWorkspaceArchiveObject(
-          services.objectStorage,
-          workspaceArchiveRef,
-          services.sandboxMetrics
-        );
-        if (candidate) await abandonCandidate(candidate.id, "snapshot_capture_failed");
+        if (candidate && !publicationAttempted)
+          await abandonCandidate(candidate.id, "snapshot_capture_failed");
         throw error;
       } finally {
+        await disposeWorkspaceArchive(archive).catch(() => {
+          console.warn("workspace archive spool cleanup failed");
+        });
         await releaseWorkspaceArchiveCapture(db, {
           accountId: ids.accountId,
           workspaceId: ids.workspaceId,
@@ -2681,6 +2698,24 @@ async function resumeBoxForTurn(services, ids, kind, holderId) {
       const established = await establishSandboxSessionFromEnvelope(settings, hydrateEnvelope, {
         sessionId: ids.sessionId,
         recovery: "create-or-restore",
+        ...services.objectStorage ? {
+          loadHostWorkspaceArchive: async (ref) => {
+            try {
+              return await downloadWorkspaceArchiveSpool(services.objectStorage, ref.key, {
+                bytes: ref.bytes,
+                sha256: ref.sha256
+              });
+            } catch (error) {
+              if (error instanceof WorkspaceArchiveStorageError) {
+                throw new WorkspaceArchiveIntegrityError(error.code, error.message, {
+                  retryable: error.retryable,
+                  cause: error
+                });
+              }
+              throw error;
+            }
+          }
+        } : {},
         backendOverride: ids.backend,
         ...ids.environment ? { environment: ids.environment } : {},
         ...services.sandboxMetrics ? { metrics: services.sandboxMetrics } : {},
@@ -3259,11 +3294,11 @@ function afterPersistableHomeMutation(services, ids, home) {
 function beforeRetainedProcessMutation(services, ids) {
   const fence = ids.workspaceMutationFence;
   if (!fence) return void 0;
-  return async ({ op, process }) => await advanceWorkspaceGenerationForRetainedProcess(services.db, {
+  return async ({ op, process: process2 }) => await advanceWorkspaceGenerationForRetainedProcess(services.db, {
     accountId: fence.accountId,
     workspaceId: ids.workspaceId,
     sessionId: ids.sessionId,
-    processId: process.id,
+    processId: process2.id,
     operation: op,
     captureWaitMs: sandboxLifecycleTransitionWaitMs2(services.settings),
     ...services.waitSignal ? { waitSignal: services.waitSignal } : {}
@@ -3272,7 +3307,7 @@ function beforeRetainedProcessMutation(services, ids) {
 function afterRetainedProcessMutation(services, ids) {
   const fence = ids.workspaceMutationFence;
   if (!fence) return void 0;
-  return async ({ op, process, admission, outcome }) => {
+  return async ({ op, process: process2, admission, outcome }) => {
     if (!admission || typeof admission !== "object" || typeof admission.id !== "string" || typeof admission.workspaceGeneration !== "number") {
       throw new Error("Retained-process mutation settlement lacked its exact admission");
     }
@@ -3280,7 +3315,7 @@ function afterRetainedProcessMutation(services, ids) {
       accountId: fence.accountId,
       workspaceId: ids.workspaceId,
       sessionId: ids.sessionId,
-      processId: process.id,
+      processId: process2.id,
       admission,
       operation: op,
       outcome
@@ -3306,7 +3341,7 @@ function captureRetainedProcessOutputForTurn(services, ids) {
   const accountId = ids.workspaceMutationFence?.accountId;
   if (!accountId) return void 0;
   return async ({
-    process,
+    process: process2,
     chunkId,
     chunk,
     stream,
@@ -3316,7 +3351,7 @@ function captureRetainedProcessOutputForTurn(services, ids) {
       accountId,
       workspaceId: ids.workspaceId,
       sessionId: ids.sessionId,
-      commandId: process.id,
+      commandId: process2.id,
       chunkId,
       chunk,
       stream,
@@ -3329,32 +3364,32 @@ function captureRetainedProcessOutputForTurn(services, ids) {
 function observeRetainedProcessTerminalForTurn(services, ids) {
   const fence = ids.workspaceMutationFence;
   if (!fence) return void 0;
-  return async ({ process }) => {
+  return async ({ process: process2 }) => {
     await observeSessionBackgroundCommandCompletion(services.db, {
       accountId: fence.accountId,
       workspaceId: ids.workspaceId,
       sessionId: ids.sessionId,
-      commandId: process.id
+      commandId: process2.id
     });
   };
 }
 function settleRetainedProcessForTurn(services, ids) {
   const fence = ids.workspaceMutationFence;
   if (!fence) return void 0;
-  return async ({ backend, process, proof }) => {
+  return async ({ backend, process: process2, proof }) => {
     const durable = await getRetainedProcess(services.db, {
       workspaceId: ids.workspaceId,
       sessionId: ids.sessionId,
-      processId: process.id
+      processId: process2.id
     });
-    if (!durable || durable.providerSessionId !== process.providerSessionId || durable.providerBackend !== backend.kind || durable.providerInstanceId !== backend.providerInstanceId || durable.leaseEpoch !== backend.leaseEpoch || durable.routeKind !== (backend.sandboxId === null ? "home" : "active") || durable.routeTargetId !== backend.sandboxId || durable.routeEpoch !== backend.activeEpoch) {
+    if (!durable || durable.providerSessionId !== process2.providerSessionId || durable.providerBackend !== backend.kind || durable.providerInstanceId !== backend.providerInstanceId || durable.leaseEpoch !== backend.leaseEpoch || durable.routeKind !== (backend.sandboxId === null ? "home" : "active") || durable.routeTargetId !== backend.sandboxId || durable.routeEpoch !== backend.activeEpoch) {
       throw new Error("Retained-process settlement lost its exact durable backend identity");
     }
     const settlement = await settleRetainedProcess(services.db, {
       accountId: fence.accountId,
       workspaceId: ids.workspaceId,
       sessionId: ids.sessionId,
-      processId: process.id,
+      processId: process2.id,
       expected: retainedProcessSettlementIdentity(durable),
       outcome: proof.outcome,
       exitCode: proof.exitCode,
@@ -3372,13 +3407,13 @@ function settleRetainedProcessForTurn(services, ids) {
 function adoptRetainedProcessAsBackgroundCommandForTurn(services, ids) {
   const fence = ids.workspaceMutationFence;
   if (!fence) return void 0;
-  return async ({ backend, process }) => {
+  return async ({ backend, process: process2 }) => {
     const durable = await getRetainedProcess(services.db, {
       workspaceId: ids.workspaceId,
       sessionId: ids.sessionId,
-      processId: process.id
+      processId: process2.id
     });
-    if (!durable || durable.providerSessionId !== process.providerSessionId || durable.providerBackend !== backend.kind || durable.providerInstanceId !== backend.providerInstanceId || durable.leaseEpoch !== backend.leaseEpoch || durable.routeKind !== (backend.sandboxId === null ? "home" : "active") || durable.routeTargetId !== backend.sandboxId || durable.routeEpoch !== backend.activeEpoch) {
+    if (!durable || durable.providerSessionId !== process2.providerSessionId || durable.providerBackend !== backend.kind || durable.providerInstanceId !== backend.providerInstanceId || durable.leaseEpoch !== backend.leaseEpoch || durable.routeKind !== (backend.sandboxId === null ? "home" : "active") || durable.routeTargetId !== backend.sandboxId || durable.routeEpoch !== backend.activeEpoch) {
       throw new Error("Background command adoption lost its exact retained-process identity");
     }
     await adoptManagedSessionBackgroundCommand(services.db, {
@@ -3388,7 +3423,7 @@ function adoptRetainedProcessAsBackgroundCommandForTurn(services, ids) {
       turnId: fence.turnId,
       executionGeneration: fence.executionGeneration,
       attemptId: fence.attemptId,
-      processId: process.id,
+      processId: process2.id,
       expected: retainedProcessSettlementIdentity(durable),
       command: "execCommand"
     });
@@ -3565,11 +3600,11 @@ function wrapTurnBoxWithRouting(services, ids, established) {
     ...services.onSandboxOperation ? { onOperation: services.onSandboxOperation } : {},
     providerCommandHandle: admittedCommandHandle,
     ...ids.workspaceMutationFence ? {
-      providerCommandPersistence: (process) => retainedProviderCommandPersistence(db, {
+      providerCommandPersistence: (process2) => retainedProviderCommandPersistence(db, {
         accountId: ids.workspaceMutationFence.accountId,
         workspaceId: ids.workspaceId,
         sessionId: ids.sessionId,
-        processId: process.id
+        processId: process2.id
       })
     } : {},
     ...beforeMutation ? { beforeMutation } : {},
@@ -3762,11 +3797,11 @@ function wrapLazyTurnBoxWithRouting(services, ids, args) {
     ...args.onFirstOperation ? { onFirstOperation: args.onFirstOperation } : {},
     providerCommandHandle: admittedCommandHandle,
     ...ids.workspaceMutationFence ? {
-      providerCommandPersistence: (process) => retainedProviderCommandPersistence(db, {
+      providerCommandPersistence: (process2) => retainedProviderCommandPersistence(db, {
         accountId: ids.workspaceMutationFence.accountId,
         workspaceId: ids.workspaceId,
         sessionId: ids.sessionId,
-        processId: process.id
+        processId: process2.id
       })
     } : {},
     ...beforeMutation ? { beforeMutation } : {},
@@ -4785,7 +4820,7 @@ import {
   classifyProviderSandboxFailure
 } from "@opengeni/runtime";
 import { CancelledFailure } from "@temporalio/activity";
-import { randomUUID as randomUUID3 } from "crypto";
+import { randomUUID as randomUUID4 } from "crypto";
 var TurnOperationCancelledError = class extends Error {
   constructor(reason) {
     super("Turn operation was cancelled with its owning turn", {
@@ -5076,7 +5111,7 @@ function createTurnSandboxProvisioner(establish, options = {}) {
   return {
     get() {
       if (!memo) {
-        const provisionId = options.provisionIdFactory?.() ?? randomUUID3();
+        const provisionId = options.provisionIdFactory?.() ?? randomUUID4();
         memo = (async () => {
           const startedAt = performance.now();
           throwIfTurnOperationCancelled(options.signal);
@@ -6641,10 +6676,8 @@ export {
   currentActivityContext,
   startActivityHeartbeat,
   nextStreamEvent,
-  collectWorkspaceArchiveObjectKeys,
+  persistWorkspaceArchiveCandidate,
   putVersion1TarArchiveOrInline,
-  deleteUnpublishedWorkspaceArchiveObject,
-  deleteWorkspaceArchiveObjectKeys,
   sandboxLeaseHolderIdForAttempt,
   waitForWarmSnapshot,
   maybePersistWarmWorkspaceSnapshot,
@@ -6726,4 +6759,4 @@ export {
   codexUsageLimitFailurePayload,
   CODEX_USAGE_LIMIT_MAX_RESUME_MS
 };
-//# sourceMappingURL=chunk-NNXLW6V7.js.map
+//# sourceMappingURL=chunk-ZNIKZQMN.js.map