@opengeni/api-router 0.9.0 → 0.11.1

package/src/sandbox/channel-a.ts

@@ -3,7 +3,7 @@
 // The structured services (FileSystem / Git / Terminal) are SYNCHRONOUS point
 // queries served client -> API -> box IN-PROCESS. Each call:
 //
-//   1. acquires a viewer-kind lease holder (warming the box when cold — the
+//   1. acquires an exact direct-request lease holder (warming the box when cold — the
 //      same cold->warming CAS attachViewer runs; a Postgres txn the API OWNS),
 //   2. resumes the box BY ID from the group lease's resume_state envelope,
 //   3. builds ONE SandboxChannelAService around the live `session` handle,
@@ -28,8 +28,6 @@ import { githubAppBotIdentity } from "@opengeni/github";
 import type { Session } from "@opengeni/contracts";
 import {
   acquireLease,
-  commitWarmingToWarm,
-  failWarmingToCold,
   getSandboxSessionEnvelope,
   loadWorkspaceEnvironmentForRun,
   markWarmLeaseInstanceLost,
@@ -44,8 +42,6 @@ import { HTTPException } from "hono/http-exception";
 import {
   establishSandboxSessionFromEnvelope,
   isProviderSandboxNotFoundError,
-  SandboxResumeStateUnavailableError,
-  serializeEstablishedSandboxEnvelope,
   SandboxChannelAService,
   ChannelAConflictError,
   ChannelANotFoundError,
@@ -57,8 +53,10 @@ import {
   withRunCredentialsSession,
   type ChannelASession,
   type EstablishedSandboxSession,
+  type RoutingSandboxSession,
 } from "@opengeni/runtime/sandbox";
-import { routingEnabled, wrapChannelABoxWithRouting } from "@opengeni/core";
+import { wrapChannelABoxWithRouting } from "@opengeni/core";
+import { establishApiSandboxSpawner } from "./rematerialize";
 
 export type ChannelAServices = {
   db: Database;
@@ -79,10 +77,12 @@ export type ChannelAContext = {
 export type ChannelAHandle = {
   service: SandboxChannelAService;
   lease: LeaseSnapshot;
+  routingSession: RoutingSandboxSession;
+  requestId: string;
 };
 
 /**
- * Run a Channel-A op against a live box, API-direct. Acquires a viewer holder
+ * Run a Channel-A op against a live box, API-direct. Acquires an exact direct holder
  * (warming the box when cold), resumes by id, builds the service, runs `fn`, and
  * ALWAYS releases the holder + drops the handle in `finally`. Maps the service's
  * typed errors to HTTP status (the route never sees a raw ChannelA*Error).
@@ -103,7 +103,8 @@ export async function withChannelA<T>(
   }
 
   const sandboxGroupId = session.sandboxGroupId;
-  const viewerId = crypto.randomUUID();
+  const requestId = crypto.randomUUID();
+  const holderId = `direct:${requestId}`;
   const leaseTtlMs = settings.sandboxLeaseTtlMs;
 
   const release = async (): Promise<void> => {
@@ -111,19 +112,19 @@ export async function withChannelA<T>(
       accountId,
       workspaceId,
       sandboxGroupId,
-      kind: "viewer",
-      holderId: viewerId,
+      kind: "direct",
+      holderId,
       idleGraceMs: settings.sandboxIdleGraceMs,
     });
   };
 
-  // Acquire a viewer holder; the cold->warming CAS spawns the box when cold.
+  // Acquire exact request authority; the cold->warming CAS spawns the box when cold.
   const acquired = await acquireLease(db, {
     accountId,
     workspaceId,
     sandboxGroupId,
-    kind: "viewer",
-    holderId: viewerId,
+    kind: "direct",
+    holderId,
     subjectId: session.id,
     backend: session.sandboxBackend,
     os: session.sandboxOs,
@@ -131,6 +132,12 @@ export async function withChannelA<T>(
     warmingLeaseTtlMs: settings.sandboxWarmingTimeoutMs,
   });
 
+  if (acquired.role === "blocked") {
+    await release();
+    throw new HTTPException(409, {
+      message: `sandbox recovery ${acquired.lease.recovery.restore.status} at epoch ${acquired.lease.leaseEpoch}`,
+    });
+  }
   if (acquired.role === "fenced") {
     await release();
     throw new HTTPException(409, {
@@ -185,43 +192,28 @@ export async function withChannelA<T>(
       // the bare session envelope (a never-warmed cold start). The order matters:
       // resume_state is the lease's authoritative box descriptor; the session
       // `_sandbox` envelope is only the per-session fallback.
-      const spawnEnvelope = acquired.lease.resumeState ?? envelope;
       try {
-        established = await establishSandboxSessionFromEnvelope(settings, spawnEnvelope, {
+        const result = await establishApiSandboxSpawner({
+          db,
+          settings,
+          accountId,
+          workspaceId,
+          sandboxGroupId,
           sessionId: session.id,
-          recovery: "create-or-restore",
-          backendOverride: session.sandboxBackend,
+          backend: session.sandboxBackend,
           environment,
+          expectedEpoch,
+          acquiredLease: acquired.lease,
+          fallbackEnvelope: envelope,
+          dataPlaneUrl: acquired.lease.dataPlaneUrl,
         });
+        established = result.established;
+        leaseSnapshot = result.lease;
       } catch (error) {
-        await failWarmingToCold(db, { accountId, workspaceId, sandboxGroupId, expectedEpoch });
         throw new HTTPException(409, {
           message: `sandbox not available (${error instanceof Error ? error.message : "spawn failed"})`,
         });
       }
-      // Persist the LIVE box as the lease's resume_state so the NEXT op resumes
-      // this box by id rather than cold-creating a rival (the box-churn the
-      // prove-it surfaced). Fall back to the session envelope when serialize is
-      // unavailable.
-      const resumeEnvelope =
-        (await serializeEstablishedSandboxEnvelope(established)) ?? envelope ?? null;
-      const committed = await commitWarmingToWarm(db, {
-        accountId,
-        workspaceId,
-        sandboxGroupId,
-        expectedEpoch,
-        instanceId: established.instanceId,
-        dataPlaneUrl: acquired.lease.dataPlaneUrl,
-        resumeBackendId: established.backendId,
-        resumeState: resumeEnvelope,
-        leaseTtlMs,
-      });
-      if (!committed.committed || !committed.lease) {
-        throw new HTTPException(409, {
-          message: `sandbox lease superseded (epoch ${expectedEpoch}); retry`,
-        });
-      }
-      leaseSnapshot = committed.lease;
     } else {
       // ATTACHED / REARMED: the box is live. Read the lease to get the
       // authoritative resume_state, then resume by id for this op.
@@ -245,10 +237,7 @@ export async function withChannelA<T>(
           environment,
         });
       } catch (error) {
-        if (
-          !(error instanceof SandboxResumeStateUnavailableError) &&
-          !isProviderSandboxNotFoundError(session.sandboxBackend, error)
-        ) {
+        if (!isProviderSandboxNotFoundError(session.sandboxBackend, error)) {
           throw error;
         }
         const marked = await markWarmLeaseInstanceLost(db, {
@@ -284,19 +273,25 @@ export async function withChannelA<T>(
       );
     };
 
-    // M7 hot-swap: when the selfhosted feature is on, route the Channel-A op to
-    // the session's currently-active sandbox (not always the group box). The
-    // proxy re-reads (active_sandbox_id, active_epoch) on each session method the
-    // service calls and dispatches to the active backend (the group box by
-    // default, or a swapped-to selfhosted machine). With the flag off the
-    // established group session is used unchanged.
-    const routedSession = routingEnabled(settings)
-      ? wrapChannelABoxWithRouting(
-          { db, settings, bus },
-          { workspaceId, sessionId: session.id },
-          established,
-        ).session
-      : established.session;
+    // Route every call through the same proxy, even when hot-swap is disabled:
+    // routing may be dormant, but its direct mutation admission is mandatory for
+    // every persistable provider write.
+    const routedSession = wrapChannelABoxWithRouting(
+      { db, settings, bus },
+      {
+        accountId,
+        workspaceId,
+        sessionId: session.id,
+        homeLease: {
+          sandboxGroupId,
+          leaseEpoch: leaseSnapshot.leaseEpoch,
+          instanceId: leaseSnapshot.instanceId!,
+          backend: session.sandboxBackend,
+        },
+        directRequest: { requestId, holderId },
+      },
+      established,
+    ).session as RoutingSandboxSession;
     const credentialSession = withRunCredentialsSession(routedSession as object, session.id);
     const scopedSession = environment.OPENGENI_TOOLSPACE_TOKEN_FILE
       ? withToolspaceTokenSession(
@@ -311,7 +306,7 @@ export async function withChannelA<T>(
       emit,
     });
 
-    return await fn({ service, lease: leaseSnapshot });
+    return await fn({ service, lease: leaseSnapshot, routingSession: routedSession, requestId });
   } catch (error) {
     throw mapChannelAError(error);
   } finally {

package/src/sandbox/machines.ts

@@ -188,6 +188,7 @@ export async function listMachines(
   // null active pointer routes to. Only present in an in-session view.
   if (session) {
     const groupActive = activeSandboxId === null;
+    const groupLease = await readLease(db, workspaceId, session.sandboxGroupId);
     machines.push(
       MachineView.parse({
         sandboxId: session.sandboxGroupId,
@@ -197,6 +198,9 @@ export async function listMachines(
         state: "online",
         active: groupActive,
         isSessionGroup: true,
+        workspaceGeneration: groupLease?.workspaceGeneration ?? null,
+        archiveGeneration: groupLease?.archiveGeneration ?? null,
+        archiveComplete: groupLease?.archiveComplete ?? false,
         // The Modal group box is a cloud Linux box; its precise OS/arch is not
         // surfaced as a metric, so the dashboard shows the canonical linux/x86_64.
         os: "linux",
@@ -249,6 +253,9 @@ export async function listMachines(
         state,
         active: activeSandboxId === sandbox.id,
         isSessionGroup: false,
+        workspaceGeneration: null,
+        archiveGeneration: null,
+        archiveComplete: false,
         os: enrollment.os,
         arch: enrollment.arch,
         hasDisplay: enrollment.hasDisplay,

package/src/sandbox/rematerialize.ts

@@ -0,0 +1,287 @@
+import type { Settings } from "@opengeni/config";
+import {
+  beginSandboxRematerialization,
+  commitWarmingToWarm,
+  failSandboxRematerialization,
+  failWarmingToCold,
+  markSandboxRestoreVerifying,
+  recordWarmingSandboxCreated,
+  SandboxLeaseRecoveryBlockedError,
+  SandboxLeaseSupersededError,
+  type Database,
+  type LeaseSnapshot,
+} from "@opengeni/db";
+import {
+  establishSandboxSessionFromEnvelope,
+  isProviderSandboxNotFoundError,
+  requirePersistableReplacementSandboxEnvelope,
+  serializeReplacementSandboxEnvelope,
+  tagModalSandbox,
+  verifySandboxExecReadiness,
+  WorkspaceArchiveIntegrityError,
+  type EstablishedSandboxSession,
+  type WorkspaceArchiveDescriptor,
+} from "@opengeni/runtime/sandbox";
+
+function hasWorkspaceArchive(envelope: Record<string, unknown> | null): boolean {
+  const sessionState =
+    envelope?.sessionState && typeof envelope.sessionState === "object"
+      ? (envelope.sessionState as Record<string, unknown>)
+      : null;
+  return (
+    typeof sessionState?.workspaceArchive === "string" && sessionState.workspaceArchive.length > 0
+  );
+}
+
+function withoutProviderIdentity(
+  envelope: Record<string, unknown> | null,
+): Record<string, unknown> | null {
+  if (!envelope) return null;
+  const sessionState =
+    envelope.sessionState && typeof envelope.sessionState === "object"
+      ? (envelope.sessionState as Record<string, unknown>)
+      : null;
+  if (!sessionState) return envelope;
+  const { providerState: _providerState, ...providerIndependentState } = sessionState;
+  return { ...envelope, sessionState: providerIndependentState };
+}
+
+async function terminateCreated(established: EstablishedSandboxSession | null): Promise<boolean> {
+  if (!established) return true;
+  const client = established.client as { delete?: (state: unknown) => Promise<unknown> };
+  try {
+    if (typeof client.delete === "function" && established.sessionState !== undefined) {
+      await client.delete(established.sessionState);
+      return true;
+    }
+    const session = established.session as {
+      terminate?: () => Promise<unknown>;
+      kill?: () => Promise<unknown>;
+      close?: () => Promise<unknown>;
+      closed?: boolean;
+    };
+    if (session.terminate) await session.terminate();
+    else if (session.kill) await session.kill();
+    else if (session.close && !session.closed) await session.close();
+    else return false;
+    return true;
+  } catch (error) {
+    return isProviderSandboxNotFoundError(established.backendId, error);
+  }
+}
+
+/** The sole API-direct cold->warming owner path used by Channel A and viewer
+ * attach. It never publishes warm until archive identity, hydrated tree, command
+ * routing, provider identity, and the selected rematerialization attempt all
+ * agree under one lease epoch. */
+export async function establishApiSandboxSpawner(input: {
+  db: Database;
+  settings: Settings;
+  accountId: string;
+  workspaceId: string;
+  sandboxGroupId: string;
+  sessionId: string;
+  backend: string;
+  environment: Record<string, string>;
+  expectedEpoch: number;
+  acquiredLease: LeaseSnapshot;
+  fallbackEnvelope: Record<string, unknown> | null;
+  dataPlaneUrl: string | null;
+}): Promise<{ established: EstablishedSandboxSession; lease: LeaseSnapshot }> {
+  const fallbackArchiveEnvelope =
+    input.acquiredLease.recovery.archive.status === "none" &&
+    hasWorkspaceArchive(input.fallbackEnvelope)
+      ? withoutProviderIdentity(input.fallbackEnvelope)
+      : null;
+  const spawnEnvelope =
+    fallbackArchiveEnvelope ?? input.acquiredLease.resumeState ?? input.fallbackEnvelope;
+  const archiveSource =
+    input.acquiredLease.recovery.archive.status === "none"
+      ? fallbackArchiveEnvelope
+      : input.acquiredLease.resumeState;
+  let established: EstablishedSandboxSession | null = null;
+  let rematerialization: { id: string; selectedRevision: string } | null = null;
+  try {
+    if (
+      input.acquiredLease.recovery.archive.status === "available" ||
+      hasWorkspaceArchive(archiveSource)
+    ) {
+      const id = crypto.randomUUID();
+      const begun = await beginSandboxRematerialization(input.db, {
+        accountId: input.accountId,
+        workspaceId: input.workspaceId,
+        sandboxGroupId: input.sandboxGroupId,
+        expectedEpoch: input.expectedEpoch,
+        rematerializationId: id,
+        archiveSource,
+      });
+      if (begun.status !== "started") {
+        if (begun.code === "stale_epoch" || begun.code === "attempt_conflict") {
+          throw new SandboxLeaseSupersededError(
+            input.sandboxGroupId,
+            begun.lease?.leaseEpoch ?? input.expectedEpoch,
+          );
+        }
+        throw new SandboxLeaseRecoveryBlockedError(
+          input.sandboxGroupId,
+          begun.lease?.leaseEpoch ?? input.expectedEpoch,
+          begun.code === "archive_unverified" ? "restore_degraded" : "restore_unrecoverable",
+          begun.lease?.recovery ?? input.acquiredLease.recovery,
+        );
+      }
+      const selectedRevision = begun.lease.recovery.restore.selectedRevision;
+      if (!selectedRevision) {
+        throw new WorkspaceArchiveIntegrityError(
+          "archive_metadata_invalid",
+          "sandbox rematerialization selected no durable archive revision",
+        );
+      }
+      rematerialization = { id, selectedRevision };
+    } else if (input.acquiredLease.recovery.archive.status !== "none") {
+      throw new SandboxLeaseRecoveryBlockedError(
+        input.sandboxGroupId,
+        input.expectedEpoch,
+        "restore_degraded",
+        input.acquiredLease.recovery,
+      );
+    }
+
+    established = await establishSandboxSessionFromEnvelope(input.settings, spawnEnvelope, {
+      sessionId: input.sessionId,
+      recovery: "create-or-restore",
+      backendOverride: input.backend as never,
+      environment: input.environment,
+      onSandboxCreated: async (created) => {
+        established = created;
+        const resumeState = requirePersistableReplacementSandboxEnvelope(
+          await serializeReplacementSandboxEnvelope(created, spawnEnvelope),
+          created.backendId,
+        );
+        const recorded = await recordWarmingSandboxCreated(input.db, {
+          accountId: input.accountId,
+          workspaceId: input.workspaceId,
+          sandboxGroupId: input.sandboxGroupId,
+          expectedEpoch: input.expectedEpoch,
+          rematerializationId: rematerialization?.id ?? null,
+          instanceId: created.instanceId,
+          resumeBackendId: created.backendId,
+          resumeState,
+          leaseTtlMs: input.settings.sandboxLeaseTtlMs,
+          warmingLeaseTtlMs: input.settings.sandboxWarmingTimeoutMs,
+        });
+        if (!recorded.recorded) {
+          throw new SandboxLeaseSupersededError(input.sandboxGroupId, input.expectedEpoch);
+        }
+        if (created.backendId === "modal") {
+          await tagModalSandbox(input.settings, created.instanceId, {
+            leaseId: input.acquiredLease.id,
+            workspaceId: input.workspaceId,
+            sandboxGroupId: input.sandboxGroupId,
+          }).catch(() => undefined);
+        }
+      },
+      onWorkspaceRestoreVerifying: async (descriptor: WorkspaceArchiveDescriptor) => {
+        if (!rematerialization || descriptor.revision !== rematerialization.selectedRevision) {
+          throw new WorkspaceArchiveIntegrityError(
+            "archive_metadata_invalid",
+            `hydrated archive revision ${descriptor.revision} does not match the selected rematerialization revision`,
+          );
+        }
+        const verifying = await markSandboxRestoreVerifying(input.db, {
+          accountId: input.accountId,
+          workspaceId: input.workspaceId,
+          sandboxGroupId: input.sandboxGroupId,
+          expectedEpoch: input.expectedEpoch,
+          rematerializationId: rematerialization.id,
+        });
+        if (!verifying.wrote) {
+          throw new SandboxLeaseSupersededError(input.sandboxGroupId, input.expectedEpoch);
+        }
+      },
+    });
+
+    await verifySandboxExecReadiness(established);
+    if (
+      rematerialization &&
+      established.restoredArchive?.revision !== rematerialization.selectedRevision
+    ) {
+      throw new WorkspaceArchiveIntegrityError(
+        "workspace_fingerprint_mismatch",
+        "sandbox restore completed without the exact selected durable archive revision",
+      );
+    }
+    const resumeState = requirePersistableReplacementSandboxEnvelope(
+      await serializeReplacementSandboxEnvelope(established, spawnEnvelope),
+      established.backendId,
+    );
+    const committed = await commitWarmingToWarm(input.db, {
+      accountId: input.accountId,
+      workspaceId: input.workspaceId,
+      sandboxGroupId: input.sandboxGroupId,
+      expectedEpoch: input.expectedEpoch,
+      instanceId: established.instanceId,
+      dataPlaneUrl: input.dataPlaneUrl,
+      resumeBackendId: established.backendId,
+      resumeState,
+      ...(rematerialization
+        ? {
+            rematerialization: {
+              id: rematerialization.id,
+              verifiedRevision: rematerialization.selectedRevision,
+            },
+          }
+        : {}),
+      leaseTtlMs: input.settings.sandboxLeaseTtlMs,
+    });
+    if (!committed.committed || !committed.lease) {
+      const terminated = await terminateCreated(established);
+      if (terminated && rematerialization) {
+        await failSandboxRematerialization(input.db, {
+          accountId: input.accountId,
+          workspaceId: input.workspaceId,
+          sandboxGroupId: input.sandboxGroupId,
+          expectedEpoch: input.expectedEpoch,
+          rematerializationId: rematerialization.id,
+          failureCode: committed.reason ?? "warm_commit_rejected",
+          retryable: false,
+        });
+      } else if (terminated) {
+        await failWarmingToCold(input.db, {
+          accountId: input.accountId,
+          workspaceId: input.workspaceId,
+          sandboxGroupId: input.sandboxGroupId,
+          expectedEpoch: input.expectedEpoch,
+        });
+      }
+      throw new SandboxLeaseSupersededError(input.sandboxGroupId, input.expectedEpoch);
+    }
+    return { established, lease: committed.lease };
+  } catch (error) {
+    if (error instanceof SandboxLeaseSupersededError) throw error;
+    const terminated = await terminateCreated(established);
+    if (terminated) {
+      if (rematerialization) {
+        await failSandboxRematerialization(input.db, {
+          accountId: input.accountId,
+          workspaceId: input.workspaceId,
+          sandboxGroupId: input.sandboxGroupId,
+          expectedEpoch: input.expectedEpoch,
+          rematerializationId: rematerialization.id,
+          failureCode:
+            error instanceof WorkspaceArchiveIntegrityError
+              ? error.code
+              : "sandbox_rematerialization_failed",
+          retryable: error instanceof WorkspaceArchiveIntegrityError ? error.retryable : true,
+        });
+      } else {
+        await failWarmingToCold(input.db, {
+          accountId: input.accountId,
+          workspaceId: input.workspaceId,
+          sandboxGroupId: input.sandboxGroupId,
+          expectedEpoch: input.expectedEpoch,
+        });
+      }
+    }
+    throw error;
+  }
+}