@opengeni/core 0.10.0 → 0.11.2

package/src/sandbox/fleet.ts

@@ -17,6 +17,7 @@ import {
   getSandbox,
   listSandboxes,
   readActiveSandbox,
+  readLease,
   requireSession,
   setActiveSandbox,
   type Database,
@@ -40,6 +41,15 @@ export type FleetServices = {
   db: Database;
   settings: Settings;
   bus?: EventBus;
+  /** API-direct readiness owner for the session's home group. Production wires
+   * this to the same viewer/provider verification + rematerialization path; core
+   * tests may omit it when exercising pointer mechanics only. */
+  ensureSessionGroupReady?: (ctx: FleetContext) => Promise<FleetReadinessHold>;
+};
+
+export type FleetReadinessHold = {
+  /** Release target liveness only after route publication settles. */
+  release: () => Promise<void>;
 };
 
 export type FleetContext = {
@@ -111,6 +121,28 @@ export type FleetSandboxEntry = {
   /** Selfhosted only: whether a display (real/Xvfb) is present. */
   hasDisplay?: boolean;
   lastSeenAt?: string | null;
+  /** Orthogonal truth dimensions. `liveness` is only their conservative UI
+   * projection and is never evidence for a specific dimension. */
+  providerStatus: "not_created" | "creating" | "exists" | "missing" | "unknown";
+  leaseLiveness: "cold" | "warming" | "warm" | "draining" | null;
+  routeStatus: "attached" | "detached";
+  archiveStatus: "none" | "available" | "unverified" | "invalid";
+  restoreStatus:
+    | "not_required"
+    | "pending"
+    | "restoring"
+    | "verifying"
+    | "ready"
+    | "degraded"
+    | "unrecoverable";
+  workspaceStatus: "unknown" | "not_ready" | "ready" | "degraded" | "unrecoverable";
+  leaseEpoch: number | null;
+  routeEpoch: number;
+  /** Numeric/boolean persistence truth only. Archive locations, content hashes,
+   * provider identities, and storage handles are intentionally not projected. */
+  workspaceGeneration: number | null;
+  archiveGeneration: number | null;
+  archiveComplete: boolean;
 };
 
 export type FleetListResult = {
@@ -127,7 +159,12 @@ export type FleetSwapResult = {
   activeSandboxId: string | null;
   activeEpoch: number;
   reason?: string;
-  code?: BackendUnresolvableCode | "concurrent_swap";
+  code?:
+    | BackendUnresolvableCode
+    | "concurrent_swap"
+    | "recovery_in_progress"
+    | "recovery_degraded"
+    | "recovery_unrecoverable";
 };
 
 const PROBE_TIMEOUT_MS = 5_000;
@@ -200,17 +237,42 @@ export async function listFleet(
   const entries: FleetSandboxEntry[] = [];
 
   // The session's own group box (the default/home sandbox; null active pointer ==
-  // this box). It is live by virtue of being the session's resumable group.
+  // this box). A session/group row is not provider existence. Online requires a
+  // warm lease, observed provider existence, and verified workspace readiness.
   const groupActive = pointer.activeSandboxId === null;
+  const groupLease = await readLease(db, ctx.workspaceId, ctx.sessionGroupId);
+  const groupOnline = Boolean(
+    groupLease?.liveness === "warm" &&
+    groupLease.recovery.provider.status === "exists" &&
+    groupLease.recovery.workspace.status === "ready",
+  );
+  const groupRecovering = Boolean(
+    groupLease &&
+    (groupLease.liveness === "warming" ||
+      groupLease.recovery.restore.status === "pending" ||
+      groupLease.recovery.restore.status === "restoring" ||
+      groupLease.recovery.restore.status === "verifying"),
+  );
   entries.push({
     id: ctx.sessionGroupId,
     kind: ctx.sessionBackend === "selfhosted" ? "selfhosted" : "modal",
     name: "session sandbox",
-    liveness: "online",
+    liveness: groupOnline ? "online" : groupRecovering ? "reconnecting" : "offline",
     active: groupActive,
     isSessionGroup: true,
     enrollmentId: null,
-    attachable: true,
+    attachable: groupOnline,
+    providerStatus: groupLease?.recovery.provider.status ?? "not_created",
+    leaseLiveness: groupLease?.liveness ?? null,
+    routeStatus: groupActive ? "attached" : "detached",
+    archiveStatus: groupLease?.recovery.archive.status ?? "none",
+    restoreStatus: groupLease?.recovery.restore.status ?? "not_required",
+    workspaceStatus: groupLease?.recovery.workspace.status ?? "unknown",
+    leaseEpoch: groupLease?.leaseEpoch ?? null,
+    routeEpoch: pointer.activeEpoch,
+    workspaceGeneration: groupLease?.workspaceGeneration ?? null,
+    archiveGeneration: groupLease?.archiveGeneration ?? null,
+    archiveComplete: groupLease?.archiveComplete ?? false,
   });
 
   // The workspace's first-class selfhosted sandboxes (enrolled machines). Probe
@@ -236,6 +298,22 @@ export async function listFleet(
       consented: probe.consented,
       hasDisplay: probe.hasDisplay,
       lastSeenAt: enrollment?.lastSeenAt ?? null,
+      providerStatus:
+        probe.liveness === "online"
+          ? "exists"
+          : probe.liveness === "reconnecting"
+            ? "unknown"
+            : "missing",
+      leaseLiveness: null,
+      routeStatus: pointer.activeSandboxId === sandbox.id ? "attached" : "detached",
+      archiveStatus: "none",
+      restoreStatus: "not_required",
+      workspaceStatus: probe.liveness === "online" ? "ready" : "not_ready",
+      leaseEpoch: null,
+      routeEpoch: pointer.activeEpoch,
+      workspaceGeneration: null,
+      archiveGeneration: null,
+      archiveComplete: false,
     });
   }
 
@@ -345,49 +423,81 @@ export async function swapActiveSandbox(
     };
   }
 
-  // Read the current epoch, then CAS on it (the fence). One retry on a lost race
-  // (a concurrent swap bumped the epoch between read and write).
-  for (let attempt = 0; attempt < 2; attempt += 1) {
-    const pointer = (await readActiveSandbox(services.db, ctx.workspaceId, ctx.sessionId)) ?? {
-      activeSandboxId: null,
-      activeEpoch: 0,
-    };
-    // No-op swap (already pointed there) is a success without an epoch bump churn.
-    if (pointer.activeSandboxId === resolved.targetSandboxId) {
+  let readinessHold: FleetReadinessHold | undefined;
+  if (resolved.targetSandboxId === null && services.ensureSessionGroupReady) {
+    try {
+      readinessHold = await services.ensureSessionGroupReady(ctx);
+    } catch (error) {
+      const lease = await readLease(services.db, ctx.workspaceId, ctx.sessionGroupId);
+      const restore = lease?.recovery.restore.status;
+      const code =
+        restore === "degraded"
+          ? ("recovery_degraded" as const)
+          : restore === "unrecoverable"
+            ? ("recovery_unrecoverable" as const)
+            : ("recovery_in_progress" as const);
+      const pointer = (await readActiveSandbox(services.db, ctx.workspaceId, ctx.sessionId)) ?? {
+        activeSandboxId: null,
+        activeEpoch: 0,
+      };
       return {
-        swapped: true,
+        swapped: false,
         activeSandboxId: pointer.activeSandboxId,
         activeEpoch: pointer.activeEpoch,
+        reason:
+          error instanceof Error
+            ? error.message
+            : "session sandbox did not reach verified readiness",
+        code,
       };
     }
-    const result = await setActiveSandbox(services.db, {
-      accountId: ctx.accountId,
-      workspaceId: ctx.workspaceId,
-      sessionId: ctx.sessionId,
-      targetSandboxId: resolved.targetSandboxId,
-      expectedEpoch: pointer.activeEpoch,
-      ...(workingDir !== undefined ? { workingDir } : {}),
-    });
-    if (result.swapped && result.pointer) {
-      return {
-        swapped: true,
-        activeSandboxId: result.pointer.activeSandboxId,
-        activeEpoch: result.pointer.activeEpoch,
+  }
+
+  try {
+    // Read the current epoch, then CAS on it (the fence). One retry on a lost race
+    // (a concurrent swap bumped the epoch between read and write).
+    for (let attempt = 0; attempt < 2; attempt += 1) {
+      const pointer = (await readActiveSandbox(services.db, ctx.workspaceId, ctx.sessionId)) ?? {
+        activeSandboxId: null,
+        activeEpoch: 0,
       };
+      // Even a same-target attach advances active_epoch. It is a repair/fence
+      // request, not a no-op acknowledgment: any cached stale route is invalidated
+      // only after target readiness has been proved above.
+      const result = await setActiveSandbox(services.db, {
+        accountId: ctx.accountId,
+        workspaceId: ctx.workspaceId,
+        sessionId: ctx.sessionId,
+        targetSandboxId: resolved.targetSandboxId,
+        expectedEpoch: pointer.activeEpoch,
+        ...(workingDir !== undefined ? { workingDir } : {}),
+      });
+      if (result.swapped && result.pointer) {
+        return {
+          swapped: true,
+          activeSandboxId: result.pointer.activeSandboxId,
+          activeEpoch: result.pointer.activeEpoch,
+        };
+      }
+      // CAS lost (a concurrent swap won) — re-read + retry once.
     }
-    // CAS lost (a concurrent swap won) — re-read + retry once.
+    const pointer = (await readActiveSandbox(services.db, ctx.workspaceId, ctx.sessionId)) ?? {
+      activeSandboxId: null,
+      activeEpoch: 0,
+    };
+    return {
+      swapped: false,
+      activeSandboxId: pointer.activeSandboxId,
+      activeEpoch: pointer.activeEpoch,
+      reason: "a concurrent swap won the epoch fence; re-read and retry",
+      code: "concurrent_swap",
+    };
+  } finally {
+    // Do not let a cleanup failure turn an already-committed route CAS into a
+    // false failure. Viewer holders are TTL-bounded and will be reaped if this
+    // best-effort explicit release cannot complete.
+    await readinessHold?.release().catch(() => undefined);
   }
-  const pointer = (await readActiveSandbox(services.db, ctx.workspaceId, ctx.sessionId)) ?? {
-    activeSandboxId: null,
-    activeEpoch: 0,
-  };
-  return {
-    swapped: false,
-    activeSandboxId: pointer.activeSandboxId,
-    activeEpoch: pointer.activeEpoch,
-    reason: "a concurrent swap won the epoch fence; re-read and retry",
-    code: "concurrent_swap",
-  };
 }
 
 export type RunOnOp =

package/src/sandbox/routing.ts

@@ -13,17 +13,33 @@
 // over the events bus) lives here, not in the leaf (which stays db-free).
 
 import type { Settings } from "@opengeni/config";
-import { getSandbox, readActiveSandbox, type Database } from "@opengeni/db";
-import type { EventBus } from "@opengeni/events";
 import {
+  advanceWorkspaceGenerationForDirectRequest,
+  advanceWorkspaceGenerationForRetainedProcess,
+  getSandbox,
+  markWarmLeaseInstanceLost,
+  readActiveSandbox,
+  retainWorkspaceMutationProcess,
+  settleRetainedProcess,
+  verifyDirectWorkspaceMutationSettlement,
+  verifyRetainedProcessMutationSettlement,
+  type Database,
+  type SandboxWorkspaceMutationAdmission,
+} from "@opengeni/db";
+import { appendAndPublishEvents, type EventBus } from "@opengeni/events";
+import {
+  isProviderSandboxGoneDuringRoutedOperation,
   makeActiveBackendResolver,
   NatsControlRpc,
   RoutingSandboxSession,
   type ControlRpc,
   type EstablishedSandboxSession,
   type NatsRequestConnection,
+  type ResolvedActiveBackend,
   type RoutableBackendSession,
   type RoutableSandbox,
+  type RoutingRetainedProcess,
+  type RoutingRetainedProcessTerminalProof,
   type SelfhostedRelayConfig,
 } from "@opengeni/runtime/sandbox";
 
@@ -97,10 +113,198 @@ export function routingEnabled(settings: Settings): boolean {
  */
 export function wrapChannelABoxWithRouting(
   services: ChannelARoutingServices,
-  ids: { workspaceId: string; sessionId: string },
+  ids: {
+    accountId: string;
+    workspaceId: string;
+    sessionId: string;
+    homeLease: {
+      sandboxGroupId: string;
+      leaseEpoch: number;
+      instanceId: string;
+      backend: string;
+    };
+    directRequest: {
+      requestId: string;
+      holderId: string;
+    };
+  },
   established: EstablishedSandboxSession,
 ): EstablishedSandboxSession {
   const { db, settings, bus } = services;
+  const beforeMutation = async ({
+    op,
+    backend,
+  }: {
+    op: string;
+    backend: ResolvedActiveBackend;
+  }): Promise<SandboxWorkspaceMutationAdmission | null> => {
+    // Connected Machines and other non-persistable targets intentionally do
+    // not dirty or advance the cloud-home archive generation.
+    if (
+      backend.sandboxId !== null ||
+      backend.leaseEpoch === undefined ||
+      backend.providerInstanceId === undefined
+    ) {
+      return null;
+    }
+    if (backend.activeEpoch === undefined) {
+      throw new Error("API-direct workspace mutation resolved without an active route epoch");
+    }
+    return await advanceWorkspaceGenerationForDirectRequest(db, {
+      accountId: ids.accountId,
+      workspaceId: ids.workspaceId,
+      sessionId: ids.sessionId,
+      requestId: ids.directRequest.requestId,
+      holderId: ids.directRequest.holderId,
+      sandboxGroupId: ids.homeLease.sandboxGroupId,
+      expectedEpoch: backend.leaseEpoch,
+      expectedInstanceId: backend.providerInstanceId,
+      routeTargetId: backend.sandboxId,
+      routeEpoch: backend.activeEpoch,
+      operation: op,
+    });
+  };
+  const afterMutation = async ({
+    op,
+    backend,
+    admission,
+    outcome,
+    retainedProcess,
+  }: {
+    op: string;
+    backend: ResolvedActiveBackend;
+    admission: unknown;
+    outcome: "resolved" | "rejected";
+    result?: unknown;
+    retainedProcess?: RoutingRetainedProcess;
+  }): Promise<void> => {
+    if (admission === null) return;
+    if (
+      !admission ||
+      typeof admission !== "object" ||
+      typeof (admission as Partial<SandboxWorkspaceMutationAdmission>).id !== "string" ||
+      typeof (admission as Partial<SandboxWorkspaceMutationAdmission>).workspaceGeneration !==
+        "number" ||
+      backend.leaseEpoch === undefined ||
+      backend.providerInstanceId === undefined ||
+      backend.activeEpoch === undefined
+    ) {
+      throw new Error("API-direct workspace mutation settlement lacked its exact admission");
+    }
+    const exactAdmission = admission as SandboxWorkspaceMutationAdmission;
+    if (outcome === "resolved" && retainedProcess) {
+      await retainWorkspaceMutationProcess(db, {
+        accountId: ids.accountId,
+        workspaceId: ids.workspaceId,
+        sessionId: ids.sessionId,
+        processId: retainedProcess.id,
+        providerSessionId: retainedProcess.providerSessionId,
+        admissionId: exactAdmission.id,
+        admittedWorkspaceGeneration: exactAdmission.workspaceGeneration,
+        operation: op,
+        owner: {
+          kind: "direct",
+          requestId: ids.directRequest.requestId,
+          holderId: ids.directRequest.holderId,
+          sandboxGroupId: ids.homeLease.sandboxGroupId,
+          expectedEpoch: backend.leaseEpoch,
+          expectedInstanceId: backend.providerInstanceId,
+          routeTargetId: exactAdmission.routeTargetId,
+          routeEpoch: exactAdmission.routeEpoch,
+        },
+      });
+      return;
+    }
+    await verifyDirectWorkspaceMutationSettlement(db, {
+      accountId: ids.accountId,
+      workspaceId: ids.workspaceId,
+      sessionId: ids.sessionId,
+      requestId: ids.directRequest.requestId,
+      holderId: ids.directRequest.holderId,
+      sandboxGroupId: ids.homeLease.sandboxGroupId,
+      expectedEpoch: backend.leaseEpoch,
+      expectedInstanceId: backend.providerInstanceId,
+      routeTargetId: exactAdmission.routeTargetId,
+      routeEpoch: exactAdmission.routeEpoch,
+      admission: exactAdmission,
+      operation: op,
+      outcome,
+    });
+  };
+  const beforeProcessMutation = async ({
+    op,
+    process,
+  }: {
+    op: string;
+    backend: ResolvedActiveBackend;
+    process: RoutingRetainedProcess;
+  }): Promise<SandboxWorkspaceMutationAdmission> =>
+    await advanceWorkspaceGenerationForRetainedProcess(db, {
+      accountId: ids.accountId,
+      workspaceId: ids.workspaceId,
+      sessionId: ids.sessionId,
+      processId: process.id,
+      operation: op,
+    });
+  const afterProcessMutation = async ({
+    op,
+    process,
+    admission,
+    outcome,
+  }: {
+    op: string;
+    backend: ResolvedActiveBackend;
+    process: RoutingRetainedProcess;
+    admission: unknown;
+    outcome: "resolved" | "rejected";
+    result?: unknown;
+  }): Promise<void> => {
+    if (
+      !admission ||
+      typeof admission !== "object" ||
+      typeof (admission as Partial<SandboxWorkspaceMutationAdmission>).id !== "string" ||
+      typeof (admission as Partial<SandboxWorkspaceMutationAdmission>).workspaceGeneration !==
+        "number"
+    ) {
+      throw new Error("API retained-process mutation settlement lacked its exact admission");
+    }
+    await verifyRetainedProcessMutationSettlement(db, {
+      accountId: ids.accountId,
+      workspaceId: ids.workspaceId,
+      sessionId: ids.sessionId,
+      processId: process.id,
+      admission: admission as SandboxWorkspaceMutationAdmission,
+      operation: op,
+      outcome,
+    });
+  };
+  const settleProcess = async ({
+    backend,
+    process,
+    proof,
+  }: {
+    backend: ResolvedActiveBackend;
+    process: RoutingRetainedProcess;
+    proof: RoutingRetainedProcessTerminalProof;
+  }): Promise<void> => {
+    if (
+      backend.sandboxId !== null ||
+      backend.leaseEpoch === undefined ||
+      backend.providerInstanceId === undefined
+    ) {
+      return;
+    }
+    await settleRetainedProcess(db, {
+      accountId: ids.accountId,
+      workspaceId: ids.workspaceId,
+      sessionId: ids.sessionId,
+      processId: process.id,
+      outcome: proof.outcome,
+      exitCode: proof.exitCode,
+      reason: proof.reason,
+      idleGraceMs: settings.sandboxIdleGraceMs,
+    });
+  };
   const resolver = makeActiveBackendResolver({
     workspaceId: ids.workspaceId,
     defaultBackend: established.session as RoutableBackendSession,
@@ -118,14 +322,68 @@ export function wrapChannelABoxWithRouting(
     },
     controlRpcFactory: controlRpcFactory(bus),
     relay: relayConfigFromSettings(settings),
+    resolveDefaultBackend: async () => ({
+      session: established.session as RoutableBackendSession,
+      sandboxId: null,
+      kind: established.backendId,
+      leaseEpoch: ids.homeLease.leaseEpoch,
+      providerInstanceId: ids.homeLease.instanceId,
+    }),
   });
 
   const proxy = new RoutingSandboxSession({
+    defaultResolved: {
+      session: established.session as RoutableBackendSession,
+      sandboxId: null,
+      kind: established.backendId,
+      leaseEpoch: ids.homeLease.leaseEpoch,
+      providerInstanceId: ids.homeLease.instanceId,
+    },
     readPointer: async () => {
+      if (!routingEnabled(settings)) {
+        return { activeSandboxId: null, activeEpoch: 0 };
+      }
       const pointer = await readActiveSandbox(db, ids.workspaceId, ids.sessionId);
       return pointer ?? { activeSandboxId: null, activeEpoch: 0 };
     },
     resolveActiveBackend: resolver,
+    beforeMutation,
+    afterMutation,
+    beforeProcessMutation,
+    afterProcessMutation,
+    settleProcess,
+    onDefaultBackendError: async ({ error }) => {
+      if (!isProviderSandboxGoneDuringRoutedOperation(ids.homeLease.backend, error)) return null;
+      const marked = await markWarmLeaseInstanceLost(db, {
+        accountId: ids.accountId,
+        workspaceId: ids.workspaceId,
+        sandboxGroupId: ids.homeLease.sandboxGroupId,
+        expectedEpoch: ids.homeLease.leaseEpoch,
+        expectedInstanceId: ids.homeLease.instanceId,
+        diagnostic: "provider_not_found_during_routed_operation",
+      });
+      if (marked.status === "marked" && bus) {
+        await appendAndPublishEvents(db, bus, ids.workspaceId, ids.sessionId, [
+          {
+            type: "sandbox.box.lost",
+            payload: { sandboxId: ids.homeLease.instanceId },
+          },
+        ]).catch(() => undefined);
+      }
+      const lease = marked.lease;
+      const restore = lease?.recovery.restore.status;
+      return {
+        leaseEpoch: lease?.leaseEpoch ?? ids.homeLease.leaseEpoch,
+        recovery:
+          marked.status === "stale"
+            ? ("superseded" as const)
+            : restore === "pending"
+              ? ("pending" as const)
+              : restore === "degraded"
+                ? ("degraded" as const)
+                : ("unrecoverable" as const),
+      };
+    },
   });
 
   return { ...established, session: proxy };