@opengeni/api-router 0.23.1 → 0.30.0

package/src/sandbox/machines.ts

@@ -156,8 +156,8 @@ function machineStateFor(
 
 /**
  * Build the Machines dashboard response for a workspace. When `sessionId` is
- * supplied (an in-session view) the session's synthetic Modal group box is
- * prepended (`isSessionGroup:true`) and the active-sandbox pointer is echoed;
+ * supplied (an in-session view) the session's synthetic home group box is
+ * prepended when one exists (`isSessionGroup:true`) and the active-sandbox pointer is echoed;
  * without it (the pure workspace dashboard) `activeSandboxId` is null and only
  * the enrolled machines are listed.
  */
@@ -184,9 +184,9 @@ export async function listMachines(
 
   const machines: MachineView[] = [];
 
-  // The session's own Modal group box (synthetic): the default/home sandbox a
-  // null active pointer routes to. Only present in an in-session view.
-  if (session) {
+  // The session's own group box (synthetic): the default/home sandbox a null
+  // active pointer routes to. A backend:none session has no home box.
+  if (session && session.sandboxBackend !== "none") {
     const groupActive = activeSandboxId === null;
     const groupLease = await readLease(db, workspaceId, session.sandboxGroupId);
     machines.push(

package/src/sandbox/metrics-ingestion.ts

@@ -32,8 +32,10 @@
 
 import {
   clearEnrollmentWentOffline,
+  disconnectAttachedBrowserDevices,
   getEnrollment,
   ingestMachineMetricsSample,
+  reconcileAttachedBrowserInventory,
   sessionsWithActiveOpOnEnrollment,
   setEnrollmentDisplayState,
   setEnrollmentOpStreamState,
@@ -43,13 +45,20 @@ import {
   type Database,
   type MachineMetricsSample,
 } from "@opengeni/db";
+import {
+  AttachedBrowserInventorySnapshot as AttachedBrowserInventoryContract,
+  type AttachedBrowserInventorySnapshot as AttachedBrowserInventoryContractValue,
+} from "@opengeni/contracts";
 import { appendAndPublishEvents, type EventBus } from "@opengeni/events";
 import type { Observability } from "@opengeni/observability";
 import {
   AgentEvent,
+  Arch,
   GoingOfflineReason,
   Hello,
+  Os,
   goingOfflineReasonToJSON,
+  type AttachedBrowserInventorySnapshot as WireAttachedBrowserInventorySnapshot,
   type MetricsSample,
 } from "@opengeni/agent-proto";
 
@@ -124,6 +133,90 @@ export function wireSampleToDbSample(wire: MetricsSample): MachineMetricsSample
   };
 }
 
+/** Validate and project one wire inventory before it reaches durable browser
+ *  discovery. Unknown enum values and unsafe uint64 counters reject the whole
+ *  authoritative snapshot instead of partially disconnecting good endpoints. */
+export function wireAttachedBrowserInventoryToContract(
+  wire: WireAttachedBrowserInventorySnapshot,
+): AttachedBrowserInventoryContractValue {
+  return AttachedBrowserInventoryContract.parse({
+    bridgeGeneration: wire.bridgeGeneration,
+    revision: safeWireInteger(wire.revision, "browser inventory revision"),
+    devices: wire.devices.map((device) => ({
+      id: device.id,
+      name: device.name,
+      profileLabel: device.profileLabel.trim() || null,
+      browserName: device.browserName,
+      browserVersion: device.browserVersion,
+      extensionVersion: device.extensionVersion,
+      platform: wireBrowserPlatform(device.platform),
+      architecture: wireBrowserArchitecture(device.arch),
+      connectionGeneration: device.connectionGeneration,
+      inventoryRevision: safeWireInteger(
+        device.inventoryRevision,
+        "attached browser inventory revision",
+      ),
+      tabCount: safeWireInteger(device.tabCount, "attached browser tab count"),
+      capabilities: device.capabilities,
+    })),
+  });
+}
+
+function safeWireInteger(value: string | number, label: string): number {
+  let parsed: bigint;
+  try {
+    parsed = BigInt(value);
+  } catch {
+    throw new Error(`${label} is not an integer`);
+  }
+  if (parsed < 0n || parsed > BigInt(Number.MAX_SAFE_INTEGER)) {
+    throw new Error(`${label} is outside the safe integer range`);
+  }
+  return Number(parsed);
+}
+
+function wireBrowserPlatform(platform: Os): "linux" | "macos" | "windows" {
+  switch (platform) {
+    case Os.OS_LINUX:
+      return "linux";
+    case Os.OS_MACOS:
+      return "macos";
+    case Os.OS_WINDOWS:
+      return "windows";
+    default:
+      throw new Error("attached browser platform is unspecified or unsupported");
+  }
+}
+
+function wireBrowserArchitecture(architecture: Arch): "x64" | "arm64" {
+  switch (architecture) {
+    case Arch.ARCH_X86_64:
+      return "x64";
+    case Arch.ARCH_AARCH64:
+      return "arm64";
+    default:
+      throw new Error("attached browser architecture is unspecified or unsupported");
+  }
+}
+
+async function ingestAttachedBrowserInventory(
+  db: Database,
+  input: {
+    workspaceId: string;
+    agentId: string;
+    inventory: WireAttachedBrowserInventorySnapshot;
+  },
+): Promise<void> {
+  const enrollment = await getEnrollment(db, input.workspaceId, input.agentId);
+  if (!enrollment) return;
+  await reconcileAttachedBrowserInventory(db, {
+    accountId: enrollment.accountId,
+    workspaceId: input.workspaceId,
+    enrollmentId: input.agentId,
+    snapshot: wireAttachedBrowserInventoryToContract(input.inventory),
+  });
+}
+
 /**
  * Ingest ONE decoded heartbeat for an enrolled machine. Resolves the enrollment's
  * accountId (needed for the RLS-scoped writes) from the enrollment row; an
@@ -253,6 +346,18 @@ export async function handleAgentEventPayload(
           enrollmentId: ids.agentId,
           reason,
         });
+        try {
+          await disconnectAttachedBrowserDevices(db, {
+            accountId: enrollment.accountId,
+            workspaceId: ids.workspaceId,
+            enrollmentId: ids.agentId,
+          });
+        } catch (error) {
+          observability?.warn?.("Failed to disconnect attached browsers with their machine", {
+            subject,
+            error: error instanceof Error ? error.message : String(error),
+          });
+        }
         // Fan out the link-plane events to the sessions with an active op on this
         // machine: machine.link.lost (its control link is going away) for every
         // clean going-offline, PLUS machine.runner.restarted when the reason is a
@@ -294,10 +399,23 @@ export async function handleAgentEventPayload(
   if (event.event?.$case !== "heartbeat") {
     return; // an unknown event kind → not a metrics point.
   }
-  const metrics = event.event.heartbeat.metrics;
-  if (!metrics) {
-    return; // a heartbeat without a sample → liveness already touched elsewhere.
+  const heartbeat = event.event.heartbeat;
+  if (heartbeat.attachedBrowserInventory) {
+    try {
+      await ingestAttachedBrowserInventory(db, {
+        workspaceId: ids.workspaceId,
+        agentId: ids.agentId,
+        inventory: heartbeat.attachedBrowserInventory,
+      });
+    } catch (error) {
+      observability?.warn?.("Failed to ingest an attached-browser inventory", {
+        subject,
+        error: error instanceof Error ? error.message : String(error),
+      });
+    }
   }
+  const metrics = heartbeat.metrics;
+  if (!metrics) return;
   try {
     await ingestHeartbeat(db, {
       workspaceId: ids.workspaceId,

package/src/sandbox/rematerialize.ts

@@ -21,8 +21,11 @@ import {
   requirePersistableReplacementSandboxEnvelope,
   resolveModalCheckpointProviderBindingForSession,
   serializeReplacementSandboxEnvelope,
+  SandboxProviderContinuityUnavailableError,
   tagModalSandbox,
+  terminateUnpublishedSandboxSession,
   verifySandboxExecReadiness,
+  withoutSandboxProviderIdentity,
   WorkspaceArchiveIntegrityError,
   type EstablishedSandboxSession,
   type WorkspaceArchiveDescriptor,
@@ -52,39 +55,10 @@ function legacyNativeArchiveFromEnvelope(envelope: Record<string, unknown> | nul
   );
 }
 
-function withoutProviderIdentity(
-  envelope: Record<string, unknown> | null,
-): Record<string, unknown> | null {
-  if (!envelope) return null;
-  const sessionState =
-    envelope.sessionState && typeof envelope.sessionState === "object"
-      ? (envelope.sessionState as Record<string, unknown>)
-      : null;
-  if (!sessionState) return envelope;
-  const { providerState: _providerState, ...providerIndependentState } = sessionState;
-  return { ...envelope, sessionState: providerIndependentState };
-}
-
 async function terminateCreated(established: EstablishedSandboxSession | null): Promise<boolean> {
   if (!established) return true;
-  const client = established.client as {
-    delete?: (state: unknown) => Promise<unknown>;
-  };
   try {
-    if (typeof client.delete === "function" && established.sessionState !== undefined) {
-      await client.delete(established.sessionState);
-      return true;
-    }
-    const session = established.session as {
-      terminate?: () => Promise<unknown>;
-      kill?: () => Promise<unknown>;
-      close?: () => Promise<unknown>;
-      closed?: boolean;
-    };
-    if (session.terminate) await session.terminate();
-    else if (session.kill) await session.kill();
-    else if (session.close && !session.closed) await session.close();
-    else return false;
+    await terminateUnpublishedSandboxSession(established);
     return true;
   } catch (error) {
     return isProviderSandboxNotFoundError(established.backendId, error);
@@ -112,7 +86,7 @@ export async function establishApiSandboxSpawner(input: {
   const fallbackArchiveEnvelope =
     input.acquiredLease.recovery.archive.status === "none" &&
     hasWorkspaceArchive(input.fallbackEnvelope)
-      ? withoutProviderIdentity(input.fallbackEnvelope)
+      ? withoutSandboxProviderIdentity(input.fallbackEnvelope)
       : null;
   let spawnEnvelope =
     fallbackArchiveEnvelope ?? input.acquiredLease.resumeState ?? input.fallbackEnvelope;
@@ -131,10 +105,12 @@ export async function establishApiSandboxSpawner(input: {
       ReturnType<typeof resolveModalCheckpointProviderBindingForSession>
     > | null;
   } | null = null;
+  const continuityRecovery = input.acquiredLease.recovery.continuity;
   try {
     if (
-      input.acquiredLease.recovery.archive.status === "available" ||
-      hasWorkspaceArchive(archiveSource)
+      (input.acquiredLease.recovery.archive.status === "available" &&
+        input.acquiredLease.archiveComplete) ||
+      (input.acquiredLease.recovery.archive.status === "none" && hasWorkspaceArchive(archiveSource))
     ) {
       const id = crypto.randomUUID();
       const legacyNativeArchive = legacyNativeArchiveFromEnvelope(archiveSource);
@@ -177,7 +153,7 @@ export async function establishApiSandboxSpawner(input: {
         legacyCheckpoint: begun.checkpointArtifact === null ? legacyNativeArchive : null,
         legacyProviderBinding: null,
       };
-    } else if (input.acquiredLease.recovery.archive.status !== "none") {
+    } else if (input.acquiredLease.recovery.archive.status !== "none" && !continuityRecovery) {
       throw new SandboxLeaseRecoveryBlockedError(
         input.sandboxGroupId,
         input.expectedEpoch,
@@ -235,6 +211,7 @@ export async function establishApiSandboxSpawner(input: {
           sandboxGroupId: input.sandboxGroupId,
           expectedEpoch: input.expectedEpoch,
           rematerializationId: rematerialization?.id ?? null,
+          ...(created.providerContinuity ? { continuityRecovery: created.providerContinuity } : {}),
           instanceId: created.instanceId,
           resumeBackendId: created.backendId,
           resumeState,
@@ -310,6 +287,7 @@ export async function establishApiSandboxSpawner(input: {
     await verifySandboxExecReadiness(established);
     if (
       rematerialization &&
+      !established.providerContinuity &&
       established.restoredArchive?.revision !== rematerialization.selectedRevision
     ) {
       throw new WorkspaceArchiveIntegrityError(
@@ -330,19 +308,21 @@ export async function establishApiSandboxSpawner(input: {
       dataPlaneUrl: input.dataPlaneUrl,
       resumeBackendId: established.backendId,
       resumeState,
-      ...(rematerialization
-        ? {
-            rematerialization: {
-              id: rematerialization.id,
-              verifiedRevision: rematerialization.selectedRevision,
-            },
-          }
-        : {}),
+      ...(established.providerContinuity
+        ? { continuityRecovery: established.providerContinuity }
+        : rematerialization
+          ? {
+              rematerialization: {
+                id: rematerialization.id,
+                verifiedRevision: rematerialization.selectedRevision,
+              },
+            }
+          : {}),
       leaseTtlMs: input.settings.sandboxLeaseTtlMs,
     });
     if (!committed.committed || !committed.lease) {
       const terminated = await terminateCreated(established);
-      if (terminated && rematerialization) {
+      if (terminated && rematerialization && !established.providerContinuity) {
         await failSandboxRematerialization(input.db, {
           accountId: input.accountId,
           workspaceId: input.workspaceId,
@@ -367,18 +347,25 @@ export async function establishApiSandboxSpawner(input: {
     if (error instanceof SandboxLeaseSupersededError) throw error;
     const terminated = await terminateCreated(established);
     if (terminated) {
-      if (rematerialization) {
+      const continuityUnavailable = error instanceof SandboxProviderContinuityUnavailableError;
+      if (rematerialization && !established?.providerContinuity) {
         await failSandboxRematerialization(input.db, {
           accountId: input.accountId,
           workspaceId: input.workspaceId,
           sandboxGroupId: input.sandboxGroupId,
           expectedEpoch: input.expectedEpoch,
           rematerializationId: rematerialization.id,
-          failureCode:
-            error instanceof WorkspaceArchiveIntegrityError
+          failureCode: continuityUnavailable
+            ? "provider_continuity_unavailable"
+            : error instanceof WorkspaceArchiveIntegrityError
               ? error.code
               : "sandbox_rematerialization_failed",
-          retryable: error instanceof WorkspaceArchiveIntegrityError ? error.retryable : true,
+          retryable: continuityUnavailable
+            ? error.retryable
+            : error instanceof WorkspaceArchiveIntegrityError
+              ? error.retryable
+              : true,
+          ...(continuityUnavailable ? { discardContinuity: true } : {}),
         });
       } else {
         await failWarmingToCold(input.db, {
@@ -386,6 +373,7 @@ export async function establishApiSandboxSpawner(input: {
           workspaceId: input.workspaceId,
           sandboxGroupId: input.sandboxGroupId,
           expectedEpoch: input.expectedEpoch,
+          ...(continuityUnavailable ? { discardContinuity: true } : {}),
         });
       }
     }

package/src/sandbox/viewer.ts

@@ -9,7 +9,8 @@
 //   attach  -> acquireLease(kind:'viewer') under FOR UPDATE + cold->warming CAS.
 //              spawner role  -> establish the box in-process + commitWarmingToWarm.
 //              attached/rearmed -> the holder alone keeps the box warm.
-//              fenced -> release + surface a 409 (a newer epoch re-established it).
+//              lifecycle claim -> wait for settlement; true epoch supersession
+//              -> release + surface a 409.
 //   heartbeat -> heartbeatLeaseHolder (epoch-fenced) refreshes the holder TTL.
 //   detach  -> releaseLeaseHolder (idempotent); the reaper (P1.3) stop()s the
 //              box at refcount 0 past the drain grace.
@@ -24,7 +25,7 @@ import {
   hasGitCredentialRepositorySelection,
   hasGitHubRepositorySelection,
   resolveStreamTokenSecret,
-  sandboxArchiveCaptureTimeoutMs,
+  sandboxLifecycleTransitionWaitMs,
   stableSandboxEnvironmentForRun,
 } from "@opengeni/config";
 import type { Settings } from "@opengeni/config";
@@ -62,6 +63,8 @@ import {
   exposeStreamPort,
   desktopCapableBackend,
   NatsControlRpc,
+  SandboxResumeIdentityMismatchError,
+  SandboxResumeIdentityUnavailableError,
   SelfhostedSandboxClient,
   TERMINAL_STREAM_PORT,
   DisplayStackUnsupportedError,
@@ -179,7 +182,14 @@ export async function sessionAttachEnvironment(
  */
 export async function attachViewer(
   services: ViewerServices,
-  input: { accountId: string; workspaceId: string; session: Session; viewerId?: string },
+  input: {
+    accountId: string;
+    workspaceId: string;
+    session: Session;
+    viewerId?: string;
+    /** Cancel lifecycle waiting when the originating HTTP request disconnects. */
+    waitSignal?: AbortSignal;
+  },
 ): Promise<ViewerAttachResult> {
   const { db, settings } = services;
   const { accountId, workspaceId, session } = input;
@@ -211,7 +221,8 @@ export async function attachViewer(
       os: session.sandboxOs,
       leaseTtlMs,
       warmingLeaseTtlMs: settings.sandboxWarmingTimeoutMs,
-      captureWaitMs: sandboxArchiveCaptureTimeoutMs(settings),
+      captureWaitMs: sandboxLifecycleTransitionWaitMs(settings),
+      ...(input.waitSignal ? { waitSignal: input.waitSignal } : {}),
     });
   } catch (error) {
     if (error instanceof SandboxViewerAdmissionBlockedError) {
@@ -237,7 +248,10 @@ export async function attachViewer(
   if (acquired.role === "fenced") {
     await release();
     throw new HTTPException(409, {
-      message: `sandbox lease superseded (epoch ${acquired.lease.leaseEpoch}); re-read capabilities and re-attach`,
+      message:
+        acquired.reason === "superseded"
+          ? `sandbox lease superseded (epoch ${acquired.lease.leaseEpoch}); re-read capabilities and re-attach`
+          : `sandbox lifecycle transition in progress (${acquired.reason}, epoch ${acquired.lease.leaseEpoch}, backend ${acquired.lease.backend}, instance ${acquired.lease.instanceId ?? "none"}); retry`,
     });
   }
 
@@ -362,6 +376,12 @@ export async function attachViewer(
           message: `sandbox instance was lost; retry to restore it`,
         });
       }
+      if (
+        error instanceof SandboxResumeIdentityMismatchError ||
+        error instanceof SandboxResumeIdentityUnavailableError
+      ) {
+        throw new HTTPException(409, { message: error.message });
+      }
       throw error;
     } finally {
       await dropEstablishedHandle(observed);
@@ -625,7 +645,7 @@ export type MintDesktopStreamInput = {
   session: Session;
   /** The viewer holder id the scoped token is minted for. */
   viewerId: string;
-  /** The live lease (must be warm/draining — the box is up). A selfhosted-active
+  /** The live lease (must be holder-confirmed warm). A selfhosted-active
    *  session may have no Modal group lease; omit and the selfhosted branch handles it. */
   lease?: LeaseSnapshot;
   /** The epoch the CALLER last observed the URL minted under. When the live
@@ -717,7 +737,7 @@ export async function mintDesktopStream(
 
   // GATE 2: the box must be live (the handshake never spins one up — a cold box
   // returns lease_cold; the viewer-attach path warms it first, then mints).
-  if (!lease || (lease.liveness !== "warm" && lease.liveness !== "draining")) {
+  if (!lease || lease.liveness !== "warm") {
     return null;
   }
 
@@ -871,15 +891,18 @@ export type TerminalStreamMint = {
   token: string;
   expiresAt: string;
   leaseEpoch: number;
+  transport: "pty-ws" | "relay-pty";
 };
 
+type SelfhostedStreamMint = Omit<TerminalStreamMint, "transport">;
+
 export type MintTerminalStreamInput = {
   accountId: string;
   workspaceId: string;
   session: Session;
   /** The viewer holder / principal id the scoped token is minted for. */
   viewerId: string;
-  /** The live lease (must be warm/draining — the box is up). A selfhosted-active
+  /** The live lease (must be holder-confirmed warm). A selfhosted-active
    *  session may have no Modal group lease; omit and the selfhosted branch handles it. */
   lease?: LeaseSnapshot;
   /** Test seam: override how the box is re-established by id (see
@@ -930,7 +953,7 @@ export async function mintTerminalStream(
   if (session.activeSandboxId) {
     const active = await getSandbox(db, workspaceId, session.activeSandboxId);
     if (active?.kind === "selfhosted") {
-      return await tryMintActiveSelfhostedStream(
+      const stream = await tryMintActiveSelfhostedStream(
         services,
         {
           session,
@@ -941,13 +964,14 @@ export async function mintTerminalStream(
         },
         input.resolveSelfhostedSession,
       );
+      return stream ? { ...stream, transport: "relay-pty" } : null;
     }
     // A Modal swap target (or unknown) falls through to the existing group-box path
     // (unchanged — Modal swap-target streaming is out of scope for this fix).
   }
 
   // GATE 2: the box must be live (the handshake never spins one up).
-  if (!lease || (lease.liveness !== "warm" && lease.liveness !== "draining")) {
+  if (!lease || lease.liveness !== "warm") {
     return null;
   }
 
@@ -1032,6 +1056,7 @@ export async function mintTerminalStream(
       token: exposed.token,
       expiresAt: exposed.expiresAt,
       leaseEpoch: lease.leaseEpoch,
+      transport: "pty-ws",
     };
   } catch {
     // Any other failure degrades the terminal pty-ws cell to the sse-events
@@ -1087,7 +1112,7 @@ async function tryMintActiveSelfhostedStream(
   resolveSelfhostedSession?: (
     sandbox: SandboxRecord,
   ) => Promise<{ resolveExposedPort?: (port: number) => Promise<unknown> }>,
-): Promise<TerminalStreamMint | null> {
+): Promise<SelfhostedStreamMint | null> {
   const { settings, bus } = services;
   const { session, workspaceId, port, sandbox } = input;
   if (!sandbox.enrollmentId) {
@@ -1108,6 +1133,8 @@ async function tryMintActiveSelfhostedStream(
         controlRpcFactory: () => controlRpc(bus),
         agentId: sandbox.enrollmentId,
         epoch: session.activeEpoch,
+        // Stream authority rotates; terminal process identity does not.
+        terminalScopeId: session.id,
       });
       shSession = await client.resume({ agentId: sandbox.enrollmentId });
     }
@@ -1165,7 +1192,7 @@ export type MintSelfhostedStreamInput = {
 export async function mintSelfhostedStream(
   services: ViewerServices,
   input: MintSelfhostedStreamInput,
-): Promise<TerminalStreamMint | null> {
+): Promise<SelfhostedStreamMint | null> {
   const { settings } = services;
   const secret = resolveStreamTokenSecret(settings);
   if (!secret) {