@opengeni/api-router 0.22.2 → 0.26.1

package/src/sandbox/enrollment.ts

@@ -75,8 +75,9 @@ export const DEVICE_POLL_INTERVAL_SECONDS = 5;
 // box) caused a self-hosted agent to drop PERMANENTLY one hour after connecting: the
 // bearer expired and the auth-callout rejected every reconnect ("re-enroll may be
 // required"). A long-lived bearer is safe because the auth-callout RE-CHECKS the
-// enrollment status on every (re)connect (auth-callout.ts) — a revoked machine is
-// denied regardless of bearer life — exactly as the long-lived relay token relies on.
+// enrollment status AND credential generation on every (re)connect
+// (auth-callout.ts) — a revoked machine or an old pre-re-enrollment bearer is denied
+// regardless of bearer life. The short NATS user-JWT cap bounds already-live access.
 export const ENROLLMENT_BEARER_TTL_SECONDS = 30 * 24 * 3600;
 // The relay PRODUCER token (the `ogr_` token; M8b) is ENROLLMENT-scoped,
 // NOT per-stream: the agent presents it on every channel registration for the life
@@ -370,6 +371,7 @@ export async function exchangeEnrollToken(
     secret,
     workspaceId: claims.workspaceId,
     agentId: enrollment.id,
+    credentialGeneration: enrollment.credentialGeneration,
     consentedScreenControl: enrollment.allowScreenControl,
   });
   return { ok: true, credentials };
@@ -430,6 +432,7 @@ export async function pollDeviceEnrollment(
     secret,
     workspaceId: request.workspaceId,
     agentId: enrollment.id,
+    credentialGeneration: enrollment.credentialGeneration,
     consentedScreenControl: enrollment.allowScreenControl,
   });
 
@@ -455,7 +458,13 @@ export async function pollDeviceEnrollment(
  *  bearer so an agent using it as the connect-token credential works uniformly. */
 async function buildEnrollmentCredentials(
   services: EnrollmentServices,
-  input: { secret: string; workspaceId: string; agentId: string; consentedScreenControl: boolean },
+  input: {
+    secret: string;
+    workspaceId: string;
+    agentId: string;
+    credentialGeneration: number;
+    consentedScreenControl: boolean;
+  },
 ): Promise<EnrollmentCredentialsResponse> {
   const { settings } = services;
   // The control-plane subject prefix the agent subscribes to: agent.<ws>.<id>.
@@ -466,6 +475,7 @@ async function buildEnrollmentCredentials(
     workspaceId: input.workspaceId,
     agentId: input.agentId,
     enrollmentId: input.agentId,
+    credentialGeneration: input.credentialGeneration,
     subjectPrefix,
     exp,
   });

package/src/sandbox/machines.ts

@@ -219,7 +219,7 @@ export async function listMachines(
   // onto the machines (no N+1). Each machine is probed for liveness.
   const [sandboxes, enrollments, metricsByEnrollment] = await Promise.all([
     listSandboxes(db, workspaceId),
-    listEnrollments(db, workspaceId),
+    listEnrollments(db, workspaceId, { status: "active" }),
     readMachineMetricsLatestForWorkspace(db, workspaceId),
   ]);
   const enrollmentById = new Map(enrollments.map((e) => [e.id, e]));

package/src/sandbox/metrics-ingestion.ts

@@ -32,8 +32,10 @@
 
 import {
   clearEnrollmentWentOffline,
+  disconnectAttachedBrowserDevices,
   getEnrollment,
   ingestMachineMetricsSample,
+  reconcileAttachedBrowserInventory,
   sessionsWithActiveOpOnEnrollment,
   setEnrollmentDisplayState,
   setEnrollmentOpStreamState,
@@ -43,13 +45,20 @@ import {
   type Database,
   type MachineMetricsSample,
 } from "@opengeni/db";
+import {
+  AttachedBrowserInventorySnapshot as AttachedBrowserInventoryContract,
+  type AttachedBrowserInventorySnapshot as AttachedBrowserInventoryContractValue,
+} from "@opengeni/contracts";
 import { appendAndPublishEvents, type EventBus } from "@opengeni/events";
 import type { Observability } from "@opengeni/observability";
 import {
   AgentEvent,
+  Arch,
   GoingOfflineReason,
   Hello,
+  Os,
   goingOfflineReasonToJSON,
+  type AttachedBrowserInventorySnapshot as WireAttachedBrowserInventorySnapshot,
   type MetricsSample,
 } from "@opengeni/agent-proto";
 
@@ -124,6 +133,90 @@ export function wireSampleToDbSample(wire: MetricsSample): MachineMetricsSample
   };
 }
 
+/** Validate and project one wire inventory before it reaches durable browser
+ *  discovery. Unknown enum values and unsafe uint64 counters reject the whole
+ *  authoritative snapshot instead of partially disconnecting good endpoints. */
+export function wireAttachedBrowserInventoryToContract(
+  wire: WireAttachedBrowserInventorySnapshot,
+): AttachedBrowserInventoryContractValue {
+  return AttachedBrowserInventoryContract.parse({
+    bridgeGeneration: wire.bridgeGeneration,
+    revision: safeWireInteger(wire.revision, "browser inventory revision"),
+    devices: wire.devices.map((device) => ({
+      id: device.id,
+      name: device.name,
+      profileLabel: device.profileLabel.trim() || null,
+      browserName: device.browserName,
+      browserVersion: device.browserVersion,
+      extensionVersion: device.extensionVersion,
+      platform: wireBrowserPlatform(device.platform),
+      architecture: wireBrowserArchitecture(device.arch),
+      connectionGeneration: device.connectionGeneration,
+      inventoryRevision: safeWireInteger(
+        device.inventoryRevision,
+        "attached browser inventory revision",
+      ),
+      tabCount: safeWireInteger(device.tabCount, "attached browser tab count"),
+      capabilities: device.capabilities,
+    })),
+  });
+}
+
+function safeWireInteger(value: string | number, label: string): number {
+  let parsed: bigint;
+  try {
+    parsed = BigInt(value);
+  } catch {
+    throw new Error(`${label} is not an integer`);
+  }
+  if (parsed < 0n || parsed > BigInt(Number.MAX_SAFE_INTEGER)) {
+    throw new Error(`${label} is outside the safe integer range`);
+  }
+  return Number(parsed);
+}
+
+function wireBrowserPlatform(platform: Os): "linux" | "macos" | "windows" {
+  switch (platform) {
+    case Os.OS_LINUX:
+      return "linux";
+    case Os.OS_MACOS:
+      return "macos";
+    case Os.OS_WINDOWS:
+      return "windows";
+    default:
+      throw new Error("attached browser platform is unspecified or unsupported");
+  }
+}
+
+function wireBrowserArchitecture(architecture: Arch): "x64" | "arm64" {
+  switch (architecture) {
+    case Arch.ARCH_X86_64:
+      return "x64";
+    case Arch.ARCH_AARCH64:
+      return "arm64";
+    default:
+      throw new Error("attached browser architecture is unspecified or unsupported");
+  }
+}
+
+async function ingestAttachedBrowserInventory(
+  db: Database,
+  input: {
+    workspaceId: string;
+    agentId: string;
+    inventory: WireAttachedBrowserInventorySnapshot;
+  },
+): Promise<void> {
+  const enrollment = await getEnrollment(db, input.workspaceId, input.agentId);
+  if (!enrollment) return;
+  await reconcileAttachedBrowserInventory(db, {
+    accountId: enrollment.accountId,
+    workspaceId: input.workspaceId,
+    enrollmentId: input.agentId,
+    snapshot: wireAttachedBrowserInventoryToContract(input.inventory),
+  });
+}
+
 /**
  * Ingest ONE decoded heartbeat for an enrolled machine. Resolves the enrollment's
  * accountId (needed for the RLS-scoped writes) from the enrollment row; an
@@ -253,6 +346,18 @@ export async function handleAgentEventPayload(
           enrollmentId: ids.agentId,
           reason,
         });
+        try {
+          await disconnectAttachedBrowserDevices(db, {
+            accountId: enrollment.accountId,
+            workspaceId: ids.workspaceId,
+            enrollmentId: ids.agentId,
+          });
+        } catch (error) {
+          observability?.warn?.("Failed to disconnect attached browsers with their machine", {
+            subject,
+            error: error instanceof Error ? error.message : String(error),
+          });
+        }
         // Fan out the link-plane events to the sessions with an active op on this
         // machine: machine.link.lost (its control link is going away) for every
         // clean going-offline, PLUS machine.runner.restarted when the reason is a
@@ -294,10 +399,23 @@ export async function handleAgentEventPayload(
   if (event.event?.$case !== "heartbeat") {
     return; // an unknown event kind → not a metrics point.
   }
-  const metrics = event.event.heartbeat.metrics;
-  if (!metrics) {
-    return; // a heartbeat without a sample → liveness already touched elsewhere.
+  const heartbeat = event.event.heartbeat;
+  if (heartbeat.attachedBrowserInventory) {
+    try {
+      await ingestAttachedBrowserInventory(db, {
+        workspaceId: ids.workspaceId,
+        agentId: ids.agentId,
+        inventory: heartbeat.attachedBrowserInventory,
+      });
+    } catch (error) {
+      observability?.warn?.("Failed to ingest an attached-browser inventory", {
+        subject,
+        error: error instanceof Error ? error.message : String(error),
+      });
+    }
   }
+  const metrics = heartbeat.metrics;
+  if (!metrics) return;
   try {
     await ingestHeartbeat(db, {
       workspaceId: ids.workspaceId,

package/src/sandbox/rematerialize.ts

@@ -21,8 +21,11 @@ import {
   requirePersistableReplacementSandboxEnvelope,
   resolveModalCheckpointProviderBindingForSession,
   serializeReplacementSandboxEnvelope,
+  SandboxProviderContinuityUnavailableError,
   tagModalSandbox,
+  terminateUnpublishedSandboxSession,
   verifySandboxExecReadiness,
+  withoutSandboxProviderIdentity,
   WorkspaceArchiveIntegrityError,
   type EstablishedSandboxSession,
   type WorkspaceArchiveDescriptor,
@@ -52,39 +55,10 @@ function legacyNativeArchiveFromEnvelope(envelope: Record<string, unknown> | nul
   );
 }
 
-function withoutProviderIdentity(
-  envelope: Record<string, unknown> | null,
-): Record<string, unknown> | null {
-  if (!envelope) return null;
-  const sessionState =
-    envelope.sessionState && typeof envelope.sessionState === "object"
-      ? (envelope.sessionState as Record<string, unknown>)
-      : null;
-  if (!sessionState) return envelope;
-  const { providerState: _providerState, ...providerIndependentState } = sessionState;
-  return { ...envelope, sessionState: providerIndependentState };
-}
-
 async function terminateCreated(established: EstablishedSandboxSession | null): Promise<boolean> {
   if (!established) return true;
-  const client = established.client as {
-    delete?: (state: unknown) => Promise<unknown>;
-  };
   try {
-    if (typeof client.delete === "function" && established.sessionState !== undefined) {
-      await client.delete(established.sessionState);
-      return true;
-    }
-    const session = established.session as {
-      terminate?: () => Promise<unknown>;
-      kill?: () => Promise<unknown>;
-      close?: () => Promise<unknown>;
-      closed?: boolean;
-    };
-    if (session.terminate) await session.terminate();
-    else if (session.kill) await session.kill();
-    else if (session.close && !session.closed) await session.close();
-    else return false;
+    await terminateUnpublishedSandboxSession(established);
     return true;
   } catch (error) {
     return isProviderSandboxNotFoundError(established.backendId, error);
@@ -112,7 +86,7 @@ export async function establishApiSandboxSpawner(input: {
   const fallbackArchiveEnvelope =
     input.acquiredLease.recovery.archive.status === "none" &&
     hasWorkspaceArchive(input.fallbackEnvelope)
-      ? withoutProviderIdentity(input.fallbackEnvelope)
+      ? withoutSandboxProviderIdentity(input.fallbackEnvelope)
       : null;
   let spawnEnvelope =
     fallbackArchiveEnvelope ?? input.acquiredLease.resumeState ?? input.fallbackEnvelope;
@@ -131,10 +105,12 @@ export async function establishApiSandboxSpawner(input: {
       ReturnType<typeof resolveModalCheckpointProviderBindingForSession>
     > | null;
   } | null = null;
+  const continuityRecovery = input.acquiredLease.recovery.continuity;
   try {
     if (
-      input.acquiredLease.recovery.archive.status === "available" ||
-      hasWorkspaceArchive(archiveSource)
+      (input.acquiredLease.recovery.archive.status === "available" &&
+        input.acquiredLease.archiveComplete) ||
+      (input.acquiredLease.recovery.archive.status === "none" && hasWorkspaceArchive(archiveSource))
     ) {
       const id = crypto.randomUUID();
       const legacyNativeArchive = legacyNativeArchiveFromEnvelope(archiveSource);
@@ -177,7 +153,7 @@ export async function establishApiSandboxSpawner(input: {
         legacyCheckpoint: begun.checkpointArtifact === null ? legacyNativeArchive : null,
         legacyProviderBinding: null,
       };
-    } else if (input.acquiredLease.recovery.archive.status !== "none") {
+    } else if (input.acquiredLease.recovery.archive.status !== "none" && !continuityRecovery) {
       throw new SandboxLeaseRecoveryBlockedError(
         input.sandboxGroupId,
         input.expectedEpoch,
@@ -235,6 +211,7 @@ export async function establishApiSandboxSpawner(input: {
           sandboxGroupId: input.sandboxGroupId,
           expectedEpoch: input.expectedEpoch,
           rematerializationId: rematerialization?.id ?? null,
+          ...(created.providerContinuity ? { continuityRecovery: created.providerContinuity } : {}),
           instanceId: created.instanceId,
           resumeBackendId: created.backendId,
           resumeState,
@@ -310,6 +287,7 @@ export async function establishApiSandboxSpawner(input: {
     await verifySandboxExecReadiness(established);
     if (
       rematerialization &&
+      !established.providerContinuity &&
       established.restoredArchive?.revision !== rematerialization.selectedRevision
     ) {
       throw new WorkspaceArchiveIntegrityError(
@@ -330,19 +308,21 @@ export async function establishApiSandboxSpawner(input: {
       dataPlaneUrl: input.dataPlaneUrl,
       resumeBackendId: established.backendId,
       resumeState,
-      ...(rematerialization
-        ? {
-            rematerialization: {
-              id: rematerialization.id,
-              verifiedRevision: rematerialization.selectedRevision,
-            },
-          }
-        : {}),
+      ...(established.providerContinuity
+        ? { continuityRecovery: established.providerContinuity }
+        : rematerialization
+          ? {
+              rematerialization: {
+                id: rematerialization.id,
+                verifiedRevision: rematerialization.selectedRevision,
+              },
+            }
+          : {}),
       leaseTtlMs: input.settings.sandboxLeaseTtlMs,
     });
     if (!committed.committed || !committed.lease) {
       const terminated = await terminateCreated(established);
-      if (terminated && rematerialization) {
+      if (terminated && rematerialization && !established.providerContinuity) {
         await failSandboxRematerialization(input.db, {
           accountId: input.accountId,
           workspaceId: input.workspaceId,
@@ -367,18 +347,25 @@ export async function establishApiSandboxSpawner(input: {
     if (error instanceof SandboxLeaseSupersededError) throw error;
     const terminated = await terminateCreated(established);
     if (terminated) {
-      if (rematerialization) {
+      const continuityUnavailable = error instanceof SandboxProviderContinuityUnavailableError;
+      if (rematerialization && !established?.providerContinuity) {
         await failSandboxRematerialization(input.db, {
           accountId: input.accountId,
           workspaceId: input.workspaceId,
           sandboxGroupId: input.sandboxGroupId,
           expectedEpoch: input.expectedEpoch,
           rematerializationId: rematerialization.id,
-          failureCode:
-            error instanceof WorkspaceArchiveIntegrityError
+          failureCode: continuityUnavailable
+            ? "provider_continuity_unavailable"
+            : error instanceof WorkspaceArchiveIntegrityError
               ? error.code
               : "sandbox_rematerialization_failed",
-          retryable: error instanceof WorkspaceArchiveIntegrityError ? error.retryable : true,
+          retryable: continuityUnavailable
+            ? error.retryable
+            : error instanceof WorkspaceArchiveIntegrityError
+              ? error.retryable
+              : true,
+          ...(continuityUnavailable ? { discardContinuity: true } : {}),
         });
       } else {
         await failWarmingToCold(input.db, {
@@ -386,6 +373,7 @@ export async function establishApiSandboxSpawner(input: {
           workspaceId: input.workspaceId,
           sandboxGroupId: input.sandboxGroupId,
           expectedEpoch: input.expectedEpoch,
+          ...(continuityUnavailable ? { discardContinuity: true } : {}),
         });
       }
     }

package/src/sandbox/viewer.ts

@@ -9,7 +9,8 @@
 //   attach  -> acquireLease(kind:'viewer') under FOR UPDATE + cold->warming CAS.
 //              spawner role  -> establish the box in-process + commitWarmingToWarm.
 //              attached/rearmed -> the holder alone keeps the box warm.
-//              fenced -> release + surface a 409 (a newer epoch re-established it).
+//              lifecycle claim -> wait for settlement; true epoch supersession
+//              -> release + surface a 409.
 //   heartbeat -> heartbeatLeaseHolder (epoch-fenced) refreshes the holder TTL.
 //   detach  -> releaseLeaseHolder (idempotent); the reaper (P1.3) stop()s the
 //              box at refcount 0 past the drain grace.
@@ -24,7 +25,7 @@ import {
   hasGitCredentialRepositorySelection,
   hasGitHubRepositorySelection,
   resolveStreamTokenSecret,
-  sandboxArchiveCaptureTimeoutMs,
+  sandboxLifecycleTransitionWaitMs,
   stableSandboxEnvironmentForRun,
 } from "@opengeni/config";
 import type { Settings } from "@opengeni/config";
@@ -62,6 +63,8 @@ import {
   exposeStreamPort,
   desktopCapableBackend,
   NatsControlRpc,
+  SandboxResumeIdentityMismatchError,
+  SandboxResumeIdentityUnavailableError,
   SelfhostedSandboxClient,
   TERMINAL_STREAM_PORT,
   DisplayStackUnsupportedError,
@@ -74,6 +77,7 @@ import {
 } from "@opengeni/runtime/sandbox";
 import { relayConfigFromSettings } from "@opengeni/core";
 import { establishApiSandboxSpawner } from "./rematerialize";
+import { establishCachedChannelAHandle } from "./channel-a";
 
 /** The minimal services a viewer op needs: the DB + settings (lease cadence +
  *  the sandbox client construction the leaf reads from settings). The bus is
@@ -178,7 +182,14 @@ export async function sessionAttachEnvironment(
  */
 export async function attachViewer(
   services: ViewerServices,
-  input: { accountId: string; workspaceId: string; session: Session; viewerId?: string },
+  input: {
+    accountId: string;
+    workspaceId: string;
+    session: Session;
+    viewerId?: string;
+    /** Cancel lifecycle waiting when the originating HTTP request disconnects. */
+    waitSignal?: AbortSignal;
+  },
 ): Promise<ViewerAttachResult> {
   const { db, settings } = services;
   const { accountId, workspaceId, session } = input;
@@ -210,7 +221,8 @@ export async function attachViewer(
       os: session.sandboxOs,
       leaseTtlMs,
       warmingLeaseTtlMs: settings.sandboxWarmingTimeoutMs,
-      captureWaitMs: sandboxArchiveCaptureTimeoutMs(settings),
+      captureWaitMs: sandboxLifecycleTransitionWaitMs(settings),
+      ...(input.waitSignal ? { waitSignal: input.waitSignal } : {}),
     });
   } catch (error) {
     if (error instanceof SandboxViewerAdmissionBlockedError) {
@@ -236,7 +248,10 @@ export async function attachViewer(
   if (acquired.role === "fenced") {
     await release();
     throw new HTTPException(409, {
-      message: `sandbox lease superseded (epoch ${acquired.lease.leaseEpoch}); re-read capabilities and re-attach`,
+      message:
+        acquired.reason === "superseded"
+          ? `sandbox lease superseded (epoch ${acquired.lease.leaseEpoch}); re-read capabilities and re-attach`
+          : `sandbox lifecycle transition in progress (${acquired.reason}, epoch ${acquired.lease.leaseEpoch}, backend ${acquired.lease.backend}, instance ${acquired.lease.instanceId ?? "none"}); retry`,
     });
   }
 
@@ -361,6 +376,12 @@ export async function attachViewer(
           message: `sandbox instance was lost; retry to restore it`,
         });
       }
+      if (
+        error instanceof SandboxResumeIdentityMismatchError ||
+        error instanceof SandboxResumeIdentityUnavailableError
+      ) {
+        throw new HTTPException(409, { message: error.message });
+      }
       throw error;
     } finally {
       await dropEstablishedHandle(observed);
@@ -624,7 +645,7 @@ export type MintDesktopStreamInput = {
   session: Session;
   /** The viewer holder id the scoped token is minted for. */
   viewerId: string;
-  /** The live lease (must be warm/draining — the box is up). A selfhosted-active
+  /** The live lease (must be holder-confirmed warm). A selfhosted-active
    *  session may have no Modal group lease; omit and the selfhosted branch handles it. */
   lease?: LeaseSnapshot;
   /** The epoch the CALLER last observed the URL minted under. When the live
@@ -716,7 +737,7 @@ export async function mintDesktopStream(
 
   // GATE 2: the box must be live (the handshake never spins one up — a cold box
   // returns lease_cold; the viewer-attach path warms it first, then mints).
-  if (!lease || (lease.liveness !== "warm" && lease.liveness !== "draining")) {
+  if (!lease || lease.liveness !== "warm") {
     return null;
   }
 
@@ -730,18 +751,22 @@ export async function mintDesktopStream(
     // SAME stable run-env the turn declares, so a later turn finds no env delta.
     const environment = await sessionAttachEnvironment(services, workspaceId, session);
     try {
+      const establish = () =>
+        (services.establishSandboxSession ?? establishSandboxSessionFromEnvelope)(
+          settings,
+          envelope,
+          {
+            sessionId: session.id,
+            recovery: "resume-only",
+            backendOverride: session.sandboxBackend,
+            environment,
+          },
+        );
       established = input.establish
         ? await input.establish(envelope)
-        : await (services.establishSandboxSession ?? establishSandboxSessionFromEnvelope)(
-            settings,
-            envelope,
-            {
-              sessionId: session.id,
-              recovery: "resume-only",
-              backendOverride: session.sandboxBackend,
-              environment,
-            },
-          );
+        : services.establishSandboxSession
+          ? await establish()
+          : await establishCachedChannelAHandle(workspaceId, session.id, lease, establish);
     } catch (error) {
       await retireMissingWarmLease(services, { accountId, workspaceId, session, lease }, error);
       return null;
@@ -874,7 +899,7 @@ export type MintTerminalStreamInput = {
   session: Session;
   /** The viewer holder / principal id the scoped token is minted for. */
   viewerId: string;
-  /** The live lease (must be warm/draining — the box is up). A selfhosted-active
+  /** The live lease (must be holder-confirmed warm). A selfhosted-active
    *  session may have no Modal group lease; omit and the selfhosted branch handles it. */
   lease?: LeaseSnapshot;
   /** Test seam: override how the box is re-established by id (see
@@ -942,7 +967,7 @@ export async function mintTerminalStream(
   }
 
   // GATE 2: the box must be live (the handshake never spins one up).
-  if (!lease || (lease.liveness !== "warm" && lease.liveness !== "draining")) {
+  if (!lease || lease.liveness !== "warm") {
     return null;
   }
 
@@ -955,18 +980,22 @@ export async function mintTerminalStream(
     // declares, so a later turn finds no manifest-env delta.
     const environment = await sessionAttachEnvironment(services, workspaceId, session);
     try {
+      const establish = () =>
+        (services.establishSandboxSession ?? establishSandboxSessionFromEnvelope)(
+          settings,
+          envelope,
+          {
+            sessionId: session.id,
+            recovery: "resume-only",
+            backendOverride: session.sandboxBackend,
+            environment,
+          },
+        );
       established = input.establish
         ? await input.establish(envelope)
-        : await (services.establishSandboxSession ?? establishSandboxSessionFromEnvelope)(
-            settings,
-            envelope,
-            {
-              sessionId: session.id,
-              recovery: "resume-only",
-              backendOverride: session.sandboxBackend,
-              environment,
-            },
-          );
+        : services.establishSandboxSession
+          ? await establish()
+          : await establishCachedChannelAHandle(workspaceId, session.id, lease, establish);
     } catch (error) {
       await retireMissingWarmLease(services, { accountId, workspaceId, session, lease }, error);
       return null;