@opengeni/api-router 0.23.1 → 0.26.1

package/src/sandbox/metrics-ingestion.ts

@@ -32,8 +32,10 @@
 
 import {
   clearEnrollmentWentOffline,
+  disconnectAttachedBrowserDevices,
   getEnrollment,
   ingestMachineMetricsSample,
+  reconcileAttachedBrowserInventory,
   sessionsWithActiveOpOnEnrollment,
   setEnrollmentDisplayState,
   setEnrollmentOpStreamState,
@@ -43,13 +45,20 @@ import {
   type Database,
   type MachineMetricsSample,
 } from "@opengeni/db";
+import {
+  AttachedBrowserInventorySnapshot as AttachedBrowserInventoryContract,
+  type AttachedBrowserInventorySnapshot as AttachedBrowserInventoryContractValue,
+} from "@opengeni/contracts";
 import { appendAndPublishEvents, type EventBus } from "@opengeni/events";
 import type { Observability } from "@opengeni/observability";
 import {
   AgentEvent,
+  Arch,
   GoingOfflineReason,
   Hello,
+  Os,
   goingOfflineReasonToJSON,
+  type AttachedBrowserInventorySnapshot as WireAttachedBrowserInventorySnapshot,
   type MetricsSample,
 } from "@opengeni/agent-proto";
 
@@ -124,6 +133,90 @@ export function wireSampleToDbSample(wire: MetricsSample): MachineMetricsSample
   };
 }
 
+/** Validate and project one wire inventory before it reaches durable browser
+ *  discovery. Unknown enum values and unsafe uint64 counters reject the whole
+ *  authoritative snapshot instead of partially disconnecting good endpoints. */
+export function wireAttachedBrowserInventoryToContract(
+  wire: WireAttachedBrowserInventorySnapshot,
+): AttachedBrowserInventoryContractValue {
+  return AttachedBrowserInventoryContract.parse({
+    bridgeGeneration: wire.bridgeGeneration,
+    revision: safeWireInteger(wire.revision, "browser inventory revision"),
+    devices: wire.devices.map((device) => ({
+      id: device.id,
+      name: device.name,
+      profileLabel: device.profileLabel.trim() || null,
+      browserName: device.browserName,
+      browserVersion: device.browserVersion,
+      extensionVersion: device.extensionVersion,
+      platform: wireBrowserPlatform(device.platform),
+      architecture: wireBrowserArchitecture(device.arch),
+      connectionGeneration: device.connectionGeneration,
+      inventoryRevision: safeWireInteger(
+        device.inventoryRevision,
+        "attached browser inventory revision",
+      ),
+      tabCount: safeWireInteger(device.tabCount, "attached browser tab count"),
+      capabilities: device.capabilities,
+    })),
+  });
+}
+
+function safeWireInteger(value: string | number, label: string): number {
+  let parsed: bigint;
+  try {
+    parsed = BigInt(value);
+  } catch {
+    throw new Error(`${label} is not an integer`);
+  }
+  if (parsed < 0n || parsed > BigInt(Number.MAX_SAFE_INTEGER)) {
+    throw new Error(`${label} is outside the safe integer range`);
+  }
+  return Number(parsed);
+}
+
+function wireBrowserPlatform(platform: Os): "linux" | "macos" | "windows" {
+  switch (platform) {
+    case Os.OS_LINUX:
+      return "linux";
+    case Os.OS_MACOS:
+      return "macos";
+    case Os.OS_WINDOWS:
+      return "windows";
+    default:
+      throw new Error("attached browser platform is unspecified or unsupported");
+  }
+}
+
+function wireBrowserArchitecture(architecture: Arch): "x64" | "arm64" {
+  switch (architecture) {
+    case Arch.ARCH_X86_64:
+      return "x64";
+    case Arch.ARCH_AARCH64:
+      return "arm64";
+    default:
+      throw new Error("attached browser architecture is unspecified or unsupported");
+  }
+}
+
+async function ingestAttachedBrowserInventory(
+  db: Database,
+  input: {
+    workspaceId: string;
+    agentId: string;
+    inventory: WireAttachedBrowserInventorySnapshot;
+  },
+): Promise<void> {
+  const enrollment = await getEnrollment(db, input.workspaceId, input.agentId);
+  if (!enrollment) return;
+  await reconcileAttachedBrowserInventory(db, {
+    accountId: enrollment.accountId,
+    workspaceId: input.workspaceId,
+    enrollmentId: input.agentId,
+    snapshot: wireAttachedBrowserInventoryToContract(input.inventory),
+  });
+}
+
 /**
  * Ingest ONE decoded heartbeat for an enrolled machine. Resolves the enrollment's
  * accountId (needed for the RLS-scoped writes) from the enrollment row; an
@@ -253,6 +346,18 @@ export async function handleAgentEventPayload(
           enrollmentId: ids.agentId,
           reason,
         });
+        try {
+          await disconnectAttachedBrowserDevices(db, {
+            accountId: enrollment.accountId,
+            workspaceId: ids.workspaceId,
+            enrollmentId: ids.agentId,
+          });
+        } catch (error) {
+          observability?.warn?.("Failed to disconnect attached browsers with their machine", {
+            subject,
+            error: error instanceof Error ? error.message : String(error),
+          });
+        }
         // Fan out the link-plane events to the sessions with an active op on this
         // machine: machine.link.lost (its control link is going away) for every
         // clean going-offline, PLUS machine.runner.restarted when the reason is a
@@ -294,10 +399,23 @@ export async function handleAgentEventPayload(
   if (event.event?.$case !== "heartbeat") {
     return; // an unknown event kind → not a metrics point.
   }
-  const metrics = event.event.heartbeat.metrics;
-  if (!metrics) {
-    return; // a heartbeat without a sample → liveness already touched elsewhere.
+  const heartbeat = event.event.heartbeat;
+  if (heartbeat.attachedBrowserInventory) {
+    try {
+      await ingestAttachedBrowserInventory(db, {
+        workspaceId: ids.workspaceId,
+        agentId: ids.agentId,
+        inventory: heartbeat.attachedBrowserInventory,
+      });
+    } catch (error) {
+      observability?.warn?.("Failed to ingest an attached-browser inventory", {
+        subject,
+        error: error instanceof Error ? error.message : String(error),
+      });
+    }
   }
+  const metrics = heartbeat.metrics;
+  if (!metrics) return;
   try {
     await ingestHeartbeat(db, {
       workspaceId: ids.workspaceId,

package/src/sandbox/rematerialize.ts

@@ -21,8 +21,11 @@ import {
   requirePersistableReplacementSandboxEnvelope,
   resolveModalCheckpointProviderBindingForSession,
   serializeReplacementSandboxEnvelope,
+  SandboxProviderContinuityUnavailableError,
   tagModalSandbox,
+  terminateUnpublishedSandboxSession,
   verifySandboxExecReadiness,
+  withoutSandboxProviderIdentity,
   WorkspaceArchiveIntegrityError,
   type EstablishedSandboxSession,
   type WorkspaceArchiveDescriptor,
@@ -52,39 +55,10 @@ function legacyNativeArchiveFromEnvelope(envelope: Record<string, unknown> | nul
   );
 }
 
-function withoutProviderIdentity(
-  envelope: Record<string, unknown> | null,
-): Record<string, unknown> | null {
-  if (!envelope) return null;
-  const sessionState =
-    envelope.sessionState && typeof envelope.sessionState === "object"
-      ? (envelope.sessionState as Record<string, unknown>)
-      : null;
-  if (!sessionState) return envelope;
-  const { providerState: _providerState, ...providerIndependentState } = sessionState;
-  return { ...envelope, sessionState: providerIndependentState };
-}
-
 async function terminateCreated(established: EstablishedSandboxSession | null): Promise<boolean> {
   if (!established) return true;
-  const client = established.client as {
-    delete?: (state: unknown) => Promise<unknown>;
-  };
   try {
-    if (typeof client.delete === "function" && established.sessionState !== undefined) {
-      await client.delete(established.sessionState);
-      return true;
-    }
-    const session = established.session as {
-      terminate?: () => Promise<unknown>;
-      kill?: () => Promise<unknown>;
-      close?: () => Promise<unknown>;
-      closed?: boolean;
-    };
-    if (session.terminate) await session.terminate();
-    else if (session.kill) await session.kill();
-    else if (session.close && !session.closed) await session.close();
-    else return false;
+    await terminateUnpublishedSandboxSession(established);
     return true;
   } catch (error) {
     return isProviderSandboxNotFoundError(established.backendId, error);
@@ -112,7 +86,7 @@ export async function establishApiSandboxSpawner(input: {
   const fallbackArchiveEnvelope =
     input.acquiredLease.recovery.archive.status === "none" &&
     hasWorkspaceArchive(input.fallbackEnvelope)
-      ? withoutProviderIdentity(input.fallbackEnvelope)
+      ? withoutSandboxProviderIdentity(input.fallbackEnvelope)
       : null;
   let spawnEnvelope =
     fallbackArchiveEnvelope ?? input.acquiredLease.resumeState ?? input.fallbackEnvelope;
@@ -131,10 +105,12 @@ export async function establishApiSandboxSpawner(input: {
       ReturnType<typeof resolveModalCheckpointProviderBindingForSession>
     > | null;
   } | null = null;
+  const continuityRecovery = input.acquiredLease.recovery.continuity;
   try {
     if (
-      input.acquiredLease.recovery.archive.status === "available" ||
-      hasWorkspaceArchive(archiveSource)
+      (input.acquiredLease.recovery.archive.status === "available" &&
+        input.acquiredLease.archiveComplete) ||
+      (input.acquiredLease.recovery.archive.status === "none" && hasWorkspaceArchive(archiveSource))
     ) {
       const id = crypto.randomUUID();
       const legacyNativeArchive = legacyNativeArchiveFromEnvelope(archiveSource);
@@ -177,7 +153,7 @@ export async function establishApiSandboxSpawner(input: {
         legacyCheckpoint: begun.checkpointArtifact === null ? legacyNativeArchive : null,
         legacyProviderBinding: null,
       };
-    } else if (input.acquiredLease.recovery.archive.status !== "none") {
+    } else if (input.acquiredLease.recovery.archive.status !== "none" && !continuityRecovery) {
       throw new SandboxLeaseRecoveryBlockedError(
         input.sandboxGroupId,
         input.expectedEpoch,
@@ -235,6 +211,7 @@ export async function establishApiSandboxSpawner(input: {
           sandboxGroupId: input.sandboxGroupId,
           expectedEpoch: input.expectedEpoch,
           rematerializationId: rematerialization?.id ?? null,
+          ...(created.providerContinuity ? { continuityRecovery: created.providerContinuity } : {}),
           instanceId: created.instanceId,
           resumeBackendId: created.backendId,
           resumeState,
@@ -310,6 +287,7 @@ export async function establishApiSandboxSpawner(input: {
     await verifySandboxExecReadiness(established);
     if (
       rematerialization &&
+      !established.providerContinuity &&
       established.restoredArchive?.revision !== rematerialization.selectedRevision
     ) {
       throw new WorkspaceArchiveIntegrityError(
@@ -330,19 +308,21 @@ export async function establishApiSandboxSpawner(input: {
       dataPlaneUrl: input.dataPlaneUrl,
       resumeBackendId: established.backendId,
       resumeState,
-      ...(rematerialization
-        ? {
-            rematerialization: {
-              id: rematerialization.id,
-              verifiedRevision: rematerialization.selectedRevision,
-            },
-          }
-        : {}),
+      ...(established.providerContinuity
+        ? { continuityRecovery: established.providerContinuity }
+        : rematerialization
+          ? {
+              rematerialization: {
+                id: rematerialization.id,
+                verifiedRevision: rematerialization.selectedRevision,
+              },
+            }
+          : {}),
       leaseTtlMs: input.settings.sandboxLeaseTtlMs,
     });
     if (!committed.committed || !committed.lease) {
       const terminated = await terminateCreated(established);
-      if (terminated && rematerialization) {
+      if (terminated && rematerialization && !established.providerContinuity) {
         await failSandboxRematerialization(input.db, {
           accountId: input.accountId,
           workspaceId: input.workspaceId,
@@ -367,18 +347,25 @@ export async function establishApiSandboxSpawner(input: {
     if (error instanceof SandboxLeaseSupersededError) throw error;
     const terminated = await terminateCreated(established);
     if (terminated) {
-      if (rematerialization) {
+      const continuityUnavailable = error instanceof SandboxProviderContinuityUnavailableError;
+      if (rematerialization && !established?.providerContinuity) {
         await failSandboxRematerialization(input.db, {
           accountId: input.accountId,
           workspaceId: input.workspaceId,
           sandboxGroupId: input.sandboxGroupId,
           expectedEpoch: input.expectedEpoch,
           rematerializationId: rematerialization.id,
-          failureCode:
-            error instanceof WorkspaceArchiveIntegrityError
+          failureCode: continuityUnavailable
+            ? "provider_continuity_unavailable"
+            : error instanceof WorkspaceArchiveIntegrityError
               ? error.code
               : "sandbox_rematerialization_failed",
-          retryable: error instanceof WorkspaceArchiveIntegrityError ? error.retryable : true,
+          retryable: continuityUnavailable
+            ? error.retryable
+            : error instanceof WorkspaceArchiveIntegrityError
+              ? error.retryable
+              : true,
+          ...(continuityUnavailable ? { discardContinuity: true } : {}),
         });
       } else {
         await failWarmingToCold(input.db, {
@@ -386,6 +373,7 @@ export async function establishApiSandboxSpawner(input: {
           workspaceId: input.workspaceId,
           sandboxGroupId: input.sandboxGroupId,
           expectedEpoch: input.expectedEpoch,
+          ...(continuityUnavailable ? { discardContinuity: true } : {}),
         });
       }
     }

package/src/sandbox/viewer.ts

@@ -9,7 +9,8 @@
 //   attach  -> acquireLease(kind:'viewer') under FOR UPDATE + cold->warming CAS.
 //              spawner role  -> establish the box in-process + commitWarmingToWarm.
 //              attached/rearmed -> the holder alone keeps the box warm.
-//              fenced -> release + surface a 409 (a newer epoch re-established it).
+//              lifecycle claim -> wait for settlement; true epoch supersession
+//              -> release + surface a 409.
 //   heartbeat -> heartbeatLeaseHolder (epoch-fenced) refreshes the holder TTL.
 //   detach  -> releaseLeaseHolder (idempotent); the reaper (P1.3) stop()s the
 //              box at refcount 0 past the drain grace.
@@ -24,7 +25,7 @@ import {
   hasGitCredentialRepositorySelection,
   hasGitHubRepositorySelection,
   resolveStreamTokenSecret,
-  sandboxArchiveCaptureTimeoutMs,
+  sandboxLifecycleTransitionWaitMs,
   stableSandboxEnvironmentForRun,
 } from "@opengeni/config";
 import type { Settings } from "@opengeni/config";
@@ -62,6 +63,8 @@ import {
   exposeStreamPort,
   desktopCapableBackend,
   NatsControlRpc,
+  SandboxResumeIdentityMismatchError,
+  SandboxResumeIdentityUnavailableError,
   SelfhostedSandboxClient,
   TERMINAL_STREAM_PORT,
   DisplayStackUnsupportedError,
@@ -179,7 +182,14 @@ export async function sessionAttachEnvironment(
  */
 export async function attachViewer(
   services: ViewerServices,
-  input: { accountId: string; workspaceId: string; session: Session; viewerId?: string },
+  input: {
+    accountId: string;
+    workspaceId: string;
+    session: Session;
+    viewerId?: string;
+    /** Cancel lifecycle waiting when the originating HTTP request disconnects. */
+    waitSignal?: AbortSignal;
+  },
 ): Promise<ViewerAttachResult> {
   const { db, settings } = services;
   const { accountId, workspaceId, session } = input;
@@ -211,7 +221,8 @@ export async function attachViewer(
       os: session.sandboxOs,
       leaseTtlMs,
       warmingLeaseTtlMs: settings.sandboxWarmingTimeoutMs,
-      captureWaitMs: sandboxArchiveCaptureTimeoutMs(settings),
+      captureWaitMs: sandboxLifecycleTransitionWaitMs(settings),
+      ...(input.waitSignal ? { waitSignal: input.waitSignal } : {}),
     });
   } catch (error) {
     if (error instanceof SandboxViewerAdmissionBlockedError) {
@@ -237,7 +248,10 @@ export async function attachViewer(
   if (acquired.role === "fenced") {
     await release();
     throw new HTTPException(409, {
-      message: `sandbox lease superseded (epoch ${acquired.lease.leaseEpoch}); re-read capabilities and re-attach`,
+      message:
+        acquired.reason === "superseded"
+          ? `sandbox lease superseded (epoch ${acquired.lease.leaseEpoch}); re-read capabilities and re-attach`
+          : `sandbox lifecycle transition in progress (${acquired.reason}, epoch ${acquired.lease.leaseEpoch}, backend ${acquired.lease.backend}, instance ${acquired.lease.instanceId ?? "none"}); retry`,
     });
   }
 
@@ -362,6 +376,12 @@ export async function attachViewer(
           message: `sandbox instance was lost; retry to restore it`,
         });
       }
+      if (
+        error instanceof SandboxResumeIdentityMismatchError ||
+        error instanceof SandboxResumeIdentityUnavailableError
+      ) {
+        throw new HTTPException(409, { message: error.message });
+      }
       throw error;
     } finally {
       await dropEstablishedHandle(observed);
@@ -625,7 +645,7 @@ export type MintDesktopStreamInput = {
   session: Session;
   /** The viewer holder id the scoped token is minted for. */
   viewerId: string;
-  /** The live lease (must be warm/draining — the box is up). A selfhosted-active
+  /** The live lease (must be holder-confirmed warm). A selfhosted-active
    *  session may have no Modal group lease; omit and the selfhosted branch handles it. */
   lease?: LeaseSnapshot;
   /** The epoch the CALLER last observed the URL minted under. When the live
@@ -717,7 +737,7 @@ export async function mintDesktopStream(
 
   // GATE 2: the box must be live (the handshake never spins one up — a cold box
   // returns lease_cold; the viewer-attach path warms it first, then mints).
-  if (!lease || (lease.liveness !== "warm" && lease.liveness !== "draining")) {
+  if (!lease || lease.liveness !== "warm") {
     return null;
   }
 
@@ -879,7 +899,7 @@ export type MintTerminalStreamInput = {
   session: Session;
   /** The viewer holder / principal id the scoped token is minted for. */
   viewerId: string;
-  /** The live lease (must be warm/draining — the box is up). A selfhosted-active
+  /** The live lease (must be holder-confirmed warm). A selfhosted-active
    *  session may have no Modal group lease; omit and the selfhosted branch handles it. */
   lease?: LeaseSnapshot;
   /** Test seam: override how the box is re-established by id (see
@@ -947,7 +967,7 @@ export async function mintTerminalStream(
   }
 
   // GATE 2: the box must be live (the handshake never spins one up).
-  if (!lease || (lease.liveness !== "warm" && lease.liveness !== "draining")) {
+  if (!lease || lease.liveness !== "warm") {
     return null;
   }
 

package/src/temporal-schedule-cleanup.ts

@@ -0,0 +1,135 @@
+import { randomUUID } from "node:crypto";
+import {
+  claimTemporalScheduleCleanups,
+  settleTemporalScheduleCleanup,
+  type Database,
+  type TemporalScheduleCleanupClaim,
+} from "@opengeni/db";
+import type { Observability } from "@opengeni/observability";
+
+const CLEANUP_BATCH_SIZE = 32;
+const CLEANUP_CLAIM_SECONDS = 15;
+const CLEANUP_POLL_INTERVAL_MS = 1_000;
+
+type TemporalScheduleCleanupDependencies = {
+  db: Database;
+  deleteSchedule: (temporalScheduleId: string) => Promise<void>;
+  observability?: Pick<Observability, "info" | "warn" | "error">;
+};
+
+export type TemporalScheduleCleanupBatchResult = {
+  claimed: number;
+  deleted: number;
+  failed: number;
+  stale: number;
+};
+
+function cleanupError(error: unknown): string {
+  if (error instanceof Error) {
+    return `${error.name}: ${error.message}`.slice(0, 2000);
+  }
+  return String(error).slice(0, 2000);
+}
+
+/** Settle exact claims concurrently; every external delete is idempotent. */
+export async function processTemporalScheduleCleanupClaims(
+  deps: TemporalScheduleCleanupDependencies,
+  claims: readonly TemporalScheduleCleanupClaim[],
+): Promise<TemporalScheduleCleanupBatchResult> {
+  const results = await Promise.all(
+    claims.map(async (claim): Promise<"deleted" | "failed" | "stale"> => {
+      try {
+        await deps.deleteSchedule(claim.temporalScheduleId);
+        const settled = await settleTemporalScheduleCleanup(deps.db, {
+          id: claim.id,
+          claimId: claim.claimId,
+        });
+        return settled ? "deleted" : "stale";
+      } catch (error) {
+        const message = cleanupError(error);
+        try {
+          const settled = await settleTemporalScheduleCleanup(deps.db, {
+            id: claim.id,
+            claimId: claim.claimId,
+            error: message,
+          });
+          if (!settled) return "stale";
+        } catch (settlementError) {
+          deps.observability?.error("Temporal schedule cleanup settlement failed", {
+            cleanupId: claim.id,
+            temporalScheduleId: claim.temporalScheduleId,
+            attemptCount: claim.attemptCount,
+            error: cleanupError(settlementError),
+          });
+        }
+        deps.observability?.warn("Temporal schedule cleanup will retry", {
+          cleanupId: claim.id,
+          temporalScheduleId: claim.temporalScheduleId,
+          attemptCount: claim.attemptCount,
+          error: message,
+        });
+        return "failed";
+      }
+    }),
+  );
+
+  return {
+    claimed: claims.length,
+    deleted: results.filter((result) => result === "deleted").length,
+    failed: results.filter((result) => result === "failed").length,
+    stale: results.filter((result) => result === "stale").length,
+  };
+}
+
+/** Claim and process one globally bounded batch. */
+export async function drainTemporalScheduleCleanupOutbox(
+  deps: TemporalScheduleCleanupDependencies,
+): Promise<TemporalScheduleCleanupBatchResult> {
+  const claims = await claimTemporalScheduleCleanups(deps.db, {
+    claimId: randomUUID(),
+    limit: CLEANUP_BATCH_SIZE,
+    claimSeconds: CLEANUP_CLAIM_SECONDS,
+  });
+  return await processTemporalScheduleCleanupClaims(deps, claims);
+}
+
+/**
+ * Start one non-overlapping reconciliation loop. PostgreSQL claims coordinate
+ * replicas; an API crash merely delays the exact row until its short claim
+ * expires.
+ */
+export function startTemporalScheduleCleanupPump(
+  deps: TemporalScheduleCleanupDependencies,
+  options: { intervalMs?: number } = {},
+): () => Promise<void> {
+  const intervalMs = Math.max(100, options.intervalMs ?? CLEANUP_POLL_INTERVAL_MS);
+  let stopped = false;
+  let running: Promise<void> | undefined;
+
+  const tick = (): void => {
+    if (stopped || running) return;
+    running = drainTemporalScheduleCleanupOutbox(deps)
+      .then((result) => {
+        if (result.claimed > 0) {
+          deps.observability?.info("Temporal schedule cleanup batch settled", result);
+        }
+      })
+      .catch((error: unknown) => {
+        deps.observability?.error("Temporal schedule cleanup claim failed", {
+          error: cleanupError(error),
+        });
+      })
+      .finally(() => {
+        running = undefined;
+      });
+  };
+
+  const timer = setInterval(tick, intervalMs);
+  queueMicrotask(tick);
+
+  return async () => {
+    stopped = true;
+    clearInterval(timer);
+    await running;
+  };
+}