@opengeni/api-router 2.5.0 → 2.6.4-canary.0

package/src/sandbox/channel-a.ts

@@ -86,6 +86,8 @@ import {
   type RoutingSandboxSession,
 } from "@opengeni/runtime/sandbox";
 import {
+  managedSessionGroupBackend,
+  managedSessionGroupOs,
   providerSettingsForSessionSandboxRuntime,
   relayConfigFromSettings,
   resolveSessionSandboxRuntime,
@@ -643,15 +645,19 @@ async function withChannelAOperation<T>(
         // A machine-home session with no selected machine uses the deployment's
         // managed group box, exactly like the worker turn path. Keep the durable
         // home label honest; only this request's effective backend changes.
-        if (settings.sandboxBackend !== "none" && settings.sandboxBackend !== "selfhosted") {
+        const groupBackend = managedSessionGroupBackend(
+          settings.sandboxBackend,
+          session.sandboxBackend,
+        );
+        if (groupBackend) {
           return await withChannelAOperation(
             services,
             {
               ...ctx,
               session: {
                 ...session,
-                sandboxBackend: settings.sandboxBackend,
-                sandboxOs: "linux",
+                sandboxBackend: groupBackend,
+                sandboxOs: managedSessionGroupOs(session.sandboxBackend, session.sandboxOs),
               },
             },
             readOnly,
@@ -659,7 +665,7 @@ async function withChannelAOperation<T>(
           );
         }
         throw new HTTPException(409, {
-          message: "machine-home session has no active Connected Machine",
+          message: "machine-home session has no active Connected Machine or managed sandbox",
         });
       }
       const sandbox = await getSandbox(

package/src/sandbox/machines.ts

@@ -9,8 +9,9 @@
 //     heartbeat), projected to the contract's MetricSample;
 //   * sharedSessionCount — the lease refcount (how many sessions share this one
 //     whole machine, the maxSandboxes:1 disclosure).
-// PLUS, when a session context is supplied, the session's synthetic Modal group
-// box (isSessionGroup:true) + the active-sandbox pointer (activeSandboxId/Epoch).
+// PLUS, when a session context is supplied, the session's synthetic managed
+// group box (isSessionGroup:true) + the active-sandbox pointer
+// (activeSandboxId/Epoch).
 //
 // This is workspace-scoped (perm enrollments:read) and flag-gated upstream
 // (sandboxSelfhostedEnabled). It deliberately does NOT depend on a FleetContext
@@ -32,6 +33,7 @@ import {
   type MachineMetricsRow,
 } from "@opengeni/db";
 import { MachineView, MetricSample, type MachinesResponse } from "@opengeni/contracts";
+import { managedSessionGroupBackend } from "@opengeni/core";
 import { selfhostedHeartbeatLiveness } from "@opengeni/runtime/sandbox";
 
 export type MachinesServices = {
@@ -244,10 +246,15 @@ export async function listMachines(
   }
 
   const machines: MachineView[] = [];
+  const groupBackend = session
+    ? managedSessionGroupBackend(services.settings.sandboxBackend, session.sandboxBackend)
+    : null;
 
-  // The session's own group box (synthetic): the default/home sandbox a null
-  // active pointer routes to. A backend:none session has no home box.
-  if (session && session.sandboxBackend !== "none") {
+  // The session's own managed group box (synthetic): the sandbox a null active
+  // pointer routes to. A machine-home session uses the deployment's managed
+  // backend only after an explicit switch; none/selfhosted-only deployments do
+  // not invent a group they cannot establish.
+  if (session && groupBackend) {
     const groupActive = activeSandboxId === null;
     const groupLease = await readLease(db, workspaceId, session.sandboxGroupId);
     machines.push(
@@ -255,7 +262,7 @@ export async function listMachines(
         sandboxId: session.sandboxGroupId,
         enrollmentId: null,
         name: "session sandbox",
-        kind: session.sandboxBackend === "selfhosted" ? "selfhosted" : "modal",
+        kind: groupBackend === "opensandbox" ? "opensandbox" : "modal",
         state: "online",
         active: groupActive,
         isSessionGroup: true,

package/src/sandbox/metrics-ingestion.ts

@@ -28,7 +28,11 @@
 // Both consumers are BEST-EFFORT and fail-soft: a decode/DB error for one message
 // is logged + swallowed (the bus subscription already swallows handler throws) so
 // a metrics blip / a display-refresh write failure never tears down the consumer,
-// back-pressures the agent, or breaks its connect.
+// back-pressures the agent, or breaks its connect. Event ingestion drains the NATS
+// subscription immediately into exact-subject queues: different runner connections
+// progress concurrently, one runner's ordering is preserved, and consecutive queued
+// heartbeats collapse latest-wins. A delayed heartbeat backlog therefore cannot keep
+// renewing a dead runner's short connection lease one stale message at a time.
 
 import {
   clearEnrollmentWentOffline,
@@ -74,6 +78,141 @@ export const AGENT_EVENTS_SUBJECT = "agent.*.*.connection.*.events";
 /** The wildcard subject the agent publishes its connect Hello on. */
 export const AGENT_HELLO_SUBJECT = "agent.*.*.connection.*.hello";
 
+type QueuedAgentEvent = {
+  payload: Uint8Array;
+  subject: string;
+  heartbeat: boolean;
+};
+
+type AgentEventQueueState = {
+  scheduled: boolean;
+  pending: QueuedAgentEvent[];
+};
+
+type ScheduledAgentEventQueue = {
+  subject: string;
+  state: AgentEventQueueState;
+};
+
+export type AgentEventIngestionScheduler = {
+  enqueue: (payload: Uint8Array, subject: string) => void;
+  whenIdle: () => Promise<void>;
+  close: () => void;
+};
+
+function payloadIsHeartbeat(payload: Uint8Array): boolean {
+  try {
+    return AgentEvent.decode(payload).event?.$case === "heartbeat";
+  } catch {
+    // Preserve the normal decoder/logging path for malformed events. They are
+    // barriers rather than heartbeat-coalescing candidates.
+    return false;
+  }
+}
+
+/**
+ * Drain agent events without making every Connected Machine wait behind one
+ * deployment-wide serial DB queue. Each exact process subject retains FIFO order;
+ * separate subjects drain concurrently. Consecutive pending heartbeats for one
+ * subject are latest-wins because they are point-in-time liveness/metrics samples,
+ * while GoingOffline and update-progress events remain ordered barriers.
+ */
+export function createAgentEventIngestionScheduler(
+  handler: (payload: Uint8Array, subject: string) => void | Promise<void>,
+  options: { maxConcurrentSubjects?: number; onHeartbeatCoalesced?: () => void } = {},
+): AgentEventIngestionScheduler {
+  const maxConcurrentSubjects = options.maxConcurrentSubjects ?? 32;
+  if (!Number.isSafeInteger(maxConcurrentSubjects) || maxConcurrentSubjects <= 0) {
+    throw new RangeError("maxConcurrentSubjects must be a positive safe integer");
+  }
+
+  const states = new Map<string, AgentEventQueueState>();
+  const readySubjects: ScheduledAgentEventQueue[] = [];
+  const idleWaiters = new Set<() => void>();
+  let accepting = true;
+  let activeSubjects = 0;
+
+  const resolveIdleWaiters = (): void => {
+    if (states.size !== 0) return;
+    for (const resolve of idleWaiters) resolve();
+    idleWaiters.clear();
+  };
+
+  async function drain(subject: string, state: AgentEventQueueState): Promise<void> {
+    while (true) {
+      const event = state.pending.shift();
+      if (!event) break;
+      try {
+        await handler(event.payload, event.subject);
+      } catch {
+        // Agent-event ingestion is best-effort. One bad event must not strand the
+        // exact-subject queue or prevent later heartbeats from restoring liveness.
+      }
+    }
+
+    activeSubjects -= 1;
+    state.scheduled = false;
+    if (state.pending.length > 0) {
+      schedule(subject, state);
+    } else {
+      if (states.get(subject) === state) states.delete(subject);
+      resolveIdleWaiters();
+    }
+    pump();
+  }
+
+  function pump(): void {
+    while (activeSubjects < maxConcurrentSubjects) {
+      const next = readySubjects.shift();
+      if (!next) return;
+      if (states.get(next.subject) !== next.state || !next.state.scheduled) continue;
+      activeSubjects += 1;
+      void drain(next.subject, next.state);
+    }
+  }
+
+  function schedule(subject: string, state: AgentEventQueueState): void {
+    if (state.scheduled) return;
+    state.scheduled = true;
+    readySubjects.push({ subject, state });
+    pump();
+  }
+
+  return {
+    enqueue(payload, subject) {
+      if (!accepting) return;
+      const heartbeat = payloadIsHeartbeat(payload);
+      let state = states.get(subject);
+      if (!state) {
+        state = { scheduled: false, pending: [] };
+        states.set(subject, state);
+      }
+
+      const queued = { payload, subject, heartbeat };
+      const lastIndex = state.pending.length - 1;
+      if (heartbeat && lastIndex >= 0 && state.pending[lastIndex]!.heartbeat) {
+        state.pending[lastIndex] = queued;
+        options.onHeartbeatCoalesced?.();
+      } else {
+        state.pending.push(queued);
+      }
+
+      schedule(subject, state);
+    },
+    async whenIdle() {
+      if (states.size === 0) return;
+      await new Promise<void>((resolve) => idleWaiters.add(resolve));
+    },
+    close() {
+      accepting = false;
+      for (const state of states.values()) {
+        state.pending.length = 0;
+      }
+      resolveIdleWaiters();
+    },
+  };
+}
+
 /**
  * Parse `agent.<ws>.<id>.connection.<instance>.<tail>` into its exact authority,
  * expected tail token. Returns null for a subject that does not match the shape
@@ -521,9 +660,24 @@ export function startMetricsIngestion(deps: {
   bus: EventBus;
   observability?: Observability;
 }): () => void {
-  return deps.bus.subscribeAgentEvents(AGENT_EVENTS_SUBJECT, (payload, subject) =>
-    handleAgentEventPayload(deps.db, deps.observability, payload, subject, deps.bus),
+  const scheduler = createAgentEventIngestionScheduler(
+    (payload, subject) =>
+      handleAgentEventPayload(deps.db, deps.observability, payload, subject, deps.bus),
+    {
+      onHeartbeatCoalesced: () =>
+        deps.observability?.incrementCounter({
+          name: "opengeni_machine_heartbeat_coalesced_total",
+          help: "Connected Machine heartbeats collapsed while an exact runner event queue was busy.",
+        }),
+    },
   );
+  const unsubscribe = deps.bus.subscribeAgentEvents(AGENT_EVENTS_SUBJECT, (payload, subject) => {
+    scheduler.enqueue(payload, subject);
+  });
+  return () => {
+    unsubscribe();
+    scheduler.close();
+  };
 }
 
 // ── Connect-Hello display refresh ─────────────────────────────────────────────

package/src/sandbox/viewer.ts

@@ -81,6 +81,8 @@ import {
   type NatsRequestConnection,
 } from "@opengeni/runtime/sandbox";
 import {
+  managedSessionGroupBackend,
+  managedSessionGroupOs,
   providerSettingsForSessionSandboxRuntime,
   relayConfigFromSettings,
   resolveSessionSandboxRuntime,
@@ -246,7 +248,24 @@ export async function attachViewer(
   },
 ): Promise<ViewerAttachResult> {
   const { db, settings } = services;
-  const { accountId, workspaceId, session } = input;
+  const { accountId, workspaceId } = input;
+  const groupBackend = managedSessionGroupBackend(
+    settings.sandboxBackend,
+    input.session.sandboxBackend,
+  );
+  if (!groupBackend) {
+    throw new HTTPException(409, {
+      message: "session has no managed sandbox group to attach",
+    });
+  }
+  const session =
+    groupBackend === input.session.sandboxBackend
+      ? input.session
+      : {
+          ...input.session,
+          sandboxBackend: groupBackend,
+          sandboxOs: managedSessionGroupOs(input.session.sandboxBackend, input.session.sandboxOs),
+        };
   const viewerId = input.viewerId ?? crypto.randomUUID();
   const attachSubjectId = claimableSubjectId(input.viewerSubjectId ?? null);
   const attachAuthorityEpoch = attachSubjectId

package/src/workspace-delete-observability.ts

@@ -0,0 +1,75 @@
+import type { WorkspaceDeleteObservation, WorkspaceDeleteObserver } from "@opengeni/db";
+import type { Observability } from "@opengeni/observability";
+
+const DURATION_BUCKETS = [
+  0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300, 600, 1_200, 2_400,
+  3_600,
+];
+const INVENTORY_BUCKETS = [0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1_000, 5_000, 10_000];
+
+function recordWorkspaceDeleteObservation(
+  observability: Observability,
+  identity: { accountId: string; workspaceId: string },
+  observation: WorkspaceDeleteObservation,
+): void {
+  const labels = { phase: observation.phase, outcome: observation.outcome };
+  observability.observeHistogram({
+    name: "opengeni_workspace_delete_phase_seconds",
+    help: "Workspace deletion transaction and bounded internal phase duration in seconds.",
+    labels,
+    buckets: DURATION_BUCKETS,
+    value: Math.max(0, observation.durationSeconds),
+  });
+  if (observation.phase === "transaction") {
+    observability.incrementCounter({
+      name: "opengeni_workspace_delete_attempts_total",
+      help: "Workspace deletion attempts by terminal database outcome.",
+      labels: { outcome: observation.outcome },
+    });
+  }
+  for (const [kind, count] of Object.entries(observation.inventory ?? {})) {
+    observability.observeHistogram({
+      name: "opengeni_workspace_delete_inventory_rows",
+      help: "Rows or live owners observed by workspace deletion preflight inventory class.",
+      labels: { kind },
+      buckets: INVENTORY_BUCKETS,
+      value: Math.max(0, Math.floor(count ?? 0)),
+    });
+  }
+  observability.info("Workspace deletion phase", {
+    accountId: identity.accountId,
+    workspaceId: identity.workspaceId,
+    phase: observation.phase,
+    outcome: observation.outcome,
+    durationSeconds: observation.durationSeconds,
+    inventoryJson: JSON.stringify(observation.inventory ?? {}),
+  });
+}
+
+/**
+ * Retain exact workspace deletion timing and inventory without putting tenant
+ * identifiers in metric labels. Observer failures never change deletion truth.
+ */
+export function workspaceDeleteObserver(
+  observability: Observability | null | undefined,
+  identity: { accountId: string; workspaceId: string },
+): WorkspaceDeleteObserver | undefined {
+  if (!observability) return undefined;
+  return {
+    onPhase: (observation) => {
+      try {
+        recordWorkspaceDeleteObservation(observability, identity, observation);
+      } catch {
+        try {
+          observability.incrementCounter({
+            name: "opengeni_observability_observer_errors_total",
+            help: "Observability observer failures isolated from product execution.",
+            labels: { observer: "workspace_delete" },
+          });
+        } catch {
+          // The metrics registry itself is unhealthy. Deletion remains authoritative.
+        }
+      }
+    },
+  };
+}