@opengeni/api-router 2.5.0 → 2.6.4

package/src/sandbox/metrics-ingestion.ts

@@ -28,7 +28,11 @@
 // Both consumers are BEST-EFFORT and fail-soft: a decode/DB error for one message
 // is logged + swallowed (the bus subscription already swallows handler throws) so
 // a metrics blip / a display-refresh write failure never tears down the consumer,
-// back-pressures the agent, or breaks its connect.
+// back-pressures the agent, or breaks its connect. Event ingestion drains the NATS
+// subscription immediately into exact-subject queues: different runner connections
+// progress concurrently, one runner's ordering is preserved, and consecutive queued
+// heartbeats collapse latest-wins. A delayed heartbeat backlog therefore cannot keep
+// renewing a dead runner's short connection lease one stale message at a time.
 
 import {
   clearEnrollmentWentOffline,
@@ -74,6 +78,141 @@ export const AGENT_EVENTS_SUBJECT = "agent.*.*.connection.*.events";
 /** The wildcard subject the agent publishes its connect Hello on. */
 export const AGENT_HELLO_SUBJECT = "agent.*.*.connection.*.hello";
 
+type QueuedAgentEvent = {
+  payload: Uint8Array;
+  subject: string;
+  heartbeat: boolean;
+};
+
+type AgentEventQueueState = {
+  scheduled: boolean;
+  pending: QueuedAgentEvent[];
+};
+
+type ScheduledAgentEventQueue = {
+  subject: string;
+  state: AgentEventQueueState;
+};
+
+export type AgentEventIngestionScheduler = {
+  enqueue: (payload: Uint8Array, subject: string) => void;
+  whenIdle: () => Promise<void>;
+  close: () => void;
+};
+
+function payloadIsHeartbeat(payload: Uint8Array): boolean {
+  try {
+    return AgentEvent.decode(payload).event?.$case === "heartbeat";
+  } catch {
+    // Preserve the normal decoder/logging path for malformed events. They are
+    // barriers rather than heartbeat-coalescing candidates.
+    return false;
+  }
+}
+
+/**
+ * Drain agent events without making every Connected Machine wait behind one
+ * deployment-wide serial DB queue. Each exact process subject retains FIFO order;
+ * separate subjects drain concurrently. Consecutive pending heartbeats for one
+ * subject are latest-wins because they are point-in-time liveness/metrics samples,
+ * while GoingOffline and update-progress events remain ordered barriers.
+ */
+export function createAgentEventIngestionScheduler(
+  handler: (payload: Uint8Array, subject: string) => void | Promise<void>,
+  options: { maxConcurrentSubjects?: number; onHeartbeatCoalesced?: () => void } = {},
+): AgentEventIngestionScheduler {
+  const maxConcurrentSubjects = options.maxConcurrentSubjects ?? 32;
+  if (!Number.isSafeInteger(maxConcurrentSubjects) || maxConcurrentSubjects <= 0) {
+    throw new RangeError("maxConcurrentSubjects must be a positive safe integer");
+  }
+
+  const states = new Map<string, AgentEventQueueState>();
+  const readySubjects: ScheduledAgentEventQueue[] = [];
+  const idleWaiters = new Set<() => void>();
+  let accepting = true;
+  let activeSubjects = 0;
+
+  const resolveIdleWaiters = (): void => {
+    if (states.size !== 0) return;
+    for (const resolve of idleWaiters) resolve();
+    idleWaiters.clear();
+  };
+
+  async function drain(subject: string, state: AgentEventQueueState): Promise<void> {
+    while (true) {
+      const event = state.pending.shift();
+      if (!event) break;
+      try {
+        await handler(event.payload, event.subject);
+      } catch {
+        // Agent-event ingestion is best-effort. One bad event must not strand the
+        // exact-subject queue or prevent later heartbeats from restoring liveness.
+      }
+    }
+
+    activeSubjects -= 1;
+    state.scheduled = false;
+    if (state.pending.length > 0) {
+      schedule(subject, state);
+    } else {
+      if (states.get(subject) === state) states.delete(subject);
+      resolveIdleWaiters();
+    }
+    pump();
+  }
+
+  function pump(): void {
+    while (activeSubjects < maxConcurrentSubjects) {
+      const next = readySubjects.shift();
+      if (!next) return;
+      if (states.get(next.subject) !== next.state || !next.state.scheduled) continue;
+      activeSubjects += 1;
+      void drain(next.subject, next.state);
+    }
+  }
+
+  function schedule(subject: string, state: AgentEventQueueState): void {
+    if (state.scheduled) return;
+    state.scheduled = true;
+    readySubjects.push({ subject, state });
+    pump();
+  }
+
+  return {
+    enqueue(payload, subject) {
+      if (!accepting) return;
+      const heartbeat = payloadIsHeartbeat(payload);
+      let state = states.get(subject);
+      if (!state) {
+        state = { scheduled: false, pending: [] };
+        states.set(subject, state);
+      }
+
+      const queued = { payload, subject, heartbeat };
+      const lastIndex = state.pending.length - 1;
+      if (heartbeat && lastIndex >= 0 && state.pending[lastIndex]!.heartbeat) {
+        state.pending[lastIndex] = queued;
+        options.onHeartbeatCoalesced?.();
+      } else {
+        state.pending.push(queued);
+      }
+
+      schedule(subject, state);
+    },
+    async whenIdle() {
+      if (states.size === 0) return;
+      await new Promise<void>((resolve) => idleWaiters.add(resolve));
+    },
+    close() {
+      accepting = false;
+      for (const state of states.values()) {
+        state.pending.length = 0;
+      }
+      resolveIdleWaiters();
+    },
+  };
+}
+
 /**
  * Parse `agent.<ws>.<id>.connection.<instance>.<tail>` into its exact authority,
  * expected tail token. Returns null for a subject that does not match the shape
@@ -521,9 +660,24 @@ export function startMetricsIngestion(deps: {
   bus: EventBus;
   observability?: Observability;
 }): () => void {
-  return deps.bus.subscribeAgentEvents(AGENT_EVENTS_SUBJECT, (payload, subject) =>
-    handleAgentEventPayload(deps.db, deps.observability, payload, subject, deps.bus),
+  const scheduler = createAgentEventIngestionScheduler(
+    (payload, subject) =>
+      handleAgentEventPayload(deps.db, deps.observability, payload, subject, deps.bus),
+    {
+      onHeartbeatCoalesced: () =>
+        deps.observability?.incrementCounter({
+          name: "opengeni_machine_heartbeat_coalesced_total",
+          help: "Connected Machine heartbeats collapsed while an exact runner event queue was busy.",
+        }),
+    },
   );
+  const unsubscribe = deps.bus.subscribeAgentEvents(AGENT_EVENTS_SUBJECT, (payload, subject) => {
+    scheduler.enqueue(payload, subject);
+  });
+  return () => {
+    unsubscribe();
+    scheduler.close();
+  };
 }
 
 // ── Connect-Hello display refresh ─────────────────────────────────────────────

package/src/sandbox/viewer.ts

@@ -81,6 +81,8 @@ import {
   type NatsRequestConnection,
 } from "@opengeni/runtime/sandbox";
 import {
+  managedSessionGroupBackend,
+  managedSessionGroupOs,
   providerSettingsForSessionSandboxRuntime,
   relayConfigFromSettings,
   resolveSessionSandboxRuntime,
@@ -246,7 +248,24 @@ export async function attachViewer(
   },
 ): Promise<ViewerAttachResult> {
   const { db, settings } = services;
-  const { accountId, workspaceId, session } = input;
+  const { accountId, workspaceId } = input;
+  const groupBackend = managedSessionGroupBackend(
+    settings.sandboxBackend,
+    input.session.sandboxBackend,
+  );
+  if (!groupBackend) {
+    throw new HTTPException(409, {
+      message: "session has no managed sandbox group to attach",
+    });
+  }
+  const session =
+    groupBackend === input.session.sandboxBackend
+      ? input.session
+      : {
+          ...input.session,
+          sandboxBackend: groupBackend,
+          sandboxOs: managedSessionGroupOs(input.session.sandboxBackend, input.session.sandboxOs),
+        };
   const viewerId = input.viewerId ?? crypto.randomUUID();
   const attachSubjectId = claimableSubjectId(input.viewerSubjectId ?? null);
   const attachAuthorityEpoch = attachSubjectId

package/src/workspace-delete-observability.ts

@@ -0,0 +1,75 @@
+import type { WorkspaceDeleteObservation, WorkspaceDeleteObserver } from "@opengeni/db";
+import type { Observability } from "@opengeni/observability";
+
+const DURATION_BUCKETS = [
+  0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120, 300, 600, 1_200, 2_400,
+  3_600,
+];
+const INVENTORY_BUCKETS = [0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1_000, 5_000, 10_000];
+
+function recordWorkspaceDeleteObservation(
+  observability: Observability,
+  identity: { accountId: string; workspaceId: string },
+  observation: WorkspaceDeleteObservation,
+): void {
+  const labels = { phase: observation.phase, outcome: observation.outcome };
+  observability.observeHistogram({
+    name: "opengeni_workspace_delete_phase_seconds",
+    help: "Workspace deletion transaction and bounded internal phase duration in seconds.",
+    labels,
+    buckets: DURATION_BUCKETS,
+    value: Math.max(0, observation.durationSeconds),
+  });
+  if (observation.phase === "transaction") {
+    observability.incrementCounter({
+      name: "opengeni_workspace_delete_attempts_total",
+      help: "Workspace deletion attempts by terminal database outcome.",
+      labels: { outcome: observation.outcome },
+    });
+  }
+  for (const [kind, count] of Object.entries(observation.inventory ?? {})) {
+    observability.observeHistogram({
+      name: "opengeni_workspace_delete_inventory_rows",
+      help: "Rows or live owners observed by workspace deletion preflight inventory class.",
+      labels: { kind },
+      buckets: INVENTORY_BUCKETS,
+      value: Math.max(0, Math.floor(count ?? 0)),
+    });
+  }
+  observability.info("Workspace deletion phase", {
+    accountId: identity.accountId,
+    workspaceId: identity.workspaceId,
+    phase: observation.phase,
+    outcome: observation.outcome,
+    durationSeconds: observation.durationSeconds,
+    inventoryJson: JSON.stringify(observation.inventory ?? {}),
+  });
+}
+
+/**
+ * Retain exact workspace deletion timing and inventory without putting tenant
+ * identifiers in metric labels. Observer failures never change deletion truth.
+ */
+export function workspaceDeleteObserver(
+  observability: Observability | null | undefined,
+  identity: { accountId: string; workspaceId: string },
+): WorkspaceDeleteObserver | undefined {
+  if (!observability) return undefined;
+  return {
+    onPhase: (observation) => {
+      try {
+        recordWorkspaceDeleteObservation(observability, identity, observation);
+      } catch {
+        try {
+          observability.incrementCounter({
+            name: "opengeni_observability_observer_errors_total",
+            help: "Observability observer failures isolated from product execution.",
+            labels: { observer: "workspace_delete" },
+          });
+        } catch {
+          // The metrics registry itself is unhealthy. Deletion remains authoritative.
+        }
+      }
+    },
+  };
+}