@opengeni/api-router 0.30.3 → 2.2.0-canary.0

package/src/sandbox/machines.ts

@@ -1,10 +1,10 @@
 // apps/api/src/sandbox/machines.ts — the M10 Machines-DASHBOARD service (design
 // §10.7). Builds the `MachinesResponse` the dashboard renders: the workspace's
 // enrolled selfhosted machines, each enriched with
-//   * STATE — the M3 liveness (online/reconnecting/offline) overlaid with the
-//     enrollment-derived consent/display reasons (consent_required /
-//     display_unavailable) — a real ControlRpc ping (the subject IS the registry),
-//     reusing the M7 fleet probe;
+//   * STATE — heartbeat + goodbye liveness (online/offline) overlaid with the
+//     enrollment-derived display reason (display_unavailable). This list does
+//     not ControlRpc-ping; attach, capability negotiation, and fleet tools
+//     still probe when they need a live responder.
 //   * METRICS — the latest machine_metrics_latest row (or null before a first
 //     heartbeat), projected to the contract's MetricSample;
 //   * sharedSessionCount — the lease refcount (how many sessions share this one
@@ -21,6 +21,7 @@
 import type { Settings } from "@opengeni/config";
 import {
   getSession,
+  getLiveEnrollmentConnection,
   listEnrollments,
   listSandboxes,
   readActiveSandbox,
@@ -30,32 +31,116 @@ import {
   type EnrollmentRecord,
   type MachineMetricsRow,
 } from "@opengeni/db";
-import type { EventBus } from "@opengeni/events";
 import { MachineView, MetricSample, type MachinesResponse } from "@opengeni/contracts";
-import {
-  NatsControlRpc,
-  selfhostedLiveness,
-  SelfhostedSession,
-  type ControlRpc,
-  type NatsRequestConnection,
-} from "@opengeni/runtime/sandbox";
-import { relayConfigFromSettings } from "@opengeni/core";
+import { selfhostedHeartbeatLiveness } from "@opengeni/runtime/sandbox";
 
 export type MachinesServices = {
   db: Database;
   settings: Settings;
-  bus?: EventBus;
 };
 
-const PROBE_TIMEOUT_MS = 5_000;
+const ACTIVE_UPDATE_STATUSES = new Set([
+  "requested",
+  "accepted",
+  "waiting_for_idle",
+  "downloading",
+  "verifying",
+  "applying",
+  "restarting",
+]);
 
-function controlRpc(bus: EventBus | undefined): ControlRpc {
-  return new NatsControlRpc(async (): Promise<NatsRequestConnection | null> => {
-    if (!bus) {
-      return null;
-    }
-    return bus.getRequestConnection();
-  });
+function compareAgentVersions(left: string, right: string): number | null {
+  const parse = (value: string): { core: number[]; prerelease: string | null } | null => {
+    const match = /^(?:v)?(\d+)\.(\d+)\.(\d+)(?:-([0-9A-Za-z.-]+))?$/.exec(value.trim());
+    if (!match) return null;
+    return {
+      core: [Number(match[1]), Number(match[2]), Number(match[3])],
+      prerelease: match[4] ?? null,
+    };
+  };
+  const a = parse(left);
+  const b = parse(right);
+  if (!a || !b) return null;
+  for (let index = 0; index < 3; index += 1) {
+    const delta = a.core[index]! - b.core[index]!;
+    if (delta !== 0) return Math.sign(delta);
+  }
+  if (a.prerelease === b.prerelease) return 0;
+  if (a.prerelease === null) return 1;
+  if (b.prerelease === null) return -1;
+  return a.prerelease.localeCompare(b.prerelease);
+}
+
+function runtimeFor(settings: Settings, enrollment: EnrollmentRecord): MachineView["runtime"] {
+  const desiredVersion =
+    enrollment.agentUpdateChannel === "beta"
+      ? (settings.agentBetaVersion ?? settings.agentStableVersion)
+      : settings.agentStableVersion;
+  const update = enrollment.agentUpdate;
+  const activeUpdate = update ? ACTIVE_UPDATE_STATUSES.has(update.status) : false;
+  const order =
+    enrollment.agentVersion && desiredVersion
+      ? compareAgentVersions(enrollment.agentVersion, desiredVersion)
+      : null;
+  const versionState = activeUpdate
+    ? "updating"
+    : update?.status === "failed"
+      ? "update_failed"
+      : order === null
+        ? "unknown"
+        : order === 0
+          ? "current"
+          : order < 0
+            ? "outdated"
+            : "ahead";
+  const capability = (name: string): boolean => enrollment.agentCapabilities[name] === true;
+  return {
+    installedVersion: enrollment.agentVersion,
+    binarySha256: enrollment.agentBinarySha256,
+    updateChannel: enrollment.agentUpdateChannel,
+    desiredVersion,
+    versionState,
+    capabilities: {
+      exec: capability("exec"),
+      filesystem: capability("filesystem"),
+      git: capability("git"),
+      pty: capability("pty"),
+      desktop: capability("desktop"),
+      opStream: capability("opStream"),
+      browserBridge: capability("browserBridge"),
+      operationResourcePolicy: capability("operationResourcePolicy"),
+      operationCpuQuota: capability("operationCpuQuota"),
+    },
+    update,
+  };
+}
+
+function connectionAuthorityFor(
+  enrollment: EnrollmentRecord | null,
+  liveConnection: EnrollmentRecord | null,
+): MachineView["connectionAuthority"] {
+  if (!enrollment) {
+    return {
+      state: "not_applicable",
+      generation: 0,
+      supersededCount: 0,
+      leaseExpiresAt: null,
+      duplicateRunnerDeniedCount: 0,
+      duplicateRunnerDeniedAt: null,
+    };
+  }
+  return {
+    state: liveConnection
+      ? "active"
+      : enrollment.connectionInstanceId && enrollment.connectionLeaseExpiresAt
+        ? "expired"
+        : "unclaimed",
+    generation: enrollment.connectionGeneration,
+    supersededCount: Math.max(0, enrollment.connectionGeneration - 1),
+    leaseExpiresAt: enrollment.connectionLeaseExpiresAt,
+    duplicateRunnerDeniedCount: enrollment.connectionDuplicateDeniedCount,
+    duplicateRunnerDeniedAt: enrollment.connectionDuplicateDeniedAt,
+  };
 }
 
 /**
@@ -83,36 +168,8 @@ export function metricRowToSample(row: MachineMetricsRow): MetricSample {
   });
 }
 
-/** Probe an enrolled machine's liveness — a real ControlRpc ping mapped through
- *  `selfhostedLiveness` (the enrollment status/consent/display + lastSeenAt
- *  disambiguate a probe-miss into reconnecting vs offline). Mirrors the M7 fleet
- *  probe. A non-active enrollment is offline without a probe. */
-async function probeEnrollment(
-  services: MachinesServices,
-  workspaceId: string,
-  enrollment: EnrollmentRecord,
-): Promise<{
-  state: "online" | "reconnecting" | "offline";
-  consented: boolean;
-  hasDisplay: boolean;
-}> {
-  const { settings, bus } = services;
-  let probeResponded = false;
-  if (enrollment.status === "active") {
-    const session = new SelfhostedSession({
-      workspaceId,
-      agentId: enrollment.id,
-      controlRpc: controlRpc(bus),
-      relay: relayConfigFromSettings(settings),
-      timeoutMs: PROBE_TIMEOUT_MS,
-    });
-    try {
-      probeResponded = await session.ping();
-    } catch {
-      probeResponded = false;
-    }
-  }
-  const derived = selfhostedLiveness({
+function enrollmentLiveness(enrollment: EnrollmentRecord) {
+  return selfhostedHeartbeatLiveness({
     enrollment: {
       status: enrollment.status,
       exposure: enrollment.exposure,
@@ -122,16 +179,15 @@ async function probeEnrollment(
       wentOfflineAt: enrollment.wentOfflineAt,
       wentOfflineReason: enrollment.wentOfflineReason,
     },
-    probeResponded,
   });
-  return { state: derived.state, consented: derived.consented, hasDisplay: derived.hasDisplay };
 }
 
 /**
  * Resolve the dashboard STATE of a machine. State reflects REACHABILITY + the
  * VIEW plane only: an online machine with no display → `display_unavailable` (no
  * desktop stream, but compute — exec/fs/git/terminal — still works); otherwise
- * the liveness state (online/reconnecting/offline). It deliberately does NOT fold
+ * the liveness state (online/offline; list heartbeat never yields reconnecting).
+ * It deliberately does NOT fold
  * in screen-control consent: a displayed machine can be VIEWED (read-only) and
  * used for compute regardless of `allowScreenControl` — only INPUT (ComputerUse /
  * an interactive stream) needs that consent, which is a per-capability concern
@@ -163,7 +219,12 @@ function machineStateFor(
  */
 export async function listMachines(
   services: MachinesServices,
-  input: { workspaceId: string; sessionId?: string | null },
+  input: {
+    accountId?: string;
+    workspaceId: string;
+    subjectId?: string;
+    sessionId?: string | null;
+  },
 ): Promise<MachinesResponse> {
   const { db } = services;
   const { workspaceId } = input;
@@ -210,18 +271,59 @@ export async function listMachines(
         allowScreenControl: false,
         sharedSessionCount: 1,
         lastSeenAt: null,
+        connectionAuthority: connectionAuthorityFor(null, null),
+        runtime: null,
+        operationPolicy: null,
         metrics: null,
       }),
     );
   }
 
   // The workspace's enrolled selfhosted machines. One bulk metrics read joined
-  // onto the machines (no N+1). Each machine is probed for liveness.
-  const [sandboxes, enrollments, metricsByEnrollment] = await Promise.all([
+  // onto the machines (no N+1). Liveness is the durable heartbeat cursor.
+  const [legacySandboxes, enrollments] = await Promise.all([
     listSandboxes(db, workspaceId),
-    listEnrollments(db, workspaceId, { status: "active" }),
-    readMachineMetricsLatestForWorkspace(db, workspaceId),
+    listEnrollments(
+      db,
+      input.accountId && input.subjectId
+        ? {
+            accountId: input.accountId,
+            workspaceId,
+            subjectId: input.subjectId,
+          }
+        : workspaceId,
+      { status: "active" },
+    ),
   ]);
+  const metricsByEnrollment = new Map<string, MachineMetricsRow>();
+  for (const originWorkspaceId of new Set(enrollments.map((entry) => entry.workspaceId))) {
+    const originMetrics = await readMachineMetricsLatestForWorkspace(db, originWorkspaceId);
+    for (const [enrollmentId, metrics] of originMetrics) {
+      metricsByEnrollment.set(enrollmentId, metrics);
+    }
+  }
+  const scopedSandboxes = enrollments.flatMap((enrollment) =>
+    enrollment.sandboxId
+      ? [
+          {
+            id: enrollment.sandboxId,
+            accountId: enrollment.accountId,
+            workspaceId: enrollment.workspaceId,
+            kind: "selfhosted" as const,
+            name: enrollment.sandboxName ?? `${enrollment.os} machine`,
+            enrollmentId: enrollment.id,
+            createdAt: enrollment.createdAt,
+            updatedAt: enrollment.updatedAt,
+          },
+        ]
+      : [],
+  );
+  const sandboxes = [
+    ...legacySandboxes.filter(
+      (sandbox) => !scopedSandboxes.some((scoped) => scoped.id === sandbox.id),
+    ),
+    ...scopedSandboxes,
+  ];
   const enrollmentById = new Map(enrollments.map((e) => [e.id, e]));
 
   const machineViews = await Promise.all(
@@ -233,11 +335,26 @@ export async function listMachines(
       if (!enrollment) {
         return null;
       }
-      const [probe, lease] = await Promise.all([
-        probeEnrollment(services, workspaceId, enrollment),
-        readLease(db, workspaceId, sandbox.id),
+      const [liveConnection, lease] = await Promise.all([
+        enrollment.connectionInstanceId &&
+        enrollment.connectionLeaseExpiresAt &&
+        Date.parse(enrollment.connectionLeaseExpiresAt) > Date.now()
+          ? Promise.resolve(enrollment)
+          : getLiveEnrollmentConnection(
+              db,
+              input.accountId && input.subjectId
+                ? {
+                    accountId: input.accountId,
+                    workspaceId,
+                    subjectId: input.subjectId,
+                  }
+                : enrollment.workspaceId,
+              enrollment.id,
+            ),
+        readLease(db, enrollment.workspaceId, sandbox.id),
       ]);
-      const state = machineStateFor(probe.state, probe.hasDisplay);
+      const liveness = enrollmentLiveness(enrollment);
+      const state = machineStateFor(liveness.state, liveness.hasDisplay);
 
       // sharedSessionCount = the lease refcount for this machine's group. The
       // selfhosted sandbox id IS the lease group key (maxSandboxes:1, N sessions
@@ -248,6 +365,8 @@ export async function listMachines(
       return MachineView.parse({
         sandboxId: sandbox.id,
         enrollmentId: enrollment.id,
+        scope: enrollment.scope,
+        generation: enrollment.generation,
         name: sandbox.name,
         kind: "selfhosted",
         state,
@@ -263,6 +382,9 @@ export async function listMachines(
         allowScreenControl: enrollment.allowScreenControl,
         sharedSessionCount,
         lastSeenAt: enrollment.lastSeenAt,
+        connectionAuthority: connectionAuthorityFor(enrollment, liveConnection),
+        runtime: runtimeFor(services.settings, enrollment),
+        operationPolicy: enrollment.operationPolicy,
         metrics: metricsRow ? metricRowToSample(metricsRow) : null,
       });
     }),