@opengeni/api-router 0.2.0

package/src/sandbox/machines.ts

@@ -0,0 +1,255 @@
+// apps/api/src/sandbox/machines.ts — the M10 Machines-DASHBOARD service (dossier
+// §10.7). Builds the `MachinesResponse` the dashboard renders: the workspace's
+// enrolled selfhosted machines, each enriched with
+//   * STATE — the M3 liveness (online/reconnecting/offline) overlaid with the
+//     enrollment-derived consent/display reasons (consent_required /
+//     display_unavailable) — a real ControlRpc ping (the subject IS the registry),
+//     reusing the M7 fleet probe;
+//   * METRICS — the latest machine_metrics_latest row (or null before a first
+//     heartbeat), projected to the contract's MetricSample;
+//   * sharedSessionCount — the lease refcount (how many sessions share this one
+//     whole machine, the maxSandboxes:1 disclosure).
+// PLUS, when a session context is supplied, the session's synthetic Modal group
+// box (isSessionGroup:true) + the active-sandbox pointer (activeSandboxId/Epoch).
+//
+// This is workspace-scoped (perm enrollments:read) and flag-gated upstream
+// (sandboxSelfhostedEnabled). It deliberately does NOT depend on a FleetContext
+// (which is session-coupled): the pure workspace dashboard works without a
+// session; an in-session view passes the optional session to add the group box +
+// active pointer.
+
+import type { Settings } from "@opengeni/config";
+import {
+  getSession,
+  listEnrollments,
+  listSandboxes,
+  readActiveSandbox,
+  readLease,
+  readMachineMetricsLatestForWorkspace,
+  type Database,
+  type EnrollmentRecord,
+  type MachineMetricsRow,
+} from "@opengeni/db";
+import type { EventBus } from "@opengeni/events";
+import {
+  MachineView,
+  MetricSample,
+  type MachinesResponse,
+} from "@opengeni/contracts";
+import {
+  NatsControlRpc,
+  selfhostedLiveness,
+  SelfhostedSession,
+  type ControlRpc,
+  type NatsRequestConnection,
+} from "@opengeni/runtime/sandbox";
+import { relayConfigFromSettings } from "@opengeni/core";
+
+export type MachinesServices = {
+  db: Database;
+  settings: Settings;
+  bus?: EventBus;
+};
+
+const PROBE_TIMEOUT_MS = 5_000;
+
+function controlRpc(bus: EventBus | undefined): ControlRpc {
+  return new NatsControlRpc(async (): Promise<NatsRequestConnection | null> => {
+    if (!bus) {
+      return null;
+    }
+    return bus.getRequestConnection();
+  });
+}
+
+/**
+ * Project a stored `machine_metrics_latest` row to the contract `MetricSample`.
+ * The DB carries `gpuUtilPercent` + `gpuMemUsedBytes`/`gpuMemTotalBytes`; the wire
+ * `MetricSample` exposes the single `gpuUtilPct` + `gpuMemBytes` (USED bytes — the
+ * "how much VRAM is in use" the dashboard reads). A null any-numeric stays null
+ * (the not-reported contract); the byte/load fields default to 0 when a sample
+ * carried no value (the agent reports 0 == not-reported for those).
+ */
+export function metricRowToSample(row: MachineMetricsRow): MetricSample {
+  return MetricSample.parse({
+    cpuPct: row.cpuPercent ?? 0,
+    load1: row.load1 ?? 0,
+    load5: row.load5 ?? 0,
+    load15: row.load15 ?? 0,
+    memUsedBytes: row.memUsedBytes ?? 0,
+    memTotalBytes: row.memTotalBytes ?? 0,
+    diskUsedBytes: row.diskUsedBytes ?? 0,
+    diskTotalBytes: row.diskTotalBytes ?? 0,
+    gpuUtilPct: row.gpuUtilPercent,
+    gpuMemBytes: row.gpuMemUsedBytes,
+    runQueue: row.contention ?? 0,
+    sampledAt: row.sampledAt,
+  });
+}
+
+/** Probe an enrolled machine's liveness — a real ControlRpc ping mapped through
+ *  `selfhostedLiveness` (the enrollment status/consent/display + lastSeenAt
+ *  disambiguate a probe-miss into reconnecting vs offline). Mirrors the M7 fleet
+ *  probe. A non-active enrollment is offline without a probe. */
+async function probeEnrollment(
+  services: MachinesServices,
+  workspaceId: string,
+  enrollment: EnrollmentRecord,
+): Promise<{ state: "online" | "reconnecting" | "offline"; consented: boolean; hasDisplay: boolean }> {
+  const { settings, bus } = services;
+  let probeResponded = false;
+  if (enrollment.status === "active") {
+    const session = new SelfhostedSession({
+      workspaceId,
+      agentId: enrollment.id,
+      controlRpc: controlRpc(bus),
+      relay: relayConfigFromSettings(settings),
+      timeoutMs: PROBE_TIMEOUT_MS,
+    });
+    try {
+      probeResponded = await session.ping();
+    } catch {
+      probeResponded = false;
+    }
+  }
+  const derived = selfhostedLiveness({
+    enrollment: {
+      status: enrollment.status,
+      exposure: enrollment.exposure,
+      allowScreenControl: enrollment.allowScreenControl,
+      hasDisplay: enrollment.hasDisplay,
+      lastSeenAt: enrollment.lastSeenAt,
+    },
+    probeResponded,
+  });
+  return { state: derived.state, consented: derived.consented, hasDisplay: derived.hasDisplay };
+}
+
+/**
+ * Resolve the dashboard STATE of a machine. State reflects REACHABILITY + the
+ * VIEW plane only: an online machine with no display → `display_unavailable` (no
+ * desktop stream, but compute — exec/fs/git/terminal — still works); otherwise
+ * the liveness state (online/reconnecting/offline). It deliberately does NOT fold
+ * in screen-control consent: a displayed machine can be VIEWED (read-only) and
+ * used for compute regardless of `allowScreenControl` — only INPUT (ComputerUse /
+ * an interactive stream) needs that consent, which is a per-capability concern
+ * carried by the separate `allowScreenControl` field (surfaced in the viewer's
+ * Take-control affordance), NOT a blocking machine state. This mirrors the
+ * view/control split in the selfhosted capability negotiation so the dashboard
+ * pill, the dock, and the "Run on" picker agree (a machine is never wrongly
+ * un-selectable just because its input isn't consented).
+ */
+function machineStateFor(
+  liveness: "online" | "reconnecting" | "offline",
+  hasDisplay: boolean,
+): MachinesResponse["machines"][number]["state"] {
+  if (liveness !== "online") {
+    return liveness;
+  }
+  if (!hasDisplay) {
+    return "display_unavailable";
+  }
+  return "online";
+}
+
+/**
+ * Build the Machines dashboard response for a workspace. When `sessionId` is
+ * supplied (an in-session view) the session's synthetic Modal group box is
+ * prepended (`isSessionGroup:true`) and the active-sandbox pointer is echoed;
+ * without it (the pure workspace dashboard) `activeSandboxId` is null and only
+ * the enrolled machines are listed.
+ */
+export async function listMachines(
+  services: MachinesServices,
+  input: { workspaceId: string; sessionId?: string | null },
+): Promise<MachinesResponse> {
+  const { db } = services;
+  const { workspaceId } = input;
+
+  // The session's active pointer (in-session view only). Absent session → the
+  // default null pointer (the workspace dashboard has no "active" machine).
+  let activeSandboxId: string | null = null;
+  let activeEpoch = 0;
+  let session: Awaited<ReturnType<typeof getSession>> | null = null;
+  if (input.sessionId) {
+    session = await getSession(db, workspaceId, input.sessionId);
+    if (session) {
+      const pointer = await readActiveSandbox(db, workspaceId, input.sessionId);
+      activeSandboxId = pointer?.activeSandboxId ?? null;
+      activeEpoch = pointer?.activeEpoch ?? 0;
+    }
+  }
+
+  const machines: MachineView[] = [];
+
+  // The session's own Modal group box (synthetic): the default/home sandbox a
+  // null active pointer routes to. Only present in an in-session view.
+  if (session) {
+    const groupActive = activeSandboxId === null;
+    machines.push(MachineView.parse({
+      sandboxId: session.sandboxGroupId,
+      enrollmentId: null,
+      name: "session sandbox",
+      kind: session.sandboxBackend === "selfhosted" ? "selfhosted" : "modal",
+      state: "online",
+      active: groupActive,
+      isSessionGroup: true,
+      // The Modal group box is a cloud Linux box; its precise OS/arch is not
+      // surfaced as a metric, so the dashboard shows the canonical linux/x86_64.
+      os: "linux",
+      arch: "x86_64",
+      hasDisplay: false,
+      allowScreenControl: false,
+      sharedSessionCount: 1,
+      lastSeenAt: null,
+      metrics: null,
+    }));
+  }
+
+  // The workspace's enrolled selfhosted machines. One bulk metrics read joined
+  // onto the machines (no N+1). Each machine is probed for liveness.
+  const [sandboxes, enrollments, metricsByEnrollment] = await Promise.all([
+    listSandboxes(db, workspaceId),
+    listEnrollments(db, workspaceId),
+    readMachineMetricsLatestForWorkspace(db, workspaceId),
+  ]);
+  const enrollmentById = new Map(enrollments.map((e) => [e.id, e]));
+
+  for (const sandbox of sandboxes) {
+    if (sandbox.kind !== "selfhosted" || !sandbox.enrollmentId) {
+      continue;
+    }
+    const enrollment = enrollmentById.get(sandbox.enrollmentId) ?? null;
+    if (!enrollment) {
+      continue;
+    }
+    const probe = await probeEnrollment(services, workspaceId, enrollment);
+    const state = machineStateFor(probe.state, probe.hasDisplay);
+
+    // sharedSessionCount = the lease refcount for this machine's group. The
+    // selfhosted sandbox id IS the lease group key (maxSandboxes:1, N sessions
+    // share via refcount). No lease yet → 0 sessions sharing.
+    const lease = await readLease(db, workspaceId, sandbox.id);
+    const sharedSessionCount = lease?.refcount ?? 0;
+
+    const metricsRow = metricsByEnrollment.get(enrollment.id) ?? null;
+    machines.push(MachineView.parse({
+      sandboxId: sandbox.id,
+      enrollmentId: enrollment.id,
+      name: sandbox.name,
+      kind: "selfhosted",
+      state,
+      active: activeSandboxId === sandbox.id,
+      isSessionGroup: false,
+      os: enrollment.os,
+      arch: enrollment.arch,
+      hasDisplay: enrollment.hasDisplay,
+      allowScreenControl: enrollment.allowScreenControl,
+      sharedSessionCount,
+      lastSeenAt: enrollment.lastSeenAt,
+      metrics: metricsRow ? metricRowToSample(metricsRow) : null,
+    }));
+  }
+
+  return { activeSandboxId, activeEpoch, machines };
+}

package/src/sandbox/metrics-ingestion.ts

@@ -0,0 +1,289 @@
+// apps/api/src/sandbox/metrics-ingestion.ts — the M10 metrics INGESTION consumer
+// (dossier §10.7 + §10.6) + the connect-Hello DISPLAY-REFRESH consumer. The
+// enrolled agent piggybacks a `MetricsSample` on its ~5s heartbeat (an
+// `AgentEvent` published one-way on `agent.<ws>.<id>.events`) and publishes a
+// `Hello` (its live self-description) on `agent.<ws>.<id>.hello` on every connect
+// /reconnect. This module owns the two agent→control-plane inbound consumers:
+//
+//   `agent.*.*.events` (heartbeat) →
+//     1. touchEnrollmentLastSeen  — the liveness cursor (online/reconnecting/offline
+//        derivation + the M3 probe disambiguation).
+//     2. ingestMachineMetricsSample — UPSERT machine_metrics_latest (the "now" row)
+//        + APPEND a machine_metrics_series row downsampled to ~1/min.
+//     A GOING-OFFLINE event is not a metrics point — liveness flips via the lease/
+//     probe path; we skip it here (no-op).
+//
+//   `agent.*.*.hello` (connect) →
+//     refreshEnrollmentDisplay — reconcile `enrollments.has_display` to the LIVE
+//     capability the Hello reports. `has_display` was previously FROZEN at the
+//     enroll-time offer snapshot; a machine that GAINS a display later (a Mac that
+//     grants Screen Recording, a box whose Xvfb starts) or LOSES one never
+//     re-surfaced. Consuming the Hello's `capabilities.desktop` / `display` makes
+//     `has_display` track reality (both directions), which the desktop-capability
+//     gate (packages/runtime capabilities.ts) keys off.
+//
+// Both consumers are BEST-EFFORT and fail-soft: a decode/DB error for one message
+// is logged + swallowed (the bus subscription already swallows handler throws) so
+// a metrics blip / a display-refresh write failure never tears down the consumer,
+// back-pressures the agent, or breaks its connect.
+
+import {
+  getEnrollment,
+  ingestMachineMetricsSample,
+  setEnrollmentHasDisplay,
+  touchEnrollmentLastSeen,
+  type Database,
+  type MachineMetricsSample,
+} from "@opengeni/db";
+import type { EventBus } from "@opengeni/events";
+import type { Observability } from "@opengeni/observability";
+import { AgentEvent, Hello, type MetricsSample } from "@opengeni/agent-proto";
+
+/** The wildcard subject the agent event plane publishes heartbeats on. */
+export const AGENT_EVENTS_SUBJECT = "agent.*.*.events";
+
+/** The wildcard subject the agent publishes its connect Hello on. */
+export const AGENT_HELLO_SUBJECT = "agent.*.*.hello";
+
+/**
+ * Parse `agent.<ws>.<id>.<tail>` → `{ workspaceId, agentId }`, requiring the
+ * expected tail token. Returns null for a subject that does not match the shape
+ * (defensive — the subscription pattern already constrains it).
+ */
+function parseAgentSubject(subject: string, tail: "events" | "hello"): { workspaceId: string; agentId: string } | null {
+  const parts = subject.split(".");
+  if (parts.length !== 4 || parts[0] !== "agent" || parts[3] !== tail) {
+    return null;
+  }
+  return { workspaceId: parts[1]!, agentId: parts[2]! };
+}
+
+/** Parse `agent.<ws>.<id>.events` → `{ workspaceId, agentId }` (heartbeat plane). */
+export function parseAgentEventSubject(subject: string): { workspaceId: string; agentId: string } | null {
+  return parseAgentSubject(subject, "events");
+}
+
+/** Parse `agent.<ws>.<id>.hello` → `{ workspaceId, agentId }` (connect plane). */
+export function parseAgentHelloSubject(subject: string): { workspaceId: string; agentId: string } | null {
+  return parseAgentSubject(subject, "hello");
+}
+
+/**
+ * Project a wire `MetricsSample` (proto, ms-stamped, GPU as a repeated list) to
+ * the DB `MachineMetricsSample`. The proto byte/count fields are protobuf-encoded
+ * as decimal strings (uint64) on the TS side (ts-proto `string`); coerce to
+ * numbers. The DB carries a single `gpuUtilPercent` + `gpuMemUsedBytes`/Total —
+ * we take the FIRST GPU (the dashboard surfaces the primary accelerator); absent
+ * GPUs stay null (the not-reported contract). A zero on a non-GPU field is the
+ * agent's "not reported" (we keep it null-friendly via `nullIfZero` only for the
+ * GPU plane; cpu/mem/disk 0 is a legitimate reading the dashboard shows as 0).
+ */
+export function wireSampleToDbSample(wire: MetricsSample): MachineMetricsSample {
+  const num = (v: string | number): number => (typeof v === "number" ? v : Number(v));
+  const firstGpu = wire.gpus[0];
+  return {
+    cpuPercent: wire.cpuPercent,
+    load1: wire.load1,
+    load5: wire.load5,
+    load15: wire.load15,
+    memUsedBytes: num(wire.memUsedBytes),
+    memTotalBytes: num(wire.memTotalBytes),
+    diskUsedBytes: num(wire.diskUsedBytes),
+    diskTotalBytes: num(wire.diskTotalBytes),
+    gpuUtilPercent: firstGpu ? firstGpu.utilPercent : null,
+    gpuMemUsedBytes: firstGpu ? num(firstGpu.memUsedBytes) : null,
+    gpuMemTotalBytes: firstGpu ? num(firstGpu.memTotalBytes) : null,
+    contention: wire.runQueue,
+    // The sample carries its own wall-clock stamp (epoch ms); fall back to now on
+    // a missing/zero stamp so a series row is never NULL-dated.
+    sampledAt: wire.sampledAtMs && Number(wire.sampledAtMs) > 0 ? new Date(Number(wire.sampledAtMs)) : new Date(),
+  };
+}
+
+/**
+ * Ingest ONE decoded heartbeat for an enrolled machine. Resolves the enrollment's
+ * accountId (needed for the RLS-scoped writes) from the enrollment row; an
+ * unknown/cross-workspace agentId is ignored (no row → no write). Touches
+ * last-seen + upserts latest + downsamples the series.
+ */
+export async function ingestHeartbeat(
+  db: Database,
+  input: { workspaceId: string; agentId: string; sample: MetricsSample },
+): Promise<{ ingested: boolean; seriesAppended: boolean }> {
+  // The enrollment row is the source of the accountId (the RLS principal) and the
+  // existence check. A revoked machine still reports its accountId, so we ingest
+  // (the dashboard shows its last sample); a truly unknown id is a no-op.
+  const enrollment = await getEnrollment(db, input.workspaceId, input.agentId);
+  if (!enrollment) {
+    return { ingested: false, seriesAppended: false };
+  }
+  const sample = wireSampleToDbSample(input.sample);
+  await touchEnrollmentLastSeen(db, {
+    accountId: enrollment.accountId,
+    workspaceId: input.workspaceId,
+    enrollmentId: input.agentId,
+  });
+  const result = await ingestMachineMetricsSample(db, {
+    accountId: enrollment.accountId,
+    workspaceId: input.workspaceId,
+    enrollmentId: input.agentId,
+    sample,
+  });
+  return { ingested: true, seriesAppended: result.seriesAppended };
+}
+
+/**
+ * Decode a raw `AgentEvent` payload + ingest it (the per-message handler). A
+ * heartbeat carrying a metrics sample is ingested; a going-offline (or a
+ * heartbeat without metrics) is a no-op. Decode failures are reported + swallowed.
+ */
+export async function handleAgentEventPayload(
+  db: Database,
+  observability: Observability | undefined,
+  payload: Uint8Array,
+  subject: string,
+): Promise<void> {
+  const ids = parseAgentEventSubject(subject);
+  if (!ids) {
+    return;
+  }
+  let event: AgentEvent;
+  try {
+    event = AgentEvent.decode(payload);
+  } catch (error) {
+    observability?.warn?.("Failed to decode an agent event for metrics ingestion", {
+      subject,
+      error: error instanceof Error ? error.message : String(error),
+    });
+    return;
+  }
+  if (event.event?.$case !== "heartbeat") {
+    return; // going-offline / unknown → not a metrics point.
+  }
+  const metrics = event.event.heartbeat.metrics;
+  if (!metrics) {
+    return; // a heartbeat without a sample → liveness already touched elsewhere.
+  }
+  try {
+    await ingestHeartbeat(db, { workspaceId: ids.workspaceId, agentId: ids.agentId, sample: metrics });
+  } catch (error) {
+    observability?.warn?.("Failed to ingest a machine metrics heartbeat", {
+      subject,
+      error: error instanceof Error ? error.message : String(error),
+    });
+  }
+}
+
+/**
+ * Start the metrics-ingestion consumer: subscribe `agent.*.*.events` and ingest
+ * every heartbeat. Gated by sandboxSelfhostedEnabled (the caller checks the flag;
+ * a disabled deployment never starts the consumer). Returns the unsubscribe fn.
+ */
+export function startMetricsIngestion(deps: {
+  db: Database;
+  bus: EventBus;
+  observability?: Observability;
+}): () => void {
+  return deps.bus.subscribeAgentEvents(AGENT_EVENTS_SUBJECT, (payload, subject) =>
+    handleAgentEventPayload(deps.db, deps.observability, payload, subject),
+  );
+}
+
+// ── Connect-Hello display refresh ─────────────────────────────────────────────
+
+/**
+ * The LIVE display presence the agent's Hello reports: a desktop framebuffer is
+ * available (`capabilities.desktop`, which the agent sets true only when a display
+ * probes AND it can stream it) OR a `Display` detail is present. An unset
+ * Capabilities (or a headless machine) → false. This is what `has_display` should
+ * track, replacing the enroll-time snapshot.
+ */
+export function helloReportsDisplay(hello: Hello): boolean {
+  const caps = hello.capabilities;
+  if (!caps) {
+    return false;
+  }
+  return caps.desktop === true || caps.display != null;
+}
+
+/**
+ * Reconcile `enrollments.has_display` to the display presence a Hello reports.
+ * Resolves the enrollment (the accountId is the RLS principal + the existence
+ * check + the current value). A no-change Hello short-circuits BEFORE issuing any
+ * write (and the DB writer is itself change-guarded as a backstop), so a steady
+ * state never churns. An unknown/cross-workspace agentId is a no-op.
+ */
+export async function refreshEnrollmentDisplay(
+  db: Database,
+  input: { workspaceId: string; agentId: string; hasDisplay: boolean },
+): Promise<{ updated: boolean }> {
+  const enrollment = await getEnrollment(db, input.workspaceId, input.agentId);
+  if (!enrollment) {
+    return { updated: false };
+  }
+  if (enrollment.hasDisplay === input.hasDisplay) {
+    // Unchanged — do not even issue the UPDATE (no churn on a steady-state Hello).
+    return { updated: false };
+  }
+  return await setEnrollmentHasDisplay(db, {
+    accountId: enrollment.accountId,
+    workspaceId: input.workspaceId,
+    enrollmentId: input.agentId,
+    hasDisplay: input.hasDisplay,
+  });
+}
+
+/**
+ * Decode a raw `Hello` payload + refresh the enrollment's display cursor (the
+ * per-message handler for the hello plane). Decode failures + write failures are
+ * reported + swallowed — a display refresh must NEVER break the agent's connect.
+ */
+export async function handleHelloPayload(
+  db: Database,
+  observability: Observability | undefined,
+  payload: Uint8Array,
+  subject: string,
+): Promise<void> {
+  const ids = parseAgentHelloSubject(subject);
+  if (!ids) {
+    return;
+  }
+  let hello: Hello;
+  try {
+    hello = Hello.decode(payload);
+  } catch (error) {
+    observability?.warn?.("Failed to decode an agent Hello for display refresh", {
+      subject,
+      error: error instanceof Error ? error.message : String(error),
+    });
+    return;
+  }
+  try {
+    await refreshEnrollmentDisplay(db, {
+      workspaceId: ids.workspaceId,
+      agentId: ids.agentId,
+      hasDisplay: helloReportsDisplay(hello),
+    });
+  } catch (error) {
+    observability?.warn?.("Failed to refresh an enrollment's display from a Hello", {
+      subject,
+      error: error instanceof Error ? error.message : String(error),
+    });
+  }
+}
+
+/**
+ * Start the Hello display-refresh consumer: subscribe `agent.*.*.hello` and
+ * reconcile `has_display` to the live capability the agent reports on every
+ * connect. Gated by sandboxSelfhostedEnabled (the caller checks the flag). Returns
+ * the unsubscribe fn.
+ */
+export function startHelloIngestion(deps: {
+  db: Database;
+  bus: EventBus;
+  observability?: Observability;
+}): () => void {
+  return deps.bus.subscribeAgentEvents(AGENT_HELLO_SUBJECT, (payload, subject) =>
+    handleHelloPayload(deps.db, deps.observability, payload, subject),
+  );
+}