@opengeni/api-router 0.30.1 → 1.0.0

package/src/sandbox/metrics-ingestion.ts

@@ -1,11 +1,11 @@
 // apps/api/src/sandbox/metrics-ingestion.ts — the M10 metrics INGESTION consumer
 // + the connect-Hello DISPLAY-REFRESH consumer. The
 // enrolled agent piggybacks a `MetricsSample` on its ~5s heartbeat (an
-// `AgentEvent` published one-way on `agent.<ws>.<id>.events`) and publishes a
-// `Hello` (its live self-description) on `agent.<ws>.<id>.hello` on every connect
+// `AgentEvent` published one-way on the exact process events subject) and publishes a
+// `Hello` (its live self-description) on the exact process hello subject on every connect
 // /reconnect. This module owns the two agent→control-plane inbound consumers:
 //
-//   `agent.*.*.events` (heartbeat) →
+//   `agent.*.*.connection.*.events` (heartbeat) →
 //     1. touchEnrollmentLastSeen  — the liveness cursor (online/reconnecting/offline
 //        derivation + the M3 probe disambiguation).
 //     2. ingestMachineMetricsSample — UPSERT machine_metrics_latest (the "now" row)
@@ -13,7 +13,7 @@
 //     A GOING-OFFLINE event is not a metrics point — liveness flips via the lease/
 //     probe path; we skip it here (no-op).
 //
-//   `agent.*.*.hello` (connect) →
+//   `agent.*.*.connection.*.hello` (connect) →
 //     refreshEnrollmentDisplay — reconcile `enrollments.has_display` to the LIVE
 //     capability the Hello reports. `has_display` was previously FROZEN at the
 //     enroll-time offer snapshot; a machine that GAINS a display later (a Mac that
@@ -22,25 +22,32 @@
 //     `has_display` track reality (both directions), which the desktop-capability
 //     gate (packages/runtime capabilities.ts) keys off.
 //     refreshEnrollmentOpStream — reconcile `enrollments.op_stream` to the LIVE
-//     runner capability the Hello reports, leaving legacy request/reply exec as the
-//     fallback unless the runner advertises the streaming engine.
+//     runner capability the Hello reports. Exec is unavailable unless the runner
+//     advertises the streaming engine.
 //
 // Both consumers are BEST-EFFORT and fail-soft: a decode/DB error for one message
 // is logged + swallowed (the bus subscription already swallows handler throws) so
 // a metrics blip / a display-refresh write failure never tears down the consumer,
-// back-pressures the agent, or breaks its connect.
+// back-pressures the agent, or breaks its connect. Event ingestion drains the NATS
+// subscription immediately into exact-subject queues: different runner connections
+// progress concurrently, one runner's ordering is preserved, and consecutive queued
+// heartbeats collapse latest-wins. A delayed heartbeat backlog therefore cannot keep
+// renewing a dead runner's short connection lease one stale message at a time.
 
 import {
   clearEnrollmentWentOffline,
   disconnectAttachedBrowserDevices,
+  advanceEnrollmentAgentUpdate,
   getEnrollment,
+  getLiveEnrollmentConnection,
   ingestMachineMetricsSample,
   reconcileAttachedBrowserInventory,
+  releaseEnrollmentConnection,
+  renewEnrollmentConnection,
   sessionsWithActiveOpOnEnrollment,
   setEnrollmentDisplayState,
+  setEnrollmentAgentRuntime,
   setEnrollmentOpStreamState,
-  setEnrollmentWentOffline,
-  touchEnrollmentLastSeen,
   type AppendEventInput,
   type Database,
   type MachineMetricsSample,
@@ -51,8 +58,10 @@ import {
 } from "@opengeni/contracts";
 import { appendAndPublishEvents, type EventBus } from "@opengeni/events";
 import type { Observability } from "@opengeni/observability";
+import { normalizeConnectedMachineWorkspaceRoot } from "@opengeni/runtime/sandbox";
 import {
   AgentEvent,
+  AgentUpdateStage,
   Arch,
   GoingOfflineReason,
   Hello,
@@ -61,40 +70,186 @@ import {
   type AttachedBrowserInventorySnapshot as WireAttachedBrowserInventorySnapshot,
   type MetricsSample,
 } from "@opengeni/agent-proto";
+import { AGENT_CONNECTION_LEASE_MS } from "./connection-authority";
 
 /** The wildcard subject the agent event plane publishes heartbeats on. */
-export const AGENT_EVENTS_SUBJECT = "agent.*.*.events";
+export const AGENT_EVENTS_SUBJECT = "agent.*.*.connection.*.events";
 
 /** The wildcard subject the agent publishes its connect Hello on. */
-export const AGENT_HELLO_SUBJECT = "agent.*.*.hello";
+export const AGENT_HELLO_SUBJECT = "agent.*.*.connection.*.hello";
+
+type QueuedAgentEvent = {
+  payload: Uint8Array;
+  subject: string;
+  heartbeat: boolean;
+};
+
+type AgentEventQueueState = {
+  scheduled: boolean;
+  pending: QueuedAgentEvent[];
+};
+
+type ScheduledAgentEventQueue = {
+  subject: string;
+  state: AgentEventQueueState;
+};
+
+export type AgentEventIngestionScheduler = {
+  enqueue: (payload: Uint8Array, subject: string) => void;
+  whenIdle: () => Promise<void>;
+  close: () => void;
+};
+
+function payloadIsHeartbeat(payload: Uint8Array): boolean {
+  try {
+    return AgentEvent.decode(payload).event?.$case === "heartbeat";
+  } catch {
+    // Preserve the normal decoder/logging path for malformed events. They are
+    // barriers rather than heartbeat-coalescing candidates.
+    return false;
+  }
+}
 
 /**
- * Parse `agent.<ws>.<id>.<tail>` → `{ workspaceId, agentId }`, requiring the
+ * Drain agent events without making every Connected Machine wait behind one
+ * deployment-wide serial DB queue. Each exact process subject retains FIFO order;
+ * separate subjects drain concurrently. Consecutive pending heartbeats for one
+ * subject are latest-wins because they are point-in-time liveness/metrics samples,
+ * while GoingOffline and update-progress events remain ordered barriers.
+ */
+export function createAgentEventIngestionScheduler(
+  handler: (payload: Uint8Array, subject: string) => void | Promise<void>,
+  options: { maxConcurrentSubjects?: number; onHeartbeatCoalesced?: () => void } = {},
+): AgentEventIngestionScheduler {
+  const maxConcurrentSubjects = options.maxConcurrentSubjects ?? 32;
+  if (!Number.isSafeInteger(maxConcurrentSubjects) || maxConcurrentSubjects <= 0) {
+    throw new RangeError("maxConcurrentSubjects must be a positive safe integer");
+  }
+
+  const states = new Map<string, AgentEventQueueState>();
+  const readySubjects: ScheduledAgentEventQueue[] = [];
+  const idleWaiters = new Set<() => void>();
+  let accepting = true;
+  let activeSubjects = 0;
+
+  const resolveIdleWaiters = (): void => {
+    if (states.size !== 0) return;
+    for (const resolve of idleWaiters) resolve();
+    idleWaiters.clear();
+  };
+
+  async function drain(subject: string, state: AgentEventQueueState): Promise<void> {
+    while (true) {
+      const event = state.pending.shift();
+      if (!event) break;
+      try {
+        await handler(event.payload, event.subject);
+      } catch {
+        // Agent-event ingestion is best-effort. One bad event must not strand the
+        // exact-subject queue or prevent later heartbeats from restoring liveness.
+      }
+    }
+
+    activeSubjects -= 1;
+    state.scheduled = false;
+    if (state.pending.length > 0) {
+      schedule(subject, state);
+    } else {
+      if (states.get(subject) === state) states.delete(subject);
+      resolveIdleWaiters();
+    }
+    pump();
+  }
+
+  function pump(): void {
+    while (activeSubjects < maxConcurrentSubjects) {
+      const next = readySubjects.shift();
+      if (!next) return;
+      if (states.get(next.subject) !== next.state || !next.state.scheduled) continue;
+      activeSubjects += 1;
+      void drain(next.subject, next.state);
+    }
+  }
+
+  function schedule(subject: string, state: AgentEventQueueState): void {
+    if (state.scheduled) return;
+    state.scheduled = true;
+    readySubjects.push({ subject, state });
+    pump();
+  }
+
+  return {
+    enqueue(payload, subject) {
+      if (!accepting) return;
+      const heartbeat = payloadIsHeartbeat(payload);
+      let state = states.get(subject);
+      if (!state) {
+        state = { scheduled: false, pending: [] };
+        states.set(subject, state);
+      }
+
+      const queued = { payload, subject, heartbeat };
+      const lastIndex = state.pending.length - 1;
+      if (heartbeat && lastIndex >= 0 && state.pending[lastIndex]!.heartbeat) {
+        state.pending[lastIndex] = queued;
+        options.onHeartbeatCoalesced?.();
+      } else {
+        state.pending.push(queued);
+      }
+
+      schedule(subject, state);
+    },
+    async whenIdle() {
+      if (states.size === 0) return;
+      await new Promise<void>((resolve) => idleWaiters.add(resolve));
+    },
+    close() {
+      accepting = false;
+      for (const state of states.values()) {
+        state.pending.length = 0;
+      }
+      resolveIdleWaiters();
+    },
+  };
+}
+
+/**
+ * Parse `agent.<ws>.<id>.connection.<instance>.<tail>` into its exact authority,
  * expected tail token. Returns null for a subject that does not match the shape
  * (defensive — the subscription pattern already constrains it).
  */
 function parseAgentSubject(
   subject: string,
   tail: "events" | "hello",
-): { workspaceId: string; agentId: string } | null {
+): { workspaceId: string; agentId: string; connectionInstanceId: string } | null {
   const parts = subject.split(".");
-  if (parts.length !== 4 || parts[0] !== "agent" || parts[3] !== tail) {
+  if (
+    parts.length !== 6 ||
+    parts[0] !== "agent" ||
+    parts[3] !== "connection" ||
+    !parts[4] ||
+    parts[5] !== tail
+  ) {
     return null;
   }
-  return { workspaceId: parts[1]!, agentId: parts[2]! };
+  return {
+    workspaceId: parts[1]!,
+    agentId: parts[2]!,
+    connectionInstanceId: parts[4]!,
+  };
 }
 
-/** Parse `agent.<ws>.<id>.events` → `{ workspaceId, agentId }` (heartbeat plane). */
+/** Parse the exact process events subject (heartbeat plane). */
 export function parseAgentEventSubject(
   subject: string,
-): { workspaceId: string; agentId: string } | null {
+): { workspaceId: string; agentId: string; connectionInstanceId: string } | null {
   return parseAgentSubject(subject, "events");
 }
 
-/** Parse `agent.<ws>.<id>.hello` → `{ workspaceId, agentId }` (connect plane). */
+/** Parse the exact process hello subject (connect plane). */
 export function parseAgentHelloSubject(
   subject: string,
-): { workspaceId: string; agentId: string } | null {
+): { workspaceId: string; agentId: string; connectionInstanceId: string } | null {
   return parseAgentSubject(subject, "hello");
 }
 
@@ -218,30 +373,23 @@ async function ingestAttachedBrowserInventory(
 }
 
 /**
- * Ingest ONE decoded heartbeat for an enrolled machine. Resolves the enrollment's
- * accountId (needed for the RLS-scoped writes) from the enrollment row; an
- * unknown/cross-workspace agentId is ignored (no row → no write). Touches
- * last-seen + upserts latest + downsamples the series.
+ * Ingest the OPTIONAL metrics sample carried by an already-authorized heartbeat.
+ * Connection renewal is deliberately owned by the outer event handler and runs
+ * before optional telemetry/inventory work, so telemetry absence or failure can
+ * never make a healthy runner lose authority.
  */
-export async function ingestHeartbeat(
+async function ingestHeartbeatMetrics(
   db: Database,
-  input: { workspaceId: string; agentId: string; sample: MetricsSample },
+  input: {
+    accountId: string;
+    workspaceId: string;
+    agentId: string;
+    sample: MetricsSample;
+  },
 ): Promise<{ ingested: boolean; seriesAppended: boolean }> {
-  // The enrollment row is the source of the accountId (the RLS principal) and the
-  // existence check. A revoked machine still reports its accountId, so we ingest
-  // (the dashboard shows its last sample); a truly unknown id is a no-op.
-  const enrollment = await getEnrollment(db, input.workspaceId, input.agentId);
-  if (!enrollment) {
-    return { ingested: false, seriesAppended: false };
-  }
   const sample = wireSampleToDbSample(input.sample);
-  await touchEnrollmentLastSeen(db, {
-    accountId: enrollment.accountId,
-    workspaceId: input.workspaceId,
-    enrollmentId: input.agentId,
-  });
   const result = await ingestMachineMetricsSample(db, {
-    accountId: enrollment.accountId,
+    accountId: input.accountId,
     workspaceId: input.workspaceId,
     enrollmentId: input.agentId,
     sample,
@@ -340,12 +488,16 @@ export async function handleAgentEventPayload(
     try {
       const enrollment = await getEnrollment(db, ids.workspaceId, ids.agentId);
       if (enrollment) {
-        await setEnrollmentWentOffline(db, {
+        const released = await releaseEnrollmentConnection(db, {
           accountId: enrollment.accountId,
           workspaceId: ids.workspaceId,
           enrollmentId: ids.agentId,
+          connectionInstanceId: ids.connectionInstanceId,
           reason,
         });
+        // A late goodbye from a superseded process is not the current machine
+        // going offline. It must not disconnect browsers or fan out link loss.
+        if (!released.released) return;
         try {
           await disconnectAttachedBrowserDevices(db, {
             accountId: enrollment.accountId,
@@ -396,10 +548,77 @@ export async function handleAgentEventPayload(
     }
     return;
   }
+  if (event.event?.$case === "agentUpdateProgress") {
+    const enrollment = await getEnrollment(db, ids.workspaceId, ids.agentId).catch(() => null);
+    if (!enrollment || enrollment.connectionInstanceId !== ids.connectionInstanceId) return;
+    const progress = event.event.agentUpdateProgress;
+    const status = (() => {
+      switch (progress.stage) {
+        case AgentUpdateStage.AGENT_UPDATE_STAGE_ACCEPTED:
+          return "accepted" as const;
+        case AgentUpdateStage.AGENT_UPDATE_STAGE_WAITING_FOR_IDLE:
+          return "waiting_for_idle" as const;
+        case AgentUpdateStage.AGENT_UPDATE_STAGE_DOWNLOADING:
+          return "downloading" as const;
+        case AgentUpdateStage.AGENT_UPDATE_STAGE_VERIFYING:
+          return "verifying" as const;
+        case AgentUpdateStage.AGENT_UPDATE_STAGE_APPLYING:
+          return "applying" as const;
+        case AgentUpdateStage.AGENT_UPDATE_STAGE_RESTARTING:
+          return "restarting" as const;
+        case AgentUpdateStage.AGENT_UPDATE_STAGE_FAILED:
+          return "failed" as const;
+        default:
+          return null;
+      }
+    })();
+    if (!status) return;
+    try {
+      await advanceEnrollmentAgentUpdate(db, {
+        accountId: enrollment.accountId,
+        workspaceId: ids.workspaceId,
+        enrollmentId: ids.agentId,
+        connectionInstanceId: ids.connectionInstanceId,
+        connectionGeneration: enrollment.connectionGeneration,
+        operationId: progress.operationId,
+        status,
+        expectedBinarySha256: /^[0-9a-f]{64}$/.test(progress.expectedBinarySha256)
+          ? progress.expectedBinarySha256
+          : null,
+        errorCode: progress.errorCode || null,
+        retryable: progress.retryable,
+        rolledBack: progress.rolledBack,
+      });
+    } catch (error) {
+      observability?.warn?.("Failed to ingest a machine self-update progress event", {
+        subject,
+        error: error instanceof Error ? error.message : String(error),
+      });
+    }
+    return;
+  }
   if (event.event?.$case !== "heartbeat") {
     return; // an unknown event kind → not a metrics point.
   }
   const heartbeat = event.event.heartbeat;
+  const enrollment = await getEnrollment(db, ids.workspaceId, ids.agentId).catch(() => null);
+  if (!enrollment || enrollment.connectionInstanceId !== ids.connectionInstanceId) return;
+  try {
+    const renewed = await renewEnrollmentConnection(db, {
+      accountId: enrollment.accountId,
+      workspaceId: ids.workspaceId,
+      enrollmentId: ids.agentId,
+      connectionInstanceId: ids.connectionInstanceId,
+      leaseMs: AGENT_CONNECTION_LEASE_MS,
+    });
+    if (!renewed.renewed) return;
+  } catch (error) {
+    observability?.warn?.("Failed to renew a machine connection heartbeat", {
+      subject,
+      error: error instanceof Error ? error.message : String(error),
+    });
+    return;
+  }
   if (heartbeat.attachedBrowserInventory) {
     try {
       await ingestAttachedBrowserInventory(db, {
@@ -414,10 +633,39 @@ export async function handleAgentEventPayload(
       });
     }
   }
+  if (heartbeat.desktopStatus) {
+    try {
+      const status = heartbeat.desktopStatus;
+      const available = status.available && !status.unavailableReason;
+      const reason = status.unavailableReason || null;
+      // The heartbeat already read the exact enrollment. Steady state adds no
+      // extra query; the change path re-reads and fences its eventual write.
+      if (
+        enrollment.hasDisplay !== available ||
+        (enrollment.desktopUnavailableReason ?? null) !== reason ||
+        enrollment.agentCapabilities?.desktop !== available
+      ) {
+        await refreshEnrollmentDisplay(db, {
+          workspaceId: ids.workspaceId,
+          agentId: ids.agentId,
+          hasDisplay: available,
+          desktopUnavailableReason: reason,
+          runtimeDesktop: available,
+          connectionInstanceId: ids.connectionInstanceId,
+        });
+      }
+    } catch (error) {
+      observability?.warn?.("Failed to refresh a machine desktop heartbeat", {
+        subject,
+        error: error instanceof Error ? error.message : String(error),
+      });
+    }
+  }
   const metrics = heartbeat.metrics;
   if (!metrics) return;
   try {
-    await ingestHeartbeat(db, {
+    await ingestHeartbeatMetrics(db, {
+      accountId: enrollment.accountId,
       workspaceId: ids.workspaceId,
       agentId: ids.agentId,
       sample: metrics,
@@ -431,7 +679,7 @@ export async function handleAgentEventPayload(
 }
 
 /**
- * Start the metrics-ingestion consumer: subscribe `agent.*.*.events` and ingest
+ * Start the metrics-ingestion consumer: subscribe exact process events and ingest
  * every heartbeat. Gated by sandboxSelfhostedEnabled (the caller checks the flag;
  * a disabled deployment never starts the consumer). Returns the unsubscribe fn.
  */
@@ -440,9 +688,24 @@ export function startMetricsIngestion(deps: {
   bus: EventBus;
   observability?: Observability;
 }): () => void {
-  return deps.bus.subscribeAgentEvents(AGENT_EVENTS_SUBJECT, (payload, subject) =>
-    handleAgentEventPayload(deps.db, deps.observability, payload, subject, deps.bus),
+  const scheduler = createAgentEventIngestionScheduler(
+    (payload, subject) =>
+      handleAgentEventPayload(deps.db, deps.observability, payload, subject, deps.bus),
+    {
+      onHeartbeatCoalesced: () =>
+        deps.observability?.incrementCounter({
+          name: "opengeni_machine_heartbeat_coalesced_total",
+          help: "Connected Machine heartbeats collapsed while an exact runner event queue was busy.",
+        }),
+    },
   );
+  const unsubscribe = deps.bus.subscribeAgentEvents(AGENT_EVENTS_SUBJECT, (payload, subject) => {
+    scheduler.enqueue(payload, subject);
+  });
+  return () => {
+    unsubscribe();
+    scheduler.close();
+  };
 }
 
 // ── Connect-Hello display refresh ─────────────────────────────────────────────
@@ -490,6 +753,47 @@ export function helloReportsOpStream(hello: Hello): boolean {
   return hello.capabilities?.opStream === true;
 }
 
+export function helloRuntimeCapabilities(hello: Hello): Record<string, boolean> {
+  const caps = hello.capabilities;
+  // Absence is an older-agent/unknown signal, not a fabricated set of false claims.
+  // Keep the durable cursor empty so reconnects from legacy agents remain a
+  // no-op and future capability additions do not get silently fabricated.
+  if (!caps) return {};
+  return {
+    exec: caps.exec === true,
+    filesystem: caps.filesystem === true,
+    git: caps.git === true,
+    pty: caps.pty === true,
+    desktop: caps.desktop === true,
+    opStream: caps.opStream === true,
+    browserBridge: caps.browserBridge === true,
+    operationResourcePolicy: caps.operationResourcePolicy === true,
+    operationCpuQuota: caps.operationCpuQuota === true,
+    transactionalFsWrite: caps.transactionalFsWrite === true,
+  };
+}
+
+function helloCompletedUpdate(hello: Hello): {
+  operationId: string;
+  targetVersion: string;
+  binarySha256: string;
+} | null {
+  if (
+    !/^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i.test(
+      hello.completedUpdateOperationId,
+    ) ||
+    !/^(?:v)?\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?$/.test(hello.completedUpdateTargetVersion) ||
+    !/^[0-9a-f]{64}$/.test(hello.completedUpdateBinarySha256)
+  ) {
+    return null;
+  }
+  return {
+    operationId: hello.completedUpdateOperationId,
+    targetVersion: hello.completedUpdateTargetVersion,
+    binarySha256: hello.completedUpdateBinarySha256,
+  };
+}
+
 /**
  * Reconcile `enrollments.has_display` (+ the capture-blocked reason) to what a Hello
  * reports. Resolves the enrollment (the accountId is the RLS principal + the
@@ -505,6 +809,8 @@ export async function refreshEnrollmentDisplay(
     agentId: string;
     hasDisplay: boolean;
     desktopUnavailableReason?: string | null;
+    runtimeDesktop?: boolean;
+    connectionInstanceId?: string;
   },
 ): Promise<{ updated: boolean }> {
   const desktopUnavailableReason = input.desktopUnavailableReason ?? null;
@@ -514,7 +820,9 @@ export async function refreshEnrollmentDisplay(
   }
   if (
     enrollment.hasDisplay === input.hasDisplay &&
-    (enrollment.desktopUnavailableReason ?? null) === desktopUnavailableReason
+    (enrollment.desktopUnavailableReason ?? null) === desktopUnavailableReason &&
+    (input.runtimeDesktop === undefined ||
+      enrollment.agentCapabilities?.desktop === input.runtimeDesktop)
   ) {
     // Both fields unchanged — do not even issue the UPDATE (no churn on a
     // steady-state Hello).
@@ -526,6 +834,8 @@ export async function refreshEnrollmentDisplay(
     enrollmentId: input.agentId,
     hasDisplay: input.hasDisplay,
     desktopUnavailableReason,
+    ...(input.runtimeDesktop !== undefined ? { runtimeDesktop: input.runtimeDesktop } : {}),
+    ...(input.connectionInstanceId ? { connectionInstanceId: input.connectionInstanceId } : {}),
   });
 }
 
@@ -541,6 +851,7 @@ export async function refreshEnrollmentOpStream(
     workspaceId: string;
     agentId: string;
     opStream: boolean;
+    connectionInstanceId?: string;
   },
 ): Promise<{ updated: boolean }> {
   const enrollment = await getEnrollment(db, input.workspaceId, input.agentId);
@@ -557,6 +868,7 @@ export async function refreshEnrollmentOpStream(
     workspaceId: input.workspaceId,
     enrollmentId: input.agentId,
     opStream: input.opStream,
+    ...(input.connectionInstanceId ? { connectionInstanceId: input.connectionInstanceId } : {}),
   });
 }
 
@@ -589,17 +901,56 @@ export async function handleHelloPayload(
     });
     return;
   }
+  const authority = await getLiveEnrollmentConnection(db, ids.workspaceId, ids.agentId).catch(
+    () => null,
+  );
+  if (!authority || authority.connectionInstanceId !== ids.connectionInstanceId) {
+    observability?.warn?.("Ignored a Hello from a superseded runner instance", {
+      workspaceId: ids.workspaceId,
+      agentId: ids.agentId,
+    });
+    return;
+  }
+  let workspaceRoot: string | undefined;
+  if (hello.workspaceRoot.length > 0) {
+    try {
+      workspaceRoot = normalizeConnectedMachineWorkspaceRoot(hello.workspaceRoot, authority.os);
+    } catch (error) {
+      observability?.warn?.("Ignored an invalid Connected Machine workspace root", {
+        workspaceId: ids.workspaceId,
+        agentId: ids.agentId,
+        error: error instanceof Error ? error.message : String(error),
+      });
+    }
+  }
   try {
     await refreshEnrollmentDisplay(db, {
       workspaceId: ids.workspaceId,
       agentId: ids.agentId,
       hasDisplay: helloReportsDisplay(hello),
       desktopUnavailableReason: helloDesktopUnavailableReason(hello),
+      connectionInstanceId: ids.connectionInstanceId,
     });
     await refreshEnrollmentOpStream(db, {
       workspaceId: ids.workspaceId,
       agentId: ids.agentId,
       opStream: helloReportsOpStream(hello),
+      connectionInstanceId: ids.connectionInstanceId,
+    });
+    await setEnrollmentAgentRuntime(db, {
+      accountId: authority.accountId,
+      workspaceId: ids.workspaceId,
+      enrollmentId: ids.agentId,
+      connectionInstanceId: ids.connectionInstanceId,
+      agentVersion: hello.agentVersion.trim() || null,
+      binarySha256: /^[0-9a-f]{64}$/.test(hello.binarySha256) ? hello.binarySha256 : null,
+      updateChannel:
+        hello.updateChannel === "stable" || hello.updateChannel === "beta"
+          ? hello.updateChannel
+          : null,
+      capabilities: helloRuntimeCapabilities(hello),
+      ...(workspaceRoot !== undefined ? { workspaceRoot } : {}),
+      completedUpdate: helloCompletedUpdate(hello),
     });
   } catch (error) {
     observability?.warn?.("Failed to refresh an enrollment's capabilities from a Hello", {
@@ -616,23 +967,21 @@ export async function handleHelloPayload(
   // sessions with an active op on it — a restored only ever pairs a prior lost, so
   // a routine connect Hello (no marker) emits nothing.
   try {
-    const enrollment = await getEnrollment(db, ids.workspaceId, ids.agentId);
-    if (enrollment) {
-      const { cleared } = await clearEnrollmentWentOffline(db, {
-        accountId: enrollment.accountId,
-        workspaceId: ids.workspaceId,
-        enrollmentId: ids.agentId,
-      });
-      if (cleared && bus) {
-        await fanOutMachineLinkEvents(
-          db,
-          bus,
-          observability,
-          ids.workspaceId,
-          ids.agentId,
-          (activeTurnId) => [{ type: "machine.link.restored", turnId: activeTurnId, payload: {} }],
-        );
-      }
+    const { cleared } = await clearEnrollmentWentOffline(db, {
+      accountId: authority.accountId,
+      workspaceId: ids.workspaceId,
+      enrollmentId: ids.agentId,
+      connectionInstanceId: ids.connectionInstanceId,
+    });
+    if (cleared && bus) {
+      await fanOutMachineLinkEvents(
+        db,
+        bus,
+        observability,
+        ids.workspaceId,
+        ids.agentId,
+        (activeTurnId) => [{ type: "machine.link.restored", turnId: activeTurnId, payload: {} }],
+      );
     }
   } catch (error) {
     observability?.warn?.("Failed to clear a machine going-offline marker on a Hello", {
@@ -643,7 +992,7 @@ export async function handleHelloPayload(
 }
 
 /**
- * Start the Hello display-refresh consumer: subscribe `agent.*.*.hello` and
+ * Start the Hello display-refresh consumer: subscribe exact process hellos and
  * reconcile `has_display` to the live capability the agent reports on every
  * connect. Gated by sandboxSelfhostedEnabled (the caller checks the flag). Returns
  * the unsubscribe fn.