@ultimat3/realtime 29.4.0 → 30.1.0

package/src/device-bus.ts

@@ -0,0 +1,258 @@
+// Addressing ONE connected device from any process: where it is (a TTL'd entry in the bus's shared
+// set, written by the node holding its socket) and how a frame reaches it and an answer comes back
+// (a subject per node, correlated by id). The transport is the realtime bus every role already
+// holds, so a job on a worker reaches a device on a web pod with no second channel.
+
+import { type Clock, logger, systemClock } from '@ultimat3/core';
+import { DeviceOfflineError, DeviceRequestTimeoutError } from './device-errors';
+import {
+  type BusClaim,
+  type BusMessage,
+  type BusReply,
+  type BusRequest,
+  decodeBusMessage,
+} from './device-wire';
+import type { Transport, TransportSubscription } from './fanout';
+import type { JsonValue } from './json';
+import { encodeToken } from './nats-kv';
+import type { Scheduler } from './thundering-herd';
+
+/** What the node holding sockets does with what the bus brings it. */
+export interface DeviceBusHandlers {
+  /** A request for a device this node may hold; answer through `reply`. */
+  onRequest(request: BusRequest): void;
+  /** A newer socket for a device opened on `claim.node`. */
+  onClaim(claim: BusClaim): void;
+}
+
+export interface DeviceBusOptions {
+  readonly transport: Transport;
+  readonly path: string;
+  readonly nodeId: string;
+  readonly schedule: Scheduler;
+  readonly clock?: Clock | undefined;
+  readonly newId: () => string;
+}
+
+interface Pending {
+  readonly deviceId: string;
+  readonly resolve: (frame: JsonValue) => void;
+  readonly reject: (error: unknown) => void;
+  readonly cancel: () => void;
+}
+
+/** Where a device's registration lives. JSON, so no spelling of a tenant or id can collide. */
+const locationKey = (path: string, tenant: string, deviceId: string): string =>
+  `device.${JSON.stringify([path, tenant, deviceId])}`;
+
+/** The holder's half of a registration: which socket, opened when (the newest one wins). */
+interface Location {
+  readonly socketId: string;
+  readonly since: number;
+}
+
+const parseLocation = (value: string): Location | undefined => {
+  try {
+    const parsed: unknown = JSON.parse(value);
+    if (typeof parsed !== 'object' || parsed === null) return undefined;
+    const { socketId, since } = parsed as { socketId?: unknown; since?: unknown };
+    return typeof socketId === 'string' && typeof since === 'number'
+      ? { socketId, since }
+      : undefined;
+  } catch {
+    return undefined;
+  }
+};
+
+export class DeviceBus {
+  readonly #options: DeviceBusOptions;
+  readonly #clock: Clock;
+  readonly #prefix: string;
+  /** The bus's own subject, never a `channel()`: no browser subscribes to it. */
+  readonly #claimSubject: string;
+  readonly #pending = new Map<string, Pending>();
+  #subscriptions: TransportSubscription[] = [];
+
+  constructor(options: DeviceBusOptions) {
+    this.#options = options;
+    this.#clock = options.clock ?? systemClock;
+    // The path is declared text and a subject token may not carry `/`: base64url, as presence does.
+    this.#prefix = `x.device.${encodeToken(options.path)}`;
+    this.#claimSubject = `${this.#prefix}.claim`;
+  }
+
+  /** Requests in flight from THIS process — what a leak test reads. */
+  get pendingCount(): number {
+    return this.#pending.size;
+  }
+
+  async start(handlers: DeviceBusHandlers | null): Promise<void> {
+    const own = await this.#options.transport.subscribe(
+      this.#nodeSubject(this.#options.nodeId),
+      (payload) => this.#receive(payload, handlers),
+    );
+    this.#subscriptions.push(own);
+    if (handlers === null) return;
+    try {
+      this.#subscriptions.push(
+        await this.#options.transport.subscribe(this.#claimSubject, (payload) =>
+          this.#receive(payload, handlers),
+        ),
+      );
+    } catch (error) {
+      // The caller never holds a bus whose start threw, so nothing else would release `own`.
+      this.stop();
+      throw error;
+    }
+  }
+
+  /** Every request still waiting is told the device went away: none is left to its deadline. */
+  stop(): void {
+    for (const subscription of this.#subscriptions) subscription.unsubscribe();
+    this.#subscriptions = [];
+    for (const [id, pending] of this.#pending) {
+      this.#settle(id, pending, { kind: 'offline', id, reason: 'this process stopped' });
+    }
+  }
+
+  async register(tenant: string, deviceId: string, socketId: string, ttlMs: number): Promise<void> {
+    const location: Location = { socketId, since: this.#clock.now().getTime() };
+    await this.#options.transport.shared.put(
+      locationKey(this.#options.path, tenant, deviceId),
+      this.#options.nodeId,
+      JSON.stringify(location),
+      ttlMs,
+    );
+    const claim: BusClaim = {
+      kind: 'claim',
+      tenant,
+      deviceId,
+      node: this.#options.nodeId,
+      socketId,
+    };
+    await this.#options.transport.publish(this.#claimSubject, JSON.stringify(claim));
+  }
+
+  /** The heartbeat's write: kept alive, or written again when the bus had already let it go. */
+  async refresh(tenant: string, deviceId: string, socketId: string, ttlMs: number): Promise<void> {
+    const key = locationKey(this.#options.path, tenant, deviceId);
+    const alive = await this.#options.transport.shared.touch(key, this.#options.nodeId, ttlMs);
+    if (alive) return;
+    const location: Location = { socketId, since: this.#clock.now().getTime() };
+    await this.#options.transport.shared.put(
+      key,
+      this.#options.nodeId,
+      JSON.stringify(location),
+      ttlMs,
+    );
+  }
+
+  async unregister(tenant: string, deviceId: string): Promise<void> {
+    await this.#options.transport.shared.drop(
+      locationKey(this.#options.path, tenant, deviceId),
+      this.#options.nodeId,
+    );
+  }
+
+  /**
+   * One frame to one device and its answer. Located first, so a device nobody holds is
+   * `X_DEVICE_OFFLINE` at once; a holder that finds no socket says so, and a device that never
+   * answers is `X_DEVICE_REQUEST_TIMEOUT` with its pending entry gone.
+   */
+  async request(
+    tenant: string,
+    deviceId: string,
+    frame: JsonValue,
+    deadlineMs: number,
+  ): Promise<JsonValue> {
+    const holder = await this.#locate(tenant, deviceId);
+    if (holder === null) {
+      throw new DeviceOfflineError({
+        path: this.#options.path,
+        deviceId,
+        reason: 'no node holds a socket for it',
+      });
+    }
+    const id = this.#options.newId();
+    const answer = new Promise<JsonValue>((resolve, reject) => {
+      const cancel = this.#options.schedule(() => {
+        if (!this.#pending.delete(id)) return;
+        reject(new DeviceRequestTimeoutError({ path: this.#options.path, deviceId, deadlineMs }));
+      }, deadlineMs);
+      this.#pending.set(id, { deviceId, resolve, reject, cancel });
+    });
+    const request: BusRequest = {
+      kind: 'request',
+      id,
+      from: this.#options.nodeId,
+      tenant,
+      deviceId,
+      deadlineMs,
+      frame,
+    };
+    try {
+      await this.#options.transport.publish(this.#nodeSubject(holder), JSON.stringify(request));
+    } catch (error) {
+      const pending = this.#pending.get(id);
+      this.#pending.delete(id);
+      pending?.cancel();
+      throw error;
+    }
+    return await answer;
+  }
+
+  /** The holder's answer, onto the requesting node's own subject. */
+  async reply(to: string, reply: BusReply): Promise<void> {
+    await this.#options.transport.publish(this.#nodeSubject(to), JSON.stringify(reply));
+  }
+
+  async #locate(tenant: string, deviceId: string): Promise<string | null> {
+    const entries = await this.#options.transport.shared.entries(
+      locationKey(this.#options.path, tenant, deviceId),
+    );
+    let newest: { node: string; since: number } | null = null;
+    for (const entry of entries) {
+      const location = parseLocation(entry.value);
+      if (location === undefined) continue;
+      if (newest === null || location.since > newest.since) {
+        newest = { node: entry.member, since: location.since };
+      }
+    }
+    return newest?.node ?? null;
+  }
+
+  #nodeSubject(node: string): string {
+    return `${this.#prefix}.node.${node}`;
+  }
+
+  #receive(payload: string, handlers: DeviceBusHandlers | null): void {
+    const message: BusMessage | undefined = decodeBusMessage(payload);
+    if (message === undefined) {
+      // Another build's envelope, or a stray publisher on this prefix: said once per message, and
+      // never answered — a reply to a message that could not be read has nobody to go to.
+      logger.warn('device.bus_message_unreadable', { path: this.#options.path });
+      return;
+    }
+    if (message.kind === 'request') handlers?.onRequest(message);
+    else if (message.kind === 'claim') handlers?.onClaim(message);
+    else {
+      const pending = this.#pending.get(message.id);
+      if (pending !== undefined) this.#settle(message.id, pending, message);
+    }
+  }
+
+  #settle(id: string, pending: Pending, reply: BusReply): void {
+    this.#pending.delete(id);
+    pending.cancel();
+    if (reply.kind === 'response') pending.resolve(reply.frame);
+    else {
+      pending.reject(
+        new DeviceOfflineError({
+          path: this.#options.path,
+          deviceId: pending.deviceId,
+          reason: reply.reason,
+        }),
+      );
+    }
+  }
+}

package/src/device-declarer.ts

@@ -0,0 +1,32 @@
+// The FILE that called `deviceSocket()`: the one fact that tells `x dev`'s rescan an edited
+// declaration (same module, re-evaluated) from another module taking its path. The policy
+// package's `declaration-site.ts` reads a stack for the same reason — nothing else names the
+// declaring module once its top level is running.
+
+/** `\` → `/`: a Windows frame and `import.meta.path` must compare on both platforms. */
+const toPosix = (text: string): string => text.replaceAll('\\', '/');
+
+/** The path of a frame (`at f (/a/b.ts:1:2)` or `at /a/b.ts:1:2`), without its line and column. */
+const fileOf = (frame: string): string | null =>
+  /(?:\(|at )([^()]+?):\d+:\d+\)?$/.exec(frame.trim())?.[1] ?? null;
+
+/**
+ * The first file in `stack` that is none of `internal`, or `null` when no frame names one — then
+ * nothing can be compared, and the caller decides as it did before it could. JSC drops the frame
+ * of a strict-mode tail call, so an app wrapper that `return`s `deviceSocket(…)` reads as its own
+ * caller: the same answer on every evaluation of one module, which is all the comparison needs.
+ */
+export function pickDeclarer(stack: string, internal: readonly string[]): string | null {
+  const own = new Set(internal.map(toPosix));
+  for (const frame of stack.split('\n').slice(1)) {
+    const file = fileOf(frame);
+    if (file !== null && !own.has(toPosix(file))) return toPosix(file);
+  }
+  return null;
+}
+
+/** Not a throw: the `Error` exists only for its stack. Line and column drop — an edit moves them. */
+export function declarerFile(internal: readonly string[]): string | null {
+  const stack = new Error().stack;
+  return stack === undefined ? null : pickDeclarer(stack, [import.meta.path, ...internal]);
+}

package/src/device-errors.ts

@@ -0,0 +1,89 @@
+// Every refusal a `deviceSocket()` makes. The three device codes are its own; an upgrade refusal
+// reuses the sync node's socket codes (the condition is the same one), with a fix that names the
+// device declaration instead of `syncNode`, because that is the file the reader has to edit.
+
+import { renderFixShellArg } from '@ultimat3/core';
+import { RealtimeError } from './realtime-error';
+
+/** The longest `deadlineMs` a request may wait — here, so the timeout's `fix:` never prints past it. */
+export const DEVICE_MAX_DEADLINE_MS = 600_000;
+
+/** The device's id rendered for a pasteable call: verbatim when shell-safe, a placeholder otherwise. */
+const idArg = (deviceId: string): string => renderFixShellArg(deviceId, '<deviceId>');
+
+/**
+ * No socket is open for this device, for this tenant, on any node. Answered at once — never by
+ * waiting out the deadline — and identically for a device of another tenant, so a request cannot
+ * learn that a device id exists somewhere it may not reach.
+ */
+export class DeviceOfflineError extends RealtimeError {
+  constructor(args: { path: string; deviceId: string; reason: string }) {
+    super({
+      code: 'X_DEVICE_OFFLINE',
+      cause: `deviceSocket("${args.path}") holds no open socket for device ${JSON.stringify(args.deviceId)} under the caller's tenant: ${args.reason}`,
+      fix: `x doctor --json   # then retry deviceLink.request(ctx, '${idArg(args.deviceId)}', frame) once the device redials; across replicas realtime.transport must be 'nats'`,
+    });
+  }
+}
+
+/** The device took the request frame and sent no `response` carrying its id before the deadline. */
+export class DeviceRequestTimeoutError extends RealtimeError {
+  constructor(args: { path: string; deviceId: string; deadlineMs: number }) {
+    super({
+      code: 'X_DEVICE_REQUEST_TIMEOUT',
+      cause: `device ${JSON.stringify(args.deviceId)} on deviceSocket("${args.path}") did not answer within ${args.deadlineMs}ms`,
+      fix: `deviceLink.request(ctx, '${idArg(args.deviceId)}', frame, { deadlineMs: ${Math.min(args.deadlineMs * 2, DEVICE_MAX_DEADLINE_MS)} })   # or answer every request with { type: 'response', id, frame } on the device`,
+    });
+  }
+}
+
+/** A frame past `DEVICE_FRAME_MAX_BYTES`, in either direction. */
+export class DeviceFrameTooLargeError extends RealtimeError {
+  constructor(args: { path: string; bytes: number; limit: number }) {
+    super({
+      code: 'X_DEVICE_FRAME_TOO_LARGE',
+      cause: `a frame of ${args.bytes} bytes on deviceSocket("${args.path}") is past the framework's ${args.limit}-byte frame limit`,
+      fix: `import { DEVICE_FRAME_MAX_BYTES } from '@ultimat3/realtime/server'   # then split the payload under it, or store the bytes and send their key`,
+    });
+  }
+}
+
+/** No bearer token, or one the resolver refused or that has expired: 401 before any socket exists. */
+export function deviceUnauthenticated(path: string, reason: string): RealtimeError {
+  return new RealtimeError({
+    code: 'X_SOCKET_UNAUTHENTICATED',
+    cause: `the device upgrade on ${path} was refused: ${reason}`,
+    fix: `curl -i -H 'Authorization: Bearer <token>' -H 'Upgrade: websocket' "$APP_URL${renderFixShellArg(path, '/link')}"   # a token deviceSocket({ resolveToken }) resolves to a grant whose expiresAt is in the future`,
+  });
+}
+
+/** The actor already holds `limit` device sockets on this node. */
+export function deviceSocketLimit(path: string, limit: number): RealtimeError {
+  return new RealtimeError({
+    code: 'X_SOCKET_LIMIT',
+    cause: `this actor already holds ${limit} sockets on deviceSocket("${path}") on this node — its maxSocketsPerActor`,
+    fix: `deviceSocket({ path: '${path}', maxSocketsPerActor: ${limit * 2}, … })   # or close the actor's other device connections`,
+  });
+}
+
+/**
+ * The upgrade carried an `Origin`, so a BROWSER is dialling — and a device socket is for clients
+ * that are not one. Only an origin the declaration lists exactly is admitted.
+ */
+export function deviceOriginRefused(path: string, admitted: readonly string[]): RealtimeError {
+  const list = admitted.length === 0 ? 'no browser origin at all' : `only ${admitted.join(', ')}`;
+  return new RealtimeError({
+    code: 'X_SOCKET_ORIGIN_REFUSED',
+    cause: `the device upgrade on ${path} carried an Origin header, and this device socket admits ${list}`,
+    fix: `deviceSocket({ path: '${path}', allowedOrigins: ['https://www.example.com'], … })   # only when a browser page really is a device; a non-browser client sends no Origin`,
+  });
+}
+
+/** The resolver raised, or did not decide within the upgrade's deadline: come back, not "signed out". */
+export function deviceAuthUnavailable(detail: string): RealtimeError {
+  return new RealtimeError({
+    code: 'X_SOCKET_AUTH_UNAVAILABLE',
+    cause: `deviceSocket({ resolveToken }) did not decide who a connecting device is: ${detail}`,
+    fix: 'x doctor --json',
+  });
+}

package/src/device-host.ts

@@ -0,0 +1,288 @@
+// The device sockets ONE node holds: open, frames in (in a lane, bounded), requests out, the beat
+// and expiry sweep, and the drain. The bus side — where a device is, and how a request from
+// another process reaches this one — is `device-bus.ts`; this file answers what arrives.
+
+import type { Clock, Ctx } from '@ultimat3/core';
+import { ctxOf, isUltimateError, reportError, runWithContext } from '@ultimat3/core';
+import { CLOSE } from './close-codes';
+import type { DeviceBus, DeviceBusHandlers } from './device-bus';
+import { DeviceFrameTooLargeError } from './device-errors';
+import type { DeviceWsData } from './device-upgrade';
+import {
+  type BusClaim,
+  type BusRequest,
+  bytesOverLimit,
+  DEVICE_FRAME_MAX_BYTES,
+  type DeviceOutbound,
+  decodeDeviceFrame,
+} from './device-wire';
+import type { JsonValue } from './json';
+import type { PrincipalSockets } from './principal-sockets';
+import { drainPlan, type Rng, reconnectFrame, type Scheduler } from './thundering-herd';
+
+/** The socket as much as this file touches it — Bun's `ServerWebSocket<DeviceWsData>` fits. */
+export interface DeviceWs {
+  readonly data: DeviceWsData;
+  /** Bun's answer: bytes written, `-1` queued under backpressure, `0` dropped. */
+  send(data: string): number;
+  close(code?: number, reason?: string): void;
+}
+
+/** Who sent a frame — what `onFrame` reads alongside it. */
+export interface DeviceInfo {
+  readonly deviceId: string;
+  readonly socketId: string;
+  readonly tenant: string;
+}
+
+export type DeviceFrameHandler = (input: {
+  readonly frame: JsonValue;
+  readonly device: DeviceInfo;
+  readonly ctx: Ctx;
+}) => Promise<JsonValue | undefined> | JsonValue | undefined;
+
+/** Frames queued behind `onFrame` on one socket before the device is told it is too fast. */
+export const DEVICE_FRAME_QUEUE_MAX = 32;
+/** Requests one device may hold unanswered on this node; past it the next is answered offline. */
+export const DEVICE_REQUESTS_OUTSTANDING_MAX = 256;
+/** A device that is silent for this many beats is gone. */
+const SILENT_BEATS = 2;
+/** The registration outlives a beat this many times over, so one slow write is never a vanish. */
+const REGISTRATION_BEATS = 3;
+
+interface Held {
+  readonly ws: DeviceWs;
+  lastSeen: number;
+  lane: Promise<void>;
+  queued: number;
+  /** Request id → the requesting node and when this node stops waiting for the device. */
+  readonly requests: Map<string, { readonly from: string; readonly until: number }>;
+}
+
+export interface DeviceHostOptions {
+  readonly path: string;
+  readonly heartbeatMs: number;
+  readonly onFrame: DeviceFrameHandler;
+  readonly bus: DeviceBus;
+  readonly principals: PrincipalSockets;
+  readonly clock: Clock;
+  readonly rng: Rng;
+  readonly schedule: Scheduler;
+}
+
+/** A device's key on this node: tenant and id together, never one without the other. */
+const deviceKey = (tenant: string, deviceId: string): string => JSON.stringify([tenant, deviceId]);
+
+export class DeviceHost implements DeviceBusHandlers {
+  readonly #options: DeviceHostOptions;
+  readonly #held = new Map<string, Held>();
+  readonly #byDevice = new Map<string, string>();
+
+  constructor(options: DeviceHostOptions) {
+    this.#options = options;
+  }
+
+  get count(): number {
+    return this.#held.size;
+  }
+
+  open(ws: DeviceWs): void {
+    const { socketId, tenant, grant } = ws.data;
+    const key = deviceKey(tenant, grant.deviceId);
+    // One socket per device: the newest wins here and, through the claim, on every other node.
+    const previous = this.#byDevice.get(key);
+    if (previous !== undefined) this.#held.get(previous)?.ws.close(CLOSE.policy, 'replaced');
+    this.#held.set(socketId, {
+      ws,
+      lastSeen: this.#options.clock.monotonic(),
+      lane: Promise.resolve(),
+      queued: 0,
+      requests: new Map(),
+    });
+    this.#byDevice.set(key, socketId);
+    this.#send(ws, {
+      type: 'hello',
+      heartbeatMs: this.#options.heartbeatMs,
+      maxFrameBytes: DEVICE_FRAME_MAX_BYTES,
+    });
+    this.#background(
+      this.#options.bus.register(tenant, grant.deviceId, socketId, this.#registrationTtl()),
+      'device.register',
+    );
+  }
+
+  message(ws: DeviceWs, message: string | Uint8Array): void {
+    const held = this.#held.get(ws.data.socketId);
+    if (held === undefined) return;
+    if (typeof message !== 'string') {
+      ws.close(1003, 'text frames only');
+      return;
+    }
+    // Bun's `maxPayloadLength` refuses first; this is the same limit for a host that applied none.
+    if (bytesOverLimit(message) !== null) {
+      ws.close(1009, 'X_DEVICE_FRAME_TOO_LARGE');
+      return;
+    }
+    held.lastSeen = this.#options.clock.monotonic();
+    const frame = decodeDeviceFrame(message);
+    if (frame === undefined) {
+      ws.close(CLOSE.policy, 'X_PROTOCOL_VERSION');
+      return;
+    }
+    if (frame.type === 'response') {
+      const asked = held.requests.get(frame.id);
+      if (asked === undefined) return;
+      held.requests.delete(frame.id);
+      this.#background(
+        this.#options.bus.reply(asked.from, { kind: 'response', id: frame.id, frame: frame.frame }),
+        'device.reply',
+      );
+    } else if (frame.type === 'frame') this.#enqueue(held, frame.frame);
+  }
+
+  close(ws: DeviceWs): void {
+    const { socketId, tenant, grant } = ws.data;
+    const held = this.#held.get(socketId);
+    this.#options.principals.release(socketId);
+    if (held === undefined) return;
+    this.#held.delete(socketId);
+    for (const [id, asked] of held.requests)
+      this.#offline(asked.from, id, 'the device disconnected');
+    const key = deviceKey(tenant, grant.deviceId);
+    if (this.#byDevice.get(key) !== socketId) return;
+    this.#byDevice.delete(key);
+    this.#background(this.#options.bus.unregister(tenant, grant.deviceId), 'device.unregister');
+  }
+
+  onRequest(request: BusRequest): void {
+    const socketId = this.#byDevice.get(deviceKey(request.tenant, request.deviceId));
+    const held = socketId === undefined ? undefined : this.#held.get(socketId);
+    if (held === undefined) {
+      this.#offline(request.from, request.id, 'no socket for it on the node it was registered on');
+      return;
+    }
+    if (held.requests.size >= DEVICE_REQUESTS_OUTSTANDING_MAX) {
+      this.#offline(
+        request.from,
+        request.id,
+        `it holds ${DEVICE_REQUESTS_OUTSTANDING_MAX} unanswered requests`,
+      );
+      return;
+    }
+    if (!this.#send(held.ws, { type: 'request', id: request.id, frame: request.frame })) {
+      this.#offline(request.from, request.id, 'its socket dropped the request frame');
+      return;
+    }
+    const until = this.#options.clock.monotonic() + request.deadlineMs;
+    held.requests.set(request.id, { from: request.from, until });
+  }
+
+  onClaim(claim: BusClaim): void {
+    const socketId = this.#byDevice.get(deviceKey(claim.tenant, claim.deviceId));
+    if (socketId === undefined || socketId === claim.socketId) return;
+    this.#held.get(socketId)?.ws.close(CLOSE.policy, 'replaced');
+  }
+
+  /** One pass: silent and expired sockets closed, live ones re-registered, stale requests forgotten. */
+  sweep(): void {
+    const now = this.#options.clock.monotonic();
+    const wall = this.#options.clock.now().getTime();
+    const silentAfter = this.#options.heartbeatMs * SILENT_BEATS;
+    for (const held of this.#held.values()) {
+      const { grant, tenant, socketId } = held.ws.data;
+      if (now - held.lastSeen > silentAfter) {
+        held.ws.close(CLOSE.idle, 'no beat');
+        continue;
+      }
+      if (grant.expiresAt !== undefined && !(grant.expiresAt > wall)) {
+        held.ws.close(CLOSE.policy, 'grant expired');
+        continue;
+      }
+      for (const [id, asked] of held.requests) if (asked.until <= now) held.requests.delete(id);
+      this.#background(
+        this.#options.bus.refresh(tenant, grant.deviceId, socketId, this.#registrationTtl()),
+        'device.refresh',
+      );
+    }
+  }
+
+  /** Every socket told when to come back (slots across `spreadMs`), then the grace, then closed. */
+  async drain(options: { graceMs: number; spreadMs: number }): Promise<void> {
+    const held = [...this.#held.values()];
+    const plan = drainPlan(
+      held.map((entry) => entry.ws.data.socketId),
+      { spreadMs: options.spreadMs, rng: this.#options.rng },
+    );
+    for (const entry of plan) {
+      const socket = this.#held.get(entry.socketId);
+      if (socket !== undefined) this.#send(socket.ws, reconnectFrame(entry.afterMs, 'drain'));
+    }
+    if (options.graceMs > 0 && this.#held.size > 0) {
+      await new Promise<void>((resolve) => this.#options.schedule(resolve, options.graceMs));
+    }
+    // Closed, not removed: each `close` callback releases its slot, its requests and its entry.
+    for (const entry of [...this.#held.values()]) entry.ws.close(CLOSE.goingAway, 'drain');
+  }
+
+  #enqueue(held: Held, frame: JsonValue): void {
+    if (held.queued >= DEVICE_FRAME_QUEUE_MAX) {
+      held.ws.close(CLOSE.policy, 'X_FRAME_RATE_LIMIT');
+      return;
+    }
+    held.queued += 1;
+    held.lane = held.lane.then(async () => {
+      try {
+        await this.#handle(held, frame);
+      } finally {
+        held.queued -= 1;
+      }
+    });
+  }
+
+  async #handle(held: Held, frame: JsonValue): Promise<void> {
+    const { grant, socketId, tenant } = held.ws.data;
+    const device: DeviceInfo = { deviceId: grant.deviceId, socketId, tenant };
+    const ctx = ctxOf({ actor: grant.actor });
+    try {
+      const answer = await runWithContext(ctx, () => this.#options.onFrame({ frame, device, ctx }));
+      if (answer === undefined || !this.#held.has(socketId)) return;
+      const text = JSON.stringify({ type: 'frame', frame: answer });
+      const over = bytesOverLimit(text);
+      if (over !== null) {
+        throw new DeviceFrameTooLargeError({
+          path: this.#options.path,
+          bytes: over,
+          limit: DEVICE_FRAME_MAX_BYTES,
+        });
+      }
+      held.ws.send(text);
+    } catch (error) {
+      reportError(error, { source: 'realtime', scope: { operation: 'device.on_frame' } });
+      // The code only: the device is a client, and a cause is this node's to read in its log.
+      this.#send(held.ws, {
+        type: 'error',
+        code: isUltimateError(error) ? error.code : 'X_INTERNAL',
+      });
+    }
+  }
+
+  /** `reconnect` is the sync protocol's own frame, built by its one builder. */
+  #send(ws: DeviceWs, frame: DeviceOutbound | ReturnType<typeof reconnectFrame>): boolean {
+    return ws.send(JSON.stringify(frame)) !== 0;
+  }
+
+  #offline(to: string, id: string, reason: string): void {
+    this.#background(this.#options.bus.reply(to, { kind: 'offline', id, reason }), 'device.reply');
+  }
+
+  #registrationTtl(): number {
+    return this.#options.heartbeatMs * REGISTRATION_BEATS;
+  }
+
+  /** A bus write nobody awaits: its failure is reported, never left unhandled. */
+  #background(work: Promise<void>, operation: string): void {
+    work.catch((error: unknown) => {
+      reportError(error, { source: 'realtime', scope: { operation } });
+    });
+  }
+}